diff options
57 files changed, 1685 insertions, 923 deletions
diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index 218f5eeb63..13693a2ecf 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -11,16 +11,19 @@ if [ -z "$NO_WERROR" ]; then BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror" fi +CI_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" +PRINT_CCACHE_STATISTICS="ccache --version | head -n 1 && ccache --show-stats" + if [ -n "$ANDROID_TOOLS_URL" ]; then CI_EXEC make distclean || true CI_EXEC ./autogen.sh CI_EXEC ./configure "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) CI_EXEC "make $MAKEJOBS && cd src/qt && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk" + CI_EXEC "${PRINT_CCACHE_STATISTICS}" exit 0 fi BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" -CI_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" if [ -n "$CONFIG_SHELL" ]; then CI_EXEC "$CONFIG_SHELL" -c "./autogen.sh" @@ -57,6 +60,6 @@ fi CI_EXEC "${MAYBE_BEAR}" "${MAYBE_TOKEN}" make "$MAKEJOBS" "$GOAL" || ( echo "Build failure. Verbose build follows." && CI_EXEC make "$GOAL" V=1 ; false ) -CI_EXEC "ccache --version | head -n 1 && ccache --show-stats" +CI_EXEC "${PRINT_CCACHE_STATISTICS}" CI_EXEC du -sh "${DEPENDS_DIR}"/*/ CI_EXEC du -sh "${PREVIOUS_RELEASES_DIR}" diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index bdb68e0f6f..32f0ea5e42 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -46,6 +46,7 @@ if [ "${RUN_TIDY}" = "true" ]; then " src/policy/settings.cpp"\ " src/rpc/fees.cpp"\ " src/rpc/signmessage.cpp"\ + " src/test/fuzz/txorphan.cpp"\ " -p . ${MAKEJOBS} -- -Xiwyu --cxx17ns -Xiwyu --mapping_file=${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp" fi diff --git a/contrib/builder-keys/keys.txt b/contrib/builder-keys/keys.txt index c70069b440..f8377cce33 100644 --- a/contrib/builder-keys/keys.txt +++ b/contrib/builder-keys/keys.txt @@ -19,6 +19,7 @@ BF6273FAEF7CC0BA1F562E50989F6B3048A116B5 Dev Random (devrandom) D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece (fivepiece) 6F993B250557E7B016ADE5713BDCDA2D87A881D9 Fuzzbawls (Fuzzbawls) 01CDF4627A3B88AAE4A571C87588242FBE38D3A8 Gavin Andresen (gavinandresen) +6B002C6EA3F91B1B0DF0C9BC8F617F1200A6D25C Gloria Zhao (glozow) D1DBF2C4B96F2DEBF4C16654410108112E7EA81F Hennadii Stepanov (hebasto) A2FD494D0021AA9B4FA58F759102B7AE654A4A5A Ilyas Ridhuan (IlyasRidhuan) 2688F5A9A4BE0F295E921E8A25F27A38A47AD566 James O'Beirne (jamesob) diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index ad3129184c..28cad05013 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -223,6 +223,7 @@ CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disab # CFLAGS HOST_CFLAGS="-O2 -g" +HOST_CFLAGS+=$(find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) case "$HOST" in *linux*) HOST_CFLAGS+=" -ffile-prefix-map=${PWD}=." ;; *mingw*) HOST_CFLAGS+=" -fno-ident" ;; diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index df51d659e8..12b563eac2 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -147,7 +147,10 @@ desirable for building Bitcoin Core release binaries." base-gcc)) (define (make-gcc-with-pthreads gcc) - (package-with-extra-configure-variable gcc "--enable-threads" "posix")) + (package-with-extra-configure-variable + (package-with-extra-patches gcc + (search-our-patches "gcc-10-remap-guix-store.patch")) + "--enable-threads" "posix")) (define (make-mingw-w64-cross-gcc cross-gcc) (package-with-extra-patches cross-gcc @@ -253,7 +256,7 @@ thus should be able to compile on most platforms where these exist.") (license license:gpl3+))) ; license is with openssl exception (define-public python-elfesteem - (let ((commit "87bbd79ab7e361004c98cc8601d4e5f029fd8bd5")) + (let ((commit "2eb1e5384ff7a220fd1afacd4a0170acff54fe56")) (package (name "python-elfesteem") (version (git-version "0.1" "1" commit)) @@ -266,8 +269,7 @@ thus should be able to compile on most platforms where these exist.") (file-name (git-file-name name commit)) (sha256 (base32 - "1nyvjisvyxyxnd0023xjf5846xd03lwawp5pfzr8vrky7wwm5maz")) - (patches (search-our-patches "elfsteem-value-error-python-39.patch")))) + "07x6p8clh11z8s1n2kdxrqwqm2almgc5qpkcr9ckb6y5ivjdr5r6")))) (build-system python-build-system) ;; There are no tests, but attempting to run python setup.py test leads to ;; PYTHONPATH problems, just disable the test @@ -535,7 +537,8 @@ inspecting signatures in Mach-O binaries.") "glibc-versioned-locpath.patch" "glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch" "glibc-2.24-no-build-time-cxx-header-run.patch" - "glibc-2.24-fcommon.patch")))))) + "glibc-2.24-fcommon.patch" + "glibc-2.24-guix-prefix.patch")))))) (define-public glibc-2.27/bitcoin-patched (package @@ -552,7 +555,8 @@ inspecting signatures in Mach-O binaries.") "1b2n1gxv9f4fd5yy68qjbnarhf8mf4vmlxk10i3328c1w5pmp0ca")) (patches (search-our-patches "glibc-ldd-x86_64.patch" "glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch" - "glibc-2.27-dont-redefine-nss-database.patch")))))) + "glibc-2.27-dont-redefine-nss-database.patch" + "glibc-2.27-guix-prefix.patch")))))) (packages->manifest (append diff --git a/contrib/guix/patches/elfsteem-value-error-python-39.patch b/contrib/guix/patches/elfsteem-value-error-python-39.patch deleted file mode 100644 index 21e1228afd..0000000000 --- a/contrib/guix/patches/elfsteem-value-error-python-39.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/examples/otool.py b/examples/otool.py -index 2b8efc0..d797b2e 100755 ---- a/examples/otool.py -+++ b/examples/otool.py -@@ -342,7 +342,7 @@ if __name__ == '__main__': - try: - e = macho_init.MACHO(raw, - parseSymbols = False) -- except ValueError, err: -+ except ValueError as err: - print("%s:" %file) - print(" %s" % err) - continue diff --git a/contrib/guix/patches/gcc-10-remap-guix-store.patch b/contrib/guix/patches/gcc-10-remap-guix-store.patch new file mode 100644 index 0000000000..a47ef7a2df --- /dev/null +++ b/contrib/guix/patches/gcc-10-remap-guix-store.patch @@ -0,0 +1,25 @@ +From aad25427e74f387412e8bc9a9d7bbc6c496c792f Mon Sep 17 00:00:00 2001 +From: Andrew Chow <achow101-github@achow101.com> +Date: Wed, 6 Jul 2022 16:49:41 -0400 +Subject: [PATCH] guix: remap guix store paths to /usr + +--- + libgcc/Makefile.in | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/libgcc/Makefile.in b/libgcc/Makefile.in +index 851e7657d07..476c2becd1c 100644 +--- a/libgcc/Makefile.in ++++ b/libgcc/Makefile.in +@@ -854,7 +854,7 @@ endif + # libgcc_eh.a, only LIB2ADDEH matters. If we do, only LIB2ADDEHSTATIC and + # LIB2ADDEHSHARED matter. (Usually all three are identical.) + +-c_flags := -fexceptions ++c_flags := -fexceptions $(shell find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) + + ifeq ($(enable_shared),yes) + +-- +2.37.0 + diff --git a/contrib/guix/patches/glibc-2.24-guix-prefix.patch b/contrib/guix/patches/glibc-2.24-guix-prefix.patch new file mode 100644 index 0000000000..cba2f59a8d --- /dev/null +++ b/contrib/guix/patches/glibc-2.24-guix-prefix.patch @@ -0,0 +1,25 @@ +Without ffile-prefix-map, the debug symbols will contain paths for the +guix store which will include the hashes of each package. However, the +hash for the same package will differ when on different architectures. +In order to be reproducible regardless of the architecture used to build +the package, map all guix store prefixes to something fixed, e.g. /usr. + +We might be able to drop this in favour of using --with-nonshared-cflags +when we being using newer versions of glibc. + +--- a/Makeconfig ++++ b/Makeconfig +@@ -950,6 +950,10 @@ object-suffixes-for-libc += .oS + # shared objects. We don't want to use CFLAGS-os because users may, for + # example, make that processor-specific. + CFLAGS-.oS = $(CFLAGS-.o) $(PIC-ccflag) ++ ++# Map Guix store paths to /usr ++CFLAGS-.oS += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -fdebug-prefix-map={}=/usr" \;` ++ + CPPFLAGS-.oS = $(CPPFLAGS-.o) -DPIC -DLIBC_NONSHARED=1 + libtype.oS = lib%_nonshared.a + endif +-- +2.35.1 + diff --git a/contrib/guix/patches/glibc-2.27-guix-prefix.patch b/contrib/guix/patches/glibc-2.27-guix-prefix.patch new file mode 100644 index 0000000000..cdb3971f7a --- /dev/null +++ b/contrib/guix/patches/glibc-2.27-guix-prefix.patch @@ -0,0 +1,25 @@ +Without ffile-prefix-map, the debug symbols will contain paths for the +guix store which will include the hashes of each package. However, the +hash for the same package will differ when on different architectures. +In order to be reproducible regardless of the architecture used to build +the package, map all guix store prefixes to something fixed, e.g. /usr. + +We might be able to drop this in favour of using --with-nonshared-cflags +when we being using newer versions of glibc. + +--- a/Makeconfig ++++ b/Makeconfig +@@ -992,6 +992,10 @@ object-suffixes := + CPPFLAGS-.o = $(pic-default) + # libc.a must be compiled with -fPIE/-fpie for static PIE. + CFLAGS-.o = $(filter %frame-pointer,$(+cflags)) $(pie-default) ++ ++# Map Guix store paths to /usr ++CFLAGS-.o += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -fdebug-prefix-map={}=/usr" \;` ++ + libtype.o := lib%.a + object-suffixes += .o + ifeq (yes,$(build-shared)) +-- +2.35.1 + diff --git a/contrib/signet/miner b/contrib/signet/miner index b366b98e2d..61415cb2dd 100755 --- a/contrib/signet/miner +++ b/contrib/signet/miner @@ -21,8 +21,8 @@ PATH_BASE_CONTRIB_SIGNET = os.path.abspath(os.path.dirname(os.path.realpath(__fi PATH_BASE_TEST_FUNCTIONAL = os.path.abspath(os.path.join(PATH_BASE_CONTRIB_SIGNET, "..", "..", "test", "functional")) sys.path.insert(0, PATH_BASE_TEST_FUNCTIONAL) -from test_framework.blocktools import WITNESS_COMMITMENT_HEADER, script_BIP34_coinbase_height # noqa: E402 -from test_framework.messages import CBlock, CBlockHeader, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, from_hex, deser_string, hash256, ser_compact_size, ser_string, ser_uint256, tx_from_hex, uint256_from_str # noqa: E402 +from test_framework.blocktools import get_witness_script, script_BIP34_coinbase_height # noqa: E402 +from test_framework.messages import CBlock, CBlockHeader, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, from_hex, deser_string, ser_compact_size, ser_string, ser_uint256, tx_from_hex # noqa: E402 from test_framework.script import CScriptOp # noqa: E402 logging.basicConfig( @@ -123,10 +123,6 @@ def create_coinbase(height, value, spk): cb.vout = [CTxOut(value, spk)] return cb -def get_witness_script(witness_root, witness_nonce): - commitment = uint256_from_str(hash256(ser_uint256(witness_root) + ser_uint256(witness_nonce))) - return b"\x6a" + CScriptOp.encode_op_pushdata(WITNESS_COMMITMENT_HEADER + ser_uint256(commitment)) - def signet_txs(block, challenge): # assumes signet solution has not been added yet so does not need # to be removed @@ -222,7 +218,7 @@ def generate_psbt(tmpl, reward_spk, *, blocktime=None): cbwit = CTxInWitness() cbwit.scriptWitness.stack = [ser_uint256(witnonce)] block.vtx[0].wit.vtxinwit = [cbwit] - block.vtx[0].vout.append(CTxOut(0, get_witness_script(witroot, witnonce))) + block.vtx[0].vout.append(CTxOut(0, bytes(get_witness_script(witroot, witnonce)))) signme, spendme = signet_txs(block, signet_spk_bin) @@ -627,5 +623,3 @@ def main(): if __name__ == "__main__": main() - - diff --git a/contrib/verify-commits/trusted-keys b/contrib/verify-commits/trusted-keys index 2f8a21009a..5ca65e7b0d 100644 --- a/contrib/verify-commits/trusted-keys +++ b/contrib/verify-commits/trusted-keys @@ -1,6 +1,6 @@ 71A3B16735405025D447E8F274810B012346C9A6 -133EAC179436F14A5CF1B794860FEB804E669320 B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B E777299FC265DD04793070EB944D35F9AC3DB76A D1DBF2C4B96F2DEBF4C16654410108112E7EA81F 152812300785C96444D3334D17565732E08E5E41 +6B002C6EA3F91B1B0DF0C9BC8F617F1200A6D25C diff --git a/doc/build-unix.md b/doc/build-unix.md index bcfa25dc76..874015707a 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -288,26 +288,3 @@ This example lists the steps necessary to setup and build a command line only di ./src/bitcoind If you intend to work with legacy Berkeley DB wallets, see [Berkeley DB](#berkeley-db) section. - -ARM Cross-compilation -------------------- -These steps can be performed on, for example, an Ubuntu VM. The depends system -will also work on other Linux distributions, however the commands for -installing the toolchain will be different. - -Make sure you install the build requirements mentioned above. -Then, install the toolchain and curl: - - sudo apt-get install g++-arm-linux-gnueabihf curl - -To build executables for ARM: - - cd depends - make HOST=arm-linux-gnueabihf NO_QT=1 - cd .. - ./autogen.sh - CONFIG_SITE=$PWD/depends/arm-linux-gnueabihf/share/config.site ./configure --enable-reduce-exports LDFLAGS=-static-libstdc++ - make - - -For further documentation on the depends system see [README.md](../depends/README.md) in the depends directory. diff --git a/src/Makefile.am b/src/Makefile.am index 488ff0e273..3fbbe180fc 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -139,6 +139,7 @@ BITCOIN_CORE_H = \ compat/cpuid.h \ compat/endian.h \ compressor.h \ + node/connection_types.h \ consensus/consensus.h \ consensus/tx_check.h \ consensus/tx_verify.h \ @@ -148,6 +149,7 @@ BITCOIN_CORE_H = \ dbwrapper.h \ deploymentinfo.h \ deploymentstatus.h \ + node/eviction.h \ external_signer.h \ flatfile.h \ fs.h \ @@ -373,7 +375,9 @@ libbitcoin_node_a_SOURCES = \ node/caches.cpp \ node/chainstate.cpp \ node/coin.cpp \ + node/connection_types.cpp \ node/context.cpp \ + node/eviction.cpp \ node/interfaces.cpp \ node/miner.cpp \ node/minisketchwrapper.cpp \ @@ -854,8 +858,7 @@ endif # TODO: libbitcoinkernel is a work in progress consensus engine library, as more # and more modules are decoupled from the consensus engine, this list will -# shrink to only those which are absolutely necessary. For example, things -# like index/*.cpp will be removed. +# shrink to only those which are absolutely necessary. libbitcoinkernel_la_SOURCES = \ kernel/bitcoinkernel.cpp \ arith_uint256.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 098feacb3d..b806b62d5b 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -325,6 +325,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/tx_in.cpp \ test/fuzz/tx_out.cpp \ test/fuzz/tx_pool.cpp \ + test/fuzz/txorphan.cpp \ test/fuzz/txrequest.cpp \ test/fuzz/utxo_snapshot.cpp \ test/fuzz/validation_load_mempool.cpp \ diff --git a/src/bench/gcs_filter.cpp b/src/bench/gcs_filter.cpp index 607e4392b7..80babb213b 100644 --- a/src/bench/gcs_filter.cpp +++ b/src/bench/gcs_filter.cpp @@ -5,39 +5,84 @@ #include <bench/bench.h> #include <blockfilter.h> -static void ConstructGCSFilter(benchmark::Bench& bench) +static const GCSFilter::ElementSet GenerateGCSTestElements() { GCSFilter::ElementSet elements; - for (int i = 0; i < 10000; ++i) { + + // Testing the benchmarks with different number of elements show that a filter + // with at least 100,000 elements results in benchmarks that have the same + // ns/op. This makes it easy to reason about how long (in nanoseconds) a single + // filter element takes to process. + for (int i = 0; i < 100000; ++i) { GCSFilter::Element element(32); element[0] = static_cast<unsigned char>(i); element[1] = static_cast<unsigned char>(i >> 8); elements.insert(std::move(element)); } + return elements; +} + +static void GCSBlockFilterGetHash(benchmark::Bench& bench) +{ + auto elements = GenerateGCSTestElements(); + + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); + BlockFilter block_filter(BlockFilterType::BASIC, {}, filter.GetEncoded(), /*skip_decode_check=*/false); + + bench.run([&] { + block_filter.GetHash(); + }); +} + +static void GCSFilterConstruct(benchmark::Bench& bench) +{ + auto elements = GenerateGCSTestElements(); + uint64_t siphash_k0 = 0; - bench.batch(elements.size()).unit("elem").run([&] { - GCSFilter filter({siphash_k0, 0, 20, 1 << 20}, elements); + bench.run([&]{ + GCSFilter filter({siphash_k0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); siphash_k0++; }); } -static void MatchGCSFilter(benchmark::Bench& bench) +static void GCSFilterDecode(benchmark::Bench& bench) { - GCSFilter::ElementSet elements; - for (int i = 0; i < 10000; ++i) { - GCSFilter::Element element(32); - element[0] = static_cast<unsigned char>(i); - element[1] = static_cast<unsigned char>(i >> 8); - elements.insert(std::move(element)); - } - GCSFilter filter({0, 0, 20, 1 << 20}, elements); + auto elements = GenerateGCSTestElements(); - bench.unit("elem").run([&] { - filter.Match(GCSFilter::Element()); + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); + auto encoded = filter.GetEncoded(); + + bench.run([&] { + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, encoded, /*skip_decode_check=*/false); }); } -BENCHMARK(ConstructGCSFilter); -BENCHMARK(MatchGCSFilter); +static void GCSFilterDecodeSkipCheck(benchmark::Bench& bench) +{ + auto elements = GenerateGCSTestElements(); + + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); + auto encoded = filter.GetEncoded(); + + bench.run([&] { + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, encoded, /*skip_decode_check=*/true); + }); +} + +static void GCSFilterMatch(benchmark::Bench& bench) +{ + auto elements = GenerateGCSTestElements(); + + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); + + bench.run([&] { + filter.Match(GCSFilter::Element()); + }); +} +BENCHMARK(GCSBlockFilterGetHash); +BENCHMARK(GCSFilterConstruct); +BENCHMARK(GCSFilterDecode); +BENCHMARK(GCSFilterDecodeSkipCheck); +BENCHMARK(GCSFilterMatch); diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index 63a9ba498f..1ad6872143 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -47,7 +47,7 @@ GCSFilter::GCSFilter(const Params& params) : m_params(params), m_N(0), m_F(0), m_encoded{0} {} -GCSFilter::GCSFilter(const Params& params, std::vector<unsigned char> encoded_filter) +GCSFilter::GCSFilter(const Params& params, std::vector<unsigned char> encoded_filter, bool skip_decode_check) : m_params(params), m_encoded(std::move(encoded_filter)) { SpanReader stream{GCS_SER_TYPE, GCS_SER_VERSION, m_encoded}; @@ -59,6 +59,8 @@ GCSFilter::GCSFilter(const Params& params, std::vector<unsigned char> encoded_fi } m_F = static_cast<uint64_t>(m_N) * static_cast<uint64_t>(m_params.m_M); + if (skip_decode_check) return; + // Verify that the encoded filter contains exactly N elements. If it has too much or too little // data, a std::ios_base::failure exception will be raised. BitStreamReader<SpanReader> bitreader{stream}; @@ -219,14 +221,14 @@ static GCSFilter::ElementSet BasicFilterElements(const CBlock& block, } BlockFilter::BlockFilter(BlockFilterType filter_type, const uint256& block_hash, - std::vector<unsigned char> filter) + std::vector<unsigned char> filter, bool skip_decode_check) : m_filter_type(filter_type), m_block_hash(block_hash) { GCSFilter::Params params; if (!BuildParams(params)) { throw std::invalid_argument("unknown filter_type"); } - m_filter = GCSFilter(params, std::move(filter)); + m_filter = GCSFilter(params, std::move(filter), skip_decode_check); } BlockFilter::BlockFilter(BlockFilterType filter_type, const CBlock& block, const CBlockUndo& block_undo) diff --git a/src/blockfilter.h b/src/blockfilter.h index 96cefbf3b2..d6a51e95c2 100644 --- a/src/blockfilter.h +++ b/src/blockfilter.h @@ -59,7 +59,7 @@ public: explicit GCSFilter(const Params& params = Params()); /** Reconstructs an already-created filter from an encoding. */ - GCSFilter(const Params& params, std::vector<unsigned char> encoded_filter); + GCSFilter(const Params& params, std::vector<unsigned char> encoded_filter, bool skip_decode_check); /** Builds a new filter from the params and set of elements. */ GCSFilter(const Params& params, const ElementSet& elements); @@ -122,7 +122,7 @@ public: //! Reconstruct a BlockFilter from parts. BlockFilter(BlockFilterType filter_type, const uint256& block_hash, - std::vector<unsigned char> filter); + std::vector<unsigned char> filter, bool skip_decode_check); //! Construct a new BlockFilter of the specified type from a block. BlockFilter(BlockFilterType filter_type, const CBlock& block, const CBlockUndo& block_undo); @@ -164,7 +164,7 @@ public: if (!BuildParams(params)) { throw std::ios_base::failure("unknown filter_type"); } - m_filter = GCSFilter(params, std::move(encoded_filter)); + m_filter = GCSFilter(params, std::move(encoded_filter), /*skip_decode_check=*/false); } }; diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index c92b8c7e19..e7fad8eb64 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -5,6 +5,7 @@ #include <map> #include <dbwrapper.h> +#include <hash.h> #include <index/blockfilterindex.h> #include <node/blockstorage.h> #include <util/system.h> @@ -143,18 +144,22 @@ bool BlockFilterIndex::CommitInternal(CDBBatch& batch) return BaseIndex::CommitInternal(batch); } -bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const +bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, const uint256& hash, BlockFilter& filter) const { CAutoFile filein(m_filter_fileseq->Open(pos, true), SER_DISK, CLIENT_VERSION); if (filein.IsNull()) { return false; } + // Check that the hash of the encoded_filter matches the one stored in the db. uint256 block_hash; std::vector<uint8_t> encoded_filter; try { filein >> block_hash >> encoded_filter; - filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter)); + uint256 result; + CHash256().Write(encoded_filter).Finalize(result); + if (result != hash) return error("Checksum mismatch in filter decode."); + filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter), /*skip_decode_check=*/true); } catch (const std::exception& e) { return error("%s: Failed to deserialize block filter from disk: %s", __func__, e.what()); @@ -381,7 +386,7 @@ bool BlockFilterIndex::LookupFilter(const CBlockIndex* block_index, BlockFilter& return false; } - return ReadFilterFromDisk(entry.pos, filter_out); + return ReadFilterFromDisk(entry.pos, entry.hash, filter_out); } bool BlockFilterIndex::LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) @@ -425,7 +430,7 @@ bool BlockFilterIndex::LookupFilterRange(int start_height, const CBlockIndex* st filters_out.resize(entries.size()); auto filter_pos_it = filters_out.begin(); for (const auto& entry : entries) { - if (!ReadFilterFromDisk(entry.pos, *filter_pos_it)) { + if (!ReadFilterFromDisk(entry.pos, entry.hash, *filter_pos_it)) { return false; } ++filter_pos_it; diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h index 6deff59000..fef8b573e8 100644 --- a/src/index/blockfilterindex.h +++ b/src/index/blockfilterindex.h @@ -31,7 +31,7 @@ private: FlatFilePos m_next_filter_pos; std::unique_ptr<FlatFileSeq> m_filter_fileseq; - bool ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const; + bool ReadFilterFromDisk(const FlatFilePos& pos, const uint256& hash, BlockFilter& filter) const; size_t WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter); Mutex m_cs_headers_cache; diff --git a/src/init.cpp b/src/init.cpp index e17728373b..eff37e1a83 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -967,7 +967,7 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb peer_connect_timeout = args.GetIntArg("-peertimeout", DEFAULT_PEER_CONNECT_TIMEOUT); if (peer_connect_timeout <= 0) { - return InitError(Untranslated("peertimeout cannot be configured with a negative value.")); + return InitError(Untranslated("peertimeout must be a positive integer.")); } if (args.IsArgSet("-minrelaytxfee")) { diff --git a/src/net.cpp b/src/net.cpp index 7f4e571c8d..c37d90519c 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -16,6 +16,7 @@ #include <compat.h> #include <consensus/consensus.h> #include <crypto/sha256.h> +#include <node/eviction.h> #include <fs.h> #include <i2p.h> #include <net_permissions.h> @@ -576,26 +577,6 @@ void CConnman::AddWhitelistPermissionFlags(NetPermissionFlags& flags, const CNet } } -std::string ConnectionTypeAsString(ConnectionType conn_type) -{ - switch (conn_type) { - case ConnectionType::INBOUND: - return "inbound"; - case ConnectionType::MANUAL: - return "manual"; - case ConnectionType::FEELER: - return "feeler"; - case ConnectionType::OUTBOUND_FULL_RELAY: - return "outbound-full-relay"; - case ConnectionType::BLOCK_RELAY: - return "block-relay-only"; - case ConnectionType::ADDR_FETCH: - return "addr-fetch"; - } // no default case, so the compiler can warn about missing cases - - assert(false); -} - CService CNode::GetAddrLocal() const { AssertLockNotHeld(m_addr_local_mutex); @@ -877,210 +858,6 @@ size_t CConnman::SocketSendData(CNode& node) const return nSentSize; } -static bool ReverseCompareNodeMinPingTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) -{ - return a.m_min_ping_time > b.m_min_ping_time; -} - -static bool ReverseCompareNodeTimeConnected(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) -{ - return a.m_connected > b.m_connected; -} - -static bool CompareNetGroupKeyed(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) { - return a.nKeyedNetGroup < b.nKeyedNetGroup; -} - -static bool CompareNodeBlockTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) -{ - // There is a fall-through here because it is common for a node to have many peers which have not yet relayed a block. - if (a.m_last_block_time != b.m_last_block_time) return a.m_last_block_time < b.m_last_block_time; - if (a.fRelevantServices != b.fRelevantServices) return b.fRelevantServices; - return a.m_connected > b.m_connected; -} - -static bool CompareNodeTXTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) -{ - // There is a fall-through here because it is common for a node to have more than a few peers that have not yet relayed txn. - if (a.m_last_tx_time != b.m_last_tx_time) return a.m_last_tx_time < b.m_last_tx_time; - if (a.m_relay_txs != b.m_relay_txs) return b.m_relay_txs; - if (a.fBloomFilter != b.fBloomFilter) return a.fBloomFilter; - return a.m_connected > b.m_connected; -} - -// Pick out the potential block-relay only peers, and sort them by last block time. -static bool CompareNodeBlockRelayOnlyTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) -{ - if (a.m_relay_txs != b.m_relay_txs) return a.m_relay_txs; - if (a.m_last_block_time != b.m_last_block_time) return a.m_last_block_time < b.m_last_block_time; - if (a.fRelevantServices != b.fRelevantServices) return b.fRelevantServices; - return a.m_connected > b.m_connected; -} - -/** - * Sort eviction candidates by network/localhost and connection uptime. - * Candidates near the beginning are more likely to be evicted, and those - * near the end are more likely to be protected, e.g. less likely to be evicted. - * - First, nodes that are not `is_local` and that do not belong to `network`, - * sorted by increasing uptime (from most recently connected to connected longer). - * - Then, nodes that are `is_local` or belong to `network`, sorted by increasing uptime. - */ -struct CompareNodeNetworkTime { - const bool m_is_local; - const Network m_network; - CompareNodeNetworkTime(bool is_local, Network network) : m_is_local(is_local), m_network(network) {} - bool operator()(const NodeEvictionCandidate& a, const NodeEvictionCandidate& b) const - { - if (m_is_local && a.m_is_local != b.m_is_local) return b.m_is_local; - if ((a.m_network == m_network) != (b.m_network == m_network)) return b.m_network == m_network; - return a.m_connected > b.m_connected; - }; -}; - -//! Sort an array by the specified comparator, then erase the last K elements where predicate is true. -template <typename T, typename Comparator> -static void EraseLastKElements( - std::vector<T>& elements, Comparator comparator, size_t k, - std::function<bool(const NodeEvictionCandidate&)> predicate = [](const NodeEvictionCandidate& n) { return true; }) -{ - std::sort(elements.begin(), elements.end(), comparator); - size_t eraseSize = std::min(k, elements.size()); - elements.erase(std::remove_if(elements.end() - eraseSize, elements.end(), predicate), elements.end()); -} - -void ProtectEvictionCandidatesByRatio(std::vector<NodeEvictionCandidate>& eviction_candidates) -{ - // Protect the half of the remaining nodes which have been connected the longest. - // This replicates the non-eviction implicit behavior, and precludes attacks that start later. - // To favorise the diversity of our peer connections, reserve up to half of these protected - // spots for Tor/onion, localhost, I2P, and CJDNS peers, even if they're not longest uptime - // overall. This helps protect these higher-latency peers that tend to be otherwise - // disadvantaged under our eviction criteria. - const size_t initial_size = eviction_candidates.size(); - const size_t total_protect_size{initial_size / 2}; - - // Disadvantaged networks to protect. In the case of equal counts, earlier array members - // have the first opportunity to recover unused slots from the previous iteration. - struct Net { bool is_local; Network id; size_t count; }; - std::array<Net, 4> networks{ - {{false, NET_CJDNS, 0}, {false, NET_I2P, 0}, {/*localhost=*/true, NET_MAX, 0}, {false, NET_ONION, 0}}}; - - // Count and store the number of eviction candidates per network. - for (Net& n : networks) { - n.count = std::count_if(eviction_candidates.cbegin(), eviction_candidates.cend(), - [&n](const NodeEvictionCandidate& c) { - return n.is_local ? c.m_is_local : c.m_network == n.id; - }); - } - // Sort `networks` by ascending candidate count, to give networks having fewer candidates - // the first opportunity to recover unused protected slots from the previous iteration. - std::stable_sort(networks.begin(), networks.end(), [](Net a, Net b) { return a.count < b.count; }); - - // Protect up to 25% of the eviction candidates by disadvantaged network. - const size_t max_protect_by_network{total_protect_size / 2}; - size_t num_protected{0}; - - while (num_protected < max_protect_by_network) { - // Count the number of disadvantaged networks from which we have peers to protect. - auto num_networks = std::count_if(networks.begin(), networks.end(), [](const Net& n) { return n.count; }); - if (num_networks == 0) { - break; - } - const size_t disadvantaged_to_protect{max_protect_by_network - num_protected}; - const size_t protect_per_network{std::max(disadvantaged_to_protect / num_networks, static_cast<size_t>(1))}; - // Early exit flag if there are no remaining candidates by disadvantaged network. - bool protected_at_least_one{false}; - - for (Net& n : networks) { - if (n.count == 0) continue; - const size_t before = eviction_candidates.size(); - EraseLastKElements(eviction_candidates, CompareNodeNetworkTime(n.is_local, n.id), - protect_per_network, [&n](const NodeEvictionCandidate& c) { - return n.is_local ? c.m_is_local : c.m_network == n.id; - }); - const size_t after = eviction_candidates.size(); - if (before > after) { - protected_at_least_one = true; - const size_t delta{before - after}; - num_protected += delta; - if (num_protected >= max_protect_by_network) { - break; - } - n.count -= delta; - } - } - if (!protected_at_least_one) { - break; - } - } - - // Calculate how many we removed, and update our total number of peers that - // we want to protect based on uptime accordingly. - assert(num_protected == initial_size - eviction_candidates.size()); - const size_t remaining_to_protect{total_protect_size - num_protected}; - EraseLastKElements(eviction_candidates, ReverseCompareNodeTimeConnected, remaining_to_protect); -} - -[[nodiscard]] std::optional<NodeId> SelectNodeToEvict(std::vector<NodeEvictionCandidate>&& vEvictionCandidates) -{ - // Protect connections with certain characteristics - - // Deterministically select 4 peers to protect by netgroup. - // An attacker cannot predict which netgroups will be protected - EraseLastKElements(vEvictionCandidates, CompareNetGroupKeyed, 4); - // Protect the 8 nodes with the lowest minimum ping time. - // An attacker cannot manipulate this metric without physically moving nodes closer to the target. - EraseLastKElements(vEvictionCandidates, ReverseCompareNodeMinPingTime, 8); - // Protect 4 nodes that most recently sent us novel transactions accepted into our mempool. - // An attacker cannot manipulate this metric without performing useful work. - EraseLastKElements(vEvictionCandidates, CompareNodeTXTime, 4); - // Protect up to 8 non-tx-relay peers that have sent us novel blocks. - EraseLastKElements(vEvictionCandidates, CompareNodeBlockRelayOnlyTime, 8, - [](const NodeEvictionCandidate& n) { return !n.m_relay_txs && n.fRelevantServices; }); - - // Protect 4 nodes that most recently sent us novel blocks. - // An attacker cannot manipulate this metric without performing useful work. - EraseLastKElements(vEvictionCandidates, CompareNodeBlockTime, 4); - - // Protect some of the remaining eviction candidates by ratios of desirable - // or disadvantaged characteristics. - ProtectEvictionCandidatesByRatio(vEvictionCandidates); - - if (vEvictionCandidates.empty()) return std::nullopt; - - // If any remaining peers are preferred for eviction consider only them. - // This happens after the other preferences since if a peer is really the best by other criteria (esp relaying blocks) - // then we probably don't want to evict it no matter what. - if (std::any_of(vEvictionCandidates.begin(),vEvictionCandidates.end(),[](NodeEvictionCandidate const &n){return n.prefer_evict;})) { - vEvictionCandidates.erase(std::remove_if(vEvictionCandidates.begin(),vEvictionCandidates.end(), - [](NodeEvictionCandidate const &n){return !n.prefer_evict;}),vEvictionCandidates.end()); - } - - // Identify the network group with the most connections and youngest member. - // (vEvictionCandidates is already sorted by reverse connect time) - uint64_t naMostConnections; - unsigned int nMostConnections = 0; - std::chrono::seconds nMostConnectionsTime{0}; - std::map<uint64_t, std::vector<NodeEvictionCandidate> > mapNetGroupNodes; - for (const NodeEvictionCandidate &node : vEvictionCandidates) { - std::vector<NodeEvictionCandidate> &group = mapNetGroupNodes[node.nKeyedNetGroup]; - group.push_back(node); - const auto grouptime{group[0].m_connected}; - - if (group.size() > nMostConnections || (group.size() == nMostConnections && grouptime > nMostConnectionsTime)) { - nMostConnections = group.size(); - nMostConnectionsTime = grouptime; - naMostConnections = node.nKeyedNetGroup; - } - } - - // Reduce to the network group with the most connections - vEvictionCandidates = std::move(mapNetGroupNodes[naMostConnections]); - - // Disconnect from the network group with the most connections - return vEvictionCandidates.front().id; -} - /** Try to find a connection to evict when the node is full. * Extreme care must be taken to avoid opening the node to attacker * triggered network partitioning. @@ -1096,10 +873,6 @@ bool CConnman::AttemptToEvictConnection() LOCK(m_nodes_mutex); for (const CNode* node : m_nodes) { - if (node->HasPermission(NetPermissionFlags::NoBan)) - continue; - if (!node->IsInboundConn()) - continue; if (node->fDisconnect) continue; NodeEvictionCandidate candidate{ @@ -1115,6 +888,8 @@ bool CConnman::AttemptToEvictConnection() Desig(prefer_evict) node->m_prefer_evict, Desig(m_is_local) node->addr.IsLocal(), Desig(m_network) node->ConnectedThroughNetwork(), + Desig(m_noban) node->HasPermission(NetPermissionFlags::NoBan), + Desig(m_conn_type) node->m_conn_type, }; vEvictionCandidates.push_back(candidate); } @@ -9,6 +9,7 @@ #include <chainparams.h> #include <common/bloom.h> #include <compat.h> +#include <node/connection_types.h> #include <consensus/amount.h> #include <crypto/siphash.h> #include <hash.h> @@ -121,78 +122,6 @@ struct CSerializedNetMsg { std::string m_type; }; -/** Different types of connections to a peer. This enum encapsulates the - * information we have available at the time of opening or accepting the - * connection. Aside from INBOUND, all types are initiated by us. - * - * If adding or removing types, please update CONNECTION_TYPE_DOC in - * src/rpc/net.cpp and src/qt/rpcconsole.cpp, as well as the descriptions in - * src/qt/guiutil.cpp and src/bitcoin-cli.cpp::NetinfoRequestHandler. */ -enum class ConnectionType { - /** - * Inbound connections are those initiated by a peer. This is the only - * property we know at the time of connection, until P2P messages are - * exchanged. - */ - INBOUND, - - /** - * These are the default connections that we use to connect with the - * network. There is no restriction on what is relayed; by default we relay - * blocks, addresses & transactions. We automatically attempt to open - * MAX_OUTBOUND_FULL_RELAY_CONNECTIONS using addresses from our AddrMan. - */ - OUTBOUND_FULL_RELAY, - - - /** - * We open manual connections to addresses that users explicitly requested - * via the addnode RPC or the -addnode/-connect configuration options. Even if a - * manual connection is misbehaving, we do not automatically disconnect or - * add it to our discouragement filter. - */ - MANUAL, - - /** - * Feeler connections are short-lived connections made to check that a node - * is alive. They can be useful for: - * - test-before-evict: if one of the peers is considered for eviction from - * our AddrMan because another peer is mapped to the same slot in the tried table, - * evict only if this longer-known peer is offline. - * - move node addresses from New to Tried table, so that we have more - * connectable addresses in our AddrMan. - * Note that in the literature ("Eclipse Attacks on Bitcoin’s Peer-to-Peer Network") - * only the latter feature is referred to as "feeler connections", - * although in our codebase feeler connections encompass test-before-evict as well. - * We make these connections approximately every FEELER_INTERVAL: - * first we resolve previously found collisions if they exist (test-before-evict), - * otherwise we connect to a node from the new table. - */ - FEELER, - - /** - * We use block-relay-only connections to help prevent against partition - * attacks. By not relaying transactions or addresses, these connections - * are harder to detect by a third party, thus helping obfuscate the - * network topology. We automatically attempt to open - * MAX_BLOCK_RELAY_ONLY_ANCHORS using addresses from our anchors.dat. Then - * addresses from our AddrMan if MAX_BLOCK_RELAY_ONLY_CONNECTIONS - * isn't reached yet. - */ - BLOCK_RELAY, - - /** - * AddrFetch connections are short lived connections used to solicit - * addresses from peers. These are initiated to addresses submitted via the - * -seednode command line argument, or under certain conditions when the - * AddrMan is empty. - */ - ADDR_FETCH, -}; - -/** Convert ConnectionType enum to a string value */ -std::string ConnectionTypeAsString(ConnectionType conn_type); - /** * Look up IP addresses from all interfaces on the machine and add them to the * list of local addresses to self-advertise. @@ -1247,54 +1176,4 @@ extern std::function<void(const CAddress& addr, bool is_incoming)> CaptureMessage; -struct NodeEvictionCandidate -{ - NodeId id; - std::chrono::seconds m_connected; - std::chrono::microseconds m_min_ping_time; - std::chrono::seconds m_last_block_time; - std::chrono::seconds m_last_tx_time; - bool fRelevantServices; - bool m_relay_txs; - bool fBloomFilter; - uint64_t nKeyedNetGroup; - bool prefer_evict; - bool m_is_local; - Network m_network; -}; - -/** - * Select an inbound peer to evict after filtering out (protecting) peers having - * distinct, difficult-to-forge characteristics. The protection logic picks out - * fixed numbers of desirable peers per various criteria, followed by (mostly) - * ratios of desirable or disadvantaged peers. If any eviction candidates - * remain, the selection logic chooses a peer to evict. - */ -[[nodiscard]] std::optional<NodeId> SelectNodeToEvict(std::vector<NodeEvictionCandidate>&& vEvictionCandidates); - -/** Protect desirable or disadvantaged inbound peers from eviction by ratio. - * - * This function protects half of the peers which have been connected the - * longest, to replicate the non-eviction implicit behavior and preclude attacks - * that start later. - * - * Half of these protected spots (1/4 of the total) are reserved for the - * following categories of peers, sorted by longest uptime, even if they're not - * longest uptime overall: - * - * - onion peers connected via our tor control service - * - * - localhost peers, as manually configured hidden services not using - * `-bind=addr[:port]=onion` will not be detected as inbound onion connections - * - * - I2P peers - * - * - CJDNS peers - * - * This helps protect these privacy network peers, which tend to be otherwise - * disadvantaged under our eviction criteria for their higher min ping times - * relative to IPv4/IPv6 peers, and favorise the diversity of peer connections. - */ -void ProtectEvictionCandidatesByRatio(std::vector<NodeEvictionCandidate>& vEvictionCandidates); - #endif // BITCOIN_NET_H diff --git a/src/net_processing.cpp b/src/net_processing.cpp index db711be130..c33dd29923 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -61,6 +61,8 @@ static constexpr auto UNCONDITIONAL_RELAY_DELAY = 2min; * Timeout = base + per_header * (expected number of headers) */ static constexpr auto HEADERS_DOWNLOAD_TIMEOUT_BASE = 15min; static constexpr auto HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER = 1ms; +/** How long to wait for a peer to respond to a getheaders request */ +static constexpr auto HEADERS_RESPONSE_TIME{2min}; /** Protect at least this many outbound peers from disconnection due to slow/ * behind headers chain. */ @@ -355,6 +357,9 @@ struct Peer { /** Work queue of items requested by this peer **/ std::deque<CInv> m_getdata_requests GUARDED_BY(m_getdata_requests_mutex); + /** Time of the last getheaders message to this peer */ + std::atomic<NodeClock::time_point> m_last_getheaders_timestamp{NodeSeconds{}}; + Peer(NodeId id) : m_id{id} {} @@ -501,7 +506,7 @@ public: private: /** Consider evicting an outbound peer based on the amount of time they've been behind our tip */ - void ConsiderEviction(CNode& pto, std::chrono::seconds time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void ConsiderEviction(CNode& pto, Peer& peer, std::chrono::seconds time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** If we have extra outbound peers, try to disconnect the one with the oldest block announcement */ void EvictExtraOutboundPeers(std::chrono::seconds now) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -560,6 +565,22 @@ private: const std::vector<CBlockHeader>& headers, bool via_compact_block) EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); + /** Various helpers for headers processing, invoked by ProcessHeadersMessage() */ + /** Deal with state tracking and headers sync for peers that send the + * occasional non-connecting header (this can happen due to BIP 130 headers + * announcements for blocks interacting with the 2hr (MAX_FUTURE_BLOCK_TIME) rule). */ + void HandleFewUnconnectingHeaders(CNode& pfrom, Peer& peer, const std::vector<CBlockHeader>& headers); + /** Return true if the headers connect to each other, false otherwise */ + bool CheckHeadersAreContinuous(const std::vector<CBlockHeader>& headers) const; + /** Request further headers from this peer with a given locator. + * We don't issue a getheaders message if we have a recent one outstanding. + * This returns true if a getheaders is actually sent, and false otherwise. + */ + bool MaybeSendGetHeaders(CNode& pfrom, const CBlockLocator& locator, Peer& peer); + /** Potentially fetch blocks from this peer upon receipt of a new headers tip */ + void HeadersDirectFetchBlocks(CNode& pfrom, const CBlockIndex* pindexLast); + /** Update peer state based on received headers message */ + void UpdatePeerStateForReceivedHeaders(CNode& pfrom, const CBlockIndex *pindexLast, bool received_new_header, bool may_have_more_headers); void SendBlockTransactions(CNode& pfrom, Peer& peer, const CBlock& block, const BlockTransactionsRequest& req); @@ -2194,6 +2215,203 @@ void PeerManagerImpl::SendBlockTransactions(CNode& pfrom, Peer& peer, const CBlo m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCKTXN, resp)); } +/** + * Special handling for unconnecting headers that might be part of a block + * announcement. + * + * We'll send a getheaders message in response to try to connect the chain. + * + * The peer can send up to MAX_UNCONNECTING_HEADERS in a row that + * don't connect before given DoS points. + * + * Once a headers message is received that is valid and does connect, + * nUnconnectingHeaders gets reset back to 0. + */ +void PeerManagerImpl::HandleFewUnconnectingHeaders(CNode& pfrom, Peer& peer, + const std::vector<CBlockHeader>& headers) +{ + const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); + + LOCK(cs_main); + CNodeState *nodestate = State(pfrom.GetId()); + + nodestate->nUnconnectingHeaders++; + // Try to fill in the missing headers. + if (MaybeSendGetHeaders(pfrom, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), peer)) { + LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", + headers[0].GetHash().ToString(), + headers[0].hashPrevBlock.ToString(), + m_chainman.m_best_header->nHeight, + pfrom.GetId(), nodestate->nUnconnectingHeaders); + } + // Set hashLastUnknownBlock for this peer, so that if we + // eventually get the headers - even from a different peer - + // we can use this peer to download. + UpdateBlockAvailability(pfrom.GetId(), headers.back().GetHash()); + + // The peer may just be broken, so periodically assign DoS points if this + // condition persists. + if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { + Misbehaving(peer, 20, strprintf("%d non-connecting headers", nodestate->nUnconnectingHeaders)); + } +} + +bool PeerManagerImpl::CheckHeadersAreContinuous(const std::vector<CBlockHeader>& headers) const +{ + uint256 hashLastBlock; + for (const CBlockHeader& header : headers) { + if (!hashLastBlock.IsNull() && header.hashPrevBlock != hashLastBlock) { + return false; + } + hashLastBlock = header.GetHash(); + } + return true; +} + +bool PeerManagerImpl::MaybeSendGetHeaders(CNode& pfrom, const CBlockLocator& locator, Peer& peer) +{ + const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); + + const auto current_time = NodeClock::now(); + + // Only allow a new getheaders message to go out if we don't have a recent + // one already in-flight + if (current_time - peer.m_last_getheaders_timestamp.load() > HEADERS_RESPONSE_TIME) { + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, locator, uint256())); + peer.m_last_getheaders_timestamp = current_time; + return true; + } + return false; +} + +/* + * Given a new headers tip ending in pindexLast, potentially request blocks towards that tip. + * We require that the given tip have at least as much work as our tip, and for + * our current tip to be "close to synced" (see CanDirectFetch()). + */ +void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const CBlockIndex* pindexLast) +{ + const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); + + LOCK(cs_main); + CNodeState *nodestate = State(pfrom.GetId()); + + if (CanDirectFetch() && pindexLast->IsValid(BLOCK_VALID_TREE) && m_chainman.ActiveChain().Tip()->nChainWork <= pindexLast->nChainWork) { + + std::vector<const CBlockIndex*> vToFetch; + const CBlockIndex *pindexWalk = pindexLast; + // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. + while (pindexWalk && !m_chainman.ActiveChain().Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && + !IsBlockRequested(pindexWalk->GetBlockHash()) && + (!DeploymentActiveAt(*pindexWalk, m_chainman, Consensus::DEPLOYMENT_SEGWIT) || State(pfrom.GetId())->fHaveWitness)) { + // We don't have this block, and it's not yet in flight. + vToFetch.push_back(pindexWalk); + } + pindexWalk = pindexWalk->pprev; + } + // If pindexWalk still isn't on our main chain, we're looking at a + // very large reorg at a time we think we're close to caught up to + // the main chain -- this shouldn't really happen. Bail out on the + // direct fetch and rely on parallel download instead. + if (!m_chainman.ActiveChain().Contains(pindexWalk)) { + LogPrint(BCLog::NET, "Large reorg, won't direct fetch to %s (%d)\n", + pindexLast->GetBlockHash().ToString(), + pindexLast->nHeight); + } else { + std::vector<CInv> vGetData; + // Download as much as possible, from earliest to latest. + for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { + if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + // Can't download any more from this peer + break; + } + uint32_t nFetchFlags = GetFetchFlags(pfrom); + vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); + BlockRequested(pfrom.GetId(), *pindex); + LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", + pindex->GetBlockHash().ToString(), pfrom.GetId()); + } + if (vGetData.size() > 1) { + LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", + pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); + } + if (vGetData.size() > 0) { + if (!m_ignore_incoming_txs && + nodestate->m_provides_cmpctblocks && + vGetData.size() == 1 && + mapBlocksInFlight.size() == 1 && + pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { + // In any case, we want to download using a compact block, not a regular one + vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); + } + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + } + } + } +} + +/** + * Given receipt of headers from a peer ending in pindexLast, along with + * whether that header was new and whether the headers message was full, + * update the state we keep for the peer. + */ +void PeerManagerImpl::UpdatePeerStateForReceivedHeaders(CNode& pfrom, + const CBlockIndex *pindexLast, bool received_new_header, bool may_have_more_headers) +{ + LOCK(cs_main); + CNodeState *nodestate = State(pfrom.GetId()); + if (nodestate->nUnconnectingHeaders > 0) { + LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom.GetId(), nodestate->nUnconnectingHeaders); + } + nodestate->nUnconnectingHeaders = 0; + + assert(pindexLast); + UpdateBlockAvailability(pfrom.GetId(), pindexLast->GetBlockHash()); + + // From here, pindexBestKnownBlock should be guaranteed to be non-null, + // because it is set in UpdateBlockAvailability. Some nullptr checks + // are still present, however, as belt-and-suspenders. + + if (received_new_header && pindexLast->nChainWork > m_chainman.ActiveChain().Tip()->nChainWork) { + nodestate->m_last_block_announcement = GetTime(); + } + + // If we're in IBD, we want outbound peers that will serve us a useful + // chain. Disconnect peers that are on chains with insufficient work. + if (m_chainman.ActiveChainstate().IsInitialBlockDownload() && !may_have_more_headers) { + // If the peer has no more headers to give us, then we know we have + // their tip. + if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { + // This peer has too little work on their headers chain to help + // us sync -- disconnect if it is an outbound disconnection + // candidate. + // Note: We compare their tip to nMinimumChainWork (rather than + // m_chainman.ActiveChain().Tip()) because we won't start block download + // until we have a headers chain that has at least + // nMinimumChainWork, even if a peer has a chain past our tip, + // as an anti-DoS measure. + if (pfrom.IsOutboundOrBlockRelayConn()) { + LogPrintf("Disconnecting outbound peer %d -- headers chain has insufficient work\n", pfrom.GetId()); + pfrom.fDisconnect = true; + } + } + } + + // If this is an outbound full-relay peer, check to see if we should protect + // it from the bad/lagging chain logic. + // Note that outbound block-relay peers are excluded from this protection, and + // thus always subject to eviction under the bad/lagging chain logic. + // See ChainSyncTimeoutState. + if (!pfrom.fDisconnect && pfrom.IsFullOutboundConn() && nodestate->pindexBestKnownBlock != nullptr) { + if (m_outbound_peers_with_protect_from_disconnect < MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT && nodestate->pindexBestKnownBlock->nChainWork >= m_chainman.ActiveChain().Tip()->nChainWork && !nodestate->m_chain_sync.m_protect) { + LogPrint(BCLog::NET, "Protecting outbound peer=%d from eviction\n", pfrom.GetId()); + nodestate->m_chain_sync.m_protect = true; + ++m_outbound_peers_with_protect_from_disconnect; + } + } +} + void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, const std::vector<CBlockHeader>& headers, bool via_compact_block) @@ -2206,55 +2424,33 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, return; } - bool received_new_header = false; const CBlockIndex *pindexLast = nullptr; - { - LOCK(cs_main); - CNodeState *nodestate = State(pfrom.GetId()); - // If this looks like it could be a block announcement (nCount <= - // MAX_BLOCKS_TO_ANNOUNCE), use special logic for handling headers that - // don't connect: - // - Send a getheaders message in response to try to connect the chain. - // - The peer can send up to MAX_UNCONNECTING_HEADERS in a row that - // don't connect before giving DoS points - // - Once a headers message is received that is valid and does connect, - // nUnconnectingHeaders gets reset back to 0. - if (!m_chainman.m_blockman.LookupBlockIndex(headers[0].hashPrevBlock) && nCount <= MAX_BLOCKS_TO_ANNOUNCE) { - nodestate->nUnconnectingHeaders++; - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), uint256())); - LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", - headers[0].GetHash().ToString(), - headers[0].hashPrevBlock.ToString(), - m_chainman.m_best_header->nHeight, - pfrom.GetId(), nodestate->nUnconnectingHeaders); - // Set hashLastUnknownBlock for this peer, so that if we - // eventually get the headers - even from a different peer - - // we can use this peer to download. - UpdateBlockAvailability(pfrom.GetId(), headers.back().GetHash()); - - if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { - Misbehaving(peer, 20, strprintf("%d non-connecting headers", nodestate->nUnconnectingHeaders)); - } - return; - } + // Do these headers connect to something in our block index? + bool headers_connect_blockindex{WITH_LOCK(::cs_main, return m_chainman.m_blockman.LookupBlockIndex(headers[0].hashPrevBlock) != nullptr)}; - uint256 hashLastBlock; - for (const CBlockHeader& header : headers) { - if (!hashLastBlock.IsNull() && header.hashPrevBlock != hashLastBlock) { - Misbehaving(peer, 20, "non-continuous headers sequence"); - return; - } - hashLastBlock = header.GetHash(); + if (!headers_connect_blockindex) { + if (nCount <= MAX_BLOCKS_TO_ANNOUNCE) { + // If this looks like it could be a BIP 130 block announcement, use + // special logic for handling headers that don't connect, as this + // could be benign. + HandleFewUnconnectingHeaders(pfrom, peer, headers); + } else { + Misbehaving(peer, 10, "invalid header received"); } + return; + } - // If we don't have the last header, then they'll have given us - // something new (if these headers are valid). - if (!m_chainman.m_blockman.LookupBlockIndex(hashLastBlock)) { - received_new_header = true; - } + // At this point, the headers connect to something in our block index. + if (!CheckHeadersAreContinuous(headers)) { + Misbehaving(peer, 20, "non-continuous headers sequence"); + return; } + // If we don't have the last header, then this peer will have given us + // something new (if these headers are valid). + bool received_new_header{WITH_LOCK(::cs_main, return m_chainman.m_blockman.LookupBlockIndex(headers.back().GetHash()) == nullptr)}; + BlockValidationState state; if (!m_chainman.ProcessNewBlockHeaders(headers, state, &pindexLast)) { if (state.IsInvalid()) { @@ -2263,123 +2459,20 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, } } - { - LOCK(cs_main); - CNodeState *nodestate = State(pfrom.GetId()); - if (nodestate->nUnconnectingHeaders > 0) { - LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom.GetId(), nodestate->nUnconnectingHeaders); - } - nodestate->nUnconnectingHeaders = 0; - - assert(pindexLast); - UpdateBlockAvailability(pfrom.GetId(), pindexLast->GetBlockHash()); - - // From here, pindexBestKnownBlock should be guaranteed to be non-null, - // because it is set in UpdateBlockAvailability. Some nullptr checks - // are still present, however, as belt-and-suspenders. - - if (received_new_header && pindexLast->nChainWork > m_chainman.ActiveChain().Tip()->nChainWork) { - nodestate->m_last_block_announcement = GetTime(); - } - - if (nCount == MAX_HEADERS_RESULTS) { - // Headers message had its maximum size; the peer may have more headers. - // TODO: optimize: if pindexLast is an ancestor of m_chainman.ActiveChain().Tip or m_chainman.m_best_header, continue - // from there instead. + // Consider fetching more headers. + if (nCount == MAX_HEADERS_RESULTS) { + // Headers message had its maximum size; the peer may have more headers. + if (MaybeSendGetHeaders(pfrom, m_chainman.ActiveChain().GetLocator(pindexLast), peer)) { LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", - pindexLast->nHeight, pfrom.GetId(), peer.m_starting_height); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(pindexLast), uint256())); - } - - // If this set of headers is valid and ends in a block with at least as - // much work as our tip, download as much as possible. - if (CanDirectFetch() && pindexLast->IsValid(BLOCK_VALID_TREE) && m_chainman.ActiveChain().Tip()->nChainWork <= pindexLast->nChainWork) { - std::vector<const CBlockIndex*> vToFetch; - const CBlockIndex *pindexWalk = pindexLast; - // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. - while (pindexWalk && !m_chainman.ActiveChain().Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && - !IsBlockRequested(pindexWalk->GetBlockHash()) && - (!DeploymentActiveAt(*pindexWalk, m_chainman, Consensus::DEPLOYMENT_SEGWIT) || State(pfrom.GetId())->fHaveWitness)) { - // We don't have this block, and it's not yet in flight. - vToFetch.push_back(pindexWalk); - } - pindexWalk = pindexWalk->pprev; - } - // If pindexWalk still isn't on our main chain, we're looking at a - // very large reorg at a time we think we're close to caught up to - // the main chain -- this shouldn't really happen. Bail out on the - // direct fetch and rely on parallel download instead. - if (!m_chainman.ActiveChain().Contains(pindexWalk)) { - LogPrint(BCLog::NET, "Large reorg, won't direct fetch to %s (%d)\n", - pindexLast->GetBlockHash().ToString(), - pindexLast->nHeight); - } else { - std::vector<CInv> vGetData; - // Download as much as possible, from earliest to latest. - for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { - if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - // Can't download any more from this peer - break; - } - uint32_t nFetchFlags = GetFetchFlags(pfrom); - vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); - BlockRequested(pfrom.GetId(), *pindex); - LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", - pindex->GetBlockHash().ToString(), pfrom.GetId()); - } - if (vGetData.size() > 1) { - LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", - pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); - } - if (vGetData.size() > 0) { - if (!m_ignore_incoming_txs && - nodestate->m_provides_cmpctblocks && - vGetData.size() == 1 && - mapBlocksInFlight.size() == 1 && - pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { - // In any case, we want to download using a compact block, not a regular one - vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); - } - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); - } - } - } - // If we're in IBD, we want outbound peers that will serve us a useful - // chain. Disconnect peers that are on chains with insufficient work. - if (m_chainman.ActiveChainstate().IsInitialBlockDownload() && nCount != MAX_HEADERS_RESULTS) { - // When nCount < MAX_HEADERS_RESULTS, we know we have no more - // headers to fetch from this peer. - if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { - // This peer has too little work on their headers chain to help - // us sync -- disconnect if it is an outbound disconnection - // candidate. - // Note: We compare their tip to nMinimumChainWork (rather than - // m_chainman.ActiveChain().Tip()) because we won't start block download - // until we have a headers chain that has at least - // nMinimumChainWork, even if a peer has a chain past our tip, - // as an anti-DoS measure. - if (pfrom.IsOutboundOrBlockRelayConn()) { - LogPrintf("Disconnecting outbound peer %d -- headers chain has insufficient work\n", pfrom.GetId()); - pfrom.fDisconnect = true; - } - } - } - - // If this is an outbound full-relay peer, check to see if we should protect - // it from the bad/lagging chain logic. - // Note that outbound block-relay peers are excluded from this protection, and - // thus always subject to eviction under the bad/lagging chain logic. - // See ChainSyncTimeoutState. - if (!pfrom.fDisconnect && pfrom.IsFullOutboundConn() && nodestate->pindexBestKnownBlock != nullptr) { - if (m_outbound_peers_with_protect_from_disconnect < MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT && nodestate->pindexBestKnownBlock->nChainWork >= m_chainman.ActiveChain().Tip()->nChainWork && !nodestate->m_chain_sync.m_protect) { - LogPrint(BCLog::NET, "Protecting outbound peer=%d from eviction\n", pfrom.GetId()); - nodestate->m_chain_sync.m_protect = true; - ++m_outbound_peers_with_protect_from_disconnect; - } + pindexLast->nHeight, pfrom.GetId(), peer.m_starting_height); } } + UpdatePeerStateForReceivedHeaders(pfrom, pindexLast, received_new_header, nCount == MAX_HEADERS_RESULTS); + + // Consider immediately downloading blocks. + HeadersDirectFetchBlocks(pfrom, pindexLast); + return; } @@ -3142,8 +3235,11 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } if (best_block != nullptr) { - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), *best_block)); - LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", m_chainman.m_best_header->nHeight, best_block->ToString(), pfrom.GetId()); + if (MaybeSendGetHeaders(pfrom, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), *peer)) { + LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", + m_chainman.m_best_header->nHeight, best_block->ToString(), + pfrom.GetId()); + } } return; @@ -3316,7 +3412,10 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // others. if (m_chainman.ActiveTip() == nullptr || (m_chainman.ActiveTip()->nChainWork < nMinimumChainWork && !pfrom.HasPermission(NetPermissionFlags::Download))) { - LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because active chain has too little work\n", pfrom.GetId()); + LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because active chain has too little work; sending empty response\n", pfrom.GetId()); + // Just respond with an empty headers message, to tell the peer to + // go away but not treat us as unresponsive. + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, std::vector<CBlock>())); return; } @@ -3597,8 +3696,9 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (!m_chainman.m_blockman.LookupBlockIndex(cmpctblock.header.hashPrevBlock)) { // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers - if (!m_chainman.ActiveChainstate().IsInitialBlockDownload()) - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), uint256())); + if (!m_chainman.ActiveChainstate().IsInitialBlockDownload()) { + MaybeSendGetHeaders(pfrom, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), *peer); + } return; } @@ -3872,6 +3972,10 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } + // Assume that this is in response to any outstanding getheaders + // request we may have sent, and clear out the time of our last request + peer->m_last_getheaders_timestamp.store(NodeSeconds{}); + std::vector<CBlockHeader> headers; // Bypass the normal CBlock deserialization, as we don't want to risk deserializing 2000 full blocks. @@ -4300,7 +4404,7 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt return fMoreWork; } -void PeerManagerImpl::ConsiderEviction(CNode& pto, std::chrono::seconds time_in_seconds) +void PeerManagerImpl::ConsiderEviction(CNode& pto, Peer& peer, std::chrono::seconds time_in_seconds) { AssertLockHeld(cs_main); @@ -4338,10 +4442,15 @@ void PeerManagerImpl::ConsiderEviction(CNode& pto, std::chrono::seconds time_in_ pto.fDisconnect = true; } else { assert(state.m_chain_sync.m_work_header); + // Here, we assume that the getheaders message goes out, + // because it'll either go out or be skipped because of a + // getheaders in-flight already, in which case the peer should + // still respond to us with a sufficiently high work chain tip. + MaybeSendGetHeaders(pto, + m_chainman.ActiveChain().GetLocator(state.m_chain_sync.m_work_header->pprev), + peer); LogPrint(BCLog::NET, "sending getheaders to outbound peer=%d to verify chain work (current best known block:%s, benchmark blockhash: %s)\n", pto.GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "<none>", state.m_chain_sync.m_work_header->GetBlockHash().ToString()); - m_connman.PushMessage(&pto, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(state.m_chain_sync.m_work_header->pprev), uint256())); state.m_chain_sync.m_sent_getheaders = true; - constexpr auto HEADERS_RESPONSE_TIME{2min}; // Bump the timeout to allow a response, which could clear the timeout // (if the response shows the peer has synced), reset the timeout (if // the peer syncs to the required work but not to our tip), or result @@ -4749,15 +4858,6 @@ bool PeerManagerImpl::SendMessages(CNode* pto) if (!state.fSyncStarted && !pto->fClient && !fImporting && !fReindex) { // Only actively request headers from a single peer, unless we're close to today. if ((nSyncStarted == 0 && sync_blocks_and_headers_from_peer) || m_chainman.m_best_header->GetBlockTime() > GetAdjustedTime() - 24 * 60 * 60) { - state.fSyncStarted = true; - state.m_headers_sync_timeout = current_time + HEADERS_DOWNLOAD_TIMEOUT_BASE + - ( - // Convert HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER to microseconds before scaling - // to maintain precision - std::chrono::microseconds{HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER} * - (GetAdjustedTime() - m_chainman.m_best_header->GetBlockTime()) / consensusParams.nPowTargetSpacing - ); - nSyncStarted++; const CBlockIndex* pindexStart = m_chainman.m_best_header; /* If possible, start at the block preceding the currently best known header. This ensures that we always get a @@ -4768,8 +4868,19 @@ bool PeerManagerImpl::SendMessages(CNode* pto) got back an empty response. */ if (pindexStart->pprev) pindexStart = pindexStart->pprev; - LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), peer->m_starting_height); - m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(pindexStart), uint256())); + if (MaybeSendGetHeaders(*pto, m_chainman.ActiveChain().GetLocator(pindexStart), *peer)) { + LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), peer->m_starting_height); + + state.fSyncStarted = true; + state.m_headers_sync_timeout = current_time + HEADERS_DOWNLOAD_TIMEOUT_BASE + + ( + // Convert HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER to microseconds before scaling + // to maintain precision + std::chrono::microseconds{HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER} * + (GetAdjustedTime() - m_chainman.m_best_header->GetBlockTime()) / consensusParams.nPowTargetSpacing + ); + nSyncStarted++; + } } } @@ -5115,7 +5226,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // Check that outbound peers have reasonable chains // GetTime() is used by this anti-DoS logic so we can test this using mocktime - ConsiderEviction(*pto, GetTime<std::chrono::seconds>()); + ConsiderEviction(*pto, *peer, GetTime<std::chrono::seconds>()); // // Message: getdata (blocks) diff --git a/src/node/connection_types.cpp b/src/node/connection_types.cpp new file mode 100644 index 0000000000..904f4371aa --- /dev/null +++ b/src/node/connection_types.cpp @@ -0,0 +1,26 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/connection_types.h> +#include <cassert> + +std::string ConnectionTypeAsString(ConnectionType conn_type) +{ + switch (conn_type) { + case ConnectionType::INBOUND: + return "inbound"; + case ConnectionType::MANUAL: + return "manual"; + case ConnectionType::FEELER: + return "feeler"; + case ConnectionType::OUTBOUND_FULL_RELAY: + return "outbound-full-relay"; + case ConnectionType::BLOCK_RELAY: + return "block-relay-only"; + case ConnectionType::ADDR_FETCH: + return "addr-fetch"; + } // no default case, so the compiler can warn about missing cases + + assert(false); +} diff --git a/src/node/connection_types.h b/src/node/connection_types.h new file mode 100644 index 0000000000..5e1abcace6 --- /dev/null +++ b/src/node/connection_types.h @@ -0,0 +1,82 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_CONNECTION_TYPES_H +#define BITCOIN_NODE_CONNECTION_TYPES_H + +#include <string> + +/** Different types of connections to a peer. This enum encapsulates the + * information we have available at the time of opening or accepting the + * connection. Aside from INBOUND, all types are initiated by us. + * + * If adding or removing types, please update CONNECTION_TYPE_DOC in + * src/rpc/net.cpp and src/qt/rpcconsole.cpp, as well as the descriptions in + * src/qt/guiutil.cpp and src/bitcoin-cli.cpp::NetinfoRequestHandler. */ +enum class ConnectionType { + /** + * Inbound connections are those initiated by a peer. This is the only + * property we know at the time of connection, until P2P messages are + * exchanged. + */ + INBOUND, + + /** + * These are the default connections that we use to connect with the + * network. There is no restriction on what is relayed; by default we relay + * blocks, addresses & transactions. We automatically attempt to open + * MAX_OUTBOUND_FULL_RELAY_CONNECTIONS using addresses from our AddrMan. + */ + OUTBOUND_FULL_RELAY, + + + /** + * We open manual connections to addresses that users explicitly requested + * via the addnode RPC or the -addnode/-connect configuration options. Even if a + * manual connection is misbehaving, we do not automatically disconnect or + * add it to our discouragement filter. + */ + MANUAL, + + /** + * Feeler connections are short-lived connections made to check that a node + * is alive. They can be useful for: + * - test-before-evict: if one of the peers is considered for eviction from + * our AddrMan because another peer is mapped to the same slot in the tried table, + * evict only if this longer-known peer is offline. + * - move node addresses from New to Tried table, so that we have more + * connectable addresses in our AddrMan. + * Note that in the literature ("Eclipse Attacks on Bitcoin’s Peer-to-Peer Network") + * only the latter feature is referred to as "feeler connections", + * although in our codebase feeler connections encompass test-before-evict as well. + * We make these connections approximately every FEELER_INTERVAL: + * first we resolve previously found collisions if they exist (test-before-evict), + * otherwise we connect to a node from the new table. + */ + FEELER, + + /** + * We use block-relay-only connections to help prevent against partition + * attacks. By not relaying transactions or addresses, these connections + * are harder to detect by a third party, thus helping obfuscate the + * network topology. We automatically attempt to open + * MAX_BLOCK_RELAY_ONLY_ANCHORS using addresses from our anchors.dat. Then + * addresses from our AddrMan if MAX_BLOCK_RELAY_ONLY_CONNECTIONS + * isn't reached yet. + */ + BLOCK_RELAY, + + /** + * AddrFetch connections are short lived connections used to solicit + * addresses from peers. These are initiated to addresses submitted via the + * -seednode command line argument, or under certain conditions when the + * AddrMan is empty. + */ + ADDR_FETCH, +}; + +/** Convert ConnectionType enum to a string value */ +std::string ConnectionTypeAsString(ConnectionType conn_type); + +#endif // BITCOIN_NODE_CONNECTION_TYPES_H diff --git a/src/node/eviction.cpp b/src/node/eviction.cpp new file mode 100644 index 0000000000..33406931d4 --- /dev/null +++ b/src/node/eviction.cpp @@ -0,0 +1,240 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/eviction.h> + +#include <algorithm> +#include <array> +#include <chrono> +#include <cstdint> +#include <functional> +#include <map> +#include <vector> + + +static bool ReverseCompareNodeMinPingTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) +{ + return a.m_min_ping_time > b.m_min_ping_time; +} + +static bool ReverseCompareNodeTimeConnected(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) +{ + return a.m_connected > b.m_connected; +} + +static bool CompareNetGroupKeyed(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) { + return a.nKeyedNetGroup < b.nKeyedNetGroup; +} + +static bool CompareNodeBlockTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) +{ + // There is a fall-through here because it is common for a node to have many peers which have not yet relayed a block. + if (a.m_last_block_time != b.m_last_block_time) return a.m_last_block_time < b.m_last_block_time; + if (a.fRelevantServices != b.fRelevantServices) return b.fRelevantServices; + return a.m_connected > b.m_connected; +} + +static bool CompareNodeTXTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) +{ + // There is a fall-through here because it is common for a node to have more than a few peers that have not yet relayed txn. + if (a.m_last_tx_time != b.m_last_tx_time) return a.m_last_tx_time < b.m_last_tx_time; + if (a.m_relay_txs != b.m_relay_txs) return b.m_relay_txs; + if (a.fBloomFilter != b.fBloomFilter) return a.fBloomFilter; + return a.m_connected > b.m_connected; +} + +// Pick out the potential block-relay only peers, and sort them by last block time. +static bool CompareNodeBlockRelayOnlyTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) +{ + if (a.m_relay_txs != b.m_relay_txs) return a.m_relay_txs; + if (a.m_last_block_time != b.m_last_block_time) return a.m_last_block_time < b.m_last_block_time; + if (a.fRelevantServices != b.fRelevantServices) return b.fRelevantServices; + return a.m_connected > b.m_connected; +} + +/** + * Sort eviction candidates by network/localhost and connection uptime. + * Candidates near the beginning are more likely to be evicted, and those + * near the end are more likely to be protected, e.g. less likely to be evicted. + * - First, nodes that are not `is_local` and that do not belong to `network`, + * sorted by increasing uptime (from most recently connected to connected longer). + * - Then, nodes that are `is_local` or belong to `network`, sorted by increasing uptime. + */ +struct CompareNodeNetworkTime { + const bool m_is_local; + const Network m_network; + CompareNodeNetworkTime(bool is_local, Network network) : m_is_local(is_local), m_network(network) {} + bool operator()(const NodeEvictionCandidate& a, const NodeEvictionCandidate& b) const + { + if (m_is_local && a.m_is_local != b.m_is_local) return b.m_is_local; + if ((a.m_network == m_network) != (b.m_network == m_network)) return b.m_network == m_network; + return a.m_connected > b.m_connected; + }; +}; + +//! Sort an array by the specified comparator, then erase the last K elements where predicate is true. +template <typename T, typename Comparator> +static void EraseLastKElements( + std::vector<T>& elements, Comparator comparator, size_t k, + std::function<bool(const NodeEvictionCandidate&)> predicate = [](const NodeEvictionCandidate& n) { return true; }) +{ + std::sort(elements.begin(), elements.end(), comparator); + size_t eraseSize = std::min(k, elements.size()); + elements.erase(std::remove_if(elements.end() - eraseSize, elements.end(), predicate), elements.end()); +} + +void ProtectNoBanConnections(std::vector<NodeEvictionCandidate>& eviction_candidates) +{ + eviction_candidates.erase(std::remove_if(eviction_candidates.begin(), eviction_candidates.end(), + [](NodeEvictionCandidate const& n) { + return n.m_noban; + }), + eviction_candidates.end()); +} + +void ProtectOutboundConnections(std::vector<NodeEvictionCandidate>& eviction_candidates) +{ + eviction_candidates.erase(std::remove_if(eviction_candidates.begin(), eviction_candidates.end(), + [](NodeEvictionCandidate const& n) { + return n.m_conn_type != ConnectionType::INBOUND; + }), + eviction_candidates.end()); +} + +void ProtectEvictionCandidatesByRatio(std::vector<NodeEvictionCandidate>& eviction_candidates) +{ + // Protect the half of the remaining nodes which have been connected the longest. + // This replicates the non-eviction implicit behavior, and precludes attacks that start later. + // To favorise the diversity of our peer connections, reserve up to half of these protected + // spots for Tor/onion, localhost, I2P, and CJDNS peers, even if they're not longest uptime + // overall. This helps protect these higher-latency peers that tend to be otherwise + // disadvantaged under our eviction criteria. + const size_t initial_size = eviction_candidates.size(); + const size_t total_protect_size{initial_size / 2}; + + // Disadvantaged networks to protect. In the case of equal counts, earlier array members + // have the first opportunity to recover unused slots from the previous iteration. + struct Net { bool is_local; Network id; size_t count; }; + std::array<Net, 4> networks{ + {{false, NET_CJDNS, 0}, {false, NET_I2P, 0}, {/*localhost=*/true, NET_MAX, 0}, {false, NET_ONION, 0}}}; + + // Count and store the number of eviction candidates per network. + for (Net& n : networks) { + n.count = std::count_if(eviction_candidates.cbegin(), eviction_candidates.cend(), + [&n](const NodeEvictionCandidate& c) { + return n.is_local ? c.m_is_local : c.m_network == n.id; + }); + } + // Sort `networks` by ascending candidate count, to give networks having fewer candidates + // the first opportunity to recover unused protected slots from the previous iteration. + std::stable_sort(networks.begin(), networks.end(), [](Net a, Net b) { return a.count < b.count; }); + + // Protect up to 25% of the eviction candidates by disadvantaged network. + const size_t max_protect_by_network{total_protect_size / 2}; + size_t num_protected{0}; + + while (num_protected < max_protect_by_network) { + // Count the number of disadvantaged networks from which we have peers to protect. + auto num_networks = std::count_if(networks.begin(), networks.end(), [](const Net& n) { return n.count; }); + if (num_networks == 0) { + break; + } + const size_t disadvantaged_to_protect{max_protect_by_network - num_protected}; + const size_t protect_per_network{std::max(disadvantaged_to_protect / num_networks, static_cast<size_t>(1))}; + // Early exit flag if there are no remaining candidates by disadvantaged network. + bool protected_at_least_one{false}; + + for (Net& n : networks) { + if (n.count == 0) continue; + const size_t before = eviction_candidates.size(); + EraseLastKElements(eviction_candidates, CompareNodeNetworkTime(n.is_local, n.id), + protect_per_network, [&n](const NodeEvictionCandidate& c) { + return n.is_local ? c.m_is_local : c.m_network == n.id; + }); + const size_t after = eviction_candidates.size(); + if (before > after) { + protected_at_least_one = true; + const size_t delta{before - after}; + num_protected += delta; + if (num_protected >= max_protect_by_network) { + break; + } + n.count -= delta; + } + } + if (!protected_at_least_one) { + break; + } + } + + // Calculate how many we removed, and update our total number of peers that + // we want to protect based on uptime accordingly. + assert(num_protected == initial_size - eviction_candidates.size()); + const size_t remaining_to_protect{total_protect_size - num_protected}; + EraseLastKElements(eviction_candidates, ReverseCompareNodeTimeConnected, remaining_to_protect); +} + +[[nodiscard]] std::optional<NodeId> SelectNodeToEvict(std::vector<NodeEvictionCandidate>&& vEvictionCandidates) +{ + // Protect connections with certain characteristics + + ProtectNoBanConnections(vEvictionCandidates); + + ProtectOutboundConnections(vEvictionCandidates); + + // Deterministically select 4 peers to protect by netgroup. + // An attacker cannot predict which netgroups will be protected + EraseLastKElements(vEvictionCandidates, CompareNetGroupKeyed, 4); + // Protect the 8 nodes with the lowest minimum ping time. + // An attacker cannot manipulate this metric without physically moving nodes closer to the target. + EraseLastKElements(vEvictionCandidates, ReverseCompareNodeMinPingTime, 8); + // Protect 4 nodes that most recently sent us novel transactions accepted into our mempool. + // An attacker cannot manipulate this metric without performing useful work. + EraseLastKElements(vEvictionCandidates, CompareNodeTXTime, 4); + // Protect up to 8 non-tx-relay peers that have sent us novel blocks. + EraseLastKElements(vEvictionCandidates, CompareNodeBlockRelayOnlyTime, 8, + [](const NodeEvictionCandidate& n) { return !n.m_relay_txs && n.fRelevantServices; }); + + // Protect 4 nodes that most recently sent us novel blocks. + // An attacker cannot manipulate this metric without performing useful work. + EraseLastKElements(vEvictionCandidates, CompareNodeBlockTime, 4); + + // Protect some of the remaining eviction candidates by ratios of desirable + // or disadvantaged characteristics. + ProtectEvictionCandidatesByRatio(vEvictionCandidates); + + if (vEvictionCandidates.empty()) return std::nullopt; + + // If any remaining peers are preferred for eviction consider only them. + // This happens after the other preferences since if a peer is really the best by other criteria (esp relaying blocks) + // then we probably don't want to evict it no matter what. + if (std::any_of(vEvictionCandidates.begin(),vEvictionCandidates.end(),[](NodeEvictionCandidate const &n){return n.prefer_evict;})) { + vEvictionCandidates.erase(std::remove_if(vEvictionCandidates.begin(),vEvictionCandidates.end(), + [](NodeEvictionCandidate const &n){return !n.prefer_evict;}),vEvictionCandidates.end()); + } + + // Identify the network group with the most connections and youngest member. + // (vEvictionCandidates is already sorted by reverse connect time) + uint64_t naMostConnections; + unsigned int nMostConnections = 0; + std::chrono::seconds nMostConnectionsTime{0}; + std::map<uint64_t, std::vector<NodeEvictionCandidate> > mapNetGroupNodes; + for (const NodeEvictionCandidate &node : vEvictionCandidates) { + std::vector<NodeEvictionCandidate> &group = mapNetGroupNodes[node.nKeyedNetGroup]; + group.push_back(node); + const auto grouptime{group[0].m_connected}; + + if (group.size() > nMostConnections || (group.size() == nMostConnections && grouptime > nMostConnectionsTime)) { + nMostConnections = group.size(); + nMostConnectionsTime = grouptime; + naMostConnections = node.nKeyedNetGroup; + } + } + + // Reduce to the network group with the most connections + vEvictionCandidates = std::move(mapNetGroupNodes[naMostConnections]); + + // Disconnect from the network group with the most connections + return vEvictionCandidates.front().id; +} diff --git a/src/node/eviction.h b/src/node/eviction.h new file mode 100644 index 0000000000..1bb32e5327 --- /dev/null +++ b/src/node/eviction.h @@ -0,0 +1,69 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_EVICTION_H +#define BITCOIN_NODE_EVICTION_H + +#include <node/connection_types.h> +#include <net_permissions.h> + +#include <chrono> +#include <cstdint> +#include <optional> +#include <vector> + +typedef int64_t NodeId; + +struct NodeEvictionCandidate { + NodeId id; + std::chrono::seconds m_connected; + std::chrono::microseconds m_min_ping_time; + std::chrono::seconds m_last_block_time; + std::chrono::seconds m_last_tx_time; + bool fRelevantServices; + bool m_relay_txs; + bool fBloomFilter; + uint64_t nKeyedNetGroup; + bool prefer_evict; + bool m_is_local; + Network m_network; + bool m_noban; + ConnectionType m_conn_type; +}; + +/** + * Select an inbound peer to evict after filtering out (protecting) peers having + * distinct, difficult-to-forge characteristics. The protection logic picks out + * fixed numbers of desirable peers per various criteria, followed by (mostly) + * ratios of desirable or disadvantaged peers. If any eviction candidates + * remain, the selection logic chooses a peer to evict. + */ +[[nodiscard]] std::optional<NodeId> SelectNodeToEvict(std::vector<NodeEvictionCandidate>&& vEvictionCandidates); + +/** Protect desirable or disadvantaged inbound peers from eviction by ratio. + * + * This function protects half of the peers which have been connected the + * longest, to replicate the non-eviction implicit behavior and preclude attacks + * that start later. + * + * Half of these protected spots (1/4 of the total) are reserved for the + * following categories of peers, sorted by longest uptime, even if they're not + * longest uptime overall: + * + * - onion peers connected via our tor control service + * + * - localhost peers, as manually configured hidden services not using + * `-bind=addr[:port]=onion` will not be detected as inbound onion connections + * + * - I2P peers + * + * - CJDNS peers + * + * This helps protect these privacy network peers, which tend to be otherwise + * disadvantaged under our eviction criteria for their higher min ping times + * relative to IPv4/IPv6 peers, and favorise the diversity of peer connections. + */ +void ProtectEvictionCandidatesByRatio(std::vector<NodeEvictionCandidate>& vEvictionCandidates); + +#endif // BITCOIN_NODE_EVICTION_H diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index af9458206e..6846e992d4 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -845,7 +845,7 @@ static RPCHelpMan gettxoutsetinfo() "Note this call may take some time if you are not using coinstatsindex.\n", { {"hash_type", RPCArg::Type::STR, RPCArg::Default{"hash_serialized_2"}, "Which UTXO set hash should be calculated. Options: 'hash_serialized_2' (the legacy algorithm), 'muhash', 'none'."}, - {"hash_or_height", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "The block hash or height of the target height (only available with coinstatsindex).", "", {"", "string or numeric"}}, + {"hash_or_height", RPCArg::Type::NUM, RPCArg::DefaultHint{"the current best block"}, "The block hash or height of the target height (only available with coinstatsindex).", "", {"", "string or numeric"}}, {"use_index", RPCArg::Type::BOOL, RPCArg::Default{true}, "Use coinstatsindex, if available."}, }, RPCResult{ @@ -881,6 +881,7 @@ static RPCHelpMan gettxoutsetinfo() HelpExampleCli("gettxoutsetinfo", R"("none")") + HelpExampleCli("gettxoutsetinfo", R"("none" 1000)") + HelpExampleCli("gettxoutsetinfo", R"("none" '"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09"')") + + HelpExampleCli("-named gettxoutsetinfo", R"(hash_type='muhash' use_index='false')") + HelpExampleRpc("gettxoutsetinfo", "") + HelpExampleRpc("gettxoutsetinfo", R"("none")") + HelpExampleRpc("gettxoutsetinfo", R"("none", 1000)") + @@ -917,6 +918,9 @@ static RPCHelpMan gettxoutsetinfo() throw JSONRPCError(RPC_INVALID_PARAMETER, "hash_serialized_2 hash type cannot be queried for a specific block"); } + if (!index_requested) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot set use_index to false when querying for a specific block"); + } pindex = ParseHashOrHeight(request.params[1], chainman); } diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index ae0d0112ba..9be3ab7df0 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -110,6 +110,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "sendrawtransaction", 1, "maxfeerate" }, { "testmempoolaccept", 0, "rawtxs" }, { "testmempoolaccept", 1, "maxfeerate" }, + { "submitpackage", 0, "package" }, { "combinerawtransaction", 0, "txs" }, { "fundrawtransaction", 1, "options" }, { "fundrawtransaction", 2, "iswitness" }, diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index c8208feaab..84d43e7818 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -5,6 +5,7 @@ #include <rpc/blockchain.h> +#include <chainparams.h> #include <core_io.h> #include <fs.h> #include <policy/rbf.h> @@ -732,6 +733,150 @@ static RPCHelpMan savemempool() }; } +static RPCHelpMan submitpackage() +{ + return RPCHelpMan{"submitpackage", + "Submit a package of raw transactions (serialized, hex-encoded) to local node (-regtest only).\n" + "The package will be validated according to consensus and mempool policy rules. If all transactions pass, they will be accepted to mempool.\n" + "This RPC is experimental and the interface may be unstable. Refer to doc/policy/packages.md for documentation on package policies.\n" + "Warning: until package relay is in use, successful submission does not mean the transaction will propagate to other nodes on the network.\n" + "Currently, each transaction is broadcasted individually after submission, which means they must meet other nodes' feerate requirements alone.\n" + , + { + {"package", RPCArg::Type::ARR, RPCArg::Optional::NO, "An array of raw transactions.", + { + {"rawtx", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, + }, + }, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::OBJ_DYN, "tx-results", "transaction results keyed by wtxid", + { + {RPCResult::Type::OBJ, "wtxid", "transaction wtxid", { + {RPCResult::Type::STR_HEX, "txid", "The transaction hash in hex"}, + {RPCResult::Type::STR_HEX, "other-wtxid", /*optional=*/true, "The wtxid of a different transaction with the same txid but different witness found in the mempool. This means the submitted transaction was ignored."}, + {RPCResult::Type::NUM, "vsize", "Virtual transaction size as defined in BIP 141."}, + {RPCResult::Type::OBJ, "fees", "Transaction fees", { + {RPCResult::Type::STR_AMOUNT, "base", "transaction fee in " + CURRENCY_UNIT}, + }}, + }} + }}, + {RPCResult::Type::STR_AMOUNT, "package-feerate", /*optional=*/true, "package feerate used for feerate checks in " + CURRENCY_UNIT + " per KvB. Excludes transactions which were deduplicated or accepted individually."}, + {RPCResult::Type::ARR, "replaced-transactions", /*optional=*/true, "List of txids of replaced transactions", + { + {RPCResult::Type::STR_HEX, "", "The transaction id"}, + }}, + }, + }, + RPCExamples{ + HelpExampleCli("testmempoolaccept", "[rawtx1, rawtx2]") + + HelpExampleCli("submitpackage", "[rawtx1, rawtx2]") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + if (!Params().IsMockableChain()) { + throw std::runtime_error("submitpackage is for regression testing (-regtest mode) only"); + } + RPCTypeCheck(request.params, { + UniValue::VARR, + }); + const UniValue raw_transactions = request.params[0].get_array(); + if (raw_transactions.size() < 1 || raw_transactions.size() > MAX_PACKAGE_COUNT) { + throw JSONRPCError(RPC_INVALID_PARAMETER, + "Array must contain between 1 and " + ToString(MAX_PACKAGE_COUNT) + " transactions."); + } + + std::vector<CTransactionRef> txns; + txns.reserve(raw_transactions.size()); + for (const auto& rawtx : raw_transactions.getValues()) { + CMutableTransaction mtx; + if (!DecodeHexTx(mtx, rawtx.get_str())) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, + "TX decode failed: " + rawtx.get_str() + " Make sure the tx has at least one input."); + } + txns.emplace_back(MakeTransactionRef(std::move(mtx))); + } + + NodeContext& node = EnsureAnyNodeContext(request.context); + CTxMemPool& mempool = EnsureMemPool(node); + CChainState& chainstate = EnsureChainman(node).ActiveChainstate(); + const auto package_result = WITH_LOCK(::cs_main, return ProcessNewPackage(chainstate, mempool, txns, /*test_accept=*/ false)); + + // First catch any errors. + switch(package_result.m_state.GetResult()) { + case PackageValidationResult::PCKG_RESULT_UNSET: break; + case PackageValidationResult::PCKG_POLICY: + { + throw JSONRPCTransactionError(TransactionError::INVALID_PACKAGE, + package_result.m_state.GetRejectReason()); + } + case PackageValidationResult::PCKG_MEMPOOL_ERROR: + { + throw JSONRPCTransactionError(TransactionError::MEMPOOL_ERROR, + package_result.m_state.GetRejectReason()); + } + case PackageValidationResult::PCKG_TX: + { + for (const auto& tx : txns) { + auto it = package_result.m_tx_results.find(tx->GetWitnessHash()); + if (it != package_result.m_tx_results.end() && it->second.m_state.IsInvalid()) { + throw JSONRPCTransactionError(TransactionError::MEMPOOL_REJECTED, + strprintf("%s failed: %s", tx->GetHash().ToString(), it->second.m_state.GetRejectReason())); + } + } + // If a PCKG_TX error was returned, there must have been an invalid transaction. + NONFATAL_UNREACHABLE(); + } + } + for (const auto& tx : txns) { + size_t num_submitted{0}; + std::string err_string; + const auto err = BroadcastTransaction(node, tx, err_string, 0, true, true); + if (err != TransactionError::OK) { + throw JSONRPCTransactionError(err, + strprintf("transaction broadcast failed: %s (all transactions were submitted, %d transactions were broadcast successfully)", + err_string, num_submitted)); + } + } + UniValue rpc_result{UniValue::VOBJ}; + UniValue tx_result_map{UniValue::VOBJ}; + std::set<uint256> replaced_txids; + for (const auto& tx : txns) { + auto it = package_result.m_tx_results.find(tx->GetWitnessHash()); + CHECK_NONFATAL(it != package_result.m_tx_results.end()); + UniValue result_inner{UniValue::VOBJ}; + result_inner.pushKV("txid", tx->GetHash().GetHex()); + if (it->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS) { + result_inner.pushKV("other-wtxid", it->second.m_other_wtxid.value().GetHex()); + } + if (it->second.m_result_type == MempoolAcceptResult::ResultType::VALID || + it->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY) { + result_inner.pushKV("vsize", int64_t{it->second.m_vsize.value()}); + UniValue fees(UniValue::VOBJ); + fees.pushKV("base", ValueFromAmount(it->second.m_base_fees.value())); + result_inner.pushKV("fees", fees); + if (it->second.m_replaced_transactions.has_value()) { + for (const auto& ptx : it->second.m_replaced_transactions.value()) { + replaced_txids.insert(ptx->GetHash()); + } + } + } + tx_result_map.pushKV(tx->GetWitnessHash().GetHex(), result_inner); + } + rpc_result.pushKV("tx-results", tx_result_map); + if (package_result.m_package_feerate.has_value()) { + rpc_result.pushKV("package-feerate", ValueFromAmount(package_result.m_package_feerate.value().GetFeePerK())); + } + UniValue replaced_list(UniValue::VARR); + for (const uint256& hash : replaced_txids) replaced_list.push_back(hash.ToString()); + rpc_result.pushKV("replaced-transactions", replaced_list); + return rpc_result; + }, + }; +} + void RegisterMempoolRPCCommands(CRPCTable& t) { static const CRPCCommand commands[]{ @@ -744,6 +889,7 @@ void RegisterMempoolRPCCommands(CRPCTable& t) {"blockchain", &getmempoolinfo}, {"blockchain", &getrawmempool}, {"blockchain", &savemempool}, + {"hidden", &submitpackage}, }; for (const auto& c : commands) { t.appendCommand(c.name, &c); diff --git a/src/test/fuzz/node_eviction.cpp b/src/test/fuzz/node_eviction.cpp index 6a363f00f7..e27b254580 100644 --- a/src/test/fuzz/node_eviction.cpp +++ b/src/test/fuzz/node_eviction.cpp @@ -32,6 +32,8 @@ FUZZ_TARGET(node_eviction) /*prefer_evict=*/fuzzed_data_provider.ConsumeBool(), /*m_is_local=*/fuzzed_data_provider.ConsumeBool(), /*m_network=*/fuzzed_data_provider.PickValueInArray(ALL_NETWORKS), + /*m_noban=*/fuzzed_data_provider.ConsumeBool(), + /*m_conn_type=*/fuzzed_data_provider.PickValueInArray(ALL_CONNECTION_TYPES), }); } // Make a copy since eviction_candidates may be in some valid but otherwise diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index e4e83c3f32..26913a41d2 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -159,6 +159,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ "signrawtransactionwithkey", "submitblock", "submitheader", + "submitpackage", "syncwithvalidationinterfacequeue", "testmempoolaccept", "uptime", diff --git a/src/test/fuzz/txorphan.cpp b/src/test/fuzz/txorphan.cpp new file mode 100644 index 0000000000..d318baa6a2 --- /dev/null +++ b/src/test/fuzz/txorphan.cpp @@ -0,0 +1,143 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <consensus/amount.h> +#include <net.h> +#include <net_processing.h> +#include <primitives/transaction.h> +#include <script/script.h> +#include <sync.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/setup_common.h> +#include <txorphanage.h> +#include <uint256.h> +#include <util/check.h> +#include <util/time.h> + +#include <algorithm> +#include <cstdint> +#include <memory> +#include <set> +#include <utility> +#include <vector> + +void initialize_orphanage() +{ + static const auto testing_setup = MakeNoLogFileContext(); +} + +FUZZ_TARGET_INIT(txorphan, initialize_orphanage) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + SetMockTime(ConsumeTime(fuzzed_data_provider)); + + TxOrphanage orphanage; + std::set<uint256> orphan_work_set; + std::vector<COutPoint> outpoints; + // initial outpoints used to construct transactions later + for (uint8_t i = 0; i < 4; i++) { + outpoints.emplace_back(uint256{i}, 0); + } + // if true, allow duplicate input when constructing tx + const bool duplicate_input = fuzzed_data_provider.ConsumeBool(); + + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10 * DEFAULT_MAX_ORPHAN_TRANSACTIONS) + { + // construct transaction + const CTransactionRef tx = [&] { + CMutableTransaction tx_mut; + const auto num_in = fuzzed_data_provider.ConsumeIntegralInRange<uint32_t>(1, outpoints.size()); + const auto num_out = fuzzed_data_provider.ConsumeIntegralInRange<uint32_t>(1, outpoints.size()); + // pick unique outpoints from outpoints as input + for (uint32_t i = 0; i < num_in; i++) { + auto& prevout = PickValue(fuzzed_data_provider, outpoints); + tx_mut.vin.emplace_back(prevout); + // pop the picked outpoint if duplicate input is not allowed + if (!duplicate_input) { + std::swap(prevout, outpoints.back()); + outpoints.pop_back(); + } + } + // output amount will not affect txorphanage + for (uint32_t i = 0; i < num_out; i++) { + tx_mut.vout.emplace_back(CAmount{0}, CScript{}); + } + // restore previously poped outpoints + for (auto& in : tx_mut.vin) { + outpoints.push_back(in.prevout); + } + const auto new_tx = MakeTransactionRef(tx_mut); + // add newly constructed transaction to outpoints + for (uint32_t i = 0; i < num_out; i++) { + outpoints.emplace_back(new_tx->GetHash(), i); + } + return new_tx; + }(); + + // trigger orphanage functions + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10 * DEFAULT_MAX_ORPHAN_TRANSACTIONS) + { + NodeId peer_id = fuzzed_data_provider.ConsumeIntegral<NodeId>(); + + CallOneOf( + fuzzed_data_provider, + [&] { + LOCK(g_cs_orphans); + orphanage.AddChildrenToWorkSet(*tx, orphan_work_set); + }, + [&] { + bool have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); + { + LOCK(g_cs_orphans); + bool get_tx = orphanage.GetTx(tx->GetHash()).first != nullptr; + Assert(have_tx == get_tx); + } + }, + [&] { + bool have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); + // AddTx should return false if tx is too big or already have it + { + LOCK(g_cs_orphans); + Assert(have_tx != orphanage.AddTx(tx, peer_id)); + } + have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); + // tx should already be added since it will not be too big in the test + // have_tx should be true and AddTx should fail + { + LOCK(g_cs_orphans); + Assert(have_tx && !orphanage.AddTx(tx, peer_id)); + } + }, + [&] { + bool have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); + // EraseTx should return 0 if m_orphans doesn't have the tx + { + LOCK(g_cs_orphans); + Assert(have_tx == orphanage.EraseTx(tx->GetHash())); + } + have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); + // have_tx should be false and EraseTx should fail + { + LOCK(g_cs_orphans); + Assert(!have_tx && !orphanage.EraseTx(tx->GetHash())); + } + }, + [&] { + LOCK(g_cs_orphans); + orphanage.EraseForPeer(peer_id); + }, + [&] { + // test mocktime and expiry + SetMockTime(ConsumeTime(fuzzed_data_provider)); + auto size_before = orphanage.Size(); + auto limit = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); + auto n_evicted = WITH_LOCK(g_cs_orphans, return orphanage.LimitOrphans(limit)); + Assert(size_before - n_evicted <= limit); + Assert(orphanage.Size() <= limit); + }); + } + } +} diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index 62b770753a..071193b550 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -5,6 +5,7 @@ #include <test/util/net.h> #include <chainparams.h> +#include <node/eviction.h> #include <net.h> #include <span.h> @@ -58,6 +59,8 @@ std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(int n_candida /*prefer_evict=*/random_context.randbool(), /*m_is_local=*/random_context.randbool(), /*m_network=*/ALL_NETWORKS[random_context.randrange(ALL_NETWORKS.size())], + /*m_noban=*/false, + /*m_conn_type=*/ConnectionType::INBOUND, }); } return candidates; diff --git a/src/test/util/net.h b/src/test/util/net.h index c5dbaeca3e..34ab9958c6 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -6,6 +6,7 @@ #define BITCOIN_TEST_UTIL_NET_H #include <compat.h> +#include <node/eviction.h> #include <netaddress.h> #include <net.h> #include <util/sock.h> diff --git a/src/util/error.cpp b/src/util/error.cpp index af8cbd0353..22a5964279 100644 --- a/src/util/error.cpp +++ b/src/util/error.cpp @@ -35,6 +35,8 @@ bilingual_str TransactionErrorString(const TransactionError err) return Untranslated("External signer not found"); case TransactionError::EXTERNAL_SIGNER_FAILED: return Untranslated("External signer failed to sign"); + case TransactionError::INVALID_PACKAGE: + return Untranslated("Transaction rejected due to invalid package"); // no default case, so the compiler can warn about missing cases } assert(false); diff --git a/src/util/error.h b/src/util/error.h index 4cc35eb1fd..0429de651a 100644 --- a/src/util/error.h +++ b/src/util/error.h @@ -32,6 +32,7 @@ enum class TransactionError { MAX_FEE_EXCEEDED, EXTERNAL_SIGNER_NOT_FOUND, EXTERNAL_SIGNER_FAILED, + INVALID_PACKAGE, }; bilingual_str TransactionErrorString(const TransactionError error); diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index f8230f7a1d..dbd768a758 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -315,12 +315,6 @@ BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const bool read_only, b env = database.env.get(); pdb = database.m_db.get(); strFile = fs::PathToString(database.m_filename); - if (!Exists(std::string("version"))) { - bool fTmp = fReadOnly; - fReadOnly = false; - Write(std::string("version"), CLIENT_VERSION); - fReadOnly = fTmp; - } } void BerkeleyDatabase::Open() diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 81055a5a1a..1d22d0993e 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -397,10 +397,13 @@ std::optional<SelectionResult> AttemptSelection(const CWallet& wallet, const CAm // The knapsack solver has some legacy behavior where it will spend dust outputs. We retain this behavior, so don't filter for positive only here. std::vector<OutputGroup> all_groups = GroupOutputs(wallet, coins, coin_selection_params, eligibility_filter, false /* positive_only */); + CAmount target_with_change = nTargetValue; // While nTargetValue includes the transaction fees for non-input things, it does not include the fee for creating a change output. - // So we need to include that for KnapsackSolver as well, as we are expecting to create a change output. - if (auto knapsack_result{KnapsackSolver(all_groups, nTargetValue + coin_selection_params.m_change_fee, - coin_selection_params.m_min_change_target, coin_selection_params.rng_fast)}) { + // So we need to include that for KnapsackSolver and SRD as well, as we are expecting to create a change output. + if (!coin_selection_params.m_subtract_fee_outputs) { + target_with_change += coin_selection_params.m_change_fee; + } + if (auto knapsack_result{KnapsackSolver(all_groups, target_with_change, coin_selection_params.m_min_change_target, coin_selection_params.rng_fast)}) { knapsack_result->ComputeAndSetWaste(coin_selection_params.m_cost_of_change); results.push_back(*knapsack_result); } @@ -409,7 +412,7 @@ std::optional<SelectionResult> AttemptSelection(const CWallet& wallet, const CAm // barely meets the target. Just use the lower bound change target instead of the randomly // generated one, since SRD will result in a random change amount anyway; avoid making the // target needlessly large. - const CAmount srd_target = nTargetValue + coin_selection_params.m_change_fee + CHANGE_LOWER; + const CAmount srd_target = target_with_change + CHANGE_LOWER; if (auto srd_result{SelectCoinsSRD(positive_groups, srd_target, coin_selection_params.rng_fast)}) { srd_result->ComputeAndSetWaste(coin_selection_params.m_cost_of_change); results.push_back(*srd_result); diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index a04446c840..8afd3f416d 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -883,12 +883,10 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) if (result != DBErrors::LOAD_OK) return result; - // Last client version to open this wallet, was previously the file version number + // Last client version to open this wallet int last_client = CLIENT_VERSION; - m_batch->Read(DBKeys::VERSION, last_client); - - int wallet_version = pwallet->GetVersion(); - pwallet->WalletLogPrintf("Wallet File Version = %d\n", wallet_version > 0 ? wallet_version : last_client); + bool has_last_client = m_batch->Read(DBKeys::VERSION, last_client); + pwallet->WalletLogPrintf("Wallet file version = %d, last client version = %d\n", pwallet->GetVersion(), last_client); pwallet->WalletLogPrintf("Keys: %u plaintext, %u encrypted, %u w/ metadata, %u total. Unknown wallet records: %u\n", wss.nKeys, wss.nCKeys, wss.nKeyMeta, wss.nKeys + wss.nCKeys, wss.m_unknown_records); @@ -909,7 +907,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) if (wss.fIsEncrypted && (last_client == 40000 || last_client == 50000)) return DBErrors::NEED_REWRITE; - if (last_client < CLIENT_VERSION) // Update + if (!has_last_client || last_client != CLIENT_VERSION) // Update m_batch->Write(DBKeys::VERSION, CLIENT_VERSION); if (wss.fAnyUnordered) diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index 62e9bec663..f606f26e70 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -62,8 +62,8 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): self.node2_args = ["-dbcrashratio=24", "-dbcache=16"] + self.base_args # Node3 is a normal node with default args, except will mine full blocks - # and non-standard txs (e.g. txs with "dust" outputs) - self.node3_args = ["-blockmaxweight=4000000", "-acceptnonstdtxn"] + # and txs with "dust" outputs + self.node3_args = ["-blockmaxweight=4000000", "-dustrelayfee=0"] self.extra_args = [self.node0_args, self.node1_args, self.node2_args, self.node3_args] def setup_network(self): @@ -211,7 +211,9 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): self.crashed_on_restart = 0 # Track count of crashes during recovery # Start by creating a lot of utxos on node3 - utxo_list = self.wallet.send_self_transfer_multi(from_node=self.nodes[3], num_outputs=5000)['new_utxos'] + utxo_list = [] + for _ in range(5): + utxo_list.extend(self.wallet.send_self_transfer_multi(from_node=self.nodes[3], num_outputs=1000)['new_utxos']) self.generate(self.nodes[3], 1, sync_fun=self.no_op) assert_equal(len(self.nodes[3].getrawmempool()), 0) self.log.info(f"Prepped {len(utxo_list)} utxo entries") diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index 0b9d651226..3ea412002a 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -46,7 +46,7 @@ class MaxUploadTest(BitcoinTestFramework): self.num_nodes = 1 self.extra_args = [[ "-maxuploadtarget=800M", - "-acceptnonstdtxn=1", + "-datacarriersize=100000", ]] self.supports_cli = False diff --git a/test/functional/feature_minchainwork.py b/test/functional/feature_minchainwork.py index fa10855a98..9d0ad5ef9d 100755 --- a/test/functional/feature_minchainwork.py +++ b/test/functional/feature_minchainwork.py @@ -82,7 +82,7 @@ class MinimumChainWorkTest(BitcoinTestFramework): msg.hashstop = 0 peer.send_and_ping(msg) time.sleep(5) - assert "headers" not in peer.last_message + assert ("headers" not in peer.last_message or len(peer.last_message["headers"].headers) == 0) self.log.info("Generating one more block") self.generate(self.nodes[0], 1) diff --git a/test/functional/feature_nulldummy.py b/test/functional/feature_nulldummy.py index 7a84098a83..9bfb79057e 100755 --- a/test/functional/feature_nulldummy.py +++ b/test/functional/feature_nulldummy.py @@ -19,9 +19,11 @@ from test_framework.blocktools import ( NORMAL_GBT_REQUEST_PARAMS, add_witness_commitment, create_block, - create_transaction, ) -from test_framework.messages import CTransaction +from test_framework.messages import ( + CTransaction, + tx_from_hex, +) from test_framework.script import ( OP_0, OP_TRUE, @@ -31,6 +33,9 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, ) +from test_framework.wallet import getnewdestination +from test_framework.key import ECKey +from test_framework.wallet_util import bytes_to_wif NULLDUMMY_ERROR = "non-mandatory-script-verify-flag (Dummy CHECKMULTISIG argument must be zero)" @@ -55,22 +60,26 @@ class NULLDUMMYTest(BitcoinTestFramework): '-par=1', # Use only one script thread to get the exact reject reason for testing ]] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + def create_transaction(self, *, txid, input_details=None, addr, amount, privkey): + input = {"txid": txid, "vout": 0} + output = {addr: amount} + rawtx = self.nodes[0].createrawtransaction([input], output) + # Details only needed for scripthash or witness spends + input = None if not input_details else [{**input, **input_details}] + signedtx = self.nodes[0].signrawtransactionwithkey(rawtx, [privkey], input) + return tx_from_hex(signedtx["hex"]) def run_test(self): - self.nodes[0].createwallet(wallet_name='wmulti', disable_private_keys=True) - wmulti = self.nodes[0].get_wallet_rpc('wmulti') - w0 = self.nodes[0].get_wallet_rpc(self.default_wallet_name) - self.address = w0.getnewaddress() - self.pubkey = w0.getaddressinfo(self.address)['pubkey'] - self.ms_address = wmulti.addmultisigaddress(1, [self.pubkey])['address'] - self.wit_address = w0.getnewaddress(address_type='p2sh-segwit') - self.wit_ms_address = wmulti.addmultisigaddress(1, [self.pubkey], '', 'p2sh-segwit')['address'] - if not self.options.descriptors: - # Legacy wallets need to import these so that they are watched by the wallet. This is unnecessary (and does not need to be tested) for descriptor wallets - wmulti.importaddress(self.ms_address) - wmulti.importaddress(self.wit_ms_address) + eckey = ECKey() + eckey.generate() + self.privkey = bytes_to_wif(eckey.get_bytes()) + self.pubkey = eckey.get_pubkey().get_bytes().hex() + cms = self.nodes[0].createmultisig(1, [self.pubkey]) + wms = self.nodes[0].createmultisig(1, [self.pubkey], 'p2sh-segwit') + self.ms_address = cms["address"] + ms_unlock_details = {"scriptPubKey": self.nodes[0].validateaddress(self.ms_address)["scriptPubKey"], + "redeemScript": cms["redeemScript"]} + self.wit_ms_address = wms['address'] self.coinbase_blocks = self.generate(self.nodes[0], 2) # block height = 2 coinbase_txid = [] @@ -82,16 +91,23 @@ class NULLDUMMYTest(BitcoinTestFramework): self.lastblocktime = int(time.time()) + self.lastblockheight self.log.info(f"Test 1: NULLDUMMY compliant base transactions should be accepted to mempool and mined before activation [{COINBASE_MATURITY + 3}]") - test1txs = [create_transaction(self.nodes[0], coinbase_txid[0], self.ms_address, amount=49)] + test1txs = [self.create_transaction(txid=coinbase_txid[0], addr=self.ms_address, amount=49, + privkey=self.nodes[0].get_deterministic_priv_key().key)] txid1 = self.nodes[0].sendrawtransaction(test1txs[0].serialize_with_witness().hex(), 0) - test1txs.append(create_transaction(self.nodes[0], txid1, self.ms_address, amount=48)) + test1txs.append(self.create_transaction(txid=txid1, input_details=ms_unlock_details, + addr=self.ms_address, amount=48, + privkey=self.privkey)) txid2 = self.nodes[0].sendrawtransaction(test1txs[1].serialize_with_witness().hex(), 0) - test1txs.append(create_transaction(self.nodes[0], coinbase_txid[1], self.wit_ms_address, amount=49)) + test1txs.append(self.create_transaction(txid=coinbase_txid[1], + addr=self.wit_ms_address, amount=49, + privkey=self.nodes[0].get_deterministic_priv_key().key)) txid3 = self.nodes[0].sendrawtransaction(test1txs[2].serialize_with_witness().hex(), 0) self.block_submit(self.nodes[0], test1txs, accept=True) self.log.info("Test 2: Non-NULLDUMMY base multisig transaction should not be accepted to mempool before activation") - test2tx = create_transaction(self.nodes[0], txid2, self.ms_address, amount=47) + test2tx = self.create_transaction(txid=txid2, input_details=ms_unlock_details, + addr=self.ms_address, amount=47, + privkey=self.privkey) invalidate_nulldummy_tx(test2tx) assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test2tx.serialize_with_witness().hex(), 0) @@ -99,14 +115,19 @@ class NULLDUMMYTest(BitcoinTestFramework): self.block_submit(self.nodes[0], [test2tx], accept=True) self.log.info("Test 4: Non-NULLDUMMY base multisig transaction is invalid after activation") - test4tx = create_transaction(self.nodes[0], test2tx.hash, self.address, amount=46) + test4tx = self.create_transaction(txid=test2tx.hash, input_details=ms_unlock_details, + addr=getnewdestination()[2], amount=46, + privkey=self.privkey) test6txs = [CTransaction(test4tx)] invalidate_nulldummy_tx(test4tx) assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test4tx.serialize_with_witness().hex(), 0) self.block_submit(self.nodes[0], [test4tx], accept=False) self.log.info("Test 5: Non-NULLDUMMY P2WSH multisig transaction invalid after activation") - test5tx = create_transaction(self.nodes[0], txid3, self.wit_address, amount=48) + test5tx = self.create_transaction(txid=txid3, input_details={"scriptPubKey": test1txs[2].vout[0].scriptPubKey.hex(), + "amount": 49, "witnessScript": wms["redeemScript"]}, + addr=getnewdestination(address_type='p2sh-segwit')[2], amount=48, + privkey=self.privkey) test6txs.append(CTransaction(test5tx)) test5tx.wit.vtxinwit[0].scriptWitness.stack[0] = b'\x01' assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test5tx.serialize_with_witness().hex(), 0) diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index 5558c76138..8e5cbba01a 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -4,28 +4,18 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RBF code.""" -from copy import deepcopy from decimal import Decimal from test_framework.messages import ( BIP125_SEQUENCE_NUMBER, COIN, - COutPoint, - CTransaction, - CTxIn, - CTxOut, SEQUENCE_FINAL, ) -from test_framework.script import CScript, OP_DROP from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, ) -from test_framework.script_util import ( - DUMMY_P2WPKH_SCRIPT, - DUMMY_2_P2WPKH_SCRIPT, -) from test_framework.wallet import MiniWallet from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE @@ -35,7 +25,6 @@ class ReplaceByFeeTest(BitcoinTestFramework): self.num_nodes = 2 self.extra_args = [ [ - "-acceptnonstdtxn=1", "-maxorphantx=1000", "-limitancestorcount=50", "-limitancestorsize=101", @@ -99,15 +88,14 @@ class ReplaceByFeeTest(BitcoinTestFramework): self.log.info("Passed") - def make_utxo(self, node, amount, confirmed=True, scriptPubKey=DUMMY_P2WPKH_SCRIPT): + def make_utxo(self, node, amount, *, confirmed=True, scriptPubKey=None): """Create a txout with a given amount and scriptPubKey - confirmed - txouts created will be confirmed in the blockchain; + confirmed - txout created will be confirmed in the blockchain; unconfirmed otherwise. """ - txid, n = self.wallet.send_to(from_node=node, scriptPubKey=scriptPubKey, amount=amount) + txid, n = self.wallet.send_to(from_node=node, scriptPubKey=scriptPubKey or self.wallet.get_scriptPubKey(), amount=amount) - # If requested, ensure txouts are confirmed. if confirmed: mempool_size = len(node.getrawmempool()) while mempool_size > 0: @@ -118,30 +106,24 @@ class ReplaceByFeeTest(BitcoinTestFramework): assert new_size < mempool_size mempool_size = new_size - return COutPoint(int(txid, 16), n) + return self.wallet.get_utxo(txid=txid, vout=n) def test_simple_doublespend(self): """Simple doublespend""" # we use MiniWallet to create a transaction template with inputs correctly set, # and modify the output (amount, scriptPubKey) according to our needs - tx_template = self.wallet.create_self_transfer()['tx'] - - tx1a = deepcopy(tx_template) - tx1a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx1a_hex = tx1a.serialize().hex() - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) + tx = self.wallet.create_self_transfer()["tx"] + tx1a_txid = self.nodes[0].sendrawtransaction(tx.serialize().hex()) # Should fail because we haven't changed the fee - tx1b = deepcopy(tx_template) - tx1b.vout = [CTxOut(1 * COIN, DUMMY_2_P2WPKH_SCRIPT)] - tx1b_hex = tx1b.serialize().hex() + tx.vout[0].scriptPubKey[-1] ^= 1 # This will raise an exception due to insufficient fee - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, 0) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx.serialize().hex(), 0) # Extra 0.1 BTC fee - tx1b.vout[0].nValue -= int(0.1 * COIN) - tx1b_hex = tx1b.serialize().hex() + tx.vout[0].nValue -= int(0.1 * COIN) + tx1b_hex = tx.serialize().hex() # Works when enabled tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, 0) @@ -163,28 +145,28 @@ class ReplaceByFeeTest(BitcoinTestFramework): chain_txids = [] while remaining_value > 1 * COIN: remaining_value -= int(0.1 * COIN) - tx = CTransaction() - tx.vin = [CTxIn(prevout, nSequence=0)] - tx.vout = [CTxOut(remaining_value, CScript([1, OP_DROP] * 15 + [1]))] - tx_hex = tx.serialize().hex() - txid = self.nodes[0].sendrawtransaction(tx_hex, 0) - chain_txids.append(txid) - prevout = COutPoint(int(txid, 16), 0) + prevout = self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=prevout, + sequence=0, + fee=Decimal("0.1"), + )["new_utxo"] + chain_txids.append(prevout["txid"]) # Whether the double-spend is allowed is evaluated by including all # child fees - 4 BTC - so this attempt is rejected. - dbl_tx = CTransaction() - dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] - dbl_tx.vout = [CTxOut(initial_nValue - 3 * COIN, DUMMY_P2WPKH_SCRIPT)] + dbl_tx = self.wallet.create_self_transfer( + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=Decimal("3"), + )["tx"] dbl_tx_hex = dbl_tx.serialize().hex() # This will raise an exception due to insufficient fee assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) # Accepted with sufficient fee - dbl_tx = CTransaction() - dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] - dbl_tx.vout = [CTxOut(int(0.1 * COIN), DUMMY_P2WPKH_SCRIPT)] + dbl_tx.vout[0].nValue = int(0.1 * COIN) dbl_tx_hex = dbl_tx.serialize().hex() self.nodes[0].sendrawtransaction(dbl_tx_hex, 0) @@ -208,22 +190,19 @@ class ReplaceByFeeTest(BitcoinTestFramework): if txout_value < fee: return - vout = [CTxOut(txout_value, CScript([i+1])) - for i in range(tree_width)] - tx = CTransaction() - tx.vin = [CTxIn(prevout, nSequence=0)] - tx.vout = vout - tx_hex = tx.serialize().hex() + tx = self.wallet.send_self_transfer_multi( + utxos_to_spend=[prevout], + from_node=self.nodes[0], + sequence=0, + num_outputs=tree_width, + amount_per_output=txout_value, + ) - assert len(tx.serialize()) < 100000 - txid = self.nodes[0].sendrawtransaction(tx_hex, 0) - yield tx + yield tx["txid"] _total_txs[0] += 1 - txid = int(txid, 16) - - for i, txout in enumerate(tx.vout): - for x in branch(COutPoint(txid, i), txout_value, + for utxo in tx["new_utxos"]: + for x in branch(utxo, txout_value, max_txs, tree_width=tree_width, fee=fee, _total_txs=_total_txs): @@ -235,25 +214,26 @@ class ReplaceByFeeTest(BitcoinTestFramework): assert_equal(len(tree_txs), n) # Attempt double-spend, will fail because too little fee paid - dbl_tx = CTransaction() - dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] - dbl_tx.vout = [CTxOut(initial_nValue - fee * n, DUMMY_P2WPKH_SCRIPT)] - dbl_tx_hex = dbl_tx.serialize().hex() + dbl_tx_hex = self.wallet.create_self_transfer( + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=(Decimal(fee) / COIN) * n, + )["hex"] # This will raise an exception due to insufficient fee assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) # 0.1 BTC fee is enough - dbl_tx = CTransaction() - dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] - dbl_tx.vout = [CTxOut(initial_nValue - fee * n - int(0.1 * COIN), DUMMY_P2WPKH_SCRIPT)] - dbl_tx_hex = dbl_tx.serialize().hex() + dbl_tx_hex = self.wallet.create_self_transfer( + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=(Decimal(fee) / COIN) * n + Decimal("0.1"), + )["hex"] self.nodes[0].sendrawtransaction(dbl_tx_hex, 0) mempool = self.nodes[0].getrawmempool() - for tx in tree_txs: - tx.rehash() - assert tx.hash not in mempool + for txid in tree_txs: + assert txid not in mempool # Try again, but with more total transactions than the "max txs # double-spent at once" anti-DoS limit. @@ -263,33 +243,36 @@ class ReplaceByFeeTest(BitcoinTestFramework): tree_txs = list(branch(tx0_outpoint, initial_nValue, n, fee=fee)) assert_equal(len(tree_txs), n) - dbl_tx = CTransaction() - dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] - dbl_tx.vout = [CTxOut(initial_nValue - 2 * fee * n, DUMMY_P2WPKH_SCRIPT)] - dbl_tx_hex = dbl_tx.serialize().hex() + dbl_tx_hex = self.wallet.create_self_transfer( + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=2 * (Decimal(fee) / COIN) * n, + )["hex"] # This will raise an exception assert_raises_rpc_error(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) - for tx in tree_txs: - tx.rehash() - self.nodes[0].getrawtransaction(tx.hash) + for txid in tree_txs: + self.nodes[0].getrawtransaction(txid) def test_replacement_feeperkb(self): """Replacement requires fee-per-KB to be higher""" tx0_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) - tx1a = CTransaction() - tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0)] - tx1a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx1a_hex = tx1a.serialize().hex() - self.nodes[0].sendrawtransaction(tx1a_hex, 0) + self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=Decimal("0.1"), + ) # Higher fee, but the fee per KB is much lower, so the replacement is # rejected. - tx1b = CTransaction() - tx1b.vin = [CTxIn(tx0_outpoint, nSequence=0)] - tx1b.vout = [CTxOut(int(0.001 * COIN), CScript([b'a' * 999000]))] - tx1b_hex = tx1b.serialize().hex() + tx1b_hex = self.wallet.create_self_transfer_multi( + utxos_to_spend=[tx0_outpoint], + sequence=0, + num_outputs=100, + amount_per_output=1000, + )["hex"] # This will raise an exception due to insufficient fee assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, 0) @@ -299,37 +282,36 @@ class ReplaceByFeeTest(BitcoinTestFramework): utxo1 = self.make_utxo(self.nodes[0], int(1.2 * COIN)) utxo2 = self.make_utxo(self.nodes[0], 3 * COIN) - tx1a = CTransaction() - tx1a.vin = [CTxIn(utxo1, nSequence=0)] - tx1a.vout = [CTxOut(int(1.1 * COIN), DUMMY_P2WPKH_SCRIPT)] - tx1a_hex = tx1a.serialize().hex() - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) - - tx1a_txid = int(tx1a_txid, 16) + tx1a_utxo = self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=utxo1, + sequence=0, + fee=Decimal("0.1"), + )["new_utxo"] # Direct spend an output of the transaction we're replacing. - tx2 = CTransaction() - tx2.vin = [CTxIn(utxo1, nSequence=0), CTxIn(utxo2, nSequence=0)] - tx2.vin.append(CTxIn(COutPoint(tx1a_txid, 0), nSequence=0)) - tx2.vout = tx1a.vout - tx2_hex = tx2.serialize().hex() + tx2_hex = self.wallet.create_self_transfer_multi( + utxos_to_spend=[utxo1, utxo2, tx1a_utxo], + sequence=0, + amount_per_output=int(COIN * tx1a_utxo["value"]), + )["hex"] # This will raise an exception assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, 0) # Spend tx1a's output to test the indirect case. - tx1b = CTransaction() - tx1b.vin = [CTxIn(COutPoint(tx1a_txid, 0), nSequence=0)] - tx1b.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx1b_hex = tx1b.serialize().hex() - tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, 0) - tx1b_txid = int(tx1b_txid, 16) + tx1b_utxo = self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx1a_utxo, + sequence=0, + fee=Decimal("0.1"), + )["new_utxo"] - tx2 = CTransaction() - tx2.vin = [CTxIn(utxo1, nSequence=0), CTxIn(utxo2, nSequence=0), - CTxIn(COutPoint(tx1b_txid, 0))] - tx2.vout = tx1a.vout - tx2_hex = tx2.serialize().hex() + tx2_hex = self.wallet.create_self_transfer_multi( + utxos_to_spend=[utxo1, utxo2, tx1b_utxo], + sequence=0, + amount_per_output=int(COIN * tx1a_utxo["value"]), + )["hex"] # This will raise an exception assert_raises_rpc_error(-26, "bad-txns-spends-conflicting-tx", self.nodes[0].sendrawtransaction, tx2_hex, 0) @@ -337,18 +319,20 @@ class ReplaceByFeeTest(BitcoinTestFramework): def test_new_unconfirmed_inputs(self): """Replacements that add new unconfirmed inputs are rejected""" confirmed_utxo = self.make_utxo(self.nodes[0], int(1.1 * COIN)) - unconfirmed_utxo = self.make_utxo(self.nodes[0], int(0.1 * COIN), False) + unconfirmed_utxo = self.make_utxo(self.nodes[0], int(0.1 * COIN), confirmed=False) - tx1 = CTransaction() - tx1.vin = [CTxIn(confirmed_utxo)] - tx1.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx1_hex = tx1.serialize().hex() - self.nodes[0].sendrawtransaction(tx1_hex, 0) + self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=confirmed_utxo, + sequence=0, + fee=Decimal("0.1"), + ) - tx2 = CTransaction() - tx2.vin = [CTxIn(confirmed_utxo), CTxIn(unconfirmed_utxo)] - tx2.vout = tx1.vout - tx2_hex = tx2.serialize().hex() + tx2_hex = self.wallet.create_self_transfer_multi( + utxos_to_spend=[confirmed_utxo, unconfirmed_utxo], + sequence=0, + amount_per_output=1 * COIN, + )["hex"] # This will raise an exception assert_raises_rpc_error(-26, "replacement-adds-unconfirmed", self.nodes[0].sendrawtransaction, tx2_hex, 0) @@ -364,45 +348,39 @@ class ReplaceByFeeTest(BitcoinTestFramework): fee = int(0.0001 * COIN) split_value = int((initial_nValue - fee) / (MAX_REPLACEMENT_LIMIT + 1)) - outputs = [] - for _ in range(MAX_REPLACEMENT_LIMIT + 1): - outputs.append(CTxOut(split_value, CScript([1]))) - - splitting_tx = CTransaction() - splitting_tx.vin = [CTxIn(utxo, nSequence=0)] - splitting_tx.vout = outputs - splitting_tx_hex = splitting_tx.serialize().hex() - - txid = self.nodes[0].sendrawtransaction(splitting_tx_hex, 0) - txid = int(txid, 16) + splitting_tx_utxos = self.wallet.send_self_transfer_multi( + from_node=self.nodes[0], + utxos_to_spend=[utxo], + sequence=0, + num_outputs=MAX_REPLACEMENT_LIMIT + 1, + amount_per_output=split_value, + )["new_utxos"] # Now spend each of those outputs individually - for i in range(MAX_REPLACEMENT_LIMIT + 1): - tx_i = CTransaction() - tx_i.vin = [CTxIn(COutPoint(txid, i), nSequence=0)] - tx_i.vout = [CTxOut(split_value - fee, DUMMY_P2WPKH_SCRIPT)] - tx_i_hex = tx_i.serialize().hex() - self.nodes[0].sendrawtransaction(tx_i_hex, 0) + for utxo in splitting_tx_utxos: + self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=utxo, + sequence=0, + fee=Decimal(fee) / COIN, + ) # Now create doublespend of the whole lot; should fail. # Need a big enough fee to cover all spending transactions and have # a higher fee rate double_spend_value = (split_value - 100 * fee) * (MAX_REPLACEMENT_LIMIT + 1) - inputs = [] - for i in range(MAX_REPLACEMENT_LIMIT + 1): - inputs.append(CTxIn(COutPoint(txid, i), nSequence=0)) - double_tx = CTransaction() - double_tx.vin = inputs - double_tx.vout = [CTxOut(double_spend_value, CScript([b'a']))] + double_tx = self.wallet.create_self_transfer_multi( + utxos_to_spend=splitting_tx_utxos, + sequence=0, + amount_per_output=double_spend_value, + )["tx"] double_tx_hex = double_tx.serialize().hex() # This will raise an exception assert_raises_rpc_error(-26, "too many potential replacements", self.nodes[0].sendrawtransaction, double_tx_hex, 0) # If we remove an input, it should pass - double_tx = CTransaction() - double_tx.vin = inputs[0:-1] - double_tx.vout = [CTxOut(double_spend_value, CScript([b'a']))] + double_tx.vin.pop() double_tx_hex = double_tx.serialize().hex() self.nodes[0].sendrawtransaction(double_tx_hex, 0) @@ -497,20 +475,22 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx0_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) # Create a non-opting in transaction - tx1a = CTransaction() - tx1a.vin = [CTxIn(tx0_outpoint, nSequence=SEQUENCE_FINAL)] - tx1a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx1a_hex = tx1a.serialize().hex() - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) + tx1a_utxo = self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx0_outpoint, + sequence=SEQUENCE_FINAL, + fee=Decimal("0.1"), + )["new_utxo"] # This transaction isn't shown as replaceable - assert_equal(self.nodes[0].getmempoolentry(tx1a_txid)['bip125-replaceable'], False) + assert_equal(self.nodes[0].getmempoolentry(tx1a_utxo["txid"])['bip125-replaceable'], False) # Shouldn't be able to double-spend - tx1b = CTransaction() - tx1b.vin = [CTxIn(tx0_outpoint, nSequence=0)] - tx1b.vout = [CTxOut(int(0.9 * COIN), DUMMY_P2WPKH_SCRIPT)] - tx1b_hex = tx1b.serialize().hex() + tx1b_hex = self.wallet.create_self_transfer( + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=Decimal("0.2"), + )["hex"] # This will raise an exception assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx1b_hex, 0) @@ -518,17 +498,19 @@ class ReplaceByFeeTest(BitcoinTestFramework): tx1_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) # Create a different non-opting in transaction - tx2a = CTransaction() - tx2a.vin = [CTxIn(tx1_outpoint, nSequence=0xfffffffe)] - tx2a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx2a_hex = tx2a.serialize().hex() - tx2a_txid = self.nodes[0].sendrawtransaction(tx2a_hex, 0) + tx2a_utxo = self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx1_outpoint, + sequence=0xfffffffe, + fee=Decimal("0.1"), + )["new_utxo"] # Still shouldn't be able to double-spend - tx2b = CTransaction() - tx2b.vin = [CTxIn(tx1_outpoint, nSequence=0)] - tx2b.vout = [CTxOut(int(0.9 * COIN), DUMMY_P2WPKH_SCRIPT)] - tx2b_hex = tx2b.serialize().hex() + tx2b_hex = self.wallet.create_self_transfer( + utxo_to_spend=tx1_outpoint, + sequence=0, + fee=Decimal("0.2"), + )["hex"] # This will raise an exception assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx2b_hex, 0) @@ -537,34 +519,31 @@ class ReplaceByFeeTest(BitcoinTestFramework): # opt-in on one of the inputs # Transaction should be replaceable on either input - tx1a_txid = int(tx1a_txid, 16) - tx2a_txid = int(tx2a_txid, 16) - - tx3a = CTransaction() - tx3a.vin = [CTxIn(COutPoint(tx1a_txid, 0), nSequence=SEQUENCE_FINAL), - CTxIn(COutPoint(tx2a_txid, 0), nSequence=0xfffffffd)] - tx3a.vout = [CTxOut(int(0.9 * COIN), CScript([b'c'])), CTxOut(int(0.9 * COIN), CScript([b'd']))] - tx3a_hex = tx3a.serialize().hex() - - tx3a_txid = self.nodes[0].sendrawtransaction(tx3a_hex, 0) + tx3a_txid = self.wallet.send_self_transfer_multi( + from_node=self.nodes[0], + utxos_to_spend=[tx1a_utxo, tx2a_utxo], + sequence=[SEQUENCE_FINAL, 0xfffffffd], + fee_per_output=int(0.1 * COIN), + )["txid"] # This transaction is shown as replaceable assert_equal(self.nodes[0].getmempoolentry(tx3a_txid)['bip125-replaceable'], True) - tx3b = CTransaction() - tx3b.vin = [CTxIn(COutPoint(tx1a_txid, 0), nSequence=0)] - tx3b.vout = [CTxOut(int(0.5 * COIN), DUMMY_P2WPKH_SCRIPT)] - tx3b_hex = tx3b.serialize().hex() - - tx3c = CTransaction() - tx3c.vin = [CTxIn(COutPoint(tx2a_txid, 0), nSequence=0)] - tx3c.vout = [CTxOut(int(0.5 * COIN), DUMMY_P2WPKH_SCRIPT)] - tx3c_hex = tx3c.serialize().hex() + self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx1a_utxo, + sequence=0, + fee=Decimal("0.4"), + ) - self.nodes[0].sendrawtransaction(tx3b_hex, 0) # If tx3b was accepted, tx3c won't look like a replacement, # but make sure it is accepted anyway - self.nodes[0].sendrawtransaction(tx3c_hex, 0) + self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx2a_utxo, + sequence=0, + fee=Decimal("0.4"), + ) def test_prioritised_transactions(self): # Ensure that fee deltas used via prioritisetransaction are @@ -573,17 +552,20 @@ class ReplaceByFeeTest(BitcoinTestFramework): # 1. Check that feeperkb uses modified fees tx0_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) - tx1a = CTransaction() - tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0)] - tx1a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx1a_hex = tx1a.serialize().hex() - tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) + tx1a_txid = self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx0_outpoint, + sequence=0, + fee=Decimal("0.1"), + )["txid"] # Higher fee, but the actual fee per KB is much lower. - tx1b = CTransaction() - tx1b.vin = [CTxIn(tx0_outpoint, nSequence=0)] - tx1b.vout = [CTxOut(int(0.001 * COIN), CScript([b'a' * 740000]))] - tx1b_hex = tx1b.serialize().hex() + tx1b_hex = self.wallet.create_self_transfer_multi( + utxos_to_spend=[tx0_outpoint], + sequence=0, + num_outputs=100, + amount_per_output=int(0.00001 * COIN), + )["hex"] # Verify tx1b cannot replace tx1a. assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, 0) @@ -599,27 +581,29 @@ class ReplaceByFeeTest(BitcoinTestFramework): # 2. Check that absolute fee checks use modified fee. tx1_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) - tx2a = CTransaction() - tx2a.vin = [CTxIn(tx1_outpoint, nSequence=0)] - tx2a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] - tx2a_hex = tx2a.serialize().hex() - self.nodes[0].sendrawtransaction(tx2a_hex, 0) + # tx2a + self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=tx1_outpoint, + sequence=0, + fee=Decimal("0.1"), + ) # Lower fee, but we'll prioritise it - tx2b = CTransaction() - tx2b.vin = [CTxIn(tx1_outpoint, nSequence=0)] - tx2b.vout = [CTxOut(int(1.01 * COIN), DUMMY_P2WPKH_SCRIPT)] - tx2b.rehash() - tx2b_hex = tx2b.serialize().hex() + tx2b = self.wallet.create_self_transfer( + utxo_to_spend=tx1_outpoint, + sequence=0, + fee=Decimal("0.09"), + ) # Verify tx2b cannot replace tx2a. - assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx2b_hex, 0) + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx2b["hex"], 0) # Now prioritise tx2b to have a higher modified fee - self.nodes[0].prioritisetransaction(txid=tx2b.hash, fee_delta=int(0.1 * COIN)) + self.nodes[0].prioritisetransaction(txid=tx2b["txid"], fee_delta=int(0.1 * COIN)) # tx2b should now be accepted - tx2b_txid = self.nodes[0].sendrawtransaction(tx2b_hex, 0) + tx2b_txid = self.nodes[0].sendrawtransaction(tx2b["hex"], 0) assert tx2b_txid in self.nodes[0].getrawmempool() diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index e92f73304b..7080662b49 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -23,7 +23,7 @@ class MempoolLimitTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 1 self.extra_args = [[ - "-acceptnonstdtxn=1", + "-datacarriersize=100000", "-maxmempool=5", "-spendzeroconfchange=0", ]] diff --git a/test/functional/mempool_updatefromblock.py b/test/functional/mempool_updatefromblock.py index 51de582ce0..f97c2223a6 100755 --- a/test/functional/mempool_updatefromblock.py +++ b/test/functional/mempool_updatefromblock.py @@ -12,6 +12,9 @@ import time from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal +from test_framework.address import key_to_p2pkh +from test_framework.wallet_util import bytes_to_wif +from test_framework.key import ECKey class MempoolUpdateFromBlockTest(BitcoinTestFramework): @@ -19,8 +22,13 @@ class MempoolUpdateFromBlockTest(BitcoinTestFramework): self.num_nodes = 1 self.extra_args = [['-limitdescendantsize=1000', '-limitancestorsize=1000', '-limitancestorcount=100']] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + def get_new_address(self): + key = ECKey() + key.generate() + pubkey = key.get_pubkey().get_bytes() + address = key_to_p2pkh(pubkey) + self.priv_keys.append(bytes_to_wif(key.get_bytes())) + return address def transaction_graph_test(self, size, n_tx_to_mine=None, start_input_txid='', end_address='', fee=Decimal(0.00100000)): """Create an acyclic tournament (a type of directed graph) of transactions and use it for testing. @@ -38,11 +46,12 @@ class MempoolUpdateFromBlockTest(BitcoinTestFramework): More details: https://en.wikipedia.org/wiki/Tournament_(graph_theory) """ + self.priv_keys = [self.nodes[0].get_deterministic_priv_key().key] if not start_input_txid: start_input_txid = self.nodes[0].getblock(self.nodes[0].getblockhash(1))['tx'][0] if not end_address: - end_address = self.nodes[0].getnewaddress() + end_address = self.get_new_address() first_block_hash = '' tx_id = [] @@ -74,7 +83,7 @@ class MempoolUpdateFromBlockTest(BitcoinTestFramework): output_value = ((inputs_value - fee) / Decimal(n_outputs)).quantize(Decimal('0.00000001')) outputs = {} for _ in range(n_outputs): - outputs[self.nodes[0].getnewaddress()] = output_value + outputs[self.get_new_address()] = output_value else: output_value = (inputs_value - fee).quantize(Decimal('0.00000001')) outputs = {end_address: output_value} @@ -84,7 +93,7 @@ class MempoolUpdateFromBlockTest(BitcoinTestFramework): # Create a new transaction. unsigned_raw_tx = self.nodes[0].createrawtransaction(inputs, outputs) - signed_raw_tx = self.nodes[0].signrawtransactionwithwallet(unsigned_raw_tx) + signed_raw_tx = self.nodes[0].signrawtransactionwithkey(unsigned_raw_tx, self.priv_keys) tx_id.append(self.nodes[0].sendrawtransaction(signed_raw_tx['hex'])) tx_size.append(self.nodes[0].getmempoolentry(tx_id[-1])['vsize']) diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index fb3974c1d5..64e66ac30a 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -26,7 +26,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): self.num_nodes = 1 self.extra_args = [[ "-printpriority=1", - "-acceptnonstdtxn=1", + "-datacarriersize=100000", ]] * self.num_nodes self.supports_cli = False diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index 139f4d64e7..28efd5a81e 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -60,7 +60,6 @@ class InvalidTxRequestTest(BitcoinTestFramework): block.solve() # Save the coinbase for later block1 = block - tip = block.sha256 node.p2ps[0].send_blocks_and_test([block], node, success=True) self.log.info("Mature the block.") @@ -93,24 +92,24 @@ class InvalidTxRequestTest(BitcoinTestFramework): SCRIPT_PUB_KEY_OP_TRUE = b'\x51\x75' * 15 + b'\x51' tx_withhold = CTransaction() tx_withhold.vin.append(CTxIn(outpoint=COutPoint(block1.vtx[0].sha256, 0))) - tx_withhold.vout.append(CTxOut(nValue=50 * COIN - 12000, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_withhold.vout = [CTxOut(nValue=25 * COIN - 12000, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)] * 2 tx_withhold.calc_sha256() # Our first orphan tx with some outputs to create further orphan txs tx_orphan_1 = CTransaction() tx_orphan_1.vin.append(CTxIn(outpoint=COutPoint(tx_withhold.sha256, 0))) - tx_orphan_1.vout = [CTxOut(nValue=10 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)] * 3 + tx_orphan_1.vout = [CTxOut(nValue=8 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)] * 3 tx_orphan_1.calc_sha256() # A valid transaction with low fee tx_orphan_2_no_fee = CTransaction() tx_orphan_2_no_fee.vin.append(CTxIn(outpoint=COutPoint(tx_orphan_1.sha256, 0))) - tx_orphan_2_no_fee.vout.append(CTxOut(nValue=10 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_orphan_2_no_fee.vout.append(CTxOut(nValue=8 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) # A valid transaction with sufficient fee tx_orphan_2_valid = CTransaction() tx_orphan_2_valid.vin.append(CTxIn(outpoint=COutPoint(tx_orphan_1.sha256, 1))) - tx_orphan_2_valid.vout.append(CTxOut(nValue=10 * COIN - 12000, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_orphan_2_valid.vout.append(CTxOut(nValue=8 * COIN - 12000, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) tx_orphan_2_valid.calc_sha256() # An invalid transaction with negative fee @@ -157,6 +156,7 @@ class InvalidTxRequestTest(BitcoinTestFramework): with node.assert_debug_log(['orphanage overflow, removed 1 tx']): node.p2ps[0].send_txs_and_test(orphan_tx_pool, node, success=False) + self.log.info('Test orphan with rejected parents') rejected_parent = CTransaction() rejected_parent.vin.append(CTxIn(outpoint=COutPoint(tx_orphan_2_invalid.sha256, 0))) rejected_parent.vout.append(CTxOut(nValue=11 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) @@ -164,6 +164,64 @@ class InvalidTxRequestTest(BitcoinTestFramework): with node.assert_debug_log(['not keeping orphan with rejected parents {}'.format(rejected_parent.hash)]): node.p2ps[0].send_txs_and_test([rejected_parent], node, success=False) + self.log.info('Test that a peer disconnection causes erase its transactions from the orphan pool') + with node.assert_debug_log(['Erased 100 orphan tx from peer=25']): + self.reconnect_p2p(num_connections=1) + + self.log.info('Test that a transaction in the orphan pool is included in a new tip block causes erase this transaction from the orphan pool') + tx_withhold_until_block_A = CTransaction() + tx_withhold_until_block_A.vin.append(CTxIn(outpoint=COutPoint(tx_withhold.sha256, 1))) + tx_withhold_until_block_A.vout = [CTxOut(nValue=12 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)] * 2 + tx_withhold_until_block_A.calc_sha256() + + tx_orphan_include_by_block_A = CTransaction() + tx_orphan_include_by_block_A.vin.append(CTxIn(outpoint=COutPoint(tx_withhold_until_block_A.sha256, 0))) + tx_orphan_include_by_block_A.vout.append(CTxOut(nValue=12 * COIN - 12000, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_orphan_include_by_block_A.calc_sha256() + + self.log.info('Send the orphan ... ') + node.p2ps[0].send_txs_and_test([tx_orphan_include_by_block_A], node, success=False) + + tip = int(node.getbestblockhash(), 16) + height = node.getblockcount() + 1 + block_A = create_block(tip, create_coinbase(height)) + block_A.vtx.extend([tx_withhold, tx_withhold_until_block_A, tx_orphan_include_by_block_A]) + block_A.hashMerkleRoot = block_A.calc_merkle_root() + block_A.solve() + + self.log.info('Send the block that includes the previous orphan ... ') + with node.assert_debug_log(["Erased 1 orphan tx included or conflicted by block"]): + node.p2ps[0].send_blocks_and_test([block_A], node, success=True) + + self.log.info('Test that a transaction in the orphan pool conflicts with a new tip block causes erase this transaction from the orphan pool') + tx_withhold_until_block_B = CTransaction() + tx_withhold_until_block_B.vin.append(CTxIn(outpoint=COutPoint(tx_withhold_until_block_A.sha256, 1))) + tx_withhold_until_block_B.vout.append(CTxOut(nValue=11 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_withhold_until_block_B.calc_sha256() + + tx_orphan_include_by_block_B = CTransaction() + tx_orphan_include_by_block_B.vin.append(CTxIn(outpoint=COutPoint(tx_withhold_until_block_B.sha256, 0))) + tx_orphan_include_by_block_B.vout.append(CTxOut(nValue=10 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_orphan_include_by_block_B.calc_sha256() + + tx_orphan_conflict_by_block_B = CTransaction() + tx_orphan_conflict_by_block_B.vin.append(CTxIn(outpoint=COutPoint(tx_withhold_until_block_B.sha256, 0))) + tx_orphan_conflict_by_block_B.vout.append(CTxOut(nValue=9 * COIN, scriptPubKey=SCRIPT_PUB_KEY_OP_TRUE)) + tx_orphan_conflict_by_block_B.calc_sha256() + self.log.info('Send the orphan ... ') + node.p2ps[0].send_txs_and_test([tx_orphan_conflict_by_block_B], node, success=False) + + tip = int(node.getbestblockhash(), 16) + height = node.getblockcount() + 1 + block_B = create_block(tip, create_coinbase(height)) + block_B.vtx.extend([tx_withhold_until_block_B, tx_orphan_include_by_block_B]) + block_B.hashMerkleRoot = block_B.calc_merkle_root() + block_B.solve() + + self.log.info('Send the block that includes a transaction which conflicts with the previous orphan ... ') + with node.assert_debug_log(["Erased 1 orphan tx included or conflicted by block"]): + node.p2ps[0].send_blocks_and_test([block_B], node, success=True) + if __name__ == '__main__': InvalidTxRequestTest().main() diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 952f1e5cc5..eedb7e6fa1 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -371,6 +371,10 @@ class SegWitTest(BitcoinTestFramework): block1 = self.build_next_block() block1.solve() + # Send an empty headers message, to clear out any prior getheaders + # messages that our peer may be waiting for us on. + self.test_node.send_message(msg_headers()) + self.test_node.announce_block_and_wait_for_getdata(block1, use_header=False) assert self.test_node.last_message["getdata"].inv[0].type == blocktype test_witness_block(self.nodes[0], self.test_node, block1, True) diff --git a/test/functional/p2p_timeouts.py b/test/functional/p2p_timeouts.py index f0abbc7d8b..15a879ae3c 100755 --- a/test/functional/p2p_timeouts.py +++ b/test/functional/p2p_timeouts.py @@ -94,6 +94,11 @@ class TimeoutsTest(BitcoinTestFramework): no_version_node.wait_for_disconnect(timeout=1) no_send_node.wait_for_disconnect(timeout=1) + self.stop_nodes(0) + self.nodes[0].assert_start_raises_init_error( + expected_msg='Error: peertimeout must be a positive integer.', + extra_args=['-peertimeout=0'], + ) if __name__ == '__main__': TimeoutsTest().main() diff --git a/test/functional/rpc_packages.py b/test/functional/rpc_packages.py index 63533affd0..f687849287 100755 --- a/test/functional/rpc_packages.py +++ b/test/functional/rpc_packages.py @@ -15,16 +15,20 @@ from test_framework.messages import ( CTxInWitness, tx_from_hex, ) +from test_framework.p2p import P2PTxInvStore from test_framework.script import ( CScript, OP_TRUE, ) from test_framework.util import ( assert_equal, + assert_fee_amount, + assert_raises_rpc_error, ) from test_framework.wallet import ( create_child_with_parents, create_raw_chain, + DEFAULT_FEE, make_chain, ) @@ -51,7 +55,7 @@ class RPCPackagesTest(BitcoinTestFramework): self.address = node.get_deterministic_priv_key().address self.coins = [] # The last 100 coinbase transactions are premature - for b in self.generatetoaddress(node, 200, self.address)[:100]: + for b in self.generatetoaddress(node, 220, self.address)[:-100]: coinbase = node.getblock(blockhash=b, verbosity=2)["tx"][0] self.coins.append({ "txid": coinbase["txid"], @@ -82,7 +86,7 @@ class RPCPackagesTest(BitcoinTestFramework): self.test_multiple_parents() self.test_conflicting() self.test_rbf() - + self.test_submitpackage() def test_independent(self): self.log.info("Test multiple independent transactions in a package") @@ -132,8 +136,7 @@ class RPCPackagesTest(BitcoinTestFramework): def test_chain(self): node = self.nodes[0] - first_coin = self.coins.pop() - (chain_hex, chain_txns) = create_raw_chain(node, first_coin, self.address, self.privkeys) + (chain_hex, chain_txns) = create_raw_chain(node, self.coins.pop(), self.address, self.privkeys) self.log.info("Check that testmempoolaccept requires packages to be sorted by dependency") assert_equal(node.testmempoolaccept(rawtxs=chain_hex[::-1]), [{"txid": tx.rehash(), "wtxid": tx.getwtxid(), "package-error": "package-not-sorted"} for tx in chain_txns[::-1]]) @@ -306,5 +309,127 @@ class RPCPackagesTest(BitcoinTestFramework): }] self.assert_testres_equal(self.independent_txns_hex + [signed_replacement_tx["hex"]], testres_rbf_package) + def assert_equal_package_results(self, node, testmempoolaccept_result, submitpackage_result): + """Assert that a successful submitpackage result is consistent with testmempoolaccept + results and getmempoolentry info. Note that the result structs are different and, due to + policy differences between testmempoolaccept and submitpackage (i.e. package feerate), + some information may be different. + """ + for testres_tx in testmempoolaccept_result: + # Grab this result from the submitpackage_result + submitres_tx = submitpackage_result["tx-results"][testres_tx["wtxid"]] + assert_equal(submitres_tx["txid"], testres_tx["txid"]) + # No "allowed" if the tx was already in the mempool + if "allowed" in testres_tx and testres_tx["allowed"]: + assert_equal(submitres_tx["vsize"], testres_tx["vsize"]) + assert_equal(submitres_tx["fees"]["base"], testres_tx["fees"]["base"]) + entry_info = node.getmempoolentry(submitres_tx["txid"]) + assert_equal(submitres_tx["vsize"], entry_info["vsize"]) + assert_equal(submitres_tx["fees"]["base"], entry_info["fees"]["base"]) + + def test_submit_child_with_parents(self, num_parents, partial_submit): + node = self.nodes[0] + peer = node.add_p2p_connection(P2PTxInvStore()) + # Test a package with num_parents parents and 1 child transaction. + package_hex = [] + package_txns = [] + values = [] + scripts = [] + for _ in range(num_parents): + parent_coin = self.coins.pop() + value = parent_coin["amount"] + (tx, txhex, value, spk) = make_chain(node, self.address, self.privkeys, parent_coin["txid"], value) + package_hex.append(txhex) + package_txns.append(tx) + values.append(value) + scripts.append(spk) + if partial_submit and random.choice([True, False]): + node.sendrawtransaction(txhex) + child_hex = create_child_with_parents(node, self.address, self.privkeys, package_txns, values, scripts) + package_hex.append(child_hex) + package_txns.append(tx_from_hex(child_hex)) + + testmempoolaccept_result = node.testmempoolaccept(rawtxs=package_hex) + submitpackage_result = node.submitpackage(package=package_hex) + + # Check that each result is present, with the correct size and fees + for i in range(num_parents + 1): + tx = package_txns[i] + wtxid = tx.getwtxid() + assert wtxid in submitpackage_result["tx-results"] + tx_result = submitpackage_result["tx-results"][wtxid] + assert_equal(tx_result, { + "txid": tx.rehash(), + "vsize": tx.get_vsize(), + "fees": { + "base": DEFAULT_FEE, + } + }) + + # submitpackage result should be consistent with testmempoolaccept and getmempoolentry + self.assert_equal_package_results(node, testmempoolaccept_result, submitpackage_result) + + # Package feerate is calculated for the remaining transactions after deduplication and + # individual submission. If only 0 or 1 transaction is left, e.g. because all transactions + # had high-feerates or were already in the mempool, no package feerate is provided. + # In this case, since all of the parents have high fees, each is accepted individually. + assert "package-feerate" not in submitpackage_result + + # The node should announce each transaction. No guarantees for propagation. + peer.wait_for_broadcast([tx.getwtxid() for tx in package_txns]) + self.generate(node, 1) + + + def test_submit_cpfp(self): + node = self.nodes[0] + peer = node.add_p2p_connection(P2PTxInvStore()) + + # 2 parent 1 child CPFP. First parent pays high fees, second parent pays 0 fees and is + # fee-bumped by the child. + coin_rich = self.coins.pop() + coin_poor = self.coins.pop() + tx_rich, hex_rich, value_rich, spk_rich = make_chain(node, self.address, self.privkeys, coin_rich["txid"], coin_rich["amount"]) + tx_poor, hex_poor, value_poor, spk_poor = make_chain(node, self.address, self.privkeys, coin_poor["txid"], coin_poor["amount"], fee=0) + package_txns = [tx_rich, tx_poor] + hex_child = create_child_with_parents(node, self.address, self.privkeys, package_txns, [value_rich, value_poor], [spk_rich, spk_poor]) + tx_child = tx_from_hex(hex_child) + package_txns.append(tx_child) + + submitpackage_result = node.submitpackage([hex_rich, hex_poor, hex_child]) + + rich_parent_result = submitpackage_result["tx-results"][tx_rich.getwtxid()] + poor_parent_result = submitpackage_result["tx-results"][tx_poor.getwtxid()] + child_result = submitpackage_result["tx-results"][tx_child.getwtxid()] + assert_equal(rich_parent_result["fees"]["base"], DEFAULT_FEE) + assert_equal(poor_parent_result["fees"]["base"], 0) + assert_equal(child_result["fees"]["base"], DEFAULT_FEE) + # Package feerate is calculated for the remaining transactions after deduplication and + # individual submission. Since this package had a 0-fee parent, package feerate must have + # been used and returned. + assert "package-feerate" in submitpackage_result + assert_fee_amount(DEFAULT_FEE, rich_parent_result["vsize"] + child_result["vsize"], submitpackage_result["package-feerate"]) + + # The node will broadcast each transaction, still abiding by its peer's fee filter + peer.wait_for_broadcast([tx.getwtxid() for tx in package_txns]) + self.generate(node, 1) + + + def test_submitpackage(self): + node = self.nodes[0] + + self.log.info("Submitpackage valid packages with 1 child and some number of parents") + for num_parents in [1, 2, 24]: + self.test_submit_child_with_parents(num_parents, False) + self.test_submit_child_with_parents(num_parents, True) + + self.log.info("Submitpackage valid packages with CPFP") + self.test_submit_cpfp() + + self.log.info("Submitpackage only allows packages of 1 child with its parents") + # Chain of 3 transactions has too many generations + chain_hex, _ = create_raw_chain(node, self.coins.pop(), self.address, self.privkeys, 3) + assert_raises_rpc_error(-25, "not-child-with-parents", node.submitpackage, chain_hex) + + if __name__ == "__main__": RPCPackagesTest().main() diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 40fcbf7761..574ea10356 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -162,30 +162,6 @@ def create_tx_with_script(prevtx, n, script_sig=b"", *, amount, script_pub_key=C tx.calc_sha256() return tx -def create_transaction(node, txid, to_address, *, amount): - """ Return signed transaction spending the first output of the - input txid. Note that the node must have a wallet that can - sign for the output that is being spent. - """ - raw_tx = create_raw_transaction(node, txid, to_address, amount=amount) - tx = tx_from_hex(raw_tx) - return tx - -def create_raw_transaction(node, txid, to_address, *, amount): - """ Return raw signed transaction spending the first output of the - input txid. Note that the node must have a wallet that can sign - for the output that is being spent. - """ - psbt = node.createpsbt(inputs=[{"txid": txid, "vout": 0}], outputs={to_address: amount}) - for _ in range(2): - for w in node.listwallets(): - wrpc = node.get_wallet_rpc(w) - signed_psbt = wrpc.walletprocesspsbt(psbt) - psbt = signed_psbt['psbt'] - final_psbt = node.finalizepsbt(psbt) - assert_equal(final_psbt["complete"], True) - return final_psbt['hex'] - def get_legacy_sigopcount_block(block, accurate=True): count = 0 for tx in block.vtx: diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 6a588275ea..1ee23f7574 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -499,22 +499,13 @@ def chain_transaction(node, parent_txids, vouts, value, fee, num_outputs): # Create large OP_RETURN txouts that can be appended to a transaction -# to make it large (helper for constructing large transactions). +# to make it large (helper for constructing large transactions). The +# total serialized size of the txouts is about 66k vbytes. def gen_return_txouts(): - # Some pre-processing to create a bunch of OP_RETURN txouts to insert into transactions we create - # So we have big transactions (and therefore can't fit very many into each block) - # create one script_pubkey - script_pubkey = "6a4d0200" # OP_RETURN OP_PUSH2 512 bytes - for _ in range(512): - script_pubkey = script_pubkey + "01" - # concatenate 128 txouts of above script_pubkey which we'll insert before the txout for change - txouts = [] from .messages import CTxOut - txout = CTxOut() - txout.nValue = 0 - txout.scriptPubKey = bytes.fromhex(script_pubkey) - for _ in range(128): - txouts.append(txout) + from .script import CScript, OP_RETURN + txouts = [CTxOut(nValue=0, scriptPubKey=CScript([OP_RETURN, b'\x01'*67437]))] + assert_equal(sum([len(txout.serialize()) for txout in txouts]), 67456) return txouts diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index 68d5dfa880..2164627781 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -197,7 +197,7 @@ class MiniWallet: return utxos def send_self_transfer(self, *, from_node, **kwargs): - """Create and send a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed.""" + """Call create_self_transfer and send the transaction.""" tx = self.create_self_transfer(**kwargs) self.sendrawtransaction(from_node=from_node, tx_hex=tx['hex']) return tx @@ -232,21 +232,27 @@ class MiniWallet: *, utxos_to_spend: Optional[List[dict]] = None, num_outputs=1, + amount_per_output=0, sequence=0, fee_per_output=1000, ): """ Create and return a transaction that spends the given UTXOs and creates a - certain number of outputs with equal amounts. + certain number of outputs with equal amounts. The output amounts can be + set by amount_per_output or automatically calculated with a fee_per_output. """ utxos_to_spend = utxos_to_spend or [self.get_utxo()] + sequence = [sequence] * len(utxos_to_spend) if type(sequence) is int else sequence + assert_equal(len(utxos_to_spend), len(sequence)) # create simple tx template (1 input, 1 output) tx = self.create_self_transfer( fee_rate=0, - utxo_to_spend=utxos_to_spend[0], sequence=sequence)["tx"] + utxo_to_spend=utxos_to_spend[0])["tx"] # duplicate inputs, witnesses and outputs tx.vin = [deepcopy(tx.vin[0]) for _ in range(len(utxos_to_spend))] + for txin, seq in zip(tx.vin, sequence): + txin.nSequence = seq tx.wit.vtxinwit = [deepcopy(tx.wit.vtxinwit[0]) for _ in range(len(utxos_to_spend))] tx.vout = [deepcopy(tx.vout[0]) for _ in range(num_outputs)] @@ -258,7 +264,7 @@ class MiniWallet: inputs_value_total = sum([int(COIN * utxo['value']) for utxo in utxos_to_spend]) outputs_value_total = inputs_value_total - fee_per_output * num_outputs for o in tx.vout: - o.nValue = outputs_value_total // num_outputs + o.nValue = amount_per_output or (outputs_value_total // num_outputs) txid = tx.rehash() return { "new_utxos": [self._create_utxo( @@ -272,21 +278,23 @@ class MiniWallet: "tx": tx, } - def create_self_transfer(self, *, fee_rate=Decimal("0.003"), utxo_to_spend=None, locktime=0, sequence=0): - """Create and return a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed.""" + def create_self_transfer(self, *, fee_rate=Decimal("0.003"), fee=Decimal("0"), utxo_to_spend=None, locktime=0, sequence=0): + """Create and return a tx with the specified fee. If fee is 0, use fee_rate, where the resulting fee may be exact or at most one satoshi higher than needed.""" utxo_to_spend = utxo_to_spend or self.get_utxo() + assert fee_rate >= 0 + assert fee >= 0 if self._mode in (MiniWalletMode.RAW_OP_TRUE, MiniWalletMode.ADDRESS_OP_TRUE): vsize = Decimal(104) # anyone-can-spend elif self._mode == MiniWalletMode.RAW_P2PK: vsize = Decimal(168) # P2PK (73 bytes scriptSig + 35 bytes scriptPubKey + 60 bytes other) else: assert False - send_value = utxo_to_spend["value"] - (fee_rate * vsize / 1000) + send_value = utxo_to_spend["value"] - (fee or (fee_rate * vsize / 1000)) assert send_value > 0 tx = CTransaction() tx.vin = [CTxIn(COutPoint(int(utxo_to_spend['txid'], 16), utxo_to_spend['vout']), nSequence=sequence)] - tx.vout = [CTxOut(int(COIN * send_value), self._scriptPubKey)] + tx.vout = [CTxOut(int(COIN * send_value), bytearray(self._scriptPubKey))] tx.nLockTime = locktime if self._mode == MiniWalletMode.RAW_P2PK: self.sign_tx(tx) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 6a44f9d21d..08229de57a 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -130,6 +130,7 @@ BASE_SCRIPTS = [ 'wallet_address_types.py --descriptors', 'feature_bip68_sequence.py', 'p2p_feefilter.py', + 'rpc_packages.py', 'feature_reindex.py', 'feature_abortnode.py', # vv Tests less than 30s vv @@ -231,7 +232,6 @@ BASE_SCRIPTS = [ 'mempool_packages.py', 'mempool_package_onemore.py', 'rpc_createmultisig.py', - 'rpc_packages.py', 'mempool_package_limits.py', 'feature_versionbits_warning.py', 'rpc_preciousblock.py', @@ -244,8 +244,7 @@ BASE_SCRIPTS = [ 'rpc_generate.py', 'wallet_balance.py --legacy-wallet', 'wallet_balance.py --descriptors', - 'feature_nulldummy.py --legacy-wallet', - 'feature_nulldummy.py --descriptors', + 'feature_nulldummy.py', 'mempool_accept.py', 'mempool_expiry.py', 'wallet_import_rescan.py --legacy-wallet', diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index a6c93ba5f9..f66fab19ac 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -25,7 +25,7 @@ class WalletTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 self.extra_args = [[ - "-acceptnonstdtxn=1", "-walletrejectlongchains=0" + "-dustrelayfee=0", "-walletrejectlongchains=0" ]] * self.num_nodes self.setup_clean_chain = True self.supports_cli = False |