diff options
Diffstat (limited to 'src')
53 files changed, 1198 insertions, 761 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index 96e56915a6..e03c21f16e 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -29,9 +29,7 @@ LIBBITCOIN_COMMON=libbitcoin_common.a LIBBITCOIN_CONSENSUS=libbitcoin_consensus.a LIBBITCOIN_CLI=libbitcoin_cli.a LIBBITCOIN_UTIL=libbitcoin_util.a -LIBBITCOIN_CRYPTO=crypto/libbitcoin_crypto.a -LIBBITCOIN_CRYPTO_SSE41=crypto/libbitcoin_crypto_sse41.a -LIBBITCOIN_CRYPTO_AVX2=crypto/libbitcoin_crypto_avx2.a +LIBBITCOIN_CRYPTO_BASE=crypto/libbitcoin_crypto_base.a LIBBITCOINQT=qt/libbitcoinqt.a LIBSECP256K1=secp256k1/libsecp256k1.la @@ -45,6 +43,16 @@ if ENABLE_WALLET LIBBITCOIN_WALLET=libbitcoin_wallet.a endif +LIBBITCOIN_CRYPTO= $(LIBBITCOIN_CRYPTO_BASE) +if ENABLE_SSE41 +LIBBITCOIN_CRYPTO_SSE41 = crypto/libbitcoin_crypto_sse41.a +LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_SSE41) +endif +if ENABLE_AVX2 +LIBBITCOIN_CRYPTO_AVX2 = crypto/libbitcoin_crypto_avx2.a +LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_AVX2) +endif + $(LIBSECP256K1): $(wildcard secp256k1/src/*) $(wildcard secp256k1/include/*) $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) @@ -52,8 +60,6 @@ $(LIBSECP256K1): $(wildcard secp256k1/src/*) $(wildcard secp256k1/include/*) # But to build the less dependent modules first, we manually select their order here: EXTRA_LIBRARIES += \ $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_CONSENSUS) \ @@ -107,6 +113,7 @@ BITCOIN_CORE_H = \ fs.h \ httprpc.h \ httpserver.h \ + index/base.h \ index/txindex.h \ indirectmap.h \ init.h \ @@ -208,6 +215,7 @@ libbitcoin_server_a_SOURCES = \ consensus/tx_verify.cpp \ httprpc.cpp \ httpserver.cpp \ + index/base.cpp \ index/txindex.cpp \ init.cpp \ dbwrapper.cpp \ @@ -268,9 +276,9 @@ libbitcoin_wallet_a_SOURCES = \ $(BITCOIN_CORE_H) # crypto primitives library -crypto_libbitcoin_crypto_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_a_SOURCES = \ +crypto_libbitcoin_crypto_base_a_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_base_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +crypto_libbitcoin_crypto_base_a_SOURCES = \ crypto/aes.cpp \ crypto/aes.h \ crypto/chacha20.h \ @@ -290,23 +298,19 @@ crypto_libbitcoin_crypto_a_SOURCES = \ crypto/sha512.h if USE_ASM -crypto_libbitcoin_crypto_a_SOURCES += crypto/sha256_sse4.cpp +crypto_libbitcoin_crypto_base_a_SOURCES += crypto/sha256_sse4.cpp endif crypto_libbitcoin_crypto_sse41_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) crypto_libbitcoin_crypto_sse41_a_CPPFLAGS = $(AM_CPPFLAGS) -if ENABLE_SSE41 crypto_libbitcoin_crypto_sse41_a_CXXFLAGS += $(SSE41_CXXFLAGS) crypto_libbitcoin_crypto_sse41_a_CPPFLAGS += -DENABLE_SSE41 -endif crypto_libbitcoin_crypto_sse41_a_SOURCES = crypto/sha256_sse41.cpp crypto_libbitcoin_crypto_avx2_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) crypto_libbitcoin_crypto_avx2_a_CPPFLAGS = $(AM_CPPFLAGS) -if ENABLE_AVX2 crypto_libbitcoin_crypto_avx2_a_CXXFLAGS += $(AVX2_CXXFLAGS) crypto_libbitcoin_crypto_avx2_a_CPPFLAGS += -DENABLE_AVX2 -endif crypto_libbitcoin_crypto_avx2_a_SOURCES = crypto/sha256_avx2.cpp # consensus: shared between all executables that validate any consensus rules. @@ -431,8 +435,6 @@ bitcoind_LDADD = \ $(LIBBITCOIN_ZMQ) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ $(LIBLEVELDB) \ $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ @@ -454,9 +456,7 @@ bitcoin_cli_LDADD = \ $(LIBBITCOIN_CLI) \ $(LIBUNIVALUE) \ $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) + $(LIBBITCOIN_CRYPTO) bitcoin_cli_LDADD += $(BOOST_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(EVENT_LIBS) # @@ -477,8 +477,6 @@ bitcoin_tx_LDADD = \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ $(LIBSECP256K1) bitcoin_tx_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) @@ -487,7 +485,7 @@ bitcoin_tx_LDADD += $(BOOST_LIBS) $(CRYPTO_LIBS) # bitcoinconsensus library # if BUILD_BITCOIN_LIBS include_HEADERS = script/bitcoinconsensus.h -libbitcoinconsensus_la_SOURCES = $(crypto_libbitcoin_crypto_a_SOURCES) $(libbitcoin_consensus_a_SOURCES) +libbitcoinconsensus_la_SOURCES = $(crypto_libbitcoin_crypto_base_a_SOURCES) $(libbitcoin_consensus_a_SOURCES) if GLIBC_BACK_COMPAT libbitcoinconsensus_la_SOURCES += compat/glibc_compat.cpp diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 804df3bf21..bc2c46f228 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -17,7 +17,7 @@ bench_bench_bitcoin_SOURCES = \ bench/bench.h \ bench/checkblock.cpp \ bench/checkqueue.cpp \ - bench/Examples.cpp \ + bench/examples.cpp \ bench/rollingbloom.cpp \ bench/crypto_hash.cpp \ bench/ccoins_caching.cpp \ @@ -39,8 +39,6 @@ bench_bench_bitcoin_LDADD = \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_CRYPTO_SSE41) \ - $(LIBBITCOIN_CRYPTO_AVX2) \ $(LIBLEVELDB) \ $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index f8c31be3d4..a84a11ac45 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -408,7 +408,7 @@ endif if ENABLE_ZMQ qt_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif -qt_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBBITCOIN_CRYPTO_SSE41) $(LIBBITCOIN_CRYPTO_AVX2) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ +qt_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(PROTOBUF_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) qt_bitcoin_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index a4356f1cbd..4b14212b2e 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -62,7 +62,7 @@ endif if ENABLE_ZMQ qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif -qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBBITCOIN_CRYPTO_SSE41) $(LIBBITCOIN_CRYPTO_AVX2) $(LIBUNIVALUE) $(LIBLEVELDB) \ +qt_test_test_bitcoin_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) \ $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \ $(QR_LIBS) $(PROTOBUF_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index cbd63cd53d..a4d31795ec 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -46,7 +46,7 @@ BITCOIN_TESTS =\ test/compress_tests.cpp \ test/crypto_tests.cpp \ test/cuckoocache_tests.cpp \ - test/DoS_tests.cpp \ + test/denialofservice_tests.cpp \ test/getarg_tests.cpp \ test/hash_tests.cpp \ test/key_io_tests.cpp \ @@ -71,7 +71,7 @@ BITCOIN_TESTS =\ test/rpc_tests.cpp \ test/sanity_tests.cpp \ test/scheduler_tests.cpp \ - test/script_P2SH_tests.cpp \ + test/script_p2sh_tests.cpp \ test/script_tests.cpp \ test/script_standard_tests.cpp \ test/scriptnum_tests.cpp \ @@ -110,7 +110,7 @@ if ENABLE_WALLET test_test_bitcoin_LDADD += $(LIBBITCOIN_WALLET) endif -test_test_bitcoin_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBBITCOIN_CRYPTO_SSE41) $(LIBBITCOIN_CRYPTO_AVX2) $(LIBUNIVALUE) \ +test_test_bitcoin_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \ $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) diff --git a/src/arith_uint256.h b/src/arith_uint256.h index 3f4cc8c2bf..e4c7575e2d 100644 --- a/src/arith_uint256.h +++ b/src/arith_uint256.h @@ -64,14 +64,6 @@ public: explicit base_uint(const std::string& str); - bool operator!() const - { - for (int i = 0; i < WIDTH; i++) - if (pn[i] != 0) - return false; - return true; - } - const base_uint operator~() const { base_uint ret; diff --git a/src/bench/Examples.cpp b/src/bench/examples.cpp index b68c9cd156..b68c9cd156 100644 --- a/src/bench/Examples.cpp +++ b/src/bench/examples.cpp diff --git a/src/bench/merkle_root.cpp b/src/bench/merkle_root.cpp index ae2a0a28dc..fab12da311 100644 --- a/src/bench/merkle_root.cpp +++ b/src/bench/merkle_root.cpp @@ -2,11 +2,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "bench.h" +#include <bench/bench.h> -#include "uint256.h" -#include "random.h" -#include "consensus/merkle.h" +#include <uint256.h> +#include <random.h> +#include <consensus/merkle.h> static void MerkleRoot(benchmark::State& state) { diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 2a594c3051..e6eb723cf4 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -591,7 +591,7 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) if (!prevOut.checkObject(types)) throw std::runtime_error("prevtxs internal object typecheck fail"); - uint256 txid = ParseHashUV(prevOut["txid"], "txid"); + uint256 txid = ParseHashStr(prevOut["txid"].get_str(), "txid"); int nOut = atoi(prevOut["vout"].getValStr()); if (nOut < 0) diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index a9b952e5a4..4b9abb2a1b 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -62,9 +62,6 @@ static bool AppInit(int argc, char* argv[]) // // If Qt is used, parameters/bitcoin.conf are parsed in qt/bitcoin.cpp's main() SetupServerArgs(); -#if HAVE_DECL_DAEMON - gArgs.AddArg("-daemon", "Run in the background as a daemon and accept commands", false, OptionsCategory::OPTIONS); -#endif std::string error; if (!gArgs.ParseParameters(argc, argv, error)) { fprintf(stderr, "Error parsing command line arguments: %s\n", error.c_str()); diff --git a/src/core_io.h b/src/core_io.h index 377633ac77..1d87d21d40 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -22,7 +22,6 @@ CScript ParseScript(const std::string& s); std::string ScriptToAsmStr(const CScript& script, const bool fAttemptSighashDecode = false); bool DecodeHexTx(CMutableTransaction& tx, const std::string& hex_tx, bool try_no_witness = false, bool try_witness = true); bool DecodeHexBlk(CBlock&, const std::string& strHexBlk); -uint256 ParseHashUV(const UniValue& v, const std::string& strName); uint256 ParseHashStr(const std::string&, const std::string& strName); std::vector<unsigned char> ParseHexUV(const UniValue& v, const std::string& strName); diff --git a/src/core_read.cpp b/src/core_read.cpp index aade7e21ca..4d851610ef 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -160,14 +160,6 @@ bool DecodeHexBlk(CBlock& block, const std::string& strHexBlk) return true; } -uint256 ParseHashUV(const UniValue& v, const std::string& strName) -{ - std::string strHex; - if (v.isStr()) - strHex = v.getValStr(); - return ParseHashStr(strHex, strName); // Note: ParseHashStr("") throws a runtime_error -} - uint256 ParseHashStr(const std::string& strHex, const std::string& strName) { if (!IsHex(strHex)) // Note: IsHex("") is false diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 6ac51d11cd..100d0d1fac 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -446,39 +446,91 @@ void TransformD64Wrapper(unsigned char* out, const unsigned char* in) WriteBE32(out + 28, s[7]); } -bool SelfTest(TransformType tr) { - static const unsigned char in1[65] = {0, 0x80}; - static const unsigned char in2[129] = { - 0, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0 - }; - static const uint32_t init[8] = {0x6a09e667ul, 0xbb67ae85ul, 0x3c6ef372ul, 0xa54ff53aul, 0x510e527ful, 0x9b05688cul, 0x1f83d9abul, 0x5be0cd19ul}; - static const uint32_t out1[8] = {0xe3b0c442ul, 0x98fc1c14ul, 0x9afbf4c8ul, 0x996fb924ul, 0x27ae41e4ul, 0x649b934cul, 0xa495991bul, 0x7852b855ul}; - static const uint32_t out2[8] = {0xce4153b0ul, 0x147c2a86ul, 0x3ed4298eul, 0xe0676bc8ul, 0x79fc77a1ul, 0x2abe1f49ul, 0xb2b055dful, 0x1069523eul}; - uint32_t buf[8]; - memcpy(buf, init, sizeof(buf)); - // Process nothing, and check we remain in the initial state. - tr(buf, nullptr, 0); - if (memcmp(buf, init, sizeof(buf))) return false; - // Process the padded empty string (unaligned) - tr(buf, in1 + 1, 1); - if (memcmp(buf, out1, sizeof(buf))) return false; - // Process 64 spaces (unaligned) - memcpy(buf, init, sizeof(buf)); - tr(buf, in2 + 1, 2); - if (memcmp(buf, out2, sizeof(buf))) return false; - return true; -} - TransformType Transform = sha256::Transform; TransformD64Type TransformD64 = sha256::TransformD64; TransformD64Type TransformD64_4way = nullptr; TransformD64Type TransformD64_8way = nullptr; -#if defined(USE_ASM) && (defined(__x86_64__) || defined(__amd64__)) +bool SelfTest() { + // Input state (equal to the initial SHA256 state) + static const uint32_t init[8] = { + 0x6a09e667ul, 0xbb67ae85ul, 0x3c6ef372ul, 0xa54ff53aul, 0x510e527ful, 0x9b05688cul, 0x1f83d9abul, 0x5be0cd19ul + }; + // Some random input data to test with + static const unsigned char data[641] = "-" // Intentionally not aligned + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do " + "eiusmod tempor incididunt ut labore et dolore magna aliqua. Et m" + "olestie ac feugiat sed lectus vestibulum mattis ullamcorper. Mor" + "bi blandit cursus risus at ultrices mi tempus imperdiet nulla. N" + "unc congue nisi vita suscipit tellus mauris. Imperdiet proin fer" + "mentum leo vel orci. Massa tempor nec feugiat nisl pretium fusce" + " id velit. Telus in metus vulputate eu scelerisque felis. Mi tem" + "pus imperdiet nulla malesuada pellentesque. Tristique magna sit."; + // Expected output state for hashing the i*64 first input bytes above (excluding SHA256 padding). + static const uint32_t result[9][8] = { + {0x6a09e667ul, 0xbb67ae85ul, 0x3c6ef372ul, 0xa54ff53aul, 0x510e527ful, 0x9b05688cul, 0x1f83d9abul, 0x5be0cd19ul}, + {0x91f8ec6bul, 0x4da10fe3ul, 0x1c9c292cul, 0x45e18185ul, 0x435cc111ul, 0x3ca26f09ul, 0xeb954caeul, 0x402a7069ul}, + {0xcabea5acul, 0x374fb97cul, 0x182ad996ul, 0x7bd69cbful, 0x450ff900ul, 0xc1d2be8aul, 0x6a41d505ul, 0xe6212dc3ul}, + {0xbcff09d6ul, 0x3e76f36eul, 0x3ecb2501ul, 0x78866e97ul, 0xe1c1e2fdul, 0x32f4eafful, 0x8aa6c4e5ul, 0xdfc024bcul}, + {0xa08c5d94ul, 0x0a862f93ul, 0x6b7f2f40ul, 0x8f9fae76ul, 0x6d40439ful, 0x79dcee0cul, 0x3e39ff3aul, 0xdc3bdbb1ul}, + {0x216a0895ul, 0x9f1a3662ul, 0xe99946f9ul, 0x87ba4364ul, 0x0fb5db2cul, 0x12bed3d3ul, 0x6689c0c7ul, 0x292f1b04ul}, + {0xca3067f8ul, 0xbc8c2656ul, 0x37cb7e0dul, 0x9b6b8b0ful, 0x46dc380bul, 0xf1287f57ul, 0xc42e4b23ul, 0x3fefe94dul}, + {0x3e4c4039ul, 0xbb6fca8cul, 0x6f27d2f7ul, 0x301e44a4ul, 0x8352ba14ul, 0x5769ce37ul, 0x48a1155ful, 0xc0e1c4c6ul}, + {0xfe2fa9ddul, 0x69d0862bul, 0x1ae0db23ul, 0x471f9244ul, 0xf55c0145ul, 0xc30f9c3bul, 0x40a84ea0ul, 0x5b8a266cul}, + }; + // Expected output for each of the individual 8 64-byte messages under full double SHA256 (including padding). + static const unsigned char result_d64[256] = { + 0x09, 0x3a, 0xc4, 0xd0, 0x0f, 0xf7, 0x57, 0xe1, 0x72, 0x85, 0x79, 0x42, 0xfe, 0xe7, 0xe0, 0xa0, + 0xfc, 0x52, 0xd7, 0xdb, 0x07, 0x63, 0x45, 0xfb, 0x53, 0x14, 0x7d, 0x17, 0x22, 0x86, 0xf0, 0x52, + 0x48, 0xb6, 0x11, 0x9e, 0x6e, 0x48, 0x81, 0x6d, 0xcc, 0x57, 0x1f, 0xb2, 0x97, 0xa8, 0xd5, 0x25, + 0x9b, 0x82, 0xaa, 0x89, 0xe2, 0xfd, 0x2d, 0x56, 0xe8, 0x28, 0x83, 0x0b, 0xe2, 0xfa, 0x53, 0xb7, + 0xd6, 0x6b, 0x07, 0x85, 0x83, 0xb0, 0x10, 0xa2, 0xf5, 0x51, 0x3c, 0xf9, 0x60, 0x03, 0xab, 0x45, + 0x6c, 0x15, 0x6e, 0xef, 0xb5, 0xac, 0x3e, 0x6c, 0xdf, 0xb4, 0x92, 0x22, 0x2d, 0xce, 0xbf, 0x3e, + 0xe9, 0xe5, 0xf6, 0x29, 0x0e, 0x01, 0x4f, 0xd2, 0xd4, 0x45, 0x65, 0xb3, 0xbb, 0xf2, 0x4c, 0x16, + 0x37, 0x50, 0x3c, 0x6e, 0x49, 0x8c, 0x5a, 0x89, 0x2b, 0x1b, 0xab, 0xc4, 0x37, 0xd1, 0x46, 0xe9, + 0x3d, 0x0e, 0x85, 0xa2, 0x50, 0x73, 0xa1, 0x5e, 0x54, 0x37, 0xd7, 0x94, 0x17, 0x56, 0xc2, 0xd8, + 0xe5, 0x9f, 0xed, 0x4e, 0xae, 0x15, 0x42, 0x06, 0x0d, 0x74, 0x74, 0x5e, 0x24, 0x30, 0xce, 0xd1, + 0x9e, 0x50, 0xa3, 0x9a, 0xb8, 0xf0, 0x4a, 0x57, 0x69, 0x78, 0x67, 0x12, 0x84, 0x58, 0xbe, 0xc7, + 0x36, 0xaa, 0xee, 0x7c, 0x64, 0xa3, 0x76, 0xec, 0xff, 0x55, 0x41, 0x00, 0x2a, 0x44, 0x68, 0x4d, + 0xb6, 0x53, 0x9e, 0x1c, 0x95, 0xb7, 0xca, 0xdc, 0x7f, 0x7d, 0x74, 0x27, 0x5c, 0x8e, 0xa6, 0x84, + 0xb5, 0xac, 0x87, 0xa9, 0xf3, 0xff, 0x75, 0xf2, 0x34, 0xcd, 0x1a, 0x3b, 0x82, 0x2c, 0x2b, 0x4e, + 0x6a, 0x46, 0x30, 0xa6, 0x89, 0x86, 0x23, 0xac, 0xf8, 0xa5, 0x15, 0xe9, 0x0a, 0xaa, 0x1e, 0x9a, + 0xd7, 0x93, 0x6b, 0x28, 0xe4, 0x3b, 0xfd, 0x59, 0xc6, 0xed, 0x7c, 0x5f, 0xa5, 0x41, 0xcb, 0x51 + }; + + + // Test Transform() for 0 through 8 transformations. + for (size_t i = 0; i <= 8; ++i) { + uint32_t state[8]; + std::copy(init, init + 8, state); + Transform(state, data + 1, i); + if (!std::equal(state, state + 8, result[i])) return false; + } + + // Test TransformD64 + unsigned char out[32]; + TransformD64(out, data + 1); + if (!std::equal(out, out + 32, result_d64)) return false; + + // Test TransformD64_4way, if available. + if (TransformD64_4way) { + unsigned char out[128]; + TransformD64_4way(out, data + 1); + if (!std::equal(out, out + 128, result_d64)) return false; + } + + // Test TransformD64_8way, if available. + if (TransformD64_8way) { + unsigned char out[256]; + TransformD64_8way(out, data + 1); + if (!std::equal(out, out + 256, result_d64)) return false; + } + + return true; +} + + +#if defined(USE_ASM) && (defined(__x86_64__) || defined(__amd64__) || defined(__i386__)) // We can't use cpuid.h's __get_cpuid as it does not support subleafs. void inline cpuid(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) { @@ -491,12 +543,14 @@ void inline cpuid(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uin std::string SHA256AutoDetect() { std::string ret = "standard"; -#if defined(USE_ASM) && (defined(__x86_64__) || defined(__amd64__)) +#if defined(USE_ASM) && (defined(__x86_64__) || defined(__amd64__) || defined(__i386__)) uint32_t eax, ebx, ecx, edx; cpuid(1, 0, eax, ebx, ecx, edx); if ((ecx >> 19) & 1) { +#if defined(__x86_64__) || defined(__amd64__) Transform = sha256_sse4::Transform; TransformD64 = TransformD64Wrapper<sha256_sse4::Transform>; +#endif #if defined(ENABLE_SSE41) && !defined(BUILD_BITCOIN_INTERNAL) TransformD64_4way = sha256d64_sse41::Transform_4way; ret = "sse4(1way+4way)"; @@ -513,7 +567,7 @@ std::string SHA256AutoDetect() } #endif - assert(SelfTest(Transform)); + assert(SelfTest()); return ret; } diff --git a/src/crypto/sha256_avx2.cpp b/src/crypto/sha256_avx2.cpp index f45c1d4ab6..b338b06927 100644 --- a/src/crypto/sha256_avx2.cpp +++ b/src/crypto/sha256_avx2.cpp @@ -7,8 +7,8 @@ #include <x86intrin.h> #endif -#include "crypto/sha256.h" -#include "crypto/common.h" +#include <crypto/sha256.h> +#include <crypto/common.h> namespace sha256d64_avx2 { namespace { diff --git a/src/crypto/sha256_sse41.cpp b/src/crypto/sha256_sse41.cpp index a11d658c70..be71dd8fb8 100644 --- a/src/crypto/sha256_sse41.cpp +++ b/src/crypto/sha256_sse41.cpp @@ -7,8 +7,8 @@ #include <x86intrin.h> #endif -#include "crypto/sha256.h" -#include "crypto/common.h" +#include <crypto/sha256.h> +#include <crypto/common.h> namespace sha256d64_sse41 { namespace { diff --git a/src/index/base.cpp b/src/index/base.cpp new file mode 100644 index 0000000000..738166dc94 --- /dev/null +++ b/src/index/base.cpp @@ -0,0 +1,278 @@ +// Copyright (c) 2017-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <index/base.h> +#include <init.h> +#include <tinyformat.h> +#include <ui_interface.h> +#include <util.h> +#include <validation.h> +#include <warnings.h> + +constexpr char DB_BEST_BLOCK = 'B'; + +constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds +constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds + +template<typename... Args> +static void FatalError(const char* fmt, const Args&... args) +{ + std::string strMessage = tfm::format(fmt, args...); + SetMiscWarning(strMessage); + LogPrintf("*** %s\n", strMessage); + uiInterface.ThreadSafeMessageBox( + "Error: A fatal internal error occurred, see debug.log for details", + "", CClientUIInterface::MSG_ERROR); + StartShutdown(); +} + +BaseIndex::DB::DB(const fs::path& path, size_t n_cache_size, bool f_memory, bool f_wipe, bool f_obfuscate) : + CDBWrapper(path, n_cache_size, f_memory, f_wipe, f_obfuscate) +{} + +bool BaseIndex::DB::ReadBestBlock(CBlockLocator& locator) const +{ + bool success = Read(DB_BEST_BLOCK, locator); + if (!success) { + locator.SetNull(); + } + return success; +} + +bool BaseIndex::DB::WriteBestBlock(const CBlockLocator& locator) +{ + return Write(DB_BEST_BLOCK, locator); +} + +BaseIndex::~BaseIndex() +{ + Interrupt(); + Stop(); +} + +bool BaseIndex::Init() +{ + CBlockLocator locator; + if (!GetDB().ReadBestBlock(locator)) { + locator.SetNull(); + } + + LOCK(cs_main); + m_best_block_index = FindForkInGlobalIndex(chainActive, locator); + m_synced = m_best_block_index.load() == chainActive.Tip(); + return true; +} + +static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev) +{ + AssertLockHeld(cs_main); + + if (!pindex_prev) { + return chainActive.Genesis(); + } + + const CBlockIndex* pindex = chainActive.Next(pindex_prev); + if (pindex) { + return pindex; + } + + return chainActive.Next(chainActive.FindFork(pindex_prev)); +} + +void BaseIndex::ThreadSync() +{ + const CBlockIndex* pindex = m_best_block_index.load(); + if (!m_synced) { + auto& consensus_params = Params().GetConsensus(); + + int64_t last_log_time = 0; + int64_t last_locator_write_time = 0; + while (true) { + if (m_interrupt) { + WriteBestBlock(pindex); + return; + } + + { + LOCK(cs_main); + const CBlockIndex* pindex_next = NextSyncBlock(pindex); + if (!pindex_next) { + WriteBestBlock(pindex); + m_best_block_index = pindex; + m_synced = true; + break; + } + pindex = pindex_next; + } + + int64_t current_time = GetTime(); + if (last_log_time + SYNC_LOG_INTERVAL < current_time) { + LogPrintf("Syncing %s with block chain from height %d\n", + GetName(), pindex->nHeight); + last_log_time = current_time; + } + + if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) { + WriteBestBlock(pindex); + last_locator_write_time = current_time; + } + + CBlock block; + if (!ReadBlockFromDisk(block, pindex, consensus_params)) { + FatalError("%s: Failed to read block %s from disk", + __func__, pindex->GetBlockHash().ToString()); + return; + } + if (!WriteBlock(block, pindex)) { + FatalError("%s: Failed to write block %s to index database", + __func__, pindex->GetBlockHash().ToString()); + return; + } + } + } + + if (pindex) { + LogPrintf("%s is enabled at height %d\n", GetName(), pindex->nHeight); + } else { + LogPrintf("%s is enabled\n", GetName()); + } +} + +bool BaseIndex::WriteBestBlock(const CBlockIndex* block_index) +{ + LOCK(cs_main); + if (!GetDB().WriteBestBlock(chainActive.GetLocator(block_index))) { + return error("%s: Failed to write locator to disk", __func__); + } + return true; +} + +void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex, + const std::vector<CTransactionRef>& txn_conflicted) +{ + if (!m_synced) { + return; + } + + const CBlockIndex* best_block_index = m_best_block_index.load(); + if (!best_block_index) { + if (pindex->nHeight != 0) { + FatalError("%s: First block connected is not the genesis block (height=%d)", + __func__, pindex->nHeight); + return; + } + } else { + // Ensure block connects to an ancestor of the current best block. This should be the case + // most of the time, but may not be immediately after the sync thread catches up and sets + // m_synced. Consider the case where there is a reorg and the blocks on the stale branch are + // in the ValidationInterface queue backlog even after the sync thread has caught up to the + // new chain tip. In this unlikely event, log a warning and let the queue clear. + if (best_block_index->GetAncestor(pindex->nHeight - 1) != pindex->pprev) { + LogPrintf("%s: WARNING: Block %s does not connect to an ancestor of " /* Continued */ + "known best chain (tip=%s); not updating index\n", + __func__, pindex->GetBlockHash().ToString(), + best_block_index->GetBlockHash().ToString()); + return; + } + } + + if (WriteBlock(*block, pindex)) { + m_best_block_index = pindex; + } else { + FatalError("%s: Failed to write block %s to index", + __func__, pindex->GetBlockHash().ToString()); + return; + } +} + +void BaseIndex::ChainStateFlushed(const CBlockLocator& locator) +{ + if (!m_synced) { + return; + } + + const uint256& locator_tip_hash = locator.vHave.front(); + const CBlockIndex* locator_tip_index; + { + LOCK(cs_main); + locator_tip_index = LookupBlockIndex(locator_tip_hash); + } + + if (!locator_tip_index) { + FatalError("%s: First block (hash=%s) in locator was not found", + __func__, locator_tip_hash.ToString()); + return; + } + + // This checks that ChainStateFlushed callbacks are received after BlockConnected. The check may fail + // immediately after the sync thread catches up and sets m_synced. Consider the case where + // there is a reorg and the blocks on the stale branch are in the ValidationInterface queue + // backlog even after the sync thread has caught up to the new chain tip. In this unlikely + // event, log a warning and let the queue clear. + const CBlockIndex* best_block_index = m_best_block_index.load(); + if (best_block_index->GetAncestor(locator_tip_index->nHeight) != locator_tip_index) { + LogPrintf("%s: WARNING: Locator contains block (hash=%s) not on known best " /* Continued */ + "chain (tip=%s); not writing index locator\n", + __func__, locator_tip_hash.ToString(), + best_block_index->GetBlockHash().ToString()); + return; + } + + if (!GetDB().WriteBestBlock(locator)) { + error("%s: Failed to write locator to disk", __func__); + } +} + +bool BaseIndex::BlockUntilSyncedToCurrentChain() +{ + AssertLockNotHeld(cs_main); + + if (!m_synced) { + return false; + } + + { + // Skip the queue-draining stuff if we know we're caught up with + // chainActive.Tip(). + LOCK(cs_main); + const CBlockIndex* chain_tip = chainActive.Tip(); + const CBlockIndex* best_block_index = m_best_block_index.load(); + if (best_block_index->GetAncestor(chain_tip->nHeight) == chain_tip) { + return true; + } + } + + LogPrintf("%s: %s is catching up on block notifications\n", __func__, GetName()); + SyncWithValidationInterfaceQueue(); + return true; +} + +void BaseIndex::Interrupt() +{ + m_interrupt(); +} + +void BaseIndex::Start() +{ + // Need to register this ValidationInterface before running Init(), so that + // callbacks are not missed if Init sets m_synced to true. + RegisterValidationInterface(this); + if (!Init()) { + FatalError("%s: %s failed to initialize", __func__, GetName()); + return; + } + + m_thread_sync = std::thread(&TraceThread<std::function<void()>>, GetName(), + std::bind(&BaseIndex::ThreadSync, this)); +} + +void BaseIndex::Stop() +{ + UnregisterValidationInterface(this); + + if (m_thread_sync.joinable()) { + m_thread_sync.join(); + } +} diff --git a/src/index/base.h b/src/index/base.h new file mode 100644 index 0000000000..04ee6e6cc2 --- /dev/null +++ b/src/index/base.h @@ -0,0 +1,98 @@ +// Copyright (c) 2017-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INDEX_BASE_H +#define BITCOIN_INDEX_BASE_H + +#include <dbwrapper.h> +#include <primitives/block.h> +#include <primitives/transaction.h> +#include <threadinterrupt.h> +#include <uint256.h> +#include <validationinterface.h> + +class CBlockIndex; + +/** + * Base class for indices of blockchain data. This implements + * CValidationInterface and ensures blocks are indexed sequentially according + * to their position in the active chain. + */ +class BaseIndex : public CValidationInterface +{ +protected: + class DB : public CDBWrapper + { + public: + DB(const fs::path& path, size_t n_cache_size, + bool f_memory = false, bool f_wipe = false, bool f_obfuscate = false); + + /// Read block locator of the chain that the txindex is in sync with. + bool ReadBestBlock(CBlockLocator& locator) const; + + /// Write block locator of the chain that the txindex is in sync with. + bool WriteBestBlock(const CBlockLocator& locator); + }; + +private: + /// Whether the index is in sync with the main chain. The flag is flipped + /// from false to true once, after which point this starts processing + /// ValidationInterface notifications to stay in sync. + std::atomic<bool> m_synced{false}; + + /// The last block in the chain that the index is in sync with. + std::atomic<const CBlockIndex*> m_best_block_index{nullptr}; + + std::thread m_thread_sync; + CThreadInterrupt m_interrupt; + + /// Sync the index with the block index starting from the current best block. + /// Intended to be run in its own thread, m_thread_sync, and can be + /// interrupted with m_interrupt. Once the index gets in sync, the m_synced + /// flag is set and the BlockConnected ValidationInterface callback takes + /// over and the sync thread exits. + void ThreadSync(); + + /// Write the current chain block locator to the DB. + bool WriteBestBlock(const CBlockIndex* block_index); + +protected: + void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex, + const std::vector<CTransactionRef>& txn_conflicted) override; + + void ChainStateFlushed(const CBlockLocator& locator) override; + + /// Initialize internal state from the database and block index. + virtual bool Init(); + + /// Write update index entries for a newly connected block. + virtual bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) { return true; } + + virtual DB& GetDB() const = 0; + + /// Get the name of the index for display in logs. + virtual const char* GetName() const = 0; + +public: + /// Destructor interrupts sync thread if running and blocks until it exits. + virtual ~BaseIndex(); + + /// Blocks the current thread until the index is caught up to the current + /// state of the block chain. This only blocks if the index has gotten in + /// sync once and only needs to process blocks in the ValidationInterface + /// queue. If the index is catching up from far behind, this method does + /// not block and immediately returns false. + bool BlockUntilSyncedToCurrentChain(); + + void Interrupt(); + + /// Start initializes the sync state and registers the instance as a + /// ValidationInterface so that it stays in sync with blockchain updates. + void Start(); + + /// Stops the instance from staying in sync with blockchain updates. + void Stop(); +}; + +#endif // BITCOIN_INDEX_BASE_H diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 3ff16b7664..e106b9b420 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -2,258 +2,261 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <chainparams.h> #include <index/txindex.h> #include <init.h> -#include <tinyformat.h> #include <ui_interface.h> #include <util.h> #include <validation.h> -#include <warnings.h> -constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds -constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds +#include <boost/thread.hpp> + +constexpr char DB_BEST_BLOCK = 'B'; +constexpr char DB_TXINDEX = 't'; +constexpr char DB_TXINDEX_BLOCK = 'T'; std::unique_ptr<TxIndex> g_txindex; -template<typename... Args> -static void FatalError(const char* fmt, const Args&... args) +struct CDiskTxPos : public CDiskBlockPos { - std::string strMessage = tfm::format(fmt, args...); - SetMiscWarning(strMessage); - LogPrintf("*** %s\n", strMessage); - uiInterface.ThreadSafeMessageBox( - "Error: A fatal internal error occurred, see debug.log for details", - "", CClientUIInterface::MSG_ERROR); - StartShutdown(); -} + unsigned int nTxOffset; // after header -TxIndex::TxIndex(std::unique_ptr<TxIndexDB> db) : - m_db(std::move(db)), m_synced(false), m_best_block_index(nullptr) -{} + ADD_SERIALIZE_METHODS; -TxIndex::~TxIndex() -{ - Interrupt(); - Stop(); -} - -bool TxIndex::Init() -{ - LOCK(cs_main); - - // Attempt to migrate txindex from the old database to the new one. Even if - // chain_tip is null, the node could be reindexing and we still want to - // delete txindex records in the old database. - if (!m_db->MigrateData(*pblocktree, chainActive.GetLocator())) { - return false; + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITEAS(CDiskBlockPos, *this); + READWRITE(VARINT(nTxOffset)); } - CBlockLocator locator; - if (!m_db->ReadBestBlock(locator)) { - locator.SetNull(); + CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { } - m_best_block_index = FindForkInGlobalIndex(chainActive, locator); - m_synced = m_best_block_index.load() == chainActive.Tip(); - return true; -} - -static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev) -{ - AssertLockHeld(cs_main); - - if (!pindex_prev) { - return chainActive.Genesis(); + CDiskTxPos() { + SetNull(); } - const CBlockIndex* pindex = chainActive.Next(pindex_prev); - if (pindex) { - return pindex; + void SetNull() { + CDiskBlockPos::SetNull(); + nTxOffset = 0; } - - return chainActive.Next(chainActive.FindFork(pindex_prev)); -} - -void TxIndex::ThreadSync() +}; + +/** + * Access to the txindex database (indexes/txindex/) + * + * The database stores a block locator of the chain the database is synced to + * so that the TxIndex can efficiently determine the point it last stopped at. + * A locator is used instead of a simple hash of the chain tip because blocks + * and block index entries may not be flushed to disk until after this database + * is updated. + */ +class TxIndex::DB : public BaseIndex::DB { - const CBlockIndex* pindex = m_best_block_index.load(); - if (!m_synced) { - auto& consensus_params = Params().GetConsensus(); - - int64_t last_log_time = 0; - int64_t last_locator_write_time = 0; - while (true) { - if (m_interrupt) { - WriteBestBlock(pindex); - return; - } +public: + explicit DB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); - { - LOCK(cs_main); - const CBlockIndex* pindex_next = NextSyncBlock(pindex); - if (!pindex_next) { - WriteBestBlock(pindex); - m_best_block_index = pindex; - m_synced = true; - break; - } - pindex = pindex_next; - } + /// Read the disk location of the transaction data with the given hash. Returns false if the + /// transaction hash is not indexed. + bool ReadTxPos(const uint256& txid, CDiskTxPos& pos) const; - int64_t current_time = GetTime(); - if (last_log_time + SYNC_LOG_INTERVAL < current_time) { - LogPrintf("Syncing txindex with block chain from height %d\n", pindex->nHeight); - last_log_time = current_time; - } + /// Write a batch of transaction positions to the DB. + bool WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos); - if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) { - WriteBestBlock(pindex); - last_locator_write_time = current_time; - } + /// Migrate txindex data from the block tree DB, where it may be for older nodes that have not + /// been upgraded yet to the new database. + bool MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator); +}; - CBlock block; - if (!ReadBlockFromDisk(block, pindex, consensus_params)) { - FatalError("%s: Failed to read block %s from disk", - __func__, pindex->GetBlockHash().ToString()); - return; - } - if (!WriteBlock(block, pindex)) { - FatalError("%s: Failed to write block %s to tx index database", - __func__, pindex->GetBlockHash().ToString()); - return; - } - } - } +TxIndex::DB::DB(size_t n_cache_size, bool f_memory, bool f_wipe) : + BaseIndex::DB(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe) +{} - if (pindex) { - LogPrintf("txindex is enabled at height %d\n", pindex->nHeight); - } else { - LogPrintf("txindex is enabled\n"); - } +bool TxIndex::DB::ReadTxPos(const uint256 &txid, CDiskTxPos& pos) const +{ + return Read(std::make_pair(DB_TXINDEX, txid), pos); } -bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) +bool TxIndex::DB::WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos) { - CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())); - std::vector<std::pair<uint256, CDiskTxPos>> vPos; - vPos.reserve(block.vtx.size()); - for (const auto& tx : block.vtx) { - vPos.emplace_back(tx->GetHash(), pos); - pos.nTxOffset += ::GetSerializeSize(*tx, SER_DISK, CLIENT_VERSION); + CDBBatch batch(*this); + for (const auto& tuple : v_pos) { + batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second); } - return m_db->WriteTxs(vPos); + return WriteBatch(batch); } -bool TxIndex::WriteBestBlock(const CBlockIndex* block_index) +/* + * Safely persist a transfer of data from the old txindex database to the new one, and compact the + * range of keys updated. This is used internally by MigrateData. + */ +static void WriteTxIndexMigrationBatches(CDBWrapper& newdb, CDBWrapper& olddb, + CDBBatch& batch_newdb, CDBBatch& batch_olddb, + const std::pair<unsigned char, uint256>& begin_key, + const std::pair<unsigned char, uint256>& end_key) { - LOCK(cs_main); - if (!m_db->WriteBestBlock(chainActive.GetLocator(block_index))) { - return error("%s: Failed to write locator to disk", __func__); - } - return true; + // Sync new DB changes to disk before deleting from old DB. + newdb.WriteBatch(batch_newdb, /*fSync=*/ true); + olddb.WriteBatch(batch_olddb); + olddb.CompactRange(begin_key, end_key); + + batch_newdb.Clear(); + batch_olddb.Clear(); } -void TxIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex, - const std::vector<CTransactionRef>& txn_conflicted) +bool TxIndex::DB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator) { - if (!m_synced) { - return; - } - - const CBlockIndex* best_block_index = m_best_block_index.load(); - if (!best_block_index) { - if (pindex->nHeight != 0) { - FatalError("%s: First block connected is not the genesis block (height=%d)", - __func__, pindex->nHeight); - return; + // The prior implementation of txindex was always in sync with block index + // and presence was indicated with a boolean DB flag. If the flag is set, + // this means the txindex from a previous version is valid and in sync with + // the chain tip. The first step of the migration is to unset the flag and + // write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the + // index entries are copied over in batches to the new database. Finally, + // DB_TXINDEX_BLOCK is erased from the old database and the block hash is + // written to the new database. + // + // Unsetting the boolean flag ensures that if the node is downgraded to a + // previous version, it will not see a corrupted, partially migrated index + // -- it will see that the txindex is disabled. When the node is upgraded + // again, the migration will pick up where it left off and sync to the block + // with hash DB_TXINDEX_BLOCK. + bool f_legacy_flag = false; + block_tree_db.ReadFlag("txindex", f_legacy_flag); + if (f_legacy_flag) { + if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) { + return error("%s: cannot write block indicator", __func__); } - } else { - // Ensure block connects to an ancestor of the current best block. This should be the case - // most of the time, but may not be immediately after the sync thread catches up and sets - // m_synced. Consider the case where there is a reorg and the blocks on the stale branch are - // in the ValidationInterface queue backlog even after the sync thread has caught up to the - // new chain tip. In this unlikely event, log a warning and let the queue clear. - if (best_block_index->GetAncestor(pindex->nHeight - 1) != pindex->pprev) { - LogPrintf("%s: WARNING: Block %s does not connect to an ancestor of " /* Continued */ - "known best chain (tip=%s); not updating txindex\n", - __func__, pindex->GetBlockHash().ToString(), - best_block_index->GetBlockHash().ToString()); - return; + if (!block_tree_db.WriteFlag("txindex", false)) { + return error("%s: cannot write block index db flag", __func__); } } - if (WriteBlock(*block, pindex)) { - m_best_block_index = pindex; - } else { - FatalError("%s: Failed to write block %s to txindex", - __func__, pindex->GetBlockHash().ToString()); - return; + CBlockLocator locator; + if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) { + return true; } -} -void TxIndex::ChainStateFlushed(const CBlockLocator& locator) -{ - if (!m_synced) { - return; - } + int64_t count = 0; + LogPrintf("Upgrading txindex database... [0%%]\n"); + uiInterface.ShowProgress(_("Upgrading txindex database"), 0, true); + int report_done = 0; + const size_t batch_size = 1 << 24; // 16 MiB + + CDBBatch batch_newdb(*this); + CDBBatch batch_olddb(block_tree_db); + + std::pair<unsigned char, uint256> key; + std::pair<unsigned char, uint256> begin_key{DB_TXINDEX, uint256()}; + std::pair<unsigned char, uint256> prev_key = begin_key; + + bool interrupted = false; + std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator()); + for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) { + boost::this_thread::interruption_point(); + if (ShutdownRequested()) { + interrupted = true; + break; + } - const uint256& locator_tip_hash = locator.vHave.front(); - const CBlockIndex* locator_tip_index; - { - LOCK(cs_main); - locator_tip_index = LookupBlockIndex(locator_tip_hash); - } + if (!cursor->GetKey(key)) { + return error("%s: cannot get key from valid cursor", __func__); + } + if (key.first != DB_TXINDEX) { + break; + } - if (!locator_tip_index) { - FatalError("%s: First block (hash=%s) in locator was not found", - __func__, locator_tip_hash.ToString()); - return; + // Log progress every 10%. + if (++count % 256 == 0) { + // Since txids are uniformly random and traversed in increasing order, the high 16 bits + // of the hash can be used to estimate the current progress. + const uint256& txid = key.second; + uint32_t high_nibble = + (static_cast<uint32_t>(*(txid.begin() + 0)) << 8) + + (static_cast<uint32_t>(*(txid.begin() + 1)) << 0); + int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5); + + uiInterface.ShowProgress(_("Upgrading txindex database"), percentage_done, true); + if (report_done < percentage_done/10) { + LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done); + report_done = percentage_done/10; + } + } + + CDiskTxPos value; + if (!cursor->GetValue(value)) { + return error("%s: cannot parse txindex record", __func__); + } + batch_newdb.Write(key, value); + batch_olddb.Erase(key); + + if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) { + // NOTE: it's OK to delete the key pointed at by the current DB cursor while iterating + // because LevelDB iterators are guaranteed to provide a consistent view of the + // underlying data, like a lightweight snapshot. + WriteTxIndexMigrationBatches(*this, block_tree_db, + batch_newdb, batch_olddb, + prev_key, key); + prev_key = key; + } } - // This checks that ChainStateFlushed callbacks are received after BlockConnected. The check may fail - // immediately after the sync thread catches up and sets m_synced. Consider the case where - // there is a reorg and the blocks on the stale branch are in the ValidationInterface queue - // backlog even after the sync thread has caught up to the new chain tip. In this unlikely - // event, log a warning and let the queue clear. - const CBlockIndex* best_block_index = m_best_block_index.load(); - if (best_block_index->GetAncestor(locator_tip_index->nHeight) != locator_tip_index) { - LogPrintf("%s: WARNING: Locator contains block (hash=%s) not on known best " /* Continued */ - "chain (tip=%s); not writing txindex locator\n", - __func__, locator_tip_hash.ToString(), - best_block_index->GetBlockHash().ToString()); - return; + // If these final DB batches complete the migration, write the best block + // hash marker to the new database and delete from the old one. This signals + // that the former is fully caught up to that point in the blockchain and + // that all txindex entries have been removed from the latter. + if (!interrupted) { + batch_olddb.Erase(DB_TXINDEX_BLOCK); + batch_newdb.Write(DB_BEST_BLOCK, locator); } - if (!m_db->WriteBestBlock(locator)) { - error("%s: Failed to write locator to disk", __func__); + WriteTxIndexMigrationBatches(*this, block_tree_db, + batch_newdb, batch_olddb, + begin_key, key); + + if (interrupted) { + LogPrintf("[CANCELLED].\n"); + return false; } + + uiInterface.ShowProgress("", 100, false); + + LogPrintf("[DONE].\n"); + return true; } -bool TxIndex::BlockUntilSyncedToCurrentChain() +TxIndex::TxIndex(size_t n_cache_size, bool f_memory, bool f_wipe) + : m_db(MakeUnique<TxIndex::DB>(n_cache_size, f_memory, f_wipe)) +{} + +TxIndex::~TxIndex() {} + +bool TxIndex::Init() { - AssertLockNotHeld(cs_main); + LOCK(cs_main); - if (!m_synced) { + // Attempt to migrate txindex from the old database to the new one. Even if + // chain_tip is null, the node could be reindexing and we still want to + // delete txindex records in the old database. + if (!m_db->MigrateData(*pblocktree, chainActive.GetLocator())) { return false; } - { - // Skip the queue-draining stuff if we know we're caught up with - // chainActive.Tip(). - LOCK(cs_main); - const CBlockIndex* chain_tip = chainActive.Tip(); - const CBlockIndex* best_block_index = m_best_block_index.load(); - if (best_block_index->GetAncestor(chain_tip->nHeight) == chain_tip) { - return true; - } - } + return BaseIndex::Init(); +} - LogPrintf("%s: txindex is catching up on block notifications\n", __func__); - SyncWithValidationInterfaceQueue(); - return true; +bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) +{ + CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())); + std::vector<std::pair<uint256, CDiskTxPos>> vPos; + vPos.reserve(block.vtx.size()); + for (const auto& tx : block.vtx) { + vPos.emplace_back(tx->GetHash(), pos); + pos.nTxOffset += ::GetSerializeSize(*tx, SER_DISK, CLIENT_VERSION); + } + return m_db->WriteTxs(vPos); } +BaseIndex::DB& TxIndex::GetDB() const { return *m_db; } + bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const { CDiskTxPos postx; @@ -281,31 +284,3 @@ bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRe block_hash = header.GetHash(); return true; } - -void TxIndex::Interrupt() -{ - m_interrupt(); -} - -void TxIndex::Start() -{ - // Need to register this ValidationInterface before running Init(), so that - // callbacks are not missed if Init sets m_synced to true. - RegisterValidationInterface(this); - if (!Init()) { - FatalError("%s: txindex failed to initialize", __func__); - return; - } - - m_thread_sync = std::thread(&TraceThread<std::function<void()>>, "txindex", - std::bind(&TxIndex::ThreadSync, this)); -} - -void TxIndex::Stop() -{ - UnregisterValidationInterface(this); - - if (m_thread_sync.joinable()) { - m_thread_sync.join(); - } -} diff --git a/src/index/txindex.h b/src/index/txindex.h index 4937bd64e9..8202c3c951 100644 --- a/src/index/txindex.h +++ b/src/index/txindex.h @@ -5,70 +5,39 @@ #ifndef BITCOIN_INDEX_TXINDEX_H #define BITCOIN_INDEX_TXINDEX_H -#include <primitives/block.h> -#include <primitives/transaction.h> -#include <threadinterrupt.h> +#include <chain.h> +#include <index/base.h> #include <txdb.h> -#include <uint256.h> -#include <validationinterface.h> - -class CBlockIndex; /** * TxIndex is used to look up transactions included in the blockchain by hash. * The index is written to a LevelDB database and records the filesystem * location of each transaction by transaction hash. */ -class TxIndex final : public CValidationInterface +class TxIndex final : public BaseIndex { -private: - const std::unique_ptr<TxIndexDB> m_db; - - /// Whether the index is in sync with the main chain. The flag is flipped - /// from false to true once, after which point this starts processing - /// ValidationInterface notifications to stay in sync. - std::atomic<bool> m_synced; - - /// The last block in the chain that the TxIndex is in sync with. - std::atomic<const CBlockIndex*> m_best_block_index; - - std::thread m_thread_sync; - CThreadInterrupt m_interrupt; - - /// Initialize internal state from the database and block index. - bool Init(); +protected: + class DB; - /// Sync the tx index with the block index starting from the current best - /// block. Intended to be run in its own thread, m_thread_sync, and can be - /// interrupted with m_interrupt. Once the txindex gets in sync, the - /// m_synced flag is set and the BlockConnected ValidationInterface callback - /// takes over and the sync thread exits. - void ThreadSync(); +private: + const std::unique_ptr<DB> m_db; - /// Write update index entries for a newly connected block. - bool WriteBlock(const CBlock& block, const CBlockIndex* pindex); +protected: + /// Override base class init to migrate from old database. + bool Init() override; - /// Write the current chain block locator to the DB. - bool WriteBestBlock(const CBlockIndex* block_index); + bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; -protected: - void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex, - const std::vector<CTransactionRef>& txn_conflicted) override; + BaseIndex::DB& GetDB() const override; - void ChainStateFlushed(const CBlockLocator& locator) override; + const char* GetName() const override { return "txindex"; } public: - /// Constructs the TxIndex, which becomes available to be queried. - explicit TxIndex(std::unique_ptr<TxIndexDB> db); - - /// Destructor interrupts sync thread if running and blocks until it exits. - ~TxIndex(); + /// Constructs the index, which becomes available to be queried. + explicit TxIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); - /// Blocks the current thread until the transaction index is caught up to - /// the current state of the block chain. This only blocks if the index has gotten in sync once - /// and only needs to process blocks in the ValidationInterface queue. If the index is catching - /// up from far behind, this method does not block and immediately returns false. - bool BlockUntilSyncedToCurrentChain(); + // Destructor is declared because this class contains a unique_ptr to an incomplete type. + virtual ~TxIndex() override; /// Look up a transaction by hash. /// @@ -77,15 +46,6 @@ public: /// @param[out] tx The transaction itself. /// @return true if transaction is found, false otherwise bool FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const; - - void Interrupt(); - - /// Start initializes the sync state and registers the instance as a - /// ValidationInterface so that it stays in sync with blockchain updates. - void Start(); - - /// Stops the instance from staying in sync with blockchain updates. - void Stop(); }; /// The global transaction index, used in GetTransaction. May be null. diff --git a/src/init.cpp b/src/init.cpp index b4e2eec0d2..1b5507703c 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -76,7 +76,7 @@ std::unique_ptr<PeerLogicValidation> peerLogic; class DummyWalletInit : public WalletInitInterface { public: - void AddWalletOptions() const override {} + void AddWalletOptions() const override; bool ParameterInteraction() const override {return true;} void RegisterRPC(CRPCTable &) const override {} bool Verify() const override {return true;} @@ -87,6 +87,15 @@ public: void Close() const override {} }; +void DummyWalletInit::AddWalletOptions() const +{ + std::vector<std::string> opts = {"-addresstype", "-changetype", "-disablewallet", "-discardfee=<amt>", "-fallbackfee=<amt>", + "-keypool=<n>", "-mintxfee=<amt>", "-paytxfee=<amt>", "-rescan", "-salvagewallet", "-spendzeroconfchange", "-txconfirmtarget=<n>", + "-upgradewallet", "-wallet=<path>", "-walletbroadcast", "-walletdir=<dir>", "-walletnotify=<cmd>", "-walletrbf", "-zapwallettxes=<mode>", + "-dblogsize=<n>", "-flushwallet", "-privdb", "-walletrejectlongchains"}; + gArgs.AddHiddenArgs(opts); +} + const WalletInitInterface& g_wallet_init_interface = DummyWalletInit(); #endif @@ -348,6 +357,12 @@ void SetupServerArgs() const auto defaultChainParams = CreateChainParams(CBaseChainParams::MAIN); const auto testnetChainParams = CreateChainParams(CBaseChainParams::TESTNET); + // Hidden Options + std::vector<std::string> hidden_args = {"-rpcssl", "-benchmark", "-h", "-help", "-socks", "-tor", "-debugnet", "-whitelistalwaysrelay", + "-prematurewitness", "-walletprematurewitness", "-promiscuousmempoolflags", "-blockminsize", "-dbcrashratio", "-forcecompactdb", "-usehd", + // GUI args. These will be overwritten by SetupUIArgs for the GUI + "-allowselfsignedrootcertificates", "-choosedatadir", "-lang=<lang>", "-min", "-resetguisettings", "-rootcertificates=<file>", "-splash", "-uiplatform"}; + // Set all of the args and their help // When adding new options to the categories, please keep and ensure alphabetical ordering. gArgs.AddArg("-?", "Print this help message and exit", false, OptionsCategory::OPTIONS); @@ -375,6 +390,8 @@ void SetupServerArgs() gArgs.AddArg("-persistmempool", strprintf("Whether to save the mempool on shutdown and load on restart (default: %u)", DEFAULT_PERSIST_MEMPOOL), false, OptionsCategory::OPTIONS); #ifndef WIN32 gArgs.AddArg("-pid=<file>", strprintf("Specify pid file. Relative paths will be prefixed by a net-specific datadir location. (default: %s)", BITCOIN_PID_FILENAME), false, OptionsCategory::OPTIONS); +#else + hidden_args.emplace_back("-pid"); #endif gArgs.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex and -rescan. " "Warning: Reverting this setting requires re-downloading the entire blockchain. " @@ -383,6 +400,8 @@ void SetupServerArgs() gArgs.AddArg("-reindex-chainstate", "Rebuild chain state from the currently indexed blocks", false, OptionsCategory::OPTIONS); #ifndef WIN32 gArgs.AddArg("-sysperms", "Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)", false, OptionsCategory::OPTIONS); +#else + hidden_args.emplace_back("-sysperms"); #endif gArgs.AddArg("-txindex", strprintf("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)", DEFAULT_TXINDEX), false, OptionsCategory::OPTIONS); @@ -421,6 +440,8 @@ void SetupServerArgs() #else gArgs.AddArg("-upnp", strprintf("Use UPnP to map the listening port (default: %u)", 0), false, OptionsCategory::CONNECTION); #endif +#else + hidden_args.emplace_back("-upnp"); #endif gArgs.AddArg("-whitebind=<addr>", "Bind to given address and whitelist peers connecting to it. Use [host]:port notation for IPv6", false, OptionsCategory::CONNECTION); gArgs.AddArg("-whitelist=<IP address or network>", "Whitelist peers connecting from the given IP address (e.g. 1.2.3.4) or CIDR notated network (e.g. 1.2.3.0/24). Can be specified multiple times." @@ -433,6 +454,11 @@ void SetupServerArgs() gArgs.AddArg("-zmqpubhashtx=<address>", "Enable publish hash transaction in <address>", false, OptionsCategory::ZMQ); gArgs.AddArg("-zmqpubrawblock=<address>", "Enable publish raw block in <address>", false, OptionsCategory::ZMQ); gArgs.AddArg("-zmqpubrawtx=<address>", "Enable publish raw transaction in <address>", false, OptionsCategory::ZMQ); +#else + hidden_args.emplace_back("-zmqpubhashblock=<address>"); + hidden_args.emplace_back("-zmqpubhashtx=<address>"); + hidden_args.emplace_back("-zmqpubrawblock=<address>"); + hidden_args.emplace_back("-zmqpubrawtx=<address>"); #endif gArgs.AddArg("-checkblocks=<n>", strprintf("How many blocks to check at startup (default: %u, 0 = all)", DEFAULT_CHECKBLOCKS), true, OptionsCategory::DEBUG_TEST); @@ -500,22 +526,14 @@ void SetupServerArgs() gArgs.AddArg("-rpcworkqueue=<n>", strprintf("Set the depth of the work queue to service RPC calls (default: %d)", DEFAULT_HTTP_WORKQUEUE), true, OptionsCategory::RPC); gArgs.AddArg("-server", "Accept command line and JSON-RPC commands", false, OptionsCategory::RPC); - // Hidden options - gArgs.AddArg("-rpcssl", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-benchmark", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-h", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-help", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-socks", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-tor", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-debugnet", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-whitelistalwaysrelay", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-prematurewitness", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-walletprematurewitness", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-promiscuousmempoolflags", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-blockminsize", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-dbcrashratio", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-forcecompactdb", "", false, OptionsCategory::HIDDEN); - gArgs.AddArg("-usehd", "", false, OptionsCategory::HIDDEN); +#if HAVE_DECL_DAEMON + gArgs.AddArg("-daemon", "Run in the background as a daemon and accept commands", false, OptionsCategory::OPTIONS); +#else + hidden_args.emplace_back("-daemon"); +#endif + + // Add the hidden options + gArgs.AddHiddenArgs(hidden_args); } std::string LicenseInfo() @@ -614,7 +632,7 @@ static void CleanupBlockRevFiles() // keeping a separate counter. Once we hit a gap (or if 0 doesn't exist) // start removing block files. int nContigCounter = 0; - for (const std::pair<std::string, fs::path>& item : mapBlockFiles) { + for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) { if (atoi(item.first) == nContigCounter) { nContigCounter++; continue; @@ -1606,8 +1624,7 @@ bool AppInitMain() // ********************************************************* Step 8: start indexers if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - auto txindex_db = MakeUnique<TxIndexDB>(nTxIndexCache, false, fReindex); - g_txindex = MakeUnique<TxIndex>(std::move(txindex_db)); + g_txindex = MakeUnique<TxIndex>(nTxIndexCache, false, fReindex); g_txindex->Start(); } diff --git a/src/net_processing.cpp b/src/net_processing.cpp index fc05dd2ad2..de456e87f4 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -561,7 +561,7 @@ static void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vec } // namespace // This function is used for testing the stale tip eviction logic, see -// DoS_tests.cpp +// denialofservice_tests.cpp void UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds) { LOCK(cs_main); diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index 5963bf371a..aac3fe5c14 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -54,7 +54,7 @@ bool IsDust(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) return (txout.nValue < GetDustThreshold(txout, dustRelayFeeIn)); } -bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType, const bool witnessEnabled) +bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType) { std::vector<std::vector<unsigned char> > vSolutions; if (!Solver(scriptPubKey, whichType, vSolutions)) @@ -73,13 +73,10 @@ bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType, const bool w (!fAcceptDatacarrier || scriptPubKey.size() > nMaxDatacarrierBytes)) return false; - else if (!witnessEnabled && (whichType == TX_WITNESS_V0_KEYHASH || whichType == TX_WITNESS_V0_SCRIPTHASH)) - return false; - return whichType != TX_NONSTANDARD && whichType != TX_WITNESS_UNKNOWN; } -bool IsStandardTx(const CTransaction& tx, std::string& reason, const bool witnessEnabled) +bool IsStandardTx(const CTransaction& tx, std::string& reason) { if (tx.nVersion > CTransaction::MAX_STANDARD_VERSION || tx.nVersion < 1) { reason = "version"; @@ -118,7 +115,7 @@ bool IsStandardTx(const CTransaction& tx, std::string& reason, const bool witnes unsigned int nDataOut = 0; txnouttype whichType; for (const CTxOut& txout : tx.vout) { - if (!::IsStandard(txout.scriptPubKey, whichType, witnessEnabled)) { + if (!::IsStandard(txout.scriptPubKey, whichType)) { reason = "scriptpubkey"; return false; } diff --git a/src/policy/policy.h b/src/policy/policy.h index 5ce019df4c..035627bd60 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -79,12 +79,12 @@ CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFee); bool IsDust(const CTxOut& txout, const CFeeRate& dustRelayFee); -bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType, const bool witnessEnabled = false); +bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType); /** * Check for standard transaction types * @return True if all outputs (scriptPubKeys) use only standard transaction forms */ -bool IsStandardTx(const CTransaction& tx, std::string& reason, const bool witnessEnabled = false); +bool IsStandardTx(const CTransaction& tx, std::string& reason); /** * Check for standard transaction types * @param[in] mapInputs Map of previous transactions that have outputs we're spending diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 1c846d38ec..360615ec56 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -388,11 +388,6 @@ struct CMutableTransaction */ uint256 GetHash() const; - friend bool operator==(const CMutableTransaction& a, const CMutableTransaction& b) - { - return a.GetHash() == b.GetHash(); - } - bool HasWitness() const { for (size_t i = 0; i < vin.size(); i++) { diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index a3721991ee..8f34e6bc82 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -38,6 +38,69 @@ </widget> </item> <item> + <spacer name="horizontalSpacer_0_Main"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>40</width> + <height>5</height> + </size> + </property> + </spacer> + </item> + <item> + <layout class="QHBoxLayout" name="horizontalLayout_Main_Prune"> + <item> + <widget class="QCheckBox" name="prune"> + <property name="toolTip"> + <string>Disables some advanced features but all blocks will still be fully validated. Reverting this setting requires re-downloading the entire blockchain. Actual disk usage may be somewhat higher.</string> + </property> + <property name="text"> + <string>Prune &block storage to</string> + </property> + </widget> + </item> + <item> + <widget class="QSpinBox" name="pruneSize"/> + </item> + <item> + <widget class="QLabel" name="pruneSizeUnitLabel"> + <property name="text"> + <string>GB</string> + </property> + <property name="textFormat"> + <enum>Qt::PlainText</enum> + </property> + </widget> + </item> + <item> + <spacer name="horizontalSpacer_Main_Prune"> + <property name="orientation"> + <enum>Qt::Horizontal</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>40</width> + <height>20</height> + </size> + </property> + </spacer> + </item> + </layout> + </item> + <item> + <widget class="QLabel" name="pruneWarning"> + <property name="text"> + <string>Reverting this setting requires re-downloading the entire blockchain.</string> + </property> + <property name="textFormat"> + <enum>Qt::PlainText</enum> + </property> + </widget> + </item> + <item> <layout class="QHBoxLayout" name="horizontalLayout_2_Main"> <item> <widget class="QLabel" name="databaseCacheLabel"> @@ -81,7 +144,7 @@ </layout> </item> <item> - <layout class="QHBoxLayout" name="horizontalLayout_3_Main"> + <layout class="QHBoxLayout" name="horizontalLayout_Main_VerifyLabel"> <item> <widget class="QLabel" name="threadsScriptVerifLabel"> <property name="text"> @@ -103,7 +166,7 @@ </widget> </item> <item> - <spacer name="horizontalSpacer_3_Main"> + <spacer name="horizontalSpacer_Main_Threads"> <property name="orientation"> <enum>Qt::Horizontal</enum> </property> diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index c0ddb89b40..108aa4f99c 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -36,8 +36,17 @@ OptionsDialog::OptionsDialog(QWidget *parent, bool enableWallet) : /* Main elements init */ ui->databaseCache->setMinimum(nMinDbCache); ui->databaseCache->setMaximum(nMaxDbCache); + static const uint64_t GiB = 1024 * 1024 * 1024; + static const uint64_t nMinDiskSpace = MIN_DISK_SPACE_FOR_BLOCK_FILES / GiB + + (MIN_DISK_SPACE_FOR_BLOCK_FILES % GiB) ? 1 : 0; + ui->pruneSize->setMinimum(nMinDiskSpace); ui->threadsScriptVerif->setMinimum(-GetNumCores()); ui->threadsScriptVerif->setMaximum(MAX_SCRIPTCHECK_THREADS); + ui->pruneWarning->setVisible(false); + ui->pruneWarning->setStyleSheet("QLabel { color: red; }"); + + ui->pruneSize->setEnabled(false); + connect(ui->prune, SIGNAL(toggled(bool)), ui->pruneSize, SLOT(setEnabled(bool))); /* Network elements init */ #ifndef USE_UPNP @@ -157,6 +166,9 @@ void OptionsDialog::setModel(OptionsModel *_model) /* warn when one of the following settings changes by user action (placed here so init via mapper doesn't trigger them) */ /* Main */ + connect(ui->prune, SIGNAL(clicked(bool)), this, SLOT(showRestartWarning())); + connect(ui->prune, SIGNAL(clicked(bool)), this, SLOT(togglePruneWarning(bool))); + connect(ui->pruneSize, SIGNAL(valueChanged(int)), this, SLOT(showRestartWarning())); connect(ui->databaseCache, SIGNAL(valueChanged(int)), this, SLOT(showRestartWarning())); connect(ui->threadsScriptVerif, SIGNAL(valueChanged(int)), this, SLOT(showRestartWarning())); /* Wallet */ @@ -176,6 +188,8 @@ void OptionsDialog::setMapper() mapper->addMapping(ui->bitcoinAtStartup, OptionsModel::StartAtStartup); mapper->addMapping(ui->threadsScriptVerif, OptionsModel::ThreadsScriptVerif); mapper->addMapping(ui->databaseCache, OptionsModel::DatabaseCache); + mapper->addMapping(ui->prune, OptionsModel::Prune); + mapper->addMapping(ui->pruneSize, OptionsModel::PruneSize); /* Wallet */ mapper->addMapping(ui->spendZeroConfChange, OptionsModel::SpendZeroConfChange); @@ -266,6 +280,11 @@ void OptionsDialog::on_hideTrayIcon_stateChanged(int fState) } } +void OptionsDialog::togglePruneWarning(bool enabled) +{ + ui->pruneWarning->setVisible(!ui->pruneWarning->isVisible()); +} + void OptionsDialog::showRestartWarning(bool fPersistent) { ui->statusLabel->setStyleSheet("QLabel { color: red; }"); diff --git a/src/qt/optionsdialog.h b/src/qt/optionsdialog.h index faf9ff8959..5aad484ce7 100644 --- a/src/qt/optionsdialog.h +++ b/src/qt/optionsdialog.h @@ -53,6 +53,7 @@ private Q_SLOTS: void on_hideTrayIcon_stateChanged(int fState); + void togglePruneWarning(bool enabled); void showRestartWarning(bool fPersistent = false); void clearStatusLabel(); void updateProxyValidationState(); diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index cae9dace4c..31a85f4e23 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -88,6 +88,16 @@ void OptionsModel::Init(bool resetSettings) // by command-line and show this in the UI. // Main + if (!settings.contains("bPrune")) + settings.setValue("bPrune", false); + if (!settings.contains("nPruneSize")) + settings.setValue("nPruneSize", 2); + // Convert prune size to MB: + const uint64_t nPruneSizeMB = settings.value("nPruneSize").toInt() * 1000; + if (!m_node.softSetArg("-prune", settings.value("bPrune").toBool() ? std::to_string(nPruneSizeMB) : "0")) { + addOverriddenOption("-prune"); + } + if (!settings.contains("nDatabaseCache")) settings.setValue("nDatabaseCache", (qint64)nDefaultDbCache); if (!m_node.softSetArg("-dbcache", settings.value("nDatabaseCache").toString().toStdString())) @@ -281,6 +291,10 @@ QVariant OptionsModel::data(const QModelIndex & index, int role) const return settings.value("language"); case CoinControlFeatures: return fCoinControlFeatures; + case Prune: + return settings.value("bPrune"); + case PruneSize: + return settings.value("nPruneSize"); case DatabaseCache: return settings.value("nDatabaseCache"); case ThreadsScriptVerif: @@ -405,6 +419,18 @@ bool OptionsModel::setData(const QModelIndex & index, const QVariant & value, in settings.setValue("fCoinControlFeatures", fCoinControlFeatures); Q_EMIT coinControlFeaturesChanged(fCoinControlFeatures); break; + case Prune: + if (settings.value("bPrune") != value) { + settings.setValue("bPrune", value); + setRestartRequired(true); + } + break; + case PruneSize: + if (settings.value("nPruneSize") != value) { + settings.setValue("nPruneSize", value); + setRestartRequired(true); + } + break; case DatabaseCache: if (settings.value("nDatabaseCache") != value) { settings.setValue("nDatabaseCache", value); diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index fc1d119a71..2777cbeaf2 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -50,6 +50,8 @@ public: Language, // QString CoinControlFeatures, // bool ThreadsScriptVerif, // int + Prune, // bool + PruneSize, // int DatabaseCache, // int SpendZeroConfChange, // bool Listen, // bool diff --git a/src/qt/res/movies/makespinner.sh b/src/qt/res/movies/makespinner.sh index d0deb1238c..76e36e4f31 100755 --- a/src/qt/res/movies/makespinner.sh +++ b/src/qt/res/movies/makespinner.sh @@ -2,6 +2,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. +export LC_ALL=C FRAMEDIR=$(dirname $0) for i in {0..35} do diff --git a/src/rest.cpp b/src/rest.cpp index ffa75c241f..a5f164497d 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -217,7 +217,7 @@ static bool rest_block(HTTPRequest* req, return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } - if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) + if (IsBlockPruned(pblockindex)) return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not available (pruned data)"); if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 48e0e02d1a..68279d7ffd 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -95,6 +95,7 @@ UniValue blockheaderToJSON(const CBlockIndex* blockindex) result.pushKV("bits", strprintf("%08x", blockindex->nBits)); result.pushKV("difficulty", GetDifficulty(blockindex)); result.pushKV("chainwork", blockindex->nChainWork.GetHex()); + result.pushKV("nTx", (uint64_t)blockindex->nTx); if (blockindex->pprev) result.pushKV("previousblockhash", blockindex->pprev->GetBlockHash().GetHex()); @@ -140,6 +141,7 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool tx result.pushKV("bits", strprintf("%08x", block.nBits)); result.pushKV("difficulty", GetDifficulty(blockindex)); result.pushKV("chainwork", blockindex->nChainWork.GetHex()); + result.pushKV("nTx", (uint64_t)blockindex->nTx); if (blockindex->pprev) result.pushKV("previousblockhash", blockindex->pprev->GetBlockHash().GetHex()); @@ -694,6 +696,7 @@ static UniValue getblockheader(const JSONRPCRequest& request) " \"bits\" : \"1d00ffff\", (string) The bits\n" " \"difficulty\" : x.xxx, (numeric) The difficulty\n" " \"chainwork\" : \"0000...1f3\" (string) Expected number of hashes required to produce the current chain (in hex)\n" + " \"nTx\" : n, (numeric) The number of transactions in the block.\n" " \"previousblockhash\" : \"hash\", (string) The hash of the previous block\n" " \"nextblockhash\" : \"hash\", (string) The hash of the next block\n" "}\n" @@ -732,7 +735,7 @@ static UniValue getblockheader(const JSONRPCRequest& request) static CBlock GetBlockChecked(const CBlockIndex* pblockindex) { CBlock block; - if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) { + if (IsBlockPruned(pblockindex)) { throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)"); } @@ -782,6 +785,7 @@ static UniValue getblock(const JSONRPCRequest& request) " \"bits\" : \"1d00ffff\", (string) The bits\n" " \"difficulty\" : x.xxx, (numeric) The difficulty\n" " \"chainwork\" : \"xxxx\", (string) Expected number of hashes required to produce the chain up to this block (in hex)\n" + " \"nTx\" : n, (numeric) The number of transactions in the block.\n" " \"previousblockhash\" : \"hash\", (string) The hash of the previous block\n" " \"nextblockhash\" : \"hash\" (string) The hash of the next block\n" "}\n" diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index bb68f72ccc..0f35fd3770 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -52,7 +52,6 @@ static const CRPCConvertParam vRPCConvertParams[] = { "listreceivedbylabel", 0, "minconf" }, { "listreceivedbylabel", 1, "include_empty" }, { "listreceivedbylabel", 2, "include_watchonly" }, - { "getlabeladdress", 1, "force" }, { "getbalance", 1, "minconf" }, { "getbalance", 2, "include_watchonly" }, { "getblockhash", 0, "height" }, diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 1530d8578b..8fa56e9335 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -475,7 +475,7 @@ static UniValue getnetworkinfo(const JSONRPCRequest& request) UniValue localAddresses(UniValue::VARR); { LOCK(cs_mapLocalHost); - for (const std::pair<CNetAddr, LocalServiceInfo> &item : mapLocalHost) + for (const std::pair<const CNetAddr, LocalServiceInfo> &item : mapLocalHost) { UniValue rec(UniValue::VOBJ); rec.pushKV("address", item.first.ToString()); diff --git a/src/test/arith_uint256_tests.cpp b/src/test/arith_uint256_tests.cpp index 13ec19834a..8644aea371 100644 --- a/src/test/arith_uint256_tests.cpp +++ b/src/test/arith_uint256_tests.cpp @@ -198,13 +198,6 @@ BOOST_AUTO_TEST_CASE( shifts ) { // "<<" ">>" "<<=" ">>=" BOOST_AUTO_TEST_CASE( unaryOperators ) // ! ~ - { - BOOST_CHECK(!ZeroL); - BOOST_CHECK(!(!OneL)); - for (unsigned int i = 0; i < 256; ++i) - BOOST_CHECK(!(!(OneL<<i))); - BOOST_CHECK(!(!R1L)); - BOOST_CHECK(!(!MaxL)); - BOOST_CHECK(~ZeroL == MaxL); unsigned char TmpArray[32]; diff --git a/src/test/blockchain_tests.cpp b/src/test/blockchain_tests.cpp index d2d0008122..7d8ae46fb8 100644 --- a/src/test/blockchain_tests.cpp +++ b/src/test/blockchain_tests.cpp @@ -1,9 +1,9 @@ #include <boost/test/unit_test.hpp> -#include "stdlib.h" +#include <stdlib.h> -#include "rpc/blockchain.h" -#include "test/test_bitcoin.h" +#include <rpc/blockchain.h> +#include <test/test_bitcoin.h> /* Equality between doubles is imprecise. Comparison should be done * with a small threshold of tolerance, rather than exact equality. diff --git a/src/test/DoS_tests.cpp b/src/test/denialofservice_tests.cpp index 1868aed7dd..e5f914ba8a 100644 --- a/src/test/DoS_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -42,7 +42,7 @@ static NodeId id = 0; void UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds); -BOOST_FIXTURE_TEST_SUITE(DoS_tests, TestingSetup) +BOOST_FIXTURE_TEST_SUITE(denialofservice_tests, TestingSetup) // Test eviction of an outbound peer whose chain never advances // Mock a node connection, and use mocktime to simulate a peer diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 5ca243f42e..0264d29455 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -571,4 +571,179 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) SetMockTime(0); } +inline CTransactionRef make_tx(std::vector<CAmount>&& output_values, std::vector<CTransactionRef>&& inputs=std::vector<CTransactionRef>(), std::vector<uint32_t>&& input_indices=std::vector<uint32_t>()) +{ + CMutableTransaction tx = CMutableTransaction(); + tx.vin.resize(inputs.size()); + tx.vout.resize(output_values.size()); + for (size_t i = 0; i < inputs.size(); ++i) { + tx.vin[i].prevout.hash = inputs[i]->GetHash(); + tx.vin[i].prevout.n = input_indices.size() > i ? input_indices[i] : 0; + } + for (size_t i = 0; i < output_values.size(); ++i) { + tx.vout[i].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + tx.vout[i].nValue = output_values[i]; + } + return MakeTransactionRef(tx); +} + + +BOOST_AUTO_TEST_CASE(MempoolAncestryTests) +{ + size_t ancestors, descendants; + + CTxMemPool pool; + TestMemPoolEntryHelper entry; + + /* Base transaction */ + // + // [tx1] + // + CTransactionRef tx1 = make_tx(/* output_values */ {10 * COIN}); + pool.addUnchecked(tx1->GetHash(), entry.Fee(10000LL).FromTx(tx1)); + + // Ancestors / descendants should be 1 / 1 (itself / itself) + pool.GetTransactionAncestry(tx1->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 1ULL); + BOOST_CHECK_EQUAL(descendants, 1ULL); + + /* Child transaction */ + // + // [tx1].0 <- [tx2] + // + CTransactionRef tx2 = make_tx(/* output_values */ {495 * CENT, 5 * COIN}, /* inputs */ {tx1}); + pool.addUnchecked(tx2->GetHash(), entry.Fee(10000LL).FromTx(tx2)); + + // Ancestors / descendants should be: + // transaction ancestors descendants + // ============ =========== =========== + // tx1 1 (tx1) 2 (tx1,2) + // tx2 2 (tx1,2) 2 (tx1,2) + pool.GetTransactionAncestry(tx1->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 1ULL); + BOOST_CHECK_EQUAL(descendants, 2ULL); + pool.GetTransactionAncestry(tx2->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 2ULL); + BOOST_CHECK_EQUAL(descendants, 2ULL); + + /* Grand-child 1 */ + // + // [tx1].0 <- [tx2].0 <- [tx3] + // + CTransactionRef tx3 = make_tx(/* output_values */ {290 * CENT, 200 * CENT}, /* inputs */ {tx2}); + pool.addUnchecked(tx3->GetHash(), entry.Fee(10000LL).FromTx(tx3)); + + // Ancestors / descendants should be: + // transaction ancestors descendants + // ============ =========== =========== + // tx1 1 (tx1) 3 (tx1,2,3) + // tx2 2 (tx1,2) 3 (tx1,2,3) + // tx3 3 (tx1,2,3) 3 (tx1,2,3) + pool.GetTransactionAncestry(tx1->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 1ULL); + BOOST_CHECK_EQUAL(descendants, 3ULL); + pool.GetTransactionAncestry(tx2->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 2ULL); + BOOST_CHECK_EQUAL(descendants, 3ULL); + pool.GetTransactionAncestry(tx3->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 3ULL); + BOOST_CHECK_EQUAL(descendants, 3ULL); + + /* Grand-child 2 */ + // + // [tx1].0 <- [tx2].0 <- [tx3] + // | + // \---1 <- [tx4] + // + CTransactionRef tx4 = make_tx(/* output_values */ {290 * CENT, 250 * CENT}, /* inputs */ {tx2}, /* input_indices */ {1}); + pool.addUnchecked(tx4->GetHash(), entry.Fee(10000LL).FromTx(tx4)); + + // Ancestors / descendants should be: + // transaction ancestors descendants + // ============ =========== =========== + // tx1 1 (tx1) 4 (tx1,2,3,4) + // tx2 2 (tx1,2) 4 (tx1,2,3,4) + // tx3 3 (tx1,2,3) 4 (tx1,2,3,4) + // tx4 3 (tx1,2,4) 4 (tx1,2,3,4) + pool.GetTransactionAncestry(tx1->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 1ULL); + BOOST_CHECK_EQUAL(descendants, 4ULL); + pool.GetTransactionAncestry(tx2->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 2ULL); + BOOST_CHECK_EQUAL(descendants, 4ULL); + pool.GetTransactionAncestry(tx3->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 3ULL); + BOOST_CHECK_EQUAL(descendants, 4ULL); + pool.GetTransactionAncestry(tx4->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 3ULL); + BOOST_CHECK_EQUAL(descendants, 4ULL); + + /* Make an alternate branch that is longer and connect it to tx3 */ + // + // [ty1].0 <- [ty2].0 <- [ty3].0 <- [ty4].0 <- [ty5].0 + // | + // [tx1].0 <- [tx2].0 <- [tx3].0 <- [ty6] --->--/ + // | + // \---1 <- [tx4] + // + CTransactionRef ty1, ty2, ty3, ty4, ty5; + CTransactionRef* ty[5] = {&ty1, &ty2, &ty3, &ty4, &ty5}; + CAmount v = 5 * COIN; + for (uint64_t i = 0; i < 5; i++) { + CTransactionRef& tyi = *ty[i]; + tyi = make_tx(/* output_values */ {v}, /* inputs */ i > 0 ? std::vector<CTransactionRef>{*ty[i - 1]} : std::vector<CTransactionRef>{}); + v -= 50 * CENT; + pool.addUnchecked(tyi->GetHash(), entry.Fee(10000LL).FromTx(tyi)); + pool.GetTransactionAncestry(tyi->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, i+1); + BOOST_CHECK_EQUAL(descendants, i+1); + } + CTransactionRef ty6 = make_tx(/* output_values */ {5 * COIN}, /* inputs */ {tx3, ty5}); + pool.addUnchecked(ty6->GetHash(), entry.Fee(10000LL).FromTx(ty6)); + + // Ancestors / descendants should be: + // transaction ancestors descendants + // ============ =================== =========== + // tx1 1 (tx1) 5 (tx1,2,3,4, ty6) + // tx2 2 (tx1,2) 5 (tx1,2,3,4, ty6) + // tx3 3 (tx1,2,3) 5 (tx1,2,3,4, ty6) + // tx4 3 (tx1,2,4) 5 (tx1,2,3,4, ty6) + // ty1 1 (ty1) 6 (ty1,2,3,4,5,6) + // ty2 2 (ty1,2) 6 (ty1,2,3,4,5,6) + // ty3 3 (ty1,2,3) 6 (ty1,2,3,4,5,6) + // ty4 4 (y1234) 6 (ty1,2,3,4,5,6) + // ty5 5 (y12345) 6 (ty1,2,3,4,5,6) + // ty6 9 (tx123, ty123456) 6 (ty1,2,3,4,5,6) + pool.GetTransactionAncestry(tx1->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 1ULL); + BOOST_CHECK_EQUAL(descendants, 5ULL); + pool.GetTransactionAncestry(tx2->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 2ULL); + BOOST_CHECK_EQUAL(descendants, 5ULL); + pool.GetTransactionAncestry(tx3->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 3ULL); + BOOST_CHECK_EQUAL(descendants, 5ULL); + pool.GetTransactionAncestry(tx4->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 3ULL); + BOOST_CHECK_EQUAL(descendants, 5ULL); + pool.GetTransactionAncestry(ty1->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 1ULL); + BOOST_CHECK_EQUAL(descendants, 6ULL); + pool.GetTransactionAncestry(ty2->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 2ULL); + BOOST_CHECK_EQUAL(descendants, 6ULL); + pool.GetTransactionAncestry(ty3->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 3ULL); + BOOST_CHECK_EQUAL(descendants, 6ULL); + pool.GetTransactionAncestry(ty4->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 4ULL); + BOOST_CHECK_EQUAL(descendants, 6ULL); + pool.GetTransactionAncestry(ty5->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 5ULL); + BOOST_CHECK_EQUAL(descendants, 6ULL); + pool.GetTransactionAncestry(ty6->GetHash(), ancestors, descendants); + BOOST_CHECK_EQUAL(ancestors, 9ULL); + BOOST_CHECK_EQUAL(descendants, 6ULL); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/script_P2SH_tests.cpp b/src/test/script_p2sh_tests.cpp index 63d211dd97..803a673fab 100644 --- a/src/test/script_P2SH_tests.cpp +++ b/src/test/script_p2sh_tests.cpp @@ -46,7 +46,7 @@ Verify(const CScript& scriptSig, const CScript& scriptPubKey, bool fStrict, Scri } -BOOST_FIXTURE_TEST_SUITE(script_P2SH_tests, BasicTestingSetup) +BOOST_FIXTURE_TEST_SUITE(script_p2sh_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(sign) { diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index 14158f2875..be7ee2428b 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -15,7 +15,7 @@ BOOST_AUTO_TEST_SUITE(txindex_tests) BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) { - TxIndex txindex(MakeUnique<TxIndexDB>(1 << 20, true)); + TxIndex txindex(1 << 20, true); CTransactionRef tx_disk; uint256 block_hash; diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 2b00064cd0..d32d4b267c 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -102,7 +102,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) // should fail. // Capture this interaction with the upgraded_nop argument: set it when evaluating // any script flag that is implemented as an upgraded NOP code. -static void ValidateCheckInputsForAllFlags(CMutableTransaction &tx, uint32_t failing_flags, bool add_to_cache) +static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t failing_flags, bool add_to_cache) { PrecomputedTransactionData txdata(tx); // If we add many more flags, this loop can get too expensive, but we can diff --git a/src/txdb.cpp b/src/txdb.cpp index 333d3596c1..b1d5879c83 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -21,8 +21,6 @@ static const char DB_COIN = 'C'; static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; -static const char DB_TXINDEX = 't'; -static const char DB_TXINDEX_BLOCK = 'T'; static const char DB_BLOCK_INDEX = 'b'; static const char DB_BEST_BLOCK = 'B'; @@ -237,17 +235,6 @@ bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockF return WriteBatch(batch, true); } -bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) { - return Read(std::make_pair(DB_TXINDEX, txid), pos); -} - -bool CBlockTreeDB::WriteTxIndex(const std::vector<std::pair<uint256, CDiskTxPos> >&vect) { - CDBBatch batch(*this); - for (std::vector<std::pair<uint256,CDiskTxPos> >::const_iterator it=vect.begin(); it!=vect.end(); it++) - batch.Write(std::make_pair(DB_TXINDEX, it->first), it->second); - return WriteBatch(batch); -} - bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) { return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0'); } @@ -425,173 +412,3 @@ bool CCoinsViewDB::Upgrade() { LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE"); return !ShutdownRequested(); } - -TxIndexDB::TxIndexDB(size_t n_cache_size, bool f_memory, bool f_wipe) : - CDBWrapper(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe) -{} - -bool TxIndexDB::ReadTxPos(const uint256 &txid, CDiskTxPos& pos) const -{ - return Read(std::make_pair(DB_TXINDEX, txid), pos); -} - -bool TxIndexDB::WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos) -{ - CDBBatch batch(*this); - for (const auto& tuple : v_pos) { - batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second); - } - return WriteBatch(batch); -} - -bool TxIndexDB::ReadBestBlock(CBlockLocator& locator) const -{ - bool success = Read(DB_BEST_BLOCK, locator); - if (!success) { - locator.SetNull(); - } - return success; -} - -bool TxIndexDB::WriteBestBlock(const CBlockLocator& locator) -{ - return Write(DB_BEST_BLOCK, locator); -} - -/* - * Safely persist a transfer of data from the old txindex database to the new one, and compact the - * range of keys updated. This is used internally by MigrateData. - */ -static void WriteTxIndexMigrationBatches(TxIndexDB& newdb, CBlockTreeDB& olddb, - CDBBatch& batch_newdb, CDBBatch& batch_olddb, - const std::pair<unsigned char, uint256>& begin_key, - const std::pair<unsigned char, uint256>& end_key) -{ - // Sync new DB changes to disk before deleting from old DB. - newdb.WriteBatch(batch_newdb, /*fSync=*/ true); - olddb.WriteBatch(batch_olddb); - olddb.CompactRange(begin_key, end_key); - - batch_newdb.Clear(); - batch_olddb.Clear(); -} - -bool TxIndexDB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator) -{ - // The prior implementation of txindex was always in sync with block index - // and presence was indicated with a boolean DB flag. If the flag is set, - // this means the txindex from a previous version is valid and in sync with - // the chain tip. The first step of the migration is to unset the flag and - // write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the - // index entries are copied over in batches to the new database. Finally, - // DB_TXINDEX_BLOCK is erased from the old database and the block hash is - // written to the new database. - // - // Unsetting the boolean flag ensures that if the node is downgraded to a - // previous version, it will not see a corrupted, partially migrated index - // -- it will see that the txindex is disabled. When the node is upgraded - // again, the migration will pick up where it left off and sync to the block - // with hash DB_TXINDEX_BLOCK. - bool f_legacy_flag = false; - block_tree_db.ReadFlag("txindex", f_legacy_flag); - if (f_legacy_flag) { - if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) { - return error("%s: cannot write block indicator", __func__); - } - if (!block_tree_db.WriteFlag("txindex", false)) { - return error("%s: cannot write block index db flag", __func__); - } - } - - CBlockLocator locator; - if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) { - return true; - } - - int64_t count = 0; - LogPrintf("Upgrading txindex database... [0%%]\n"); - uiInterface.ShowProgress(_("Upgrading txindex database"), 0, true); - int report_done = 0; - const size_t batch_size = 1 << 24; // 16 MiB - - CDBBatch batch_newdb(*this); - CDBBatch batch_olddb(block_tree_db); - - std::pair<unsigned char, uint256> key; - std::pair<unsigned char, uint256> begin_key{DB_TXINDEX, uint256()}; - std::pair<unsigned char, uint256> prev_key = begin_key; - - bool interrupted = false; - std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator()); - for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) { - boost::this_thread::interruption_point(); - if (ShutdownRequested()) { - interrupted = true; - break; - } - - if (!cursor->GetKey(key)) { - return error("%s: cannot get key from valid cursor", __func__); - } - if (key.first != DB_TXINDEX) { - break; - } - - // Log progress every 10%. - if (++count % 256 == 0) { - // Since txids are uniformly random and traversed in increasing order, the high 16 bits - // of the hash can be used to estimate the current progress. - const uint256& txid = key.second; - uint32_t high_nibble = - (static_cast<uint32_t>(*(txid.begin() + 0)) << 8) + - (static_cast<uint32_t>(*(txid.begin() + 1)) << 0); - int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5); - - uiInterface.ShowProgress(_("Upgrading txindex database"), percentage_done, true); - if (report_done < percentage_done/10) { - LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done); - report_done = percentage_done/10; - } - } - - CDiskTxPos value; - if (!cursor->GetValue(value)) { - return error("%s: cannot parse txindex record", __func__); - } - batch_newdb.Write(key, value); - batch_olddb.Erase(key); - - if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) { - // NOTE: it's OK to delete the key pointed at by the current DB cursor while iterating - // because LevelDB iterators are guaranteed to provide a consistent view of the - // underlying data, like a lightweight snapshot. - WriteTxIndexMigrationBatches(*this, block_tree_db, - batch_newdb, batch_olddb, - prev_key, key); - prev_key = key; - } - } - - // If these final DB batches complete the migration, write the best block - // hash marker to the new database and delete from the old one. This signals - // that the former is fully caught up to that point in the blockchain and - // that all txindex entries have been removed from the latter. - if (!interrupted) { - batch_olddb.Erase(DB_TXINDEX_BLOCK); - batch_newdb.Write(DB_BEST_BLOCK, locator); - } - - WriteTxIndexMigrationBatches(*this, block_tree_db, - batch_newdb, batch_olddb, - begin_key, key); - - if (interrupted) { - LogPrintf("[CANCELLED].\n"); - return false; - } - - uiInterface.ShowProgress("", 100, false); - - LogPrintf("[DONE].\n"); - return true; -} diff --git a/src/txdb.h b/src/txdb.h index 4193f98de1..100adb428d 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -40,31 +40,6 @@ static const int64_t nMaxTxIndexCache = 1024; //! Max memory allocated to coin DB specific cache (MiB) static const int64_t nMaxCoinsDBCache = 8; -struct CDiskTxPos : public CDiskBlockPos -{ - unsigned int nTxOffset; // after header - - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(CDiskBlockPos, *this); - READWRITE(VARINT(nTxOffset)); - } - - CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { - } - - CDiskTxPos() { - SetNull(); - } - - void SetNull() { - CDiskBlockPos::SetNull(); - nTxOffset = 0; - } -}; - /** CCoinsView backed by the coin database (chainstate/) */ class CCoinsViewDB final : public CCoinsView { @@ -118,43 +93,9 @@ public: bool ReadLastBlockFile(int &nFile); bool WriteReindexing(bool fReindexing); bool ReadReindexing(bool &fReindexing); - bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); - bool WriteTxIndex(const std::vector<std::pair<uint256, CDiskTxPos> > &vect); bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex); }; -/** - * Access to the txindex database (indexes/txindex/) - * - * The database stores a block locator of the chain the database is synced to - * so that the TxIndex can efficiently determine the point it last stopped at. - * A locator is used instead of a simple hash of the chain tip because blocks - * and block index entries may not be flushed to disk until after this database - * is updated. - */ -class TxIndexDB : public CDBWrapper -{ -public: - explicit TxIndexDB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); - - /// Read the disk location of the transaction data with the given hash. Returns false if the - /// transaction hash is not indexed. - bool ReadTxPos(const uint256& txid, CDiskTxPos& pos) const; - - /// Write a batch of transaction positions to the DB. - bool WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos); - - /// Read block locator of the chain that the txindex is in sync with. - bool ReadBestBlock(CBlockLocator& locator) const; - - /// Write block locator of the chain that the txindex is in sync with. - bool WriteBestBlock(const CBlockLocator& locator); - - /// Migrate txindex data from the block tree DB, where it may be for older nodes that have not - /// been upgraded yet to the new database. - bool MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator); -}; - #endif // BITCOIN_TXDB_H diff --git a/src/txmempool.cpp b/src/txmempool.cpp index bb585fc075..1c6dba0c9c 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -693,18 +693,18 @@ void CTxMemPool::check(const CCoinsViewCache *pcoins) const // Check children against mapNextTx CTxMemPool::setEntries setChildrenCheck; auto iter = mapNextTx.lower_bound(COutPoint(it->GetTx().GetHash(), 0)); - int64_t childSizes = 0; + uint64_t child_sizes = 0; for (; iter != mapNextTx.end() && iter->first->hash == it->GetTx().GetHash(); ++iter) { txiter childit = mapTx.find(iter->second->GetHash()); assert(childit != mapTx.end()); // mapNextTx points to in-mempool transactions if (setChildrenCheck.insert(childit).second) { - childSizes += childit->GetTxSize(); + child_sizes += childit->GetTxSize(); } } assert(setChildrenCheck == GetMemPoolChildren(it)); // Also check to make sure size is greater than sum with immediate children. // just a sanity check, not definitive that this calc is correct... - assert(it->GetSizeWithDescendants() >= childSizes + it->GetTxSize()); + assert(it->GetSizeWithDescendants() >= child_sizes + it->GetTxSize()); if (fDependsWait) waitingOnDependants.push_back(&(*it)); @@ -1055,11 +1055,36 @@ void CTxMemPool::TrimToSize(size_t sizelimit, std::vector<COutPoint>* pvNoSpends } } -bool CTxMemPool::TransactionWithinChainLimit(const uint256& txid, size_t chainLimit) const { +uint64_t CTxMemPool::CalculateDescendantMaximum(txiter entry) const { + // find parent with highest descendant count + std::vector<txiter> candidates; + setEntries counted; + candidates.push_back(entry); + uint64_t maximum = 0; + while (candidates.size()) { + txiter candidate = candidates.back(); + candidates.pop_back(); + if (!counted.insert(candidate).second) continue; + const setEntries& parents = GetMemPoolParents(candidate); + if (parents.size() == 0) { + maximum = std::max(maximum, candidate->GetCountWithDescendants()); + } else { + for (txiter i : parents) { + candidates.push_back(i); + } + } + } + return maximum; +} + +void CTxMemPool::GetTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants) const { LOCK(cs); auto it = mapTx.find(txid); - return it == mapTx.end() || (it->GetCountWithAncestors() < chainLimit && - it->GetCountWithDescendants() < chainLimit); + ancestors = descendants = 0; + if (it != mapTx.end()) { + ancestors = it->GetCountWithAncestors(); + descendants = CalculateDescendantMaximum(it); + } } SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {} diff --git a/src/txmempool.h b/src/txmempool.h index ca7b1cd4be..bda812b42f 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -498,6 +498,7 @@ public: const setEntries & GetMemPoolParents(txiter entry) const EXCLUSIVE_LOCKS_REQUIRED(cs); const setEntries & GetMemPoolChildren(txiter entry) const EXCLUSIVE_LOCKS_REQUIRED(cs); + uint64_t CalculateDescendantMaximum(txiter entry) const EXCLUSIVE_LOCKS_REQUIRED(cs); private: typedef std::map<txiter, setEntries, CompareIteratorByHash> cacheMap; @@ -619,8 +620,11 @@ public: /** Expire all transaction (and their dependencies) in the mempool older than time. Return the number of removed transactions. */ int Expire(int64_t time); - /** Returns false if the transaction is in the mempool and not within the chain limit specified. */ - bool TransactionWithinChainLimit(const uint256& txid, size_t chainLimit) const; + /** + * Calculate the ancestor and descendant count for the given transaction. + * The counts include the transaction itself. + */ + void GetTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants) const; unsigned long size() { diff --git a/src/util.cpp b/src/util.cpp index 48d64e3eec..ab262b4063 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -585,6 +585,13 @@ void ArgsManager::AddArg(const std::string& name, const std::string& help, const assert(ret.second); // Make sure an insertion actually happened } +void ArgsManager::AddHiddenArgs(const std::vector<std::string>& names) +{ + for (const std::string& name : names) { + AddArg(name, "", false, OptionsCategory::HIDDEN); + } +} + std::string ArgsManager::GetHelpMessage() { const bool show_debug = gArgs.GetBoolArg("-help-debug", false); diff --git a/src/util.h b/src/util.h index efd8a4bd9d..8094d72d6b 100644 --- a/src/util.h +++ b/src/util.h @@ -264,6 +264,11 @@ public: void AddArg(const std::string& name, const std::string& help, const bool debug_only, const OptionsCategory& cat); /** + * Add many hidden arguments + */ + void AddHiddenArgs(const std::vector<std::string>& args); + + /** * Clear available arguments */ void ClearArgs() { m_available_args.clear(); } diff --git a/src/validation.cpp b/src/validation.cpp index ae8d3aa3a4..1027780f38 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -577,15 +577,9 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool if (tx.IsCoinBase()) return state.DoS(100, false, REJECT_INVALID, "coinbase"); - // Reject transactions with witness before segregated witness activates (override with -prematurewitness) - bool witnessEnabled = IsWitnessEnabled(chainActive.Tip(), chainparams.GetConsensus()); - if (!gArgs.GetBoolArg("-prematurewitness", false) && tx.HasWitness() && !witnessEnabled) { - return state.DoS(0, false, REJECT_NONSTANDARD, "no-witness-yet", true); - } - // Rather not work on nonstandard transactions (unless -testnet/-regtest) std::string reason; - if (fRequireStandard && !IsStandardTx(tx, reason, witnessEnabled)) + if (fRequireStandard && !IsStandardTx(tx, reason)) return state.DoS(0, false, REJECT_NONSTANDARD, reason); // Do not work on transactions that are too small. @@ -2704,6 +2698,9 @@ bool CChainState::ActivateBestChain(CValidationState &state, const CChainParams& // Block until the validation queue drains. This should largely // never happen in normal operation, however may happen during // reindex, causing memory blowup if we run too far ahead. + // Note that if a validationinterface callback ends up calling + // ActivateBestChain this may lead to a deadlock! We should + // probably have a DEBUG_LOCKORDER test for this in the future. SyncWithValidationInterfaceQueue(); } @@ -3837,7 +3834,7 @@ bool CChainState::LoadBlockIndex(const Consensus::Params& consensus_params, CBlo // Calculate nChainWork std::vector<std::pair<int, CBlockIndex*> > vSortedByHeight; vSortedByHeight.reserve(mapBlockIndex.size()); - for (const std::pair<uint256, CBlockIndex*>& item : mapBlockIndex) + for (const std::pair<const uint256, CBlockIndex*>& item : mapBlockIndex) { CBlockIndex* pindex = item.second; vSortedByHeight.push_back(std::make_pair(pindex->nHeight, pindex)); @@ -3904,7 +3901,7 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) // Check presence of blk files LogPrintf("Checking all blk files are present...\n"); std::set<int> setBlkDataFiles; - for (const std::pair<uint256, CBlockIndex*>& item : mapBlockIndex) + for (const std::pair<const uint256, CBlockIndex*>& item : mapBlockIndex) { CBlockIndex* pindex = item.second; if (pindex->nStatus & BLOCK_HAVE_DATA) { @@ -4003,7 +4000,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, reportDone = percentageDone/10; } uiInterface.ShowProgress(_("Verifying blocks..."), percentageDone, false); - if (pindex->nHeight < chainActive.Height()-nCheckDepth) + if (pindex->nHeight <= chainActive.Height()-nCheckDepth) break; if (fPruneMode && !(pindex->nStatus & BLOCK_HAVE_DATA)) { // If pruning, only go back as far as we have data. diff --git a/src/validation.h b/src/validation.h index b5ab10786a..b9c2f6c023 100644 --- a/src/validation.h +++ b/src/validation.h @@ -233,7 +233,8 @@ static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024; * Note that we guarantee that either the proof-of-work is valid on pblock, or * (and possibly also) BlockChecked will have been called. * - * Call without cs_main held. + * May not be called with cs_main held. May not be called in a + * validationinterface callback. * * @param[in] pblock The block we want to process. * @param[in] fForceProcessing Process this block even if unrequested; used for non-network block sources and whitelisted peers. @@ -245,7 +246,8 @@ bool ProcessNewBlock(const CChainParams& chainparams, const std::shared_ptr<cons /** * Process incoming block headers. * - * Call without cs_main held. + * May not be called with cs_main held. May not be called in a + * validationinterface callback. * * @param[in] block The block headers themselves * @param[out] state This may be set to an Error state if any error occurred processing them @@ -278,7 +280,12 @@ void ThreadScriptCheck(); bool IsInitialBlockDownload(); /** Retrieve a transaction (from memory pool, or from disk, if possible) */ bool GetTransaction(const uint256& hash, CTransactionRef& tx, const Consensus::Params& params, uint256& hashBlock, bool fAllowSlow = false, CBlockIndex* blockIndex = nullptr); -/** Find the best known block, and make it the tip of the block chain */ +/** + * Find the best known block, and make it the tip of the block chain + * + * May not be called with cs_main held. May not be called in a + * validationinterface callback. + */ bool ActivateBestChain(CValidationState& state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock = std::shared_ptr<const CBlock>()); CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams); @@ -445,7 +452,11 @@ inline CBlockIndex* LookupBlockIndex(const uint256& hash) /** Find the last common block between the parameter chain and a locator. */ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator); -/** Mark a block as precious and reorganize. */ +/** Mark a block as precious and reorganize. + * + * May not be called with cs_main held. May not be called in a + * validationinterface callback. + */ bool PreciousBlock(CValidationState& state, const CChainParams& params, CBlockIndex *pindex); /** Mark a block as invalid. */ @@ -497,4 +508,10 @@ bool DumpMempool(); /** Load the mempool from disk. */ bool LoadMempool(); +//! Check whether the block associated with this index entry is pruned or not. +inline bool IsBlockPruned(const CBlockIndex* pblockindex) +{ + return (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0); +} + #endif // BITCOIN_VALIDATION_H diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 456f08bc14..fd9e23cb64 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -120,7 +120,7 @@ static void WalletTxToJSON(const CWalletTx& wtx, UniValue& entry) } entry.pushKV("bip125-replaceable", rbfStatus); - for (const std::pair<std::string, std::string>& item : wtx.mapValue) + for (const std::pair<const std::string, std::string>& item : wtx.mapValue) entry.pushKV(item.first, item.second); } @@ -198,7 +198,7 @@ CTxDestination GetLabelDestination(CWallet* const pwallet, const std::string& la return dest; } -static UniValue getlabeladdress(const JSONRPCRequest& request) +static UniValue getaccountaddress(const JSONRPCRequest& request) { std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); CWallet* const pwallet = wallet.get(); @@ -207,53 +207,36 @@ static UniValue getlabeladdress(const JSONRPCRequest& request) return NullUniValue; } - if (!IsDeprecatedRPCEnabled("accounts") && request.strMethod == "getaccountaddress") { + if (!IsDeprecatedRPCEnabled("accounts")) { if (request.fHelp) { throw std::runtime_error("getaccountaddress (Deprecated, will be removed in V0.18. To use this command, start bitcoind with -deprecatedrpc=accounts)"); } throw JSONRPCError(RPC_METHOD_DEPRECATED, "getaccountaddress is deprecated and will be removed in V0.18. To use this command, start bitcoind with -deprecatedrpc=accounts."); } - if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) + if (request.fHelp || request.params.size() != 1) throw std::runtime_error( - "getlabeladdress \"label\" ( force ) \n" - "\nReturns the default receiving address for this label. This will reset to a fresh address once there's a transaction that spends to it.\n" + "getaccountaddress \"account\"\n" + "\n\nDEPRECATED. Returns the current Bitcoin address for receiving payments to this account.\n" "\nArguments:\n" - "1. \"label\" (string, required) The label for the address. It can also be set to the empty string \"\" to represent the default label.\n" - "2. \"force\" (bool, optional) Whether the label should be created if it does not yet exist. If False, the RPC will return an error if called with a label that doesn't exist.\n" - " Defaults to false (unless the getaccountaddress method alias is being called, in which case defaults to true for backwards compatibility).\n" + "1. \"account\" (string, required) The account for the address. It can also be set to the empty string \"\" to represent the default account. The account does not need to exist, it will be created and a new address created if there is no account by the given name.\n" "\nResult:\n" - "\"address\" (string) The current receiving address for the label.\n" + "\"address\" (string) The account bitcoin address\n" "\nExamples:\n" - + HelpExampleCli("getlabeladdress", "") - + HelpExampleCli("getlabeladdress", "\"\"") - + HelpExampleCli("getlabeladdress", "\"mylabel\"") - + HelpExampleRpc("getlabeladdress", "\"mylabel\"") + + HelpExampleCli("getaccountaddress", "") + + HelpExampleCli("getaccountaddress", "\"\"") + + HelpExampleCli("getaccountaddress", "\"myaccount\"") + + HelpExampleRpc("getaccountaddress", "\"myaccount\"") ); LOCK2(cs_main, pwallet->cs_wallet); - // Parse the label first so we don't generate a key if there's an error - std::string label = LabelFromValue(request.params[0]); - bool force = request.strMethod == "getaccountaddress"; - if (!request.params[1].isNull()) { - force = request.params[1].get_bool(); - } - - bool label_found = false; - for (const std::pair<CTxDestination, CAddressBookData>& item : pwallet->mapAddressBook) { - if (item.second.name == label) { - label_found = true; - break; - } - } - if (!force && !label_found) { - throw JSONRPCError(RPC_WALLET_INVALID_LABEL_NAME, std::string("No addresses with label " + label)); - } + // Parse the account first so we don't generate a key if there's an error + std::string account = LabelFromValue(request.params[0]); UniValue ret(UniValue::VSTR); - ret = EncodeDestination(GetLabelDestination(pwallet, label)); + ret = EncodeDestination(GetLabelDestination(pwallet, account)); return ret; } @@ -343,23 +326,33 @@ static UniValue setlabel(const JSONRPCRequest& request) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); } + std::string old_label = pwallet->mapAddressBook[dest].name; std::string label = LabelFromValue(request.params[1]); if (IsMine(*pwallet, dest)) { - // Detect when changing the label of an address that is the receiving address of another label: - // If so, delete the account record for it. Labels, unlike addresses, can be deleted, - // and if we wouldn't do this, the record would stick around forever. - if (pwallet->mapAddressBook.count(dest)) { - std::string old_label = pwallet->mapAddressBook[dest].name; - if (old_label != label && dest == GetLabelDestination(pwallet, old_label)) { - pwallet->DeleteLabel(old_label); - } - } pwallet->SetAddressBook(dest, label, "receive"); + if (request.strMethod == "setaccount" && old_label != label && dest == GetLabelDestination(pwallet, old_label)) { + // for setaccount, call GetLabelDestination so a new receive address is created for the old account + GetLabelDestination(pwallet, old_label, true); + } } else { pwallet->SetAddressBook(dest, label, "send"); } + // Detect when there are no addresses using this label. + // If so, delete the account record for it. Labels, unlike addresses, can be deleted, + // and if we wouldn't do this, the record would stick around forever. + bool found_address = false; + for (const std::pair<const CTxDestination, CAddressBookData>& item : pwallet->mapAddressBook) { + if (item.second.name == label) { + found_address = true; + break; + } + } + if (!found_address) { + pwallet->DeleteLabel(old_label); + } + return NullUniValue; } @@ -447,7 +440,7 @@ static UniValue getaddressesbyaccount(const JSONRPCRequest& request) // Find all addresses that have the given account UniValue ret(UniValue::VARR); - for (const std::pair<CTxDestination, CAddressBookData>& item : pwallet->mapAddressBook) { + for (const std::pair<const CTxDestination, CAddressBookData>& item : pwallet->mapAddressBook) { const CTxDestination& dest = item.first; const std::string& strName = item.second.name; if (strName == strAccount) { @@ -760,7 +753,7 @@ static UniValue getreceivedbyaddress(const JSONRPCRequest& request) // Tally CAmount nAmount = 0; - for (const std::pair<uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { + for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { const CWalletTx& wtx = pairWtx.second; if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) continue; @@ -828,7 +821,7 @@ static UniValue getreceivedbylabel(const JSONRPCRequest& request) // Tally CAmount nAmount = 0; - for (const std::pair<uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { + for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { const CWalletTx& wtx = pairWtx.second; if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) continue; @@ -1457,13 +1450,6 @@ static UniValue addwitnessaddress(const JSONRPCRequest& request) "Projects should transition to using the address_type argument of getnewaddress, or option -addresstype=[bech32|p2sh-segwit] instead.\n"); } - { - LOCK(cs_main); - if (!IsWitnessEnabled(chainActive.Tip(), Params().GetConsensus()) && !gArgs.GetBoolArg("-walletprematurewitness", false)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Segregated witness not enabled on network"); - } - } - CTxDestination dest = DecodeDestination(request.params[0].get_str()); if (!IsValidDestination(dest)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); @@ -1541,7 +1527,7 @@ static UniValue ListReceived(CWallet * const pwallet, const UniValue& params, bo // Tally std::map<CTxDestination, tallyitem> mapTally; - for (const std::pair<uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { + for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { const CWalletTx& wtx = pairWtx.second; if (wtx.IsCoinBase() || !CheckFinalTx(*wtx.tx)) @@ -2120,13 +2106,13 @@ static UniValue listaccounts(const JSONRPCRequest& request) includeWatchonly = includeWatchonly | ISMINE_WATCH_ONLY; std::map<std::string, CAmount> mapAccountBalances; - for (const std::pair<CTxDestination, CAddressBookData>& entry : pwallet->mapAddressBook) { + for (const std::pair<const CTxDestination, CAddressBookData>& entry : pwallet->mapAddressBook) { if (IsMine(*pwallet, entry.first) & includeWatchonly) { // This address belongs to me mapAccountBalances[entry.second.name] = 0; } } - for (const std::pair<uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { + for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { const CWalletTx& wtx = pairWtx.second; CAmount nFee; std::string strSentAccount; @@ -2155,7 +2141,7 @@ static UniValue listaccounts(const JSONRPCRequest& request) mapAccountBalances[entry.strAccount] += entry.nCreditDebit; UniValue ret(UniValue::VOBJ); - for (const std::pair<std::string, CAmount>& accountBalance : mapAccountBalances) { + for (const std::pair<const std::string, CAmount>& accountBalance : mapAccountBalances) { ret.pushKV(accountBalance.first, ValueFromAmount(accountBalance.second)); } return ret; @@ -2264,7 +2250,7 @@ static UniValue listsinceblock(const JSONRPCRequest& request) UniValue transactions(UniValue::VARR); - for (const std::pair<uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { + for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { CWalletTx tx = pairWtx.second; if (depth == -1 || tx.GetDepthInMainChain() < depth) { @@ -4201,7 +4187,7 @@ static UniValue getaddressesbylabel(const JSONRPCRequest& request) // Find all addresses that have the given label UniValue ret(UniValue::VOBJ); - for (const std::pair<CTxDestination, CAddressBookData>& item : pwallet->mapAddressBook) { + for (const std::pair<const CTxDestination, CAddressBookData>& item : pwallet->mapAddressBook) { if (item.second.name == label) { ret.pushKV(EncodeDestination(item.first), AddressBookDataToJSON(item.second, false)); } @@ -4254,7 +4240,7 @@ static UniValue listlabels(const JSONRPCRequest& request) // Add to a set to sort by label name, then insert into Univalue array std::set<std::string> label_set; - for (const std::pair<CTxDestination, CAddressBookData>& entry : pwallet->mapAddressBook) { + for (const std::pair<const CTxDestination, CAddressBookData>& entry : pwallet->mapAddressBook) { if (purpose.empty() || entry.second.purpose == purpose) { label_set.insert(entry.second.name); } @@ -4404,7 +4390,7 @@ static const CRPCCommand commands[] = { "wallet", "sethdseed", &sethdseed, {"newkeypool","seed"} }, /** Account functions (deprecated) */ - { "wallet", "getaccountaddress", &getlabeladdress, {"account"} }, + { "wallet", "getaccountaddress", &getaccountaddress, {"account"} }, { "wallet", "getaccount", &getaccount, {"address"} }, { "wallet", "getaddressesbyaccount", &getaddressesbyaccount, {"account"} }, { "wallet", "getreceivedbyaccount", &getreceivedbylabel, {"account","minconf"} }, @@ -4414,7 +4400,6 @@ static const CRPCCommand commands[] = { "wallet", "move", &movecmd, {"fromaccount","toaccount","amount","minconf","comment"} }, /** Label functions (to replace non-balance account functions) */ - { "wallet", "getlabeladdress", &getlabeladdress, {"label","force"} }, { "wallet", "getaddressesbylabel", &getaddressesbylabel, {"label"} }, { "wallet", "getreceivedbylabel", &getreceivedbylabel, {"label","minconf"} }, { "wallet", "listlabels", &listlabels, {"purpose"} }, diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index e90370cf06..d90be33000 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -2,14 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "wallet/wallet.h" -#include "wallet/coinselection.h" -#include "wallet/coincontrol.h" -#include "amount.h" -#include "primitives/transaction.h" -#include "random.h" -#include "test/test_bitcoin.h" -#include "wallet/test/wallet_test_fixture.h" +#include <wallet/wallet.h> +#include <wallet/coinselection.h> +#include <wallet/coincontrol.h> +#include <amount.h> +#include <primitives/transaction.h> +#include <random.h> +#include <test/test_bitcoin.h> +#include <wallet/test/wallet_test_fixture.h> #include <boost/test/unit_test.hpp> #include <random> diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index c3597aace8..f55c2262e6 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2469,8 +2469,11 @@ bool CWallet::OutputEligibleForSpending(const COutput& output, const CoinEligibi if (output.nDepth < (output.tx->IsFromMe(ISMINE_ALL) ? eligibility_filter.conf_mine : eligibility_filter.conf_theirs)) return false; - if (!mempool.TransactionWithinChainLimit(output.tx->GetHash(), eligibility_filter.max_ancestors)) + size_t ancestors, descendants; + mempool.GetTransactionAncestry(output.tx->GetHash(), ancestors, descendants); + if (ancestors > eligibility_filter.max_ancestors || descendants > eligibility_filter.max_descendants) { return false; + } return true; } @@ -2582,16 +2585,17 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm ++it; } - size_t nMaxChainLength = std::min(gArgs.GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT), gArgs.GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT)); + size_t max_ancestors = (size_t)std::max<int64_t>(1, gArgs.GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT)); + size_t max_descendants = (size_t)std::max<int64_t>(1, gArgs.GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT)); bool fRejectLongChains = gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS); bool res = nTargetValue <= nValueFromPresetInputs || SelectCoinsMinConf(nTargetValue - nValueFromPresetInputs, CoinEligibilityFilter(1, 6, 0), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || SelectCoinsMinConf(nTargetValue - nValueFromPresetInputs, CoinEligibilityFilter(1, 1, 0), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || (m_spend_zero_conf_change && SelectCoinsMinConf(nTargetValue - nValueFromPresetInputs, CoinEligibilityFilter(0, 1, 2), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(nTargetValue - nValueFromPresetInputs, CoinEligibilityFilter(0, 1, std::min((size_t)4, nMaxChainLength/3)), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(nTargetValue - nValueFromPresetInputs, CoinEligibilityFilter(0, 1, nMaxChainLength/2), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || - (m_spend_zero_conf_change && SelectCoinsMinConf(nTargetValue - nValueFromPresetInputs, CoinEligibilityFilter(0, 1, nMaxChainLength), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || + (m_spend_zero_conf_change && SelectCoinsMinConf(nTargetValue - nValueFromPresetInputs, CoinEligibilityFilter(0, 1, std::min((size_t)4, max_ancestors/3), std::min((size_t)4, max_descendants/3)), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || + (m_spend_zero_conf_change && SelectCoinsMinConf(nTargetValue - nValueFromPresetInputs, CoinEligibilityFilter(0, 1, max_ancestors/2, max_descendants/2), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || + (m_spend_zero_conf_change && SelectCoinsMinConf(nTargetValue - nValueFromPresetInputs, CoinEligibilityFilter(0, 1, max_ancestors-1, max_descendants-1), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)) || (m_spend_zero_conf_change && !fRejectLongChains && SelectCoinsMinConf(nTargetValue - nValueFromPresetInputs, CoinEligibilityFilter(0, 1, std::numeric_limits<uint64_t>::max()), vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)); // because SelectCoinsMinConf clears the setCoinsRet, we now add the possible inputs to the coinset @@ -3280,7 +3284,7 @@ bool CWallet::DelAddressBook(const CTxDestination& address) // Delete destdata tuples associated with address std::string strAddress = EncodeDestination(address); - for (const std::pair<std::string, std::string> &item : mapAddressBook[address].destdata) + for (const std::pair<const std::string, std::string> &item : mapAddressBook[address].destdata) { WalletBatch(*database).EraseDestData(strAddress, item.first); } @@ -3681,7 +3685,7 @@ std::set<CTxDestination> CWallet::GetLabelAddresses(const std::string& label) co { LOCK(cs_wallet); std::set<CTxDestination> result; - for (const std::pair<CTxDestination, CAddressBookData>& item : mapAddressBook) + for (const std::pair<const CTxDestination, CAddressBookData>& item : mapAddressBook) { const CTxDestination& address = item.first; const std::string& strName = item.second.name; diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index f1761b0fcf..1ec2a9e771 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -662,8 +662,10 @@ struct CoinEligibilityFilter const int conf_mine; const int conf_theirs; const uint64_t max_ancestors; + const uint64_t max_descendants; - CoinEligibilityFilter(int conf_mine, int conf_theirs, uint64_t max_ancestors) : conf_mine(conf_mine), conf_theirs(conf_theirs), max_ancestors(max_ancestors) {} + CoinEligibilityFilter(int conf_mine, int conf_theirs, uint64_t max_ancestors) : conf_mine(conf_mine), conf_theirs(conf_theirs), max_ancestors(max_ancestors), max_descendants(max_ancestors) {} + CoinEligibilityFilter(int conf_mine, int conf_theirs, uint64_t max_ancestors, uint64_t max_descendants) : conf_mine(conf_mine), conf_theirs(conf_theirs), max_ancestors(max_ancestors), max_descendants(max_descendants) {} }; class WalletRescanReserver; //forward declarations for ScanForWalletTransactions/RescanFromTime |