diff options
31 files changed, 482 insertions, 301 deletions
diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 1de6c4cbe9..73ab5eebb6 100644 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -8,11 +8,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan export DOCKER_NAME_TAG=ubuntu:16.04 -export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" +export PACKAGES="clang-8 llvm-8 python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" export NO_DEPENDS=1 export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=thread --disable-hardening --disable-asm CC=clang CXX=clang++" - -# xenial comes with old clang versions that can not parse the sanitizer suppressions files -# Remove unparseable lines as a hacky workaround -sed -i '/^implicit-/d' "${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan" +export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=thread --disable-hardening --disable-asm CC=clang-8 CXX=clang++-8" diff --git a/configure.ac b/configure.ac index 828c7aa7b3..546d00de1b 100644 --- a/configure.ac +++ b/configure.ac @@ -1268,9 +1268,9 @@ if test x$use_pkgconfig = xyes; then BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) fi if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench != xnonononono; then - PKG_CHECK_MODULES([EVENT], [libevent], [use_libevent=yes], [AC_MSG_ERROR(libevent not found.)]) + PKG_CHECK_MODULES([EVENT], [libevent >= 2.0.21], [use_libevent=yes], [AC_MSG_ERROR(libevent version 2.0.21 or greater not found.)]) if test x$TARGET_OS != xwindows; then - PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads],, [AC_MSG_ERROR(libevent_pthreads not found.)]) + PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.0.21],, [AC_MSG_ERROR(libevent_pthreads version 2.0.21 or greater not found.)]) fi fi diff --git a/doc/dependencies.md b/doc/dependencies.md index 51a2240107..0cb5311e8b 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -13,7 +13,7 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | FreeType | [2.7.1](https://download.savannah.gnu.org/releases/freetype) | | No | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Android only) | | GCC | | [4.8+](https://gcc.gnu.org/) (C++11 support) | | | | | HarfBuzz-NG | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | -| libevent | [2.1.11-stable](https://github.com/libevent/libevent/releases) | 2.0.22 | No | | | +| libevent | [2.1.11-stable](https://github.com/libevent/libevent/releases) | [2.0.21](https://github.com/bitcoin/bitcoin/pull/18676) | No | | | | libpng | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | librsvg | | | | | | | MiniUPnPc | [2.0.20180203](https://miniupnp.tuxfamily.org/files) | | No | | | diff --git a/src/Makefile.am b/src/Makefile.am index f15852ac66..627df97cad 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -215,6 +215,7 @@ BITCOIN_CORE_H = \ util/check.h \ util/error.h \ util/fees.h \ + util/golombrice.h \ util/spanparsing.h \ util/system.h \ util/macros.h \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 5bea1efa0d..059ffd6964 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -41,6 +41,7 @@ FUZZ_TARGETS = \ test/fuzz/flat_file_pos_deserialize \ test/fuzz/flatfile \ test/fuzz/float \ + test/fuzz/golomb_rice \ test/fuzz/hex \ test/fuzz/http_request \ test/fuzz/integer \ @@ -517,6 +518,12 @@ test_fuzz_float_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_float_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_float_SOURCES = test/fuzz/float.cpp +test_fuzz_golomb_rice_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_golomb_rice_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_golomb_rice_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_golomb_rice_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_golomb_rice_SOURCES = test/fuzz/golomb_rice.cpp + test_fuzz_hex_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_hex_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_hex_LDADD = $(FUZZ_SUITE_LD_COMMON) diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index 7aff3be6e7..5f5bed5bda 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -12,6 +12,7 @@ #include <primitives/transaction.h> #include <script/script.h> #include <streams.h> +#include <util/golombrice.h> /// SerType used to serialize parameters in GCS filter encoding. static constexpr int GCS_SER_TYPE = SER_NETWORK; @@ -23,37 +24,6 @@ static const std::map<BlockFilterType, std::string> g_filter_types = { {BlockFilterType::BASIC, "basic"}, }; -template <typename OStream> -static void GolombRiceEncode(BitStreamWriter<OStream>& bitwriter, uint8_t P, uint64_t x) -{ - // Write quotient as unary-encoded: q 1's followed by one 0. - uint64_t q = x >> P; - while (q > 0) { - int nbits = q <= 64 ? static_cast<int>(q) : 64; - bitwriter.Write(~0ULL, nbits); - q -= nbits; - } - bitwriter.Write(0, 1); - - // Write the remainder in P bits. Since the remainder is just the bottom - // P bits of x, there is no need to mask first. - bitwriter.Write(x, P); -} - -template <typename IStream> -static uint64_t GolombRiceDecode(BitStreamReader<IStream>& bitreader, uint8_t P) -{ - // Read unary-encoded quotient: q 1's followed by one 0. - uint64_t q = 0; - while (bitreader.Read(1) == 1) { - ++q; - } - - uint64_t r = bitreader.Read(P); - - return (q << P) + r; -} - // Map a value x that is uniformly distributed in the range [0, 2^64) to a // value uniformly distributed in [0, n) by returning the upper 64 bits of // x * n. @@ -809,14 +809,13 @@ public: RecursiveMutex cs_inventory; struct TxRelay { - TxRelay() { pfilter = MakeUnique<CBloomFilter>(); } mutable RecursiveMutex cs_filter; // We use fRelayTxes for two purposes - // a) it allows us to not relay tx invs before receiving the peer's version message // b) the peer may tell us in its version message that we should not relay tx invs // unless it loads a bloom filter. bool fRelayTxes GUARDED_BY(cs_filter){false}; - std::unique_ptr<CBloomFilter> pfilter PT_GUARDED_BY(cs_filter) GUARDED_BY(cs_filter); + std::unique_ptr<CBloomFilter> pfilter PT_GUARDED_BY(cs_filter) GUARDED_BY(cs_filter){nullptr}; mutable RecursiveMutex cs_tx_inventory; CRollingBloomFilter filterInventoryKnown GUARDED_BY(cs_tx_inventory){50000, 0.000001}; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index d3089c4176..96e9178b6e 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -3198,7 +3198,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRec } LOCK(pfrom->m_tx_relay->cs_filter); if (pfrom->GetLocalServices() & NODE_BLOOM) { - pfrom->m_tx_relay->pfilter.reset(new CBloomFilter()); + pfrom->m_tx_relay->pfilter = nullptr; } pfrom->m_tx_relay->fRelayTxes = true; return true; diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 6ce562ffbe..94a1e0a63d 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -152,7 +152,7 @@ void TestGUI(interfaces::Node& node) wallet->SetLastBlockProcessed(105, ::ChainActive().Tip()->GetBlockHash()); } { - WalletRescanReserver reserver(wallet.get()); + WalletRescanReserver reserver(*wallet); reserver.reserve(); CWallet::ScanResult result = wallet->ScanForWalletTransactions(Params().GetConsensus().hashGenesisBlock, 0 /* block height */, {} /* max height */, reserver, true /* fUpdate */); QCOMPARE(result.status, CWallet::ScanResult::SUCCESS); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index a27fe56f8b..827b83d67e 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1718,8 +1718,10 @@ static UniValue getblockstats(const JSONRPCRequest& request) {RPCResult::Type::NUM, "utxo_size_inc", "The increase/decrease in size for the utxo index (not discounting op_return and similar)"}, }}, RPCExamples{ - HelpExampleCli("getblockstats", "1000 '[\"minfeerate\",\"avgfeerate\"]'") - + HelpExampleRpc("getblockstats", "1000 '[\"minfeerate\",\"avgfeerate\"]'") + HelpExampleCli("getblockstats", R"('"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09"' '["minfeerate","avgfeerate"]')") + + HelpExampleCli("getblockstats", R"(1000 '["minfeerate","avgfeerate"]')") + + HelpExampleRpc("getblockstats", R"("00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09", ["minfeerate","avgfeerate"])") + + HelpExampleRpc("getblockstats", R"(1000, ["minfeerate","avgfeerate"])") }, }.Check(request); diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index d7241e1e3a..5d0f2fd703 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -153,6 +153,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "logging", 0, "include" }, { "logging", 1, "exclude" }, { "disconnectnode", 1, "nodeid" }, + { "upgradewallet", 0, "version" }, // Echo with conversion (For testing only) { "echojson", 0, "arg0" }, { "echojson", 1, "arg1" }, diff --git a/src/test/fuzz/golomb_rice.cpp b/src/test/fuzz/golomb_rice.cpp new file mode 100644 index 0000000000..3e20416116 --- /dev/null +++ b/src/test/fuzz/golomb_rice.cpp @@ -0,0 +1,112 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <blockfilter.h> +#include <serialize.h> +#include <streams.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/util.h> +#include <util/bytevectorhash.h> +#include <util/golombrice.h> + +#include <algorithm> +#include <cassert> +#include <cstdint> +#include <iosfwd> +#include <unordered_set> +#include <vector> + +namespace { +uint64_t MapIntoRange(const uint64_t x, const uint64_t n) +{ + const uint64_t x_hi = x >> 32; + const uint64_t x_lo = x & 0xFFFFFFFF; + const uint64_t n_hi = n >> 32; + const uint64_t n_lo = n & 0xFFFFFFFF; + const uint64_t ac = x_hi * n_hi; + const uint64_t ad = x_hi * n_lo; + const uint64_t bc = x_lo * n_hi; + const uint64_t bd = x_lo * n_lo; + const uint64_t mid34 = (bd >> 32) + (bc & 0xFFFFFFFF) + (ad & 0xFFFFFFFF); + const uint64_t upper64 = ac + (bc >> 32) + (ad >> 32) + (mid34 >> 32); + return upper64; +} + +uint64_t HashToRange(const std::vector<uint8_t>& element, const uint64_t f) +{ + const uint64_t hash = CSipHasher(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL) + .Write(element.data(), element.size()) + .Finalize(); + return MapIntoRange(hash, f); +} + +std::vector<uint64_t> BuildHashedSet(const std::unordered_set<std::vector<uint8_t>, ByteVectorHash>& elements, const uint64_t f) +{ + std::vector<uint64_t> hashed_elements; + hashed_elements.reserve(elements.size()); + for (const std::vector<uint8_t>& element : elements) { + hashed_elements.push_back(HashToRange(element, f)); + } + std::sort(hashed_elements.begin(), hashed_elements.end()); + return hashed_elements; +} +} // namespace + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + std::vector<uint8_t> golomb_rice_data; + std::vector<uint64_t> encoded_deltas; + { + std::unordered_set<std::vector<uint8_t>, ByteVectorHash> elements; + const int n = fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 512); + for (int i = 0; i < n; ++i) { + elements.insert(ConsumeRandomLengthByteVector(fuzzed_data_provider, 16)); + } + CVectorWriter stream(SER_NETWORK, 0, golomb_rice_data, 0); + WriteCompactSize(stream, static_cast<uint32_t>(elements.size())); + BitStreamWriter<CVectorWriter> bitwriter(stream); + if (!elements.empty()) { + uint64_t last_value = 0; + for (const uint64_t value : BuildHashedSet(elements, static_cast<uint64_t>(elements.size()) * static_cast<uint64_t>(BASIC_FILTER_M))) { + const uint64_t delta = value - last_value; + encoded_deltas.push_back(delta); + GolombRiceEncode(bitwriter, BASIC_FILTER_P, delta); + last_value = value; + } + } + bitwriter.Flush(); + } + + std::vector<uint64_t> decoded_deltas; + { + VectorReader stream{SER_NETWORK, 0, golomb_rice_data, 0}; + BitStreamReader<VectorReader> bitreader(stream); + const uint32_t n = static_cast<uint32_t>(ReadCompactSize(stream)); + for (uint32_t i = 0; i < n; ++i) { + decoded_deltas.push_back(GolombRiceDecode(bitreader, BASIC_FILTER_P)); + } + } + + assert(encoded_deltas == decoded_deltas); + + { + const std::vector<uint8_t> random_bytes = ConsumeRandomLengthByteVector(fuzzed_data_provider, 1024); + VectorReader stream{SER_NETWORK, 0, random_bytes, 0}; + uint32_t n; + try { + n = static_cast<uint32_t>(ReadCompactSize(stream)); + } catch (const std::ios_base::failure&) { + return; + } + BitStreamReader<VectorReader> bitreader(stream); + for (uint32_t i = 0; i < std::min<uint32_t>(n, 1024); ++i) { + try { + (void)GolombRiceDecode(bitreader, BASIC_FILTER_P); + } catch (const std::ios_base::failure&) { + } + } + } +} diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index a3a1d3c035..cdef7dcc3c 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -108,7 +108,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) // any script flag that is implemented as an upgraded NOP code. static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t failing_flags, bool add_to_cache) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - PrecomputedTransactionData txdata(tx); + PrecomputedTransactionData txdata; // If we add many more flags, this loop can get too expensive, but we can // rewrite in the future to randomly pick a set of flags to evaluate. for (uint32_t test_flags=0; test_flags < (1U << 16); test_flags += 1) { @@ -200,7 +200,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) LOCK(cs_main); TxValidationState state; - PrecomputedTransactionData ptd_spend_tx(spend_tx); + PrecomputedTransactionData ptd_spend_tx; BOOST_CHECK(!CheckInputScripts(CTransaction(spend_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, nullptr)); @@ -269,7 +269,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) // Make it valid, and check again invalid_with_cltv_tx.vin[0].scriptSig = CScript() << vchSig << 100; TxValidationState state; - PrecomputedTransactionData txdata(invalid_with_cltv_tx); + PrecomputedTransactionData txdata; BOOST_CHECK(CheckInputScripts(CTransaction(invalid_with_cltv_tx), state, ::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY, true, true, txdata, nullptr)); } @@ -297,7 +297,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) // Make it valid, and check again invalid_with_csv_tx.vin[0].scriptSig = CScript() << vchSig << 100; TxValidationState state; - PrecomputedTransactionData txdata(invalid_with_csv_tx); + PrecomputedTransactionData txdata; BOOST_CHECK(CheckInputScripts(CTransaction(invalid_with_csv_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_CHECKSEQUENCEVERIFY, true, true, txdata, nullptr)); } @@ -358,7 +358,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) tx.vin[1].scriptWitness.SetNull(); TxValidationState state; - PrecomputedTransactionData txdata(tx); + PrecomputedTransactionData txdata; // This transaction is now invalid under segwit, because of the second input. BOOST_CHECK(!CheckInputScripts(CTransaction(tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, nullptr)); diff --git a/src/util/golombrice.h b/src/util/golombrice.h new file mode 100644 index 0000000000..425e7f6681 --- /dev/null +++ b/src/util/golombrice.h @@ -0,0 +1,43 @@ +// Copyright (c) 2018-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_GOLOMBRICE_H +#define BITCOIN_UTIL_GOLOMBRICE_H + +#include <streams.h> + +#include <cstdint> + +template <typename OStream> +void GolombRiceEncode(BitStreamWriter<OStream>& bitwriter, uint8_t P, uint64_t x) +{ + // Write quotient as unary-encoded: q 1's followed by one 0. + uint64_t q = x >> P; + while (q > 0) { + int nbits = q <= 64 ? static_cast<int>(q) : 64; + bitwriter.Write(~0ULL, nbits); + q -= nbits; + } + bitwriter.Write(0, 1); + + // Write the remainder in P bits. Since the remainder is just the bottom + // P bits of x, there is no need to mask first. + bitwriter.Write(x, P); +} + +template <typename IStream> +uint64_t GolombRiceDecode(BitStreamReader<IStream>& bitreader, uint8_t P) +{ + // Read unary-encoded quotient: q 1's followed by one 0. + uint64_t q = 0; + while (bitreader.Read(1) == 1) { + ++q; + } + + uint64_t r = bitreader.Read(P); + + return (q << P) + r; +} + +#endif // BITCOIN_UTIL_GOLOMBRICE_H diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 37ae91f95f..8688f3df5e 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -57,7 +57,6 @@ void WalletInit::AddWalletOptions() const gArgs.AddArg("-salvagewallet", "Attempt to recover private keys from a corrupt wallet on startup", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-spendzeroconfchange", strprintf("Spend unconfirmed change when sending transactions (default: %u)", DEFAULT_SPEND_ZEROCONF_CHANGE), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-txconfirmtarget=<n>", strprintf("If paytxfee is not set, include enough fee so transactions begin confirmation on average within n blocks (default: %u)", DEFAULT_TX_CONFIRM_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); - gArgs.AddArg("-upgradewallet", "Upgrade wallet to latest format on startup", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-wallet=<path>", "Specify wallet database path. Can be specified multiple times to load multiple wallets. Path is interpreted relative to <walletdir> if it is not absolute, and will be created if it does not exist (as a directory containing a wallet.dat file and log files). For backwards compatibility this will also accept names of existing data files in <walletdir>.)", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::WALLET); gArgs.AddArg("-walletbroadcast", strprintf("Make the wallet broadcast transactions (default: %u)", DEFAULT_WALLETBROADCAST), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-walletdir=<dir>", "Specify directory to hold wallets (default: <datadir>/wallets if it exists, otherwise <datadir>)", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::WALLET); @@ -116,12 +115,6 @@ bool WalletInit::ParameterInteraction() const } } - if (is_multiwallet) { - if (gArgs.GetBoolArg("-upgradewallet", false)) { - return InitError(strprintf("%s is only allowed with a single wallet file", "-upgradewallet")); - } - } - if (gArgs.GetBoolArg("-sysperms", false)) return InitError("-sysperms is not allowed in combination with enabled wallet functionality"); diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 85a474be75..ceb7a7d287 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -130,7 +130,7 @@ UniValue importprivkey(const JSONRPCRequest& request) EnsureLegacyScriptPubKeyMan(*wallet, true); - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); bool fRescan = true; { auto locked_chain = pwallet->chain().lock(); @@ -274,7 +274,7 @@ UniValue importaddress(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled when blocks are pruned"); } - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); if (fRescan && !reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } @@ -474,7 +474,7 @@ UniValue importpubkey(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled when blocks are pruned"); } - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); if (fRescan && !reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } @@ -549,7 +549,7 @@ UniValue importwallet(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Importing wallets is disabled when blocks are pruned"); } - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); if (!reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } @@ -1365,7 +1365,7 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) } } - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); if (fRescan && !reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 703365b612..46efb18793 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -572,6 +572,52 @@ static UniValue signmessage(const JSONRPCRequest& request) return signature; } +static CAmount GetReceived(interfaces::Chain::Lock& locked_chain, const CWallet& wallet, const UniValue& params, bool by_label) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +{ + std::set<CTxDestination> address_set; + + if (by_label) { + // Get the set of addresses assigned to label + std::string label = LabelFromValue(params[0]); + address_set = wallet.GetLabelAddresses(label); + } else { + // Get the address + CTxDestination dest = DecodeDestination(params[0].get_str()); + if (!IsValidDestination(dest)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); + } + CScript script_pub_key = GetScriptForDestination(dest); + if (!wallet.IsMine(script_pub_key)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Address not found in wallet"); + } + address_set.insert(dest); + } + + // Minimum confirmations + int min_depth = 1; + if (!params[1].isNull()) + min_depth = params[1].get_int(); + + // Tally + CAmount amount = 0; + for (const std::pair<const uint256, CWalletTx>& wtx_pair : wallet.mapWallet) { + const CWalletTx& wtx = wtx_pair.second; + if (wtx.IsCoinBase() || !locked_chain.checkFinalTx(*wtx.tx) || wtx.GetDepthInMainChain() < min_depth) { + continue; + } + + for (const CTxOut& txout : wtx.tx->vout) { + CTxDestination address; + if (ExtractDestination(txout.scriptPubKey, address) && wallet.IsMine(address) && address_set.count(address)) { + amount += txout.nValue; + } + } + } + + return amount; +} + + static UniValue getreceivedbyaddress(const JSONRPCRequest& request) { std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); @@ -609,36 +655,7 @@ static UniValue getreceivedbyaddress(const JSONRPCRequest& request) auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - // Bitcoin address - CTxDestination dest = DecodeDestination(request.params[0].get_str()); - if (!IsValidDestination(dest)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); - } - CScript scriptPubKey = GetScriptForDestination(dest); - if (!pwallet->IsMine(scriptPubKey)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Address not found in wallet"); - } - - // Minimum confirmations - int nMinDepth = 1; - if (!request.params[1].isNull()) - nMinDepth = request.params[1].get_int(); - - // Tally - CAmount nAmount = 0; - for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { - const CWalletTx& wtx = pairWtx.second; - if (wtx.IsCoinBase() || !locked_chain->checkFinalTx(*wtx.tx)) { - continue; - } - - for (const CTxOut& txout : wtx.tx->vout) - if (txout.scriptPubKey == scriptPubKey) - if (wtx.GetDepthInMainChain() >= nMinDepth) - nAmount += txout.nValue; - } - - return ValueFromAmount(nAmount); + return ValueFromAmount(GetReceived(*locked_chain, *pwallet, request.params, /* by_label */ false)); } @@ -679,34 +696,7 @@ static UniValue getreceivedbylabel(const JSONRPCRequest& request) auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - // Minimum confirmations - int nMinDepth = 1; - if (!request.params[1].isNull()) - nMinDepth = request.params[1].get_int(); - - // Get the set of pub keys assigned to label - std::string label = LabelFromValue(request.params[0]); - std::set<CTxDestination> setAddress = pwallet->GetLabelAddresses(label); - - // Tally - CAmount nAmount = 0; - for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { - const CWalletTx& wtx = pairWtx.second; - if (wtx.IsCoinBase() || !locked_chain->checkFinalTx(*wtx.tx)) { - continue; - } - - for (const CTxOut& txout : wtx.tx->vout) - { - CTxDestination address; - if (ExtractDestination(txout.scriptPubKey, address) && pwallet->IsMine(address) && setAddress.count(address)) { - if (wtx.GetDepthInMainChain() >= nMinDepth) - nAmount += txout.nValue; - } - } - } - - return ValueFromAmount(nAmount); + return ValueFromAmount(GetReceived(*locked_chain, *pwallet, request.params, /* by_label */ true)); } @@ -2591,7 +2581,7 @@ static UniValue loadwallet(const JSONRPCRequest& request) RPCHelpMan{"loadwallet", "\nLoads a wallet from a wallet file or directory." "\nNote that all wallet command-line options used when starting bitcoind will be" - "\napplied to the new wallet (eg -zapwallettxes, upgradewallet, rescan, etc).\n", + "\napplied to the new wallet (eg -zapwallettxes, rescan, etc).\n", { {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The wallet directory or .dat file."}, }, @@ -3540,7 +3530,7 @@ UniValue rescanblockchain(const JSONRPCRequest& request) }, }.Check(request); - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); if (!reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } @@ -4017,7 +4007,7 @@ UniValue sethdseed(const JSONRPCRequest& request) // Do not do anything to non-HD wallets if (!pwallet->CanSupportFeature(FEATURE_HD)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Cannot set a HD seed on a non-HD wallet. Start with -upgradewallet in order to upgrade a non-HD wallet to HD"); + throw JSONRPCError(RPC_WALLET_ERROR, "Cannot set a HD seed on a non-HD wallet. Use the upgradewallet RPC in order to upgrade a non-HD wallet to HD"); } EnsureWalletIsUnlocked(pwallet); @@ -4241,6 +4231,45 @@ UniValue walletcreatefundedpsbt(const JSONRPCRequest& request) return result; } +static UniValue upgradewallet(const JSONRPCRequest& request) +{ + std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); + CWallet* const pwallet = wallet.get(); + + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { + return NullUniValue; + } + + RPCHelpMan{"upgradewallet", + "\nUpgrade the wallet. Upgrades to the latest version if no version number is specified\n" + "New keys may be generated and a new wallet backup will need to be made.", + { + {"version", RPCArg::Type::NUM, /* default */ strprintf("%d", FEATURE_LATEST), "The version number to upgrade to. Default is the latest wallet version"} + }, + RPCResults{}, + RPCExamples{ + HelpExampleCli("upgradewallet", "169900") + + HelpExampleRpc("upgradewallet", "169900") + } + }.Check(request); + + RPCTypeCheck(request.params, {UniValue::VNUM}, true); + + EnsureWalletIsUnlocked(pwallet); + + int version = 0; + if (!request.params[0].isNull()) { + version = request.params[0].get_int(); + } + + std::string error; + std::vector<std::string> warnings; + if (!pwallet->UpgradeWallet(version, error, warnings)) { + throw JSONRPCError(RPC_WALLET_ERROR, error); + } + return error; +} + UniValue abortrescan(const JSONRPCRequest& request); // in rpcdump.cpp UniValue dumpprivkey(const JSONRPCRequest& request); // in rpcdump.cpp UniValue importprivkey(const JSONRPCRequest& request); @@ -4309,6 +4338,7 @@ static const CRPCCommand commands[] = { "wallet", "signmessage", &signmessage, {"address","message"} }, { "wallet", "signrawtransactionwithwallet", &signrawtransactionwithwallet, {"hexstring","prevtxs","sighashtype"} }, { "wallet", "unloadwallet", &unloadwallet, {"wallet_name"} }, + { "wallet", "upgradewallet", &upgradewallet, {"version"} }, { "wallet", "walletcreatefundedpsbt", &walletcreatefundedpsbt, {"inputs","outputs","locktime","options","bip32derivs"} }, { "wallet", "walletlock", &walletlock, {} }, { "wallet", "walletpassphrase", &walletpassphrase, {"passphrase","timeout"} }, diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 295f6c7499..9e1c1ce0be 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -54,7 +54,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); - WalletRescanReserver reserver(&wallet); + WalletRescanReserver reserver(wallet); reserver.reserve(); CWallet::ScanResult result = wallet.ScanForWalletTransactions({} /* start_block */, 0 /* start_height */, {} /* max_height */, reserver, false /* update */); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); @@ -73,7 +73,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); - WalletRescanReserver reserver(&wallet); + WalletRescanReserver reserver(wallet); reserver.reserve(); CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); @@ -96,7 +96,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); - WalletRescanReserver reserver(&wallet); + WalletRescanReserver reserver(wallet); reserver.reserve(); CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); @@ -118,7 +118,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); - WalletRescanReserver reserver(&wallet); + WalletRescanReserver reserver(wallet); reserver.reserve(); CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); @@ -463,7 +463,7 @@ public: bool firstRun; wallet->LoadWallet(firstRun); AddKey(*wallet, coinbaseKey); - WalletRescanReserver reserver(wallet.get()); + WalletRescanReserver reserver(*wallet); reserver.reserve(); CWallet::ScanResult result = wallet->ScanForWalletTransactions(::ChainActive().Genesis()->GetBlockHash(), 0 /* start_height */, {} /* max_height */, reserver, false /* update */); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 4ba74f7f92..7a6deab1a8 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -3827,44 +3827,6 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, } } - int prev_version = walletInstance->GetVersion(); - if (gArgs.GetBoolArg("-upgradewallet", fFirstRun)) - { - int nMaxVersion = gArgs.GetArg("-upgradewallet", 0); - if (nMaxVersion == 0) // the -upgradewallet without argument case - { - walletInstance->WalletLogPrintf("Performing wallet upgrade to %i\n", FEATURE_LATEST); - nMaxVersion = FEATURE_LATEST; - walletInstance->SetMinVersion(FEATURE_LATEST); // permanently upgrade the wallet immediately - } - else - walletInstance->WalletLogPrintf("Allowing wallet upgrade up to %i\n", nMaxVersion); - if (nMaxVersion < walletInstance->GetVersion()) - { - error = _("Cannot downgrade wallet").translated; - return nullptr; - } - walletInstance->SetMaxVersion(nMaxVersion); - } - - // Upgrade to HD if explicit upgrade - if (gArgs.GetBoolArg("-upgradewallet", false)) { - LOCK(walletInstance->cs_wallet); - - // Do not upgrade versions to any version between HD_SPLIT and FEATURE_PRE_SPLIT_KEYPOOL unless already supporting HD_SPLIT - int max_version = walletInstance->GetVersion(); - if (!walletInstance->CanSupportFeature(FEATURE_HD_SPLIT) && max_version >= FEATURE_HD_SPLIT && max_version < FEATURE_PRE_SPLIT_KEYPOOL) { - error = _("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use -upgradewallet=169900 or -upgradewallet with no version specified.").translated; - return nullptr; - } - - for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { - if (!spk_man->Upgrade(prev_version, error)) { - return nullptr; - } - } - } - if (fFirstRun) { // ensure this wallet.dat can only be opened by clients supporting HD with chain split and expects no default key @@ -4061,7 +4023,7 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, } { - WalletRescanReserver reserver(walletInstance.get()); + WalletRescanReserver reserver(*walletInstance); if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(locked_chain->getBlockHash(rescan_height), rescan_height, {} /* max height */, reserver, true /* update */).status)) { error = _("Failed to rescan the wallet during initialization").translated; return nullptr; @@ -4126,6 +4088,42 @@ const CAddressBookData* CWallet::FindAddressBookEntry(const CTxDestination& dest return &address_book_it->second; } +bool CWallet::UpgradeWallet(int version, std::string& error, std::vector<std::string>& warnings) +{ + int prev_version = GetVersion(); + int nMaxVersion = version; + if (nMaxVersion == 0) // the -upgradewallet without argument case + { + WalletLogPrintf("Performing wallet upgrade to %i\n", FEATURE_LATEST); + nMaxVersion = FEATURE_LATEST; + SetMinVersion(FEATURE_LATEST); // permanently upgrade the wallet immediately + } + else + WalletLogPrintf("Allowing wallet upgrade up to %i\n", nMaxVersion); + if (nMaxVersion < GetVersion()) + { + error = _("Cannot downgrade wallet").translated; + return false; + } + SetMaxVersion(nMaxVersion); + + LOCK(cs_wallet); + + // Do not upgrade versions to any version between HD_SPLIT and FEATURE_PRE_SPLIT_KEYPOOL unless already supporting HD_SPLIT + int max_version = GetVersion(); + if (!CanSupportFeature(FEATURE_HD_SPLIT) && max_version >= FEATURE_HD_SPLIT && max_version < FEATURE_PRE_SPLIT_KEYPOOL) { + error = _("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use version 169900 or no version specified.").translated; + return false; + } + + for (auto spk_man : GetActiveScriptPubKeyMans()) { + if (!spk_man->Upgrade(prev_version, error)) { + return false; + } + } + return true; +} + void CWallet::postInitProcess() { auto locked_chain = chain().lock(); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 638c8562c9..176c483572 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -1175,6 +1175,9 @@ public: LogPrintf(("%s " + fmt).c_str(), GetDisplayName(), parameters...); }; + /** Upgrade the wallet */ + bool UpgradeWallet(int version, std::string& error, std::vector<std::string>& warnings); + //! Returns all unique ScriptPubKeyMans in m_internal_spk_managers and m_external_spk_managers std::set<ScriptPubKeyMan*> GetActiveScriptPubKeyMans() const; @@ -1241,35 +1244,35 @@ void MaybeResendWalletTxs(); class WalletRescanReserver { private: - CWallet* m_wallet; + CWallet& m_wallet; bool m_could_reserve; public: - explicit WalletRescanReserver(CWallet* w) : m_wallet(w), m_could_reserve(false) {} + explicit WalletRescanReserver(CWallet& w) : m_wallet(w), m_could_reserve(false) {} bool reserve() { assert(!m_could_reserve); - std::lock_guard<std::mutex> lock(m_wallet->mutexScanning); - if (m_wallet->fScanningWallet) { + std::lock_guard<std::mutex> lock(m_wallet.mutexScanning); + if (m_wallet.fScanningWallet) { return false; } - m_wallet->m_scanning_start = GetTimeMillis(); - m_wallet->m_scanning_progress = 0; - m_wallet->fScanningWallet = true; + m_wallet.m_scanning_start = GetTimeMillis(); + m_wallet.m_scanning_progress = 0; + m_wallet.fScanningWallet = true; m_could_reserve = true; return true; } bool isReserved() const { - return (m_could_reserve && m_wallet->fScanningWallet); + return (m_could_reserve && m_wallet.fScanningWallet); } ~WalletRescanReserver() { - std::lock_guard<std::mutex> lock(m_wallet->mutexScanning); + std::lock_guard<std::mutex> lock(m_wallet.mutexScanning); if (m_could_reserve) { - m_wallet->fScanningWallet = false; + m_wallet.fScanningWallet = false; } } }; diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 919d6474de..fdd86310c0 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -28,8 +28,8 @@ from test_framework.util import ( NODE_0 = 0 NODE_2 = 2 -WIT_V0 = 0 -WIT_V1 = 1 +P2WPKH = 0 +P2WSH = 1 def getutxo(txid): utxo = {} @@ -118,8 +118,8 @@ class SegWitTest(BitcoinTestFramework): balance_presetup = self.nodes[0].getbalance() self.pubkey = [] - p2sh_ids = [] # p2sh_ids[NODE][VER] is an array of txids that spend to a witness version VER pkscript to an address for NODE embedded in p2sh - wit_ids = [] # wit_ids[NODE][VER] is an array of txids that spend to a witness version VER pkscript to an address for NODE via bare witness + p2sh_ids = [] # p2sh_ids[NODE][TYPE] is an array of txids that spend to P2WPKH (TYPE=0) or P2WSH (TYPE=1) scripts to an address for NODE embedded in p2sh + wit_ids = [] # wit_ids[NODE][TYPE] is an array of txids that spend to P2WPKH (TYPE=0) or P2WSH (TYPE=1) scripts to an address for NODE via bare witness for i in range(3): newaddress = self.nodes[i].getnewaddress() self.pubkey.append(self.nodes[i].getaddressinfo(newaddress)["pubkey"]) @@ -152,14 +152,14 @@ class SegWitTest(BitcoinTestFramework): self.sync_blocks() self.log.info("Verify witness txs are skipped for mining before the fork") - self.skip_mine(self.nodes[2], wit_ids[NODE_2][WIT_V0][0], True) # block 424 - self.skip_mine(self.nodes[2], wit_ids[NODE_2][WIT_V1][0], True) # block 425 - self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V0][0], True) # block 426 - self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][WIT_V1][0], True) # block 427 + self.skip_mine(self.nodes[2], wit_ids[NODE_2][P2WPKH][0], True) # block 424 + self.skip_mine(self.nodes[2], wit_ids[NODE_2][P2WSH][0], True) # block 425 + self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][P2WPKH][0], True) # block 426 + self.skip_mine(self.nodes[2], p2sh_ids[NODE_2][P2WSH][0], True) # block 427 self.log.info("Verify unsigned p2sh witness txs without a redeem script are invalid") - self.fail_accept(self.nodes[2], "mandatory-script-verify-flag", p2sh_ids[NODE_2][WIT_V0][1], False) - self.fail_accept(self.nodes[2], "mandatory-script-verify-flag", p2sh_ids[NODE_2][WIT_V1][1], False) + self.fail_accept(self.nodes[2], "mandatory-script-verify-flag-failed (Operation not valid with the current stack size)", p2sh_ids[NODE_2][P2WPKH][1], sign=False) + self.fail_accept(self.nodes[2], "mandatory-script-verify-flag-failed (Operation not valid with the current stack size)", p2sh_ids[NODE_2][P2WSH][1], sign=False) self.nodes[2].generate(4) # blocks 428-431 @@ -173,13 +173,13 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Verify default node can't accept txs with missing witness") # unsigned, no scriptsig - self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", wit_ids[NODE_0][WIT_V0][0], False) - self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", wit_ids[NODE_0][WIT_V1][0], False) - self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", p2sh_ids[NODE_0][WIT_V0][0], False) - self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", p2sh_ids[NODE_0][WIT_V1][0], False) + self.fail_accept(self.nodes[0], "non-mandatory-script-verify-flag (Witness program hash mismatch)", wit_ids[NODE_0][P2WPKH][0], sign=False) + self.fail_accept(self.nodes[0], "non-mandatory-script-verify-flag (Witness program was passed an empty witness)", wit_ids[NODE_0][P2WSH][0], sign=False) + self.fail_accept(self.nodes[0], "mandatory-script-verify-flag-failed (Operation not valid with the current stack size)", p2sh_ids[NODE_0][P2WPKH][0], sign=False) + self.fail_accept(self.nodes[0], "mandatory-script-verify-flag-failed (Operation not valid with the current stack size)", p2sh_ids[NODE_0][P2WSH][0], sign=False) # unsigned with redeem script - self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", p2sh_ids[NODE_0][WIT_V0][0], False, witness_script(False, self.pubkey[0])) - self.fail_accept(self.nodes[0], "mandatory-script-verify-flag", p2sh_ids[NODE_0][WIT_V1][0], False, witness_script(True, self.pubkey[0])) + self.fail_accept(self.nodes[0], "non-mandatory-script-verify-flag (Witness program hash mismatch)", p2sh_ids[NODE_0][P2WPKH][0], sign=False, redeem_script=witness_script(False, self.pubkey[0])) + self.fail_accept(self.nodes[0], "non-mandatory-script-verify-flag (Witness program was passed an empty witness)", p2sh_ids[NODE_0][P2WSH][0], sign=False, redeem_script=witness_script(True, self.pubkey[0])) self.log.info("Verify block and transaction serialization rpcs return differing serializations depending on rpc serialization flag") assert self.nodes[2].getblock(blockhash, False) != self.nodes[0].getblock(blockhash, False) @@ -194,16 +194,16 @@ class SegWitTest(BitcoinTestFramework): assert self.nodes[0].getrawtransaction(tx_id, False, blockhash) == tx.serialize_without_witness().hex() self.log.info("Verify witness txs without witness data are invalid after the fork") - self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch)', wit_ids[NODE_2][WIT_V0][2], sign=False) - self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program was passed an empty witness)', wit_ids[NODE_2][WIT_V1][2], sign=False) - self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch)', p2sh_ids[NODE_2][WIT_V0][2], sign=False, redeem_script=witness_script(False, self.pubkey[2])) - self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program was passed an empty witness)', p2sh_ids[NODE_2][WIT_V1][2], sign=False, redeem_script=witness_script(True, self.pubkey[2])) + self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch)', wit_ids[NODE_2][P2WPKH][2], sign=False) + self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program was passed an empty witness)', wit_ids[NODE_2][P2WSH][2], sign=False) + self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch)', p2sh_ids[NODE_2][P2WPKH][2], sign=False, redeem_script=witness_script(False, self.pubkey[2])) + self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program was passed an empty witness)', p2sh_ids[NODE_2][P2WSH][2], sign=False, redeem_script=witness_script(True, self.pubkey[2])) self.log.info("Verify default node can now use witness txs") - self.success_mine(self.nodes[0], wit_ids[NODE_0][WIT_V0][0], True) # block 432 - self.success_mine(self.nodes[0], wit_ids[NODE_0][WIT_V1][0], True) # block 433 - self.success_mine(self.nodes[0], p2sh_ids[NODE_0][WIT_V0][0], True) # block 434 - self.success_mine(self.nodes[0], p2sh_ids[NODE_0][WIT_V1][0], True) # block 435 + self.success_mine(self.nodes[0], wit_ids[NODE_0][P2WPKH][0], True) # block 432 + self.success_mine(self.nodes[0], wit_ids[NODE_0][P2WSH][0], True) # block 433 + self.success_mine(self.nodes[0], p2sh_ids[NODE_0][P2WPKH][0], True) # block 434 + self.success_mine(self.nodes[0], p2sh_ids[NODE_0][P2WSH][0], True) # block 435 self.log.info("Verify sigops are counted in GBT with BIP141 rules after the fork") txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index e7ece8b9b1..2e80d7a248 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -13,7 +13,6 @@ BLOCKS = 101 BALANCE = (BLOCKS - 100) * 50 class TestBitcoinCli(BitcoinTestFramework): - def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 @@ -33,12 +32,12 @@ class TestBitcoinCli(BitcoinTestFramework): user, password = get_auth_cookie(self.nodes[0].datadir, self.chain) self.log.info("Test -stdinrpcpass option") - assert_equal(BLOCKS, self.nodes[0].cli('-rpcuser=%s' % user, '-stdinrpcpass', input=password).getblockcount()) - assert_raises_process_error(1, "Incorrect rpcuser or rpcpassword", self.nodes[0].cli('-rpcuser=%s' % user, '-stdinrpcpass', input="foo").echo) + assert_equal(BLOCKS, self.nodes[0].cli('-rpcuser={}'.format(user), '-stdinrpcpass', input=password).getblockcount()) + assert_raises_process_error(1, 'Incorrect rpcuser or rpcpassword', self.nodes[0].cli('-rpcuser={}'.format(user), '-stdinrpcpass', input='foo').echo) self.log.info("Test -stdin and -stdinrpcpass") - assert_equal(["foo", "bar"], self.nodes[0].cli('-rpcuser=%s' % user, '-stdin', '-stdinrpcpass', input=password + "\nfoo\nbar").echo()) - assert_raises_process_error(1, "Incorrect rpcuser or rpcpassword", self.nodes[0].cli('-rpcuser=%s' % user, '-stdin', '-stdinrpcpass', input="foo").echo) + assert_equal(['foo', 'bar'], self.nodes[0].cli('-rpcuser={}'.format(user), '-stdin', '-stdinrpcpass', input=password + '\nfoo\nbar').echo()) + assert_raises_process_error(1, 'Incorrect rpcuser or rpcpassword', self.nodes[0].cli('-rpcuser={}'.format(user), '-stdin', '-stdinrpcpass', input='foo').echo) self.log.info("Test connecting to a non-existing server") assert_raises_process_error(1, "Could not connect to the server", self.nodes[0].cli('-rpcport=1').echo) @@ -52,7 +51,7 @@ class TestBitcoinCli(BitcoinTestFramework): self.log.info("Test -getinfo returns expected network and blockchain info") if self.is_wallet_compiled(): self.nodes[0].encryptwallet(password) - cli_get_info = self.nodes[0].cli('-getinfo').send_cli() + cli_get_info = self.nodes[0].cli().send_cli('-getinfo') network_info = self.nodes[0].getnetworkinfo() blockchain_info = self.nodes[0].getblockchaininfo() assert_equal(cli_get_info['version'], network_info['version']) @@ -76,20 +75,17 @@ class TestBitcoinCli(BitcoinTestFramework): else: self.log.info("*** Wallet not compiled; cli getwalletinfo and -getinfo wallet tests skipped") - self.stop_node(0) - self.log.info("Test -version with node stopped") - cli_response = self.nodes[0].cli("-version").send_cli() + self.stop_node(0) + cli_response = self.nodes[0].cli().send_cli('-version') assert "{} RPC client version".format(self.config['environment']['PACKAGE_NAME']) in cli_response - self.log.info("Test -rpcwait option waits for RPC connection instead of failing") - # Start node without RPC connection. - self.nodes[0].start() - # Verify failure without -rpcwait. - assert_raises_process_error(1, "Could not connect to the server", self.nodes[0].cli('getblockcount').echo) - # Verify success using -rpcwait. - assert_equal(BLOCKS, self.nodes[0].cli('-rpcwait', 'getblockcount').send_cli()) + self.log.info("Test -rpcwait option successfully waits for RPC connection") + self.nodes[0].start() # start node without RPC connection + self.nodes[0].wait_for_cookie_credentials() # ensure cookie file is available to avoid race condition + blocks = self.nodes[0].cli('-rpcwait').send_cli('getblockcount') self.nodes[0].wait_for_rpc_connection() + assert_equal(blocks, BLOCKS) if __name__ == '__main__': diff --git a/test/functional/mempool_reorg.py b/test/functional/mempool_reorg.py index e360b550a4..2d23343fd5 100755 --- a/test/functional/mempool_reorg.py +++ b/test/functional/mempool_reorg.py @@ -20,8 +20,6 @@ class MempoolCoinbaseTest(BitcoinTestFramework): def skip_test_if_missing_module(self): self.skip_if_no_wallet() - alert_filename = None # Set by setup_network - def run_test(self): # Start with a 200 block chain assert_equal(self.nodes[0].getblockcount(), 200) @@ -76,9 +74,8 @@ class MempoolCoinbaseTest(BitcoinTestFramework): spend_101_id = self.nodes[0].sendrawtransaction(spend_101_raw) spend_102_1_id = self.nodes[0].sendrawtransaction(spend_102_1_raw) - self.sync_all(timeout=720) - assert_equal(set(self.nodes[0].getrawmempool()), {spend_101_id, spend_102_1_id, timelock_tx_id}) + self.sync_all() for node in self.nodes: node.invalidateblock(last_block[0]) @@ -91,10 +88,9 @@ class MempoolCoinbaseTest(BitcoinTestFramework): for node in self.nodes: node.invalidateblock(new_blocks[0]) - self.sync_all(timeout=720) - # mempool should be empty. assert_equal(set(self.nodes[0].getrawmempool()), set()) + self.sync_all() if __name__ == '__main__': diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index 2eeb03bfaa..a8b768c144 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -7,14 +7,18 @@ Test BIP 37 """ from test_framework.messages import ( + CInv, + MAX_BLOOM_FILTER_SIZE, + MAX_BLOOM_HASH_FUNCS, MSG_BLOCK, MSG_FILTERED_BLOCK, - msg_getdata, - msg_filterload, msg_filteradd, msg_filterclear, + msg_filterload, + msg_getdata, ) from test_framework.mininode import P2PInterface +from test_framework.script import MAX_SCRIPT_ELEMENT_SIZE from test_framework.test_framework import BitcoinTestFramework @@ -35,8 +39,9 @@ class FilterNode(P2PInterface): for i in message.inv: # inv messages can only contain TX or BLOCK, so translate BLOCK to FILTERED_BLOCK if i.type == MSG_BLOCK: - i.type = MSG_FILTERED_BLOCK - want.inv.append(i) + want.inv.append(CInv(MSG_FILTERED_BLOCK, i.hash)) + else: + want.inv.append(i) if len(want.inv): self.send_message(want) @@ -64,7 +69,13 @@ class FilterTest(BitcoinTestFramework): self.log.info('Check that too large filter is rejected') with self.nodes[0].assert_debug_log(['Misbehaving']): - filter_node.send_and_ping(msg_filterload(data=b'\xaa', nHashFuncs=51, nTweak=0, nFlags=1)) + filter_node.send_and_ping(msg_filterload(data=b'\xaa', nHashFuncs=MAX_BLOOM_HASH_FUNCS+1)) + with self.nodes[0].assert_debug_log(['Misbehaving']): + filter_node.send_and_ping(msg_filterload(data=b'\xbb'*(MAX_BLOOM_FILTER_SIZE+1))) + + self.log.info('Check that too large data element to add to the filter is rejected') + with self.nodes[0].assert_debug_log(['Misbehaving']): + filter_node.send_and_ping(msg_filteradd(data=b'\xcc'*(MAX_SCRIPT_ELEMENT_SIZE+1))) self.log.info('Add filtered P2P connection to the node') filter_node.send_and_ping(filter_node.watch_filter_init) @@ -103,6 +114,20 @@ class FilterTest(BitcoinTestFramework): txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 7) filter_node.wait_for_tx(txid) + self.log.info('Check that request for filtered blocks is ignored if no filter is set') + filter_node.merkleblock_received = False + filter_node.tx_received = False + with self.nodes[0].assert_debug_log(expected_msgs=['received getdata']): + block_hash = self.nodes[0].generatetoaddress(1, self.nodes[0].getnewaddress())[0] + filter_node.wait_for_inv([CInv(MSG_BLOCK, int(block_hash, 16))]) + filter_node.sync_with_ping() + assert not filter_node.merkleblock_received + assert not filter_node.tx_received + + self.log.info('Check that sending "filteradd" if no filter is set is treated as misbehavior') + with self.nodes[0].assert_debug_log(['Misbehaving']): + filter_node.send_and_ping(msg_filteradd(data=b'letsmisbehave')) + self.log.info("Check that division-by-zero remote crash bug [CVE-2013-5700] is fixed") filter_node.send_and_ping(msg_filterload(data=b'', nHashFuncs=1)) filter_node.send_and_ping(msg_filteradd(data=b'letstrytocrashthisnode')) diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 57c01161f5..4bd832e8f7 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -19,7 +19,7 @@ from test_framework.test_framework import BitcoinTestFramework class msg_unrecognized: """Nonsensical message. Modeled after similar types in test_framework.messages.""" - command = b'badmsg' + msgtype = b'badmsg' def __init__(self, *, str_data): self.str_data = str_data.encode() if not isinstance(str_data, bytes) else str_data @@ -28,7 +28,7 @@ class msg_unrecognized: return messages.ser_string(self.str_data) def __repr__(self): - return "{}(data={})".format(self.command, self.str_data) + return "{}(data={})".format(self.msgtype, self.str_data) class InvalidMessagesTest(BitcoinTestFramework): @@ -50,7 +50,7 @@ class InvalidMessagesTest(BitcoinTestFramework): self.test_magic_bytes() self.test_checksum() self.test_size() - self.test_command() + self.test_msgtype() self.test_large_inv() node = self.nodes[0] @@ -168,7 +168,7 @@ class InvalidMessagesTest(BitcoinTestFramework): msg = conn.build_message(msg_unrecognized(str_data="d")) cut_len = ( 4 + # magic - 12 + # command + 12 + # msgtype 4 #len ) # modify checksum @@ -183,7 +183,7 @@ class InvalidMessagesTest(BitcoinTestFramework): msg = conn.build_message(msg_unrecognized(str_data="d")) cut_len = ( 4 + # magic - 12 # command + 12 # msgtype ) # modify len to MAX_SIZE + 1 msg = msg[:cut_len] + struct.pack("<I", 0x02000000 + 1) + msg[cut_len + 4:] @@ -191,13 +191,13 @@ class InvalidMessagesTest(BitcoinTestFramework): conn.wait_for_disconnect(timeout=1) self.nodes[0].disconnect_p2ps() - def test_command(self): + def test_msgtype(self): conn = self.nodes[0].add_p2p_connection(P2PDataStore()) with self.nodes[0].assert_debug_log(['PROCESSMESSAGE: ERRORS IN HEADER']): msg = msg_unrecognized(str_data="d") - msg.command = b'\xff' * 12 + msg.msgtype = b'\xff' * 12 msg = conn.build_message(msg) - # Modify command + # Modify msgtype msg = msg[:7] + b'\x00' + msg[7 + 1:] self.nodes[0].p2p.send_raw_message(msg) conn.sync_with_ping(timeout=1) diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index c4e4324738..7f7430d04e 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -37,7 +37,7 @@ class CLazyNode(P2PInterface): def bad_message(self, message): self.unexpected_msg = True - self.log.info("should not have received message: %s" % message.command) + self.log.info("should not have received message: %s" % message.msgtype) def on_open(self): self.ever_connected = True diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 7781605788..33fba1c69a 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -37,6 +37,8 @@ MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version MAX_LOCATOR_SZ = 101 MAX_BLOCK_BASE_SIZE = 1000000 +MAX_BLOOM_FILTER_SIZE = 36000 +MAX_BLOOM_HASH_FUNCS = 50 COIN = 100000000 # 1 btc in satoshis MAX_MONEY = 21000000 * COIN @@ -946,7 +948,7 @@ class CMerkleBlock: class msg_version: __slots__ = ("addrFrom", "addrTo", "nNonce", "nRelay", "nServices", "nStartingHeight", "nTime", "nVersion", "strSubVer") - command = b"version" + msgtype = b"version" def __init__(self): self.nVersion = MY_VERSION @@ -1004,7 +1006,7 @@ class msg_version: class msg_verack: __slots__ = () - command = b"verack" + msgtype = b"verack" def __init__(self): pass @@ -1021,7 +1023,7 @@ class msg_verack: class msg_addr: __slots__ = ("addrs",) - command = b"addr" + msgtype = b"addr" def __init__(self): self.addrs = [] @@ -1038,7 +1040,7 @@ class msg_addr: class msg_inv: __slots__ = ("inv",) - command = b"inv" + msgtype = b"inv" def __init__(self, inv=None): if inv is None: @@ -1058,7 +1060,7 @@ class msg_inv: class msg_getdata: __slots__ = ("inv",) - command = b"getdata" + msgtype = b"getdata" def __init__(self, inv=None): self.inv = inv if inv is not None else [] @@ -1075,7 +1077,7 @@ class msg_getdata: class msg_getblocks: __slots__ = ("locator", "hashstop") - command = b"getblocks" + msgtype = b"getblocks" def __init__(self): self.locator = CBlockLocator() @@ -1099,7 +1101,7 @@ class msg_getblocks: class msg_tx: __slots__ = ("tx",) - command = b"tx" + msgtype = b"tx" def __init__(self, tx=CTransaction()): self.tx = tx @@ -1123,7 +1125,7 @@ class msg_no_witness_tx(msg_tx): class msg_block: __slots__ = ("block",) - command = b"block" + msgtype = b"block" def __init__(self, block=None): if block is None: @@ -1142,12 +1144,12 @@ class msg_block: # for cases where a user needs tighter control over what is sent over the wire -# note that the user must supply the name of the command, and the data +# note that the user must supply the name of the msgtype, and the data class msg_generic: - __slots__ = ("command", "data") + __slots__ = ("msgtype", "data") - def __init__(self, command, data=None): - self.command = command + def __init__(self, msgtype, data=None): + self.msgtype = msgtype self.data = data def serialize(self): @@ -1165,7 +1167,7 @@ class msg_no_witness_block(msg_block): class msg_getaddr: __slots__ = () - command = b"getaddr" + msgtype = b"getaddr" def __init__(self): pass @@ -1182,7 +1184,7 @@ class msg_getaddr: class msg_ping: __slots__ = ("nonce",) - command = b"ping" + msgtype = b"ping" def __init__(self, nonce=0): self.nonce = nonce @@ -1201,7 +1203,7 @@ class msg_ping: class msg_pong: __slots__ = ("nonce",) - command = b"pong" + msgtype = b"pong" def __init__(self, nonce=0): self.nonce = nonce @@ -1220,7 +1222,7 @@ class msg_pong: class msg_mempool: __slots__ = () - command = b"mempool" + msgtype = b"mempool" def __init__(self): pass @@ -1237,7 +1239,7 @@ class msg_mempool: class msg_notfound: __slots__ = ("vec", ) - command = b"notfound" + msgtype = b"notfound" def __init__(self, vec=None): self.vec = vec or [] @@ -1254,7 +1256,7 @@ class msg_notfound: class msg_sendheaders: __slots__ = () - command = b"sendheaders" + msgtype = b"sendheaders" def __init__(self): pass @@ -1275,7 +1277,7 @@ class msg_sendheaders: # hash_stop (hash of last desired block header, 0 to get as many as possible) class msg_getheaders: __slots__ = ("hashstop", "locator",) - command = b"getheaders" + msgtype = b"getheaders" def __init__(self): self.locator = CBlockLocator() @@ -1301,7 +1303,7 @@ class msg_getheaders: # <count> <vector of block headers> class msg_headers: __slots__ = ("headers",) - command = b"headers" + msgtype = b"headers" def __init__(self, headers=None): self.headers = headers if headers is not None else [] @@ -1322,7 +1324,7 @@ class msg_headers: class msg_merkleblock: __slots__ = ("merkleblock",) - command = b"merkleblock" + msgtype = b"merkleblock" def __init__(self, merkleblock=None): if merkleblock is None: @@ -1342,7 +1344,7 @@ class msg_merkleblock: class msg_filterload: __slots__ = ("data", "nHashFuncs", "nTweak", "nFlags") - command = b"filterload" + msgtype = b"filterload" def __init__(self, data=b'00', nHashFuncs=0, nTweak=0, nFlags=0): self.data = data @@ -1371,7 +1373,7 @@ class msg_filterload: class msg_filteradd: __slots__ = ("data") - command = b"filteradd" + msgtype = b"filteradd" def __init__(self, data): self.data = data @@ -1390,7 +1392,7 @@ class msg_filteradd: class msg_filterclear: __slots__ = () - command = b"filterclear" + msgtype = b"filterclear" def __init__(self): pass @@ -1407,7 +1409,7 @@ class msg_filterclear: class msg_feefilter: __slots__ = ("feerate",) - command = b"feefilter" + msgtype = b"feefilter" def __init__(self, feerate=0): self.feerate = feerate @@ -1426,7 +1428,7 @@ class msg_feefilter: class msg_sendcmpct: __slots__ = ("announce", "version") - command = b"sendcmpct" + msgtype = b"sendcmpct" def __init__(self): self.announce = False @@ -1448,7 +1450,7 @@ class msg_sendcmpct: class msg_cmpctblock: __slots__ = ("header_and_shortids",) - command = b"cmpctblock" + msgtype = b"cmpctblock" def __init__(self, header_and_shortids = None): self.header_and_shortids = header_and_shortids @@ -1468,7 +1470,7 @@ class msg_cmpctblock: class msg_getblocktxn: __slots__ = ("block_txn_request",) - command = b"getblocktxn" + msgtype = b"getblocktxn" def __init__(self): self.block_txn_request = None @@ -1488,7 +1490,7 @@ class msg_getblocktxn: class msg_blocktxn: __slots__ = ("block_transactions",) - command = b"blocktxn" + msgtype = b"blocktxn" def __init__(self): self.block_transactions = BlockTransactions() diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index 23d8255e14..ea078fd81c 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -180,7 +180,7 @@ class P2PConnection(asyncio.Protocol): raise ValueError("magic bytes mismatch: {} != {}".format(repr(self.magic_bytes), repr(self.recvbuf))) if len(self.recvbuf) < 4 + 12 + 4 + 4: return - command = self.recvbuf[4:4+12].split(b"\x00", 1)[0] + msgtype = self.recvbuf[4:4+12].split(b"\x00", 1)[0] msglen = struct.unpack("<i", self.recvbuf[4+12:4+12+4])[0] checksum = self.recvbuf[4+12+4:4+12+4+4] if len(self.recvbuf) < 4 + 12 + 4 + 4 + msglen: @@ -191,10 +191,10 @@ class P2PConnection(asyncio.Protocol): if checksum != h[:4]: raise ValueError("got bad checksum " + repr(self.recvbuf)) self.recvbuf = self.recvbuf[4+12+4+4+msglen:] - if command not in MESSAGEMAP: - raise ValueError("Received unknown command from %s:%d: '%s' %s" % (self.dstaddr, self.dstport, command, repr(msg))) + if msgtype not in MESSAGEMAP: + raise ValueError("Received unknown msgtype from %s:%d: '%s' %s" % (self.dstaddr, self.dstport, msgtype, repr(msg))) f = BytesIO(msg) - t = MESSAGEMAP[command]() + t = MESSAGEMAP[msgtype]() t.deserialize(f) self._log_message("receive", t) self.on_message(t) @@ -233,11 +233,11 @@ class P2PConnection(asyncio.Protocol): def build_message(self, message): """Build a serialized P2P message""" - command = message.command + msgtype = message.msgtype data = message.serialize() tmsg = self.magic_bytes - tmsg += command - tmsg += b"\x00" * (12 - len(command)) + tmsg += msgtype + tmsg += b"\x00" * (12 - len(msgtype)) tmsg += struct.pack("<I", len(data)) th = sha256(data) h = sha256(th) @@ -304,10 +304,10 @@ class P2PInterface(P2PConnection): and the most recent message of each type.""" with mininode_lock: try: - command = message.command.decode('ascii') - self.message_count[command] += 1 - self.last_message[command] = message - getattr(self, 'on_' + command)(message) + msgtype = message.msgtype.decode('ascii') + self.message_count[msgtype] += 1 + self.last_message[msgtype] = message + getattr(self, 'on_' + msgtype)(message) except: print("ERROR delivering %s (%s)" % (repr(message), sys.exc_info()[0])) raise diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index ba72d1c622..64f39b8cfe 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -26,6 +26,7 @@ from .util import ( MAX_NODES, append_config, delete_cookie_file, + get_auth_cookie, get_rpc_proxy, rpc_url, wait_until, @@ -225,9 +226,6 @@ class TestNode(): self.rpc_connected = True self.url = self.rpc.url return - except IOError as e: - if e.errno != errno.ECONNREFUSED: # Port not yet open? - raise # unknown IO error except JSONRPCException as e: # Initialization phase # -28 RPC in warmup # -342 Service unavailable, RPC server started but is shutting down due to error @@ -237,12 +235,30 @@ class TestNode(): # This might happen when the RPC server is in warmup, but shut down before the call to getblockcount # succeeds. Try again to properly raise the FailedToStartError pass - except ValueError as e: # cookie file not found and no rpcuser or rpcassword. bitcoind still starting + except OSError as e: + if e.errno != errno.ECONNREFUSED: # Port not yet open? + raise # unknown OS error + except ValueError as e: # cookie file not found and no rpcuser or rpcpassword; bitcoind is still starting if "No RPC credentials" not in str(e): raise time.sleep(1.0 / poll_per_s) self._raise_assertion_error("Unable to connect to bitcoind after {}s".format(self.rpc_timeout)) + def wait_for_cookie_credentials(self): + """Ensures auth cookie credentials can be read, e.g. for testing CLI with -rpcwait before RPC connection is up.""" + self.log.debug("Waiting for cookie credentials") + # Poll at a rate of four times per second. + poll_per_s = 4 + for _ in range(poll_per_s * self.rpc_timeout): + try: + get_auth_cookie(self.datadir, self.chain) + self.log.debug("Cookie credentials successfully retrieved") + return + except ValueError: # cookie file not found and no rpcuser or rpcpassword; bitcoind is still starting + pass # so we continue polling until RPC credentials are retrieved + time.sleep(1.0 / poll_per_s) + self._raise_assertion_error("Unable to retrieve cookie credentials after {}s".format(self.rpc_timeout)) + def generate(self, nblocks, maxtries=1000000): self.log.debug("TestNode.generate() dispatches `generate` call to `generatetoaddress`") return self.generatetoaddress(nblocks=nblocks, address=self.get_deterministic_priv_key().address, maxtries=maxtries) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 9d5ff63f34..64e1aa3bbc 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -410,7 +410,10 @@ def sync_blocks(rpc_connections, *, wait=1, timeout=60): # Check that each peer has at least one connection assert (all([len(x.getpeerinfo()) for x in rpc_connections])) time.sleep(wait) - raise AssertionError("Block sync timed out:{}".format("".join("\n {!r}".format(b) for b in best_hash))) + raise AssertionError("Block sync timed out after {}s:{}".format( + timeout, + "".join("\n {!r}".format(b) for b in best_hash), + )) def sync_mempools(rpc_connections, *, wait=1, timeout=60, flush_scheduler=True): @@ -429,11 +432,16 @@ def sync_mempools(rpc_connections, *, wait=1, timeout=60, flush_scheduler=True): # Check that each peer has at least one connection assert (all([len(x.getpeerinfo()) for x in rpc_connections])) time.sleep(wait) - raise AssertionError("Mempool sync timed out:{}".format("".join("\n {!r}".format(m) for m in pool))) + raise AssertionError("Mempool sync timed out after {}s:{}".format( + timeout, + "".join("\n {!r}".format(m) for m in pool), + )) + # Transaction/Block functions ############################# + def find_output(node, txid, amount, *, blockhash=None): """ Return index to output of txid with value amount diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index 0171b594c1..c569416292 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -126,10 +126,6 @@ class MultiWalletTest(BitcoinTestFramework): self.nodes[0].assert_start_raises_init_error(['-salvagewallet', '-wallet=w1', '-wallet=w2'], "Error: -salvagewallet is only allowed with a single wallet file") self.nodes[0].assert_start_raises_init_error(['-salvagewallet=1', '-wallet=w1', '-wallet=w2'], "Error: -salvagewallet is only allowed with a single wallet file") - self.log.info("Do not allow -upgradewallet with multiwallet") - self.nodes[0].assert_start_raises_init_error(['-upgradewallet', '-wallet=w1', '-wallet=w2'], "Error: -upgradewallet is only allowed with a single wallet file") - self.nodes[0].assert_start_raises_init_error(['-upgradewallet=1', '-wallet=w1', '-wallet=w2'], "Error: -upgradewallet is only allowed with a single wallet file") - # if wallets/ doesn't exist, datadir should be the default wallet dir wallet_dir2 = data_dir('walletdir') os.rename(wallet_dir(), wallet_dir2) @@ -333,18 +329,5 @@ class MultiWalletTest(BitcoinTestFramework): self.nodes[0].unloadwallet(wallet) self.nodes[1].loadwallet(wallet) - # Fail to load if wallet is downgraded - shutil.copytree(os.path.join(self.options.data_wallets_dir, 'high_minversion'), wallet_dir('high_minversion')) - self.restart_node(0, extra_args=['-upgradewallet={}'.format(FEATURE_LATEST)]) - assert {'name': 'high_minversion'} in self.nodes[0].listwalletdir()['wallets'] - self.log.info("Fail -upgradewallet that results in downgrade") - assert_raises_rpc_error( - -4, - 'Wallet loading failed: Error loading {}: Wallet requires newer version of {}'.format( - wallet_dir('high_minversion', 'wallet.dat'), self.config['environment']['PACKAGE_NAME']), - lambda: self.nodes[0].loadwallet(filename='high_minversion'), - ) - - if __name__ == '__main__': MultiWalletTest().main() |