diff options
Diffstat (limited to 'src')
288 files changed, 5586 insertions, 1737 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index ae95902f67..627df97cad 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -215,6 +215,7 @@ BITCOIN_CORE_H = \ util/check.h \ util/error.h \ util/fees.h \ + util/golombrice.h \ util/spanparsing.h \ util/system.h \ util/macros.h \ @@ -523,9 +524,12 @@ libbitcoin_util_a_SOURCES = \ util/strencodings.cpp \ util/string.cpp \ util/time.cpp \ - util/url.cpp \ $(BITCOIN_CORE_H) +if USE_LIBEVENT +libbitcoin_util_a_SOURCES += util/url.cpp +endif + if GLIBC_BACK_COMPAT libbitcoin_util_a_SOURCES += compat/glibc_compat.cpp AM_LDFLAGS += $(COMPAT_LDFLAGS) diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index eae8b1fcd1..766c0fca54 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -12,6 +12,7 @@ GENERATED_BENCH_FILES = $(RAW_BENCH_FILES:.raw=.raw.h) bench_bench_bitcoin_SOURCES = \ $(RAW_BENCH_FILES) \ + bench/addrman.cpp \ bench/bench_bitcoin.cpp \ bench/bench.cpp \ bench/bench.h \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index c3021743f4..06dc59cf5c 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -3,6 +3,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. FUZZ_TARGETS = \ + test/fuzz/addition_overflow \ test/fuzz/addr_info_deserialize \ test/fuzz/addrdb \ test/fuzz/address_deserialize \ @@ -27,17 +28,22 @@ FUZZ_TARGETS = \ test/fuzz/bloom_filter \ test/fuzz/bloomfilter_deserialize \ test/fuzz/chain \ + test/fuzz/checkqueue \ test/fuzz/coins_deserialize \ + test/fuzz/cuckoocache \ test/fuzz/decode_tx \ test/fuzz/descriptor_parse \ test/fuzz/diskblockindex_deserialize \ test/fuzz/eval_script \ test/fuzz/fee_rate \ test/fuzz/fee_rate_deserialize \ + test/fuzz/fees \ test/fuzz/flat_file_pos_deserialize \ test/fuzz/flatfile \ test/fuzz/float \ + test/fuzz/golomb_rice \ test/fuzz/hex \ + test/fuzz/http_request \ test/fuzz/integer \ test/fuzz/inv_deserialize \ test/fuzz/key \ @@ -58,10 +64,13 @@ FUZZ_TARGETS = \ test/fuzz/parse_numbers \ test/fuzz/parse_script \ test/fuzz/parse_univalue \ + test/fuzz/prevector \ test/fuzz/partial_merkle_tree_deserialize \ test/fuzz/partially_signed_transaction_deserialize \ test/fuzz/pow \ test/fuzz/prefilled_transaction_deserialize \ + test/fuzz/primitives_transaction \ + test/fuzz/process_messages \ test/fuzz/process_message \ test/fuzz/process_message_addr \ test/fuzz/process_message_block \ @@ -194,6 +203,7 @@ BITCOIN_TESTS =\ test/fs_tests.cpp \ test/getarg_tests.cpp \ test/hash_tests.cpp \ + test/interfaces_tests.cpp \ test/key_io_tests.cpp \ test/key_tests.cpp \ test/limitedmap_tests.cpp \ @@ -238,7 +248,9 @@ BITCOIN_TESTS =\ test/uint256_tests.cpp \ test/util_tests.cpp \ test/validation_block_tests.cpp \ + test/validation_chainstatemanager_tests.cpp \ test/validation_flush_tests.cpp \ + test/validationinterface_tests.cpp \ test/versionbits_tests.cpp if ENABLE_WALLET @@ -279,6 +291,12 @@ endif if ENABLE_FUZZ +test_fuzz_addition_overflow_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_addition_overflow_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_addition_overflow_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_addition_overflow_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_addition_overflow_SOURCES = test/fuzz/addition_overflow.cpp + test_fuzz_addr_info_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DADDR_INFO_DESERIALIZE=1 test_fuzz_addr_info_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_addr_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -423,12 +441,24 @@ test_fuzz_chain_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_chain_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_chain_SOURCES = test/fuzz/chain.cpp +test_fuzz_checkqueue_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_checkqueue_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_checkqueue_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_checkqueue_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_checkqueue_SOURCES = test/fuzz/checkqueue.cpp + test_fuzz_coins_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DCOINS_DESERIALIZE=1 test_fuzz_coins_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_coins_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_coins_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_coins_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_cuckoocache_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_cuckoocache_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_cuckoocache_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_cuckoocache_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_cuckoocache_SOURCES = test/fuzz/cuckoocache.cpp + test_fuzz_decode_tx_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_decode_tx_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_decode_tx_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -465,6 +495,12 @@ test_fuzz_fee_rate_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_fee_rate_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_fee_rate_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_fees_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_fees_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_fees_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_fees_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_fees_SOURCES = test/fuzz/fees.cpp + test_fuzz_flat_file_pos_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DFLAT_FILE_POS_DESERIALIZE=1 test_fuzz_flat_file_pos_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_flat_file_pos_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -483,12 +519,24 @@ test_fuzz_float_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_float_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_float_SOURCES = test/fuzz/float.cpp +test_fuzz_golomb_rice_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_golomb_rice_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_golomb_rice_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_golomb_rice_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_golomb_rice_SOURCES = test/fuzz/golomb_rice.cpp + test_fuzz_hex_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_hex_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_hex_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_hex_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_hex_SOURCES = test/fuzz/hex.cpp +test_fuzz_http_request_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_http_request_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_http_request_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_http_request_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_http_request_SOURCES = test/fuzz/http_request.cpp + test_fuzz_integer_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_integer_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_integer_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -609,6 +657,12 @@ test_fuzz_parse_univalue_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_parse_univalue_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_parse_univalue_SOURCES = test/fuzz/parse_univalue.cpp +test_fuzz_prevector_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_prevector_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_prevector_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_prevector_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_prevector_SOURCES = test/fuzz/prevector.cpp + test_fuzz_partial_merkle_tree_deserialize_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -DPARTIAL_MERKLE_TREE_DESERIALIZE=1 test_fuzz_partial_merkle_tree_deserialize_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_partial_merkle_tree_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -633,6 +687,18 @@ test_fuzz_prefilled_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_prefilled_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_prefilled_transaction_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_primitives_transaction_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_primitives_transaction_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_primitives_transaction_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_primitives_transaction_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_primitives_transaction_SOURCES = test/fuzz/primitives_transaction.cpp + +test_fuzz_process_messages_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_process_messages_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_process_messages_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_process_messages_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_process_messages_SOURCES = test/fuzz/process_messages.cpp + test_fuzz_process_message_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_process_message_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_message_LDADD = $(FUZZ_SUITE_LD_COMMON) diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index 505d630b7d..d7bc73defb 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -11,6 +11,7 @@ TEST_UTIL_H = \ test/util/blockfilter.h \ test/util/logging.h \ test/util/mining.h \ + test/util/net.h \ test/util/setup_common.h \ test/util/str.h \ test/util/transaction_utils.h \ @@ -22,6 +23,7 @@ libtest_util_a_SOURCES = \ test/util/blockfilter.cpp \ test/util/logging.cpp \ test/util/mining.cpp \ + test/util/net.cpp \ test/util/setup_common.cpp \ test/util/str.cpp \ test/util/transaction_utils.cpp \ diff --git a/src/addrman.cpp b/src/addrman.cpp index 2f8a3a0bd5..02fb2fd491 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2012 Pieter Wuille -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp new file mode 100644 index 0000000000..cc260df2b8 --- /dev/null +++ b/src/bench/addrman.cpp @@ -0,0 +1,140 @@ +// Copyright (c) 2020-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <addrman.h> +#include <bench/bench.h> +#include <random.h> +#include <util/time.h> + +#include <vector> + +/* A "source" is a source address from which we have received a bunch of other addresses. */ + +static constexpr size_t NUM_SOURCES = 64; +static constexpr size_t NUM_ADDRESSES_PER_SOURCE = 256; + +static std::vector<CAddress> g_sources; +static std::vector<std::vector<CAddress>> g_addresses; + +static void CreateAddresses() +{ + if (g_sources.size() > 0) { // already created + return; + } + + FastRandomContext rng(uint256(std::vector<unsigned char>(32, 123))); + + auto randAddr = [&rng]() { + in6_addr addr; + memcpy(&addr, rng.randbytes(sizeof(addr)).data(), sizeof(addr)); + + uint16_t port; + memcpy(&port, rng.randbytes(sizeof(port)).data(), sizeof(port)); + if (port == 0) { + port = 1; + } + + CAddress ret(CService(addr, port), NODE_NETWORK); + + ret.nTime = GetAdjustedTime(); + + return ret; + }; + + for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { + g_sources.emplace_back(randAddr()); + g_addresses.emplace_back(); + for (size_t addr_i = 0; addr_i < NUM_ADDRESSES_PER_SOURCE; ++addr_i) { + g_addresses[source_i].emplace_back(randAddr()); + } + } +} + +static void AddAddressesToAddrMan(CAddrMan& addrman) +{ + for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { + addrman.Add(g_addresses[source_i], g_sources[source_i]); + } +} + +static void FillAddrMan(CAddrMan& addrman) +{ + CreateAddresses(); + + AddAddressesToAddrMan(addrman); +} + +/* Benchmarks */ + +static void AddrManAdd(benchmark::State& state) +{ + CreateAddresses(); + + CAddrMan addrman; + + while (state.KeepRunning()) { + AddAddressesToAddrMan(addrman); + addrman.Clear(); + } +} + +static void AddrManSelect(benchmark::State& state) +{ + CAddrMan addrman; + + FillAddrMan(addrman); + + while (state.KeepRunning()) { + const auto& address = addrman.Select(); + assert(address.GetPort() > 0); + } +} + +static void AddrManGetAddr(benchmark::State& state) +{ + CAddrMan addrman; + + FillAddrMan(addrman); + + while (state.KeepRunning()) { + const auto& addresses = addrman.GetAddr(); + assert(addresses.size() > 0); + } +} + +static void AddrManGood(benchmark::State& state) +{ + /* Create many CAddrMan objects - one to be modified at each loop iteration. + * This is necessary because the CAddrMan::Good() method modifies the + * object, affecting the timing of subsequent calls to the same method and + * we want to do the same amount of work in every loop iteration. */ + + const uint64_t numLoops = state.m_num_iters * state.m_num_evals; + + std::vector<CAddrMan> addrmans(numLoops); + for (auto& addrman : addrmans) { + FillAddrMan(addrman); + } + + auto markSomeAsGood = [](CAddrMan& addrman) { + for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { + for (size_t addr_i = 0; addr_i < NUM_ADDRESSES_PER_SOURCE; ++addr_i) { + if (addr_i % 32 == 0) { + addrman.Good(g_addresses[source_i][addr_i]); + } + } + } + }; + + uint64_t i = 0; + while (state.KeepRunning()) { + markSomeAsGood(addrmans.at(i)); + ++i; + } +} + +BENCHMARK(AddrManAdd, 5); +BENCHMARK(AddrManSelect, 1000000); +BENCHMARK(AddrManGetAddr, 500); +BENCHMARK(AddrManGood, 2); diff --git a/src/bench/bech32.cpp b/src/bench/bech32.cpp index f2fc3999fe..2107840a3a 100644 --- a/src/bench/bech32.cpp +++ b/src/bench/bech32.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,8 +7,8 @@ #include <bech32.h> #include <util/strencodings.h> -#include <vector> #include <string> +#include <vector> static void Bech32Encode(benchmark::State& state) diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 5cf7e43f4b..7b93ef688d 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,7 +15,6 @@ #include <numeric> #include <regex> -const RegTestingSetup* g_testing_setup = nullptr; const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; void benchmark::ConsolePrinter::header() @@ -115,18 +114,7 @@ void benchmark::BenchRunner::RunAll(Printer& printer, uint64_t num_evals, double printer.header(); for (const auto& p : benchmarks()) { - RegTestingSetup test{}; - assert(g_testing_setup == nullptr); - g_testing_setup = &test; - { - LOCK(cs_main); - assert(::ChainActive().Height() == 0); - const bool witness_enabled{IsWitnessEnabled(::ChainActive().Tip(), Params().GetConsensus())}; - assert(witness_enabled); - } - if (!std::regex_match(p.first, baseMatch, reFilter)) { - g_testing_setup = nullptr; continue; } @@ -139,7 +127,6 @@ void benchmark::BenchRunner::RunAll(Printer& printer, uint64_t num_evals, double p.second.func(state); } printer.result(state); - g_testing_setup = nullptr; } printer.footer(); diff --git a/src/bench/bench.h b/src/bench/bench.h index 6b7a0f76d1..629bca9a73 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -5,18 +5,15 @@ #ifndef BITCOIN_BENCH_BENCH_H #define BITCOIN_BENCH_BENCH_H +#include <chrono> #include <functional> #include <map> #include <string> #include <vector> -#include <chrono> #include <boost/preprocessor/cat.hpp> #include <boost/preprocessor/stringize.hpp> -struct RegTestingSetup; -extern const RegTestingSetup* g_testing_setup; //!< A pointer to the current testing setup - // Simple micro-benchmarking framework; API mostly matches a subset of the Google Benchmark // framework (see https://github.com/google/benchmark) // Why not use the Google Benchmark framework? Because adding Yet Another Dependency diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 9235d5fe6a..1b75854210 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -17,39 +17,40 @@ static const char* DEFAULT_PLOT_PLOTLYURL = "https://cdn.plot.ly/plotly-latest.m static const int64_t DEFAULT_PLOT_WIDTH = 1024; static const int64_t DEFAULT_PLOT_HEIGHT = 768; -static void SetupBenchArgs() +static void SetupBenchArgs(ArgsManager& argsman) { - SetupHelpOptions(gArgs); + SetupHelpOptions(argsman); - gArgs.AddArg("-list", "List benchmarks without executing them. Can be combined with -scaling and -filter", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-evals=<n>", strprintf("Number of measurement evaluations to perform. (default: %u)", DEFAULT_BENCH_EVALUATIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-filter=<regex>", strprintf("Regular expression filter to select benchmark by name (default: %s)", DEFAULT_BENCH_FILTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-scaling=<n>", strprintf("Scaling factor for benchmark's runtime (default: %u)", DEFAULT_BENCH_SCALING), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-printer=(console|plot)", strprintf("Choose printer format. console: print data to console. plot: Print results as HTML graph (default: %s)", DEFAULT_BENCH_PRINTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-plot-plotlyurl=<uri>", strprintf("URL to use for plotly.js (default: %s)", DEFAULT_PLOT_PLOTLYURL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-plot-width=<x>", strprintf("Plot width in pixel (default: %u)", DEFAULT_PLOT_WIDTH), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - gArgs.AddArg("-plot-height=<x>", strprintf("Plot height in pixel (default: %u)", DEFAULT_PLOT_HEIGHT), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-list", "List benchmarks without executing them. Can be combined with -scaling and -filter", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-evals=<n>", strprintf("Number of measurement evaluations to perform. (default: %u)", DEFAULT_BENCH_EVALUATIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-filter=<regex>", strprintf("Regular expression filter to select benchmark by name (default: %s)", DEFAULT_BENCH_FILTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-scaling=<n>", strprintf("Scaling factor for benchmark's runtime (default: %u)", DEFAULT_BENCH_SCALING), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-printer=(console|plot)", strprintf("Choose printer format. console: print data to console. plot: Print results as HTML graph (default: %s)", DEFAULT_BENCH_PRINTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-plot-plotlyurl=<uri>", strprintf("URL to use for plotly.js (default: %s)", DEFAULT_PLOT_PLOTLYURL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-plot-width=<x>", strprintf("Plot width in pixel (default: %u)", DEFAULT_PLOT_WIDTH), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-plot-height=<x>", strprintf("Plot height in pixel (default: %u)", DEFAULT_PLOT_HEIGHT), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); } int main(int argc, char** argv) { - SetupBenchArgs(); + ArgsManager argsman; + SetupBenchArgs(argsman); std::string error; - if (!gArgs.ParseParameters(argc, argv, error)) { + if (!argsman.ParseParameters(argc, argv, error)) { tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error); return EXIT_FAILURE; } - if (HelpRequested(gArgs)) { - std::cout << gArgs.GetHelpMessage(); + if (HelpRequested(argsman)) { + std::cout << argsman.GetHelpMessage(); return EXIT_SUCCESS; } - int64_t evaluations = gArgs.GetArg("-evals", DEFAULT_BENCH_EVALUATIONS); - std::string regex_filter = gArgs.GetArg("-filter", DEFAULT_BENCH_FILTER); - std::string scaling_str = gArgs.GetArg("-scaling", DEFAULT_BENCH_SCALING); - bool is_list_only = gArgs.GetBoolArg("-list", false); + int64_t evaluations = argsman.GetArg("-evals", DEFAULT_BENCH_EVALUATIONS); + std::string regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER); + std::string scaling_str = argsman.GetArg("-scaling", DEFAULT_BENCH_SCALING); + bool is_list_only = argsman.GetBoolArg("-list", false); if (evaluations == 0) { return EXIT_SUCCESS; @@ -65,12 +66,12 @@ int main(int argc, char** argv) } std::unique_ptr<benchmark::Printer> printer = MakeUnique<benchmark::ConsolePrinter>(); - std::string printer_arg = gArgs.GetArg("-printer", DEFAULT_BENCH_PRINTER); + std::string printer_arg = argsman.GetArg("-printer", DEFAULT_BENCH_PRINTER); if ("plot" == printer_arg) { printer.reset(new benchmark::PlotlyPrinter( - gArgs.GetArg("-plot-plotlyurl", DEFAULT_PLOT_PLOTLYURL), - gArgs.GetArg("-plot-width", DEFAULT_PLOT_WIDTH), - gArgs.GetArg("-plot-height", DEFAULT_PLOT_HEIGHT))); + argsman.GetArg("-plot-plotlyurl", DEFAULT_PLOT_PLOTLYURL), + argsman.GetArg("-plot-width", DEFAULT_PLOT_WIDTH), + argsman.GetArg("-plot-height", DEFAULT_PLOT_HEIGHT))); } benchmark::BenchRunner::RunAll(*printer, evaluations, scaling_factor, regex_filter, is_list_only); diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index a113a73828..1a0084c915 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -16,6 +16,7 @@ static void AssembleBlock(benchmark::State& state) { + RegTestingSetup test_setup; const std::vector<unsigned char> op_true{OP_TRUE}; CScriptWitness witness; witness.stack.push_back(op_true); @@ -30,7 +31,7 @@ static void AssembleBlock(benchmark::State& state) std::array<CTransactionRef, NUM_BLOCKS - COINBASE_MATURITY + 1> txs; for (size_t b{0}; b < NUM_BLOCKS; ++b) { CMutableTransaction tx; - tx.vin.push_back(MineBlock(g_testing_setup->m_node, SCRIPT_PUB)); + tx.vin.push_back(MineBlock(test_setup.m_node, SCRIPT_PUB)); tx.vin.back().scriptWitness = witness; tx.vout.emplace_back(1337, SCRIPT_PUB); if (NUM_BLOCKS - b >= COINBASE_MATURITY) @@ -41,13 +42,13 @@ static void AssembleBlock(benchmark::State& state) for (const auto& txr : txs) { TxValidationState state; - bool ret{::AcceptToMemoryPool(::mempool, state, txr, nullptr /* plTxnReplaced */, false /* bypass_limits */, /* nAbsurdFee */ 0)}; + bool ret{::AcceptToMemoryPool(*test_setup.m_node.mempool, state, txr, nullptr /* plTxnReplaced */, false /* bypass_limits */, /* nAbsurdFee */ 0)}; assert(ret); } } while (state.KeepRunning()) { - PrepareBlock(g_testing_setup->m_node, SCRIPT_PUB); + PrepareBlock(test_setup.m_node, SCRIPT_PUB); } } diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index e9dd40293f..d658976c3c 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,6 +18,9 @@ // (https://github.com/bitcoin/bitcoin/issues/7883#issuecomment-224807484) static void CCoinsCaching(benchmark::State& state) { + const ECCVerifyHandle verify_handle; + ECC_Start(); + FillableSigningProvider keystore; CCoinsView coinsDummy; CCoinsViewCache coins(&coinsDummy); @@ -47,6 +50,7 @@ static void CCoinsCaching(benchmark::State& state) CAmount value = coins.GetValueIn(tx_1); assert(value == (50 + 21 + 22) * COIN); } + ECC_Stop(); } BENCHMARK(CCoinsCaching, 170 * 1000); diff --git a/src/bench/chacha_poly_aead.cpp b/src/bench/chacha_poly_aead.cpp index a02a5315a4..df10f27d03 100644 --- a/src/bench/chacha_poly_aead.cpp +++ b/src/bench/chacha_poly_aead.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,8 @@ #include <crypto/poly1305.h> // for the POLY1305_TAGLEN constant #include <hash.h> -#include <limits> #include <assert.h> +#include <limits> /* Number of bytes to process per iteration */ static constexpr uint64_t BUFFER_SIZE_TINY = 64; diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 55786126b3..2b2c78905e 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,9 +6,9 @@ #include <bench/data.h> #include <chainparams.h> -#include <validation.h> -#include <streams.h> #include <consensus/validation.h> +#include <streams.h> +#include <validation.h> // These are the two major time-sinks which happen after we have fully received // a block off the wire, but before we can relay the block on to peers using diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index f5f96a0136..e052681181 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -1,14 +1,18 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> -#include <util/system.h> #include <checkqueue.h> +#include <key.h> #include <prevector.h> -#include <vector> -#include <boost/thread/thread.hpp> +#include <pubkey.h> #include <random.h> +#include <util/system.h> + +#include <boost/thread/thread.hpp> + +#include <vector> static const int MIN_CORES = 2; @@ -22,6 +26,9 @@ static const unsigned int QUEUE_BATCH_SIZE = 128; // and there is a little bit of work done between calls to Add. static void CCheckQueueSpeedPrevectorJob(benchmark::State& state) { + const ECCVerifyHandle verify_handle; + ECC_Start(); + struct PrevectorJob { prevector<PREVECTOR_SIZE, uint8_t> p; PrevectorJob(){ @@ -57,5 +64,6 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::State& state) } tg.interrupt_all(); tg.join_all(); + ECC_Stop(); } BENCHMARK(CCheckQueueSpeedPrevectorJob, 1400); diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index a9d4d78888..ddcef5121e 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -1,17 +1,17 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> -#include <hash.h> -#include <random.h> -#include <uint256.h> #include <crypto/ripemd160.h> #include <crypto/sha1.h> #include <crypto/sha256.h> #include <crypto/sha512.h> #include <crypto/siphash.h> +#include <hash.h> +#include <random.h> +#include <uint256.h> /* Number of bytes to hash per iteration */ static const uint64_t BUFFER_SIZE = 1000*1000; diff --git a/src/bench/duplicate_inputs.cpp b/src/bench/duplicate_inputs.cpp index a783370b4e..57673ccb84 100644 --- a/src/bench/duplicate_inputs.cpp +++ b/src/bench/duplicate_inputs.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,13 +7,15 @@ #include <consensus/merkle.h> #include <consensus/validation.h> #include <pow.h> +#include <test/util/setup_common.h> #include <txmempool.h> #include <validation.h> - static void DuplicateInputs(benchmark::State& state) { + RegTestingSetup test_setup; + const CScript SCRIPT_PUB{CScript(OP_TRUE)}; const CChainParams& chainparams = Params(); diff --git a/src/bench/examples.cpp b/src/bench/examples.cpp index a2fdab5609..f88150200a 100644 --- a/src/bench/examples.cpp +++ b/src/bench/examples.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index 1c9c106666..7df024def6 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -4,6 +4,7 @@ #include <bench/bench.h> #include <policy/policy.h> +#include <test/util/setup_common.h> #include <txmempool.h> @@ -24,6 +25,8 @@ static void AddTx(const CTransactionRef& tx, const CAmount& nFee, CTxMemPool& po // unique transactions for a more meaningful performance measurement. static void MempoolEviction(benchmark::State& state) { + RegTestingSetup test_setup; + CMutableTransaction tx1 = CMutableTransaction(); tx1.vin.resize(1); tx1.vin[0].scriptSig = CScript() << OP_1; diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp index 389e2c096f..38d8632318 100644 --- a/src/bench/mempool_stress.cpp +++ b/src/bench/mempool_stress.cpp @@ -4,6 +4,7 @@ #include <bench/bench.h> #include <policy/policy.h> +#include <test/util/setup_common.h> #include <txmempool.h> #include <vector> @@ -73,6 +74,7 @@ static void ComplexMemPool(benchmark::State& state) ordered_coins.emplace_back(MakeTransactionRef(tx)); available_coins.emplace_back(ordered_coins.back(), tx_counter++); } + TestingSetup test_setup; CTxMemPool pool; LOCK2(cs_main, pool.cs); while (state.KeepRunning()) { diff --git a/src/bench/merkle_root.cpp b/src/bench/merkle_root.cpp index bdb2bdbe3d..e84f92feae 100644 --- a/src/bench/merkle_root.cpp +++ b/src/bench/merkle_root.cpp @@ -1,12 +1,12 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> -#include <uint256.h> -#include <random.h> #include <consensus/merkle.h> +#include <random.h> +#include <uint256.h> static void MerkleRoot(benchmark::State& state) { diff --git a/src/bench/rpc_blockchain.cpp b/src/bench/rpc_blockchain.cpp index 2fc6f116a4..511573abac 100644 --- a/src/bench/rpc_blockchain.cpp +++ b/src/bench/rpc_blockchain.cpp @@ -1,13 +1,13 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> #include <bench/data.h> -#include <validation.h> -#include <streams.h> #include <rpc/blockchain.h> +#include <streams.h> +#include <validation.h> #include <univalue.h> diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index 0b34ae3f95..14bca5f7d1 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,6 +18,9 @@ // modified to measure performance of other types of scripts. static void VerifyScriptBench(benchmark::State& state) { + const ECCVerifyHandle verify_handle; + ECC_Start(); + const int flags = SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH; const int witnessversion = 0; @@ -69,6 +72,7 @@ static void VerifyScriptBench(benchmark::State& state) assert(csuccess == 1); #endif } + ECC_Stop(); } static void VerifyNestedIfScript(benchmark::State& state) { diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index 0381369218..05d61fca22 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,6 +14,7 @@ static void WalletBalance(benchmark::State& state, const bool set_dirty, const bool add_watchonly, const bool add_mine) { + RegTestingSetup test_setup; const auto& ADDRESS_WATCHONLY = ADDRESS_BCRT1_UNSPENDABLE; NodeContext node; @@ -30,8 +31,8 @@ static void WalletBalance(benchmark::State& state, const bool set_dirty, const b if (add_watchonly) importaddress(wallet, ADDRESS_WATCHONLY); for (int i = 0; i < 100; ++i) { - generatetoaddress(g_testing_setup->m_node, address_mine.get_value_or(ADDRESS_WATCHONLY)); - generatetoaddress(g_testing_setup->m_node, ADDRESS_WATCHONLY); + generatetoaddress(test_setup.m_node, address_mine.get_value_or(ADDRESS_WATCHONLY)); + generatetoaddress(test_setup.m_node, ADDRESS_WATCHONLY); } SyncWithValidationInterfaceQueue(); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 6982eaab61..cdaabd6fab 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,6 +15,7 @@ #include <util/strencodings.h> #include <util/system.h> #include <util/translation.h> +#include <util/url.h> #include <functional> #include <memory> @@ -29,6 +30,7 @@ #include <compat/stdin.h> const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; +UrlDecodeFn* const URL_DECODE = urlDecode; static const char DEFAULT_RPCCONNECT[] = "127.0.0.1"; static const int DEFAULT_HTTP_CLIENT_TIMEOUT=900; @@ -228,6 +230,7 @@ public: const int ID_NETWORKINFO = 0; const int ID_BLOCKCHAININFO = 1; const int ID_WALLETINFO = 2; + const int ID_BALANCES = 3; /** Create a simulated `getinfo` request. */ UniValue PrepareRequest(const std::string& method, const std::vector<std::string>& args) override @@ -239,6 +242,7 @@ public: result.push_back(JSONRPCRequestObj("getnetworkinfo", NullUniValue, ID_NETWORKINFO)); result.push_back(JSONRPCRequestObj("getblockchaininfo", NullUniValue, ID_BLOCKCHAININFO)); result.push_back(JSONRPCRequestObj("getwalletinfo", NullUniValue, ID_WALLETINFO)); + result.push_back(JSONRPCRequestObj("getbalances", NullUniValue, ID_BALANCES)); return result; } @@ -246,9 +250,9 @@ public: UniValue ProcessReply(const UniValue &batch_in) override { UniValue result(UniValue::VOBJ); - std::vector<UniValue> batch = JSONRPCProcessBatchReply(batch_in, 3); - // Errors in getnetworkinfo() and getblockchaininfo() are fatal, pass them on - // getwalletinfo() is allowed to fail in case there is no wallet. + std::vector<UniValue> batch = JSONRPCProcessBatchReply(batch_in, batch_in.size()); + // Errors in getnetworkinfo() and getblockchaininfo() are fatal, pass them on; + // getwalletinfo() and getbalances() are allowed to fail if there is no wallet. if (!batch[ID_NETWORKINFO]["error"].isNull()) { return batch[ID_NETWORKINFO]; } @@ -265,13 +269,15 @@ public: result.pushKV("difficulty", batch[ID_BLOCKCHAININFO]["result"]["difficulty"]); result.pushKV("chain", UniValue(batch[ID_BLOCKCHAININFO]["result"]["chain"])); if (!batch[ID_WALLETINFO]["result"].isNull()) { - result.pushKV("balance", batch[ID_WALLETINFO]["result"]["balance"]); result.pushKV("keypoolsize", batch[ID_WALLETINFO]["result"]["keypoolsize"]); if (!batch[ID_WALLETINFO]["result"]["unlocked_until"].isNull()) { result.pushKV("unlocked_until", batch[ID_WALLETINFO]["result"]["unlocked_until"]); } result.pushKV("paytxfee", batch[ID_WALLETINFO]["result"]["paytxfee"]); } + if (!batch[ID_BALANCES]["result"].isNull()) { + result.pushKV("balance", batch[ID_BALANCES]["result"]["mine"]["trusted"]); + } result.pushKV("relayfee", batch[ID_NETWORKINFO]["result"]["relayfee"]); result.pushKV("warnings", batch[ID_NETWORKINFO]["result"]["warnings"]); return JSONRPCReplyObj(result, NullUniValue, 1); @@ -545,11 +551,19 @@ static int CommandLineRPC(int argc, char *argv[]) return nRet; } -int main(int argc, char* argv[]) -{ #ifdef WIN32 +// Export main() and ensure working ASLR on Windows. +// Exporting a symbol will prevent the linker from stripping +// the .reloc section from the binary, which is a requirement +// for ASLR. This is a temporary workaround until a fixed +// version of binutils is used for releases. +__declspec(dllexport) int main(int argc, char* argv[]) +{ util::WinCmdLineArgs winArgs; std::tie(argc, argv) = winArgs.get(); +#else +int main(int argc, char* argv[]) +{ #endif SetupEnvironment(); if (!SetupNetworking()) { diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index b4b2d7ed52..f54a299a36 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 7f1a4a114b..7f9439788a 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,11 +11,13 @@ #include <logging.h> #include <util/system.h> #include <util/translation.h> +#include <util/url.h> #include <wallet/wallettool.h> #include <functional> const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; +UrlDecodeFn* const URL_DECODE = nullptr; static void SetupWalletToolArgs() { diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index e284dce0d5..2aa416ca44 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -20,10 +20,12 @@ #include <util/system.h> #include <util/threadnames.h> #include <util/translation.h> +#include <util/url.h> #include <functional> const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; +UrlDecodeFn* const URL_DECODE = urlDecode; static void WaitForShutdown(NodeContext& node) { @@ -51,7 +53,7 @@ static bool AppInit(int argc, char* argv[]) // Parameters // // If Qt is used, parameters/bitcoin.conf are parsed in qt/bitcoin.cpp's main() - SetupServerArgs(); + SetupServerArgs(node); std::string error; if (!gArgs.ParseParameters(argc, argv, error)) { return InitError(strprintf("Error parsing command line arguments: %s\n", error)); diff --git a/src/blockencodings.h b/src/blockencodings.h index be50166cfc..377ac3a1a6 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index 7aff3be6e7..5f5bed5bda 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -12,6 +12,7 @@ #include <primitives/transaction.h> #include <script/script.h> #include <streams.h> +#include <util/golombrice.h> /// SerType used to serialize parameters in GCS filter encoding. static constexpr int GCS_SER_TYPE = SER_NETWORK; @@ -23,37 +24,6 @@ static const std::map<BlockFilterType, std::string> g_filter_types = { {BlockFilterType::BASIC, "basic"}, }; -template <typename OStream> -static void GolombRiceEncode(BitStreamWriter<OStream>& bitwriter, uint8_t P, uint64_t x) -{ - // Write quotient as unary-encoded: q 1's followed by one 0. - uint64_t q = x >> P; - while (q > 0) { - int nbits = q <= 64 ? static_cast<int>(q) : 64; - bitwriter.Write(~0ULL, nbits); - q -= nbits; - } - bitwriter.Write(0, 1); - - // Write the remainder in P bits. Since the remainder is just the bottom - // P bits of x, there is no need to mask first. - bitwriter.Write(x, P); -} - -template <typename IStream> -static uint64_t GolombRiceDecode(BitStreamReader<IStream>& bitreader, uint8_t P) -{ - // Read unary-encoded quotient: q 1's followed by one 0. - uint64_t q = 0; - while (bitreader.Read(1) == 1) { - ++q; - } - - uint64_t r = bitreader.Read(P); - - return (q << P) + r; -} - // Map a value x that is uniformly distributed in the range [0, 2^64) to a // value uniformly distributed in [0, n) by returning the upper 64 bits of // x * n. diff --git a/src/bloom.cpp b/src/bloom.cpp index bd6069b31f..30af507243 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -102,19 +102,6 @@ bool CBloomFilter::contains(const uint256& hash) const return contains(data); } -void CBloomFilter::clear() -{ - vData.assign(vData.size(),0); - isFull = false; - isEmpty = true; -} - -void CBloomFilter::reset(const unsigned int nNewTweak) -{ - clear(); - nTweak = nNewTweak; -} - bool CBloomFilter::IsWithinSizeConstraints() const { return vData.size() <= MAX_BLOOM_FILTER_SIZE && nHashFuncs <= MAX_HASH_FUNCS; diff --git a/src/bloom.h b/src/bloom.h index 68e76a0258..8e3b7be54d 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -84,9 +84,6 @@ public: bool contains(const COutPoint& outpoint) const; bool contains(const uint256& hash) const; - void clear(); - void reset(const unsigned int nNewTweak); - //! True if the size is <= MAX_BLOOM_FILTER_SIZE and the number of hash functions is <= MAX_HASH_FUNCS //! (catch a filter which was just deserialized which was too big) bool IsWithinSizeConstraints() const; diff --git a/src/chain.h b/src/chain.h index 64c016a1d6..802e23f775 100644 --- a/src/chain.h +++ b/src/chain.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/chainparams.cpp b/src/chainparams.cpp index f8f2d41138..799474fae2 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/chainparams.h b/src/chainparams.h index 379c75e4be..542ef329da 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/checkqueue.h b/src/checkqueue.h index 9dab5a09ac..e3faa1dec0 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2018 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/coins.cpp b/src/coins.cpp index b71362c6a0..6b4cb2aec7 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -77,8 +77,21 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi } if (!possible_overwrite) { if (!it->second.coin.IsSpent()) { - throw std::logic_error("Adding new coin that replaces non-pruned entry"); + throw std::logic_error("Attempted to overwrite an unspent coin (when possible_overwrite is false)"); } + // If the coin exists in this cache as a spent coin and is DIRTY, then + // its spentness hasn't been flushed to the parent cache. We're + // re-adding the coin to this cache now but we can't mark it as FRESH. + // If we mark it FRESH and then spend it before the cache is flushed + // we would remove it from this cache and would never flush spentness + // to the parent cache. + // + // Re-adding a spent coin can happen in the case of a re-org (the coin + // is 'spent' when the block adding it is disconnected and then + // re-added when it is also added in a newly connected block). + // + // If the coin doesn't exist in the current cache, or is spent but not + // DIRTY, then it can be marked FRESH. fresh = !(it->second.flags & CCoinsCacheEntry::DIRTY); } it->second.coin = std::move(coin); @@ -86,12 +99,12 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi cachedCoinsUsage += it->second.coin.DynamicMemoryUsage(); } -void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight, bool check) { +void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight, bool check_for_overwrite) { bool fCoinbase = tx.IsCoinBase(); const uint256& txid = tx.GetHash(); for (size_t i = 0; i < tx.vout.size(); ++i) { - bool overwrite = check ? cache.HaveCoin(COutPoint(txid, i)) : fCoinbase; - // Always set the possible_overwrite flag to AddCoin for coinbase txn, in order to correctly + bool overwrite = check_for_overwrite ? cache.HaveCoin(COutPoint(txid, i)) : fCoinbase; + // Coinbase transactions can always be overwritten, in order to correctly // deal with the pre-BIP30 occurrences of duplicate coinbase transactions. cache.AddCoin(COutPoint(txid, i), Coin(tx.vout[i], nHeight, fCoinbase), overwrite); } @@ -152,11 +165,11 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn } CCoinsMap::iterator itUs = cacheCoins.find(it->first); if (itUs == cacheCoins.end()) { - // The parent cache does not have an entry, while the child does - // We can ignore it if it's both FRESH and pruned in the child + // The parent cache does not have an entry, while the child cache does. + // We can ignore it if it's both spent and FRESH in the child if (!(it->second.flags & CCoinsCacheEntry::FRESH && it->second.coin.IsSpent())) { - // Otherwise we will need to create it in the parent - // and move the data up and mark it as dirty + // Create the coin in the parent cache, move the data up + // and mark it as dirty. CCoinsCacheEntry& entry = cacheCoins[it->first]; entry.coin = std::move(it->second.coin); cachedCoinsUsage += entry.coin.DynamicMemoryUsage(); @@ -169,19 +182,18 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn } } } else { - // Assert that the child cache entry was not marked FRESH if the - // parent cache entry has unspent outputs. If this ever happens, - // it means the FRESH flag was misapplied and there is a logic - // error in the calling code. + // Found the entry in the parent cache if ((it->second.flags & CCoinsCacheEntry::FRESH) && !itUs->second.coin.IsSpent()) { - throw std::logic_error("FRESH flag misapplied to cache entry for base transaction with spendable outputs"); + // The coin was marked FRESH in the child cache, but the coin + // exists in the parent cache. If this ever happens, it means + // the FRESH flag was misapplied and there is a logic error in + // the calling code. + throw std::logic_error("FRESH flag misapplied to coin that exists in parent cache"); } - // Found the entry in the parent cache if ((itUs->second.flags & CCoinsCacheEntry::FRESH) && it->second.coin.IsSpent()) { - // The grandparent does not have an entry, and the child is - // modified and being pruned. This means we can just delete - // it from the parent. + // The grandparent cache does not have an entry, and the coin + // has been spent. We can just delete it from the parent cache. cachedCoinsUsage -= itUs->second.coin.DynamicMemoryUsage(); cacheCoins.erase(itUs); } else { @@ -190,11 +202,10 @@ bool CCoinsViewCache::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlockIn itUs->second.coin = std::move(it->second.coin); cachedCoinsUsage += itUs->second.coin.DynamicMemoryUsage(); itUs->second.flags |= CCoinsCacheEntry::DIRTY; - // NOTE: It is possible the child has a FRESH flag here in - // the event the entry we found in the parent is pruned. But - // we must not copy that FRESH flag to the parent as that - // pruned state likely still needs to be communicated to the - // grandparent. + // NOTE: It isn't safe to mark the coin as FRESH in the parent + // cache. If it already existed and was spent in the parent + // cache then marking it FRESH would prevent that spentness + // from being flushed to the grandparent. } } } diff --git a/src/coins.h b/src/coins.h index ea2e759b54..2aed56c2bd 100644 --- a/src/coins.h +++ b/src/coins.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -109,19 +109,45 @@ public: } }; +/** + * A Coin in one level of the coins database caching hierarchy. + * + * A coin can either be: + * - unspent or spent (in which case the Coin object will be nulled out - see Coin.Clear()) + * - DIRTY or not DIRTY + * - FRESH or not FRESH + * + * Out of these 2^3 = 8 states, only some combinations are valid: + * - unspent, FRESH, DIRTY (e.g. a new coin created in the cache) + * - unspent, not FRESH, DIRTY (e.g. a coin changed in the cache during a reorg) + * - unspent, not FRESH, not DIRTY (e.g. an unspent coin fetched from the parent cache) + * - spent, FRESH, not DIRTY (e.g. a spent coin fetched from the parent cache) + * - spent, not FRESH, DIRTY (e.g. a coin is spent and spentness needs to be flushed to the parent) + */ struct CCoinsCacheEntry { Coin coin; // The actual cached data. unsigned char flags; enum Flags { - DIRTY = (1 << 0), // This cache entry is potentially different from the version in the parent view. - FRESH = (1 << 1), // The parent view does not have this entry (or it is pruned). - /* Note that FRESH is a performance optimization with which we can - * erase coins that are fully spent if we know we do not need to - * flush the changes to the parent cache. It is always safe to - * not mark FRESH if that condition is not guaranteed. + /** + * DIRTY means the CCoinsCacheEntry is potentially different from the + * version in the parent cache. Failure to mark a coin as DIRTY when + * it is potentially different from the parent cache will cause a + * consensus failure, since the coin's state won't get written to the + * parent when the cache is flushed. + */ + DIRTY = (1 << 0), + /** + * FRESH means the parent cache does not have this coin or that it is a + * spent coin in the parent cache. If a FRESH coin in the cache is + * later spent, it can be deleted entirely and doesn't ever need to be + * flushed to the parent. This is a performance optimization. Marking a + * coin as FRESH when it exists unspent in the parent cache will cause a + * consensus failure, since it might not be deleted from the parent + * when this cache is flushed. */ + FRESH = (1 << 1), }; CCoinsCacheEntry() : flags(0) {} @@ -246,7 +272,7 @@ public: bool HaveCoinInCache(const COutPoint &outpoint) const; /** - * Return a reference to Coin in the cache, or a pruned one if not found. This is + * Return a reference to Coin in the cache, or coinEmpty if not found. This is * more efficient than GetCoin. * * Generally, do not hold the reference returned for more than a short scope. @@ -258,10 +284,10 @@ public: const Coin& AccessCoin(const COutPoint &output) const; /** - * Add a coin. Set potential_overwrite to true if a non-pruned version may - * already exist. + * Add a coin. Set possible_overwrite to true if an unspent version may + * already exist in the cache. */ - void AddCoin(const COutPoint& outpoint, Coin&& coin, bool potential_overwrite); + void AddCoin(const COutPoint& outpoint, Coin&& coin, bool possible_overwrite); /** * Spend a coin. Pass moveto in order to get the deleted data. diff --git a/src/compressor.h b/src/compressor.h index 223603e7e9..478bfff0b6 100644 --- a/src/compressor.h +++ b/src/compressor.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/consensus/merkle.cpp b/src/consensus/merkle.cpp index 241cc316a6..af01902c92 100644 --- a/src/consensus/merkle.cpp +++ b/src/consensus/merkle.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/core_read.cpp b/src/core_read.cpp index d036955641..df78c319ee 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/crypto/sha256_shani.cpp b/src/crypto/sha256_shani.cpp index 92f67710fb..3473f6e39f 100644 --- a/src/crypto/sha256_shani.cpp +++ b/src/crypto/sha256_shani.cpp @@ -11,13 +11,11 @@ #include <stdint.h> #include <immintrin.h> - - namespace { -const __m128i MASK = _mm_set_epi64x(0x0c0d0e0f08090a0bULL, 0x0405060700010203ULL); -const __m128i INIT0 = _mm_set_epi64x(0x6a09e667bb67ae85ull, 0x510e527f9b05688cull); -const __m128i INIT1 = _mm_set_epi64x(0x3c6ef372a54ff53aull, 0x1f83d9ab5be0cd19ull); +alignas(__m128i) const uint8_t MASK[16] = {0x03, 0x02, 0x01, 0x00, 0x07, 0x06, 0x05, 0x04, 0x0b, 0x0a, 0x09, 0x08, 0x0f, 0x0e, 0x0d, 0x0c}; +alignas(__m128i) const uint8_t INIT0[16] = {0x8c, 0x68, 0x05, 0x9b, 0x7f, 0x52, 0x0e, 0x51, 0x85, 0xae, 0x67, 0xbb, 0x67, 0xe6, 0x09, 0x6a}; +alignas(__m128i) const uint8_t INIT1[16] = {0x19, 0xcd, 0xe0, 0x5b, 0xab, 0xd9, 0x83, 0x1f, 0x3a, 0xf5, 0x4f, 0xa5, 0x72, 0xf3, 0x6e, 0x3c}; void inline __attribute__((always_inline)) QuadRound(__m128i& state0, __m128i& state1, uint64_t k1, uint64_t k0) { @@ -67,12 +65,12 @@ void inline __attribute__((always_inline)) Unshuffle(__m128i& s0, __m128i& s1) __m128i inline __attribute__((always_inline)) Load(const unsigned char* in) { - return _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)in), MASK); + return _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)in), _mm_load_si128((const __m128i*)MASK)); } void inline __attribute__((always_inline)) Save(unsigned char* out, __m128i s) { - _mm_storeu_si128((__m128i*)out, _mm_shuffle_epi8(s, MASK)); + _mm_storeu_si128((__m128i*)out, _mm_shuffle_epi8(s, _mm_load_si128((const __m128i*)MASK))); } } @@ -149,8 +147,8 @@ void Transform_2way(unsigned char* out, const unsigned char* in) __m128i bm0, bm1, bm2, bm3, bs0, bs1, bso0, bso1; /* Transform 1 */ - bs0 = as0 = INIT0; - bs1 = as1 = INIT1; + bs0 = as0 = _mm_load_si128((const __m128i*)INIT0); + bs1 = as1 = _mm_load_si128((const __m128i*)INIT1); am0 = Load(in); bm0 = Load(in + 64); QuadRound(as0, as1, am0, 0xe9b5dba5b5c0fbcfull, 0x71374491428a2f98ull); @@ -219,10 +217,10 @@ void Transform_2way(unsigned char* out, const unsigned char* in) ShiftMessageC(bm1, bm2, bm3); QuadRound(as0, as1, am3, 0xc67178f2bef9A3f7ull, 0xa4506ceb90befffaull); QuadRound(bs0, bs1, bm3, 0xc67178f2bef9A3f7ull, 0xa4506ceb90befffaull); - as0 = _mm_add_epi32(as0, INIT0); - bs0 = _mm_add_epi32(bs0, INIT0); - as1 = _mm_add_epi32(as1, INIT1); - bs1 = _mm_add_epi32(bs1, INIT1); + as0 = _mm_add_epi32(as0, _mm_load_si128((const __m128i*)INIT0)); + bs0 = _mm_add_epi32(bs0, _mm_load_si128((const __m128i*)INIT0)); + as1 = _mm_add_epi32(as1, _mm_load_si128((const __m128i*)INIT1)); + bs1 = _mm_add_epi32(bs1, _mm_load_si128((const __m128i*)INIT1)); /* Transform 2 */ aso0 = as0; @@ -275,8 +273,8 @@ void Transform_2way(unsigned char* out, const unsigned char* in) bm1 = bs1; /* Transform 3 */ - bs0 = as0 = INIT0; - bs1 = as1 = INIT1; + bs0 = as0 = _mm_load_si128((const __m128i*)INIT0); + bs1 = as1 = _mm_load_si128((const __m128i*)INIT1); QuadRound(as0, as1, am0, 0xe9b5dba5B5c0fbcfull, 0x71374491428a2f98ull); QuadRound(bs0, bs1, bm0, 0xe9b5dba5B5c0fbcfull, 0x71374491428a2f98ull); QuadRound(as0, as1, am1, 0xab1c5ed5923f82a4ull, 0x59f111f13956c25bull); @@ -339,10 +337,10 @@ void Transform_2way(unsigned char* out, const unsigned char* in) ShiftMessageC(bm1, bm2, bm3); QuadRound(as0, as1, am3, 0xc67178f2bef9a3f7ull, 0xa4506ceb90befffaull); QuadRound(bs0, bs1, bm3, 0xc67178f2bef9a3f7ull, 0xa4506ceb90befffaull); - as0 = _mm_add_epi32(as0, INIT0); - bs0 = _mm_add_epi32(bs0, INIT0); - as1 = _mm_add_epi32(as1, INIT1); - bs1 = _mm_add_epi32(bs1, INIT1); + as0 = _mm_add_epi32(as0, _mm_load_si128((const __m128i*)INIT0)); + bs0 = _mm_add_epi32(bs0, _mm_load_si128((const __m128i*)INIT0)); + as1 = _mm_add_epi32(as1, _mm_load_si128((const __m128i*)INIT1)); + bs1 = _mm_add_epi32(bs1, _mm_load_si128((const __m128i*)INIT1)); /* Extract hash into out */ Unshuffle(as0, as1); diff --git a/src/cuckoocache.h b/src/cuckoocache.h index 4ad5818cdc..2daf676c4a 100644 --- a/src/cuckoocache.h +++ b/src/cuckoocache.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_CUCKOOCACHE_H #define BITCOIN_CUCKOOCACHE_H +#include <algorithm> // std::find #include <array> #include <atomic> #include <cmath> diff --git a/src/flatfile.h b/src/flatfile.h index d80682d383..60b3503cc3 100644 --- a/src/flatfile.h +++ b/src/flatfile.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 4d49736140..60c4d06f12 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 11d73b7c9a..176284d103 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -189,7 +189,7 @@ static bool InitHTTPAllowList() } /** HTTP request method as string - use for logging only */ -static std::string RequestMethodString(HTTPRequest::RequestMethod m) +std::string RequestMethodString(HTTPRequest::RequestMethod m) { switch (m) { case HTTPRequest::GET: @@ -510,10 +510,10 @@ void HTTPEvent::trigger(struct timeval* tv) else evtimer_add(ev, tv); // trigger after timeval passed } -HTTPRequest::HTTPRequest(struct evhttp_request* _req) : req(_req), - replySent(false) +HTTPRequest::HTTPRequest(struct evhttp_request* _req, bool _replySent) : req(_req), replySent(_replySent) { } + HTTPRequest::~HTTPRequest() { if (!replySent) { diff --git a/src/httpserver.h b/src/httpserver.h index 46820e6aee..97cd63778a 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -60,7 +60,7 @@ private: bool replySent; public: - explicit HTTPRequest(struct evhttp_request* req); + explicit HTTPRequest(struct evhttp_request* req, bool replySent = false); ~HTTPRequest(); enum RequestMethod { diff --git a/src/index/base.cpp b/src/index/base.cpp index ba71830b6e..7bff463f5b 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/index/base.h b/src/index/base.h index 95d83b9b47..3fab810bb2 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/indirectmap.h b/src/indirectmap.h index 417d500bd4..a9f4b0b9f2 100644 --- a/src/indirectmap.h +++ b/src/indirectmap.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/init.cpp b/src/init.cpp index 437e934093..a213dacbe0 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -243,13 +243,12 @@ void Shutdown(NodeContext& node) } // FlushStateToDisk generates a ChainStateFlushed callback, which we should avoid missing - // - // g_chainstate is referenced here directly (instead of ::ChainstateActive()) because it - // may not have been initialized yet. { LOCK(cs_main); - if (g_chainstate && g_chainstate->CanFlushToDisk()) { - g_chainstate->ForceFlushStateToDisk(); + for (CChainState* chainstate : g_chainman.GetAll()) { + if (chainstate->CanFlushToDisk()) { + chainstate->ForceFlushStateToDisk(); + } } } @@ -273,9 +272,11 @@ void Shutdown(NodeContext& node) { LOCK(cs_main); - if (g_chainstate && g_chainstate->CanFlushToDisk()) { - g_chainstate->ForceFlushStateToDisk(); - g_chainstate->ResetCoinsViews(); + for (CChainState* chainstate : g_chainman.GetAll()) { + if (chainstate->CanFlushToDisk()) { + chainstate->ForceFlushStateToDisk(); + chainstate->ResetCoinsViews(); + } } pblocktree.reset(); } @@ -291,20 +292,23 @@ void Shutdown(NodeContext& node) } #endif - try { - if (!fs::remove(GetPidFile())) { - LogPrintf("%s: Unable to remove PID file: File does not exist\n", __func__); - } - } catch (const fs::filesystem_error& e) { - LogPrintf("%s: Unable to remove PID file: %s\n", __func__, fsbridge::get_filesystem_error_message(e)); - } node.chain_clients.clear(); UnregisterAllValidationInterfaces(); GetMainSignals().UnregisterBackgroundSignalScheduler(); globalVerifyHandle.reset(); ECC_Stop(); + node.args = nullptr; if (node.mempool) node.mempool = nullptr; node.scheduler.reset(); + + try { + if (!fs::remove(GetPidFile())) { + LogPrintf("%s: Unable to remove PID file: File does not exist\n", __func__); + } + } catch (const fs::filesystem_error& e) { + LogPrintf("%s: Unable to remove PID file: %s\n", __func__, fsbridge::get_filesystem_error_message(e)); + } + LogPrintf("%s: done\n", __func__); } @@ -357,8 +361,11 @@ static void OnRPCStopped() LogPrint(BCLog::RPC, "RPC stopped.\n"); } -void SetupServerArgs() +void SetupServerArgs(NodeContext& node) { + assert(!node.args); + node.args = &gArgs; + SetupHelpOptions(gArgs); gArgs.AddArg("-help-debug", "Print help message with debugging options and exit", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); // server-only for now @@ -513,11 +520,16 @@ void SetupServerArgs() gArgs.AddArg("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-addrmantest", "Allows to test address relay on localhost", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-debug=<category>", "Output debugging information (default: -nodebug, supplying <category> is optional). " - "If <category> is not supplied or if <category> = 1, output all debugging information. <category> can be: " + ListLogCategories() + ".", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); + "If <category> is not supplied or if <category> = 1, output all debugging information. <category> can be: " + LogInstance().LogCategoriesString() + ".", + ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-debugexclude=<category>", strprintf("Exclude debugging information for a category. Can be used in conjunction with -debug=1 to output debug logs for all categories except one or more specified categories."), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-logips", strprintf("Include IP addresses in debug output (default: %u)", DEFAULT_LOGIPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-logtimestamps", strprintf("Prepend debug output with timestamp (default: %u)", DEFAULT_LOGTIMESTAMPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); +#ifdef HAVE_THREAD_LOCAL gArgs.AddArg("-logthreadnames", strprintf("Prepend debug output with name of the originating thread (only available on platforms supporting thread_local) (default: %u)", DEFAULT_LOGTHREADNAMES), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); +#else + hidden_args.emplace_back("-logthreadnames"); +#endif gArgs.AddArg("-logtimemicros", strprintf("Add microsecond precision to debug timestamps (default: %u)", DEFAULT_LOGTIMEMICROS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-mocktime=<n>", "Replace actual time with " + UNIX_EPOCH_TIME + " (default: 0)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-maxsigcachesize=<n>", strprintf("Limit sum of signature cache and script execution cache sizes to <n> MiB (default: %u)", DEFAULT_MAX_SIG_CACHE_SIZE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); @@ -717,11 +729,17 @@ static void ThreadImport(std::vector<fs::path> vImportFiles) } // scan for better chains in the block chain database, that are not yet connected in the active best chain - BlockValidationState state; - if (!ActivateBestChain(state, chainparams)) { - LogPrintf("Failed to connect best block (%s)\n", state.ToString()); - StartShutdown(); - return; + + // We can't hold cs_main during ActivateBestChain even though we're accessing + // the g_chainman unique_ptrs since ABC requires us not to be holding cs_main, so retrieve + // the relevant pointers before the ABC call. + for (CChainState* chainstate : WITH_LOCK(::cs_main, return g_chainman.GetAll())) { + BlockValidationState state; + if (!chainstate->ActivateBestChain(state, chainparams, nullptr)) { + LogPrintf("Failed to connect best block (%s)\n", state.ToString()); + StartShutdown(); + return; + } } if (gArgs.GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) { @@ -849,7 +867,9 @@ void InitLogging() LogInstance().m_print_to_console = gArgs.GetBoolArg("-printtoconsole", !gArgs.GetBoolArg("-daemon", false)); LogInstance().m_log_timestamps = gArgs.GetBoolArg("-logtimestamps", DEFAULT_LOGTIMESTAMPS); LogInstance().m_log_time_micros = gArgs.GetBoolArg("-logtimemicros", DEFAULT_LOGTIMEMICROS); +#ifdef HAVE_THREAD_LOCAL LogInstance().m_log_threadnames = gArgs.GetBoolArg("-logthreadnames", DEFAULT_LOGTHREADNAMES); +#endif fLogIPs = gArgs.GetBoolArg("-logips", DEFAULT_LOGIPS); @@ -1511,17 +1531,18 @@ bool AppInitMain(NodeContext& node) bool fLoaded = false; while (!fLoaded && !ShutdownRequested()) { bool fReset = fReindex; + auto is_coinsview_empty = [&](CChainState* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { + return fReset || fReindexChainState || chainstate->CoinsTip().GetBestBlock().IsNull(); + }; std::string strLoadError; uiInterface.InitMessage(_("Loading block index...").translated); do { const int64_t load_block_index_start_time = GetTimeMillis(); - bool is_coinsview_empty; try { LOCK(cs_main); - // This statement makes ::ChainstateActive() usable. - g_chainstate = MakeUnique<CChainState>(); + g_chainman.InitializeChainstate(); UnloadBlockIndex(); // new CBlockTreeDB tries to delete the existing file, which @@ -1574,43 +1595,53 @@ bool AppInitMain(NodeContext& node) // At this point we're either in reindex or we've loaded a useful // block tree into BlockIndex()! - ::ChainstateActive().InitCoinsDB( - /* cache_size_bytes */ nCoinDBCache, - /* in_memory */ false, - /* should_wipe */ fReset || fReindexChainState); - - ::ChainstateActive().CoinsErrorCatcher().AddReadErrCallback([]() { - uiInterface.ThreadSafeMessageBox( - _("Error reading from database, shutting down.").translated, - "", CClientUIInterface::MSG_ERROR); - }); - - // If necessary, upgrade from older database format. - // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate - if (!::ChainstateActive().CoinsDB().Upgrade()) { - strLoadError = _("Error upgrading chainstate database").translated; - break; - } - - // ReplayBlocks is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate - if (!::ChainstateActive().ReplayBlocks(chainparams)) { - strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate.").translated; - break; - } - - // The on-disk coinsdb is now in a good state, create the cache - ::ChainstateActive().InitCoinsCache(); - assert(::ChainstateActive().CanFlushToDisk()); + bool failed_chainstate_init = false; + + for (CChainState* chainstate : g_chainman.GetAll()) { + LogPrintf("Initializing chainstate %s\n", chainstate->ToString()); + chainstate->InitCoinsDB( + /* cache_size_bytes */ nCoinDBCache, + /* in_memory */ false, + /* should_wipe */ fReset || fReindexChainState); + + chainstate->CoinsErrorCatcher().AddReadErrCallback([]() { + uiInterface.ThreadSafeMessageBox( + _("Error reading from database, shutting down.").translated, + "", CClientUIInterface::MSG_ERROR); + }); + + // If necessary, upgrade from older database format. + // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate + if (!chainstate->CoinsDB().Upgrade()) { + strLoadError = _("Error upgrading chainstate database").translated; + failed_chainstate_init = true; + break; + } - is_coinsview_empty = fReset || fReindexChainState || - ::ChainstateActive().CoinsTip().GetBestBlock().IsNull(); - if (!is_coinsview_empty) { - // LoadChainTip initializes the chain based on CoinsTip()'s best block - if (!::ChainstateActive().LoadChainTip(chainparams)) { - strLoadError = _("Error initializing block database").translated; + // ReplayBlocks is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate + if (!chainstate->ReplayBlocks(chainparams)) { + strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate.").translated; + failed_chainstate_init = true; break; } - assert(::ChainActive().Tip() != nullptr); + + // The on-disk coinsdb is now in a good state, create the cache + chainstate->InitCoinsCache(); + assert(chainstate->CanFlushToDisk()); + + if (!is_coinsview_empty(chainstate)) { + // LoadChainTip initializes the chain based on CoinsTip()'s best block + if (!chainstate->LoadChainTip(chainparams)) { + strLoadError = _("Error initializing block database").translated; + failed_chainstate_init = true; + break; // out of the per-chainstate loop + } + assert(chainstate->m_chain.Tip() != nullptr); + } + } + + if (failed_chainstate_init) { + break; // out of the chainstate activation do-while } } catch (const std::exception& e) { LogPrintf("%s\n", e.what()); @@ -1618,49 +1649,76 @@ bool AppInitMain(NodeContext& node) break; } - if (!fReset) { - // Note that RewindBlockIndex MUST run even if we're about to -reindex-chainstate. - // It both disconnects blocks based on ::ChainActive(), and drops block data in - // BlockIndex() based on lack of available witness data. - uiInterface.InitMessage(_("Rewinding blocks...").translated); - if (!RewindBlockIndex(chainparams)) { - strLoadError = _("Unable to rewind the database to a pre-fork state. You will need to redownload the blockchain").translated; - break; + bool failed_rewind{false}; + // Can't hold cs_main while calling RewindBlockIndex, so retrieve the relevant + // chainstates beforehand. + for (CChainState* chainstate : WITH_LOCK(::cs_main, return g_chainman.GetAll())) { + if (!fReset) { + // Note that RewindBlockIndex MUST run even if we're about to -reindex-chainstate. + // It both disconnects blocks based on the chainstate, and drops block data in + // BlockIndex() based on lack of available witness data. + uiInterface.InitMessage(_("Rewinding blocks...").translated); + if (!chainstate->RewindBlockIndex(chainparams)) { + strLoadError = _( + "Unable to rewind the database to a pre-fork state. " + "You will need to redownload the blockchain").translated; + failed_rewind = true; + break; // out of the per-chainstate loop + } } } + if (failed_rewind) { + break; // out of the chainstate activation do-while + } + + bool failed_verification = false; + try { LOCK(cs_main); - if (!is_coinsview_empty) { - uiInterface.InitMessage(_("Verifying blocks...").translated); - if (fHavePruned && gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { - LogPrintf("Prune: pruned datadir may not have more than %d blocks; only checking available blocks\n", - MIN_BLOCKS_TO_KEEP); - } - CBlockIndex* tip = ::ChainActive().Tip(); - RPCNotifyBlockChange(true, tip); - if (tip && tip->nTime > GetAdjustedTime() + 2 * 60 * 60) { - strLoadError = _("The block database contains a block which appears to be from the future. " - "This may be due to your computer's date and time being set incorrectly. " - "Only rebuild the block database if you are sure that your computer's date and time are correct").translated; - break; - } - - if (!CVerifyDB().VerifyDB(chainparams, &::ChainstateActive().CoinsDB(), gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL), - gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { - strLoadError = _("Corrupted block database detected").translated; - break; + for (CChainState* chainstate : g_chainman.GetAll()) { + if (!is_coinsview_empty(chainstate)) { + uiInterface.InitMessage(_("Verifying blocks...").translated); + if (fHavePruned && gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { + LogPrintf("Prune: pruned datadir may not have more than %d blocks; only checking available blocks\n", + MIN_BLOCKS_TO_KEEP); + } + + const CBlockIndex* tip = chainstate->m_chain.Tip(); + RPCNotifyBlockChange(true, tip); + if (tip && tip->nTime > GetAdjustedTime() + 2 * 60 * 60) { + strLoadError = _("The block database contains a block which appears to be from the future. " + "This may be due to your computer's date and time being set incorrectly. " + "Only rebuild the block database if you are sure that your computer's date and time are correct").translated; + failed_verification = true; + break; + } + + // Only verify the DB of the active chainstate. This is fixed in later + // work when we allow VerifyDB to be parameterized by chainstate. + if (&::ChainstateActive() == chainstate && + !CVerifyDB().VerifyDB( + chainparams, &chainstate->CoinsDB(), + gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL), + gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { + strLoadError = _("Corrupted block database detected").translated; + failed_verification = true; + break; + } } } } catch (const std::exception& e) { LogPrintf("%s\n", e.what()); strLoadError = _("Error opening block database").translated; + failed_verification = true; break; } - fLoaded = true; - LogPrintf(" block index %15dms\n", GetTimeMillis() - load_block_index_start_time); + if (!failed_verification) { + fLoaded = true; + LogPrintf(" block index %15dms\n", GetTimeMillis() - load_block_index_start_time); + } } while(false); if (!fLoaded && !ShutdownRequested()) { @@ -1724,8 +1782,11 @@ bool AppInitMain(NodeContext& node) LogPrintf("Unsetting NODE_NETWORK on prune mode\n"); nLocalServices = ServiceFlags(nLocalServices & ~NODE_NETWORK); if (!fReindex) { - uiInterface.InitMessage(_("Pruning blockstore...").translated); - ::ChainstateActive().PruneAndFlush(); + LOCK(cs_main); + for (CChainState* chainstate : g_chainman.GetAll()) { + uiInterface.InitMessage(_("Pruning blockstore...").translated); + chainstate->PruneAndFlush(); + } } } diff --git a/src/init.h b/src/init.h index f74ae5a47a..ef568b6f38 100644 --- a/src/init.h +++ b/src/init.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -54,9 +54,9 @@ bool AppInitLockDataDirectory(); bool AppInitMain(NodeContext& node); /** - * Setup the arguments for gArgs + * Register all arguments with the ArgsManager */ -void SetupServerArgs(); +void SetupServerArgs(NodeContext& node); /** Returns licensing information (for -version) */ std::string LicenseInfo(); diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index 0b3cd08e22..0e7641ae32 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -38,6 +38,21 @@ namespace interfaces { namespace { +bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<RecursiveMutex>& lock) +{ + if (!index) return false; + if (block.m_hash) *block.m_hash = index->GetBlockHash(); + if (block.m_height) *block.m_height = index->nHeight; + if (block.m_time) *block.m_time = index->GetBlockTime(); + if (block.m_max_time) *block.m_max_time = index->GetBlockTimeMax(); + if (block.m_mtp_time) *block.m_mtp_time = index->GetMedianTimePast(); + if (block.m_data) { + REVERSE_LOCK(lock); + if (!ReadBlockFromDisk(*block.m_data, index, Params().GetConsensus())) block.m_data->SetNull(); + } + return true; +} + class LockImpl : public Chain::Lock, public UniqueLock<RecursiveMutex> { Optional<int> getHeight() override @@ -65,20 +80,6 @@ class LockImpl : public Chain::Lock, public UniqueLock<RecursiveMutex> assert(block != nullptr); return block->GetBlockHash(); } - int64_t getBlockTime(int height) override - { - LockAssertion lock(::cs_main); - CBlockIndex* block = ::ChainActive()[height]; - assert(block != nullptr); - return block->GetBlockTime(); - } - int64_t getBlockMedianTimePast(int height) override - { - LockAssertion lock(::cs_main); - CBlockIndex* block = ::ChainActive()[height]; - assert(block != nullptr); - return block->GetMedianTimePast(); - } bool haveBlockOnDisk(int height) override { LockAssertion lock(::cs_main); @@ -95,20 +96,6 @@ class LockImpl : public Chain::Lock, public UniqueLock<RecursiveMutex> } return nullopt; } - Optional<int> findPruned(int start_height, Optional<int> stop_height) override - { - LockAssertion lock(::cs_main); - if (::fPruneMode) { - CBlockIndex* block = stop_height ? ::ChainActive()[*stop_height] : ::ChainActive().Tip(); - while (block && block->nHeight >= start_height) { - if ((block->nStatus & BLOCK_HAVE_DATA) == 0) { - return block->nHeight; - } - block = block->pprev; - } - } - return nullopt; - } Optional<int> findFork(const uint256& hash, Optional<int>* height) override { LockAssertion lock(::cs_main); @@ -247,26 +234,50 @@ public: std::unique_ptr<Chain::Lock> result = std::move(lock); // Temporary to avoid CWG 1579 return result; } - bool findBlock(const uint256& hash, CBlock* block, int64_t* time, int64_t* time_max) override + bool findBlock(const uint256& hash, const FoundBlock& block) override { - CBlockIndex* index; - { - LOCK(cs_main); - index = LookupBlockIndex(hash); - if (!index) { - return false; - } - if (time) { - *time = index->GetBlockTime(); - } - if (time_max) { - *time_max = index->GetBlockTimeMax(); + WAIT_LOCK(cs_main, lock); + return FillBlock(LookupBlockIndex(hash), block, lock); + } + bool findFirstBlockWithTimeAndHeight(int64_t min_time, int min_height, const FoundBlock& block) override + { + WAIT_LOCK(cs_main, lock); + return FillBlock(ChainActive().FindEarliestAtLeast(min_time, min_height), block, lock); + } + bool findNextBlock(const uint256& block_hash, int block_height, const FoundBlock& next, bool* reorg) override { + WAIT_LOCK(cs_main, lock); + CBlockIndex* block = ChainActive()[block_height]; + if (block && block->GetBlockHash() != block_hash) block = nullptr; + if (reorg) *reorg = !block; + return FillBlock(block ? ChainActive()[block_height + 1] : nullptr, next, lock); + } + bool findAncestorByHeight(const uint256& block_hash, int ancestor_height, const FoundBlock& ancestor_out) override + { + WAIT_LOCK(cs_main, lock); + if (const CBlockIndex* block = LookupBlockIndex(block_hash)) { + if (const CBlockIndex* ancestor = block->GetAncestor(ancestor_height)) { + return FillBlock(ancestor, ancestor_out, lock); } } - if (block && !ReadBlockFromDisk(*block, index, Params().GetConsensus())) { - block->SetNull(); - } - return true; + return FillBlock(nullptr, ancestor_out, lock); + } + bool findAncestorByHash(const uint256& block_hash, const uint256& ancestor_hash, const FoundBlock& ancestor_out) override + { + WAIT_LOCK(cs_main, lock); + const CBlockIndex* block = LookupBlockIndex(block_hash); + const CBlockIndex* ancestor = LookupBlockIndex(ancestor_hash); + if (block && ancestor && block->GetAncestor(ancestor->nHeight) != ancestor) ancestor = nullptr; + return FillBlock(ancestor, ancestor_out, lock); + } + bool findCommonAncestor(const uint256& block_hash1, const uint256& block_hash2, const FoundBlock& ancestor_out, const FoundBlock& block1_out, const FoundBlock& block2_out) override + { + WAIT_LOCK(cs_main, lock); + const CBlockIndex* block1 = LookupBlockIndex(block_hash1); + const CBlockIndex* block2 = LookupBlockIndex(block_hash2); + const CBlockIndex* ancestor = block1 && block2 ? LastCommonAncestor(block1, block2) : nullptr; + // Using & instead of && below to avoid short circuiting and leaving + // output uninitialized. + return FillBlock(ancestor, ancestor_out, lock) & FillBlock(block1, block1_out, lock) & FillBlock(block2, block2_out, lock); } void findCoins(std::map<COutPoint, Coin>& coins) override { return FindCoins(m_node, coins); } double guessVerificationProgress(const uint256& block_hash) override @@ -274,6 +285,25 @@ public: LOCK(cs_main); return GuessVerificationProgress(Params().TxData(), LookupBlockIndex(block_hash)); } + bool hasBlocks(const uint256& block_hash, int min_height, Optional<int> max_height) override + { + // hasBlocks returns true if all ancestors of block_hash in specified + // range have block data (are not pruned), false if any ancestors in + // specified range are missing data. + // + // For simplicity and robustness, min_height and max_height are only + // used to limit the range, and passing min_height that's too low or + // max_height that's too high will not crash or change the result. + LOCK(::cs_main); + if (CBlockIndex* block = LookupBlockIndex(block_hash)) { + if (max_height && block->nHeight >= *max_height) block = block->GetAncestor(*max_height); + for (; block->nStatus & BLOCK_HAVE_DATA; block = block->pprev) { + // Check pprev to not segfault if min_height is too low + if (block->nHeight <= min_height || !block->pprev) return true; + } + } + return false; + } RBFTransactionState isRBFOptIn(const CTransaction& tx) override { LOCK(::mempool.cs); diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index e1bc9bbbf3..fb77c81cdc 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -30,6 +30,27 @@ namespace interfaces { class Handler; class Wallet; +//! Helper for findBlock to selectively return pieces of block data. +class FoundBlock +{ +public: + FoundBlock& hash(uint256& hash) { m_hash = &hash; return *this; } + FoundBlock& height(int& height) { m_height = &height; return *this; } + FoundBlock& time(int64_t& time) { m_time = &time; return *this; } + FoundBlock& maxTime(int64_t& max_time) { m_max_time = &max_time; return *this; } + FoundBlock& mtpTime(int64_t& mtp_time) { m_mtp_time = &mtp_time; return *this; } + //! Read block data from disk. If the block exists but doesn't have data + //! (for example due to pruning), the CBlock variable will be set to null. + FoundBlock& data(CBlock& data) { m_data = &data; return *this; } + + uint256* m_hash = nullptr; + int* m_height = nullptr; + int64_t* m_time = nullptr; + int64_t* m_max_time = nullptr; + int64_t* m_mtp_time = nullptr; + CBlock* m_data = nullptr; +}; + //! Interface giving clients (wallet processes, maybe other analysis tools in //! the future) ability to access to the chain state, receive notifications, //! estimate fees, and submit transactions. @@ -79,13 +100,6 @@ public: //! Get block hash. Height must be valid or this function will abort. virtual uint256 getBlockHash(int height) = 0; - //! Get block time. Height must be valid or this function will abort. - virtual int64_t getBlockTime(int height) = 0; - - //! Get block median time past. Height must be valid or this function - //! will abort. - virtual int64_t getBlockMedianTimePast(int height) = 0; - //! Check that the block is available on disk (i.e. has not been //! pruned), and contains transactions. virtual bool haveBlockOnDisk(int height) = 0; @@ -97,10 +111,6 @@ public: //! (to avoid the cost of a second lookup in case this information is needed.) virtual Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height, uint256* hash) = 0; - //! Return height of last block in the specified range which is pruned, or - //! nullopt if no block in the range is pruned. Range is inclusive. - virtual Optional<int> findPruned(int start_height = 0, Optional<int> stop_height = nullopt) = 0; - //! Return height of the specified block if it is on the chain, otherwise //! return the height of the highest block on chain that's an ancestor //! of the specified block, or nullopt if there is no common ancestor. @@ -127,14 +137,36 @@ public: //! Return whether node has the block and optionally return block metadata //! or contents. - //! - //! If a block pointer is provided to retrieve the block contents, and the - //! block exists but doesn't have data (for example due to pruning), the - //! block will be empty and all fields set to null. - virtual bool findBlock(const uint256& hash, - CBlock* block = nullptr, - int64_t* time = nullptr, - int64_t* max_time = nullptr) = 0; + virtual bool findBlock(const uint256& hash, const FoundBlock& block={}) = 0; + + //! Find first block in the chain with timestamp >= the given time + //! and height >= than the given height, return false if there is no block + //! with a high enough timestamp and height. Optionally return block + //! information. + virtual bool findFirstBlockWithTimeAndHeight(int64_t min_time, int min_height, const FoundBlock& block={}) = 0; + + //! Find next block if block is part of current chain. Also flag if + //! there was a reorg and the specified block hash is no longer in the + //! current chain, and optionally return block information. + virtual bool findNextBlock(const uint256& block_hash, int block_height, const FoundBlock& next={}, bool* reorg=nullptr) = 0; + + //! Find ancestor of block at specified height and optionally return + //! ancestor information. + virtual bool findAncestorByHeight(const uint256& block_hash, int ancestor_height, const FoundBlock& ancestor_out={}) = 0; + + //! Return whether block descends from a specified ancestor, and + //! optionally return ancestor information. + virtual bool findAncestorByHash(const uint256& block_hash, + const uint256& ancestor_hash, + const FoundBlock& ancestor_out={}) = 0; + + //! Find most recent common ancestor between two blocks and optionally + //! return block information. + virtual bool findCommonAncestor(const uint256& block_hash1, + const uint256& block_hash2, + const FoundBlock& ancestor_out={}, + const FoundBlock& block1_out={}, + const FoundBlock& block2_out={}) = 0; //! Look up unspent output information. Returns coins in the mempool and in //! the current chain UTXO set. Iterates through all the keys in the map and @@ -145,6 +177,11 @@ public: //! the specified block hash are verified. virtual double guessVerificationProgress(const uint256& block_hash) = 0; + //! Return true if data is available for all blocks in the specified range + //! of blocks. This checks all blocks that are ancestors of block_hash in + //! the height range from min_height to max_height, inclusive. + virtual bool hasBlocks(const uint256& block_hash, int min_height = 0, Optional<int> max_height = {}) = 0; + //! Check if transaction is RBF opt in. virtual RBFTransactionState isRBFOptIn(const CTransaction& tx) = 0; @@ -283,6 +320,12 @@ public: //! Shut down client. virtual void stop() = 0; + + //! Set mock time. + virtual void setMockTime(int64_t time) = 0; + + //! Return interfaces for accessing wallets (if any). + virtual std::vector<std::unique_ptr<Wallet>> getWallets() = 0; }; //! Return implementation of Chain interface. diff --git a/src/interfaces/handler.cpp b/src/interfaces/handler.cpp index 95035c1b54..4134a4527f 100644 --- a/src/interfaces/handler.cpp +++ b/src/interfaces/handler.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/interfaces/handler.h b/src/interfaces/handler.h index fbac3c6b71..11baf9dd65 100644 --- a/src/interfaces/handler.h +++ b/src/interfaces/handler.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp index 905173d20b..5ebbd61584 100644 --- a/src/interfaces/node.cpp +++ b/src/interfaces/node.cpp @@ -97,7 +97,7 @@ public: StopMapPort(); } } - void setupServerArgs() override { return SetupServerArgs(); } + void setupServerArgs() override { return SetupServerArgs(m_context); } bool getProxy(Network net, proxyType& proxy_info) override { return GetProxy(net, proxy_info); } size_t getNodeCount(CConnman::NumConnections flags) override { @@ -253,8 +253,9 @@ public: std::vector<std::unique_ptr<Wallet>> getWallets() override { std::vector<std::unique_ptr<Wallet>> wallets; - for (const std::shared_ptr<CWallet>& wallet : GetWallets()) { - wallets.emplace_back(MakeWallet(wallet)); + for (auto& client : m_context.chain_clients) { + auto client_wallets = client->getWallets(); + std::move(client_wallets.begin(), client_wallets.end(), std::back_inserter(wallets)); } return wallets; } diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index abce09ca4a..152917ac60 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,6 +14,7 @@ #include <sync.h> #include <ui_interface.h> #include <uint256.h> +#include <util/check.h> #include <util/system.h> #include <wallet/feebumper.h> #include <wallet/fees.h> @@ -62,7 +63,7 @@ WalletTx MakeWalletTx(CWallet& wallet, const CWalletTx& wtx) WalletTxStatus MakeWalletTxStatus(interfaces::Chain::Lock& locked_chain, const CWalletTx& wtx) { WalletTxStatus result; - result.block_height = locked_chain.getBlockHeight(wtx.m_confirm.hashBlock).get_value_or(std::numeric_limits<int>::max()); + result.block_height = wtx.m_confirm.block_height > 0 ? wtx.m_confirm.block_height : std::numeric_limits<int>::max(); result.blocks_to_maturity = wtx.GetBlocksToMaturity(); result.depth_in_main_chain = wtx.GetDepthInMainChain(); result.time_received = wtx.nTimeReceived; @@ -318,13 +319,9 @@ public: if (mi == m_wallet->mapWallet.end()) { return false; } - if (Optional<int> height = locked_chain->getHeight()) { - num_blocks = *height; - block_time = locked_chain->getBlockTime(*height); - } else { - num_blocks = -1; - block_time = -1; - } + num_blocks = m_wallet->GetLastBlockHeight(); + block_time = -1; + CHECK_NONFATAL(m_wallet->chain().findBlock(m_wallet->GetLastBlockHash(), FoundBlock().time(block_time))); tx_status = MakeWalletTxStatus(*locked_chain, mi->second); return true; } @@ -373,12 +370,12 @@ public: { auto locked_chain = m_wallet->chain().lock(true /* try_lock */); if (!locked_chain) return false; - num_blocks = locked_chain->getHeight().get_value_or(-1); - if (!force && num_blocks == cached_num_blocks) return false; TRY_LOCK(m_wallet->cs_wallet, locked_wallet); if (!locked_wallet) { return false; } + num_blocks = m_wallet->GetLastBlockHeight(); + if (!force && num_blocks == cached_num_blocks) return false; balances = getBalances(); return true; } @@ -467,6 +464,7 @@ public: { RemoveWallet(m_wallet); } + bool isLegacy() override { return m_wallet->IsLegacy(); } std::unique_ptr<Handler> handleUnload(UnloadFn fn) override { return MakeHandler(m_wallet->NotifyUnload.connect(fn)); @@ -519,6 +517,15 @@ public: void start(CScheduler& scheduler) override { return StartWallets(scheduler); } void flush() override { return FlushWallets(); } void stop() override { return StopWallets(); } + void setMockTime(int64_t time) override { return SetMockTime(time); } + std::vector<std::unique_ptr<Wallet>> getWallets() override + { + std::vector<std::unique_ptr<Wallet>> wallets; + for (const auto& wallet : GetWallets()) { + wallets.emplace_back(MakeWallet(wallet)); + } + return wallets; + } ~WalletClientImpl() override { UnloadWallets(); } Chain& m_chain; diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index dbb0912230..5d870c5e3d 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,7 +15,6 @@ #include <functional> #include <map> #include <memory> -#include <psbt.h> #include <stdint.h> #include <string> #include <tuple> @@ -26,12 +25,13 @@ class CCoinControl; class CFeeRate; class CKey; class CWallet; -enum isminetype : unsigned int; enum class FeeReason; -typedef uint8_t isminefilter; - enum class OutputType; +enum class TransactionError; +enum isminetype : unsigned int; struct CRecipient; +struct PartiallySignedTransaction; +typedef uint8_t isminefilter; namespace interfaces { @@ -266,6 +266,9 @@ public: // Remove wallet. virtual void remove() = 0; + //! Return whether is a legacy wallet + virtual bool isLegacy() = 0; + //! Register handler for unload message. using UnloadFn = std::function<void()>; virtual std::unique_ptr<Handler> handleUnload(UnloadFn fn) = 0; diff --git a/src/logging.cpp b/src/logging.cpp index 6fd916b603..eb9da06d9b 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -182,30 +182,15 @@ bool GetLogCategory(BCLog::LogFlags& flag, const std::string& str) return false; } -std::string ListLogCategories() +std::vector<LogCategory> BCLog::Logger::LogCategoriesList() { - std::string ret; - int outcount = 0; + std::vector<LogCategory> ret; for (const CLogCategoryDesc& category_desc : LogCategories) { // Omit the special cases. if (category_desc.flag != BCLog::NONE && category_desc.flag != BCLog::ALL) { - if (outcount != 0) ret += ", "; - ret += category_desc.category; - outcount++; - } - } - return ret; -} - -std::vector<CLogCategoryActive> ListActiveLogCategories() -{ - std::vector<CLogCategoryActive> ret; - for (const CLogCategoryDesc& category_desc : LogCategories) { - // Omit the special cases. - if (category_desc.flag != BCLog::NONE && category_desc.flag != BCLog::ALL) { - CLogCategoryActive catActive; + LogCategory catActive; catActive.category = category_desc.category; - catActive.active = LogAcceptCategory(category_desc.flag); + catActive.active = WillLogCategory(category_desc.flag); ret.push_back(catActive); } } diff --git a/src/logging.h b/src/logging.h index b2fde1b9ea..ab07010316 100644 --- a/src/logging.h +++ b/src/logging.h @@ -8,6 +8,7 @@ #include <fs.h> #include <tinyformat.h> +#include <util/string.h> #include <atomic> #include <cstdint> @@ -24,8 +25,7 @@ extern const char * const DEFAULT_DEBUGLOGFILE; extern bool fLogIPs; -struct CLogCategoryActive -{ +struct LogCategory { std::string category; bool active; }; @@ -132,6 +132,13 @@ namespace BCLog { bool DisableCategory(const std::string& str); bool WillLogCategory(LogFlags category) const; + /** Returns a vector of the log categories */ + std::vector<LogCategory> LogCategoriesList(); + /** Returns a string with the log categories */ + std::string LogCategoriesString() + { + return Join(LogCategoriesList(), ", ", [&](const LogCategory& i) { return i.category; }); + }; bool DefaultShrinkDebugFile() const; }; @@ -146,12 +153,6 @@ static inline bool LogAcceptCategory(BCLog::LogFlags category) return LogInstance().WillLogCategory(category); } -/** Returns a string with the log categories. */ -std::string ListLogCategories(); - -/** Returns a vector of the active log categories. */ -std::vector<CLogCategoryActive> ListActiveLogCategories(); - /** Return true if str parses as a log category and set the flag */ bool GetLogCategory(BCLog::LogFlags& flag, const std::string& str); diff --git a/src/logging/timer.h b/src/logging/timer.h index 2b27c71080..21bb3d121e 100644 --- a/src/logging/timer.h +++ b/src/logging/timer.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/memusage.h b/src/memusage.h index 24eb450465..a6e894129a 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2018 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/miner.cpp b/src/miner.cpp index 61d27d17c1..d9dcbe8a70 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -39,6 +39,17 @@ int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParam return nNewTime - nOldTime; } +void RegenerateCommitments(CBlock& block) +{ + CMutableTransaction tx{*block.vtx.at(0)}; + tx.vout.erase(tx.vout.begin() + GetWitnessCommitmentIndex(block)); + block.vtx.at(0) = MakeTransactionRef(tx); + + GenerateCoinbaseCommitment(block, WITH_LOCK(cs_main, return LookupBlockIndex(block.hashPrevBlock)), Params().GetConsensus()); + + block.hashMerkleRoot = BlockMerkleRoot(block); +} + BlockAssembler::Options::Options() { blockMinFeeRate = CFeeRate(DEFAULT_BLOCK_MIN_TX_FEE); nBlockMaxWeight = DEFAULT_BLOCK_MAX_WEIGHT; diff --git a/src/miner.h b/src/miner.h index cc8fc31a9f..69296f9078 100644 --- a/src/miner.h +++ b/src/miner.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -203,4 +203,7 @@ private: void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned int& nExtraNonce); int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev); +/** Update an old GenerateCoinbaseCommitment from CreateNewBlock after the block txs have changed */ +void RegenerateCommitments(CBlock& block); + #endif // BITCOIN_MINER_H @@ -45,16 +45,10 @@ static const bool DEFAULT_WHITELISTRELAY = true; /** Default for -whitelistforcerelay. */ static const bool DEFAULT_WHITELISTFORCERELAY = false; -/** Time between pings automatically sent out for latency probing and keepalive (in seconds). */ -static const int PING_INTERVAL = 2 * 60; /** Time after which to disconnect, after waiting for a ping response (or inactivity). */ static const int TIMEOUT_INTERVAL = 20 * 60; /** Run the feeler connection loop once every 2 minutes or 120 seconds. **/ static const int FEELER_INTERVAL = 120; -/** The maximum number of entries in an 'inv' protocol message */ -static const unsigned int MAX_INV_SZ = 50000; -/** The maximum number of entries in a locator */ -static const unsigned int MAX_LOCATOR_SZ = 101; /** The maximum number of new addresses to accumulate before announcing. */ static const unsigned int MAX_ADDR_TO_SEND = 1000; /** Maximum length of incoming protocol messages (no message over 4 MB is currently acceptable). */ @@ -479,6 +473,7 @@ private: std::atomic<int64_t> m_next_send_inv_to_incoming{0}; friend struct CConnmanTest; + friend struct ConnmanTestMsg; }; void Discover(); void StartMapPort(); @@ -718,6 +713,8 @@ public: class CNode { friend class CConnman; + friend struct ConnmanTestMsg; + public: std::unique_ptr<TransportDeserializer> m_deserializer; std::unique_ptr<TransportSerializer> m_serializer; @@ -794,8 +791,8 @@ public: std::vector<CAddress> vAddrToSend; const std::unique_ptr<CRollingBloomFilter> m_addr_known; bool fGetAddr{false}; - int64_t nNextAddrSend GUARDED_BY(cs_sendProcessing){0}; - int64_t nNextLocalAddrSend GUARDED_BY(cs_sendProcessing){0}; + std::chrono::microseconds m_next_addr_send GUARDED_BY(cs_sendProcessing){0}; + std::chrono::microseconds m_next_local_addr_send GUARDED_BY(cs_sendProcessing){0}; bool IsAddrRelayPeer() const { return m_addr_known != nullptr; } @@ -806,14 +803,13 @@ public: RecursiveMutex cs_inventory; struct TxRelay { - TxRelay() { pfilter = MakeUnique<CBloomFilter>(); } mutable RecursiveMutex cs_filter; // We use fRelayTxes for two purposes - // a) it allows us to not relay tx invs before receiving the peer's version message // b) the peer may tell us in its version message that we should not relay tx invs // unless it loads a bloom filter. bool fRelayTxes GUARDED_BY(cs_filter){false}; - std::unique_ptr<CBloomFilter> pfilter PT_GUARDED_BY(cs_filter) GUARDED_BY(cs_filter); + std::unique_ptr<CBloomFilter> pfilter PT_GUARDED_BY(cs_filter) GUARDED_BY(cs_filter){nullptr}; mutable RecursiveMutex cs_tx_inventory; CRollingBloomFilter filterInventoryKnown GUARDED_BY(cs_tx_inventory){50000, 0.000001}; diff --git a/src/net_permissions.h b/src/net_permissions.h index ad74848347..962a2159fc 100644 --- a/src/net_permissions.h +++ b/src/net_permissions.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/net_processing.cpp b/src/net_processing.cpp index ab430cbe19..26327ac6eb 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -64,6 +64,12 @@ static constexpr int STALE_RELAY_AGE_LIMIT = 30 * 24 * 60 * 60; /// Age after which a block is considered historical for purposes of rate /// limiting block relay. Set to one week, denominated in seconds. static constexpr int HISTORICAL_BLOCK_AGE = 7 * 24 * 60 * 60; +/** Time between pings automatically sent out for latency probing and keepalive (in seconds). */ +static const int PING_INTERVAL = 2 * 60; +/** The maximum number of entries in a locator */ +static const unsigned int MAX_LOCATOR_SZ = 101; +/** The maximum number of entries in an 'inv' protocol message */ +static const unsigned int MAX_INV_SZ = 50000; /** Maximum number of in-flight transactions from a peer */ static constexpr int32_t MAX_PEER_TX_IN_FLIGHT = 100; /** Maximum number of announced transactions from a peer */ @@ -80,7 +86,47 @@ static_assert(INBOUND_PEER_TX_DELAY >= MAX_GETDATA_RANDOM_DELAY, "To preserve security, MAX_GETDATA_RANDOM_DELAY should not exceed INBOUND_PEER_DELAY"); /** Limit to avoid sending big packets. Not used in processing incoming GETDATA for compatibility */ static const unsigned int MAX_GETDATA_SZ = 1000; - +/** Number of blocks that can be requested at any given time from a single peer. */ +static const int MAX_BLOCKS_IN_TRANSIT_PER_PEER = 16; +/** Timeout in seconds during which a peer must stall block download progress before being disconnected. */ +static const unsigned int BLOCK_STALLING_TIMEOUT = 2; +/** Number of headers sent in one getheaders result. We rely on the assumption that if a peer sends + * less than this number, we reached its tip. Changing this value is a protocol upgrade. */ +static const unsigned int MAX_HEADERS_RESULTS = 2000; +/** Maximum depth of blocks we're willing to serve as compact blocks to peers + * when requested. For older blocks, a regular BLOCK response will be sent. */ +static const int MAX_CMPCTBLOCK_DEPTH = 5; +/** Maximum depth of blocks we're willing to respond to GETBLOCKTXN requests for. */ +static const int MAX_BLOCKTXN_DEPTH = 10; +/** Size of the "block download window": how far ahead of our current height do we fetch? + * Larger windows tolerate larger download speed differences between peer, but increase the potential + * degree of disordering of blocks on disk (which make reindexing and pruning harder). We'll probably + * want to make this a per-peer adaptive value at some point. */ +static const unsigned int BLOCK_DOWNLOAD_WINDOW = 1024; +/** Block download timeout base, expressed in millionths of the block interval (i.e. 10 min) */ +static const int64_t BLOCK_DOWNLOAD_TIMEOUT_BASE = 1000000; +/** Additional block download timeout per parallel downloading peer (i.e. 5 min) */ +static const int64_t BLOCK_DOWNLOAD_TIMEOUT_PER_PEER = 500000; +/** Maximum number of headers to announce when relaying blocks with headers message.*/ +static const unsigned int MAX_BLOCKS_TO_ANNOUNCE = 8; +/** Maximum number of unconnecting headers announcements before DoS score */ +static const int MAX_UNCONNECTING_HEADERS = 10; +/** Minimum blocks required to signal NODE_NETWORK_LIMITED */ +static const unsigned int NODE_NETWORK_LIMITED_MIN_BLOCKS = 288; +/** Average delay between local address broadcasts */ +static constexpr std::chrono::hours AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL{24}; +/** Average delay between peer address broadcasts */ +static constexpr std::chrono::seconds AVG_ADDRESS_BROADCAST_INTERVAL{30}; +/** Average delay between trickled inventory transmissions in seconds. + * Blocks and whitelisted receivers bypass this, outbound peers get half this delay. */ +static const unsigned int INVENTORY_BROADCAST_INTERVAL = 5; +/** Maximum number of inventory items to send per transmission. + * Limits the impact of low-fee transaction floods. */ +static constexpr unsigned int INVENTORY_BROADCAST_MAX = 7 * INVENTORY_BROADCAST_INTERVAL; +/** Average delay between feefilter broadcasts in seconds. */ +static constexpr unsigned int AVG_FEEFILTER_BROADCAST_INTERVAL = 10 * 60; +/** Maximum feefilter broadcast delay after significant change. */ +static constexpr unsigned int MAX_FEEFILTER_CHANGE_DELAY = 5 * 60; struct COrphanTx { // When modifying, adapt the copy of this definition in tests/DoS_tests. @@ -97,21 +143,6 @@ void EraseOrphansFor(NodeId peer); /** Increase a node's misbehavior score. */ void Misbehaving(NodeId nodeid, int howmuch, const std::string& message="") EXCLUSIVE_LOCKS_REQUIRED(cs_main); -/** Average delay between local address broadcasts in seconds. */ -static constexpr unsigned int AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL = 24 * 60 * 60; -/** Average delay between peer address broadcasts in seconds. */ -static const unsigned int AVG_ADDRESS_BROADCAST_INTERVAL = 30; -/** Average delay between trickled inventory transmissions in seconds. - * Blocks and whitelisted receivers bypass this, outbound peers get half this delay. */ -static const unsigned int INVENTORY_BROADCAST_INTERVAL = 5; -/** Maximum number of inventory items to send per transmission. - * Limits the impact of low-fee transaction floods. */ -static constexpr unsigned int INVENTORY_BROADCAST_MAX = 7 * INVENTORY_BROADCAST_INTERVAL; -/** Average delay between feefilter broadcasts in seconds. */ -static constexpr unsigned int AVG_FEEFILTER_BROADCAST_INTERVAL = 10 * 60; -/** Maximum feefilter broadcast delay after significant change. */ -static constexpr unsigned int MAX_FEEFILTER_CHANGE_DELAY = 5 * 60; - // Internal stuff namespace { /** Number of nodes with fSyncStarted. */ @@ -1365,7 +1396,7 @@ void RelayTransaction(const uint256& txid, const CConnman& connman) }); } -static void RelayAddress(const CAddress& addr, bool fReachable, CConnman* connman) +static void RelayAddress(const CAddress& addr, bool fReachable, const CConnman& connman) { unsigned int nRelayNodes = fReachable ? 2 : 1; // limited relaying of addresses outside our network(s) @@ -1373,7 +1404,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, CConnman* connma // Use deterministic randomness to send to the same nodes for 24 hours // at a time so the m_addr_knowns of the chosen nodes prevent repeats uint64_t hashAddr = addr.GetHash(); - const CSipHasher hasher = connman->GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60)); + const CSipHasher hasher = connman.GetDeterministicRandomizer(RANDOMIZER_ID_ADDRESS_RELAY).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24 * 60 * 60)); FastRandomContext insecure_rand; std::array<std::pair<uint64_t, CNode*>,2> best{{{0, nullptr}, {0, nullptr}}}; @@ -1398,7 +1429,7 @@ static void RelayAddress(const CAddress& addr, bool fReachable, CConnman* connma } }; - connman->ForEachNodeThen(std::move(sortfunc), std::move(pushfunc)); + connman.ForEachNodeThen(std::move(sortfunc), std::move(pushfunc)); } void static ProcessGetBlockData(CNode* pfrom, const CChainParams& chainparams, const CInv& inv, CConnman* connman) @@ -1911,9 +1942,9 @@ void static ProcessOrphanTx(CConnman* connman, CTxMemPool& mempool, std::set<uin } } -bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc) +bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc) { - LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->GetId()); + LogPrint(BCLog::NET, "received: %s (%u bytes) peer=%d\n", SanitizeString(msg_type), vRecv.size(), pfrom->GetId()); if (gArgs.IsArgSet("-dropmessagestest") && GetRand(gArgs.GetArg("-dropmessagestest", 0)) == 0) { LogPrintf("dropmessagestest DROPPING RECV MESSAGE\n"); @@ -1922,8 +1953,8 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR if (!(pfrom->GetLocalServices() & NODE_BLOOM) && - (strCommand == NetMsgType::FILTERLOAD || - strCommand == NetMsgType::FILTERADD)) + (msg_type == NetMsgType::FILTERLOAD || + msg_type == NetMsgType::FILTERADD)) { if (pfrom->nVersion >= NO_BLOOM_VERSION) { LOCK(cs_main); @@ -1935,7 +1966,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR } } - if (strCommand == NetMsgType::VERSION) { + if (msg_type == NetMsgType::VERSION) { // Each connection can only send one version message if (pfrom->nVersion != 0) { @@ -2107,7 +2138,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR // At this point, the outgoing message serialization version can't change. const CNetMsgMaker msgMaker(pfrom->GetSendVersion()); - if (strCommand == NetMsgType::VERACK) + if (msg_type == NetMsgType::VERACK) { pfrom->SetRecvVersion(std::min(pfrom->nVersion.load(), PROTOCOL_VERSION)); @@ -2152,7 +2183,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return false; } - if (strCommand == NetMsgType::ADDR) { + if (msg_type == NetMsgType::ADDR) { std::vector<CAddress> vAddr; vRecv >> vAddr; @@ -2192,7 +2223,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR if (addr.nTime > nSince && !pfrom->fGetAddr && vAddr.size() <= 10 && addr.IsRoutable()) { // Relay to a limited number of other nodes - RelayAddress(addr, fReachable, connman); + RelayAddress(addr, fReachable, *connman); } // Do not store addresses outside our network if (fReachable) @@ -2206,13 +2237,13 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::SENDHEADERS) { + if (msg_type == NetMsgType::SENDHEADERS) { LOCK(cs_main); State(pfrom->GetId())->fPreferHeaders = true; return true; } - if (strCommand == NetMsgType::SENDCMPCT) { + if (msg_type == NetMsgType::SENDCMPCT) { bool fAnnounceUsingCMPCTBLOCK = false; uint64_t nCMPCTBLOCKVersion = 0; vRecv >> fAnnounceUsingCMPCTBLOCK >> nCMPCTBLOCKVersion; @@ -2235,7 +2266,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::INV) { + if (msg_type == NetMsgType::INV) { std::vector<CInv> vInv; vRecv >> vInv; if (vInv.size() > MAX_INV_SZ) @@ -2297,7 +2328,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::GETDATA) { + if (msg_type == NetMsgType::GETDATA) { std::vector<CInv> vInv; vRecv >> vInv; if (vInv.size() > MAX_INV_SZ) @@ -2318,7 +2349,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::GETBLOCKS) { + if (msg_type == NetMsgType::GETBLOCKS) { CBlockLocator locator; uint256 hashStop; vRecv >> locator >> hashStop; @@ -2386,7 +2417,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::GETBLOCKTXN) { + if (msg_type == NetMsgType::GETBLOCKTXN) { BlockTransactionsRequest req; vRecv >> req; @@ -2435,7 +2466,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::GETHEADERS) { + if (msg_type == NetMsgType::GETHEADERS) { CBlockLocator locator; uint256 hashStop; vRecv >> locator >> hashStop; @@ -2502,7 +2533,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::TX) { + if (msg_type == NetMsgType::TX) { // Stop processing the transaction early if // We are in blocks only mode and peer is either not whitelisted or whitelistrelay is off // or if this peer is supposed to be a block-relay-only peer @@ -2644,7 +2675,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::CMPCTBLOCK) + if (msg_type == NetMsgType::CMPCTBLOCK) { // Ignore cmpctblock received while importing if (fImporting || fReindex) { @@ -2865,7 +2896,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::BLOCKTXN) + if (msg_type == NetMsgType::BLOCKTXN) { // Ignore blocktxn received while importing if (fImporting || fReindex) { @@ -2947,7 +2978,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::HEADERS) + if (msg_type == NetMsgType::HEADERS) { // Ignore headers received while importing if (fImporting || fReindex) { @@ -2973,7 +3004,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return ProcessHeadersMessage(pfrom, connman, mempool, headers, chainparams, /*via_compact_block=*/false); } - if (strCommand == NetMsgType::BLOCK) + if (msg_type == NetMsgType::BLOCK) { // Ignore block received while importing if (fImporting || fReindex) { @@ -3009,7 +3040,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::GETADDR) { + if (msg_type == NetMsgType::GETADDR) { // This asymmetric behavior for inbound and outbound connections was introduced // to prevent a fingerprinting attack: an attacker can send specific fake addresses // to users' AddrMan and later request them by sending getaddr messages. @@ -3043,7 +3074,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::MEMPOOL) { + if (msg_type == NetMsgType::MEMPOOL) { if (!(pfrom->GetLocalServices() & NODE_BLOOM) && !pfrom->HasPermission(PF_MEMPOOL)) { if (!pfrom->HasPermission(PF_NOBAN)) @@ -3071,7 +3102,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::PING) { + if (msg_type == NetMsgType::PING) { if (pfrom->nVersion > BIP0031_VERSION) { uint64_t nonce = 0; @@ -3092,7 +3123,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::PONG) { + if (msg_type == NetMsgType::PONG) { int64_t pingUsecEnd = nTimeReceived; uint64_t nonce = 0; size_t nAvail = vRecv.in_avail(); @@ -3148,7 +3179,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::FILTERLOAD) { + if (msg_type == NetMsgType::FILTERLOAD) { CBloomFilter filter; vRecv >> filter; @@ -3168,7 +3199,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::FILTERADD) { + if (msg_type == NetMsgType::FILTERADD) { std::vector<unsigned char> vData; vRecv >> vData; @@ -3192,19 +3223,19 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::FILTERCLEAR) { + if (msg_type == NetMsgType::FILTERCLEAR) { if (pfrom->m_tx_relay == nullptr) { return true; } LOCK(pfrom->m_tx_relay->cs_filter); if (pfrom->GetLocalServices() & NODE_BLOOM) { - pfrom->m_tx_relay->pfilter.reset(new CBloomFilter()); + pfrom->m_tx_relay->pfilter = nullptr; } pfrom->m_tx_relay->fRelayTxes = true; return true; } - if (strCommand == NetMsgType::FEEFILTER) { + if (msg_type == NetMsgType::FEEFILTER) { CAmount newFeeFilter = 0; vRecv >> newFeeFilter; if (MoneyRange(newFeeFilter)) { @@ -3217,7 +3248,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR return true; } - if (strCommand == NetMsgType::NOTFOUND) { + if (msg_type == NetMsgType::NOTFOUND) { // Remove the NOTFOUND transactions from the peer LOCK(cs_main); CNodeState *state = State(pfrom->GetId()); @@ -3243,7 +3274,7 @@ bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vR } // Ignore unknown commands for extensibility - LogPrint(BCLog::NET, "Unknown command \"%s\" from peer=%d\n", SanitizeString(strCommand), pfrom->GetId()); + LogPrint(BCLog::NET, "Unknown command \"%s\" from peer=%d\n", SanitizeString(msg_type), pfrom->GetId()); return true; } @@ -3338,7 +3369,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter LogPrint(BCLog::NET, "PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(msg.m_command), pfrom->GetId()); return fMoreWork; } - const std::string& strCommand = msg.m_command; + const std::string& msg_type = msg.m_command; // Message size unsigned int nMessageSize = msg.m_message_size; @@ -3348,7 +3379,7 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter if (!msg.m_valid_checksum) { LogPrint(BCLog::NET, "%s(%s, %u bytes): CHECKSUM ERROR peer=%d\n", __func__, - SanitizeString(strCommand), nMessageSize, pfrom->GetId()); + SanitizeString(msg_type), nMessageSize, pfrom->GetId()); return fMoreWork; } @@ -3356,19 +3387,19 @@ bool PeerLogicValidation::ProcessMessages(CNode* pfrom, std::atomic<bool>& inter bool fRet = false; try { - fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.m_time, chainparams, m_mempool, connman, m_banman, interruptMsgProc); + fRet = ProcessMessage(pfrom, msg_type, vRecv, msg.m_time, chainparams, m_mempool, connman, m_banman, interruptMsgProc); if (interruptMsgProc) return false; if (!pfrom->vRecvGetData.empty()) fMoreWork = true; } catch (const std::exception& e) { - LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' (%s) caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what(), typeid(e).name()); + LogPrint(BCLog::NET, "%s(%s, %u bytes): Exception '%s' (%s) caught\n", __func__, SanitizeString(msg_type), nMessageSize, e.what(), typeid(e).name()); } catch (...) { - LogPrint(BCLog::NET, "%s(%s, %u bytes): Unknown exception caught\n", __func__, SanitizeString(strCommand), nMessageSize); + LogPrint(BCLog::NET, "%s(%s, %u bytes): Unknown exception caught\n", __func__, SanitizeString(msg_type), nMessageSize); } if (!fRet) { - LogPrint(BCLog::NET, "%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->GetId()); + LogPrint(BCLog::NET, "%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(msg_type), nMessageSize, pfrom->GetId()); } LOCK(cs_main); @@ -3583,16 +3614,16 @@ bool PeerLogicValidation::SendMessages(CNode* pto) int64_t nNow = GetTimeMicros(); auto current_time = GetTime<std::chrono::microseconds>(); - if (pto->IsAddrRelayPeer() && !::ChainstateActive().IsInitialBlockDownload() && pto->nNextLocalAddrSend < nNow) { + if (pto->IsAddrRelayPeer() && !::ChainstateActive().IsInitialBlockDownload() && pto->m_next_local_addr_send < current_time) { AdvertiseLocal(pto); - pto->nNextLocalAddrSend = PoissonNextSend(nNow, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); + pto->m_next_local_addr_send = PoissonNextSend(current_time, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); } // // Message: addr // - if (pto->IsAddrRelayPeer() && pto->nNextAddrSend < nNow) { - pto->nNextAddrSend = PoissonNextSend(nNow, AVG_ADDRESS_BROADCAST_INTERVAL); + if (pto->IsAddrRelayPeer() && pto->m_next_addr_send < current_time) { + pto->m_next_addr_send = PoissonNextSend(current_time, AVG_ADDRESS_BROADCAST_INTERVAL); std::vector<CAddress> vAddr; vAddr.reserve(pto->vAddrToSend.size()); assert(pto->m_addr_known); diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 228caf74a9..6adb171abd 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/netaddress.h b/src/netaddress.h index b7381c1eb4..d0ab770379 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index 641b2a5d9c..ec52a08ace 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/node/context.cpp b/src/node/context.cpp index 5b19a41bd4..0238aab0d9 100644 --- a/src/node/context.cpp +++ b/src/node/context.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/node/context.h b/src/node/context.h index 1c592b456b..566ff170be 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,6 +8,7 @@ #include <memory> #include <vector> +class ArgsManager; class BanMan; class CConnman; class CScheduler; @@ -33,6 +34,7 @@ struct NodeContext { CTxMemPool* mempool{nullptr}; // Currently a raw pointer because the memory is not managed by this struct std::unique_ptr<PeerLogicValidation> peer_logic; std::unique_ptr<BanMan> banman; + ArgsManager* args{nullptr}; // Currently a raw pointer because the memory is not managed by this struct std::unique_ptr<interfaces::Chain> chain; std::vector<std::unique_ptr<interfaces::ChainClient>> chain_clients; std::unique_ptr<CScheduler> scheduler; diff --git a/src/node/psbt.cpp b/src/node/psbt.cpp index 5b16035f7d..c189018268 100644 --- a/src/node/psbt.cpp +++ b/src/node/psbt.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/node/transaction.h b/src/node/transaction.h index a85dfb8ace..6491700d44 100644 --- a/src/node/transaction.h +++ b/src/node/transaction.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,11 +6,19 @@ #define BITCOIN_NODE_TRANSACTION_H #include <attributes.h> +#include <policy/feerate.h> #include <primitives/transaction.h> #include <util/error.h> struct NodeContext; +/** Maximum fee rate for sendrawtransaction and testmempoolaccept RPC calls. + * Also used by the GUI when broadcasting a completed PSBT. + * By default, a transaction with a fee rate higher than this will be rejected + * by these RPCs and the GUI. This can be overridden with the maxfeerate argument. + */ +static const CFeeRate DEFAULT_MAX_RAW_TX_FEE_RATE{COIN / 10}; + /** * Submit a transaction to the mempool and (optionally) relay it to all P2P peers. * diff --git a/src/outputtype.cpp b/src/outputtype.cpp index 71b5cba01c..ea7a86d6d6 100644 --- a/src/outputtype.cpp +++ b/src/outputtype.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/prevector.h b/src/prevector.h index 6d690e7f96..9f2f7ba2de 100644 --- a/src/prevector.h +++ b/src/prevector.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index 6e72c1f15c..e6183cf2f4 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/protocol.cpp b/src/protocol.cpp index bd3ed25a8a..a3e844e35b 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/psbt.cpp b/src/psbt.cpp index e6b6285652..ef9781817a 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/psbt.h b/src/psbt.h index dfba261961..af57994f3a 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -40,6 +40,10 @@ static constexpr uint8_t PSBT_OUT_BIP32_DERIVATION = 0x02; // as a 0 length key which indicates that this is the separator. The separator has no value. static constexpr uint8_t PSBT_SEPARATOR = 0x00; +// BIP 174 does not specify a maximum file size, but we set a limit anyway +// to prevent reading a stream indefinately and running out of memory. +const std::streamsize MAX_FILE_SIZE_PSBT = 100000000; // 100 MiB + /** A structure for PSBTs which contain per-input information */ struct PSBTInput { diff --git a/src/pubkey.h b/src/pubkey.h index 2fc92c9bc6..261842b7f7 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -219,6 +219,11 @@ struct CExtPubKey { a.pubkey == b.pubkey; } + friend bool operator!=(const CExtPubKey &a, const CExtPubKey &b) + { + return !(a == b); + } + void Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const; void Decode(const unsigned char code[BIP32_EXTKEY_SIZE]); bool Derive(CExtPubKey& out, unsigned int nChild) const; diff --git a/src/qt/addresstablemodel.cpp b/src/qt/addresstablemodel.cpp index 3869318fea..8110f4e895 100644 --- a/src/qt/addresstablemodel.cpp +++ b/src/qt/addresstablemodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/bantablemodel.cpp b/src/qt/bantablemodel.cpp index 72f16bb09f..2739b21a9d 100644 --- a/src/qt/bantablemodel.cpp +++ b/src/qt/bantablemodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/bantablemodel.h b/src/qt/bantablemodel.h index f01c506a1e..88d5b5811b 100644 --- a/src/qt/bantablemodel.h +++ b/src/qt/bantablemodel.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 4313d6ee7f..8939b566f7 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -336,7 +336,7 @@ void BitcoinApplication::initializeResult(bool success) window->setClientModel(clientModel); #ifdef ENABLE_WALLET if (WalletModel::isWalletEnabled()) { - m_wallet_controller = new WalletController(m_node, platformStyle, optionsModel, this); + m_wallet_controller = new WalletController(*clientModel, platformStyle, this); window->setWalletController(m_wallet_controller); if (paymentServer) { paymentServer->setOptionsModel(optionsModel); @@ -381,7 +381,7 @@ void BitcoinApplication::shutdownResult() void BitcoinApplication::handleRunawayException(const QString &message) { - QMessageBox::critical(nullptr, "Runaway exception", BitcoinGUI::tr("A fatal error occurred. Bitcoin can no longer continue safely and will quit.") + QString("\n\n") + message); + QMessageBox::critical(nullptr, "Runaway exception", BitcoinGUI::tr("A fatal error occurred. %1 can no longer continue safely and will quit.").arg(PACKAGE_NAME) + QString("\n\n") + message); ::exit(EXIT_FAILURE); } diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 75f3e9bf45..71e4d59ea7 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -317,6 +317,8 @@ void BitcoinGUI::createActions() signMessageAction->setStatusTip(tr("Sign messages with your Bitcoin addresses to prove you own them")); verifyMessageAction = new QAction(tr("&Verify message..."), this); verifyMessageAction->setStatusTip(tr("Verify messages to ensure they were signed with specified Bitcoin addresses")); + m_load_psbt_action = new QAction(tr("Load PSBT..."), this); + m_load_psbt_action->setStatusTip(tr("Load Partially Signed Bitcoin Transaction")); openRPCConsoleAction = new QAction(tr("Node window"), this); openRPCConsoleAction->setStatusTip(tr("Open node debugging and diagnostic console")); @@ -366,6 +368,7 @@ void BitcoinGUI::createActions() connect(changePassphraseAction, &QAction::triggered, walletFrame, &WalletFrame::changePassphrase); connect(signMessageAction, &QAction::triggered, [this]{ showNormalIfMinimized(); }); connect(signMessageAction, &QAction::triggered, [this]{ gotoSignMessageTab(); }); + connect(m_load_psbt_action, &QAction::triggered, [this]{ gotoLoadPSBT(); }); connect(verifyMessageAction, &QAction::triggered, [this]{ showNormalIfMinimized(); }); connect(verifyMessageAction, &QAction::triggered, [this]{ gotoVerifyMessageTab(); }); connect(usedSendingAddressesAction, &QAction::triggered, walletFrame, &WalletFrame::usedSendingAddresses); @@ -438,6 +441,7 @@ void BitcoinGUI::createMenuBar() file->addAction(backupWalletAction); file->addAction(signMessageAction); file->addAction(verifyMessageAction); + file->addAction(m_load_psbt_action); file->addSeparator(); } file->addAction(quitAction); @@ -854,6 +858,10 @@ void BitcoinGUI::gotoVerifyMessageTab(QString addr) { if (walletFrame) walletFrame->gotoVerifyMessageTab(addr); } +void BitcoinGUI::gotoLoadPSBT() +{ + if (walletFrame) walletFrame->gotoLoadPSBT(); +} #endif // ENABLE_WALLET void BitcoinGUI::updateNetworkState() diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 809cf8b4ed..70366e12d1 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -135,6 +135,7 @@ private: QAction* usedReceivingAddressesAction = nullptr; QAction* signMessageAction = nullptr; QAction* verifyMessageAction = nullptr; + QAction* m_load_psbt_action = nullptr; QAction* aboutAction = nullptr; QAction* receiveCoinsAction = nullptr; QAction* receiveCoinsMenuAction = nullptr; @@ -270,6 +271,8 @@ public Q_SLOTS: void gotoSignMessageTab(QString addr = ""); /** Show Sign/Verify Message dialog and switch to verify message tab */ void gotoVerifyMessageTab(QString addr = ""); + /** Show load Partially Signed Bitcoin Transaction dialog */ + void gotoLoadPSBT(); /** Show open dialog */ void openClicked(); diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index a1ec3eaab1..b94fcc9865 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -105,6 +105,14 @@ int64_t ClientModel::getHeaderTipTime() const return cachedBestHeaderTime; } +int ClientModel::getNumBlocks() const +{ + if (m_cached_num_blocks == -1) { + m_cached_num_blocks = m_node.getNumBlocks(); + } + return m_cached_num_blocks; +} + void ClientModel::updateNumConnections(int numConnections) { Q_EMIT numConnectionsChanged(numConnections); @@ -241,6 +249,8 @@ static void BlockTipChanged(ClientModel *clientmodel, bool initialSync, int heig // cache best headers time and height to reduce future cs_main locks clientmodel->cachedBestHeaderHeight = height; clientmodel->cachedBestHeaderTime = blockTime; + } else { + clientmodel->m_cached_num_blocks = height; } // During initial sync, block notifications, and header notifications from reindexing are both throttled. diff --git a/src/qt/clientmodel.h b/src/qt/clientmodel.h index 79175e0af4..7ac4120a8f 100644 --- a/src/qt/clientmodel.h +++ b/src/qt/clientmodel.h @@ -56,6 +56,7 @@ public: //! Return number of connections, default is in- and outbound (total) int getNumConnections(unsigned int flags = CONNECTIONS_ALL) const; + int getNumBlocks() const; int getHeaderTipHeight() const; int64_t getHeaderTipTime() const; @@ -73,9 +74,10 @@ public: bool getProxyInfo(std::string& ip_port) const; - // caches for the best header + // caches for the best header, number of blocks mutable std::atomic<int> cachedBestHeaderHeight; mutable std::atomic<int64_t> cachedBestHeaderTime; + mutable std::atomic<int> m_cached_num_blocks{-1}; private: interfaces::Node& m_node; diff --git a/src/qt/coincontroltreewidget.h b/src/qt/coincontroltreewidget.h index 39dc9a5e9e..86af555385 100644 --- a/src/qt/coincontroltreewidget.h +++ b/src/qt/coincontroltreewidget.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/createwalletdialog.cpp b/src/qt/createwalletdialog.cpp index 8e6474b0d4..5056e487fc 100644 --- a/src/qt/createwalletdialog.cpp +++ b/src/qt/createwalletdialog.cpp @@ -60,3 +60,8 @@ bool CreateWalletDialog::isMakeBlankWalletChecked() const { return ui->blank_wallet_checkbox->isChecked(); } + +bool CreateWalletDialog::isDescriptorWalletChecked() const +{ + return ui->descriptor_checkbox->isChecked(); +} diff --git a/src/qt/createwalletdialog.h b/src/qt/createwalletdialog.h index 30766107b9..20cce937c8 100644 --- a/src/qt/createwalletdialog.h +++ b/src/qt/createwalletdialog.h @@ -27,6 +27,7 @@ public: bool isEncryptWalletChecked() const; bool isDisablePrivateKeysChecked() const; bool isMakeBlankWalletChecked() const; + bool isDescriptorWalletChecked() const; private: Ui::CreateWalletDialog *ui; diff --git a/src/qt/csvmodelwriter.h b/src/qt/csvmodelwriter.h index e443529335..ad247b6859 100644 --- a/src/qt/csvmodelwriter.h +++ b/src/qt/csvmodelwriter.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/forms/createwalletdialog.ui b/src/qt/forms/createwalletdialog.ui index e49bab8f3b..b592140dd7 100644 --- a/src/qt/forms/createwalletdialog.ui +++ b/src/qt/forms/createwalletdialog.ui @@ -7,7 +7,7 @@ <x>0</x> <y>0</y> <width>364</width> - <height>185</height> + <height>213</height> </rect> </property> <property name="windowTitle"> @@ -17,7 +17,7 @@ <property name="geometry"> <rect> <x>10</x> - <y>140</y> + <y>170</y> <width>341</width> <height>32</height> </rect> @@ -106,6 +106,22 @@ <string>Make Blank Wallet</string> </property> </widget> + <widget class="QCheckBox" name="descriptor_checkbox"> + <property name="geometry"> + <rect> + <x>20</x> + <y>140</y> + <width>171</width> + <height>22</height> + </rect> + </property> + <property name="toolTip"> + <string>Use descriptors for scriptPubKey management</string> + </property> + <property name="text"> + <string>Descriptor Wallet</string> + </property> + </widget> </widget> <tabstops> <tabstop>wallet_name_line_edit</tabstop> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 98dde1656a..a54b65961c 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/macnotificationhandler.mm b/src/qt/macnotificationhandler.mm index b16042e946..2d26a8383d 100644 --- a/src/qt/macnotificationhandler.mm +++ b/src/qt/macnotificationhandler.mm @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/main.cpp b/src/qt/main.cpp index 3dfd9e850e..6e772d58c5 100644 --- a/src/qt/main.cpp +++ b/src/qt/main.cpp @@ -1,10 +1,11 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <qt/bitcoin.h> #include <util/translation.h> +#include <util/url.h> #include <QCoreApplication> @@ -15,5 +16,6 @@ extern const std::function<std::string(const char*)> G_TRANSLATION_FUN = [](const char* psz) { return QCoreApplication::translate("bitcoin-core", psz).toStdString(); }; +UrlDecodeFn* const URL_DECODE = urlDecode; int main(int argc, char* argv[]) { return GuiMain(argc, argv); } diff --git a/src/qt/modaloverlay.cpp b/src/qt/modaloverlay.cpp index 6243a71c7d..49a1992803 100644 --- a/src/qt/modaloverlay.cpp +++ b/src/qt/modaloverlay.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index c376921b72..e0ae1f9e92 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -161,20 +161,27 @@ void OverviewPage::setBalance(const interfaces::WalletBalances& balances) { int unit = walletModel->getOptionsModel()->getDisplayUnit(); m_balances = balances; - if (walletModel->wallet().privateKeysDisabled()) { - ui->labelBalance->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelUnconfirmed->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance + balances.unconfirmed_watch_only_balance + balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); + if (walletModel->wallet().isLegacy()) { + if (walletModel->wallet().privateKeysDisabled()) { + ui->labelBalance->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance, false, BitcoinUnits::separatorAlways)); + ui->labelUnconfirmed->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_watch_only_balance, false, BitcoinUnits::separatorAlways)); + ui->labelImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); + ui->labelTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance + balances.unconfirmed_watch_only_balance + balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); + } else { + ui->labelBalance->setText(BitcoinUnits::formatWithUnit(unit, balances.balance, false, BitcoinUnits::separatorAlways)); + ui->labelUnconfirmed->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_balance, false, BitcoinUnits::separatorAlways)); + ui->labelImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_balance, false, BitcoinUnits::separatorAlways)); + ui->labelTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.balance + balances.unconfirmed_balance + balances.immature_balance, false, BitcoinUnits::separatorAlways)); + ui->labelWatchAvailable->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance, false, BitcoinUnits::separatorAlways)); + ui->labelWatchPending->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_watch_only_balance, false, BitcoinUnits::separatorAlways)); + ui->labelWatchImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); + ui->labelWatchTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance + balances.unconfirmed_watch_only_balance + balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); + } } else { ui->labelBalance->setText(BitcoinUnits::formatWithUnit(unit, balances.balance, false, BitcoinUnits::separatorAlways)); ui->labelUnconfirmed->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_balance, false, BitcoinUnits::separatorAlways)); ui->labelImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_balance, false, BitcoinUnits::separatorAlways)); ui->labelTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.balance + balances.unconfirmed_balance + balances.immature_balance, false, BitcoinUnits::separatorAlways)); - ui->labelWatchAvailable->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelWatchPending->setText(BitcoinUnits::formatWithUnit(unit, balances.unconfirmed_watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelWatchImmature->setText(BitcoinUnits::formatWithUnit(unit, balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); - ui->labelWatchTotal->setText(BitcoinUnits::formatWithUnit(unit, balances.watch_only_balance + balances.unconfirmed_watch_only_balance + balances.immature_watch_only_balance, false, BitcoinUnits::separatorAlways)); } // only show immature (newly mined) balance if it's non-zero, so as not to complicate things // for the non-mining users diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index a1fc791536..5220f8e138 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h index cf45c5a08f..0e20b1d9dc 100644 --- a/src/qt/peertablemodel.h +++ b/src/qt/peertablemodel.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/qvaluecombobox.h b/src/qt/qvaluecombobox.h index 5cca515079..bde9c0d1a6 100644 --- a/src/qt/qvaluecombobox.h +++ b/src/qt/qvaluecombobox.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/receivecoinsdialog.cpp b/src/qt/receivecoinsdialog.cpp index 180550c5ae..5debded4ea 100644 --- a/src/qt/receivecoinsdialog.cpp +++ b/src/qt/receivecoinsdialog.cpp @@ -157,17 +157,40 @@ void ReceiveCoinsDialog::on_receiveButton_clicked() } } address = model->getAddressTableModel()->addRow(AddressTableModel::Receive, label, "", address_type); - SendCoinsRecipient info(address, label, - ui->reqAmount->value(), ui->reqMessage->text()); - ReceiveRequestDialog *dialog = new ReceiveRequestDialog(this); - dialog->setAttribute(Qt::WA_DeleteOnClose); - dialog->setModel(model); - dialog->setInfo(info); - dialog->show(); - clear(); - /* Store request for later reference */ - model->getRecentRequestsTableModel()->addNewRequest(info); + switch(model->getAddressTableModel()->getEditStatus()) + { + case AddressTableModel::EditStatus::OK: { + // Success + SendCoinsRecipient info(address, label, + ui->reqAmount->value(), ui->reqMessage->text()); + ReceiveRequestDialog *dialog = new ReceiveRequestDialog(this); + dialog->setAttribute(Qt::WA_DeleteOnClose); + dialog->setModel(model); + dialog->setInfo(info); + dialog->show(); + + /* Store request for later reference */ + model->getRecentRequestsTableModel()->addNewRequest(info); + break; + } + case AddressTableModel::EditStatus::WALLET_UNLOCK_FAILURE: + QMessageBox::critical(this, windowTitle(), + tr("Could not unlock wallet."), + QMessageBox::Ok, QMessageBox::Ok); + break; + case AddressTableModel::EditStatus::KEY_GENERATION_FAILURE: + QMessageBox::critical(this, windowTitle(), + tr("Could not generate new %1 address").arg(QString::fromStdString(FormatOutputType(address_type))), + QMessageBox::Ok, QMessageBox::Ok); + break; + // These aren't valid return values for our action + case AddressTableModel::EditStatus::INVALID_ADDRESS: + case AddressTableModel::EditStatus::DUPLICATE_ADDRESS: + case AddressTableModel::EditStatus::NO_CHANGES: + assert(false); + } + clear(); } void ReceiveCoinsDialog::on_recentRequestsView_doubleClicked(const QModelIndex &index) diff --git a/src/qt/res/icons/bitcoin.ico b/src/qt/res/icons/bitcoin.ico Binary files differindex 8f5045015d..8f5045015d 100755..100644 --- a/src/qt/res/icons/bitcoin.ico +++ b/src/qt/res/icons/bitcoin.ico diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 0ffdc892c5..18a7e9b021 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index a8c82aaf6c..7b389a48d6 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -219,11 +219,8 @@ SendCoinsDialog::~SendCoinsDialog() delete ui; } -void SendCoinsDialog::on_sendButton_clicked() +bool SendCoinsDialog::PrepareSendText(QString& question_string, QString& informative_text, QString& detailed_text) { - if(!model || !model->getOptionsModel()) - return; - QList<SendCoinsRecipient> recipients; bool valid = true; @@ -246,7 +243,7 @@ void SendCoinsDialog::on_sendButton_clicked() if(!valid || recipients.isEmpty()) { - return; + return false; } fNewRecipientAllowed = false; @@ -255,11 +252,11 @@ void SendCoinsDialog::on_sendButton_clicked() { // Unlock wallet was cancelled fNewRecipientAllowed = true; - return; + return false; } // prepare transaction for getting txFee earlier - WalletModelTransaction currentTransaction(recipients); + m_current_transaction = MakeUnique<WalletModelTransaction>(recipients); WalletModel::SendCoinsReturn prepareStatus; // Always use a CCoinControl instance, use the CoinControlDialog instance if CoinControl has been enabled @@ -269,22 +266,20 @@ void SendCoinsDialog::on_sendButton_clicked() updateCoinControlState(ctrl); - prepareStatus = model->prepareTransaction(currentTransaction, ctrl); + prepareStatus = model->prepareTransaction(*m_current_transaction, ctrl); // process prepareStatus and on error generate message shown to user processSendCoinsReturn(prepareStatus, - BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), currentTransaction.getTransactionFee())); + BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), m_current_transaction->getTransactionFee())); if(prepareStatus.status != WalletModel::OK) { fNewRecipientAllowed = true; - return; + return false; } - CAmount txFee = currentTransaction.getTransactionFee(); - - // Format confirmation message + CAmount txFee = m_current_transaction->getTransactionFee(); QStringList formatted; - for (const SendCoinsRecipient &rcp : currentTransaction.getRecipients()) + for (const SendCoinsRecipient &rcp : m_current_transaction->getRecipients()) { // generate amount string with wallet name in case of multiwallet QString amount = BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), rcp.amount); @@ -311,72 +306,82 @@ void SendCoinsDialog::on_sendButton_clicked() formatted.append(recipientElement); } - QString questionString; if (model->wallet().privateKeysDisabled()) { - questionString.append(tr("Do you want to draft this transaction?")); + question_string.append(tr("Do you want to draft this transaction?")); } else { - questionString.append(tr("Are you sure you want to send?")); + question_string.append(tr("Are you sure you want to send?")); } - questionString.append("<br /><span style='font-size:10pt;'>"); + question_string.append("<br /><span style='font-size:10pt;'>"); if (model->wallet().privateKeysDisabled()) { - questionString.append(tr("Please, review your transaction proposal. This will produce a Partially Signed Bitcoin Transaction (PSBT) which you can copy and then sign with e.g. an offline %1 wallet, or a PSBT-compatible hardware wallet.").arg(PACKAGE_NAME)); + question_string.append(tr("Please, review your transaction proposal. This will produce a Partially Signed Bitcoin Transaction (PSBT) which you can save or copy and then sign with e.g. an offline %1 wallet, or a PSBT-compatible hardware wallet.").arg(PACKAGE_NAME)); } else { - questionString.append(tr("Please, review your transaction.")); + question_string.append(tr("Please, review your transaction.")); } - questionString.append("</span>%1"); + question_string.append("</span>%1"); if(txFee > 0) { // append fee string if a fee is required - questionString.append("<hr /><b>"); - questionString.append(tr("Transaction fee")); - questionString.append("</b>"); + question_string.append("<hr /><b>"); + question_string.append(tr("Transaction fee")); + question_string.append("</b>"); // append transaction size - questionString.append(" (" + QString::number((double)currentTransaction.getTransactionSize() / 1000) + " kB): "); + question_string.append(" (" + QString::number((double)m_current_transaction->getTransactionSize() / 1000) + " kB): "); // append transaction fee value - questionString.append("<span style='color:#aa0000; font-weight:bold;'>"); - questionString.append(BitcoinUnits::formatHtmlWithUnit(model->getOptionsModel()->getDisplayUnit(), txFee)); - questionString.append("</span><br />"); + question_string.append("<span style='color:#aa0000; font-weight:bold;'>"); + question_string.append(BitcoinUnits::formatHtmlWithUnit(model->getOptionsModel()->getDisplayUnit(), txFee)); + question_string.append("</span><br />"); // append RBF message according to transaction's signalling - questionString.append("<span style='font-size:10pt; font-weight:normal;'>"); + question_string.append("<span style='font-size:10pt; font-weight:normal;'>"); if (ui->optInRBF->isChecked()) { - questionString.append(tr("You can increase the fee later (signals Replace-By-Fee, BIP-125).")); + question_string.append(tr("You can increase the fee later (signals Replace-By-Fee, BIP-125).")); } else { - questionString.append(tr("Not signalling Replace-By-Fee, BIP-125.")); + question_string.append(tr("Not signalling Replace-By-Fee, BIP-125.")); } - questionString.append("</span>"); + question_string.append("</span>"); } // add total amount in all subdivision units - questionString.append("<hr />"); - CAmount totalAmount = currentTransaction.getTotalTransactionAmount() + txFee; + question_string.append("<hr />"); + CAmount totalAmount = m_current_transaction->getTotalTransactionAmount() + txFee; QStringList alternativeUnits; for (const BitcoinUnits::Unit u : BitcoinUnits::availableUnits()) { if(u != model->getOptionsModel()->getDisplayUnit()) alternativeUnits.append(BitcoinUnits::formatHtmlWithUnit(u, totalAmount)); } - questionString.append(QString("<b>%1</b>: <b>%2</b>").arg(tr("Total Amount")) + question_string.append(QString("<b>%1</b>: <b>%2</b>").arg(tr("Total Amount")) .arg(BitcoinUnits::formatHtmlWithUnit(model->getOptionsModel()->getDisplayUnit(), totalAmount))); - questionString.append(QString("<br /><span style='font-size:10pt; font-weight:normal;'>(=%1)</span>") + question_string.append(QString("<br /><span style='font-size:10pt; font-weight:normal;'>(=%1)</span>") .arg(alternativeUnits.join(" " + tr("or") + " "))); - QString informative_text; - QString detailed_text; if (formatted.size() > 1) { - questionString = questionString.arg(""); + question_string = question_string.arg(""); informative_text = tr("To review recipient list click \"Show Details...\""); detailed_text = formatted.join("\n\n"); } else { - questionString = questionString.arg("<br /><br />" + formatted.at(0)); + question_string = question_string.arg("<br /><br />" + formatted.at(0)); } + + return true; +} + +void SendCoinsDialog::on_sendButton_clicked() +{ + if(!model || !model->getOptionsModel()) + return; + + QString question_string, informative_text, detailed_text; + if (!PrepareSendText(question_string, informative_text, detailed_text)) return; + assert(m_current_transaction); + const QString confirmation = model->wallet().privateKeysDisabled() ? tr("Confirm transaction proposal") : tr("Confirm send coins"); - const QString confirmButtonText = model->wallet().privateKeysDisabled() ? tr("Copy PSBT to clipboard") : tr("Send"); - SendConfirmationDialog confirmationDialog(confirmation, questionString, informative_text, detailed_text, SEND_CONFIRM_DELAY, confirmButtonText, this); + const QString confirmButtonText = model->wallet().privateKeysDisabled() ? tr("Create Unsigned") : tr("Send"); + SendConfirmationDialog confirmationDialog(confirmation, question_string, informative_text, detailed_text, SEND_CONFIRM_DELAY, confirmButtonText, this); confirmationDialog.exec(); QMessageBox::StandardButton retval = static_cast<QMessageBox::StandardButton>(confirmationDialog.result()); @@ -388,7 +393,7 @@ void SendCoinsDialog::on_sendButton_clicked() bool send_failure = false; if (model->wallet().privateKeysDisabled()) { - CMutableTransaction mtx = CMutableTransaction{*(currentTransaction.getWtx())}; + CMutableTransaction mtx = CMutableTransaction{*(m_current_transaction->getWtx())}; PartiallySignedTransaction psbtx(mtx); bool complete = false; const TransactionError err = model->wallet().fillPSBT(SIGHASH_ALL, false /* sign */, true /* bip32derivs */, psbtx, complete); @@ -398,15 +403,51 @@ void SendCoinsDialog::on_sendButton_clicked() CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; GUIUtil::setClipboard(EncodeBase64(ssTx.str()).c_str()); - Q_EMIT message(tr("PSBT copied"), "Copied to clipboard", CClientUIInterface::MSG_INFORMATION); + QMessageBox msgBox; + msgBox.setText("Unsigned Transaction"); + msgBox.setInformativeText("The PSBT has been copied to the clipboard. You can also save it."); + msgBox.setStandardButtons(QMessageBox::Save | QMessageBox::Discard); + msgBox.setDefaultButton(QMessageBox::Discard); + switch (msgBox.exec()) { + case QMessageBox::Save: { + QString selectedFilter; + QString fileNameSuggestion = ""; + bool first = true; + for (const SendCoinsRecipient &rcp : m_current_transaction->getRecipients()) { + if (!first) { + fileNameSuggestion.append(" - "); + } + QString labelOrAddress = rcp.label.isEmpty() ? rcp.address : rcp.label; + QString amount = BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), rcp.amount); + fileNameSuggestion.append(labelOrAddress + "-" + amount); + first = false; + } + fileNameSuggestion.append(".psbt"); + QString filename = GUIUtil::getSaveFileName(this, + tr("Save Transaction Data"), fileNameSuggestion, + tr("Partially Signed Transaction (Binary) (*.psbt)"), &selectedFilter); + if (filename.isEmpty()) { + return; + } + std::ofstream out(filename.toLocal8Bit().data()); + out << ssTx.str(); + out.close(); + Q_EMIT message(tr("PSBT saved"), "PSBT saved to disk", CClientUIInterface::MSG_INFORMATION); + break; + } + case QMessageBox::Discard: + break; + default: + assert(false); + } } else { // now send the prepared transaction - WalletModel::SendCoinsReturn sendStatus = model->sendCoins(currentTransaction); + WalletModel::SendCoinsReturn sendStatus = model->sendCoins(*m_current_transaction); // process sendStatus and on error generate message shown to user processSendCoinsReturn(sendStatus); if (sendStatus.status == WalletModel::OK) { - Q_EMIT coinsSent(currentTransaction.getWtx()->GetHash()); + Q_EMIT coinsSent(m_current_transaction->getWtx()->GetHash()); } else { send_failure = true; } @@ -417,10 +458,13 @@ void SendCoinsDialog::on_sendButton_clicked() coinControlUpdateLabels(); } fNewRecipientAllowed = true; + m_current_transaction.reset(); } void SendCoinsDialog::clear() { + m_current_transaction.reset(); + // Clear coin control settings CoinControlDialog::coinControl()->UnSelectAll(); ui->checkBoxCoinControlChange->setChecked(false); diff --git a/src/qt/sendcoinsdialog.h b/src/qt/sendcoinsdialog.h index 86422c4030..36bc2a846b 100644 --- a/src/qt/sendcoinsdialog.h +++ b/src/qt/sendcoinsdialog.h @@ -60,6 +60,7 @@ private: Ui::SendCoinsDialog *ui; ClientModel *clientModel; WalletModel *model; + std::unique_ptr<WalletModelTransaction> m_current_transaction; bool fNewRecipientAllowed; bool fFeeMinimized; const PlatformStyle *platformStyle; @@ -69,6 +70,8 @@ private: // Additional parameter msgArg can be used via .arg(msgArg). void processSendCoinsReturn(const WalletModel::SendCoinsReturn &sendCoinsReturn, const QString &msgArg = QString()); void minimizeFeeSection(bool fMinimize); + // Format confirmation message + bool PrepareSendText(QString& question_string, QString& informative_text, QString& detailed_text); void updateFeeMinimizedLabel(); // Update the passed in CCoinControl with state from the GUI void updateCoinControlState(CCoinControl& ctrl); diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index 4552753bf6..67c2c71e9c 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index e4ffa6cd9a..d8fe4a9238 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 042387286a..476128520c 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,6 +8,7 @@ #include <interfaces/chain.h> #include <interfaces/node.h> +#include <qt/clientmodel.h> #include <qt/editaddressdialog.h> #include <qt/optionsmodel.h> #include <qt/platformstyle.h> @@ -106,8 +107,9 @@ void TestAddAddressesToSendBook(interfaces::Node& node) // Initialize relevant QT models. std::unique_ptr<const PlatformStyle> platformStyle(PlatformStyle::instantiate("other")); OptionsModel optionsModel(node); + ClientModel clientModel(node, &optionsModel); AddWallet(wallet); - WalletModel walletModel(interfaces::MakeWallet(wallet), node, platformStyle.get(), &optionsModel); + WalletModel walletModel(interfaces::MakeWallet(wallet), clientModel, platformStyle.get()); RemoveWallet(wallet); EditAddressDialog editAddressDialog(EditAddressDialog::NewSendingAddress); editAddressDialog.setModel(walletModel.getAddressTableModel()); diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index 14a75b23f3..f88d57c716 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -82,6 +82,7 @@ void AppTests::appTests() // Reset global state to avoid interfering with later tests. AbortShutdown(); UnloadBlockIndex(); + WITH_LOCK(::cs_main, g_chainman.Reset()); } //! Entry point for BitcoinGUI tests. diff --git a/src/qt/test/apptests.h b/src/qt/test/apptests.h index d16c9fe487..f0a555005d 100644 --- a/src/qt/test/apptests.h +++ b/src/qt/test/apptests.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/util.cpp b/src/qt/test/util.cpp index e09f0ad77d..759832381b 100644 --- a/src/qt/test/util.cpp +++ b/src/qt/test/util.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/util.h b/src/qt/test/util.h index 763847606a..950314da3b 100644 --- a/src/qt/test/util.h +++ b/src/qt/test/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index c1a0f63f73..94a1e0a63d 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,6 +8,7 @@ #include <interfaces/chain.h> #include <interfaces/node.h> #include <qt/bitcoinamountfield.h> +#include <qt/clientmodel.h> #include <qt/optionsmodel.h> #include <qt/platformstyle.h> #include <qt/qvalidatedlineedit.h> @@ -151,12 +152,9 @@ void TestGUI(interfaces::Node& node) wallet->SetLastBlockProcessed(105, ::ChainActive().Tip()->GetBlockHash()); } { - auto locked_chain = wallet->chain().lock(); - LockAssertion lock(::cs_main); - - WalletRescanReserver reserver(wallet.get()); + WalletRescanReserver reserver(*wallet); reserver.reserve(); - CWallet::ScanResult result = wallet->ScanForWalletTransactions(locked_chain->getBlockHash(0), {} /* stop_block */, reserver, true /* fUpdate */); + CWallet::ScanResult result = wallet->ScanForWalletTransactions(Params().GetConsensus().hashGenesisBlock, 0 /* block height */, {} /* max height */, reserver, true /* fUpdate */); QCOMPARE(result.status, CWallet::ScanResult::SUCCESS); QCOMPARE(result.last_scanned_block, ::ChainActive().Tip()->GetBlockHash()); QVERIFY(result.last_failed_block.IsNull()); @@ -168,8 +166,9 @@ void TestGUI(interfaces::Node& node) SendCoinsDialog sendCoinsDialog(platformStyle.get()); TransactionView transactionView(platformStyle.get()); OptionsModel optionsModel(node); + ClientModel clientModel(node, &optionsModel); AddWallet(wallet); - WalletModel walletModel(interfaces::MakeWallet(wallet), node, platformStyle.get(), &optionsModel); + WalletModel walletModel(interfaces::MakeWallet(wallet), clientModel, platformStyle.get()); RemoveWallet(wallet); sendCoinsDialog.setModel(&walletModel); transactionView.setModel(&walletModel); diff --git a/src/qt/trafficgraphwidget.h b/src/qt/trafficgraphwidget.h index af5890ba24..7e8bfb2337 100644 --- a/src/qt/trafficgraphwidget.h +++ b/src/qt/trafficgraphwidget.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactiondescdialog.h b/src/qt/transactiondescdialog.h index 74e34cde87..3204b22657 100644 --- a/src/qt/transactiondescdialog.h +++ b/src/qt/transactiondescdialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 64e9c856db..18554aef1f 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -5,6 +5,7 @@ #include <qt/transactiontablemodel.h> #include <qt/addresstablemodel.h> +#include <qt/clientmodel.h> #include <qt/guiconstants.h> #include <qt/guiutil.h> #include <qt/optionsmodel.h> @@ -175,7 +176,7 @@ public: return cachedWallet.size(); } - TransactionRecord *index(interfaces::Wallet& wallet, int idx) + TransactionRecord *index(interfaces::Wallet& wallet, const int cur_num_blocks, const int idx) { if(idx >= 0 && idx < cachedWallet.size()) { @@ -191,7 +192,7 @@ public: interfaces::WalletTxStatus wtx; int numBlocks; int64_t block_time; - if (wallet.tryGetTxStatus(rec->hash, wtx, numBlocks, block_time) && rec->statusUpdateNeeded(numBlocks)) { + if (rec->statusUpdateNeeded(cur_num_blocks) && wallet.tryGetTxStatus(rec->hash, wtx, numBlocks, block_time)) { rec->updateStatus(wtx, numBlocks, block_time); } return rec; @@ -663,10 +664,10 @@ QVariant TransactionTableModel::headerData(int section, Qt::Orientation orientat QModelIndex TransactionTableModel::index(int row, int column, const QModelIndex &parent) const { Q_UNUSED(parent); - TransactionRecord *data = priv->index(walletModel->wallet(), row); + TransactionRecord *data = priv->index(walletModel->wallet(), walletModel->clientModel().getNumBlocks(), row); if(data) { - return createIndex(row, column, priv->index(walletModel->wallet(), row)); + return createIndex(row, column, data); } return QModelIndex(); } diff --git a/src/qt/utilitydialog.cpp b/src/qt/utilitydialog.cpp index 3101fb01c3..d16feba1bb 100644 --- a/src/qt/utilitydialog.cpp +++ b/src/qt/utilitydialog.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/utilitydialog.h b/src/qt/utilitydialog.h index 833b86fd3e..b6a42d3d9d 100644 --- a/src/qt/utilitydialog.h +++ b/src/qt/utilitydialog.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2018 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index f076b5ba61..9c1488fb0e 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -1,10 +1,11 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <qt/walletcontroller.h> #include <qt/askpassphrasedialog.h> +#include <qt/clientmodel.h> #include <qt/createwalletdialog.h> #include <qt/guiconstants.h> #include <qt/guiutil.h> @@ -24,13 +25,14 @@ #include <QTimer> #include <QWindow> -WalletController::WalletController(interfaces::Node& node, const PlatformStyle* platform_style, OptionsModel* options_model, QObject* parent) +WalletController::WalletController(ClientModel& client_model, const PlatformStyle* platform_style, QObject* parent) : QObject(parent) , m_activity_thread(new QThread(this)) , m_activity_worker(new QObject) - , m_node(node) + , m_client_model(client_model) + , m_node(client_model.node()) , m_platform_style(platform_style) - , m_options_model(options_model) + , m_options_model(client_model.getOptionsModel()) { m_handler_load_wallet = m_node.handleLoadWallet([this](std::unique_ptr<interfaces::Wallet> wallet) { getOrCreateWallet(std::move(wallet)); @@ -104,7 +106,7 @@ WalletModel* WalletController::getOrCreateWallet(std::unique_ptr<interfaces::Wal } // Instantiate model and register it. - WalletModel* wallet_model = new WalletModel(std::move(wallet), m_node, m_platform_style, m_options_model, nullptr); + WalletModel* wallet_model = new WalletModel(std::move(wallet), m_client_model, m_platform_style, nullptr); // Handler callback runs in a different thread so fix wallet model thread affinity. wallet_model->moveToThread(thread()); wallet_model->setParent(this); @@ -224,6 +226,9 @@ void CreateWalletActivity::createWallet() if (m_create_wallet_dialog->isMakeBlankWalletChecked()) { flags |= WALLET_FLAG_BLANK_WALLET; } + if (m_create_wallet_dialog->isDescriptorWalletChecked()) { + flags |= WALLET_FLAG_DESCRIPTORS; + } QTimer::singleShot(500, worker(), [this, name, flags] { WalletCreationStatus status; diff --git a/src/qt/walletcontroller.h b/src/qt/walletcontroller.h index f30eb25308..3808b7d8bf 100644 --- a/src/qt/walletcontroller.h +++ b/src/qt/walletcontroller.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -21,6 +21,7 @@ #include <QTimer> #include <QString> +class ClientModel; class OptionsModel; class PlatformStyle; class WalletModel; @@ -47,7 +48,7 @@ class WalletController : public QObject void removeAndDeleteWallet(WalletModel* wallet_model); public: - WalletController(interfaces::Node& node, const PlatformStyle* platform_style, OptionsModel* options_model, QObject* parent); + WalletController(ClientModel& client_model, const PlatformStyle* platform_style, QObject* parent); ~WalletController(); //! Returns wallet models currently open. @@ -70,6 +71,7 @@ Q_SIGNALS: private: QThread* const m_activity_thread; QObject* const m_activity_worker; + ClientModel& m_client_model; interfaces::Node& m_node; const PlatformStyle* const m_platform_style; OptionsModel* const m_options_model; diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index dac3326cc4..02a9583ae9 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -163,6 +163,14 @@ void WalletFrame::gotoVerifyMessageTab(QString addr) walletView->gotoVerifyMessageTab(addr); } +void WalletFrame::gotoLoadPSBT() +{ + WalletView *walletView = currentWalletView(); + if (walletView) { + walletView->gotoLoadPSBT(); + } +} + void WalletFrame::encryptWallet(bool status) { WalletView *walletView = currentWalletView(); diff --git a/src/qt/walletframe.h b/src/qt/walletframe.h index 20fad08b0e..d90ade5005 100644 --- a/src/qt/walletframe.h +++ b/src/qt/walletframe.h @@ -78,6 +78,9 @@ public Q_SLOTS: /** Show Sign/Verify Message dialog and switch to verify message tab */ void gotoVerifyMessageTab(QString addr = ""); + /** Load Partially Signed Bitcoin Transaction */ + void gotoLoadPSBT(); + /** Encrypt the wallet */ void encryptWallet(bool status); /** Backup the wallet */ diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 608797d6ad..e7581cd64e 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,6 +9,7 @@ #include <qt/walletmodel.h> #include <qt/addresstablemodel.h> +#include <qt/clientmodel.h> #include <qt/guiconstants.h> #include <qt/guiutil.h> #include <qt/optionsmodel.h> @@ -20,6 +21,7 @@ #include <interfaces/handler.h> #include <interfaces/node.h> #include <key_io.h> +#include <psbt.h> #include <ui_interface.h> #include <util/system.h> // for GetBoolArg #include <wallet/coincontrol.h> @@ -33,8 +35,13 @@ #include <QTimer> -WalletModel::WalletModel(std::unique_ptr<interfaces::Wallet> wallet, interfaces::Node& node, const PlatformStyle *platformStyle, OptionsModel *_optionsModel, QObject *parent) : - QObject(parent), m_wallet(std::move(wallet)), m_node(node), optionsModel(_optionsModel), addressTableModel(nullptr), +WalletModel::WalletModel(std::unique_ptr<interfaces::Wallet> wallet, ClientModel& client_model, const PlatformStyle *platformStyle, QObject *parent) : + QObject(parent), + m_wallet(std::move(wallet)), + m_client_model(client_model), + m_node(client_model.node()), + optionsModel(client_model.getOptionsModel()), + addressTableModel(nullptr), transactionTableModel(nullptr), recentRequestsTableModel(nullptr), cachedEncryptionStatus(Unencrypted), diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 7936014af9..07004b7c6b 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -24,6 +24,7 @@ enum class OutputType; class AddressTableModel; +class ClientModel; class OptionsModel; class PlatformStyle; class RecentRequestsTableModel; @@ -52,7 +53,7 @@ class WalletModel : public QObject Q_OBJECT public: - explicit WalletModel(std::unique_ptr<interfaces::Wallet> wallet, interfaces::Node& node, const PlatformStyle *platformStyle, OptionsModel *optionsModel, QObject *parent = nullptr); + explicit WalletModel(std::unique_ptr<interfaces::Wallet> wallet, ClientModel& client_model, const PlatformStyle *platformStyle, QObject *parent = nullptr); ~WalletModel(); enum StatusCode // Returned by sendCoins @@ -143,6 +144,7 @@ public: interfaces::Node& node() const { return m_node; } interfaces::Wallet& wallet() const { return *m_wallet; } + ClientModel& clientModel() const { return m_client_model; } QString getWalletName() const; QString getDisplayName() const; @@ -159,6 +161,7 @@ private: std::unique_ptr<interfaces::Handler> m_handler_show_progress; std::unique_ptr<interfaces::Handler> m_handler_watch_only_changed; std::unique_ptr<interfaces::Handler> m_handler_can_get_addrs_changed; + ClientModel& m_client_model; interfaces::Node& m_node; bool fHaveWatchOnly; diff --git a/src/qt/walletview.cpp b/src/qt/walletview.cpp index bdcb82e06b..5d9b420df7 100644 --- a/src/qt/walletview.cpp +++ b/src/qt/walletview.cpp @@ -1,9 +1,12 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <qt/walletview.h> +#include <node/psbt.h> +#include <node/transaction.h> +#include <policy/policy.h> #include <qt/addressbookpage.h> #include <qt/askpassphrasedialog.h> #include <qt/clientmodel.h> @@ -20,6 +23,7 @@ #include <interfaces/node.h> #include <ui_interface.h> +#include <util/strencodings.h> #include <QAction> #include <QActionGroup> @@ -197,6 +201,80 @@ void WalletView::gotoVerifyMessageTab(QString addr) signVerifyMessageDialog->setAddress_VM(addr); } +void WalletView::gotoLoadPSBT() +{ + QString filename = GUIUtil::getOpenFileName(this, + tr("Load Transaction Data"), QString(), + tr("Partially Signed Transaction (*.psbt)"), nullptr); + if (filename.isEmpty()) return; + if (GetFileSize(filename.toLocal8Bit().data(), MAX_FILE_SIZE_PSBT) == MAX_FILE_SIZE_PSBT) { + Q_EMIT message(tr("Error"), tr("PSBT file must be smaller than 100 MiB"), CClientUIInterface::MSG_ERROR); + return; + } + std::ifstream in(filename.toLocal8Bit().data(), std::ios::binary); + std::string data(std::istreambuf_iterator<char>{in}, {}); + + std::string error; + PartiallySignedTransaction psbtx; + if (!DecodeRawPSBT(psbtx, data, error)) { + Q_EMIT message(tr("Error"), tr("Unable to decode PSBT file") + "\n" + QString::fromStdString(error), CClientUIInterface::MSG_ERROR); + return; + } + + CMutableTransaction mtx; + bool complete = false; + PSBTAnalysis analysis = AnalyzePSBT(psbtx); + QMessageBox msgBox; + msgBox.setText("PSBT"); + switch (analysis.next) { + case PSBTRole::CREATOR: + case PSBTRole::UPDATER: + msgBox.setInformativeText("PSBT is incomplete. Copy to clipboard for manual inspection?"); + break; + case PSBTRole::SIGNER: + msgBox.setInformativeText("Transaction needs more signatures. Copy to clipboard?"); + break; + case PSBTRole::FINALIZER: + case PSBTRole::EXTRACTOR: + complete = FinalizeAndExtractPSBT(psbtx, mtx); + if (complete) { + msgBox.setInformativeText(tr("Would you like to send this transaction?")); + } else { + // The analyzer missed something, e.g. if there are final_scriptSig/final_scriptWitness + // but with invalid signatures. + msgBox.setInformativeText(tr("There was an unexpected problem processing the PSBT. Copy to clipboard for manual inspection?")); + } + } + + msgBox.setStandardButtons(QMessageBox::Yes | QMessageBox::Cancel); + switch (msgBox.exec()) { + case QMessageBox::Yes: { + if (complete) { + std::string err_string; + CTransactionRef tx = MakeTransactionRef(mtx); + + TransactionError result = BroadcastTransaction(*clientModel->node().context(), tx, err_string, DEFAULT_MAX_RAW_TX_FEE_RATE.GetFeePerK(), /* relay */ true, /* wait_callback */ false); + if (result == TransactionError::OK) { + Q_EMIT message(tr("Success"), tr("Broadcasted transaction sucessfully."), CClientUIInterface::MSG_INFORMATION | CClientUIInterface::MODAL); + } else { + Q_EMIT message(tr("Error"), QString::fromStdString(err_string), CClientUIInterface::MSG_ERROR); + } + } else { + // Serialize the PSBT + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << psbtx; + GUIUtil::setClipboard(EncodeBase64(ssTx.str()).c_str()); + Q_EMIT message(tr("PSBT copied"), "Copied to clipboard", CClientUIInterface::MSG_INFORMATION); + return; + } + } + case QMessageBox::Cancel: + break; + default: + assert(false); + } +} + bool WalletView::handlePaymentRequest(const SendCoinsRecipient& recipient) { return sendCoinsPage->handlePaymentRequest(recipient); diff --git a/src/qt/walletview.h b/src/qt/walletview.h index 78d870f59f..11f894e7f8 100644 --- a/src/qt/walletview.h +++ b/src/qt/walletview.h @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -83,6 +83,8 @@ public Q_SLOTS: void gotoSignMessageTab(QString addr = ""); /** Show Sign/Verify Message dialog and switch to verify message tab */ void gotoVerifyMessageTab(QString addr = ""); + /** Load Partially Signed Bitcoin Transaction */ + void gotoLoadPSBT(); /** Show incoming transaction notification for new transactions. diff --git a/src/random.cpp b/src/random.cpp index 2a27e6ba0d..765239e1f5 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/random.h b/src/random.h index 4e4597cff6..0c5008685b 100644 --- a/src/random.h +++ b/src/random.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/randomenv.cpp b/src/randomenv.cpp index 8b3d478529..073d82b491 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f71ed99652..827b83d67e 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -375,8 +375,6 @@ static UniValue getdifficulty(const JSONRPCRequest& request) static std::vector<RPCResult> MempoolEntryDescription() { return { RPCResult{RPCResult::Type::NUM, "vsize", "virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted."}, - RPCResult{RPCResult::Type::NUM, "size", "(DEPRECATED) same as vsize. Only returned if bitcoind is started with -deprecatedrpc=size\n" - "size will be completely removed in v0.20."}, RPCResult{RPCResult::Type::NUM, "weight", "transaction weight as defined in BIP 141."}, RPCResult{RPCResult::Type::STR_AMOUNT, "fee", "transaction fee in " + CURRENCY_UNIT + " (DEPRECATED)"}, RPCResult{RPCResult::Type::STR_AMOUNT, "modifiedfee", "transaction fee with fee deltas used for mining priority (DEPRECATED)"}, @@ -415,7 +413,6 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool info.pushKV("fees", fees); info.pushKV("vsize", (int)e.GetTxSize()); - if (IsDeprecatedRPCEnabled("size")) info.pushKV("size", (int)e.GetTxSize()); info.pushKV("weight", (int)e.GetTxWeight()); info.pushKV("fee", ValueFromAmount(e.GetFee())); info.pushKV("modifiedfee", ValueFromAmount(e.GetModifiedFee())); @@ -812,7 +809,7 @@ static UniValue getblock(const JSONRPCRequest& request) "If verbosity is 2, returns an Object with information about block <hash> and information about each transaction. \n", { {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The block hash"}, - {"verbosity", RPCArg::Type::NUM, /* default */ "1", "0 for hex-encoded data, 1 for a json object, and 2 for json object with transaction data"}, + {"verbosity|verbose", RPCArg::Type::NUM, /* default */ "1", "0 for hex-encoded data, 1 for a json object, and 2 for json object with transaction data"}, }, { RPCResult{"for verbosity = 0", @@ -1078,13 +1075,11 @@ UniValue gettxout(const JSONRPCRequest& request) static UniValue verifychain(const JSONRPCRequest& request) { - int nCheckLevel = gArgs.GetArg("-checklevel", DEFAULT_CHECKLEVEL); - int nCheckDepth = gArgs.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS); RPCHelpMan{"verifychain", "\nVerifies blockchain database.\n", { - {"checklevel", RPCArg::Type::NUM, /* default */ strprintf("%d, range=0-4", nCheckLevel), "How thorough the block verification is."}, - {"nblocks", RPCArg::Type::NUM, /* default */ strprintf("%d, 0=all", nCheckDepth), "The number of blocks to check."}, + {"checklevel", RPCArg::Type::NUM, /* default */ strprintf("%d, range=0-4", DEFAULT_CHECKLEVEL), "How thorough the block verification is."}, + {"nblocks", RPCArg::Type::NUM, /* default */ strprintf("%d, 0=all", DEFAULT_CHECKBLOCKS), "The number of blocks to check."}, }, RPCResult{ RPCResult::Type::BOOL, "", "Verified or not"}, @@ -1094,15 +1089,12 @@ static UniValue verifychain(const JSONRPCRequest& request) }, }.Check(request); - LOCK(cs_main); + const int check_level(request.params[0].isNull() ? DEFAULT_CHECKLEVEL : request.params[0].get_int()); + const int check_depth{request.params[1].isNull() ? DEFAULT_CHECKBLOCKS : request.params[1].get_int()}; - if (!request.params[0].isNull()) - nCheckLevel = request.params[0].get_int(); - if (!request.params[1].isNull()) - nCheckDepth = request.params[1].get_int(); + LOCK(cs_main); - return CVerifyDB().VerifyDB( - Params(), &::ChainstateActive().CoinsTip(), nCheckLevel, nCheckDepth); + return CVerifyDB().VerifyDB(Params(), &::ChainstateActive().CoinsTip(), check_level, check_depth); } static void BuriedForkDescPushBack(UniValue& softforks, const std::string &name, int height) EXCLUSIVE_LOCKS_REQUIRED(cs_main) @@ -1319,7 +1311,7 @@ static UniValue getchaintips(const JSONRPCRequest& request) /* * Idea: the set of chain tips is ::ChainActive().tip, plus orphan blocks which do not have another orphan building off of them. * Algorithm: - * - Make one pass through g_blockman.m_block_index, picking out the orphan blocks, and also storing a set of the orphan block's pprev pointers. + * - Make one pass through BlockIndex(), picking out the orphan blocks, and also storing a set of the orphan block's pprev pointers. * - Iterate through the orphan blocks. If the block isn't pointed to by another orphan, it is a chain tip. * - add ::ChainActive().Tip() */ @@ -1726,8 +1718,10 @@ static UniValue getblockstats(const JSONRPCRequest& request) {RPCResult::Type::NUM, "utxo_size_inc", "The increase/decrease in size for the utxo index (not discounting op_return and similar)"}, }}, RPCExamples{ - HelpExampleCli("getblockstats", "1000 '[\"minfeerate\",\"avgfeerate\"]'") - + HelpExampleRpc("getblockstats", "1000 '[\"minfeerate\",\"avgfeerate\"]'") + HelpExampleCli("getblockstats", R"('"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09"' '["minfeerate","avgfeerate"]')") + + HelpExampleCli("getblockstats", R"(1000 '["minfeerate","avgfeerate"]')") + + HelpExampleRpc("getblockstats", R"("00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09", ["minfeerate","avgfeerate"])") + + HelpExampleRpc("getblockstats", R"(1000, ["minfeerate","avgfeerate"])") }, }.Check(request); diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index c1762483e9..3045a74d7a 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -33,6 +33,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "generatetoaddress", 2, "maxtries" }, { "generatetodescriptor", 0, "num_blocks" }, { "generatetodescriptor", 2, "maxtries" }, + { "generateblock", 1, "transactions" }, { "getnetworkhashps", 0, "nblocks" }, { "getnetworkhashps", 1, "height" }, { "sendtoaddress", 1, "amount" }, @@ -97,10 +98,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "signrawtransactionwithkey", 1, "privkeys" }, { "signrawtransactionwithkey", 2, "prevtxs" }, { "signrawtransactionwithwallet", 1, "prevtxs" }, - { "sendrawtransaction", 1, "allowhighfees" }, { "sendrawtransaction", 1, "maxfeerate" }, { "testmempoolaccept", 0, "rawtxs" }, - { "testmempoolaccept", 1, "allowhighfees" }, { "testmempoolaccept", 1, "maxfeerate" }, { "combinerawtransaction", 0, "txs" }, { "fundrawtransaction", 1, "options" }, @@ -132,6 +131,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "importpubkey", 2, "rescan" }, { "importmulti", 0, "requests" }, { "importmulti", 1, "options" }, + { "importdescriptors", 0, "requests" }, { "verifychain", 0, "checklevel" }, { "verifychain", 1, "nblocks" }, { "getblockstats", 0, "hash_or_height" }, @@ -154,6 +154,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "logging", 0, "include" }, { "logging", 1, "exclude" }, { "disconnectnode", 1, "nodeid" }, + { "upgradewallet", 0, "version" }, // Echo with conversion (For testing only) { "echojson", 0, "arg0" }, { "echojson", 1, "arg1" }, @@ -170,6 +171,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "createwallet", 1, "disable_private_keys"}, { "createwallet", 2, "blank"}, { "createwallet", 4, "avoid_reuse"}, + { "createwallet", 5, "descriptors"}, { "getnodeaddresses", 0, "count"}, { "stop", 0, "wait" }, }; diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index da9d583fa7..05d3fd6afb 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -101,6 +101,36 @@ static UniValue getnetworkhashps(const JSONRPCRequest& request) return GetNetworkHashPS(!request.params[0].isNull() ? request.params[0].get_int() : 120, !request.params[1].isNull() ? request.params[1].get_int() : -1); } +static bool GenerateBlock(CBlock& block, uint64_t& max_tries, unsigned int& extra_nonce, uint256& block_hash) +{ + block_hash.SetNull(); + + { + LOCK(cs_main); + IncrementExtraNonce(&block, ::ChainActive().Tip(), extra_nonce); + } + + CChainParams chainparams(Params()); + + while (max_tries > 0 && block.nNonce < std::numeric_limits<uint32_t>::max() && !CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus()) && !ShutdownRequested()) { + ++block.nNonce; + --max_tries; + } + if (max_tries == 0 || ShutdownRequested()) { + return false; + } + if (block.nNonce == std::numeric_limits<uint32_t>::max()) { + return true; + } + + std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block); + if (!ProcessNewBlock(chainparams, shared_pblock, true, nullptr)) + throw JSONRPCError(RPC_INTERNAL_ERROR, "ProcessNewBlock, block not accepted"); + + block_hash = block.GetHash(); + return true; +} + static UniValue generateBlocks(const CTxMemPool& mempool, const CScript& coinbase_script, int nGenerate, uint64_t nMaxTries) { int nHeightEnd = 0; @@ -119,29 +149,54 @@ static UniValue generateBlocks(const CTxMemPool& mempool, const CScript& coinbas if (!pblocktemplate.get()) throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); CBlock *pblock = &pblocktemplate->block; - { - LOCK(cs_main); - IncrementExtraNonce(pblock, ::ChainActive().Tip(), nExtraNonce); - } - while (nMaxTries > 0 && pblock->nNonce < std::numeric_limits<uint32_t>::max() && !CheckProofOfWork(pblock->GetHash(), pblock->nBits, Params().GetConsensus()) && !ShutdownRequested()) { - ++pblock->nNonce; - --nMaxTries; - } - if (nMaxTries == 0 || ShutdownRequested()) { + + uint256 block_hash; + if (!GenerateBlock(*pblock, nMaxTries, nExtraNonce, block_hash)) { break; } - if (pblock->nNonce == std::numeric_limits<uint32_t>::max()) { - continue; + + if (!block_hash.IsNull()) { + ++nHeight; + blockHashes.push_back(block_hash.GetHex()); } - std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(*pblock); - if (!ProcessNewBlock(Params(), shared_pblock, true, nullptr)) - throw JSONRPCError(RPC_INTERNAL_ERROR, "ProcessNewBlock, block not accepted"); - ++nHeight; - blockHashes.push_back(pblock->GetHash().GetHex()); } return blockHashes; } +static bool getScriptFromDescriptor(const std::string& descriptor, CScript& script, std::string& error) +{ + FlatSigningProvider key_provider; + const auto desc = Parse(descriptor, key_provider, error, /* require_checksum = */ false); + if (desc) { + if (desc->IsRange()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Ranged descriptor not accepted. Maybe pass through deriveaddresses first?"); + } + + FlatSigningProvider provider; + std::vector<CScript> scripts; + if (!desc->Expand(0, key_provider, scripts, provider)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Cannot derive script without private keys")); + } + + // Combo desriptors can have 2 or 4 scripts, so we can't just check scripts.size() == 1 + CHECK_NONFATAL(scripts.size() > 0 && scripts.size() <= 4); + + if (scripts.size() == 1) { + script = scripts.at(0); + } else if (scripts.size() == 4) { + // For uncompressed keys, take the 3rd script, since it is p2wpkh + script = scripts.at(2); + } else { + // Else take the 2nd script, since it is p2pkh + script = scripts.at(1); + } + + return true; + } else { + return false; + } +} + static UniValue generatetodescriptor(const JSONRPCRequest& request) { RPCHelpMan{ @@ -166,27 +221,15 @@ static UniValue generatetodescriptor(const JSONRPCRequest& request) const int num_blocks{request.params[0].get_int()}; const int64_t max_tries{request.params[2].isNull() ? 1000000 : request.params[2].get_int()}; - FlatSigningProvider key_provider; + CScript coinbase_script; std::string error; - const auto desc = Parse(request.params[1].get_str(), key_provider, error, /* require_checksum = */ false); - if (!desc) { + if (!getScriptFromDescriptor(request.params[1].get_str(), coinbase_script, error)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error); } - if (desc->IsRange()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Ranged descriptor not accepted. Maybe pass through deriveaddresses first?"); - } - - FlatSigningProvider provider; - std::vector<CScript> coinbase_script; - if (!desc->Expand(0, key_provider, coinbase_script, provider)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Cannot derive script without private keys")); - } const CTxMemPool& mempool = EnsureMemPool(); - CHECK_NONFATAL(coinbase_script.size() == 1); - - return generateBlocks(mempool, coinbase_script.at(0), num_blocks, max_tries); + return generateBlocks(mempool, coinbase_script, num_blocks, max_tries); } static UniValue generatetoaddress(const JSONRPCRequest& request) @@ -229,6 +272,113 @@ static UniValue generatetoaddress(const JSONRPCRequest& request) return generateBlocks(mempool, coinbase_script, nGenerate, nMaxTries); } +static UniValue generateblock(const JSONRPCRequest& request) +{ + RPCHelpMan{"generateblock", + "\nMine a block with a set of ordered transactions immediately to a specified address or descriptor (before the RPC call returns)\n", + { + {"output", RPCArg::Type::STR, RPCArg::Optional::NO, "The address or descriptor to send the newly generated bitcoin to."}, + {"transactions", RPCArg::Type::ARR, RPCArg::Optional::NO, "An array of hex strings which are either txids or raw transactions.\n" + "Txids must reference transactions currently in the mempool.\n" + "All transactions must be valid and in valid order, otherwise the block will be rejected.", + { + {"rawtx/txid", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, + }, + }, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR_HEX, "hash", "hash of generated block"}, + } + }, + RPCExamples{ + "\nGenerate a block to myaddress, with txs rawtx and mempool_txid\n" + + HelpExampleCli("generateblock", R"("myaddress" '["rawtx", "mempool_txid"]')") + }, + }.Check(request); + + const auto address_or_descriptor = request.params[0].get_str(); + CScript coinbase_script; + std::string error; + + if (!getScriptFromDescriptor(address_or_descriptor, coinbase_script, error)) { + const auto destination = DecodeDestination(address_or_descriptor); + if (!IsValidDestination(destination)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Error: Invalid address or descriptor"); + } + + coinbase_script = GetScriptForDestination(destination); + } + + const CTxMemPool& mempool = EnsureMemPool(); + + std::vector<CTransactionRef> txs; + const auto raw_txs_or_txids = request.params[1].get_array(); + for (size_t i = 0; i < raw_txs_or_txids.size(); i++) { + const auto str(raw_txs_or_txids[i].get_str()); + + uint256 hash; + CMutableTransaction mtx; + if (ParseHashStr(str, hash)) { + + const auto tx = mempool.get(hash); + if (!tx) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Transaction %s not in mempool.", str)); + } + + txs.emplace_back(tx); + + } else if (DecodeHexTx(mtx, str)) { + txs.push_back(MakeTransactionRef(std::move(mtx))); + + } else { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("Transaction decode failed for %s", str)); + } + } + + CChainParams chainparams(Params()); + CBlock block; + + { + LOCK(cs_main); + + CTxMemPool empty_mempool; + std::unique_ptr<CBlockTemplate> blocktemplate(BlockAssembler(empty_mempool, chainparams).CreateNewBlock(coinbase_script)); + if (!blocktemplate) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); + } + block = blocktemplate->block; + } + + CHECK_NONFATAL(block.vtx.size() == 1); + + // Add transactions + block.vtx.insert(block.vtx.end(), txs.begin(), txs.end()); + RegenerateCommitments(block); + + { + LOCK(cs_main); + + BlockValidationState state; + if (!TestBlockValidity(state, chainparams, block, LookupBlockIndex(block.hashPrevBlock), false, false)) { + throw JSONRPCError(RPC_VERIFY_ERROR, strprintf("TestBlockValidity failed: %s", state.ToString())); + } + } + + uint256 block_hash; + uint64_t max_tries{1000000}; + unsigned int extra_nonce{0}; + + if (!GenerateBlock(block, max_tries, extra_nonce, block_hash) || block_hash.IsNull()) { + throw JSONRPCError(RPC_MISC_ERROR, "Failed to make block."); + } + + UniValue obj(UniValue::VOBJ); + obj.pushKV("hash", block_hash.GetHex()); + return obj; +} + static UniValue getmininginfo(const JSONRPCRequest& request) { RPCHelpMan{"getmininginfo", @@ -1038,6 +1188,7 @@ static const CRPCCommand commands[] = { "generating", "generatetoaddress", &generatetoaddress, {"nblocks","address","maxtries"} }, { "generating", "generatetodescriptor", &generatetodescriptor, {"num_blocks","descriptor","maxtries"} }, + { "generating", "generateblock", &generateblock, {"output","transactions"} }, { "util", "estimatesmartfee", &estimatesmartfee, {"conf_target", "estimate_mode"} }, diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 51a9581349..f3c5fed858 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -1,9 +1,10 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <httpserver.h> +#include <interfaces/chain.h> #include <key_io.h> #include <node/context.h> #include <outputtype.h> @@ -363,7 +364,13 @@ static UniValue setmocktime(const JSONRPCRequest& request) LOCK(cs_main); RPCTypeCheck(request.params, {UniValue::VNUM}); - SetMockTime(request.params[0].get_int64()); + int64_t time = request.params[0].get_int64(); + SetMockTime(time); + if (g_rpc_node) { + for (const auto& chain_client : g_rpc_node->chain_clients) { + chain_client->setMockTime(time); + } + } return NullUniValue; } @@ -509,7 +516,7 @@ UniValue logging(const JSONRPCRequest& request) "When called with arguments, adds or removes categories from debug logging and return the lists above.\n" "The arguments are evaluated in order \"include\", \"exclude\".\n" "If an item is both included and excluded, it will thus end up being excluded.\n" - "The valid logging categories are: " + ListLogCategories() + "\n" + "The valid logging categories are: " + LogInstance().LogCategoriesString() + "\n" "In addition, the following are available as category names with special meanings:\n" " - \"all\", \"1\" : represent all logging categories.\n" " - \"none\", \"0\" : even if other logging categories are specified, ignore all of them.\n" @@ -561,8 +568,7 @@ UniValue logging(const JSONRPCRequest& request) } UniValue result(UniValue::VOBJ); - std::vector<CLogCategoryActive> vLogCatActive = ListActiveLogCategories(); - for (const auto& logCatActive : vLogCatActive) { + for (const auto& logCatActive : LogInstance().LogCategoriesList()) { result.pushKV(logCatActive.category, logCatActive.active); } diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 10562126db..d6d15f8b56 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 3c0b060439..063ee1697c 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -40,12 +40,6 @@ #include <univalue.h> -/** Maximum fee rate for sendrawtransaction and testmempoolaccept. - * By default, a transaction with a fee rate higher than this will be rejected - * by the RPCs. This can be overridden with the maxfeerate argument. - */ -static const CFeeRate DEFAULT_MAX_RAW_TX_FEE_RATE{COIN / 10}; - static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) { // Call into TxToUniv() in bitcoin-common to decode the transaction hex. @@ -825,7 +819,7 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) RPCTypeCheck(request.params, { UniValue::VSTR, - UniValueType(), // NUM or BOOL, checked later + UniValueType(), // VNUM or VSTR, checked inside AmountFromValue() }); // parse hex string from parameter @@ -834,13 +828,9 @@ static UniValue sendrawtransaction(const JSONRPCRequest& request) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); CTransactionRef tx(MakeTransactionRef(std::move(mtx))); - CFeeRate max_raw_tx_fee_rate = DEFAULT_MAX_RAW_TX_FEE_RATE; - // TODO: temporary migration code for old clients. Remove in v0.20 - if (request.params[1].isBool()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Second argument must be numeric (maxfeerate) and no longer supports a boolean. To allow a transaction with high fees, set maxfeerate to 0."); - } else if (!request.params[1].isNull()) { - max_raw_tx_fee_rate = CFeeRate(AmountFromValue(request.params[1])); - } + const CFeeRate max_raw_tx_fee_rate = request.params[1].isNull() ? + DEFAULT_MAX_RAW_TX_FEE_RATE : + CFeeRate(AmountFromValue(request.params[1])); int64_t virtual_size = GetVirtualTransactionSize(*tx); CAmount max_raw_tx_fee = max_raw_tx_fee_rate.GetFee(virtual_size); @@ -896,7 +886,7 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) RPCTypeCheck(request.params, { UniValue::VARR, - UniValueType(), // NUM or BOOL, checked later + UniValueType(), // VNUM or VSTR, checked inside AmountFromValue() }); if (request.params[0].get_array().size() != 1) { @@ -910,13 +900,9 @@ static UniValue testmempoolaccept(const JSONRPCRequest& request) CTransactionRef tx(MakeTransactionRef(std::move(mtx))); const uint256& tx_hash = tx->GetHash(); - CFeeRate max_raw_tx_fee_rate = DEFAULT_MAX_RAW_TX_FEE_RATE; - // TODO: temporary migration code for old clients. Remove in v0.20 - if (request.params[1].isBool()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Second argument must be numeric (maxfeerate) and no longer supports a boolean. To allow a transaction with high fees, set maxfeerate to 0."); - } else if (!request.params[1].isNull()) { - max_raw_tx_fee_rate = CFeeRate(AmountFromValue(request.params[1])); - } + const CFeeRate max_raw_tx_fee_rate = request.params[1].isNull() ? + DEFAULT_MAX_RAW_TX_FEE_RATE : + CFeeRate(AmountFromValue(request.params[1])); CTxMemPool& mempool = EnsureMemPool(); int64_t virtual_size = GetVirtualTransactionSize(*tx); @@ -1823,10 +1809,10 @@ static const CRPCCommand commands[] = { "rawtransactions", "createrawtransaction", &createrawtransaction, {"inputs","outputs","locktime","replaceable"} }, { "rawtransactions", "decoderawtransaction", &decoderawtransaction, {"hexstring","iswitness"} }, { "rawtransactions", "decodescript", &decodescript, {"hexstring"} }, - { "rawtransactions", "sendrawtransaction", &sendrawtransaction, {"hexstring","allowhighfees|maxfeerate"} }, + { "rawtransactions", "sendrawtransaction", &sendrawtransaction, {"hexstring","maxfeerate"} }, { "rawtransactions", "combinerawtransaction", &combinerawtransaction, {"txs"} }, { "rawtransactions", "signrawtransactionwithkey", &signrawtransactionwithkey, {"hexstring","privkeys","prevtxs","sighashtype"} }, - { "rawtransactions", "testmempoolaccept", &testmempoolaccept, {"rawtxs","allowhighfees|maxfeerate"} }, + { "rawtransactions", "testmempoolaccept", &testmempoolaccept, {"rawtxs","maxfeerate"} }, { "rawtransactions", "decodepsbt", &decodepsbt, {"psbt"} }, { "rawtransactions", "combinepsbt", &combinepsbt, {"txs"} }, { "rawtransactions", "finalizepsbt", &finalizepsbt, {"psbt", "extract"} }, diff --git a/src/rpc/rawtransaction_util.cpp b/src/rpc/rawtransaction_util.cpp index 7b701a2bbe..bd82773bf2 100644 --- a/src/rpc/rawtransaction_util.cpp +++ b/src/rpc/rawtransaction_util.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rpc/rawtransaction_util.h b/src/rpc/rawtransaction_util.h index 436db5dc60..942314eccf 100644 --- a/src/rpc/rawtransaction_util.h +++ b/src/rpc/rawtransaction_util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 7e1fb7a59d..860fa198d5 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,6 +13,9 @@ #include <tuple> +#include <boost/algorithm/string/classification.hpp> +#include <boost/algorithm/string/split.hpp> + const std::string UNIX_EPOCH_TIME = "UNIX epoch time"; const std::string EXAMPLE_ADDRESS[2] = {"bc1q09vm5lfy0j5reeulh4x5752q25uqqvz34hufdl", "bc1q02ad21edsxd23d32dfgqqsz4vv4nmtfzuklhy3"}; @@ -330,7 +333,7 @@ struct Sections { if (outer_type == OuterType::NONE) return; // Nothing more to do for non-recursive types on first recursion auto left = indent; if (arg.m_type_str.size() != 0 && push_name) { - left += "\"" + arg.m_name + "\": " + arg.m_type_str.at(0); + left += "\"" + arg.GetName() + "\": " + arg.m_type_str.at(0); } else { left += push_name ? arg.ToStringObj(/* oneline */ false) : arg.ToString(/* oneline */ false); } @@ -341,7 +344,7 @@ struct Sections { case RPCArg::Type::OBJ: case RPCArg::Type::OBJ_USER_KEYS: { const auto right = outer_type == OuterType::NONE ? "" : arg.ToDescriptionString(); - PushSection({indent + (push_name ? "\"" + arg.m_name + "\": " : "") + "{", right}); + PushSection({indent + (push_name ? "\"" + arg.GetName() + "\": " : "") + "{", right}); for (const auto& arg_inner : arg.m_inner) { Push(arg_inner, current_indent + 2, OuterType::OBJ); } @@ -353,7 +356,7 @@ struct Sections { } case RPCArg::Type::ARR: { auto left = indent; - left += push_name ? "\"" + arg.m_name + "\": " : ""; + left += push_name ? "\"" + arg.GetName() + "\": " : ""; left += "["; const auto right = outer_type == OuterType::NONE ? "" : arg.ToDescriptionString(); PushSection({left, right}); @@ -419,8 +422,12 @@ RPCHelpMan::RPCHelpMan(std::string name, std::string description, std::vector<RP { std::set<std::string> named_args; for (const auto& arg : m_args) { + std::vector<std::string> names; + boost::split(names, arg.m_names, boost::is_any_of("|")); // Should have unique named arguments - CHECK_NONFATAL(named_args.insert(arg.m_name).second); + for (const std::string& name : names) { + CHECK_NONFATAL(named_args.insert(name).second); + } } } @@ -489,7 +496,7 @@ std::string RPCHelpMan::ToString() const if (i == 0) ret += "\nArguments:\n"; // Push named argument name and description - sections.m_sections.emplace_back(::ToString(i + 1) + ". " + arg.m_name, arg.ToDescriptionString()); + sections.m_sections.emplace_back(::ToString(i + 1) + ". " + arg.GetFirstName(), arg.ToDescriptionString()); sections.m_max_pad = std::max(sections.m_max_pad, sections.m_sections.back().m_left.size()); // Recursively push nested args @@ -506,6 +513,17 @@ std::string RPCHelpMan::ToString() const return ret; } +std::string RPCArg::GetFirstName() const +{ + return m_names.substr(0, m_names.find("|")); +} + +std::string RPCArg::GetName() const +{ + CHECK_NONFATAL(std::string::npos == m_names.find("|")); + return m_names; +} + bool RPCArg::IsOptional() const { if (m_fallback.which() == 1) { @@ -681,7 +699,7 @@ std::string RPCArg::ToStringObj(const bool oneline) const { std::string res; res += "\""; - res += m_name; + res += GetFirstName(); if (oneline) { res += "\":"; } else { @@ -723,13 +741,13 @@ std::string RPCArg::ToString(const bool oneline) const switch (m_type) { case Type::STR_HEX: case Type::STR: { - return "\"" + m_name + "\""; + return "\"" + GetFirstName() + "\""; } case Type::NUM: case Type::RANGE: case Type::AMOUNT: case Type::BOOL: { - return m_name; + return GetFirstName(); } case Type::OBJ: case Type::OBJ_USER_KEYS: { diff --git a/src/rpc/util.h b/src/rpc/util.h index f65ad1246b..53dce2c397 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -142,7 +142,7 @@ struct RPCArg { OMITTED, }; using Fallback = boost::variant<Optional, /* default value for optional args */ std::string>; - const std::string m_name; //!< The name of the arg (can be empty for inner args) + const std::string m_names; //!< The name of the arg (can be empty for inner args, can contain multiple aliases separated by | for named request arguments) const Type m_type; const std::vector<RPCArg> m_inner; //!< Only used for arrays or dicts const Fallback m_fallback; @@ -157,7 +157,7 @@ struct RPCArg { const std::string description, const std::string oneline_description = "", const std::vector<std::string> type_str = {}) - : m_name{std::move(name)}, + : m_names{std::move(name)}, m_type{std::move(type)}, m_fallback{std::move(fallback)}, m_description{std::move(description)}, @@ -175,7 +175,7 @@ struct RPCArg { const std::vector<RPCArg> inner, const std::string oneline_description = "", const std::vector<std::string> type_str = {}) - : m_name{std::move(name)}, + : m_names{std::move(name)}, m_type{std::move(type)}, m_inner{std::move(inner)}, m_fallback{std::move(fallback)}, @@ -188,6 +188,12 @@ struct RPCArg { bool IsOptional() const; + /** Return the first of all aliases */ + std::string GetFirstName() const; + + /** Return the name, throws when there are aliases */ + std::string GetName() const; + /** * Return the type string of the argument. * Set oneline to allow it to be overridden by a custom oneline type string (m_oneline_description). diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 4cac5a54e0..c4bd47310b 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 83dc046ca1..5279f40506 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -575,6 +575,7 @@ public: default: return nullopt; } } + bool IsSingleType() const final { return true; } }; /** A parsed raw(H) descriptor. */ @@ -602,6 +603,7 @@ public: default: return nullopt; } } + bool IsSingleType() const final { return true; } }; /** A parsed pk(P) descriptor. */ @@ -611,6 +613,7 @@ protected: std::vector<CScript> MakeScripts(const std::vector<CPubKey>& keys, const CScript*, FlatSigningProvider&) const override { return Vector(GetScriptForRawPubKey(keys[0])); } public: PKDescriptor(std::unique_ptr<PubkeyProvider> prov) : DescriptorImpl(Vector(std::move(prov)), {}, "pk") {} + bool IsSingleType() const final { return true; } }; /** A parsed pkh(P) descriptor. */ @@ -626,6 +629,7 @@ protected: public: PKHDescriptor(std::unique_ptr<PubkeyProvider> prov) : DescriptorImpl(Vector(std::move(prov)), {}, "pkh") {} Optional<OutputType> GetOutputType() const override { return OutputType::LEGACY; } + bool IsSingleType() const final { return true; } }; /** A parsed wpkh(P) descriptor. */ @@ -641,6 +645,7 @@ protected: public: WPKHDescriptor(std::unique_ptr<PubkeyProvider> prov) : DescriptorImpl(Vector(std::move(prov)), {}, "wpkh") {} Optional<OutputType> GetOutputType() const override { return OutputType::BECH32; } + bool IsSingleType() const final { return true; } }; /** A parsed combo(P) descriptor. */ @@ -664,6 +669,7 @@ protected: } public: ComboDescriptor(std::unique_ptr<PubkeyProvider> prov) : DescriptorImpl(Vector(std::move(prov)), {}, "combo") {} + bool IsSingleType() const final { return false; } }; /** A parsed multi(...) or sortedmulti(...) descriptor */ @@ -683,6 +689,7 @@ protected: } public: MultisigDescriptor(int threshold, std::vector<std::unique_ptr<PubkeyProvider>> providers, bool sorted = false) : DescriptorImpl(std::move(providers), {}, sorted ? "sortedmulti" : "multi"), m_threshold(threshold), m_sorted(sorted) {} + bool IsSingleType() const final { return true; } }; /** A parsed sh(...) descriptor. */ @@ -699,6 +706,7 @@ public: if (m_subdescriptor_arg->GetOutputType() == OutputType::BECH32) return OutputType::P2SH_SEGWIT; return OutputType::LEGACY; } + bool IsSingleType() const final { return true; } }; /** A parsed wsh(...) descriptor. */ @@ -709,6 +717,7 @@ protected: public: WSHDescriptor(std::unique_ptr<DescriptorImpl> desc) : DescriptorImpl({}, std::move(desc), "wsh") {} Optional<OutputType> GetOutputType() const override { return OutputType::BECH32; } + bool IsSingleType() const final { return true; } }; //////////////////////////////////////////////////////////////////////////// diff --git a/src/script/descriptor.h b/src/script/descriptor.h index 34cd5760de..17b43e7c81 100644 --- a/src/script/descriptor.h +++ b/src/script/descriptor.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -87,6 +87,9 @@ struct Descriptor { /** Convert the descriptor back to a string, undoing parsing. */ virtual std::string ToString() const = 0; + /** Whether this descriptor will return one scriptPubKey or multiple (aka is or is not combo) */ + virtual bool IsSingleType() const = 0; + /** Convert the descriptor to a private string. This fails if the provided provider does not have the relevant private keys. */ virtual bool ToPrivateString(const SigningProvider& provider, std::string& out) const = 0; diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 083022fbdd..23d5b72a5c 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -342,6 +342,35 @@ public: }; } +/** Helper for OP_CHECKSIG and OP_CHECKSIGVERIFY + * + * A return value of false means the script fails entirely. When true is returned, the + * fSuccess variable indicates whether the signature check itself succeeded. + */ +static bool EvalChecksig(const valtype& vchSig, const valtype& vchPubKey, CScript::const_iterator pbegincodehash, CScript::const_iterator pend, unsigned int flags, const BaseSignatureChecker& checker, SigVersion sigversion, ScriptError* serror, bool& fSuccess) +{ + // Subset of script starting at the most recent codeseparator + CScript scriptCode(pbegincodehash, pend); + + // Drop the signature in pre-segwit scripts but not segwit scripts + if (sigversion == SigVersion::BASE) { + int found = FindAndDelete(scriptCode, CScript() << vchSig); + if (found > 0 && (flags & SCRIPT_VERIFY_CONST_SCRIPTCODE)) + return set_error(serror, SCRIPT_ERR_SIG_FINDANDDELETE); + } + + if (!CheckSignatureEncoding(vchSig, flags, serror) || !CheckPubKeyEncoding(vchPubKey, flags, sigversion, serror)) { + //serror is set + return false; + } + fSuccess = checker.CheckSig(vchSig, vchPubKey, scriptCode, sigversion); + + if (!fSuccess && (flags & SCRIPT_VERIFY_NULLFAIL) && vchSig.size()) + return set_error(serror, SCRIPT_ERR_SIG_NULLFAIL); + + return true; +} + bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript& script, unsigned int flags, const BaseSignatureChecker& checker, SigVersion sigversion, ScriptError* serror) { static const CScriptNum bnZero(0); @@ -985,25 +1014,8 @@ bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript& valtype& vchSig = stacktop(-2); valtype& vchPubKey = stacktop(-1); - // Subset of script starting at the most recent codeseparator - CScript scriptCode(pbegincodehash, pend); - - // Drop the signature in pre-segwit scripts but not segwit scripts - if (sigversion == SigVersion::BASE) { - int found = FindAndDelete(scriptCode, CScript() << vchSig); - if (found > 0 && (flags & SCRIPT_VERIFY_CONST_SCRIPTCODE)) - return set_error(serror, SCRIPT_ERR_SIG_FINDANDDELETE); - } - - if (!CheckSignatureEncoding(vchSig, flags, serror) || !CheckPubKeyEncoding(vchPubKey, flags, sigversion, serror)) { - //serror is set - return false; - } - bool fSuccess = checker.CheckSig(vchSig, vchPubKey, scriptCode, sigversion); - - if (!fSuccess && (flags & SCRIPT_VERIFY_NULLFAIL) && vchSig.size()) - return set_error(serror, SCRIPT_ERR_SIG_NULLFAIL); - + bool fSuccess = true; + if (!EvalChecksig(vchSig, vchPubKey, pbegincodehash, pend, flags, checker, sigversion, serror, fSuccess)) return false; popstack(stack); popstack(stack); stack.push_back(fSuccess ? vchTrue : vchFalse); @@ -1279,18 +1291,29 @@ uint256 GetOutputsHash(const T& txTo) } // namespace template <class T> -PrecomputedTransactionData::PrecomputedTransactionData(const T& txTo) +void PrecomputedTransactionData::Init(const T& txTo) { + assert(!m_ready); + // Cache is calculated only for transactions with witness if (txTo.HasWitness()) { hashPrevouts = GetPrevoutHash(txTo); hashSequence = GetSequenceHash(txTo); hashOutputs = GetOutputsHash(txTo); - ready = true; } + + m_ready = true; +} + +template <class T> +PrecomputedTransactionData::PrecomputedTransactionData(const T& txTo) +{ + Init(txTo); } // explicit instantiation +template void PrecomputedTransactionData::Init(const CTransaction& txTo); +template void PrecomputedTransactionData::Init(const CMutableTransaction& txTo); template PrecomputedTransactionData::PrecomputedTransactionData(const CTransaction& txTo); template PrecomputedTransactionData::PrecomputedTransactionData(const CMutableTransaction& txTo); @@ -1303,7 +1326,7 @@ uint256 SignatureHash(const CScript& scriptCode, const T& txTo, unsigned int nIn uint256 hashPrevouts; uint256 hashSequence; uint256 hashOutputs; - const bool cacheready = cache && cache->ready; + const bool cacheready = cache && cache->m_ready; if (!(nHashType & SIGHASH_ANYONECANPAY)) { hashPrevouts = cacheready ? cache->hashPrevouts : GetPrevoutHash(txTo); diff --git a/src/script/interpreter.h b/src/script/interpreter.h index 2b104a608c..71f2436369 100644 --- a/src/script/interpreter.h +++ b/src/script/interpreter.h @@ -121,7 +121,12 @@ bool CheckSignatureEncoding(const std::vector<unsigned char> &vchSig, unsigned i struct PrecomputedTransactionData { uint256 hashPrevouts, hashSequence, hashOutputs; - bool ready = false; + bool m_ready = false; + + PrecomputedTransactionData() = default; + + template <class T> + void Init(const T& tx); template <class T> explicit PrecomputedTransactionData(const T& tx); diff --git a/src/script/script.h b/src/script/script.h index 7aaa10b60b..daf4224530 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -419,28 +419,15 @@ public: READWRITEAS(CScriptBase, *this); } - CScript& operator+=(const CScript& b) - { - reserve(size() + b.size()); - insert(end(), b.begin(), b.end()); - return *this; - } - - friend CScript operator+(const CScript& a, const CScript& b) - { - CScript ret = a; - ret += b; - return ret; - } - - CScript(int64_t b) { operator<<(b); } - + explicit CScript(int64_t b) { operator<<(b); } explicit CScript(opcodetype b) { operator<<(b); } explicit CScript(const CScriptNum& b) { operator<<(b); } // delete non-existent constructor to defend against future introduction // e.g. via prevector explicit CScript(const std::vector<unsigned char>& b) = delete; + /** Delete non-existent operator to defend against future introduction */ + CScript& operator<<(const CScript& b) = delete; CScript& operator<<(int64_t b) { return push_int64(b); } @@ -487,15 +474,6 @@ public: return *this; } - CScript& operator<<(const CScript& b) - { - // I'm not sure if this should push the script or concatenate scripts. - // If there's ever a use for pushing a script onto a script, delete this member fn - assert(!"Warning: Pushing a CScript onto a CScript with << is probably not intended, use + to concatenate!"); - return *this; - } - - bool GetOp(const_iterator& pc, opcodetype& opcodeRet, std::vector<unsigned char>& vchRet) const { return GetScriptOp(pc, end(), opcodeRet, &vchRet); @@ -506,7 +484,6 @@ public: return GetScriptOp(pc, end(), opcodeRet, nullptr); } - /** Encode/decode small integers: */ static int DecodeOP_N(opcodetype opcode) { diff --git a/src/script/script_error.cpp b/src/script/script_error.cpp index ff521d5860..57e8fee539 100644 --- a/src/script/script_error.cpp +++ b/src/script/script_error.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/sign.cpp b/src/script/sign.cpp index fe8292fe57..1e00afcf89 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/script/sign.h b/src/script/sign.h index f03af0713f..b77d26c0d7 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/serialize.h b/src/serialize.h index 5045cb3c7f..fe53eeed31 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/span.h b/src/span.h index 9379b15c81..664ea8d4c6 100644 --- a/src/span.h +++ b/src/span.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/support/lockedpool.cpp b/src/support/lockedpool.cpp index f3cc12201c..9b14637b95 100644 --- a/src/support/lockedpool.cpp +++ b/src/support/lockedpool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/sync.h b/src/sync.h index ead2cdc67b..0c6f0ef0a7 100644 --- a/src/sync.h +++ b/src/sync.h @@ -210,7 +210,7 @@ public: friend class reverse_lock; }; -#define REVERSE_LOCK(g) decltype(g)::reverse_lock PASTE2(revlock, __COUNTER__)(g, #g, __FILE__, __LINE__) +#define REVERSE_LOCK(g) typename std::decay<decltype(g)>::type::reverse_lock PASTE2(revlock, __COUNTER__)(g, #g, __FILE__, __LINE__) template<typename MutexArg> using DebugLock = UniqueLock<typename std::remove_reference<typename std::remove_pointer<MutexArg>::type>::type>; diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index dfa8a6df21..bc6b38c682 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -1,18 +1,19 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <addrman.h> +#include <test/data/asmap.raw.h> #include <test/util/setup_common.h> -#include <string> -#include <boost/test/unit_test.hpp> #include <util/asmap.h> #include <util/string.h> -#include <test/data/asmap.raw.h> - #include <hash.h> #include <netbase.h> #include <random.h> +#include <boost/test/unit_test.hpp> + +#include <string> + class CAddrManTest : public CAddrMan { private: diff --git a/src/test/arith_uint256_tests.cpp b/src/test/arith_uint256_tests.cpp index 3723a48903..a135c93786 100644 --- a/src/test/arith_uint256_tests.cpp +++ b/src/test/arith_uint256_tests.cpp @@ -1,17 +1,19 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <arith_uint256.h> +#include <test/util/setup_common.h> +#include <uint256.h> + #include <boost/test/unit_test.hpp> -#include <stdint.h> -#include <sstream> + +#include <cmath> #include <iomanip> #include <limits> -#include <cmath> -#include <uint256.h> -#include <arith_uint256.h> +#include <sstream> +#include <stdint.h> #include <string> -#include <test/util/setup_common.h> BOOST_FIXTURE_TEST_SUITE(arith_uint256_tests, BasicTestingSetup) diff --git a/src/test/base32_tests.cpp b/src/test/base32_tests.cpp index 690368b177..eedab30576 100644 --- a/src/test/base32_tests.cpp +++ b/src/test/base32_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <util/strencodings.h> #include <test/util/setup_common.h> +#include <util/strencodings.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/base64_tests.cpp b/src/test/base64_tests.cpp index 94df4d1955..5927eab6cf 100644 --- a/src/test/base64_tests.cpp +++ b/src/test/base64_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <util/strencodings.h> #include <test/util/setup_common.h> +#include <util/strencodings.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/bip32_tests.cpp b/src/test/bip32_tests.cpp index 53df032252..32329eb510 100644 --- a/src/test/bip32_tests.cpp +++ b/src/test/bip32_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2013-2019 The Bitcoin Core developers +// Copyright (c) 2013-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,8 @@ #include <key.h> #include <key_io.h> #include <streams.h> -#include <util/strencodings.h> #include <test/util/setup_common.h> +#include <util/strencodings.h> #include <string> #include <vector> diff --git a/src/test/blockchain_tests.cpp b/src/test/blockchain_tests.cpp index aa704642bf..c8e8cdeeb3 100644 --- a/src/test/blockchain_tests.cpp +++ b/src/test/blockchain_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,8 @@ #include <chain.h> #include <rpc/blockchain.h> -#include <util/string.h> #include <test/util/setup_common.h> +#include <util/string.h> /* Equality between doubles is imprecise. Comparison should be done * with a small threshold of tolerance, rather than exact equality. diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index 5e52dc268f..ff01f730a3 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 4a7ad9b38b..a75d9fc9ec 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,10 +12,10 @@ #include <random.h> #include <serialize.h> #include <streams.h> +#include <test/util/setup_common.h> #include <uint256.h> -#include <util/system.h> #include <util/strencodings.h> -#include <test/util/setup_common.h> +#include <util/system.h> #include <vector> @@ -27,6 +27,7 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize) { CBloomFilter filter(3, 0.01, 0, BLOOM_UPDATE_ALL); + BOOST_CHECK_MESSAGE( !filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "Bloom filter should be empty!"); filter.insert(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")); BOOST_CHECK_MESSAGE( filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "Bloom filter doesn't contain just-inserted object!"); // One bit different in first byte @@ -50,8 +51,6 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize) BOOST_CHECK_EQUAL_COLLECTIONS(stream.begin(), stream.end(), expected.begin(), expected.end()); BOOST_CHECK_MESSAGE( filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "Bloom filter doesn't contain just-inserted object!"); - filter.clear(); - BOOST_CHECK_MESSAGE( !filter.contains(ParseHex("99108ad8ed9bb6274d3980bab5a85c048f0950c8")), "Bloom filter should be empty!"); } BOOST_AUTO_TEST_CASE(bloom_create_insert_serialize_with_tweak) diff --git a/src/test/bswap_tests.cpp b/src/test/bswap_tests.cpp index 0b4bfdb019..c89cb5488d 100644 --- a/src/test/bswap_tests.cpp +++ b/src/test/bswap_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index a9628e85f9..0565982215 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -1,23 +1,23 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <checkqueue.h> +#include <test/util/setup_common.h> #include <util/memory.h> #include <util/system.h> #include <util/time.h> -#include <test/util/setup_common.h> -#include <checkqueue.h> #include <boost/test/unit_test.hpp> #include <boost/thread.hpp> + #include <atomic> -#include <thread> -#include <vector> -#include <mutex> #include <condition_variable> - +#include <mutex> +#include <thread> #include <unordered_set> #include <utility> +#include <vector> BOOST_FIXTURE_TEST_SUITE(checkqueue_tests, TestingSetup) diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 436c1bffa0..60196c36a5 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -8,6 +8,7 @@ #include <script/standard.h> #include <streams.h> #include <test/util/setup_common.h> +#include <txdb.h> #include <uint256.h> #include <undo.h> #include <util/strencodings.h> @@ -109,7 +110,12 @@ static const unsigned int NUM_SIMULATION_ITERATIONS = 40000; // // During the process, booleans are kept to make sure that the randomized // operation hits all branches. -BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) +// +// If fake_best_block is true, assign a random uint256 to mock the recording +// of best block on flush. This is necessary when using CCoinsViewDB as the base, +// otherwise we'll hit an assertion in BatchWrite. +// +void SimulationTest(CCoinsView* base, bool fake_best_block) { // Various coverage trackers. bool removed_all_caches = false; @@ -126,9 +132,8 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) std::map<COutPoint, Coin> result; // The cache stack. - CCoinsViewTest base; // A CCoinsViewTest at the bottom. std::vector<CCoinsViewCacheTest*> stack; // A stack of CCoinsViewCaches on top. - stack.push_back(new CCoinsViewCacheTest(&base)); // Start with one cache. + stack.push_back(new CCoinsViewCacheTest(base)); // Start with one cache. // Use a limited set of random transaction ids, so we do test overwriting entries. std::vector<uint256> txids; @@ -211,6 +216,7 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) // Every 100 iterations, flush an intermediate cache if (stack.size() > 1 && InsecureRandBool() == 0) { unsigned int flushIndex = InsecureRandRange(stack.size() - 1); + if (fake_best_block) stack[flushIndex]->SetBestBlock(InsecureRand256()); BOOST_CHECK(stack[flushIndex]->Flush()); } } @@ -218,13 +224,14 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) // Every 100 iterations, change the cache stack. if (stack.size() > 0 && InsecureRandBool() == 0) { //Remove the top cache + if (fake_best_block) stack.back()->SetBestBlock(InsecureRand256()); BOOST_CHECK(stack.back()->Flush()); delete stack.back(); stack.pop_back(); } if (stack.size() == 0 || (stack.size() < 4 && InsecureRandBool())) { //Add a new cache - CCoinsView* tip = &base; + CCoinsView* tip = base; if (stack.size() > 0) { tip = stack.back(); } else { @@ -256,6 +263,16 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) BOOST_CHECK(uncached_an_entry); } +// Run the above simulation for multiple base types. +BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) +{ + CCoinsViewTest base; + SimulationTest(&base, false); + + CCoinsViewDB db_base{"test", /*nCacheSize*/ 1 << 23, /*fMemory*/ true, /*fWipe*/ false}; + SimulationTest(&db_base, true); +} + // Store of all necessary tx and undo data for next test typedef std::map<COutPoint, std::tuple<CTransaction,CTxUndo,Coin>> UtxoData; UtxoData utxoData; @@ -532,7 +549,7 @@ BOOST_AUTO_TEST_CASE(ccoins_serialization) } const static COutPoint OUTPOINT; -const static CAmount PRUNED = -1; +const static CAmount SPENT = -1; const static CAmount ABSENT = -2; const static CAmount FAIL = -3; const static CAmount VALUE1 = 100; @@ -551,7 +568,7 @@ static void SetCoinsValue(CAmount value, Coin& coin) assert(value != ABSENT); coin.Clear(); assert(coin.IsSpent()); - if (value != PRUNED) { + if (value != SPENT) { coin.out.nValue = value; coin.nHeight = 1; assert(!coin.IsSpent()); @@ -581,7 +598,7 @@ void GetCoinsMapEntry(const CCoinsMap& map, CAmount& value, char& flags) flags = NO_ENTRY; } else { if (it->second.coin.IsSpent()) { - value = PRUNED; + value = SPENT; } else { value = it->second.coin.out.nValue; } @@ -634,28 +651,28 @@ BOOST_AUTO_TEST_CASE(ccoins_access) * Value Value Value Flags Flags */ CheckAccessCoin(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckAccessCoin(ABSENT, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoin(ABSENT, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoin(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoin(ABSENT, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(ABSENT, SPENT , SPENT , 0 , 0 ); + CheckAccessCoin(ABSENT, SPENT , SPENT , FRESH , FRESH ); + CheckAccessCoin(ABSENT, SPENT , SPENT , DIRTY , DIRTY ); + CheckAccessCoin(ABSENT, SPENT , SPENT , DIRTY|FRESH, DIRTY|FRESH); CheckAccessCoin(ABSENT, VALUE2, VALUE2, 0 , 0 ); CheckAccessCoin(ABSENT, VALUE2, VALUE2, FRESH , FRESH ); CheckAccessCoin(ABSENT, VALUE2, VALUE2, DIRTY , DIRTY ); CheckAccessCoin(ABSENT, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoin(PRUNED, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckAccessCoin(PRUNED, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoin(PRUNED, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoin(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoin(PRUNED, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); - CheckAccessCoin(PRUNED, VALUE2, VALUE2, 0 , 0 ); - CheckAccessCoin(PRUNED, VALUE2, VALUE2, FRESH , FRESH ); - CheckAccessCoin(PRUNED, VALUE2, VALUE2, DIRTY , DIRTY ); - CheckAccessCoin(PRUNED, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(SPENT , ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckAccessCoin(SPENT , SPENT , SPENT , 0 , 0 ); + CheckAccessCoin(SPENT , SPENT , SPENT , FRESH , FRESH ); + CheckAccessCoin(SPENT , SPENT , SPENT , DIRTY , DIRTY ); + CheckAccessCoin(SPENT , SPENT , SPENT , DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(SPENT , VALUE2, VALUE2, 0 , 0 ); + CheckAccessCoin(SPENT , VALUE2, VALUE2, FRESH , FRESH ); + CheckAccessCoin(SPENT , VALUE2, VALUE2, DIRTY , DIRTY ); + CheckAccessCoin(SPENT , VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH); CheckAccessCoin(VALUE1, ABSENT, VALUE1, NO_ENTRY , 0 ); - CheckAccessCoin(VALUE1, PRUNED, PRUNED, 0 , 0 ); - CheckAccessCoin(VALUE1, PRUNED, PRUNED, FRESH , FRESH ); - CheckAccessCoin(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckAccessCoin(VALUE1, PRUNED, PRUNED, DIRTY|FRESH, DIRTY|FRESH); + CheckAccessCoin(VALUE1, SPENT , SPENT , 0 , 0 ); + CheckAccessCoin(VALUE1, SPENT , SPENT , FRESH , FRESH ); + CheckAccessCoin(VALUE1, SPENT , SPENT , DIRTY , DIRTY ); + CheckAccessCoin(VALUE1, SPENT , SPENT , DIRTY|FRESH, DIRTY|FRESH); CheckAccessCoin(VALUE1, VALUE2, VALUE2, 0 , 0 ); CheckAccessCoin(VALUE1, VALUE2, VALUE2, FRESH , FRESH ); CheckAccessCoin(VALUE1, VALUE2, VALUE2, DIRTY , DIRTY ); @@ -685,31 +702,31 @@ BOOST_AUTO_TEST_CASE(ccoins_spend) * Value Value Value Flags Flags */ CheckSpendCoins(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckSpendCoins(ABSENT, PRUNED, PRUNED, 0 , DIRTY ); - CheckSpendCoins(ABSENT, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckSpendCoins(ABSENT, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckSpendCoins(ABSENT, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckSpendCoins(ABSENT, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(ABSENT, SPENT , SPENT , 0 , DIRTY ); + CheckSpendCoins(ABSENT, SPENT , ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(ABSENT, SPENT , SPENT , DIRTY , DIRTY ); + CheckSpendCoins(ABSENT, SPENT , ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(ABSENT, VALUE2, SPENT , 0 , DIRTY ); CheckSpendCoins(ABSENT, VALUE2, ABSENT, FRESH , NO_ENTRY ); - CheckSpendCoins(ABSENT, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(ABSENT, VALUE2, SPENT , DIRTY , DIRTY ); CheckSpendCoins(ABSENT, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckSpendCoins(PRUNED, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); - CheckSpendCoins(PRUNED, PRUNED, PRUNED, 0 , DIRTY ); - CheckSpendCoins(PRUNED, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckSpendCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckSpendCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckSpendCoins(PRUNED, VALUE2, PRUNED, 0 , DIRTY ); - CheckSpendCoins(PRUNED, VALUE2, ABSENT, FRESH , NO_ENTRY ); - CheckSpendCoins(PRUNED, VALUE2, PRUNED, DIRTY , DIRTY ); - CheckSpendCoins(PRUNED, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckSpendCoins(VALUE1, ABSENT, PRUNED, NO_ENTRY , DIRTY ); - CheckSpendCoins(VALUE1, PRUNED, PRUNED, 0 , DIRTY ); - CheckSpendCoins(VALUE1, PRUNED, ABSENT, FRESH , NO_ENTRY ); - CheckSpendCoins(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY ); - CheckSpendCoins(VALUE1, PRUNED, ABSENT, DIRTY|FRESH, NO_ENTRY ); - CheckSpendCoins(VALUE1, VALUE2, PRUNED, 0 , DIRTY ); + CheckSpendCoins(SPENT , ABSENT, ABSENT, NO_ENTRY , NO_ENTRY ); + CheckSpendCoins(SPENT , SPENT , SPENT , 0 , DIRTY ); + CheckSpendCoins(SPENT , SPENT , ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(SPENT , SPENT , SPENT , DIRTY , DIRTY ); + CheckSpendCoins(SPENT , SPENT , ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(SPENT , VALUE2, SPENT , 0 , DIRTY ); + CheckSpendCoins(SPENT , VALUE2, ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(SPENT , VALUE2, SPENT , DIRTY , DIRTY ); + CheckSpendCoins(SPENT , VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(VALUE1, ABSENT, SPENT , NO_ENTRY , DIRTY ); + CheckSpendCoins(VALUE1, SPENT , SPENT , 0 , DIRTY ); + CheckSpendCoins(VALUE1, SPENT , ABSENT, FRESH , NO_ENTRY ); + CheckSpendCoins(VALUE1, SPENT , SPENT , DIRTY , DIRTY ); + CheckSpendCoins(VALUE1, SPENT , ABSENT, DIRTY|FRESH, NO_ENTRY ); + CheckSpendCoins(VALUE1, VALUE2, SPENT , 0 , DIRTY ); CheckSpendCoins(VALUE1, VALUE2, ABSENT, FRESH , NO_ENTRY ); - CheckSpendCoins(VALUE1, VALUE2, PRUNED, DIRTY , DIRTY ); + CheckSpendCoins(VALUE1, VALUE2, SPENT , DIRTY , DIRTY ); CheckSpendCoins(VALUE1, VALUE2, ABSENT, DIRTY|FRESH, NO_ENTRY ); } @@ -742,7 +759,7 @@ static void CheckAddCoinBase(CAmount base_value, CAmount cache_value, CAmount mo template <typename... Args> static void CheckAddCoin(Args&&... args) { - for (const CAmount base_value : {ABSENT, PRUNED, VALUE1}) + for (const CAmount base_value : {ABSENT, SPENT, VALUE1}) CheckAddCoinBase(base_value, std::forward<Args>(args)...); } @@ -751,21 +768,21 @@ BOOST_AUTO_TEST_CASE(ccoins_add) /* Check AddCoin behavior, requesting a new coin from a cache view, * writing a modification to the coin, and then checking the resulting * entry in the cache after the modification. Verify behavior with the - * with the AddCoin potential_overwrite argument set to false, and to true. + * AddCoin possible_overwrite argument set to false, and to true. * - * Cache Write Result Cache Result potential_overwrite + * Cache Write Result Cache Result possible_overwrite * Value Value Value Flags Flags */ CheckAddCoin(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY|FRESH, false); CheckAddCoin(ABSENT, VALUE3, VALUE3, NO_ENTRY , DIRTY , true ); - CheckAddCoin(PRUNED, VALUE3, VALUE3, 0 , DIRTY|FRESH, false); - CheckAddCoin(PRUNED, VALUE3, VALUE3, 0 , DIRTY , true ); - CheckAddCoin(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, false); - CheckAddCoin(PRUNED, VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); - CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , false); - CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY , DIRTY , true ); - CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, false); - CheckAddCoin(PRUNED, VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); + CheckAddCoin(SPENT , VALUE3, VALUE3, 0 , DIRTY|FRESH, false); + CheckAddCoin(SPENT , VALUE3, VALUE3, 0 , DIRTY , true ); + CheckAddCoin(SPENT , VALUE3, VALUE3, FRESH , DIRTY|FRESH, false); + CheckAddCoin(SPENT , VALUE3, VALUE3, FRESH , DIRTY|FRESH, true ); + CheckAddCoin(SPENT , VALUE3, VALUE3, DIRTY , DIRTY , false); + CheckAddCoin(SPENT , VALUE3, VALUE3, DIRTY , DIRTY , true ); + CheckAddCoin(SPENT , VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, false); + CheckAddCoin(SPENT , VALUE3, VALUE3, DIRTY|FRESH, DIRTY|FRESH, true ); CheckAddCoin(VALUE2, VALUE3, FAIL , 0 , NO_ENTRY , false); CheckAddCoin(VALUE2, VALUE3, VALUE3, 0 , DIRTY , true ); CheckAddCoin(VALUE2, VALUE3, FAIL , FRESH , NO_ENTRY , false); @@ -805,42 +822,42 @@ BOOST_AUTO_TEST_CASE(ccoins_write) * Value Value Value Flags Flags Flags */ CheckWriteCoins(ABSENT, ABSENT, ABSENT, NO_ENTRY , NO_ENTRY , NO_ENTRY ); - CheckWriteCoins(ABSENT, PRUNED, PRUNED, NO_ENTRY , DIRTY , DIRTY ); - CheckWriteCoins(ABSENT, PRUNED, ABSENT, NO_ENTRY , DIRTY|FRESH, NO_ENTRY ); + CheckWriteCoins(ABSENT, SPENT , SPENT , NO_ENTRY , DIRTY , DIRTY ); + CheckWriteCoins(ABSENT, SPENT , ABSENT, NO_ENTRY , DIRTY|FRESH, NO_ENTRY ); CheckWriteCoins(ABSENT, VALUE2, VALUE2, NO_ENTRY , DIRTY , DIRTY ); CheckWriteCoins(ABSENT, VALUE2, VALUE2, NO_ENTRY , DIRTY|FRESH, DIRTY|FRESH); - CheckWriteCoins(PRUNED, ABSENT, PRUNED, 0 , NO_ENTRY , 0 ); - CheckWriteCoins(PRUNED, ABSENT, PRUNED, FRESH , NO_ENTRY , FRESH ); - CheckWriteCoins(PRUNED, ABSENT, PRUNED, DIRTY , NO_ENTRY , DIRTY ); - CheckWriteCoins(PRUNED, ABSENT, PRUNED, DIRTY|FRESH, NO_ENTRY , DIRTY|FRESH); - CheckWriteCoins(PRUNED, PRUNED, PRUNED, 0 , DIRTY , DIRTY ); - CheckWriteCoins(PRUNED, PRUNED, PRUNED, 0 , DIRTY|FRESH, DIRTY ); - CheckWriteCoins(PRUNED, PRUNED, ABSENT, FRESH , DIRTY , NO_ENTRY ); - CheckWriteCoins(PRUNED, PRUNED, ABSENT, FRESH , DIRTY|FRESH, NO_ENTRY ); - CheckWriteCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY , DIRTY ); - CheckWriteCoins(PRUNED, PRUNED, PRUNED, DIRTY , DIRTY|FRESH, DIRTY ); - CheckWriteCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, DIRTY , NO_ENTRY ); - CheckWriteCoins(PRUNED, PRUNED, ABSENT, DIRTY|FRESH, DIRTY|FRESH, NO_ENTRY ); - CheckWriteCoins(PRUNED, VALUE2, VALUE2, 0 , DIRTY , DIRTY ); - CheckWriteCoins(PRUNED, VALUE2, VALUE2, 0 , DIRTY|FRESH, DIRTY ); - CheckWriteCoins(PRUNED, VALUE2, VALUE2, FRESH , DIRTY , DIRTY|FRESH); - CheckWriteCoins(PRUNED, VALUE2, VALUE2, FRESH , DIRTY|FRESH, DIRTY|FRESH); - CheckWriteCoins(PRUNED, VALUE2, VALUE2, DIRTY , DIRTY , DIRTY ); - CheckWriteCoins(PRUNED, VALUE2, VALUE2, DIRTY , DIRTY|FRESH, DIRTY ); - CheckWriteCoins(PRUNED, VALUE2, VALUE2, DIRTY|FRESH, DIRTY , DIRTY|FRESH); - CheckWriteCoins(PRUNED, VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH, DIRTY|FRESH); + CheckWriteCoins(SPENT , ABSENT, SPENT , 0 , NO_ENTRY , 0 ); + CheckWriteCoins(SPENT , ABSENT, SPENT , FRESH , NO_ENTRY , FRESH ); + CheckWriteCoins(SPENT , ABSENT, SPENT , DIRTY , NO_ENTRY , DIRTY ); + CheckWriteCoins(SPENT , ABSENT, SPENT , DIRTY|FRESH, NO_ENTRY , DIRTY|FRESH); + CheckWriteCoins(SPENT , SPENT , SPENT , 0 , DIRTY , DIRTY ); + CheckWriteCoins(SPENT , SPENT , SPENT , 0 , DIRTY|FRESH, DIRTY ); + CheckWriteCoins(SPENT , SPENT , ABSENT, FRESH , DIRTY , NO_ENTRY ); + CheckWriteCoins(SPENT , SPENT , ABSENT, FRESH , DIRTY|FRESH, NO_ENTRY ); + CheckWriteCoins(SPENT , SPENT , SPENT , DIRTY , DIRTY , DIRTY ); + CheckWriteCoins(SPENT , SPENT , SPENT , DIRTY , DIRTY|FRESH, DIRTY ); + CheckWriteCoins(SPENT , SPENT , ABSENT, DIRTY|FRESH, DIRTY , NO_ENTRY ); + CheckWriteCoins(SPENT , SPENT , ABSENT, DIRTY|FRESH, DIRTY|FRESH, NO_ENTRY ); + CheckWriteCoins(SPENT , VALUE2, VALUE2, 0 , DIRTY , DIRTY ); + CheckWriteCoins(SPENT , VALUE2, VALUE2, 0 , DIRTY|FRESH, DIRTY ); + CheckWriteCoins(SPENT , VALUE2, VALUE2, FRESH , DIRTY , DIRTY|FRESH); + CheckWriteCoins(SPENT , VALUE2, VALUE2, FRESH , DIRTY|FRESH, DIRTY|FRESH); + CheckWriteCoins(SPENT , VALUE2, VALUE2, DIRTY , DIRTY , DIRTY ); + CheckWriteCoins(SPENT , VALUE2, VALUE2, DIRTY , DIRTY|FRESH, DIRTY ); + CheckWriteCoins(SPENT , VALUE2, VALUE2, DIRTY|FRESH, DIRTY , DIRTY|FRESH); + CheckWriteCoins(SPENT , VALUE2, VALUE2, DIRTY|FRESH, DIRTY|FRESH, DIRTY|FRESH); CheckWriteCoins(VALUE1, ABSENT, VALUE1, 0 , NO_ENTRY , 0 ); CheckWriteCoins(VALUE1, ABSENT, VALUE1, FRESH , NO_ENTRY , FRESH ); CheckWriteCoins(VALUE1, ABSENT, VALUE1, DIRTY , NO_ENTRY , DIRTY ); CheckWriteCoins(VALUE1, ABSENT, VALUE1, DIRTY|FRESH, NO_ENTRY , DIRTY|FRESH); - CheckWriteCoins(VALUE1, PRUNED, PRUNED, 0 , DIRTY , DIRTY ); - CheckWriteCoins(VALUE1, PRUNED, FAIL , 0 , DIRTY|FRESH, NO_ENTRY ); - CheckWriteCoins(VALUE1, PRUNED, ABSENT, FRESH , DIRTY , NO_ENTRY ); - CheckWriteCoins(VALUE1, PRUNED, FAIL , FRESH , DIRTY|FRESH, NO_ENTRY ); - CheckWriteCoins(VALUE1, PRUNED, PRUNED, DIRTY , DIRTY , DIRTY ); - CheckWriteCoins(VALUE1, PRUNED, FAIL , DIRTY , DIRTY|FRESH, NO_ENTRY ); - CheckWriteCoins(VALUE1, PRUNED, ABSENT, DIRTY|FRESH, DIRTY , NO_ENTRY ); - CheckWriteCoins(VALUE1, PRUNED, FAIL , DIRTY|FRESH, DIRTY|FRESH, NO_ENTRY ); + CheckWriteCoins(VALUE1, SPENT , SPENT , 0 , DIRTY , DIRTY ); + CheckWriteCoins(VALUE1, SPENT , FAIL , 0 , DIRTY|FRESH, NO_ENTRY ); + CheckWriteCoins(VALUE1, SPENT , ABSENT, FRESH , DIRTY , NO_ENTRY ); + CheckWriteCoins(VALUE1, SPENT , FAIL , FRESH , DIRTY|FRESH, NO_ENTRY ); + CheckWriteCoins(VALUE1, SPENT , SPENT , DIRTY , DIRTY , DIRTY ); + CheckWriteCoins(VALUE1, SPENT , FAIL , DIRTY , DIRTY|FRESH, NO_ENTRY ); + CheckWriteCoins(VALUE1, SPENT , ABSENT, DIRTY|FRESH, DIRTY , NO_ENTRY ); + CheckWriteCoins(VALUE1, SPENT , FAIL , DIRTY|FRESH, DIRTY|FRESH, NO_ENTRY ); CheckWriteCoins(VALUE1, VALUE2, VALUE2, 0 , DIRTY , DIRTY ); CheckWriteCoins(VALUE1, VALUE2, FAIL , 0 , DIRTY|FRESH, NO_ENTRY ); CheckWriteCoins(VALUE1, VALUE2, VALUE2, FRESH , DIRTY , DIRTY|FRESH); @@ -854,8 +871,8 @@ BOOST_AUTO_TEST_CASE(ccoins_write) // they would be too repetitive (the parent cache is never updated in these // cases). The loop below covers these cases and makes sure the parent cache // is always left unchanged. - for (const CAmount parent_value : {ABSENT, PRUNED, VALUE1}) - for (const CAmount child_value : {ABSENT, PRUNED, VALUE2}) + for (const CAmount parent_value : {ABSENT, SPENT, VALUE1}) + for (const CAmount child_value : {ABSENT, SPENT, VALUE2}) for (const char parent_flags : parent_value == ABSENT ? ABSENT_FLAGS : FLAGS) for (const char child_flags : child_value == ABSENT ? ABSENT_FLAGS : CLEAN_FLAGS) CheckWriteCoins(parent_value, child_value, parent_value, parent_flags, child_flags, parent_flags); diff --git a/src/test/compilerbug_tests.cpp b/src/test/compilerbug_tests.cpp index 1a6fcda009..b68bc279e1 100644 --- a/src/test/compilerbug_tests.cpp +++ b/src/test/compilerbug_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> +#include <test/util/setup_common.h> BOOST_FIXTURE_TEST_SUITE(compilerbug_tests, BasicTestingSetup) diff --git a/src/test/compress_tests.cpp b/src/test/compress_tests.cpp index 22eae91cf0..df1a119d79 100644 --- a/src/test/compress_tests.cpp +++ b/src/test/compress_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <compressor.h> -#include <test/util/setup_common.h> #include <script/standard.h> +#include <test/util/setup_common.h> #include <stdint.h> diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 2deb0c5bfc..f64251fe32 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -1,21 +1,21 @@ -// Copyright (c) 2014-2019 The Bitcoin Core developers +// Copyright (c) 2014-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <crypto/aes.h> #include <crypto/chacha20.h> #include <crypto/chacha_poly_aead.h> -#include <crypto/poly1305.h> #include <crypto/hkdf_sha256_32.h> #include <crypto/hmac_sha256.h> #include <crypto/hmac_sha512.h> +#include <crypto/poly1305.h> #include <crypto/ripemd160.h> #include <crypto/sha1.h> #include <crypto/sha256.h> #include <crypto/sha512.h> #include <random.h> -#include <util/strencodings.h> #include <test/util/setup_common.h> +#include <util/strencodings.h> #include <vector> diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index 6be24c0845..3a951d28ae 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -1,13 +1,13 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <boost/test/unit_test.hpp> #include <cuckoocache.h> +#include <deque> +#include <random.h> #include <script/sigcache.h> #include <test/util/setup_common.h> -#include <random.h> #include <thread> -#include <deque> /** Test Suite for CuckooCache * diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index 3dfae29de6..c378546e8b 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <dbwrapper.h> -#include <uint256.h> #include <test/util/setup_common.h> +#include <uint256.h> #include <util/memory.h> #include <memory> diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 3154c619d2..121b80ab2d 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -1,16 +1,18 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <vector> -#include <string> +#include <script/descriptor.h> #include <script/sign.h> #include <script/standard.h> #include <test/util/setup_common.h> -#include <boost/test/unit_test.hpp> -#include <script/descriptor.h> #include <util/strencodings.h> +#include <boost/test/unit_test.hpp> + +#include <string> +#include <vector> + namespace { void CheckUnparsable(const std::string& prv, const std::string& pub, const std::string& expected_error) diff --git a/src/test/fuzz/addition_overflow.cpp b/src/test/fuzz/addition_overflow.cpp new file mode 100644 index 0000000000..a455992b13 --- /dev/null +++ b/src/test/fuzz/addition_overflow.cpp @@ -0,0 +1,55 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <string> +#include <vector> + +#if defined(__has_builtin) +#if __has_builtin(__builtin_add_overflow) +#define HAVE_BUILTIN_ADD_OVERFLOW +#endif +#elif defined(__GNUC__) && (__GNUC__ >= 5) +#define HAVE_BUILTIN_ADD_OVERFLOW +#endif + +namespace { +template <typename T> +void TestAdditionOverflow(FuzzedDataProvider& fuzzed_data_provider) +{ + const T i = fuzzed_data_provider.ConsumeIntegral<T>(); + const T j = fuzzed_data_provider.ConsumeIntegral<T>(); + const bool is_addition_overflow_custom = AdditionOverflow(i, j); +#if defined(HAVE_BUILTIN_ADD_OVERFLOW) + T result_builtin; + const bool is_addition_overflow_builtin = __builtin_add_overflow(i, j, &result_builtin); + assert(is_addition_overflow_custom == is_addition_overflow_builtin); + if (!is_addition_overflow_custom) { + assert(i + j == result_builtin); + } +#else + if (!is_addition_overflow_custom) { + (void)(i + j); + } +#endif +} +} // namespace + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + TestAdditionOverflow<int64_t>(fuzzed_data_provider); + TestAdditionOverflow<uint64_t>(fuzzed_data_provider); + TestAdditionOverflow<int32_t>(fuzzed_data_provider); + TestAdditionOverflow<uint32_t>(fuzzed_data_provider); + TestAdditionOverflow<int16_t>(fuzzed_data_provider); + TestAdditionOverflow<uint16_t>(fuzzed_data_provider); + TestAdditionOverflow<char>(fuzzed_data_provider); + TestAdditionOverflow<unsigned char>(fuzzed_data_provider); + TestAdditionOverflow<signed char>(fuzzed_data_provider); +} diff --git a/src/test/fuzz/base_encode_decode.cpp b/src/test/fuzz/base_encode_decode.cpp index adad6b3f96..8d49f93c2f 100644 --- a/src/test/fuzz/base_encode_decode.cpp +++ b/src/test/fuzz/base_encode_decode.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,8 +6,8 @@ #include <base58.h> #include <psbt.h> -#include <util/string.h> #include <util/strencodings.h> +#include <util/string.h> #include <cassert> #include <cstdint> diff --git a/src/test/fuzz/block.cpp b/src/test/fuzz/block.cpp index 9d0ad369a2..f30fa03e0b 100644 --- a/src/test/fuzz/block.cpp +++ b/src/test/fuzz/block.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,8 +7,8 @@ #include <consensus/validation.h> #include <core_io.h> #include <core_memusage.h> -#include <pubkey.h> #include <primitives/block.h> +#include <pubkey.h> #include <streams.h> #include <test/fuzz/fuzz.h> #include <validation.h> @@ -62,4 +62,8 @@ void test_one_input(const std::vector<uint8_t>& buffer) const size_t raw_memory_size = RecursiveDynamicUsage(block); const size_t raw_memory_size_as_shared_ptr = RecursiveDynamicUsage(std::make_shared<CBlock>(block)); assert(raw_memory_size_as_shared_ptr > raw_memory_size); + CBlock block_copy = block; + block_copy.SetNull(); + const bool is_null = block_copy.IsNull(); + assert(is_null); } diff --git a/src/test/fuzz/bloom_filter.cpp b/src/test/fuzz/bloom_filter.cpp index d1112f8e62..50036ce5bd 100644 --- a/src/test/fuzz/bloom_filter.cpp +++ b/src/test/fuzz/bloom_filter.cpp @@ -25,7 +25,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) fuzzed_data_provider.ConsumeIntegral<unsigned int>(), static_cast<unsigned char>(fuzzed_data_provider.PickValueInArray({BLOOM_UPDATE_NONE, BLOOM_UPDATE_ALL, BLOOM_UPDATE_P2PUBKEY_ONLY, BLOOM_UPDATE_MASK}))}; while (fuzzed_data_provider.remaining_bytes() > 0) { - switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 6)) { + switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 4)) { case 0: { const std::vector<unsigned char> b = ConsumeRandomLengthByteVector(fuzzed_data_provider); (void)bloom_filter.contains(b); @@ -56,13 +56,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) assert(present); break; } - case 3: - bloom_filter.clear(); - break; - case 4: - bloom_filter.reset(fuzzed_data_provider.ConsumeIntegral<unsigned int>()); - break; - case 5: { + case 3: { const Optional<CMutableTransaction> mut_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (!mut_tx) { break; @@ -71,7 +65,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) (void)bloom_filter.IsRelevantAndUpdate(tx); break; } - case 6: + case 4: bloom_filter.UpdateEmptyFull(); break; } diff --git a/src/test/fuzz/checkqueue.cpp b/src/test/fuzz/checkqueue.cpp new file mode 100644 index 0000000000..2ed097b827 --- /dev/null +++ b/src/test/fuzz/checkqueue.cpp @@ -0,0 +1,65 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <checkqueue.h> +#include <optional.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <string> +#include <vector> + +namespace { +struct DumbCheck { + const bool result = false; + + DumbCheck() = default; + + explicit DumbCheck(const bool _result) : result(_result) + { + } + + bool operator()() const + { + return result; + } + + void swap(DumbCheck& x) + { + } +}; +} // namespace + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + + const unsigned int batch_size = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, 1024); + CCheckQueue<DumbCheck> check_queue_1{batch_size}; + CCheckQueue<DumbCheck> check_queue_2{batch_size}; + std::vector<DumbCheck> checks_1; + std::vector<DumbCheck> checks_2; + const int size = fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 1024); + for (int i = 0; i < size; ++i) { + const bool result = fuzzed_data_provider.ConsumeBool(); + checks_1.emplace_back(result); + checks_2.emplace_back(result); + } + if (fuzzed_data_provider.ConsumeBool()) { + check_queue_1.Add(checks_1); + } + if (fuzzed_data_provider.ConsumeBool()) { + (void)check_queue_1.Wait(); + } + + CCheckQueueControl<DumbCheck> check_queue_control{&check_queue_2}; + if (fuzzed_data_provider.ConsumeBool()) { + check_queue_control.Add(checks_2); + } + if (fuzzed_data_provider.ConsumeBool()) { + (void)check_queue_control.Wait(); + } +} diff --git a/src/test/fuzz/cuckoocache.cpp b/src/test/fuzz/cuckoocache.cpp new file mode 100644 index 0000000000..f674efe1b1 --- /dev/null +++ b/src/test/fuzz/cuckoocache.cpp @@ -0,0 +1,49 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <cuckoocache.h> +#include <optional.h> +#include <script/sigcache.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/setup_common.h> + +#include <cstdint> +#include <string> +#include <vector> + +namespace { +FuzzedDataProvider* fuzzed_data_provider_ptr = nullptr; + +struct RandomHasher { + template <uint8_t> + uint32_t operator()(const bool& /* unused */) const + { + assert(fuzzed_data_provider_ptr != nullptr); + return fuzzed_data_provider_ptr->ConsumeIntegral<uint32_t>(); + } +}; +} // namespace + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + fuzzed_data_provider_ptr = &fuzzed_data_provider; + CuckooCache::cache<bool, RandomHasher> cuckoo_cache{}; + if (fuzzed_data_provider.ConsumeBool()) { + const size_t megabytes = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 16); + cuckoo_cache.setup_bytes(megabytes << 20); + } else { + cuckoo_cache.setup(fuzzed_data_provider.ConsumeIntegralInRange<uint32_t>(0, 4096)); + } + while (fuzzed_data_provider.ConsumeBool()) { + if (fuzzed_data_provider.ConsumeBool()) { + cuckoo_cache.insert(fuzzed_data_provider.ConsumeBool()); + } else { + cuckoo_cache.contains(fuzzed_data_provider.ConsumeBool(), fuzzed_data_provider.ConsumeBool()); + } + } + fuzzed_data_provider_ptr = nullptr; +} diff --git a/src/test/fuzz/descriptor_parse.cpp b/src/test/fuzz/descriptor_parse.cpp index 1e0abe94f8..001758ffdb 100644 --- a/src/test/fuzz/descriptor_parse.cpp +++ b/src/test/fuzz/descriptor_parse.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index 964fc85302..54793c890f 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/eval_script.cpp b/src/test/fuzz/eval_script.cpp index 6a1b037630..c556599db3 100644 --- a/src/test/fuzz/eval_script.cpp +++ b/src/test/fuzz/eval_script.cpp @@ -1,11 +1,11 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <pubkey.h> #include <script/interpreter.h> -#include <test/fuzz/fuzz.h> #include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> #include <util/memory.h> #include <limits> diff --git a/src/test/fuzz/fees.cpp b/src/test/fuzz/fees.cpp new file mode 100644 index 0000000000..090994263e --- /dev/null +++ b/src/test/fuzz/fees.cpp @@ -0,0 +1,26 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <amount.h> +#include <optional.h> +#include <policy/fees.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const CFeeRate minimal_incremental_fee{ConsumeMoney(fuzzed_data_provider)}; + FeeFilterRounder fee_filter_rounder{minimal_incremental_fee}; + while (fuzzed_data_provider.ConsumeBool()) { + const CAmount current_minimum_fee = ConsumeMoney(fuzzed_data_provider); + const CAmount rounded_fee = fee_filter_rounder.round(current_minimum_fee); + assert(MoneyRange(rounded_fee)); + } +} diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index a085e36911..6e2188fe86 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/golomb_rice.cpp b/src/test/fuzz/golomb_rice.cpp new file mode 100644 index 0000000000..3e20416116 --- /dev/null +++ b/src/test/fuzz/golomb_rice.cpp @@ -0,0 +1,112 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <blockfilter.h> +#include <serialize.h> +#include <streams.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/util.h> +#include <util/bytevectorhash.h> +#include <util/golombrice.h> + +#include <algorithm> +#include <cassert> +#include <cstdint> +#include <iosfwd> +#include <unordered_set> +#include <vector> + +namespace { +uint64_t MapIntoRange(const uint64_t x, const uint64_t n) +{ + const uint64_t x_hi = x >> 32; + const uint64_t x_lo = x & 0xFFFFFFFF; + const uint64_t n_hi = n >> 32; + const uint64_t n_lo = n & 0xFFFFFFFF; + const uint64_t ac = x_hi * n_hi; + const uint64_t ad = x_hi * n_lo; + const uint64_t bc = x_lo * n_hi; + const uint64_t bd = x_lo * n_lo; + const uint64_t mid34 = (bd >> 32) + (bc & 0xFFFFFFFF) + (ad & 0xFFFFFFFF); + const uint64_t upper64 = ac + (bc >> 32) + (ad >> 32) + (mid34 >> 32); + return upper64; +} + +uint64_t HashToRange(const std::vector<uint8_t>& element, const uint64_t f) +{ + const uint64_t hash = CSipHasher(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL) + .Write(element.data(), element.size()) + .Finalize(); + return MapIntoRange(hash, f); +} + +std::vector<uint64_t> BuildHashedSet(const std::unordered_set<std::vector<uint8_t>, ByteVectorHash>& elements, const uint64_t f) +{ + std::vector<uint64_t> hashed_elements; + hashed_elements.reserve(elements.size()); + for (const std::vector<uint8_t>& element : elements) { + hashed_elements.push_back(HashToRange(element, f)); + } + std::sort(hashed_elements.begin(), hashed_elements.end()); + return hashed_elements; +} +} // namespace + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + std::vector<uint8_t> golomb_rice_data; + std::vector<uint64_t> encoded_deltas; + { + std::unordered_set<std::vector<uint8_t>, ByteVectorHash> elements; + const int n = fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 512); + for (int i = 0; i < n; ++i) { + elements.insert(ConsumeRandomLengthByteVector(fuzzed_data_provider, 16)); + } + CVectorWriter stream(SER_NETWORK, 0, golomb_rice_data, 0); + WriteCompactSize(stream, static_cast<uint32_t>(elements.size())); + BitStreamWriter<CVectorWriter> bitwriter(stream); + if (!elements.empty()) { + uint64_t last_value = 0; + for (const uint64_t value : BuildHashedSet(elements, static_cast<uint64_t>(elements.size()) * static_cast<uint64_t>(BASIC_FILTER_M))) { + const uint64_t delta = value - last_value; + encoded_deltas.push_back(delta); + GolombRiceEncode(bitwriter, BASIC_FILTER_P, delta); + last_value = value; + } + } + bitwriter.Flush(); + } + + std::vector<uint64_t> decoded_deltas; + { + VectorReader stream{SER_NETWORK, 0, golomb_rice_data, 0}; + BitStreamReader<VectorReader> bitreader(stream); + const uint32_t n = static_cast<uint32_t>(ReadCompactSize(stream)); + for (uint32_t i = 0; i < n; ++i) { + decoded_deltas.push_back(GolombRiceDecode(bitreader, BASIC_FILTER_P)); + } + } + + assert(encoded_deltas == decoded_deltas); + + { + const std::vector<uint8_t> random_bytes = ConsumeRandomLengthByteVector(fuzzed_data_provider, 1024); + VectorReader stream{SER_NETWORK, 0, random_bytes, 0}; + uint32_t n; + try { + n = static_cast<uint32_t>(ReadCompactSize(stream)); + } catch (const std::ios_base::failure&) { + return; + } + BitStreamReader<VectorReader> bitreader(stream); + for (uint32_t i = 0; i < std::min<uint32_t>(n, 1024); ++i) { + try { + (void)GolombRiceDecode(bitreader, BASIC_FILTER_P); + } catch (const std::ios_base::failure&) { + } + } + } +} diff --git a/src/test/fuzz/hex.cpp b/src/test/fuzz/hex.cpp index 3bbf0084c2..5fed17c17c 100644 --- a/src/test/fuzz/hex.cpp +++ b/src/test/fuzz/hex.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <core_io.h> -#include <pubkey.h> #include <primitives/block.h> +#include <pubkey.h> #include <rpc/util.h> #include <test/fuzz/fuzz.h> #include <uint256.h> diff --git a/src/test/fuzz/http_request.cpp b/src/test/fuzz/http_request.cpp new file mode 100644 index 0000000000..ebf89749e9 --- /dev/null +++ b/src/test/fuzz/http_request.cpp @@ -0,0 +1,73 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <httpserver.h> +#include <netaddress.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <event2/buffer.h> +#include <event2/event.h> +#include <event2/http.h> +#include <event2/http_struct.h> + +#include <cassert> +#include <cstdint> +#include <string> +#include <vector> + +// workaround for libevent versions before 2.1.1, +// when internal functions didn't have underscores at the end +#if LIBEVENT_VERSION_NUMBER < 0x02010100 +extern "C" int evhttp_parse_firstline(struct evhttp_request*, struct evbuffer*); +extern "C" int evhttp_parse_headers(struct evhttp_request*, struct evbuffer*); +inline int evhttp_parse_firstline_(struct evhttp_request* r, struct evbuffer* b) +{ + return evhttp_parse_firstline(r, b); +} +inline int evhttp_parse_headers_(struct evhttp_request* r, struct evbuffer* b) +{ + return evhttp_parse_headers(r, b); +} +#else +extern "C" int evhttp_parse_firstline_(struct evhttp_request*, struct evbuffer*); +extern "C" int evhttp_parse_headers_(struct evhttp_request*, struct evbuffer*); +#endif + +std::string RequestMethodString(HTTPRequest::RequestMethod m); + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + evhttp_request* evreq = evhttp_request_new(nullptr, nullptr); + assert(evreq != nullptr); + evreq->kind = EVHTTP_REQUEST; + evbuffer* evbuf = evbuffer_new(); + assert(evbuf != nullptr); + const std::vector<uint8_t> http_buffer = ConsumeRandomLengthByteVector(fuzzed_data_provider, 4096); + evbuffer_add(evbuf, http_buffer.data(), http_buffer.size()); + if (evhttp_parse_firstline_(evreq, evbuf) != 1 || evhttp_parse_headers_(evreq, evbuf) != 1) { + evbuffer_free(evbuf); + evhttp_request_free(evreq); + return; + } + + HTTPRequest http_request{evreq, true}; + const HTTPRequest::RequestMethod request_method = http_request.GetRequestMethod(); + (void)RequestMethodString(request_method); + (void)http_request.GetURI(); + (void)http_request.GetHeader("Host"); + const std::string header = fuzzed_data_provider.ConsumeRandomLengthString(16); + (void)http_request.GetHeader(header); + (void)http_request.WriteHeader(header, fuzzed_data_provider.ConsumeRandomLengthString(16)); + (void)http_request.GetHeader(header); + const std::string body = http_request.ReadBody(); + assert(body.empty()); + const CService service = http_request.GetPeer(); + assert(service.ToString() == "[::]:0"); + + evbuffer_free(evbuf); + evhttp_request_free(evreq); +} diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index fff2fabd17..9dbf0fcc90 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -23,6 +23,7 @@ #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> #include <time.h> #include <uint256.h> #include <util/moneystr.h> @@ -35,6 +36,7 @@ #include <cassert> #include <chrono> #include <limits> +#include <set> #include <vector> void initialize() @@ -90,8 +92,12 @@ void test_one_input(const std::vector<uint8_t>& buffer) } (void)GetSizeOfCompactSize(u64); (void)GetSpecialScriptSize(u32); - // (void)GetVirtualTransactionSize(i64, i64); // function defined only for a subset of int64_t inputs - // (void)GetVirtualTransactionSize(i64, i64, u32); // function defined only for a subset of int64_t/uint32_t inputs + if (!MultiplicationOverflow(i64, static_cast<int64_t>(::nBytesPerSigOp)) && !AdditionOverflow(i64 * ::nBytesPerSigOp, static_cast<int64_t>(4))) { + (void)GetVirtualTransactionSize(i64, i64); + } + if (!MultiplicationOverflow(i64, static_cast<int64_t>(u32)) && !AdditionOverflow(i64, static_cast<int64_t>(4)) && !AdditionOverflow(i64 * u32, static_cast<int64_t>(4))) { + (void)GetVirtualTransactionSize(i64, i64, u32); + } (void)HexDigit(ch); (void)MoneyRange(i64); (void)ToString(i64); @@ -109,6 +115,12 @@ void test_one_input(const std::vector<uint8_t>& buffer) (void)memusage::DynamicUsage(u8); const unsigned char uch = static_cast<unsigned char>(u8); (void)memusage::DynamicUsage(uch); + { + const std::set<int64_t> i64s{i64, static_cast<int64_t>(u64)}; + const size_t dynamic_usage = memusage::DynamicUsage(i64s); + const size_t incremental_dynamic_usage = memusage::IncrementalDynamicUsage(i64s); + assert(dynamic_usage == incremental_dynamic_usage * i64s.size()); + } (void)MillisToTimeval(i64); const double d = ser_uint64_to_double(u64); assert(ser_double_to_uint64(d) == u64); diff --git a/src/test/fuzz/locale.cpp b/src/test/fuzz/locale.cpp index 09580c0c52..3597f51e51 100644 --- a/src/test/fuzz/locale.cpp +++ b/src/test/fuzz/locale.cpp @@ -17,8 +17,7 @@ namespace { const std::string locale_identifiers[] = { - "C", "C.UTF-8", "aa_DJ", "aa_DJ.ISO-8859-1", "aa_DJ.UTF-8", "aa_ER", "aa_ER.UTF-8", "aa_ET", "aa_ET.UTF-8", "af_ZA", "af_ZA.ISO-8859-1", "af_ZA.UTF-8", "agr_PE", "agr_PE.UTF-8", "ak_GH", "ak_GH.UTF-8", "am_ET", "am_ET.UTF-8", "an_ES", "an_ES.ISO-8859-15", "an_ES.UTF-8", "anp_IN", "anp_IN.UTF-8", "ar_AE", "ar_AE.ISO-8859-6", "ar_AE.UTF-8", "ar_BH", "ar_BH.ISO-8859-6", "ar_BH.UTF-8", "ar_DZ", "ar_DZ.ISO-8859-6", "ar_DZ.UTF-8", "ar_EG", "ar_EG.ISO-8859-6", "ar_EG.UTF-8", "ar_IN", "ar_IN.UTF-8", "ar_IQ", "ar_IQ.ISO-8859-6", "ar_IQ.UTF-8", "ar_JO", "ar_JO.ISO-8859-6", "ar_JO.UTF-8", "ar_KW", "ar_KW.ISO-8859-6", "ar_KW.UTF-8", "ar_LB", "ar_LB.ISO-8859-6", "ar_LB.UTF-8", "ar_LY", "ar_LY.ISO-8859-6", "ar_LY.UTF-8", "ar_MA", "ar_MA.ISO-8859-6", "ar_MA.UTF-8", "ar_OM", "ar_OM.ISO-8859-6", "ar_OM.UTF-8", "ar_QA", "ar_QA.ISO-8859-6", "ar_QA.UTF-8", "ar_SA", "ar_SA.ISO-8859-6", "ar_SA.UTF-8", "ar_SD", "ar_SD.ISO-8859-6", "ar_SD.UTF-8", "ar_SS", "ar_SS.UTF-8", "ar_SY", "ar_SY.ISO-8859-6", "ar_SY.UTF-8", "ar_TN", "ar_TN.ISO-8859-6", "ar_TN.UTF-8", "ar_YE", "ar_YE.ISO-8859-6", "ar_YE.UTF-8", "as_IN", "as_IN.UTF-8", "ast_ES", "ast_ES.ISO-8859-15", "ast_ES.UTF-8", "ayc_PE", "ayc_PE.UTF-8", "az_AZ", "az_AZ.UTF-8", "az_IR", "az_IR.UTF-8", "be_BY", "be_BY.CP1251", "be_BY.UTF-8", "bem_ZM", "bem_ZM.UTF-8", "ber_DZ", "ber_DZ.UTF-8", "ber_MA", "ber_MA.UTF-8", "bg_BG", "bg_BG.CP1251", "bg_BG.UTF-8", "bho_IN", "bho_IN.UTF-8", "bho_NP", "bho_NP.UTF-8", "bi_VU", "bi_VU.UTF-8", "bn_BD", "bn_BD.UTF-8", "bn_IN", "bn_IN.UTF-8", "bo_CN", "bo_CN.UTF-8", "bo_IN", "bo_IN.UTF-8", "br_FR", "br_FR.ISO-8859-1", "br_FR.UTF-8", "brx_IN", "brx_IN.UTF-8", "bs_BA", "bs_BA.ISO-8859-2", "bs_BA.UTF-8", "byn_ER", "byn_ER.UTF-8", "ca_AD", "ca_AD.ISO-8859-15", "ca_AD.UTF-8", "ca_ES", "ca_ES.ISO-8859-1", "ca_ES.UTF-8", "ca_FR", "ca_FR.ISO-8859-15", "ca_FR.UTF-8", "ca_IT", "ca_IT.ISO-8859-15", "ca_IT.UTF-8", "ce_RU", "ce_RU.UTF-8", "chr_US", "chr_US.UTF-8", "ckb_IQ", "ckb_IQ.UTF-8", "cmn_TW", "cmn_TW.UTF-8", "crh_UA", "crh_UA.UTF-8", "csb_PL", "csb_PL.UTF-8", "cs_CZ", "cs_CZ.ISO-8859-2", "cs_CZ.UTF-8", "cv_RU", "cv_RU.UTF-8", "cy_GB", "cy_GB.ISO-8859-14", "cy_GB.UTF-8", "da_DK", "da_DK.ISO-8859-1", "da_DK.UTF-8", "de_AT", "de_AT.ISO-8859-1", "de_AT.UTF-8", "de_BE", "de_BE.ISO-8859-1", "de_BE.UTF-8", "de_CH", "de_CH.ISO-8859-1", "de_CH.UTF-8", "de_DE", "de_DE.ISO-8859-1", "de_DE.UTF-8", "de_IT", "de_IT.ISO-8859-1", "de_IT.UTF-8", "de_LU", "de_LU.ISO-8859-1", "de_LU.UTF-8", "doi_IN", "doi_IN.UTF-8", "dv_MV", "dv_MV.UTF-8", "dz_BT", "dz_BT.UTF-8", "el_CY", "el_CY.ISO-8859-7", "el_CY.UTF-8", "el_GR", "el_GR.ISO-8859-7", "el_GR.UTF-8", "en_AG", "en_AG.UTF-8", "en_AU", "en_AU.ISO-8859-1", "en_AU.UTF-8", "en_BW", "en_BW.ISO-8859-1", "en_BW.UTF-8", "en_CA", "en_CA.ISO-8859-1", "en_CA.UTF-8", "en_DK", "en_DK.ISO-8859-1", "en_DK.ISO-8859-15", "en_DK.UTF-8", "en_GB", "en_GB.ISO-8859-1", "en_GB.ISO-8859-15", "en_GB.UTF-8", "en_HK", "en_HK.ISO-8859-1", "en_HK.UTF-8", "en_IE", "en_IE.ISO-8859-1", "en_IE.UTF-8", "en_IL", "en_IL.UTF-8", "en_IN", "en_IN.UTF-8", "en_NG", "en_NG.UTF-8", "en_NZ", "en_NZ.ISO-8859-1", "en_NZ.UTF-8", "en_PH", "en_PH.ISO-8859-1", "en_PH.UTF-8", "en_SG", "en_SG.ISO-8859-1", "en_SG.UTF-8", "en_US", "en_US.ISO-8859-1", "en_US.ISO-8859-15", "en_US.UTF-8", "en_ZA", "en_ZA.ISO-8859-1", "en_ZA.UTF-8", "en_ZM", "en_ZM.UTF-8", "en_ZW", "en_ZW.ISO-8859-1", "en_ZW.UTF-8", "es_AR", "es_AR.ISO-8859-1", "es_AR.UTF-8", "es_BO", "es_BO.ISO-8859-1", "es_BO.UTF-8", "es_CL", "es_CL.ISO-8859-1", "es_CL.UTF-8", "es_CO", "es_CO.ISO-8859-1", "es_CO.UTF-8", "es_CR", "es_CR.ISO-8859-1", "es_CR.UTF-8", "es_CU", "es_CU.UTF-8", "es_DO", "es_DO.ISO-8859-1", "es_DO.UTF-8", "es_EC", "es_EC.ISO-8859-1", "es_EC.UTF-8", "es_ES", "es_ES.ISO-8859-1", "es_ES.UTF-8", "es_GT", "es_GT.ISO-8859-1", "es_GT.UTF-8", "es_HN", "es_HN.ISO-8859-1", "es_HN.UTF-8", "es_MX", "es_MX.ISO-8859-1", "es_MX.UTF-8", "es_NI", "es_NI.ISO-8859-1", "es_NI.UTF-8", "es_PA", "es_PA.ISO-8859-1", "es_PA.UTF-8", "es_PE", "es_PE.ISO-8859-1", "es_PE.UTF-8", "es_PR", "es_PR.ISO-8859-1", "es_PR.UTF-8", "es_PY", "es_PY.ISO-8859-1", "es_PY.UTF-8", "es_SV", "es_SV.ISO-8859-1", "es_SV.UTF-8", "es_US", "es_US.ISO-8859-1", "es_US.UTF-8", "es_UY", "es_UY.ISO-8859-1", "es_UY.UTF-8", "es_VE", "es_VE.ISO-8859-1", "es_VE.UTF-8", "et_EE", "et_EE.ISO-8859-1", "et_EE.ISO-8859-15", "et_EE.UTF-8", "eu_ES", "eu_ES.ISO-8859-1", "eu_ES.UTF-8", "eu_FR", "eu_FR.ISO-8859-1", "eu_FR.UTF-8", "fa_IR", "fa_IR.UTF-8", "ff_SN", "ff_SN.UTF-8", "fi_FI", "fi_FI.ISO-8859-1", "fi_FI.UTF-8", "fil_PH", "fil_PH.UTF-8", "fo_FO", "fo_FO.ISO-8859-1", "fo_FO.UTF-8", "fr_BE", "fr_BE.ISO-8859-1", "fr_BE.UTF-8", "fr_CA", "fr_CA.ISO-8859-1", "fr_CA.UTF-8", "fr_CH", "fr_CH.ISO-8859-1", "fr_CH.UTF-8", "fr_FR", "fr_FR.ISO-8859-1", "fr_FR.UTF-8", "fr_LU", "fr_LU.ISO-8859-1", "fr_LU.UTF-8", "fur_IT", "fur_IT.UTF-8", "fy_DE", "fy_DE.UTF-8", "fy_NL", "fy_NL.UTF-8", "ga_IE", "ga_IE.ISO-8859-1", "ga_IE.UTF-8", "gd_GB", "gd_GB.ISO-8859-15", "gd_GB.UTF-8", "gez_ER", "gez_ER.UTF-8", "gez_ET", "gez_ET.UTF-8", "gl_ES", "gl_ES.ISO-8859-1", "gl_ES.UTF-8", "gu_IN", "gu_IN.UTF-8", "gv_GB", "gv_GB.ISO-8859-1", "gv_GB.UTF-8", "hak_TW", "hak_TW.UTF-8", "ha_NG", "ha_NG.UTF-8", "he_IL", "he_IL.ISO-8859-8", "he_IL.UTF-8", "hif_FJ", "hif_FJ.UTF-8", "hi_IN", "hi_IN.UTF-8", "hne_IN", "hne_IN.UTF-8", "hr_HR", "hr_HR.ISO-8859-2", "hr_HR.UTF-8", "hsb_DE", "hsb_DE.ISO-8859-2", "hsb_DE.UTF-8", "ht_HT", "ht_HT.UTF-8", "hu_HU", "hu_HU.ISO-8859-2", "hu_HU.UTF-8", "hy_AM", "hy_AM.ARMSCII-8", "hy_AM.UTF-8", "ia_FR", "ia_FR.UTF-8", "id_ID", "id_ID.ISO-8859-1", "id_ID.UTF-8", "ig_NG", "ig_NG.UTF-8", "ik_CA", "ik_CA.UTF-8", "is_IS", "is_IS.ISO-8859-1", "is_IS.UTF-8", "it_CH", "it_CH.ISO-8859-1", "it_CH.UTF-8", "it_IT", "it_IT.ISO-8859-1", "it_IT.UTF-8", "iu_CA", "iu_CA.UTF-8", "kab_DZ", "kab_DZ.UTF-8", "ka_GE", "ka_GE.GEORGIAN-PS", "ka_GE.UTF-8", "kk_KZ", "kk_KZ.PT154", "kk_KZ.RK1048", "kk_KZ.UTF-8", "kl_GL", "kl_GL.ISO-8859-1", "kl_GL.UTF-8", "km_KH", "km_KH.UTF-8", "kn_IN", "kn_IN.UTF-8", "kok_IN", "kok_IN.UTF-8", "ks_IN", "ks_IN.UTF-8", "ku_TR", "ku_TR.ISO-8859-9", "ku_TR.UTF-8", "kw_GB", "kw_GB.ISO-8859-1", "kw_GB.UTF-8", "ky_KG", "ky_KG.UTF-8", "lb_LU", "lb_LU.UTF-8", "lg_UG", "lg_UG.ISO-8859-10", "lg_UG.UTF-8", "li_BE", "li_BE.UTF-8", "lij_IT", "lij_IT.UTF-8", "li_NL", "li_NL.UTF-8", "ln_CD", "ln_CD.UTF-8", "lo_LA", "lo_LA.UTF-8", "lt_LT", "lt_LT.ISO-8859-13", "lt_LT.UTF-8", "lv_LV", "lv_LV.ISO-8859-13", "lv_LV.UTF-8", "lzh_TW", "lzh_TW.UTF-8", "mag_IN", "mag_IN.UTF-8", "mai_IN", "mai_IN.UTF-8", "mai_NP", "mai_NP.UTF-8", "mfe_MU", "mfe_MU.UTF-8", "mg_MG", "mg_MG.ISO-8859-15", "mg_MG.UTF-8", "mhr_RU", "mhr_RU.UTF-8", "mi_NZ", "mi_NZ.ISO-8859-13", "mi_NZ.UTF-8", "miq_NI", "miq_NI.UTF-8", "mjw_IN", "mjw_IN.UTF-8", "mk_MK", "mk_MK.ISO-8859-5", "mk_MK.UTF-8", "ml_IN", "ml_IN.UTF-8", "mni_IN", "mni_IN.UTF-8", "mn_MN", "mn_MN.UTF-8", "mr_IN", "mr_IN.UTF-8", "ms_MY", "ms_MY.ISO-8859-1", "ms_MY.UTF-8", "mt_MT", "mt_MT.ISO-8859-3", "mt_MT.UTF-8", "my_MM", "my_MM.UTF-8", "nan_TW", "nan_TW.UTF-8", "nb_NO", "nb_NO.ISO-8859-1", "nb_NO.UTF-8", "nds_DE", "nds_DE.UTF-8", "nds_NL", "nds_NL.UTF-8", "ne_NP", "ne_NP.UTF-8", "nhn_MX", "nhn_MX.UTF-8", "niu_NU", "niu_NU.UTF-8", "niu_NZ", "niu_NZ.UTF-8", "nl_AW", "nl_AW.UTF-8", "nl_BE", "nl_BE.ISO-8859-1", "nl_BE.UTF-8", "nl_NL", "nl_NL.ISO-8859-1", "nl_NL.UTF-8", "nn_NO", "nn_NO.ISO-8859-1", "nn_NO.UTF-8", "nr_ZA", "nr_ZA.UTF-8", "nso_ZA", "nso_ZA.UTF-8", "oc_FR", "oc_FR.ISO-8859-1", "oc_FR.UTF-8", "om_ET", "om_ET.UTF-8", "om_KE", "om_KE.ISO-8859-1", "om_KE.UTF-8", "or_IN", "or_IN.UTF-8", "os_RU", "os_RU.UTF-8", "pa_IN", "pa_IN.UTF-8", "pap_AW", "pap_AW.UTF-8", "pap_CW", "pap_CW.UTF-8", "pa_PK", "pa_PK.UTF-8", "pl_PL", "pl_PL.ISO-8859-2", "pl_PL.UTF-8", "ps_AF", "ps_AF.UTF-8", "pt_BR", "pt_BR.ISO-8859-1", "pt_BR.UTF-8", "pt_PT", "pt_PT.ISO-8859-1", "pt_PT.UTF-8", "quz_PE", "quz_PE.UTF-8", "raj_IN", "raj_IN.UTF-8", "ro_RO", "ro_RO.ISO-8859-2", "ro_RO.UTF-8", "ru_RU", "ru_RU.CP1251", "ru_RU.ISO-8859-5", "ru_RU.KOI8-R", "ru_RU.UTF-8", "ru_UA", "ru_UA.KOI8-U", "ru_UA.UTF-8", "rw_RW", "rw_RW.UTF-8", "sa_IN", "sa_IN.UTF-8", "sat_IN", "sat_IN.UTF-8", "sc_IT", "sc_IT.UTF-8", "sd_IN", "sd_IN.UTF-8", "sd_PK", "sd_PK.UTF-8", "se_NO", "se_NO.UTF-8", "sgs_LT", "sgs_LT.UTF-8", "shn_MM", "shn_MM.UTF-8", "shs_CA", "shs_CA.UTF-8", "sid_ET", "sid_ET.UTF-8", "si_LK", "si_LK.UTF-8", "sk_SK", "sk_SK.ISO-8859-2", "sk_SK.UTF-8", "sl_SI", "sl_SI.ISO-8859-2", "sl_SI.UTF-8", "sm_WS", "sm_WS.UTF-8", "so_DJ", "so_DJ.ISO-8859-1", "so_DJ.UTF-8", "so_ET", "so_ET.UTF-8", "so_KE", "so_KE.ISO-8859-1", "so_KE.UTF-8", "so_SO", "so_SO.ISO-8859-1", "so_SO.UTF-8", "sq_AL", "sq_AL.ISO-8859-1", "sq_AL.UTF-8", "sq_MK", "sq_MK.UTF-8", "sr_ME", "sr_ME.UTF-8", "sr_RS", "sr_RS.UTF-8", "ss_ZA", "ss_ZA.UTF-8", "st_ZA", "st_ZA.ISO-8859-1", "st_ZA.UTF-8", "sv_FI", "sv_FI.ISO-8859-1", "sv_FI.UTF-8", "sv_SE", "sv_SE.ISO-8859-1", "sv_SE.ISO-8859-15", "sv_SE.UTF-8", "sw_KE", "sw_KE.UTF-8", "sw_TZ", "sw_TZ.UTF-8", "szl_PL", "szl_PL.UTF-8", "ta_IN", "ta_IN.UTF-8", "ta_LK", "ta_LK.UTF-8", "te_IN", "te_IN.UTF-8", "tg_TJ", "tg_TJ.KOI8-T", "tg_TJ.UTF-8", "the_NP", "the_NP.UTF-8", "th_TH", "th_TH.TIS-620", "th_TH.UTF-8", "ti_ER", "ti_ER.UTF-8", "ti_ET", "ti_ET.UTF-8", "tig_ER", "tig_ER.UTF-8", "tk_TM", "tk_TM.UTF-8", "tl_PH", "tl_PH.ISO-8859-1", "tl_PH.UTF-8", "tn_ZA", "tn_ZA.UTF-8", "to_TO", "to_TO.UTF-8", "tpi_PG", "tpi_PG.UTF-8", "tr_CY", "tr_CY.ISO-8859-9", "tr_CY.UTF-8", "tr_TR", "tr_TR.ISO-8859-9", "tr_TR.UTF-8", "ts_ZA", "ts_ZA.UTF-8", "tt_RU", "tt_RU.UTF-8", "ug_CN", "ug_CN.UTF-8", "uk_UA", "uk_UA.KOI8-U", "uk_UA.UTF-8", "unm_US", "unm_US.UTF-8", "ur_IN", "ur_IN.UTF-8", "ur_PK", "ur_PK.UTF-8", "uz_UZ", "uz_UZ.ISO-8859-1", "uz_UZ.UTF-8", "ve_ZA", "ve_ZA.UTF-8", "vi_VN", "vi_VN.UTF-8", "wa_BE", "wa_BE.ISO-8859-1", "wa_BE.UTF-8", "wae_CH", "wae_CH.UTF-8", "wal_ET", "wal_ET.UTF-8", "wo_SN", "wo_SN.UTF-8", "xh_ZA", "xh_ZA.ISO-8859-1", "xh_ZA.UTF-8", "yi_US", "yi_US.CP1255", "yi_US.UTF-8", "yo_NG", "yo_NG.UTF-8", "yue_HK", "yue_HK.UTF-8", "yuw_PG", "yuw_PG.UTF-8", "zh_CN", "zh_CN.GB18030", "zh_CN.GB2312", "zh_CN.GBK", "zh_CN.UTF-8", "zh_HK", "zh_HK.BIG5-HKSCS", "zh_HK.UTF-8", "zh_SG", "zh_SG.GB2312", "zh_SG.GBK", "zh_SG.UTF-8", "zh_TW", "zh_TW.BIG5", "zh_TW.EUC-TW", "zh_TW.UTF-8", "zu_ZA", "zu_ZA.ISO-8859-1", "zu_ZA.UTF-8" -}; + "C", "C.UTF-8", "aa_DJ", "aa_DJ.ISO-8859-1", "aa_DJ.UTF-8", "aa_ER", "aa_ER.UTF-8", "aa_ET", "aa_ET.UTF-8", "af_ZA", "af_ZA.ISO-8859-1", "af_ZA.UTF-8", "agr_PE", "agr_PE.UTF-8", "ak_GH", "ak_GH.UTF-8", "am_ET", "am_ET.UTF-8", "an_ES", "an_ES.ISO-8859-15", "an_ES.UTF-8", "anp_IN", "anp_IN.UTF-8", "ar_AE", "ar_AE.ISO-8859-6", "ar_AE.UTF-8", "ar_BH", "ar_BH.ISO-8859-6", "ar_BH.UTF-8", "ar_DZ", "ar_DZ.ISO-8859-6", "ar_DZ.UTF-8", "ar_EG", "ar_EG.ISO-8859-6", "ar_EG.UTF-8", "ar_IN", "ar_IN.UTF-8", "ar_IQ", "ar_IQ.ISO-8859-6", "ar_IQ.UTF-8", "ar_JO", "ar_JO.ISO-8859-6", "ar_JO.UTF-8", "ar_KW", "ar_KW.ISO-8859-6", "ar_KW.UTF-8", "ar_LB", "ar_LB.ISO-8859-6", "ar_LB.UTF-8", "ar_LY", "ar_LY.ISO-8859-6", "ar_LY.UTF-8", "ar_MA", "ar_MA.ISO-8859-6", "ar_MA.UTF-8", "ar_OM", "ar_OM.ISO-8859-6", "ar_OM.UTF-8", "ar_QA", "ar_QA.ISO-8859-6", "ar_QA.UTF-8", "ar_SA", "ar_SA.ISO-8859-6", "ar_SA.UTF-8", "ar_SD", "ar_SD.ISO-8859-6", "ar_SD.UTF-8", "ar_SS", "ar_SS.UTF-8", "ar_SY", "ar_SY.ISO-8859-6", "ar_SY.UTF-8", "ar_TN", "ar_TN.ISO-8859-6", "ar_TN.UTF-8", "ar_YE", "ar_YE.ISO-8859-6", "ar_YE.UTF-8", "as_IN", "as_IN.UTF-8", "ast_ES", "ast_ES.ISO-8859-15", "ast_ES.UTF-8", "ayc_PE", "ayc_PE.UTF-8", "az_AZ", "az_AZ.UTF-8", "az_IR", "az_IR.UTF-8", "be_BY", "be_BY.CP1251", "be_BY.UTF-8", "bem_ZM", "bem_ZM.UTF-8", "ber_DZ", "ber_DZ.UTF-8", "ber_MA", "ber_MA.UTF-8", "bg_BG", "bg_BG.CP1251", "bg_BG.UTF-8", "bho_IN", "bho_IN.UTF-8", "bho_NP", "bho_NP.UTF-8", "bi_VU", "bi_VU.UTF-8", "bn_BD", "bn_BD.UTF-8", "bn_IN", "bn_IN.UTF-8", "bo_CN", "bo_CN.UTF-8", "bo_IN", "bo_IN.UTF-8", "br_FR", "br_FR.ISO-8859-1", "br_FR.UTF-8", "brx_IN", "brx_IN.UTF-8", "bs_BA", "bs_BA.ISO-8859-2", "bs_BA.UTF-8", "byn_ER", "byn_ER.UTF-8", "ca_AD", "ca_AD.ISO-8859-15", "ca_AD.UTF-8", "ca_ES", "ca_ES.ISO-8859-1", "ca_ES.UTF-8", "ca_FR", "ca_FR.ISO-8859-15", "ca_FR.UTF-8", "ca_IT", "ca_IT.ISO-8859-15", "ca_IT.UTF-8", "ce_RU", "ce_RU.UTF-8", "chr_US", "chr_US.UTF-8", "ckb_IQ", "ckb_IQ.UTF-8", "cmn_TW", "cmn_TW.UTF-8", "crh_UA", "crh_UA.UTF-8", "csb_PL", "csb_PL.UTF-8", "cs_CZ", "cs_CZ.ISO-8859-2", "cs_CZ.UTF-8", "cv_RU", "cv_RU.UTF-8", "cy_GB", "cy_GB.ISO-8859-14", "cy_GB.UTF-8", "da_DK", "da_DK.ISO-8859-1", "da_DK.UTF-8", "de_AT", "de_AT.ISO-8859-1", "de_AT.UTF-8", "de_BE", "de_BE.ISO-8859-1", "de_BE.UTF-8", "de_CH", "de_CH.ISO-8859-1", "de_CH.UTF-8", "de_DE", "de_DE.ISO-8859-1", "de_DE.UTF-8", "de_IT", "de_IT.ISO-8859-1", "de_IT.UTF-8", "de_LU", "de_LU.ISO-8859-1", "de_LU.UTF-8", "doi_IN", "doi_IN.UTF-8", "dv_MV", "dv_MV.UTF-8", "dz_BT", "dz_BT.UTF-8", "el_CY", "el_CY.ISO-8859-7", "el_CY.UTF-8", "el_GR", "el_GR.ISO-8859-7", "el_GR.UTF-8", "en_AG", "en_AG.UTF-8", "en_AU", "en_AU.ISO-8859-1", "en_AU.UTF-8", "en_BW", "en_BW.ISO-8859-1", "en_BW.UTF-8", "en_CA", "en_CA.ISO-8859-1", "en_CA.UTF-8", "en_DK", "en_DK.ISO-8859-1", "en_DK.ISO-8859-15", "en_DK.UTF-8", "en_GB", "en_GB.ISO-8859-1", "en_GB.ISO-8859-15", "en_GB.UTF-8", "en_HK", "en_HK.ISO-8859-1", "en_HK.UTF-8", "en_IE", "en_IE.ISO-8859-1", "en_IE.UTF-8", "en_IL", "en_IL.UTF-8", "en_IN", "en_IN.UTF-8", "en_NG", "en_NG.UTF-8", "en_NZ", "en_NZ.ISO-8859-1", "en_NZ.UTF-8", "en_PH", "en_PH.ISO-8859-1", "en_PH.UTF-8", "en_SG", "en_SG.ISO-8859-1", "en_SG.UTF-8", "en_US", "en_US.ISO-8859-1", "en_US.ISO-8859-15", "en_US.UTF-8", "en_ZA", "en_ZA.ISO-8859-1", "en_ZA.UTF-8", "en_ZM", "en_ZM.UTF-8", "en_ZW", "en_ZW.ISO-8859-1", "en_ZW.UTF-8", "es_AR", "es_AR.ISO-8859-1", "es_AR.UTF-8", "es_BO", "es_BO.ISO-8859-1", "es_BO.UTF-8", "es_CL", "es_CL.ISO-8859-1", "es_CL.UTF-8", "es_CO", "es_CO.ISO-8859-1", "es_CO.UTF-8", "es_CR", "es_CR.ISO-8859-1", "es_CR.UTF-8", "es_CU", "es_CU.UTF-8", "es_DO", "es_DO.ISO-8859-1", "es_DO.UTF-8", "es_EC", "es_EC.ISO-8859-1", "es_EC.UTF-8", "es_ES", "es_ES.ISO-8859-1", "es_ES.UTF-8", "es_GT", "es_GT.ISO-8859-1", "es_GT.UTF-8", "es_HN", "es_HN.ISO-8859-1", "es_HN.UTF-8", "es_MX", "es_MX.ISO-8859-1", "es_MX.UTF-8", "es_NI", "es_NI.ISO-8859-1", "es_NI.UTF-8", "es_PA", "es_PA.ISO-8859-1", "es_PA.UTF-8", "es_PE", "es_PE.ISO-8859-1", "es_PE.UTF-8", "es_PR", "es_PR.ISO-8859-1", "es_PR.UTF-8", "es_PY", "es_PY.ISO-8859-1", "es_PY.UTF-8", "es_SV", "es_SV.ISO-8859-1", "es_SV.UTF-8", "es_US", "es_US.ISO-8859-1", "es_US.UTF-8", "es_UY", "es_UY.ISO-8859-1", "es_UY.UTF-8", "es_VE", "es_VE.ISO-8859-1", "es_VE.UTF-8", "et_EE", "et_EE.ISO-8859-1", "et_EE.ISO-8859-15", "et_EE.UTF-8", "eu_ES", "eu_ES.ISO-8859-1", "eu_ES.UTF-8", "eu_FR", "eu_FR.ISO-8859-1", "eu_FR.UTF-8", "fa_IR", "fa_IR.UTF-8", "ff_SN", "ff_SN.UTF-8", "fi_FI", "fi_FI.ISO-8859-1", "fi_FI.UTF-8", "fil_PH", "fil_PH.UTF-8", "fo_FO", "fo_FO.ISO-8859-1", "fo_FO.UTF-8", "fr_BE", "fr_BE.ISO-8859-1", "fr_BE.UTF-8", "fr_CA", "fr_CA.ISO-8859-1", "fr_CA.UTF-8", "fr_CH", "fr_CH.ISO-8859-1", "fr_CH.UTF-8", "fr_FR", "fr_FR.ISO-8859-1", "fr_FR.UTF-8", "fr_LU", "fr_LU.ISO-8859-1", "fr_LU.UTF-8", "fur_IT", "fur_IT.UTF-8", "fy_DE", "fy_DE.UTF-8", "fy_NL", "fy_NL.UTF-8", "ga_IE", "ga_IE.ISO-8859-1", "ga_IE.UTF-8", "gd_GB", "gd_GB.ISO-8859-15", "gd_GB.UTF-8", "gez_ER", "gez_ER.UTF-8", "gez_ET", "gez_ET.UTF-8", "gl_ES", "gl_ES.ISO-8859-1", "gl_ES.UTF-8", "gu_IN", "gu_IN.UTF-8", "gv_GB", "gv_GB.ISO-8859-1", "gv_GB.UTF-8", "hak_TW", "hak_TW.UTF-8", "ha_NG", "ha_NG.UTF-8", "he_IL", "he_IL.ISO-8859-8", "he_IL.UTF-8", "hif_FJ", "hif_FJ.UTF-8", "hi_IN", "hi_IN.UTF-8", "hne_IN", "hne_IN.UTF-8", "hr_HR", "hr_HR.ISO-8859-2", "hr_HR.UTF-8", "hsb_DE", "hsb_DE.ISO-8859-2", "hsb_DE.UTF-8", "ht_HT", "ht_HT.UTF-8", "hu_HU", "hu_HU.ISO-8859-2", "hu_HU.UTF-8", "hy_AM", "hy_AM.ARMSCII-8", "hy_AM.UTF-8", "ia_FR", "ia_FR.UTF-8", "id_ID", "id_ID.ISO-8859-1", "id_ID.UTF-8", "ig_NG", "ig_NG.UTF-8", "ik_CA", "ik_CA.UTF-8", "is_IS", "is_IS.ISO-8859-1", "is_IS.UTF-8", "it_CH", "it_CH.ISO-8859-1", "it_CH.UTF-8", "it_IT", "it_IT.ISO-8859-1", "it_IT.UTF-8", "iu_CA", "iu_CA.UTF-8", "kab_DZ", "kab_DZ.UTF-8", "ka_GE", "ka_GE.GEORGIAN-PS", "ka_GE.UTF-8", "kk_KZ", "kk_KZ.PT154", "kk_KZ.RK1048", "kk_KZ.UTF-8", "kl_GL", "kl_GL.ISO-8859-1", "kl_GL.UTF-8", "km_KH", "km_KH.UTF-8", "kn_IN", "kn_IN.UTF-8", "kok_IN", "kok_IN.UTF-8", "ks_IN", "ks_IN.UTF-8", "ku_TR", "ku_TR.ISO-8859-9", "ku_TR.UTF-8", "kw_GB", "kw_GB.ISO-8859-1", "kw_GB.UTF-8", "ky_KG", "ky_KG.UTF-8", "lb_LU", "lb_LU.UTF-8", "lg_UG", "lg_UG.ISO-8859-10", "lg_UG.UTF-8", "li_BE", "li_BE.UTF-8", "lij_IT", "lij_IT.UTF-8", "li_NL", "li_NL.UTF-8", "ln_CD", "ln_CD.UTF-8", "lo_LA", "lo_LA.UTF-8", "lt_LT", "lt_LT.ISO-8859-13", "lt_LT.UTF-8", "lv_LV", "lv_LV.ISO-8859-13", "lv_LV.UTF-8", "lzh_TW", "lzh_TW.UTF-8", "mag_IN", "mag_IN.UTF-8", "mai_IN", "mai_IN.UTF-8", "mai_NP", "mai_NP.UTF-8", "mfe_MU", "mfe_MU.UTF-8", "mg_MG", "mg_MG.ISO-8859-15", "mg_MG.UTF-8", "mhr_RU", "mhr_RU.UTF-8", "mi_NZ", "mi_NZ.ISO-8859-13", "mi_NZ.UTF-8", "miq_NI", "miq_NI.UTF-8", "mjw_IN", "mjw_IN.UTF-8", "mk_MK", "mk_MK.ISO-8859-5", "mk_MK.UTF-8", "ml_IN", "ml_IN.UTF-8", "mni_IN", "mni_IN.UTF-8", "mn_MN", "mn_MN.UTF-8", "mr_IN", "mr_IN.UTF-8", "ms_MY", "ms_MY.ISO-8859-1", "ms_MY.UTF-8", "mt_MT", "mt_MT.ISO-8859-3", "mt_MT.UTF-8", "my_MM", "my_MM.UTF-8", "nan_TW", "nan_TW.UTF-8", "nb_NO", "nb_NO.ISO-8859-1", "nb_NO.UTF-8", "nds_DE", "nds_DE.UTF-8", "nds_NL", "nds_NL.UTF-8", "ne_NP", "ne_NP.UTF-8", "nhn_MX", "nhn_MX.UTF-8", "niu_NU", "niu_NU.UTF-8", "niu_NZ", "niu_NZ.UTF-8", "nl_AW", "nl_AW.UTF-8", "nl_BE", "nl_BE.ISO-8859-1", "nl_BE.UTF-8", "nl_NL", "nl_NL.ISO-8859-1", "nl_NL.UTF-8", "nn_NO", "nn_NO.ISO-8859-1", "nn_NO.UTF-8", "nr_ZA", "nr_ZA.UTF-8", "nso_ZA", "nso_ZA.UTF-8", "oc_FR", "oc_FR.ISO-8859-1", "oc_FR.UTF-8", "om_ET", "om_ET.UTF-8", "om_KE", "om_KE.ISO-8859-1", "om_KE.UTF-8", "or_IN", "or_IN.UTF-8", "os_RU", "os_RU.UTF-8", "pa_IN", "pa_IN.UTF-8", "pap_AW", "pap_AW.UTF-8", "pap_CW", "pap_CW.UTF-8", "pa_PK", "pa_PK.UTF-8", "pl_PL", "pl_PL.ISO-8859-2", "pl_PL.UTF-8", "ps_AF", "ps_AF.UTF-8", "pt_BR", "pt_BR.ISO-8859-1", "pt_BR.UTF-8", "pt_PT", "pt_PT.ISO-8859-1", "pt_PT.UTF-8", "quz_PE", "quz_PE.UTF-8", "raj_IN", "raj_IN.UTF-8", "ro_RO", "ro_RO.ISO-8859-2", "ro_RO.UTF-8", "ru_RU", "ru_RU.CP1251", "ru_RU.ISO-8859-5", "ru_RU.KOI8-R", "ru_RU.UTF-8", "ru_UA", "ru_UA.KOI8-U", "ru_UA.UTF-8", "rw_RW", "rw_RW.UTF-8", "sa_IN", "sa_IN.UTF-8", "sat_IN", "sat_IN.UTF-8", "sc_IT", "sc_IT.UTF-8", "sd_IN", "sd_IN.UTF-8", "sd_PK", "sd_PK.UTF-8", "se_NO", "se_NO.UTF-8", "sgs_LT", "sgs_LT.UTF-8", "shn_MM", "shn_MM.UTF-8", "shs_CA", "shs_CA.UTF-8", "sid_ET", "sid_ET.UTF-8", "si_LK", "si_LK.UTF-8", "sk_SK", "sk_SK.ISO-8859-2", "sk_SK.UTF-8", "sl_SI", "sl_SI.ISO-8859-2", "sl_SI.UTF-8", "sm_WS", "sm_WS.UTF-8", "so_DJ", "so_DJ.ISO-8859-1", "so_DJ.UTF-8", "so_ET", "so_ET.UTF-8", "so_KE", "so_KE.ISO-8859-1", "so_KE.UTF-8", "so_SO", "so_SO.ISO-8859-1", "so_SO.UTF-8", "sq_AL", "sq_AL.ISO-8859-1", "sq_AL.UTF-8", "sq_MK", "sq_MK.UTF-8", "sr_ME", "sr_ME.UTF-8", "sr_RS", "sr_RS.UTF-8", "ss_ZA", "ss_ZA.UTF-8", "st_ZA", "st_ZA.ISO-8859-1", "st_ZA.UTF-8", "sv_FI", "sv_FI.ISO-8859-1", "sv_FI.UTF-8", "sv_SE", "sv_SE.ISO-8859-1", "sv_SE.ISO-8859-15", "sv_SE.UTF-8", "sw_KE", "sw_KE.UTF-8", "sw_TZ", "sw_TZ.UTF-8", "szl_PL", "szl_PL.UTF-8", "ta_IN", "ta_IN.UTF-8", "ta_LK", "ta_LK.UTF-8", "te_IN", "te_IN.UTF-8", "tg_TJ", "tg_TJ.KOI8-T", "tg_TJ.UTF-8", "the_NP", "the_NP.UTF-8", "th_TH", "th_TH.TIS-620", "th_TH.UTF-8", "ti_ER", "ti_ER.UTF-8", "ti_ET", "ti_ET.UTF-8", "tig_ER", "tig_ER.UTF-8", "tk_TM", "tk_TM.UTF-8", "tl_PH", "tl_PH.ISO-8859-1", "tl_PH.UTF-8", "tn_ZA", "tn_ZA.UTF-8", "to_TO", "to_TO.UTF-8", "tpi_PG", "tpi_PG.UTF-8", "tr_CY", "tr_CY.ISO-8859-9", "tr_CY.UTF-8", "tr_TR", "tr_TR.ISO-8859-9", "tr_TR.UTF-8", "ts_ZA", "ts_ZA.UTF-8", "tt_RU", "tt_RU.UTF-8", "ug_CN", "ug_CN.UTF-8", "uk_UA", "uk_UA.KOI8-U", "uk_UA.UTF-8", "unm_US", "unm_US.UTF-8", "ur_IN", "ur_IN.UTF-8", "ur_PK", "ur_PK.UTF-8", "uz_UZ", "uz_UZ.ISO-8859-1", "uz_UZ.UTF-8", "ve_ZA", "ve_ZA.UTF-8", "vi_VN", "vi_VN.UTF-8", "wa_BE", "wa_BE.ISO-8859-1", "wa_BE.UTF-8", "wae_CH", "wae_CH.UTF-8", "wal_ET", "wal_ET.UTF-8", "wo_SN", "wo_SN.UTF-8", "xh_ZA", "xh_ZA.ISO-8859-1", "xh_ZA.UTF-8", "yi_US", "yi_US.CP1255", "yi_US.UTF-8", "yo_NG", "yo_NG.UTF-8", "yue_HK", "yue_HK.UTF-8", "yuw_PG", "yuw_PG.UTF-8", "zh_CN", "zh_CN.GB18030", "zh_CN.GB2312", "zh_CN.GBK", "zh_CN.UTF-8", "zh_HK", "zh_HK.BIG5-HKSCS", "zh_HK.UTF-8", "zh_SG", "zh_SG.GB2312", "zh_SG.GBK", "zh_SG.UTF-8", "zh_TW", "zh_TW.BIG5", "zh_TW.EUC-TW", "zh_TW.UTF-8", "zu_ZA", "zu_ZA.ISO-8859-1", "zu_ZA.UTF-8"}; std::string ConsumeLocaleIdentifier(FuzzedDataProvider& fuzzed_data_provider) { @@ -56,7 +55,10 @@ void test_one_input(const std::vector<uint8_t>& buffer) const int64_t atoi64c_without_locale = atoi64(random_string.c_str()); const int64_t random_int64 = fuzzed_data_provider.ConsumeIntegral<int64_t>(); const std::string tostring_without_locale = ToString(random_int64); + // The variable `random_int32` is no longer used, but the harness still needs to + // consume the same data that it did previously to not invalidate existing seeds. const int32_t random_int32 = fuzzed_data_provider.ConsumeIntegral<int32_t>(); + (void)random_int32; const std::string strprintf_int_without_locale = strprintf("%d", random_int64); const double random_double = fuzzed_data_provider.ConsumeFloatingPoint<double>(); const std::string strprintf_double_without_locale = strprintf("%f", random_double); diff --git a/src/test/fuzz/parse_univalue.cpp b/src/test/fuzz/parse_univalue.cpp index 571364aaa6..a269378607 100644 --- a/src/test/fuzz/parse_univalue.cpp +++ b/src/test/fuzz/parse_univalue.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/prevector.cpp b/src/test/fuzz/prevector.cpp new file mode 100644 index 0000000000..64920f4af5 --- /dev/null +++ b/src/test/fuzz/prevector.cpp @@ -0,0 +1,263 @@ +// Copyright (c) 2015-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> + +#include <prevector.h> +#include <vector> + +#include <reverse_iterator.h> +#include <serialize.h> +#include <streams.h> + +namespace { + +template<unsigned int N, typename T> +class prevector_tester { + typedef std::vector<T> realtype; + realtype real_vector; + realtype real_vector_alt; + + typedef prevector<N, T> pretype; + pretype pre_vector; + pretype pre_vector_alt; + + typedef typename pretype::size_type Size; + +public: + void test() const { + const pretype& const_pre_vector = pre_vector; + assert(real_vector.size() == pre_vector.size()); + assert(real_vector.empty() == pre_vector.empty()); + for (Size s = 0; s < real_vector.size(); s++) { + assert(real_vector[s] == pre_vector[s]); + assert(&(pre_vector[s]) == &(pre_vector.begin()[s])); + assert(&(pre_vector[s]) == &*(pre_vector.begin() + s)); + assert(&(pre_vector[s]) == &*((pre_vector.end() + s) - real_vector.size())); + } + // assert(realtype(pre_vector) == real_vector); + assert(pretype(real_vector.begin(), real_vector.end()) == pre_vector); + assert(pretype(pre_vector.begin(), pre_vector.end()) == pre_vector); + size_t pos = 0; + for (const T& v : pre_vector) { + assert(v == real_vector[pos]); + ++pos; + } + for (const T& v : reverse_iterate(pre_vector)) { + --pos; + assert(v == real_vector[pos]); + } + for (const T& v : const_pre_vector) { + assert(v == real_vector[pos]); + ++pos; + } + for (const T& v : reverse_iterate(const_pre_vector)) { + --pos; + assert(v == real_vector[pos]); + } + CDataStream ss1(SER_DISK, 0); + CDataStream ss2(SER_DISK, 0); + ss1 << real_vector; + ss2 << pre_vector; + assert(ss1.size() == ss2.size()); + for (Size s = 0; s < ss1.size(); s++) { + assert(ss1[s] == ss2[s]); + } + } + + void resize(Size s) { + real_vector.resize(s); + assert(real_vector.size() == s); + pre_vector.resize(s); + assert(pre_vector.size() == s); + } + + void reserve(Size s) { + real_vector.reserve(s); + assert(real_vector.capacity() >= s); + pre_vector.reserve(s); + assert(pre_vector.capacity() >= s); + } + + void insert(Size position, const T& value) { + real_vector.insert(real_vector.begin() + position, value); + pre_vector.insert(pre_vector.begin() + position, value); + } + + void insert(Size position, Size count, const T& value) { + real_vector.insert(real_vector.begin() + position, count, value); + pre_vector.insert(pre_vector.begin() + position, count, value); + } + + template<typename I> + void insert_range(Size position, I first, I last) { + real_vector.insert(real_vector.begin() + position, first, last); + pre_vector.insert(pre_vector.begin() + position, first, last); + } + + void erase(Size position) { + real_vector.erase(real_vector.begin() + position); + pre_vector.erase(pre_vector.begin() + position); + } + + void erase(Size first, Size last) { + real_vector.erase(real_vector.begin() + first, real_vector.begin() + last); + pre_vector.erase(pre_vector.begin() + first, pre_vector.begin() + last); + } + + void update(Size pos, const T& value) { + real_vector[pos] = value; + pre_vector[pos] = value; + } + + void push_back(const T& value) { + real_vector.push_back(value); + pre_vector.push_back(value); + } + + void pop_back() { + real_vector.pop_back(); + pre_vector.pop_back(); + } + + void clear() { + real_vector.clear(); + pre_vector.clear(); + } + + void assign(Size n, const T& value) { + real_vector.assign(n, value); + pre_vector.assign(n, value); + } + + Size size() const { + return real_vector.size(); + } + + Size capacity() const { + return pre_vector.capacity(); + } + + void shrink_to_fit() { + pre_vector.shrink_to_fit(); + } + + void swap() { + real_vector.swap(real_vector_alt); + pre_vector.swap(pre_vector_alt); + } + + void move() { + real_vector = std::move(real_vector_alt); + real_vector_alt.clear(); + pre_vector = std::move(pre_vector_alt); + pre_vector_alt.clear(); + } + + void copy() { + real_vector = real_vector_alt; + pre_vector = pre_vector_alt; + } + + void resize_uninitialized(realtype values) { + size_t r = values.size(); + size_t s = real_vector.size() / 2; + if (real_vector.capacity() < s + r) { + real_vector.reserve(s + r); + } + real_vector.resize(s); + pre_vector.resize_uninitialized(s); + for (auto v : values) { + real_vector.push_back(v); + } + auto p = pre_vector.size(); + pre_vector.resize_uninitialized(p + r); + for (auto v : values) { + pre_vector[p] = v; + ++p; + } + } +}; + +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider prov(buffer.data(), buffer.size()); + prevector_tester<8, int> test; + + while (prov.remaining_bytes()) { + switch (prov.ConsumeIntegralInRange<int>(0, 13 + 3 * (test.size() > 0))) { + case 0: + test.insert(prov.ConsumeIntegralInRange<size_t>(0, test.size()), prov.ConsumeIntegral<int>()); + break; + case 1: + test.resize(std::max(0, std::min(30, (int)test.size() + prov.ConsumeIntegralInRange<int>(0, 4) - 2))); + break; + case 2: + test.insert(prov.ConsumeIntegralInRange<size_t>(0, test.size()), 1 + prov.ConsumeBool(), prov.ConsumeIntegral<int>()); + break; + case 3: { + int del = prov.ConsumeIntegralInRange<int>(0, test.size()); + int beg = prov.ConsumeIntegralInRange<int>(0, test.size() - del); + test.erase(beg, beg + del); + break; + } + case 4: + test.push_back(prov.ConsumeIntegral<int>()); + break; + case 5: { + int values[4]; + int num = 1 + prov.ConsumeIntegralInRange<int>(0, 3); + for (int k = 0; k < num; ++k) { + values[k] = prov.ConsumeIntegral<int>(); + } + test.insert_range(prov.ConsumeIntegralInRange<size_t>(0, test.size()), values, values + num); + break; + } + case 6: { + int num = 1 + prov.ConsumeIntegralInRange<int>(0, 15); + std::vector<int> values(num); + for (auto& v : values) { + v = prov.ConsumeIntegral<int>(); + } + test.resize_uninitialized(values); + break; + } + case 7: + test.reserve(prov.ConsumeIntegralInRange<size_t>(0, 32767)); + break; + case 8: + test.shrink_to_fit(); + break; + case 9: + test.clear(); + break; + case 10: + test.assign(prov.ConsumeIntegralInRange<size_t>(0, 32767), prov.ConsumeIntegral<int>()); + break; + case 11: + test.swap(); + break; + case 12: + test.copy(); + break; + case 13: + test.move(); + break; + case 14: + test.update(prov.ConsumeIntegralInRange<size_t>(0, test.size() - 1), prov.ConsumeIntegral<int>()); + break; + case 15: + test.erase(prov.ConsumeIntegralInRange<size_t>(0, test.size() - 1)); + break; + case 16: + test.pop_back(); + break; + } + } + + test.test(); +} diff --git a/src/test/fuzz/primitives_transaction.cpp b/src/test/fuzz/primitives_transaction.cpp new file mode 100644 index 0000000000..2e5ba6bdb0 --- /dev/null +++ b/src/test/fuzz/primitives_transaction.cpp @@ -0,0 +1,34 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <optional.h> +#include <primitives/transaction.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const CScript script = ConsumeScript(fuzzed_data_provider); + const Optional<COutPoint> out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); + if (out_point) { + const CTxIn tx_in{*out_point, script, fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; + (void)tx_in; + } + const CTxOut tx_out_1{ConsumeMoney(fuzzed_data_provider), script}; + const CTxOut tx_out_2{ConsumeMoney(fuzzed_data_provider), ConsumeScript(fuzzed_data_provider)}; + assert((tx_out_1 == tx_out_2) != (tx_out_1 != tx_out_2)); + const Optional<CMutableTransaction> mutable_tx_1 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const Optional<CMutableTransaction> mutable_tx_2 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (mutable_tx_1 && mutable_tx_2) { + const CTransaction tx_1{*mutable_tx_1}; + const CTransaction tx_2{*mutable_tx_2}; + assert((tx_1 == tx_2) != (tx_1 != tx_2)); + } +} diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index dc49dd499a..c03365199a 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -19,20 +19,17 @@ #include <validationinterface.h> #include <version.h> -#include <algorithm> #include <atomic> #include <cassert> #include <chrono> #include <cstdint> #include <iosfwd> #include <iostream> -#include <map> #include <memory> -#include <set> #include <string> #include <vector> -bool ProcessMessage(CNode* pfrom, const std::string& strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc); +bool ProcessMessage(CNode* pfrom, const std::string& msg_type, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams, CTxMemPool& mempool, CConnman* connman, BanMan* banman, const std::atomic<bool>& interruptMsgProc); namespace { @@ -44,25 +41,17 @@ const std::string LIMIT_TO_MESSAGE_TYPE{TO_STRING(MESSAGE_TYPE)}; const std::string LIMIT_TO_MESSAGE_TYPE; #endif -const std::map<std::string, std::set<std::string>> EXPECTED_DESERIALIZATION_EXCEPTIONS = { - {"CDataStream::read(): end of data: iostream error", {"addr", "block", "blocktxn", "cmpctblock", "feefilter", "filteradd", "filterload", "getblocks", "getblocktxn", "getdata", "getheaders", "headers", "inv", "notfound", "ping", "sendcmpct", "tx"}}, - {"CompactSize exceeds limit of type: iostream error", {"cmpctblock"}}, - {"differential value overflow: iostream error", {"getblocktxn"}}, - {"index overflowed 16 bits: iostream error", {"getblocktxn"}}, - {"index overflowed 16-bits: iostream error", {"cmpctblock"}}, - {"indexes overflowed 16 bits: iostream error", {"getblocktxn"}}, - {"non-canonical ReadCompactSize(): iostream error", {"addr", "block", "blocktxn", "cmpctblock", "filteradd", "filterload", "getblocks", "getblocktxn", "getdata", "getheaders", "headers", "inv", "notfound", "tx"}}, - {"ReadCompactSize(): size too large: iostream error", {"addr", "block", "blocktxn", "cmpctblock", "filteradd", "filterload", "getblocks", "getblocktxn", "getdata", "getheaders", "headers", "inv", "notfound", "tx"}}, - {"Superfluous witness record: iostream error", {"block", "blocktxn", "cmpctblock", "tx"}}, - {"Unknown transaction optional data: iostream error", {"block", "blocktxn", "cmpctblock", "tx"}}, -}; - -const RegTestingSetup* g_setup; +const TestingSetup* g_setup; } // namespace void initialize() { - static RegTestingSetup setup{}; + static TestingSetup setup{ + CBaseChainParams::REGTEST, + { + "-nodebuglogfile", + }, + }; g_setup = &setup; for (int i = 0; i < 2 * COINBASE_MATURITY; i++) { @@ -86,13 +75,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) g_setup->m_node.peer_logic->InitializeNode(&p2p_node); try { (void)ProcessMessage(&p2p_node, random_message_type, random_bytes_data_stream, GetTimeMillis(), Params(), *g_setup->m_node.mempool, g_setup->m_node.connman.get(), g_setup->m_node.banman.get(), std::atomic<bool>{false}); - } catch (const std::ios_base::failure& e) { - const std::string exception_message{e.what()}; - const auto p = EXPECTED_DESERIALIZATION_EXCEPTIONS.find(exception_message); - if (p == EXPECTED_DESERIALIZATION_EXCEPTIONS.cend() || p->second.count(random_message_type) == 0) { - std::cout << "Unexpected exception when processing message type \"" << random_message_type << "\": " << exception_message << std::endl; - assert(false); - } + } catch (const std::ios_base::failure&) { } SyncWithValidationInterfaceQueue(); } diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp new file mode 100644 index 0000000000..bcbf65bdca --- /dev/null +++ b/src/test/fuzz/process_messages.cpp @@ -0,0 +1,80 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <consensus/consensus.h> +#include <net.h> +#include <net_processing.h> +#include <protocol.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/mining.h> +#include <test/util/net.h> +#include <test/util/setup_common.h> +#include <util/memory.h> +#include <validation.h> +#include <validationinterface.h> + +const TestingSetup* g_setup; + +void initialize() +{ + static TestingSetup setup{ + CBaseChainParams::REGTEST, + { + "-nodebuglogfile", + }, + }; + g_setup = &setup; + + for (int i = 0; i < 2 * COINBASE_MATURITY; i++) { + MineBlock(g_setup->m_node, CScript() << OP_TRUE); + } + SyncWithValidationInterfaceQueue(); +} + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + + ConnmanTestMsg& connman = *(ConnmanTestMsg*)g_setup->m_node.connman.get(); + std::vector<CNode*> peers; + + const auto num_peers_to_add = fuzzed_data_provider.ConsumeIntegralInRange(1, 3); + for (int i = 0; i < num_peers_to_add; ++i) { + const ServiceFlags service_flags = ServiceFlags(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + const bool inbound{fuzzed_data_provider.ConsumeBool()}; + const bool block_relay_only{fuzzed_data_provider.ConsumeBool()}; + peers.push_back(MakeUnique<CNode>(i, service_flags, 0, INVALID_SOCKET, CAddress{CService{in_addr{0x0100007f}, 7777}, NODE_NETWORK}, 0, 0, CAddress{}, std::string{}, inbound, block_relay_only).release()); + CNode& p2p_node = *peers.back(); + + p2p_node.fSuccessfullyConnected = true; + p2p_node.fPauseSend = false; + p2p_node.nVersion = PROTOCOL_VERSION; + p2p_node.SetSendVersion(PROTOCOL_VERSION); + g_setup->m_node.peer_logic->InitializeNode(&p2p_node); + + connman.AddTestNode(p2p_node); + } + + while (fuzzed_data_provider.ConsumeBool()) { + const std::string random_message_type{fuzzed_data_provider.ConsumeBytesAsString(CMessageHeader::COMMAND_SIZE).c_str()}; + + CSerializedNetMsg net_msg; + net_msg.command = random_message_type; + net_msg.data = ConsumeRandomLengthByteVector(fuzzed_data_provider); + + CNode& random_node = *peers.at(fuzzed_data_provider.ConsumeIntegralInRange<int>(0, peers.size() - 1)); + + (void)connman.ReceiveMsgFrom(random_node, net_msg); + random_node.fPauseSend = false; + + try { + connman.ProcessMessagesOnce(random_node); + } catch (const std::ios_base::failure&) { + } + } + connman.ClearTestNodes(); + SyncWithValidationInterfaceQueue(); +} diff --git a/src/test/fuzz/psbt.cpp b/src/test/fuzz/psbt.cpp index ca3e0b8586..64328fb66e 100644 --- a/src/test/fuzz/psbt.cpp +++ b/src/test/fuzz/psbt.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index 80e2f234d7..de82122dd6 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/script_flags.cpp b/src/test/fuzz/script_flags.cpp index 3d8ece7c61..ffc65eedc0 100644 --- a/src/test/fuzz/script_flags.cpp +++ b/src/test/fuzz/script_flags.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/script_ops.cpp b/src/test/fuzz/script_ops.cpp index 0cd129ba7a..7d24af20ac 100644 --- a/src/test/fuzz/script_ops.cpp +++ b/src/test/fuzz/script_ops.cpp @@ -17,12 +17,16 @@ void test_one_input(const std::vector<uint8_t>& buffer) CScript script = ConsumeScript(fuzzed_data_provider); while (fuzzed_data_provider.remaining_bytes() > 0) { switch (fuzzed_data_provider.ConsumeIntegralInRange(0, 7)) { - case 0: - script += ConsumeScript(fuzzed_data_provider); + case 0: { + CScript s = ConsumeScript(fuzzed_data_provider); + script = std::move(s); break; - case 1: - script = script + ConsumeScript(fuzzed_data_provider); + } + case 1: { + const CScript& s = ConsumeScript(fuzzed_data_provider); + script = s; break; + } case 2: script << fuzzed_data_provider.ConsumeIntegral<int64_t>(); break; diff --git a/src/test/fuzz/signature_checker.cpp b/src/test/fuzz/signature_checker.cpp index 312db27adc..4a8c7a63af 100644 --- a/src/test/fuzz/signature_checker.cpp +++ b/src/test/fuzz/signature_checker.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/transaction.cpp b/src/test/fuzz/transaction.cpp index d8e84f1a0f..d6deb7fc3d 100644 --- a/src/test/fuzz/transaction.cpp +++ b/src/test/fuzz/transaction.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index b70ea6d90e..9c7b0d47a2 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -120,4 +120,15 @@ NODISCARD bool MultiplicationOverflow(const T i, const T j) noexcept } } +template <class T> +NODISCARD bool AdditionOverflow(const T i, const T j) noexcept +{ + static_assert(std::is_integral<T>::value, "Integral required."); + if (std::numeric_limits<T>::is_signed) { + return (i > 0 && j > std::numeric_limits<T>::max() - i) || + (i < 0 && j < std::numeric_limits<T>::min() - i); + } + return std::numeric_limits<T>::max() - i < j; +} + #endif // BITCOIN_TEST_FUZZ_UTIL_H diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index 10fb05ca8a..512e48f8e5 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <test/util/setup_common.h> #include <util/strencodings.h> #include <util/system.h> -#include <test/util/setup_common.h> #include <string> #include <utility> diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index b864e6e599..87f6470afa 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -1,12 +1,12 @@ -// Copyright (c) 2013-2019 The Bitcoin Core developers +// Copyright (c) 2013-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <clientversion.h> #include <crypto/siphash.h> #include <hash.h> -#include <util/strencodings.h> #include <test/util/setup_common.h> +#include <util/strencodings.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/interfaces_tests.cpp b/src/test/interfaces_tests.cpp new file mode 100644 index 0000000000..b0d4de89f3 --- /dev/null +++ b/src/test/interfaces_tests.cpp @@ -0,0 +1,163 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <consensus/validation.h> +#include <interfaces/chain.h> +#include <script/standard.h> +#include <test/util/setup_common.h> +#include <validation.h> + +#include <boost/test/unit_test.hpp> + +using interfaces::FoundBlock; + +BOOST_FIXTURE_TEST_SUITE(interfaces_tests, TestChain100Setup) + +BOOST_AUTO_TEST_CASE(findBlock) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + + uint256 hash; + BOOST_CHECK(chain->findBlock(active[10]->GetBlockHash(), FoundBlock().hash(hash))); + BOOST_CHECK_EQUAL(hash, active[10]->GetBlockHash()); + + int height = -1; + BOOST_CHECK(chain->findBlock(active[20]->GetBlockHash(), FoundBlock().height(height))); + BOOST_CHECK_EQUAL(height, active[20]->nHeight); + + CBlock data; + BOOST_CHECK(chain->findBlock(active[30]->GetBlockHash(), FoundBlock().data(data))); + BOOST_CHECK_EQUAL(data.GetHash(), active[30]->GetBlockHash()); + + int64_t time = -1; + BOOST_CHECK(chain->findBlock(active[40]->GetBlockHash(), FoundBlock().time(time))); + BOOST_CHECK_EQUAL(time, active[40]->GetBlockTime()); + + int64_t max_time = -1; + BOOST_CHECK(chain->findBlock(active[50]->GetBlockHash(), FoundBlock().maxTime(max_time))); + BOOST_CHECK_EQUAL(max_time, active[50]->GetBlockTimeMax()); + + int64_t mtp_time = -1; + BOOST_CHECK(chain->findBlock(active[60]->GetBlockHash(), FoundBlock().mtpTime(mtp_time))); + BOOST_CHECK_EQUAL(mtp_time, active[60]->GetMedianTimePast()); + + BOOST_CHECK(!chain->findBlock({}, FoundBlock())); +} + +BOOST_AUTO_TEST_CASE(findFirstBlockWithTimeAndHeight) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + uint256 hash; + int height; + BOOST_CHECK(chain->findFirstBlockWithTimeAndHeight(/* min_time= */ 0, /* min_height= */ 5, FoundBlock().hash(hash).height(height))); + BOOST_CHECK_EQUAL(hash, active[5]->GetBlockHash()); + BOOST_CHECK_EQUAL(height, 5); + BOOST_CHECK(!chain->findFirstBlockWithTimeAndHeight(/* min_time= */ active.Tip()->GetBlockTimeMax() + 1, /* min_height= */ 0)); +} + +BOOST_AUTO_TEST_CASE(findNextBlock) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + bool reorg; + uint256 hash; + BOOST_CHECK(chain->findNextBlock(active[20]->GetBlockHash(), 20, FoundBlock().hash(hash), &reorg)); + BOOST_CHECK_EQUAL(hash, active[21]->GetBlockHash()); + BOOST_CHECK_EQUAL(reorg, false); + BOOST_CHECK(!chain->findNextBlock(uint256(), 20, {}, &reorg)); + BOOST_CHECK_EQUAL(reorg, true); + BOOST_CHECK(!chain->findNextBlock(active.Tip()->GetBlockHash(), active.Height(), {}, &reorg)); + BOOST_CHECK_EQUAL(reorg, false); +} + +BOOST_AUTO_TEST_CASE(findAncestorByHeight) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + uint256 hash; + BOOST_CHECK(chain->findAncestorByHeight(active[20]->GetBlockHash(), 10, FoundBlock().hash(hash))); + BOOST_CHECK_EQUAL(hash, active[10]->GetBlockHash()); + BOOST_CHECK(!chain->findAncestorByHeight(active[10]->GetBlockHash(), 20)); +} + +BOOST_AUTO_TEST_CASE(findAncestorByHash) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + int height = -1; + BOOST_CHECK(chain->findAncestorByHash(active[20]->GetBlockHash(), active[10]->GetBlockHash(), FoundBlock().height(height))); + BOOST_CHECK_EQUAL(height, 10); + BOOST_CHECK(!chain->findAncestorByHash(active[10]->GetBlockHash(), active[20]->GetBlockHash())); +} + +BOOST_AUTO_TEST_CASE(findCommonAncestor) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + auto* orig_tip = active.Tip(); + for (int i = 0; i < 10; ++i) { + BlockValidationState state; + ChainstateActive().InvalidateBlock(state, Params(), active.Tip()); + } + BOOST_CHECK_EQUAL(active.Height(), orig_tip->nHeight - 10); + coinbaseKey.MakeNewKey(true); + for (int i = 0; i < 20; ++i) { + CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); + } + BOOST_CHECK_EQUAL(active.Height(), orig_tip->nHeight + 10); + uint256 fork_hash; + int fork_height; + int orig_height; + BOOST_CHECK(chain->findCommonAncestor(orig_tip->GetBlockHash(), active.Tip()->GetBlockHash(), FoundBlock().height(fork_height).hash(fork_hash), FoundBlock().height(orig_height))); + BOOST_CHECK_EQUAL(orig_height, orig_tip->nHeight); + BOOST_CHECK_EQUAL(fork_height, orig_tip->nHeight - 10); + BOOST_CHECK_EQUAL(fork_hash, active[fork_height]->GetBlockHash()); + + uint256 active_hash, orig_hash; + BOOST_CHECK(!chain->findCommonAncestor(active.Tip()->GetBlockHash(), {}, {}, FoundBlock().hash(active_hash), {})); + BOOST_CHECK(!chain->findCommonAncestor({}, orig_tip->GetBlockHash(), {}, {}, FoundBlock().hash(orig_hash))); + BOOST_CHECK_EQUAL(active_hash, active.Tip()->GetBlockHash()); + BOOST_CHECK_EQUAL(orig_hash, orig_tip->GetBlockHash()); +} + +BOOST_AUTO_TEST_CASE(hasBlocks) +{ + auto chain = interfaces::MakeChain(m_node); + auto& active = ChainActive(); + + // Test ranges + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 10, 90)); + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 10, {})); + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 0, 90)); + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 0, {})); + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), -1000, 1000)); + active[5]->nStatus &= ~BLOCK_HAVE_DATA; + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 10, 90)); + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 10, {})); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 0, 90)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 0, {})); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), -1000, 1000)); + active[95]->nStatus &= ~BLOCK_HAVE_DATA; + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 10, 90)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 10, {})); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 0, 90)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 0, {})); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), -1000, 1000)); + active[50]->nStatus &= ~BLOCK_HAVE_DATA; + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 10, 90)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 10, {})); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 0, 90)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 0, {})); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), -1000, 1000)); + + // Test edge cases + BOOST_CHECK(chain->hasBlocks(active.Tip()->GetBlockHash(), 6, 49)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 5, 49)); + BOOST_CHECK(!chain->hasBlocks(active.Tip()->GetBlockHash(), 6, 50)); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/key_io_tests.cpp b/src/test/key_io_tests.cpp index b52513f4af..d465ee6759 100644 --- a/src/test/key_io_tests.cpp +++ b/src/test/key_io_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,8 @@ #include <key.h> #include <key_io.h> #include <script/script.h> -#include <util/strencodings.h> #include <test/util/setup_common.h> +#include <util/strencodings.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index 034b7938f9..cf2bd03698 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <key.h> #include <key_io.h> +#include <test/util/setup_common.h> #include <uint256.h> -#include <util/system.h> #include <util/strencodings.h> #include <util/string.h> -#include <test/util/setup_common.h> +#include <util/system.h> #include <string> #include <vector> diff --git a/src/test/main.cpp b/src/test/main.cpp index f32243d1d3..5885564074 100644 --- a/src/test/main.cpp +++ b/src/test/main.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/merkleblock_tests.cpp b/src/test/merkleblock_tests.cpp index 9f8c4ba5c5..98b27994a6 100644 --- a/src/test/merkleblock_tests.cpp +++ b/src/test/merkleblock_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <merkleblock.h> -#include <uint256.h> #include <test/util/setup_common.h> +#include <uint256.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/multisig_tests.cpp b/src/test/multisig_tests.cpp index 97a918da45..dd2890c134 100644 --- a/src/test/multisig_tests.cpp +++ b/src/test/multisig_tests.cpp @@ -1,17 +1,17 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <key.h> #include <policy/policy.h> +#include <script/interpreter.h> #include <script/script.h> #include <script/script_error.h> -#include <script/interpreter.h> #include <script/sign.h> #include <script/signingprovider.h> +#include <test/util/setup_common.h> #include <tinyformat.h> #include <uint256.h> -#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 9b5a86fef2..84bf593497 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -1,23 +1,24 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <addrdb.h> #include <addrman.h> +#include <chainparams.h> #include <clientversion.h> -#include <test/util/setup_common.h> -#include <string> -#include <boost/test/unit_test.hpp> -#include <serialize.h> -#include <streams.h> #include <net.h> #include <netbase.h> -#include <chainparams.h> +#include <serialize.h> +#include <streams.h> +#include <test/util/setup_common.h> #include <util/memory.h> -#include <util/system.h> #include <util/string.h> +#include <util/system.h> + +#include <boost/test/unit_test.hpp> #include <memory> +#include <string> class CAddrManSerializationMock : public CAddrMan { diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 9730b40580..ec6a290334 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <netbase.h> #include <net_permissions.h> +#include <netbase.h> #include <test/util/setup_common.h> #include <util/strencodings.h> diff --git a/src/test/pmt_tests.cpp b/src/test/pmt_tests.cpp index bf58bd63b9..a9d661438c 100644 --- a/src/test/pmt_tests.cpp +++ b/src/test/pmt_tests.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <arith_uint256.h> #include <consensus/merkle.h> #include <merkleblock.h> #include <serialize.h> #include <streams.h> +#include <test/util/setup_common.h> #include <uint256.h> -#include <arith_uint256.h> #include <version.h> -#include <test/util/setup_common.h> #include <vector> diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 025e2b78ca..06877898a4 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <policy/policy.h> #include <policy/fees.h> +#include <policy/policy.h> #include <txmempool.h> #include <uint256.h> #include <util/time.h> diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index 9782b78f2c..12c5848eaf 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <vector> #include <prevector.h> +#include <vector> #include <reverse_iterator.h> #include <serialize.h> diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index e0df41a971..ca3b92f2e1 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,8 @@ #include <boost/test/unit_test.hpp> -#include <random> #include <algorithm> +#include <random> BOOST_FIXTURE_TEST_SUITE(random_tests, BasicTestingSetup) diff --git a/src/test/reverselock_tests.cpp b/src/test/reverselock_tests.cpp index 4e51b8c02a..a42608a66d 100644 --- a/src/test/reverselock_tests.cpp +++ b/src/test/reverselock_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 84a3980b19..d9c66f1c19 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <rpc/server.h> #include <rpc/client.h> +#include <rpc/server.h> #include <rpc/util.h> #include <core_io.h> diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp index 801cf8e5d1..1395a7f38c 100644 --- a/src/test/scheduler_tests.cpp +++ b/src/test/scheduler_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,15 +6,17 @@ #include <scheduler.h> #include <util/time.h> -#include <boost/thread.hpp> #include <boost/test/unit_test.hpp> +#include <boost/thread.hpp> + +#include <mutex> BOOST_AUTO_TEST_SUITE(scheduler_tests) -static void microTask(CScheduler& s, boost::mutex& mutex, int& counter, int delta, std::chrono::system_clock::time_point rescheduleTime) +static void microTask(CScheduler& s, std::mutex& mutex, int& counter, int delta, std::chrono::system_clock::time_point rescheduleTime) { { - boost::unique_lock<boost::mutex> lock(mutex); + std::lock_guard<std::mutex> lock(mutex); counter += delta; } std::chrono::system_clock::time_point noTime = std::chrono::system_clock::time_point::min(); @@ -38,7 +40,7 @@ BOOST_AUTO_TEST_CASE(manythreads) // counters should sum to the number of initial tasks performed. CScheduler microTasks; - boost::mutex counterMutex[10]; + std::mutex counterMutex[10]; int counter[10] = { 0 }; FastRandomContext rng{/* fDeterministic */ true}; auto zeroToNine = [](FastRandomContext& rc) -> int { return rc.randrange(10); }; // [0, 9] diff --git a/src/test/script_p2sh_tests.cpp b/src/test/script_p2sh_tests.cpp index 8c1e843b0b..f6824a4e5e 100644 --- a/src/test/script_p2sh_tests.cpp +++ b/src/test/script_p2sh_tests.cpp @@ -1,17 +1,17 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/tx_verify.h> #include <key.h> -#include <validation.h> #include <policy/policy.h> +#include <policy/settings.h> #include <script/script.h> #include <script/script_error.h> -#include <policy/settings.h> #include <script/sign.h> #include <script/signingprovider.h> #include <test/util/setup_common.h> +#include <validation.h> #include <vector> diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 26015ca4c2..56454f61f3 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,16 +6,16 @@ #include <core_io.h> #include <key.h> +#include <rpc/util.h> #include <script/script.h> #include <script/script_error.h> #include <script/sign.h> #include <script/signingprovider.h> -#include <util/system.h> -#include <util/strencodings.h> -#include <test/util/transaction_utils.h> -#include <test/util/setup_common.h> -#include <rpc/util.h> #include <streams.h> +#include <test/util/setup_common.h> +#include <test/util/transaction_utils.h> +#include <util/strencodings.h> +#include <util/system.h> #if defined(HAVE_CONSENSUS_LIB) #include <script/bitcoinconsensus.h> @@ -217,7 +217,6 @@ struct KeyData KeyData() { - key0.Set(vchKey0, vchKey0 + 32, false); key0C.Set(vchKey0, vchKey0 + 32, true); pubkey0 = key0.GetPubKey(); @@ -272,9 +271,9 @@ private: void DoPush(const std::vector<unsigned char>& data) { - DoPush(); - push = data; - havePush = true; + DoPush(); + push = data; + havePush = true; } public: @@ -306,10 +305,10 @@ public: return *this; } - TestBuilder& Add(const CScript& _script) + TestBuilder& Opcode(const opcodetype& _op) { DoPush(); - spendTx.vin[0].scriptSig += _script; + spendTx.vin[0].scriptSig << _op; return *this; } @@ -326,8 +325,9 @@ public: return *this; } - TestBuilder& Push(const CScript& _script) { - DoPush(std::vector<unsigned char>(_script.begin(), _script.end())); + TestBuilder& Push(const CScript& _script) + { + DoPush(std::vector<unsigned char>(_script.begin(), _script.end())); return *this; } @@ -681,22 +681,22 @@ BOOST_AUTO_TEST_CASE(script_build) tests.push_back(TestBuilder(CScript() << OP_2 << ToByteVector(keys.pubkey1C) << ToByteVector(keys.pubkey1C) << OP_2 << OP_CHECKMULTISIG, "2-of-2 with two identical keys and sigs pushed using OP_DUP but no SIGPUSHONLY", 0 - ).Num(0).PushSig(keys.key1).Add(CScript() << OP_DUP)); + ).Num(0).PushSig(keys.key1).Opcode(OP_DUP)); tests.push_back(TestBuilder(CScript() << OP_2 << ToByteVector(keys.pubkey1C) << ToByteVector(keys.pubkey1C) << OP_2 << OP_CHECKMULTISIG, "2-of-2 with two identical keys and sigs pushed using OP_DUP", SCRIPT_VERIFY_SIGPUSHONLY - ).Num(0).PushSig(keys.key1).Add(CScript() << OP_DUP).ScriptError(SCRIPT_ERR_SIG_PUSHONLY)); + ).Num(0).PushSig(keys.key1).Opcode(OP_DUP).ScriptError(SCRIPT_ERR_SIG_PUSHONLY)); tests.push_back(TestBuilder(CScript() << ToByteVector(keys.pubkey2C) << OP_CHECKSIG, "P2SH(P2PK) with non-push scriptSig but no P2SH or SIGPUSHONLY", 0, true - ).PushSig(keys.key2).Add(CScript() << OP_NOP8).PushRedeem()); + ).PushSig(keys.key2).Opcode(OP_NOP8).PushRedeem()); tests.push_back(TestBuilder(CScript() << ToByteVector(keys.pubkey2C) << OP_CHECKSIG, "P2PK with non-push scriptSig but with P2SH validation", 0 - ).PushSig(keys.key2).Add(CScript() << OP_NOP8)); + ).PushSig(keys.key2).Opcode(OP_NOP8)); tests.push_back(TestBuilder(CScript() << ToByteVector(keys.pubkey2C) << OP_CHECKSIG, "P2SH(P2PK) with non-push scriptSig but no SIGPUSHONLY", SCRIPT_VERIFY_P2SH, true - ).PushSig(keys.key2).Add(CScript() << OP_NOP8).PushRedeem().ScriptError(SCRIPT_ERR_SIG_PUSHONLY)); + ).PushSig(keys.key2).Opcode(OP_NOP8).PushRedeem().ScriptError(SCRIPT_ERR_SIG_PUSHONLY)); tests.push_back(TestBuilder(CScript() << ToByteVector(keys.pubkey2C) << OP_CHECKSIG, "P2SH(P2PK) with non-push scriptSig but not P2SH", SCRIPT_VERIFY_SIGPUSHONLY, true - ).PushSig(keys.key2).Add(CScript() << OP_NOP8).PushRedeem().ScriptError(SCRIPT_ERR_SIG_PUSHONLY)); + ).PushSig(keys.key2).Opcode(OP_NOP8).PushRedeem().ScriptError(SCRIPT_ERR_SIG_PUSHONLY)); tests.push_back(TestBuilder(CScript() << OP_2 << ToByteVector(keys.pubkey1C) << ToByteVector(keys.pubkey1C) << OP_2 << OP_CHECKMULTISIG, "2-of-2 with two identical keys and sigs pushed", SCRIPT_VERIFY_SIGPUSHONLY ).Num(0).PushSig(keys.key1).PushSig(keys.key1)); @@ -1470,24 +1470,6 @@ BOOST_AUTO_TEST_CASE(script_HasValidOps) BOOST_CHECK(!script.HasValidOps()); } -BOOST_AUTO_TEST_CASE(script_can_append_self) -{ - CScript s, d; - - s = ScriptFromHex("00"); - s += s; - d = ScriptFromHex("0000"); - BOOST_CHECK(s == d); - - // check doubling a script that's large enough to require reallocation - static const char hex[] = "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f"; - s = CScript() << ParseHex(hex) << OP_CHECKSIG; - d = CScript() << ParseHex(hex) << OP_CHECKSIG << ParseHex(hex) << OP_CHECKSIG; - s += s; - BOOST_CHECK(s == d); -} - - #if defined(HAVE_CONSENSUS_LIB) /* Test simple (successful) usage of bitcoinconsensus_verify_script */ diff --git a/src/test/scriptnum10.h b/src/test/scriptnum10.h index 9f928827cb..352797f18d 100644 --- a/src/test/scriptnum10.h +++ b/src/test/scriptnum10.h @@ -1,17 +1,17 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_TEST_SCRIPTNUM10_H #define BITCOIN_TEST_SCRIPTNUM10_H +#include <assert.h> #include <limits> #include <stdexcept> #include <stdint.h> #include <string> #include <vector> -#include <assert.h> class scriptnum10_error : public std::runtime_error { diff --git a/src/test/scriptnum_tests.cpp b/src/test/scriptnum_tests.cpp index 40a6f69668..281018be9f 100644 --- a/src/test/scriptnum_tests.cpp +++ b/src/test/scriptnum_tests.cpp @@ -1,9 +1,9 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <test/scriptnum10.h> #include <script/script.h> +#include <test/scriptnum10.h> #include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index ea600499ca..9a6c721ab8 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <hash.h> #include <serialize.h> #include <streams.h> -#include <hash.h> #include <test/util/setup_common.h> #include <util/strencodings.h> diff --git a/src/test/settings_tests.cpp b/src/test/settings_tests.cpp index 10b161aa80..fcd831ccda 100644 --- a/src/test/settings_tests.cpp +++ b/src/test/settings_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index bcc4a46873..5ca136ea6e 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -1,18 +1,18 @@ -// Copyright (c) 2013-2019 The Bitcoin Core developers +// Copyright (c) 2013-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/tx_check.h> #include <consensus/validation.h> -#include <test/data/sighash.json.h> #include <hash.h> #include <script/interpreter.h> #include <script/script.h> #include <serialize.h> #include <streams.h> +#include <test/data/sighash.json.h> #include <test/util/setup_common.h> -#include <util/system.h> #include <util/strencodings.h> +#include <util/system.h> #include <version.h> #include <iostream> diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index 6462fcefe3..6e36bce7a1 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/consensus.h> #include <consensus/tx_verify.h> -#include <pubkey.h> #include <key.h> +#include <pubkey.h> #include <script/script.h> #include <script/standard.h> -#include <uint256.h> #include <test/util/setup_common.h> +#include <uint256.h> #include <vector> diff --git a/src/test/timedata_tests.cpp b/src/test/timedata_tests.cpp index 29b43e9bec..8c880babd1 100644 --- a/src/test/timedata_tests.cpp +++ b/src/test/timedata_tests.cpp @@ -1,14 +1,14 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // #include <netaddress.h> #include <noui.h> -#include <util/string.h> #include <test/util/logging.h> #include <test/util/setup_common.h> #include <timedata.h> +#include <util/string.h> #include <warnings.h> #include <string> diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 96520079d7..9d0ae56c3f 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,23 +6,23 @@ #include <test/data/tx_valid.json.h> #include <test/util/setup_common.h> -#include <clientversion.h> #include <checkqueue.h> +#include <clientversion.h> #include <consensus/tx_check.h> #include <consensus/validation.h> #include <core_io.h> #include <key.h> -#include <validation.h> #include <policy/policy.h> #include <policy/settings.h> #include <script/script.h> +#include <script/script_error.h> #include <script/sign.h> #include <script/signingprovider.h> -#include <script/script_error.h> #include <script/standard.h> #include <streams.h> -#include <util/strencodings.h> #include <test/util/transaction_utils.h> +#include <util/strencodings.h> +#include <validation.h> #include <map> #include <string> diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index 3550a02316..5fc172ee86 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -70,12 +70,8 @@ BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) // shutdown sequence (c.f. Shutdown() in init.cpp) txindex.Stop(); - // txindex job may be scheduled, so stop scheduler before destructing - m_node.scheduler->stop(); - threadGroup.interrupt_all(); - threadGroup.join_all(); - - // Rest of shutdown sequence and destructors happen in ~TestingSetup() + // Let scheduler events finish running to avoid accessing any memory related to txindex after it is destructed + SyncWithValidationInterfaceQueue(); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidation_tests.cpp b/src/test/txvalidation_tests.cpp index cace75f093..c3d7af8323 100644 --- a/src/test/txvalidation_tests.cpp +++ b/src/test/txvalidation_tests.cpp @@ -1,12 +1,12 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <validation.h> #include <consensus/validation.h> #include <primitives/transaction.h> #include <script/script.h> #include <test/util/setup_common.h> +#include <validation.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 7842594b80..cdef7dcc3c 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -4,12 +4,12 @@ #include <consensus/validation.h> #include <key.h> -#include <validation.h> -#include <txmempool.h> -#include <script/standard.h> #include <script/sign.h> #include <script/signingprovider.h> +#include <script/standard.h> #include <test/util/setup_common.h> +#include <txmempool.h> +#include <validation.h> #include <boost/test/unit_test.hpp> @@ -108,7 +108,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) // any script flag that is implemented as an upgraded NOP code. static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t failing_flags, bool add_to_cache) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - PrecomputedTransactionData txdata(tx); + PrecomputedTransactionData txdata; // If we add many more flags, this loop can get too expensive, but we can // rewrite in the future to randomly pick a set of flags to evaluate. for (uint32_t test_flags=0; test_flags < (1U << 16); test_flags += 1) { @@ -200,7 +200,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) LOCK(cs_main); TxValidationState state; - PrecomputedTransactionData ptd_spend_tx(spend_tx); + PrecomputedTransactionData ptd_spend_tx; BOOST_CHECK(!CheckInputScripts(CTransaction(spend_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_DERSIG, true, true, ptd_spend_tx, nullptr)); @@ -269,7 +269,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) // Make it valid, and check again invalid_with_cltv_tx.vin[0].scriptSig = CScript() << vchSig << 100; TxValidationState state; - PrecomputedTransactionData txdata(invalid_with_cltv_tx); + PrecomputedTransactionData txdata; BOOST_CHECK(CheckInputScripts(CTransaction(invalid_with_cltv_tx), state, ::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY, true, true, txdata, nullptr)); } @@ -297,7 +297,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) // Make it valid, and check again invalid_with_csv_tx.vin[0].scriptSig = CScript() << vchSig << 100; TxValidationState state; - PrecomputedTransactionData txdata(invalid_with_csv_tx); + PrecomputedTransactionData txdata; BOOST_CHECK(CheckInputScripts(CTransaction(invalid_with_csv_tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_CHECKSEQUENCEVERIFY, true, true, txdata, nullptr)); } @@ -358,7 +358,7 @@ BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) tx.vin[1].scriptWitness.SetNull(); TxValidationState state; - PrecomputedTransactionData txdata(tx); + PrecomputedTransactionData txdata; // This transaction is now invalid under segwit, because of the second input. BOOST_CHECK(!CheckInputScripts(CTransaction(tx), state, &::ChainstateActive().CoinsTip(), SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS, true, true, txdata, nullptr)); diff --git a/src/test/uint256_tests.cpp b/src/test/uint256_tests.cpp index 7293ecd325..c0ae2f8cf2 100644 --- a/src/test/uint256_tests.cpp +++ b/src/test/uint256_tests.cpp @@ -1,16 +1,17 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <arith_uint256.h> #include <streams.h> +#include <test/util/setup_common.h> #include <uint256.h> #include <version.h> -#include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> -#include <sstream> + #include <iomanip> +#include <sstream> #include <string> BOOST_FIXTURE_TEST_SUITE(uint256_tests, BasicTestingSetup) diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp new file mode 100644 index 0000000000..09f2f1807f --- /dev/null +++ b/src/test/util/net.cpp @@ -0,0 +1,39 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/net.h> + +#include <chainparams.h> +#include <net.h> + +void ConnmanTestMsg::NodeReceiveMsgBytes(CNode& node, const char* pch, unsigned int nBytes, bool& complete) const +{ + assert(node.ReceiveMsgBytes(pch, nBytes, complete)); + if (complete) { + size_t nSizeAdded = 0; + auto it(node.vRecvMsg.begin()); + for (; it != node.vRecvMsg.end(); ++it) { + // vRecvMsg contains only completed CNetMessage + // the single possible partially deserialized message are held by TransportDeserializer + nSizeAdded += it->m_raw_message_size; + } + { + LOCK(node.cs_vProcessMsg); + node.vProcessMsg.splice(node.vProcessMsg.end(), node.vRecvMsg, node.vRecvMsg.begin(), it); + node.nProcessQueueSize += nSizeAdded; + node.fPauseRecv = node.nProcessQueueSize > nReceiveFloodSize; + } + } +} + +bool ConnmanTestMsg::ReceiveMsgFrom(CNode& node, CSerializedNetMsg& ser_msg) const +{ + std::vector<unsigned char> ser_msg_header; + node.m_serializer->prepareForTransport(ser_msg, ser_msg_header); + + bool complete; + NodeReceiveMsgBytes(node, (const char*)ser_msg_header.data(), ser_msg_header.size(), complete); + NodeReceiveMsgBytes(node, (const char*)ser_msg.data.data(), ser_msg.data.size(), complete); + return complete; +} diff --git a/src/test/util/net.h b/src/test/util/net.h new file mode 100644 index 0000000000..ca8cb7fad5 --- /dev/null +++ b/src/test/util/net.h @@ -0,0 +1,33 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_NET_H +#define BITCOIN_TEST_UTIL_NET_H + +#include <net.h> + +struct ConnmanTestMsg : public CConnman { + using CConnman::CConnman; + void AddTestNode(CNode& node) + { + LOCK(cs_vNodes); + vNodes.push_back(&node); + } + void ClearTestNodes() + { + LOCK(cs_vNodes); + for (CNode* node : vNodes) { + delete node; + } + vNodes.clear(); + } + + void ProcessMessagesOnce(CNode& node) { m_msgproc->ProcessMessages(&node, flagInterruptMsgProc); } + + void NodeReceiveMsgBytes(CNode& node, const char* pch, unsigned int nBytes, bool& complete) const; + + bool ReceiveMsgFrom(CNode& node, CSerializedNetMsg& ser_msg) const; +}; + +#endif // BITCOIN_TEST_UTIL_NET_H diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index d684b97787..bf0afc4171 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2019 The Bitcoin Core developers +// Copyright (c) 2011-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -27,12 +27,15 @@ #include <util/string.h> #include <util/time.h> #include <util/translation.h> +#include <util/url.h> +#include <util/vector.h> #include <validation.h> #include <validationinterface.h> #include <functional> const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; +UrlDecodeFn* const URL_DECODE = nullptr; FastRandomContext g_insecure_rand_ctx; /** Random context to get unique temp data dirs. Separate from g_insecure_rand_ctx, which can be seeded from a const env var */ @@ -63,17 +66,34 @@ std::ostream& operator<<(std::ostream& os, const uint256& num) return os; } -BasicTestingSetup::BasicTestingSetup(const std::string& chainName) +BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args) : m_path_root{fs::temp_directory_path() / "test_common_" PACKAGE_NAME / g_insecure_rand_ctx_temp_path.rand256().ToString()} { + const std::vector<const char*> arguments = Cat( + { + "dummy", + "-printtoconsole=0", + "-logtimemicros", + "-debug", + "-debugexclude=libevent", + "-debugexclude=leveldb", + }, + extra_args); fs::create_directories(m_path_root); gArgs.ForceSetArg("-datadir", m_path_root.string()); ClearDatadirCache(); + { + SetupServerArgs(m_node); + std::string error; + const bool success{m_node.args->ParseParameters(arguments.size(), arguments.data(), error)}; + assert(success); + assert(error.empty()); + } SelectParams(chainName); SeedInsecureRand(); - gArgs.ForceSetArg("-printtoconsole", "0"); if (G_TEST_LOG_FUN) LogInstance().PushBackCallback(G_TEST_LOG_FUN); InitLogging(); + AppInitParameterInteraction(); LogInstance().StartLogging(); SHA256AutoDetect(); ECC_Start(); @@ -93,10 +113,12 @@ BasicTestingSetup::~BasicTestingSetup() { LogInstance().DisconnectTestLogger(); fs::remove_all(m_path_root); + gArgs.ClearArgs(); ECC_Stop(); } -TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(chainName) +TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args) + : BasicTestingSetup(chainName, extra_args) { const CChainParams& chainparams = Params(); // Ideally we'd move all the RPC tests to the functional testing framework @@ -112,7 +134,8 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha GetMainSignals().RegisterBackgroundSignalScheduler(*g_rpc_node->scheduler); pblocktree.reset(new CBlockTreeDB(1 << 20, true)); - g_chainstate = MakeUnique<CChainState>(); + + g_chainman.InitializeChainstate(); ::ChainstateActive().InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); assert(!::ChainstateActive().CanFlushToDisk()); @@ -139,6 +162,11 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha m_node.banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); m_node.connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. m_node.peer_logic = MakeUnique<PeerLogicValidation>(m_node.connman.get(), m_node.banman.get(), *m_node.scheduler, *m_node.mempool); + { + CConnman::Options options; + options.m_msgproc = m_node.peer_logic.get(); + m_node.connman->Init(options); + } } TestingSetup::~TestingSetup() @@ -151,10 +179,11 @@ TestingSetup::~TestingSetup() g_rpc_node = nullptr; m_node.connman.reset(); m_node.banman.reset(); + m_node.args = nullptr; m_node.mempool = nullptr; m_node.scheduler.reset(); UnloadBlockIndex(); - g_chainstate.reset(); + g_chainman.Reset(); pblocktree.reset(); } diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 0930309c3a..2477f9ad06 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -73,9 +73,11 @@ static constexpr CAmount CENT{1000000}; */ struct BasicTestingSetup { ECCVerifyHandle globalVerifyHandle; + NodeContext m_node; - explicit BasicTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); + explicit BasicTestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); ~BasicTestingSetup(); + private: const fs::path m_path_root; }; @@ -84,10 +86,9 @@ private: * Included are coins database, script check threads setup. */ struct TestingSetup : public BasicTestingSetup { - NodeContext m_node; boost::thread_group threadGroup; - explicit TestingSetup(const std::string& chainName = CBaseChainParams::MAIN); + explicit TestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); ~TestingSetup(); }; diff --git a/src/test/util/transaction_utils.cpp b/src/test/util/transaction_utils.cpp index 999b803a8d..7e5bb30a2c 100644 --- a/src/test/util/transaction_utils.cpp +++ b/src/test/util/transaction_utils.cpp @@ -1,10 +1,10 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <test/util/transaction_utils.h> #include <coins.h> #include <script/signingprovider.h> +#include <test/util/transaction_utils.h> CMutableTransaction BuildCreditingTransaction(const CScript& scriptPubKey, int nValue) { diff --git a/src/test/util/transaction_utils.h b/src/test/util/transaction_utils.h index f843928a5f..1beddd334b 100644 --- a/src/test/util/transaction_utils.h +++ b/src/test/util/transaction_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 73b37f909f..45b7fd4932 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -6,7 +6,7 @@ #include <clientversion.h> #include <hash.h> // For Hash() -#include <key.h> // For CKey +#include <key.h> // For CKey #include <optional.h> #include <sync.h> #include <test/util/setup_common.h> @@ -14,10 +14,10 @@ #include <uint256.h> #include <util/message.h> // For MessageSign(), MessageVerify(), MESSAGE_MAGIC #include <util/moneystr.h> +#include <util/spanparsing.h> #include <util/strencodings.h> #include <util/string.h> #include <util/time.h> -#include <util/spanparsing.h> #include <util/vector.h> #include <array> diff --git a/src/test/util_threadnames_tests.cpp b/src/test/util_threadnames_tests.cpp index cee4e0ce3c..f226caf717 100644 --- a/src/test/util_threadnames_tests.cpp +++ b/src/test/util_threadnames_tests.cpp @@ -1,15 +1,15 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <test/util/setup_common.h> #include <util/string.h> #include <util/threadnames.h> -#include <test/util/setup_common.h> +#include <mutex> +#include <set> #include <thread> #include <vector> -#include <set> -#include <mutex> #if defined(HAVE_CONFIG_H) #include <config/bitcoin-config.h> diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index afb3db36a2..b8af869b8e 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp new file mode 100644 index 0000000000..0d149285ad --- /dev/null +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -0,0 +1,107 @@ +// Copyright (c) 2019-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +#include <chainparams.h> +#include <consensus/validation.h> +#include <random.h> +#include <sync.h> +#include <test/util/setup_common.h> +#include <uint256.h> +#include <validation.h> +#include <validationinterface.h> + +#include <vector> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(validation_chainstatemanager_tests, TestingSetup) + +//! Basic tests for ChainstateManager. +//! +//! First create a legacy (IBD) chainstate, then create a snapshot chainstate. +BOOST_AUTO_TEST_CASE(chainstatemanager) +{ + ChainstateManager manager; + std::vector<CChainState*> chainstates; + const CChainParams& chainparams = Params(); + + // Create a legacy (IBD) chainstate. + // + ENTER_CRITICAL_SECTION(cs_main); + CChainState& c1 = manager.InitializeChainstate(); + LEAVE_CRITICAL_SECTION(cs_main); + chainstates.push_back(&c1); + c1.InitCoinsDB( + /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + WITH_LOCK(::cs_main, c1.InitCoinsCache()); + + BOOST_CHECK(!manager.IsSnapshotActive()); + BOOST_CHECK(!manager.IsSnapshotValidated()); + BOOST_CHECK(!manager.IsBackgroundIBD(&c1)); + auto all = manager.GetAll(); + BOOST_CHECK_EQUAL_COLLECTIONS(all.begin(), all.end(), chainstates.begin(), chainstates.end()); + + auto& active_chain = manager.ActiveChain(); + BOOST_CHECK_EQUAL(&active_chain, &c1.m_chain); + + BOOST_CHECK_EQUAL(manager.ActiveHeight(), -1); + + auto active_tip = manager.ActiveTip(); + auto exp_tip = c1.m_chain.Tip(); + BOOST_CHECK_EQUAL(active_tip, exp_tip); + + auto& validated_cs = manager.ValidatedChainstate(); + BOOST_CHECK_EQUAL(&validated_cs, &c1); + + // Create a snapshot-based chainstate. + // + ENTER_CRITICAL_SECTION(cs_main); + CChainState& c2 = manager.InitializeChainstate(GetRandHash()); + LEAVE_CRITICAL_SECTION(cs_main); + chainstates.push_back(&c2); + c2.InitCoinsDB( + /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + WITH_LOCK(::cs_main, c2.InitCoinsCache()); + // Unlike c1, which doesn't have any blocks. Gets us different tip, height. + c2.LoadGenesisBlock(chainparams); + BlockValidationState _; + BOOST_CHECK(c2.ActivateBestChain(_, chainparams, nullptr)); + + BOOST_CHECK(manager.IsSnapshotActive()); + BOOST_CHECK(!manager.IsSnapshotValidated()); + BOOST_CHECK(manager.IsBackgroundIBD(&c1)); + BOOST_CHECK(!manager.IsBackgroundIBD(&c2)); + auto all2 = manager.GetAll(); + BOOST_CHECK_EQUAL_COLLECTIONS(all2.begin(), all2.end(), chainstates.begin(), chainstates.end()); + + auto& active_chain2 = manager.ActiveChain(); + BOOST_CHECK_EQUAL(&active_chain2, &c2.m_chain); + + BOOST_CHECK_EQUAL(manager.ActiveHeight(), 0); + + auto active_tip2 = manager.ActiveTip(); + auto exp_tip2 = c2.m_chain.Tip(); + BOOST_CHECK_EQUAL(active_tip2, exp_tip2); + + // Ensure that these pointers actually correspond to different + // CCoinsViewCache instances. + BOOST_CHECK(exp_tip != exp_tip2); + + auto& validated_cs2 = manager.ValidatedChainstate(); + BOOST_CHECK_EQUAL(&validated_cs2, &c1); + + auto& validated_chain = manager.ValidatedChain(); + BOOST_CHECK_EQUAL(&validated_chain, &c1.m_chain); + + auto validated_tip = manager.ValidatedTip(); + exp_tip = c1.m_chain.Tip(); + BOOST_CHECK_EQUAL(validated_tip, exp_tip); + + // Let scheduler events finish running to avoid accessing memory that is going to be unloaded + SyncWithValidationInterfaceQueue(); + + WITH_LOCK(::cs_main, manager.Unload()); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp index c24164528f..388a2dbd13 100644 --- a/src/test/validation_flush_tests.cpp +++ b/src/test/validation_flush_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // diff --git a/src/test/validationinterface_tests.cpp b/src/test/validationinterface_tests.cpp new file mode 100644 index 0000000000..208be92852 --- /dev/null +++ b/src/test/validationinterface_tests.cpp @@ -0,0 +1,60 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <boost/test/unit_test.hpp> +#include <consensus/validation.h> +#include <primitives/block.h> +#include <scheduler.h> +#include <test/util/setup_common.h> +#include <util/check.h> +#include <validationinterface.h> + +BOOST_FIXTURE_TEST_SUITE(validationinterface_tests, TestingSetup) + +class TestInterface : public CValidationInterface +{ +public: + TestInterface(std::function<void()> on_call = nullptr, std::function<void()> on_destroy = nullptr) + : m_on_call(std::move(on_call)), m_on_destroy(std::move(on_destroy)) + { + } + virtual ~TestInterface() + { + if (m_on_destroy) m_on_destroy(); + } + void BlockChecked(const CBlock& block, const BlockValidationState& state) override + { + if (m_on_call) m_on_call(); + } + static void Call() + { + CBlock block; + BlockValidationState state; + GetMainSignals().BlockChecked(block, state); + } + std::function<void()> m_on_call; + std::function<void()> m_on_destroy; +}; + +// Regression test to ensure UnregisterAllValidationInterfaces calls don't +// destroy a validation interface while it is being called. Bug: +// https://github.com/bitcoin/bitcoin/pull/18551 +BOOST_AUTO_TEST_CASE(unregister_all_during_call) +{ + bool destroyed = false; + RegisterSharedValidationInterface(std::make_shared<TestInterface>( + [&] { + // First call should decrements reference count 2 -> 1 + UnregisterAllValidationInterfaces(); + BOOST_CHECK(!destroyed); + // Second call should not decrement reference count 1 -> 0 + UnregisterAllValidationInterfaces(); + BOOST_CHECK(!destroyed); + }, + [&] { destroyed = true; })); + TestInterface::Call(); + BOOST_CHECK(destroyed); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index 7b59d539a6..11c6bdad91 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -1,13 +1,13 @@ -// Copyright (c) 2014-2019 The Bitcoin Core developers +// Copyright (c) 2014-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chain.h> -#include <versionbits.h> -#include <test/util/setup_common.h> #include <chainparams.h> -#include <validation.h> #include <consensus/params.h> +#include <test/util/setup_common.h> +#include <validation.h> +#include <versionbits.h> #include <boost/test/unit_test.hpp> diff --git a/src/txdb.cpp b/src/txdb.cpp index acc47ab45e..071aa1336b 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 47b0d39ea4..102e50dc5c 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/undo.h b/src/undo.h index 80bbd146f3..a98f046735 100644 --- a/src/undo.h +++ b/src/undo.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/asmap.cpp b/src/util/asmap.cpp index 60bd27bf90..5354cdb962 100644 --- a/src/util/asmap.cpp +++ b/src/util/asmap.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/bip32.cpp b/src/util/bip32.cpp index 6f176dd5ec..4c7e948368 100644 --- a/src/util/bip32.cpp +++ b/src/util/bip32.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/bip32.h b/src/util/bip32.h index 7e58b79f38..347e83db9e 100644 --- a/src/util/bip32.h +++ b/src/util/bip32.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/golombrice.h b/src/util/golombrice.h new file mode 100644 index 0000000000..425e7f6681 --- /dev/null +++ b/src/util/golombrice.h @@ -0,0 +1,43 @@ +// Copyright (c) 2018-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_GOLOMBRICE_H +#define BITCOIN_UTIL_GOLOMBRICE_H + +#include <streams.h> + +#include <cstdint> + +template <typename OStream> +void GolombRiceEncode(BitStreamWriter<OStream>& bitwriter, uint8_t P, uint64_t x) +{ + // Write quotient as unary-encoded: q 1's followed by one 0. + uint64_t q = x >> P; + while (q > 0) { + int nbits = q <= 64 ? static_cast<int>(q) : 64; + bitwriter.Write(~0ULL, nbits); + q -= nbits; + } + bitwriter.Write(0, 1); + + // Write the remainder in P bits. Since the remainder is just the bottom + // P bits of x, there is no need to mask first. + bitwriter.Write(x, P); +} + +template <typename IStream> +uint64_t GolombRiceDecode(BitStreamReader<IStream>& bitreader, uint8_t P) +{ + // Read unary-encoded quotient: q 1's followed by one 0. + uint64_t q = 0; + while (bitreader.Read(1) == 1) { + ++q; + } + + uint64_t r = bitreader.Read(P); + + return (q << P) + r; +} + +#endif // BITCOIN_UTIL_GOLOMBRICE_H diff --git a/src/util/moneystr.cpp b/src/util/moneystr.cpp index 544cfb58f9..1bc8d02eab 100644 --- a/src/util/moneystr.cpp +++ b/src/util/moneystr.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/moneystr.h b/src/util/moneystr.h index d8b08adc24..9d2b6da0fc 100644 --- a/src/util/moneystr.h +++ b/src/util/moneystr.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index 16917ccb47..3a903b6897 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 11317f0432..bd988f1410 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/string.h b/src/util/string.h index 694f0a1ca4..b8e2a06235 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/system.cpp b/src/util/system.cpp index b0a538b527..0790ea0d48 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -141,6 +141,12 @@ bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes) return free_bytes_available >= min_disk_space + additional_bytes; } +std::streampos GetFileSize(const char* path, std::streamsize max) { + std::ifstream file(path, std::ios::binary); + file.ignore(max); + return file.gcount(); +} + /** * Interpret a string argument as a boolean. * @@ -226,10 +232,11 @@ static bool CheckValid(const std::string& key, const util::SettingsValue& val, u return true; } -ArgsManager::ArgsManager() -{ - // nothing to do -} +// Define default constructor and destructor that are not inline, so code instantiating this class doesn't need to +// #include class definitions for all members. +// For example, m_settings has an internal dependency on univalue. +ArgsManager::ArgsManager() {} +ArgsManager::~ArgsManager() {} const std::set<std::string> ArgsManager::GetUnsuitableSectionOnlyArgs() const { diff --git a/src/util/system.h b/src/util/system.h index 3138522b5c..a5eea5dfab 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -63,6 +63,14 @@ void UnlockDirectory(const fs::path& directory, const std::string& lockfile_name bool DirIsWritable(const fs::path& directory); bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes = 0); +/** Get the size of a file by scanning it. + * + * @param[in] path The file path + * @param[in] max Stop seeking beyond this limit + * @return The file size or max + */ +std::streampos GetFileSize(const char* path, std::streamsize max = std::numeric_limits<std::streamsize>::max()); + /** Release all directory locks. This is used for unit testing only, at runtime * the global destructor will take care of the locks. */ @@ -192,6 +200,7 @@ protected: public: ArgsManager(); + ~ArgsManager(); /** * Select the network in use diff --git a/src/util/time.cpp b/src/util/time.cpp index 0938ff36a6..e96972fe12 100644 --- a/src/util/time.cpp +++ b/src/util/time.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/time.h b/src/util/time.h index 77de1e047d..b00c25f67c 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/util/url.h b/src/util/url.h index e9ea2ab765..5a7b11fa04 100644 --- a/src/util/url.h +++ b/src/util/url.h @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2019 The Bitcoin Core developers +// Copyright (c) 2015-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,6 +7,8 @@ #include <string> -std::string urlDecode(const std::string &urlEncoded); +using UrlDecodeFn = std::string(const std::string& url_encoded); +UrlDecodeFn urlDecode; +extern UrlDecodeFn* const URL_DECODE; #endif // BITCOIN_UTIL_URL_H diff --git a/src/validation.cpp b/src/validation.cpp index 7ee94f8657..25975e3e31 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -20,6 +20,7 @@ #include <index/txindex.h> #include <logging.h> #include <logging/timer.h> +#include <optional.h> #include <policy/fees.h> #include <policy/policy.h> #include <policy/settings.h> @@ -58,6 +59,25 @@ #define MICRO 0.000001 #define MILLI 0.001 +/** + * An extra transaction can be added to a package, as long as it only has one + * ancestor and is no larger than this. Not really any reason to make this + * configurable as it doesn't materially change DoS parameters. + */ +static const unsigned int EXTRA_DESCENDANT_TX_SIZE_LIMIT = 10000; +/** Maximum kilobytes for transactions to store for processing during reorg */ +static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; +/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ +static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB +/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ +static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB +/** Time to wait (in seconds) between writing blocks/block index to disk. */ +static const unsigned int DATABASE_WRITE_INTERVAL = 60 * 60; +/** Time to wait (in seconds) between flushing chainstate to disk. */ +static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; +/** Maximum age of our tip in seconds for us to be considered current for fee estimation */ +static const int64_t MAX_FEE_ESTIMATION_TIP_AGE = 3 * 60 * 60; + bool CBlockIndexWorkComparator::operator()(const CBlockIndex *pa, const CBlockIndex *pb) const { // First sort by most total work, ... if (pa->nChainWork > pb->nChainWork) return false; @@ -76,20 +96,19 @@ bool CBlockIndexWorkComparator::operator()(const CBlockIndex *pa, const CBlockIn return false; } -namespace { -BlockManager g_blockman; -} // anon namespace - -std::unique_ptr<CChainState> g_chainstate; +ChainstateManager g_chainman; -CChainState& ChainstateActive() { - assert(g_chainstate); - return *g_chainstate; +CChainState& ChainstateActive() +{ + LOCK(::cs_main); + assert(g_chainman.m_active_chainstate); + return *g_chainman.m_active_chainstate; } -CChain& ChainActive() { - assert(g_chainstate); - return g_chainstate->m_chain; +CChain& ChainActive() +{ + LOCK(::cs_main); + return ::ChainstateActive().m_chain; } /** @@ -151,8 +170,8 @@ namespace { CBlockIndex* LookupBlockIndex(const uint256& hash) { AssertLockHeld(cs_main); - BlockMap::const_iterator it = g_blockman.m_block_index.find(hash); - return it == g_blockman.m_block_index.end() ? nullptr : it->second; + BlockMap::const_iterator it = g_chainman.BlockIndex().find(hash); + return it == g_chainman.BlockIndex().end() ? nullptr : it->second; } CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& locator) @@ -1016,7 +1035,7 @@ bool MemPoolAccept::AcceptSingleTransaction(const CTransactionRef& ptx, ATMPArgs // scripts (ie, other policy checks pass). We perform the inexpensive // checks first and avoid hashing and signature verification unless those // checks pass, to mitigate CPU exhaustion denial-of-service attacks. - PrecomputedTransactionData txdata(*ptx); + PrecomputedTransactionData txdata; if (!PolicyScriptChecks(args, workspace, txdata)) return false; @@ -1242,10 +1261,9 @@ void CoinsViews::InitCache() m_cacheview = MakeUnique<CCoinsViewCache>(&m_catcherview); } -// NOTE: for now m_blockman is set to a global, but this will be changed -// in a future commit. -CChainState::CChainState() : m_blockman(g_blockman) {} - +CChainState::CChainState(BlockManager& blockman, uint256 from_snapshot_blockhash) + : m_blockman(blockman), + m_from_snapshot_blockhash(from_snapshot_blockhash) {} void CChainState::InitCoinsDB( size_t cache_size_bytes, @@ -1253,6 +1271,10 @@ void CChainState::InitCoinsDB( bool should_wipe, std::string leveldb_name) { + if (!m_from_snapshot_blockhash.IsNull()) { + leveldb_name += "_" + m_from_snapshot_blockhash.ToString(); + } + m_coins_views = MakeUnique<CoinsViews>( leveldb_name, cache_size_bytes, in_memory, should_wipe); } @@ -1294,7 +1316,8 @@ static CBlockIndex *pindexBestForkTip = nullptr, *pindexBestForkBase = nullptr; BlockMap& BlockIndex() { - return g_blockman.m_block_index; + LOCK(::cs_main); + return g_chainman.m_blockman.m_block_index; } static void AlertNotify(const std::string& strMessage) @@ -1512,6 +1535,10 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState &state, const C return true; } + if (!txdata.m_ready) { + txdata.Init(tx); + } + for (unsigned int i = 0; i < tx.vin.size(); i++) { const COutPoint &prevout = tx.vin[i].prevout; const Coin& coin = inputs.AccessCoin(prevout); @@ -1668,10 +1695,11 @@ int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out) return DISCONNECT_FAILED; // adding output for transaction without known metadata } } - // The potential_overwrite parameter to AddCoin is only allowed to be false if we know for - // sure that the coin did not already exist in the cache. As we have queried for that above - // using HaveCoin, we don't need to guess. When fClean is false, a coin already existed and - // it is an overwrite. + // If the coin already exists as an unspent coin in the cache, then the + // possible_overwrite parameter to AddCoin must be set to true. We have + // already checked whether an unspent coin exists above using HaveCoin, so + // we don't need to guess. When fClean is false, an unspent coin already + // existed and it is an overwrite. view.AddCoin(out, std::move(undo), !fClean); return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; @@ -2075,15 +2103,19 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, CBlockUndo blockundo; + // Precomputed transaction data pointers must not be invalidated + // until after `control` has run the script checks (potentially + // in multiple threads). Preallocate the vector size so a new allocation + // doesn't invalidate pointers into the vector, and keep txsdata in scope + // for as long as `control`. CCheckQueueControl<CScriptCheck> control(fScriptChecks && g_parallel_script_checks ? &scriptcheckqueue : nullptr); + std::vector<PrecomputedTransactionData> txsdata(block.vtx.size()); std::vector<int> prevheights; CAmount nFees = 0; int nInputs = 0; int64_t nSigOpsCost = 0; blockundo.vtxundo.reserve(block.vtx.size() - 1); - std::vector<PrecomputedTransactionData> txdata; - txdata.reserve(block.vtx.size()); // Required so that pointers to individual PrecomputedTransactionData don't get invalidated for (unsigned int i = 0; i < block.vtx.size(); i++) { const CTransaction &tx = *(block.vtx[i]); @@ -2130,13 +2162,12 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-blk-sigops"); } - txdata.emplace_back(tx); if (!tx.IsCoinBase()) { std::vector<CScriptCheck> vChecks; bool fCacheResults = fJustCheck; /* Don't cache results if we're actually connecting blocks (still consult the cache, though) */ TxValidationState tx_state; - if (fScriptChecks && !CheckInputScripts(tx, tx_state, view, flags, fCacheResults, fCacheResults, txdata[i], g_parallel_script_checks ? &vChecks : nullptr)) { + if (fScriptChecks && !CheckInputScripts(tx, tx_state, view, flags, fCacheResults, fCacheResults, txsdata[i], g_parallel_script_checks ? &vChecks : nullptr)) { // Any transaction validation failure in ConnectBlock is a block consensus failure state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, tx_state.GetRejectReason(), tx_state.GetDebugMessage()); @@ -3443,7 +3474,7 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidatio if (fCheckpointsEnabled) { // Don't accept any forks from the main chain prior to last checkpoint. // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in our - // g_blockman.m_block_index. + // BlockIndex(). CBlockIndex* pcheckpoint = GetLastCheckpoint(params.Checkpoints()); if (pcheckpoint && nHeight < pcheckpoint->nHeight) { LogPrintf("ERROR: %s: forked chain older than last checkpoint (height %d)\n", __func__, nHeight); @@ -3651,7 +3682,8 @@ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& headers, BlockValid LOCK(cs_main); for (const CBlockHeader& header : headers) { CBlockIndex *pindex = nullptr; // Use a temp pindex instead of ppindex to avoid a const_cast - bool accepted = g_blockman.AcceptBlockHeader(header, state, chainparams, &pindex); + bool accepted = g_chainman.m_blockman.AcceptBlockHeader( + header, state, chainparams, &pindex); ::ChainstateActive().CheckBlockIndex(chainparams.GetConsensus()); if (!accepted) { @@ -3853,7 +3885,7 @@ void PruneOneBlockFile(const int fileNumber) { LOCK(cs_LastBlockFile); - for (const auto& entry : g_blockman.m_block_index) { + for (const auto& entry : g_chainman.BlockIndex()) { CBlockIndex* pindex = entry.second; if (pindex->nFile == fileNumber) { pindex->nStatus &= ~BLOCK_HAVE_DATA; @@ -3867,12 +3899,12 @@ void PruneOneBlockFile(const int fileNumber) // to be downloaded again in order to consider its chain, at which // point it would be considered as a candidate for // m_blocks_unlinked or setBlockIndexCandidates. - auto range = g_blockman.m_blocks_unlinked.equal_range(pindex->pprev); + auto range = g_chainman.m_blockman.m_blocks_unlinked.equal_range(pindex->pprev); while (range.first != range.second) { std::multimap<CBlockIndex *, CBlockIndex *>::iterator _it = range.first; range.first++; if (_it->second == pindex) { - g_blockman.m_blocks_unlinked.erase(_it); + g_chainman.m_blockman.m_blocks_unlinked.erase(_it); } } } @@ -4109,9 +4141,11 @@ void BlockManager::Unload() { bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - if (!g_blockman.LoadBlockIndex( - chainparams.GetConsensus(), *pblocktree, ::ChainstateActive().setBlockIndexCandidates)) + if (!g_chainman.m_blockman.LoadBlockIndex( + chainparams.GetConsensus(), *pblocktree, + ::ChainstateActive().setBlockIndexCandidates)) { return false; + } // Load block file info pblocktree->ReadLastBlockFile(nLastBlockFile); @@ -4133,7 +4167,7 @@ bool static LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_RE // Check presence of blk files LogPrintf("Checking all blk files are present...\n"); std::set<int> setBlkDataFiles; - for (const std::pair<const uint256, CBlockIndex*>& item : g_blockman.m_block_index) + for (const std::pair<const uint256, CBlockIndex*>& item : g_chainman.BlockIndex()) { CBlockIndex* pindex = item.second; if (pindex->nStatus & BLOCK_HAVE_DATA) { @@ -4510,26 +4544,15 @@ bool CChainState::RewindBlockIndex(const CChainParams& params) PruneBlockIndexCandidates(); CheckBlockIndex(params.GetConsensus()); - } - } - - return true; -} -bool RewindBlockIndex(const CChainParams& params) { - if (!::ChainstateActive().RewindBlockIndex(params)) { - return false; - } - - LOCK(cs_main); - if (::ChainActive().Tip() != nullptr) { - // FlushStateToDisk can possibly read ::ChainActive(). Be conservative - // and skip it here, we're about to -reindex-chainstate anyway, so - // it'll get called a bunch real soon. - BlockValidationState state; - if (!::ChainstateActive().FlushStateToDisk(params, state, FlushStateMode::ALWAYS)) { - LogPrintf("RewindBlockIndex: unable to flush state to disk (%s)\n", state.ToString()); - return false; + // FlushStateToDisk can possibly read ::ChainActive(). Be conservative + // and skip it here, we're about to -reindex-chainstate anyway, so + // it'll get called a bunch real soon. + BlockValidationState state; + if (!FlushStateToDisk(params, state, FlushStateMode::ALWAYS)) { + LogPrintf("RewindBlockIndex: unable to flush state to disk (%s)\n", state.ToString()); + return false; + } } } @@ -4547,8 +4570,7 @@ void CChainState::UnloadBlockIndex() { void UnloadBlockIndex() { LOCK(cs_main); - ::ChainActive().SetTip(nullptr); - g_blockman.Unload(); + g_chainman.Unload(); pindexBestInvalid = nullptr; pindexBestHeader = nullptr; mempool.clear(); @@ -4561,8 +4583,6 @@ void UnloadBlockIndex() warningcache[b].clear(); } fHavePruned = false; - - ::ChainstateActive().UnloadBlockIndex(); } bool LoadBlockIndex(const CChainParams& chainparams) @@ -4572,7 +4592,7 @@ bool LoadBlockIndex(const CChainParams& chainparams) if (!fReindex) { bool ret = LoadBlockIndexDB(chainparams); if (!ret) return false; - needs_init = g_blockman.m_block_index.empty(); + needs_init = g_chainman.m_blockman.m_block_index.empty(); } if (needs_init) { @@ -4693,7 +4713,7 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFi // Activate the genesis block so normal node progress can continue if (hash == chainparams.GetConsensus().hashGenesisBlock) { BlockValidationState state; - if (!ActivateBestChain(state, chainparams)) { + if (!ActivateBestChain(state, chainparams, nullptr)) { break; } } @@ -4923,6 +4943,14 @@ void CChainState::CheckBlockIndex(const Consensus::Params& consensusParams) assert(nNodes == forward.size()); } +std::string CChainState::ToString() +{ + CBlockIndex* tip = m_chain.Tip(); + return strprintf("Chainstate [%s] @ height %d (%s)", + m_from_snapshot_blockhash.IsNull() ? "ibd" : "snapshot", + tip ? tip->nHeight : -1, tip ? tip->GetBlockHash().ToString() : "null"); +} + std::string CBlockFileInfo::ToString() const { return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); @@ -5110,10 +5138,99 @@ public: CMainCleanup() {} ~CMainCleanup() { // block headers - BlockMap::iterator it1 = g_blockman.m_block_index.begin(); - for (; it1 != g_blockman.m_block_index.end(); it1++) + BlockMap::iterator it1 = g_chainman.BlockIndex().begin(); + for (; it1 != g_chainman.BlockIndex().end(); it1++) delete (*it1).second; - g_blockman.m_block_index.clear(); + g_chainman.BlockIndex().clear(); } }; static CMainCleanup instance_of_cmaincleanup; + +Optional<uint256> ChainstateManager::SnapshotBlockhash() const { + if (m_active_chainstate != nullptr) { + // If a snapshot chainstate exists, it will always be our active. + return m_active_chainstate->m_from_snapshot_blockhash; + } + return {}; +} + +std::vector<CChainState*> ChainstateManager::GetAll() +{ + std::vector<CChainState*> out; + + if (!IsSnapshotValidated() && m_ibd_chainstate) { + out.push_back(m_ibd_chainstate.get()); + } + + if (m_snapshot_chainstate) { + out.push_back(m_snapshot_chainstate.get()); + } + + return out; +} + +CChainState& ChainstateManager::InitializeChainstate(const uint256& snapshot_blockhash) +{ + bool is_snapshot = !snapshot_blockhash.IsNull(); + std::unique_ptr<CChainState>& to_modify = + is_snapshot ? m_snapshot_chainstate : m_ibd_chainstate; + + if (to_modify) { + throw std::logic_error("should not be overwriting a chainstate"); + } + + to_modify.reset(new CChainState(m_blockman, snapshot_blockhash)); + + // Snapshot chainstates and initial IBD chaintates always become active. + if (is_snapshot || (!is_snapshot && !m_active_chainstate)) { + LogPrintf("Switching active chainstate to %s\n", to_modify->ToString()); + m_active_chainstate = to_modify.get(); + } else { + throw std::logic_error("unexpected chainstate activation"); + } + + return *to_modify; +} + +CChain& ChainstateManager::ActiveChain() const +{ + assert(m_active_chainstate); + return m_active_chainstate->m_chain; +} + +bool ChainstateManager::IsSnapshotActive() const +{ + return m_snapshot_chainstate && m_active_chainstate == m_snapshot_chainstate.get(); +} + +CChainState& ChainstateManager::ValidatedChainstate() const +{ + if (m_snapshot_chainstate && IsSnapshotValidated()) { + return *m_snapshot_chainstate.get(); + } + assert(m_ibd_chainstate); + return *m_ibd_chainstate.get(); +} + +bool ChainstateManager::IsBackgroundIBD(CChainState* chainstate) const +{ + return (m_snapshot_chainstate && chainstate == m_ibd_chainstate.get()); +} + +void ChainstateManager::Unload() +{ + for (CChainState* chainstate : this->GetAll()) { + chainstate->m_chain.SetTip(nullptr); + chainstate->UnloadBlockIndex(); + } + + m_blockman.Unload(); +} + +void ChainstateManager::Reset() +{ + m_ibd_chainstate.reset(); + m_snapshot_chainstate.reset(); + m_active_chainstate = nullptr; + m_snapshot_validated = false; +} diff --git a/src/validation.h b/src/validation.h index a5335edc43..91b1ba6497 100644 --- a/src/validation.h +++ b/src/validation.h @@ -14,6 +14,7 @@ #include <coins.h> #include <crypto/common.h> // for ReadLE64 #include <fs.h> +#include <optional.h> #include <policy/feerate.h> #include <protocol.h> // For CMessageHeader::MessageStartChars #include <script/script_error.h> @@ -59,57 +60,15 @@ static const unsigned int DEFAULT_ANCESTOR_SIZE_LIMIT = 101; static const unsigned int DEFAULT_DESCENDANT_LIMIT = 25; /** Default for -limitdescendantsize, maximum kilobytes of in-mempool descendants */ static const unsigned int DEFAULT_DESCENDANT_SIZE_LIMIT = 101; -/** - * An extra transaction can be added to a package, as long as it only has one - * ancestor and is no larger than this. Not really any reason to make this - * configurable as it doesn't materially change DoS parameters. - */ -static const unsigned int EXTRA_DESCENDANT_TX_SIZE_LIMIT = 10000; /** Default for -mempoolexpiry, expiration time for mempool transactions in hours */ static const unsigned int DEFAULT_MEMPOOL_EXPIRY = 336; -/** Maximum kilobytes for transactions to store for processing during reorg */ -static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; /** The maximum size of a blk?????.dat file (since 0.8) */ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB -/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ -static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB -/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ -static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB - /** Maximum number of dedicated script-checking threads allowed */ static const int MAX_SCRIPTCHECK_THREADS = 15; /** -par default (number of script-checking threads, 0 = auto) */ static const int DEFAULT_SCRIPTCHECK_THREADS = 0; -/** Number of blocks that can be requested at any given time from a single peer. */ -static const int MAX_BLOCKS_IN_TRANSIT_PER_PEER = 16; -/** Timeout in seconds during which a peer must stall block download progress before being disconnected. */ -static const unsigned int BLOCK_STALLING_TIMEOUT = 2; -/** Number of headers sent in one getheaders result. We rely on the assumption that if a peer sends - * less than this number, we reached its tip. Changing this value is a protocol upgrade. */ -static const unsigned int MAX_HEADERS_RESULTS = 2000; -/** Maximum depth of blocks we're willing to serve as compact blocks to peers - * when requested. For older blocks, a regular BLOCK response will be sent. */ -static const int MAX_CMPCTBLOCK_DEPTH = 5; -/** Maximum depth of blocks we're willing to respond to GETBLOCKTXN requests for. */ -static const int MAX_BLOCKTXN_DEPTH = 10; -/** Size of the "block download window": how far ahead of our current height do we fetch? - * Larger windows tolerate larger download speed differences between peer, but increase the potential - * degree of disordering of blocks on disk (which make reindexing and pruning harder). We'll probably - * want to make this a per-peer adaptive value at some point. */ -static const unsigned int BLOCK_DOWNLOAD_WINDOW = 1024; -/** Time to wait (in seconds) between writing blocks/block index to disk. */ -static const unsigned int DATABASE_WRITE_INTERVAL = 60 * 60; -/** Time to wait (in seconds) between flushing chainstate to disk. */ -static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; -/** Block download timeout base, expressed in millionths of the block interval (i.e. 10 min) */ -static const int64_t BLOCK_DOWNLOAD_TIMEOUT_BASE = 1000000; -/** Additional block download timeout per parallel downloading peer (i.e. 5 min) */ -static const int64_t BLOCK_DOWNLOAD_TIMEOUT_PER_PEER = 500000; - static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60; -/** Maximum age of our tip in seconds for us to be considered current for fee estimation */ -static const int64_t MAX_FEE_ESTIMATION_TIP_AGE = 3 * 60 * 60; - static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; @@ -118,15 +77,21 @@ static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; static const bool DEFAULT_PERSIST_MEMPOOL = true; /** Default for using fee filter */ static const bool DEFAULT_FEEFILTER = true; - -/** Maximum number of headers to announce when relaying blocks with headers message.*/ -static const unsigned int MAX_BLOCKS_TO_ANNOUNCE = 8; - -/** Maximum number of unconnecting headers announcements before DoS score */ -static const int MAX_UNCONNECTING_HEADERS = 10; - /** Default for -stopatheight */ static const int DEFAULT_STOPATHEIGHT = 0; +/** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ::ChainActive().Tip() will not be pruned. */ +static const unsigned int MIN_BLOCKS_TO_KEEP = 288; +static const signed int DEFAULT_CHECKBLOCKS = 6; +static const unsigned int DEFAULT_CHECKLEVEL = 3; +// Require that user allocate at least 550 MiB for block & undo files (blk???.dat and rev???.dat) +// At 1MB per block, 288 blocks = 288MB. +// Add 15% for Undo data = 331MB +// Add 20% for Orphan block rate = 397MB +// We want the low water mark after pruning to be at least 397 MB and since we prune in +// full block file chunks, we need the high water mark which triggers the prune to be +// one 128MB block file + added 15% undo data = 147MB greater for a total of 545MB +// Setting the target to >= 550 MiB will make it likely we can respect the target. +static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024; struct BlockHasher { @@ -174,23 +139,6 @@ extern bool fHavePruned; extern bool fPruneMode; /** Number of MiB of block files that we're trying to stay below. */ extern uint64_t nPruneTarget; -/** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ::ChainActive().Tip() will not be pruned. */ -static const unsigned int MIN_BLOCKS_TO_KEEP = 288; -/** Minimum blocks required to signal NODE_NETWORK_LIMITED */ -static const unsigned int NODE_NETWORK_LIMITED_MIN_BLOCKS = 288; - -static const signed int DEFAULT_CHECKBLOCKS = 6; -static const unsigned int DEFAULT_CHECKLEVEL = 3; - -// Require that user allocate at least 550 MiB for block & undo files (blk???.dat and rev???.dat) -// At 1MB per block, 288 blocks = 288MB. -// Add 15% for Undo data = 331MB -// Add 20% for Orphan block rate = 397MB -// We want the low water mark after pruning to be at least 397 MB and since we prune in -// full block file chunks, we need the high water mark which triggers the prune to be -// one 128MB block file + added 15% undo data = 147MB greater for a total of 545MB -// Setting the target to >= 550 MiB will make it likely we can respect the target. -static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024; /** * Process an incoming block. This only returns after the best known valid @@ -379,9 +327,6 @@ bool TestBlockValidity(BlockValidationState& state, const CChainParams& chainpar * Note that transaction witness validation rules are always enforced when P2SH is enforced. */ bool IsWitnessEnabled(const CBlockIndex* pindexPrev, const Consensus::Params& params); -/** When there are blocks in the active chain with missing data, rewind the chainstate and remove them from the block index */ -bool RewindBlockIndex(const CChainParams& params) LOCKS_EXCLUDED(cs_main); - /** Compute at which vout of the block's coinbase transaction the witness commitment occurs, or -1 if not found */ int GetWitnessCommitmentIndex(const CBlock& block); @@ -539,6 +484,9 @@ enum class CoinsCacheSizeState OK = 0 }; +// Defined below, but needed for `friend` usage in CChainState. +class ChainstateManager; + /** * CChainState stores and provides an API to update our local knowledge of the * current best chain. @@ -591,8 +539,7 @@ private: std::unique_ptr<CoinsViews> m_coins_views; public: - CChainState(BlockManager& blockman) : m_blockman(blockman) {} - CChainState(); + explicit CChainState(BlockManager& blockman, uint256 from_snapshot_blockhash = uint256()); /** * Initialize the CoinsViews UTXO set database management data structures. The in-memory @@ -621,6 +568,13 @@ public: CChain m_chain; /** + * The blockhash which is the base of the snapshot this chainstate was created from. + * + * IsNull() if this chainstate was not created from a snapshot. + */ + const uint256 m_from_snapshot_blockhash{}; + + /** * The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself and all ancestors) and * as good as our current tip or better. Entries may be failed, though, and pruning nodes may be * missing the data for the block. @@ -741,6 +695,8 @@ public: size_t max_coins_cache_size_bytes, size_t max_mempool_size_bytes) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + std::string ToString() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + private: bool ActivateBestChainStep(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); bool ConnectTip(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions& disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, ::mempool.cs); @@ -753,6 +709,8 @@ private: //! Mark a block as not having block data void EraseBlockData(CBlockIndex* index) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + friend ChainstateManager; }; /** Mark a block as precious and reorganize. @@ -768,6 +726,150 @@ bool InvalidateBlock(BlockValidationState& state, const CChainParams& chainparam /** Remove invalidity status from a block and its descendants. */ void ResetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +/** + * Provides an interface for creating and interacting with one or two + * chainstates: an IBD chainstate generated by downloading blocks, and + * an optional snapshot chainstate loaded from a UTXO snapshot. Managed + * chainstates can be maintained at different heights simultaneously. + * + * This class provides abstractions that allow the retrieval of the current + * most-work chainstate ("Active") as well as chainstates which may be in + * background use to validate UTXO snapshots. + * + * Definitions: + * + * *IBD chainstate*: a chainstate whose current state has been "fully" + * validated by the initial block download process. + * + * *Snapshot chainstate*: a chainstate populated by loading in an + * assumeutxo UTXO snapshot. + * + * *Active chainstate*: the chainstate containing the current most-work + * chain. Consulted by most parts of the system (net_processing, + * wallet) as a reflection of the current chain and UTXO set. + * This may either be an IBD chainstate or a snapshot chainstate. + * + * *Background IBD chainstate*: an IBD chainstate for which the + * IBD process is happening in the background while use of the + * active (snapshot) chainstate allows the rest of the system to function. + * + * *Validated chainstate*: the most-work chainstate which has been validated + * locally via initial block download. This will be the snapshot chainstate + * if a snapshot was loaded and all blocks up to the snapshot starting point + * have been downloaded and validated (via background validation), otherwise + * it will be the IBD chainstate. + */ +class ChainstateManager +{ +private: + //! The chainstate used under normal operation (i.e. "regular" IBD) or, if + //! a snapshot is in use, for background validation. + //! + //! Its contents (including on-disk data) will be deleted *upon shutdown* + //! after background validation of the snapshot has completed. We do not + //! free the chainstate contents immediately after it finishes validation + //! to cautiously avoid a case where some other part of the system is still + //! using this pointer (e.g. net_processing). + //! + //! Once this pointer is set to a corresponding chainstate, it will not + //! be reset until init.cpp:Shutdown(). This means it is safe to acquire + //! the contents of this pointer with ::cs_main held, release the lock, + //! and then use the reference without concern of it being deconstructed. + //! + //! This is especially important when, e.g., calling ActivateBestChain() + //! on all chainstates because we are not able to hold ::cs_main going into + //! that call. + std::unique_ptr<CChainState> m_ibd_chainstate; + + //! A chainstate initialized on the basis of a UTXO snapshot. If this is + //! non-null, it is always our active chainstate. + //! + //! Once this pointer is set to a corresponding chainstate, it will not + //! be reset until init.cpp:Shutdown(). This means it is safe to acquire + //! the contents of this pointer with ::cs_main held, release the lock, + //! and then use the reference without concern of it being deconstructed. + //! + //! This is especially important when, e.g., calling ActivateBestChain() + //! on all chainstates because we are not able to hold ::cs_main going into + //! that call. + std::unique_ptr<CChainState> m_snapshot_chainstate; + + //! Points to either the ibd or snapshot chainstate; indicates our + //! most-work chain. + //! + //! Once this pointer is set to a corresponding chainstate, it will not + //! be reset until init.cpp:Shutdown(). This means it is safe to acquire + //! the contents of this pointer with ::cs_main held, release the lock, + //! and then use the reference without concern of it being deconstructed. + //! + //! This is especially important when, e.g., calling ActivateBestChain() + //! on all chainstates because we are not able to hold ::cs_main going into + //! that call. + CChainState* m_active_chainstate{nullptr}; + + //! If true, the assumed-valid chainstate has been fully validated + //! by the background validation chainstate. + bool m_snapshot_validated{false}; + + // For access to m_active_chainstate. + friend CChainState& ChainstateActive(); + friend CChain& ChainActive(); + +public: + //! A single BlockManager instance is shared across each constructed + //! chainstate to avoid duplicating block metadata. + BlockManager m_blockman GUARDED_BY(::cs_main); + + //! Instantiate a new chainstate and assign it based upon whether it is + //! from a snapshot. + //! + //! @param[in] snapshot_blockhash If given, signify that this chainstate + //! is based on a snapshot. + CChainState& InitializeChainstate(const uint256& snapshot_blockhash = uint256()) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + //! Get all chainstates currently being used. + std::vector<CChainState*> GetAll(); + + //! The most-work chain. + CChain& ActiveChain() const; + int ActiveHeight() const { return ActiveChain().Height(); } + CBlockIndex* ActiveTip() const { return ActiveChain().Tip(); } + + BlockMap& BlockIndex() EXCLUSIVE_LOCKS_REQUIRED(::cs_main) + { + return m_blockman.m_block_index; + } + + bool IsSnapshotActive() const; + + Optional<uint256> SnapshotBlockhash() const; + + //! Is there a snapshot in use and has it been fully validated? + bool IsSnapshotValidated() const { return m_snapshot_validated; } + + //! @returns true if this chainstate is being used to validate an active + //! snapshot in the background. + bool IsBackgroundIBD(CChainState* chainstate) const; + + //! Return the most-work chainstate that has been fully validated. + //! + //! During background validation of a snapshot, this is the IBD chain. After + //! background validation has completed, this is the snapshot chain. + CChainState& ValidatedChainstate() const; + + CChain& ValidatedChain() const { return ValidatedChainstate().m_chain; } + CBlockIndex* ValidatedTip() const { return ValidatedChain().Tip(); } + + //! Unload block index and chain data before shutdown. + void Unload() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + //! Clear (deconstruct) chainstate data. + void Reset(); +}; + +extern ChainstateManager g_chainman GUARDED_BY(::cs_main); + /** @returns the most-work valid chainstate. */ CChainState& ChainstateActive(); @@ -777,11 +879,6 @@ CChain& ChainActive(); /** @returns the global block index map. */ BlockMap& BlockIndex(); -// Most often ::ChainstateActive() should be used instead of this, but some code -// may not be able to assume that this has been initialized yet and so must use it -// directly, e.g. init.cpp. -extern std::unique_ptr<CChainState> g_chainstate; - /** Global variable that points to the active block tree (protected by cs_main) */ extern std::unique_ptr<CBlockTreeDB> pblocktree; diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index c06647cb0d..11000774c0 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -67,8 +67,8 @@ public: void Clear() { LOCK(m_mutex); - for (auto it = m_list.begin(); it != m_list.end();) { - it = --it->count ? std::next(it) : m_list.erase(it); + for (const auto& entry : m_map) { + if (!--entry.second->count) m_list.erase(entry.second); } m_map.clear(); } diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 5bbb2c0ad0..079a5d3d53 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -106,6 +106,9 @@ bool SelectCoinsBnB(std::vector<OutputGroup>& utxo_pool, const CAmount& target_v best_selection = curr_selection; best_selection.resize(utxo_pool.size()); best_waste = curr_waste; + if (best_waste == 0) { + break; + } } curr_waste -= (curr_value - actual_target); // Remove the excess value as we will be selecting different coins now backtrack = true; diff --git a/src/wallet/db.h b/src/wallet/db.h index bebaa55d05..1c5f42abca 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 1623ab9074..dafa022ded 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/feebumper.h b/src/wallet/feebumper.h index 859f754761..fc038ae731 100644 --- a/src/wallet/feebumper.h +++ b/src/wallet/feebumper.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 50f064b305..8688f3df5e 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -57,7 +57,6 @@ void WalletInit::AddWalletOptions() const gArgs.AddArg("-salvagewallet", "Attempt to recover private keys from a corrupt wallet on startup", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-spendzeroconfchange", strprintf("Spend unconfirmed change when sending transactions (default: %u)", DEFAULT_SPEND_ZEROCONF_CHANGE), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-txconfirmtarget=<n>", strprintf("If paytxfee is not set, include enough fee so transactions begin confirmation on average within n blocks (default: %u)", DEFAULT_TX_CONFIRM_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); - gArgs.AddArg("-upgradewallet", "Upgrade wallet to latest format on startup", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-wallet=<path>", "Specify wallet database path. Can be specified multiple times to load multiple wallets. Path is interpreted relative to <walletdir> if it is not absolute, and will be created if it does not exist (as a directory containing a wallet.dat file and log files). For backwards compatibility this will also accept names of existing data files in <walletdir>.)", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::WALLET); gArgs.AddArg("-walletbroadcast", strprintf("Make the wallet broadcast transactions (default: %u)", DEFAULT_WALLETBROADCAST), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); gArgs.AddArg("-walletdir=<dir>", "Specify directory to hold wallets (default: <datadir>/wallets if it exists, otherwise <datadir>)", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::WALLET); @@ -116,12 +115,6 @@ bool WalletInit::ParameterInteraction() const } } - if (is_multiwallet) { - if (gArgs.GetBoolArg("-upgradewallet", false)) { - return InitError(strprintf("%s is only allowed with a single wallet file", "-upgradewallet")); - } - } - if (gArgs.GetBoolArg("-sysperms", false)) return InitError("-sysperms is not allowed in combination with enabled wallet functionality"); diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index d6e44c7be5..0afd2dfcf0 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index ea54027c48..9417e2bd58 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -28,6 +28,8 @@ +using interfaces::FoundBlock; + std::string static EncodeDumpString(const std::string &str) { std::stringstream ret; for (const unsigned char c : str) { @@ -128,7 +130,7 @@ UniValue importprivkey(const JSONRPCRequest& request) EnsureLegacyScriptPubKeyMan(*wallet, true); - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); bool fRescan = true; { auto locked_chain = pwallet->chain().lock(); @@ -272,7 +274,7 @@ UniValue importaddress(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled when blocks are pruned"); } - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); if (fRescan && !reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } @@ -360,8 +362,9 @@ UniValue importprunedfunds(const JSONRPCRequest& request) } auto locked_chain = pwallet->chain().lock(); - Optional<int> height = locked_chain->getBlockHeight(merkleBlock.header.GetHash()); - if (height == nullopt) { + LOCK(pwallet->cs_wallet); + int height; + if (!pwallet->chain().findAncestorByHash(pwallet->GetLastBlockHash(), merkleBlock.header.GetHash(), FoundBlock().height(height))) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found in chain"); } @@ -372,11 +375,9 @@ UniValue importprunedfunds(const JSONRPCRequest& request) unsigned int txnIndex = vIndex[it - vMatch.begin()]; - CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, *height, merkleBlock.header.GetHash(), txnIndex); + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, height, merkleBlock.header.GetHash(), txnIndex); wtx.m_confirm = confirm; - LOCK(pwallet->cs_wallet); - if (pwallet->IsMine(*wtx.tx)) { pwallet->AddToWallet(wtx, false); return NullUniValue; @@ -473,7 +474,7 @@ UniValue importpubkey(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Rescan is disabled when blocks are pruned"); } - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); if (fRescan && !reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } @@ -548,7 +549,7 @@ UniValue importwallet(const JSONRPCRequest& request) throw JSONRPCError(RPC_WALLET_ERROR, "Importing wallets is disabled when blocks are pruned"); } - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); if (!reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } @@ -566,8 +567,7 @@ UniValue importwallet(const JSONRPCRequest& request) if (!file.is_open()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open wallet dump file"); } - Optional<int> tip_height = locked_chain->getHeight(); - nTimeBegin = tip_height ? locked_chain->getBlockTime(*tip_height) : 0; + CHECK_NONFATAL(pwallet->chain().findBlock(pwallet->GetLastBlockHash(), FoundBlock().time(nTimeBegin))); int64_t nFilesize = std::max((int64_t)1, (int64_t)file.tellg()); file.seekg(0, file.beg); @@ -724,9 +724,8 @@ UniValue dumpprivkey(const JSONRPCRequest& request) UniValue dumpwallet(const JSONRPCRequest& request) { - std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); - const CWallet* const pwallet = wallet.get(); - if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { + std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + if (!EnsureWalletIsAvailable(pwallet.get(), request.fHelp)) { return NullUniValue; } @@ -736,7 +735,7 @@ UniValue dumpwallet(const JSONRPCRequest& request) "Note that if your wallet contains keys which are not derived from your HD seed (e.g. imported keys), these are not covered by\n" "only backing up the seed itself, and must be backed up too (e.g. ensure you back up the whole dumpfile).\n", { - {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The filename with path (either absolute or relative to bitcoind)"}, + {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The filename with path (absolute path recommended)"}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -750,12 +749,17 @@ UniValue dumpwallet(const JSONRPCRequest& request) }, }.Check(request); - LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*wallet); + CWallet& wallet = *pwallet; + LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(wallet); + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + wallet.BlockUntilSyncedToCurrentChain(); auto locked_chain = pwallet->chain().lock(); LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); - EnsureWalletIsUnlocked(pwallet); + EnsureWalletIsUnlocked(&wallet); fs::path filepath = request.params[0].get_str(); filepath = fs::absolute(filepath); @@ -791,9 +795,10 @@ UniValue dumpwallet(const JSONRPCRequest& request) // produce output file << strprintf("# Wallet dump created by Bitcoin %s\n", CLIENT_BUILD); file << strprintf("# * Created on %s\n", FormatISO8601DateTime(GetTime())); - const Optional<int> tip_height = locked_chain->getHeight(); - file << strprintf("# * Best block at time of backup was %i (%s),\n", tip_height.get_value_or(-1), tip_height ? locked_chain->getBlockHash(*tip_height).ToString() : "(missing block hash)"); - file << strprintf("# mined on %s\n", tip_height ? FormatISO8601DateTime(locked_chain->getBlockTime(*tip_height)) : "(missing block time)"); + file << strprintf("# * Best block at time of backup was %i (%s),\n", wallet.GetLastBlockHeight(), wallet.GetLastBlockHash().ToString()); + int64_t block_time = 0; + CHECK_NONFATAL(wallet.chain().findBlock(wallet.GetLastBlockHash(), FoundBlock().time(block_time))); + file << strprintf("# mined on %s\n", FormatISO8601DateTime(block_time)); file << "\n"; // add the base58check encoded extended master if the wallet uses HD @@ -816,8 +821,8 @@ UniValue dumpwallet(const JSONRPCRequest& request) CKey key; if (spk_man.GetKey(keyid, key)) { file << strprintf("%s %s ", EncodeSecret(key), strTime); - if (GetWalletAddressesForKey(&spk_man, pwallet, keyid, strAddr, strLabel)) { - file << strprintf("label=%s", strLabel); + if (GetWalletAddressesForKey(&spk_man, &wallet, keyid, strAddr, strLabel)) { + file << strprintf("label=%s", strLabel); } else if (keyid == seed_id) { file << "hdseed=1"; } else if (mapKeyPool.count(keyid)) { @@ -1364,7 +1369,7 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) } } - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); if (fRescan && !reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } @@ -1379,20 +1384,13 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) EnsureWalletIsUnlocked(pwallet); // Verify all timestamps are present before importing any keys. - const Optional<int> tip_height = locked_chain->getHeight(); - now = tip_height ? locked_chain->getBlockMedianTimePast(*tip_height) : 0; + CHECK_NONFATAL(pwallet->chain().findBlock(pwallet->GetLastBlockHash(), FoundBlock().time(nLowestTimestamp).mtpTime(now))); for (const UniValue& data : requests.getValues()) { GetImportTimestamp(data, now); } const int64_t minimumTimestamp = 1; - if (fRescan && tip_height) { - nLowestTimestamp = locked_chain->getBlockTime(*tip_height); - } else { - fRescan = false; - } - for (const UniValue& data : requests.getValues()) { const int64_t timestamp = std::max(GetImportTimestamp(data, now), minimumTimestamp); const UniValue result = ProcessImport(pwallet, data, timestamp); @@ -1460,3 +1458,297 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) return response; } + +static UniValue ProcessDescriptorImport(CWallet * const pwallet, const UniValue& data, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) +{ + UniValue warnings(UniValue::VARR); + UniValue result(UniValue::VOBJ); + + try { + if (!data.exists("desc")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Descriptor not found."); + } + + const std::string& descriptor = data["desc"].get_str(); + const bool active = data.exists("active") ? data["active"].get_bool() : false; + const bool internal = data.exists("internal") ? data["internal"].get_bool() : false; + const std::string& label = data.exists("label") ? data["label"].get_str() : ""; + + // Parse descriptor string + FlatSigningProvider keys; + std::string error; + auto parsed_desc = Parse(descriptor, keys, error, /* require_checksum = */ true); + if (!parsed_desc) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error); + } + + // Range check + int64_t range_start = 0, range_end = 1, next_index = 0; + if (!parsed_desc->IsRange() && data.exists("range")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Range should not be specified for an un-ranged descriptor"); + } else if (parsed_desc->IsRange()) { + if (data.exists("range")) { + auto range = ParseDescriptorRange(data["range"]); + range_start = range.first; + range_end = range.second + 1; // Specified range end is inclusive, but we need range end as exclusive + } else { + warnings.push_back("Range not given, using default keypool range"); + range_start = 0; + range_end = gArgs.GetArg("-keypool", DEFAULT_KEYPOOL_SIZE); + } + next_index = range_start; + + if (data.exists("next_index")) { + next_index = data["next_index"].get_int64(); + // bound checks + if (next_index < range_start || next_index >= range_end) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "next_index is out of range"); + } + } + } + + // Active descriptors must be ranged + if (active && !parsed_desc->IsRange()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Active descriptors must be ranged"); + } + + // Ranged descriptors should not have a label + if (data.exists("range") && data.exists("label")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Ranged descriptors should not have a label"); + } + + // Internal addresses should not have a label either + if (internal && data.exists("label")) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Internal addresses should not have a label"); + } + + // Combo descriptor check + if (active && !parsed_desc->IsSingleType()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Combo descriptors cannot be set to active"); + } + + // If the wallet disabled private keys, abort if private keys exist + if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !keys.keys.empty()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Cannot import private keys to a wallet with private keys disabled"); + } + + // Need to ExpandPrivate to check if private keys are available for all pubkeys + FlatSigningProvider expand_keys; + std::vector<CScript> scripts; + parsed_desc->Expand(0, keys, scripts, expand_keys); + parsed_desc->ExpandPrivate(0, keys, expand_keys); + + // Check if all private keys are provided + bool have_all_privkeys = !expand_keys.keys.empty(); + for (const auto& entry : expand_keys.origins) { + const CKeyID& key_id = entry.first; + CKey key; + if (!expand_keys.GetKey(key_id, key)) { + have_all_privkeys = false; + break; + } + } + + // If private keys are enabled, check some things. + if (!pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + if (keys.keys.empty()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Cannot import descriptor without private keys to a wallet with private keys enabled"); + } + if (!have_all_privkeys) { + warnings.push_back("Not all private keys provided. Some wallet functionality may return unexpected errors"); + } + } + + WalletDescriptor w_desc(std::move(parsed_desc), timestamp, range_start, range_end, next_index); + + // Check if the wallet already contains the descriptor + auto existing_spk_manager = pwallet->GetDescriptorScriptPubKeyMan(w_desc); + if (existing_spk_manager) { + LOCK(existing_spk_manager->cs_desc_man); + if (range_start > existing_spk_manager->GetWalletDescriptor().range_start) { + throw JSONRPCError(RPC_INVALID_PARAMS, strprintf("range_start can only decrease; current range = [%d,%d]", existing_spk_manager->GetWalletDescriptor().range_start, existing_spk_manager->GetWalletDescriptor().range_end)); + } + } + + // Add descriptor to the wallet + auto spk_manager = pwallet->AddWalletDescriptor(w_desc, keys, label); + if (spk_manager == nullptr) { + throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Could not add descriptor '%s'", descriptor)); + } + + // Set descriptor as active if necessary + if (active) { + if (!w_desc.descriptor->GetOutputType()) { + warnings.push_back("Unknown output type, cannot set descriptor to active."); + } else { + pwallet->SetActiveScriptPubKeyMan(spk_manager->GetID(), *w_desc.descriptor->GetOutputType(), internal); + } + } + + result.pushKV("success", UniValue(true)); + } catch (const UniValue& e) { + result.pushKV("success", UniValue(false)); + result.pushKV("error", e); + } catch (...) { + result.pushKV("success", UniValue(false)); + + result.pushKV("error", JSONRPCError(RPC_MISC_ERROR, "Missing required fields")); + } + if (warnings.size()) result.pushKV("warnings", warnings); + return result; +} + +UniValue importdescriptors(const JSONRPCRequest& main_request) { + // Acquire the wallet + std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(main_request); + CWallet* const pwallet = wallet.get(); + if (!EnsureWalletIsAvailable(pwallet, main_request.fHelp)) { + return NullUniValue; + } + + RPCHelpMan{"importdescriptors", + "\nImport descriptors. This will trigger a rescan of the blockchain based on the earliest timestamp of all descriptors being imported. Requires a new wallet backup.\n" + "\nNote: This call can take over an hour to complete if using an early timestamp; during that time, other rpc calls\n" + "may report that the imported keys, addresses or scripts exist but related transactions are still missing.\n", + { + {"requests", RPCArg::Type::ARR, RPCArg::Optional::NO, "Data to be imported", + { + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + { + {"desc", RPCArg::Type::STR, RPCArg::Optional::NO, "Descriptor to import."}, + {"active", RPCArg::Type::BOOL, /* default */ "false", "Set this descriptor to be the active descriptor for the corresponding output type/externality"}, + {"range", RPCArg::Type::RANGE, RPCArg::Optional::OMITTED, "If a ranged descriptor is used, this specifies the end or the range (in the form [begin,end]) to import"}, + {"next_index", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "If a ranged descriptor is set to active, this specifies the next index to generate addresses from"}, + {"timestamp", RPCArg::Type::NUM, RPCArg::Optional::NO, "Time from which to start rescanning the blockchain for this descriptor, in " + UNIX_EPOCH_TIME + "\n" + " Use the string \"now\" to substitute the current synced blockchain time.\n" + " \"now\" can be specified to bypass scanning, for outputs which are known to never have been used, and\n" + " 0 can be specified to scan the entire blockchain. Blocks up to 2 hours before the earliest timestamp\n" + " of all descriptors being imported will be scanned.", + /* oneline_description */ "", {"timestamp | \"now\"", "integer / string"} + }, + {"internal", RPCArg::Type::BOOL, /* default */ "false", "Whether matching outputs should be treated as not incoming payments (e.g. change)"}, + {"label", RPCArg::Type::STR, /* default */ "''", "Label to assign to the address, only allowed with internal=false"}, + }, + }, + }, + "\"requests\""}, + }, + RPCResult{ + RPCResult::Type::ARR, "", "Response is an array with the same size as the input that has the execution result", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::BOOL, "success", ""}, + {RPCResult::Type::ARR, "warnings", /* optional */ true, "", + { + {RPCResult::Type::STR, "", ""}, + }}, + {RPCResult::Type::OBJ, "error", /* optional */ true, "", + { + {RPCResult::Type::ELISION, "", "JSONRPC error"}, + }}, + }}, + } + }, + RPCExamples{ + HelpExampleCli("importdescriptors", "'[{ \"desc\": \"<my descriptor>\", \"timestamp\":1455191478, \"internal\": true }, " + "{ \"desc\": \"<my desccriptor 2>\", \"label\": \"example 2\", \"timestamp\": 1455191480 }]'") + + HelpExampleCli("importdescriptors", "'[{ \"desc\": \"<my descriptor>\", \"timestamp\":1455191478, \"active\": true, \"range\": [0,100], \"label\": \"<my bech32 wallet>\" }]'") + }, + }.Check(main_request); + + // Make sure wallet is a descriptor wallet + if (!pwallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + throw JSONRPCError(RPC_WALLET_ERROR, "importdescriptors is not available for non-descriptor wallets"); + } + + RPCTypeCheck(main_request.params, {UniValue::VARR, UniValue::VOBJ}); + + WalletRescanReserver reserver(*pwallet); + if (!reserver.reserve()) { + throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); + } + + const UniValue& requests = main_request.params[0]; + const int64_t minimum_timestamp = 1; + int64_t now = 0; + int64_t lowest_timestamp = 0; + bool rescan = false; + UniValue response(UniValue::VARR); + { + auto locked_chain = pwallet->chain().lock(); + LOCK(pwallet->cs_wallet); + EnsureWalletIsUnlocked(pwallet); + + CHECK_NONFATAL(pwallet->chain().findBlock(pwallet->GetLastBlockHash(), FoundBlock().time(lowest_timestamp).mtpTime(now))); + + // Get all timestamps and extract the lowest timestamp + for (const UniValue& request : requests.getValues()) { + // This throws an error if "timestamp" doesn't exist + const int64_t timestamp = std::max(GetImportTimestamp(request, now), minimum_timestamp); + const UniValue result = ProcessDescriptorImport(pwallet, request, timestamp); + response.push_back(result); + + if (lowest_timestamp > timestamp ) { + lowest_timestamp = timestamp; + } + + // If we know the chain tip, and at least one request was successful then allow rescan + if (!rescan && result["success"].get_bool()) { + rescan = true; + } + } + pwallet->ConnectScriptPubKeyManNotifiers(); + } + + // Rescan the blockchain using the lowest timestamp + if (rescan) { + int64_t scanned_time = pwallet->RescanFromTime(lowest_timestamp, reserver, true /* update */); + { + auto locked_chain = pwallet->chain().lock(); + LOCK(pwallet->cs_wallet); + pwallet->ReacceptWalletTransactions(); + } + + if (pwallet->IsAbortingRescan()) { + throw JSONRPCError(RPC_MISC_ERROR, "Rescan aborted by user."); + } + + if (scanned_time > lowest_timestamp) { + std::vector<UniValue> results = response.getValues(); + response.clear(); + response.setArray(); + + // Compose the response + for (unsigned int i = 0; i < requests.size(); ++i) { + const UniValue& request = requests.getValues().at(i); + + // If the descriptor timestamp is within the successfully scanned + // range, or if the import result already has an error set, let + // the result stand unmodified. Otherwise replace the result + // with an error message. + if (scanned_time <= GetImportTimestamp(request, now) || results.at(i).exists("error")) { + response.push_back(results.at(i)); + } else { + UniValue result = UniValue(UniValue::VOBJ); + result.pushKV("success", UniValue(false)); + result.pushKV( + "error", + JSONRPCError( + RPC_MISC_ERROR, + strprintf("Rescan failed for descriptor with timestamp %d. There was an error reading a " + "block from time %d, which is after or within %d seconds of key creation, and " + "could contain transactions pertaining to the desc. As a result, transactions " + "and coins using this desc may not appear in the wallet. This error could be " + "caused by pruning or data corruption (see bitcoind log for details) and could " + "be dealt with by downloading and rescanning the relevant blocks (see -reindex " + "and -rescan options).", + GetImportTimestamp(request, now), scanned_time - TIMESTAMP_WINDOW - 1, TIMESTAMP_WINDOW))); + response.push_back(std::move(result)); + } + } + } + } + + return response; +} diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 61ad2f1198..49c583c422 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -37,6 +37,8 @@ #include <univalue.h> +using interfaces::FoundBlock; + static const std::string WALLET_ENDPOINT_BASE = "/wallet/"; static const std::string HELP_REQUIRING_PASSPHRASE{"\nRequires wallet passphrase to be set with walletpassphrase call if wallet is encrypted.\n"}; @@ -78,9 +80,9 @@ bool HaveKey(const SigningProvider& wallet, const CKey& key) bool GetWalletNameFromJSONRPCRequest(const JSONRPCRequest& request, std::string& wallet_name) { - if (request.URI.substr(0, WALLET_ENDPOINT_BASE.size()) == WALLET_ENDPOINT_BASE) { + if (URL_DECODE && request.URI.substr(0, WALLET_ENDPOINT_BASE.size()) == WALLET_ENDPOINT_BASE) { // wallet endpoint was used - wallet_name = urlDecode(request.URI.substr(WALLET_ENDPOINT_BASE.size())); + wallet_name = URL_DECODE(request.URI.substr(WALLET_ENDPOINT_BASE.size())); return true; } return false; @@ -143,8 +145,7 @@ static void WalletTxToJSON(interfaces::Chain& chain, interfaces::Chain::Lock& lo entry.pushKV("blockheight", wtx.m_confirm.block_height); entry.pushKV("blockindex", wtx.m_confirm.nIndex); int64_t block_time; - bool found_block = chain.findBlock(wtx.m_confirm.hashBlock, nullptr /* block */, &block_time); - CHECK_NONFATAL(found_block); + CHECK_NONFATAL(chain.findBlock(wtx.m_confirm.hashBlock, FoundBlock().time(block_time))); entry.pushKV("blocktime", block_time); } else { entry.pushKV("trusted", wtx.IsTrusted(locked_chain)); @@ -571,6 +572,52 @@ static UniValue signmessage(const JSONRPCRequest& request) return signature; } +static CAmount GetReceived(interfaces::Chain::Lock& locked_chain, const CWallet& wallet, const UniValue& params, bool by_label) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +{ + std::set<CTxDestination> address_set; + + if (by_label) { + // Get the set of addresses assigned to label + std::string label = LabelFromValue(params[0]); + address_set = wallet.GetLabelAddresses(label); + } else { + // Get the address + CTxDestination dest = DecodeDestination(params[0].get_str()); + if (!IsValidDestination(dest)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); + } + CScript script_pub_key = GetScriptForDestination(dest); + if (!wallet.IsMine(script_pub_key)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Address not found in wallet"); + } + address_set.insert(dest); + } + + // Minimum confirmations + int min_depth = 1; + if (!params[1].isNull()) + min_depth = params[1].get_int(); + + // Tally + CAmount amount = 0; + for (const std::pair<const uint256, CWalletTx>& wtx_pair : wallet.mapWallet) { + const CWalletTx& wtx = wtx_pair.second; + if (wtx.IsCoinBase() || !locked_chain.checkFinalTx(*wtx.tx) || wtx.GetDepthInMainChain() < min_depth) { + continue; + } + + for (const CTxOut& txout : wtx.tx->vout) { + CTxDestination address; + if (ExtractDestination(txout.scriptPubKey, address) && wallet.IsMine(address) && address_set.count(address)) { + amount += txout.nValue; + } + } + } + + return amount; +} + + static UniValue getreceivedbyaddress(const JSONRPCRequest& request) { std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); @@ -608,36 +655,7 @@ static UniValue getreceivedbyaddress(const JSONRPCRequest& request) auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - // Bitcoin address - CTxDestination dest = DecodeDestination(request.params[0].get_str()); - if (!IsValidDestination(dest)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); - } - CScript scriptPubKey = GetScriptForDestination(dest); - if (!pwallet->IsMine(scriptPubKey)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Address not found in wallet"); - } - - // Minimum confirmations - int nMinDepth = 1; - if (!request.params[1].isNull()) - nMinDepth = request.params[1].get_int(); - - // Tally - CAmount nAmount = 0; - for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { - const CWalletTx& wtx = pairWtx.second; - if (wtx.IsCoinBase() || !locked_chain->checkFinalTx(*wtx.tx)) { - continue; - } - - for (const CTxOut& txout : wtx.tx->vout) - if (txout.scriptPubKey == scriptPubKey) - if (wtx.GetDepthInMainChain() >= nMinDepth) - nAmount += txout.nValue; - } - - return ValueFromAmount(nAmount); + return ValueFromAmount(GetReceived(*locked_chain, *pwallet, request.params, /* by_label */ false)); } @@ -678,34 +696,7 @@ static UniValue getreceivedbylabel(const JSONRPCRequest& request) auto locked_chain = pwallet->chain().lock(); LOCK(pwallet->cs_wallet); - // Minimum confirmations - int nMinDepth = 1; - if (!request.params[1].isNull()) - nMinDepth = request.params[1].get_int(); - - // Get the set of pub keys assigned to label - std::string label = LabelFromValue(request.params[0]); - std::set<CTxDestination> setAddress = pwallet->GetLabelAddresses(label); - - // Tally - CAmount nAmount = 0; - for (const std::pair<const uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { - const CWalletTx& wtx = pairWtx.second; - if (wtx.IsCoinBase() || !locked_chain->checkFinalTx(*wtx.tx)) { - continue; - } - - for (const CTxOut& txout : wtx.tx->vout) - { - CTxDestination address; - if (ExtractDestination(txout.scriptPubKey, address) && pwallet->IsMine(address) && setAddress.count(address)) { - if (wtx.GetDepthInMainChain() >= nMinDepth) - nAmount += txout.nValue; - } - } - } - - return ValueFromAmount(nAmount); + return ValueFromAmount(GetReceived(*locked_chain, *pwallet, request.params, /* by_label */ true)); } @@ -732,9 +723,9 @@ static UniValue getbalance(const JSONRPCRequest& request) RPCResult::Type::STR_AMOUNT, "amount", "The total amount in " + CURRENCY_UNIT + " received for this wallet." }, RPCExamples{ - "\nThe total amount in the wallet with 1 or more confirmations\n" + "\nThe total amount in the wallet with 0 or more confirmations\n" + HelpExampleCli("getbalance", "") + - "\nThe total amount in the wallet at least 6 blocks confirmed\n" + "\nThe total amount in the wallet with at least 6 confirmations\n" + HelpExampleCli("getbalance", "\"*\" 6") + "\nAs a JSON-RPC call\n" + HelpExampleRpc("getbalance", "\"*\", 6") @@ -1399,8 +1390,8 @@ UniValue listtransactions(const JSONRPCRequest& request) "\nIf a label name is provided, this will return only incoming transactions paying to addresses with the specified label.\n" "\nReturns up to 'count' most recent transactions skipping the first 'from' transactions.\n", { - {"label", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "If set, should be a valid label name to return only incoming transactions\n" - " with the specified label, or \"*\" to disable filtering and return all transactions."}, + {"label|dummy", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "If set, should be a valid label name to return only incoming transactions\n" + "with the specified label, or \"*\" to disable filtering and return all transactions."}, {"count", RPCArg::Type::NUM, /* default */ "10", "The number of transactions to return"}, {"skip", RPCArg::Type::NUM, /* default */ "0", "The number of transactions to skip"}, {"include_watchonly", RPCArg::Type::BOOL, /* default */ "true for watch-only wallets, otherwise false", "Include transactions to watch-only addresses (see 'importaddress')"}, @@ -1578,8 +1569,9 @@ static UniValue listsinceblock(const JSONRPCRequest& request) uint256 blockId; if (!request.params[0].isNull() && !request.params[0].get_str().empty()) { blockId = ParseHashV(request.params[0], "blockhash"); - height = locked_chain->findFork(blockId, &altheight); - if (!height) { + height.emplace(); + altheight.emplace(); + if (!pwallet->chain().findCommonAncestor(blockId, pwallet->GetLastBlockHash(), /* ancestor out */ FoundBlock().height(*height), /* blockId out */ FoundBlock().height(*altheight))) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } } @@ -1598,8 +1590,7 @@ static UniValue listsinceblock(const JSONRPCRequest& request) bool include_removed = (request.params[3].isNull() || request.params[3].get_bool()); - const Optional<int> tip_height = locked_chain->getHeight(); - int depth = tip_height && height ? (1 + *tip_height - *height) : -1; + int depth = height ? pwallet->GetLastBlockHeight() + 1 - *height : -1; UniValue transactions(UniValue::VARR); @@ -1616,7 +1607,7 @@ static UniValue listsinceblock(const JSONRPCRequest& request) UniValue removed(UniValue::VARR); while (include_removed && altheight && *altheight > *height) { CBlock block; - if (!pwallet->chain().findBlock(blockId, &block) || block.IsNull()) { + if (!pwallet->chain().findBlock(blockId, FoundBlock().data(block)) || block.IsNull()) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Can't read block from disk"); } for (const CTransactionRef& tx : block.vtx) { @@ -1631,8 +1622,8 @@ static UniValue listsinceblock(const JSONRPCRequest& request) --*altheight; } - int last_height = tip_height ? *tip_height + 1 - target_confirms : -1; - uint256 lastblock = last_height >= 0 ? locked_chain->getBlockHash(last_height) : uint256(); + uint256 lastblock; + CHECK_NONFATAL(pwallet->chain().findAncestorByHeight(pwallet->GetLastBlockHash(), pwallet->GetLastBlockHeight() + 1 - target_confirms, FoundBlock().hash(lastblock))); UniValue ret(UniValue::VOBJ); ret.pushKV("transactions", transactions); @@ -1860,7 +1851,7 @@ static UniValue keypoolrefill(const JSONRPCRequest& request) }, }.Check(request); - if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + if (pwallet->IsLegacy() && pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { throw JSONRPCError(RPC_WALLET_ERROR, "Error: Private keys are disabled for this wallet"); } @@ -2324,7 +2315,8 @@ static UniValue settxfee(const JSONRPCRequest& request) } RPCHelpMan{"settxfee", - "\nSet the transaction fee per kB for this wallet. Overrides the global -paytxfee command line parameter.\n", + "\nSet the transaction fee per kB for this wallet. Overrides the global -paytxfee command line parameter.\n" + "Can be deactivated by passing 0 as the fee. In that case automatic fee selection will be used by default.\n", { {"amount", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "The transaction fee in " + CURRENCY_UNIT + "/kB"}, }, @@ -2342,12 +2334,15 @@ static UniValue settxfee(const JSONRPCRequest& request) CAmount nAmount = AmountFromValue(request.params[0]); CFeeRate tx_fee_rate(nAmount, 1000); + CFeeRate max_tx_fee_rate(pwallet->m_default_max_tx_fee, 1000); if (tx_fee_rate == CFeeRate(0)) { // automatic selection } else if (tx_fee_rate < pwallet->chain().relayMinFee()) { throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("txfee cannot be less than min relay tx fee (%s)", pwallet->chain().relayMinFee().ToString())); } else if (tx_fee_rate < pwallet->m_min_fee) { throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("txfee cannot be less than wallet min fee (%s)", pwallet->m_min_fee.ToString())); + } else if (tx_fee_rate > max_tx_fee_rate) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("txfee cannot be more than wallet max tx fee (%s)", max_tx_fee_rate.ToString())); } pwallet->m_pay_tx_fee = tx_fee_rate; @@ -2444,7 +2439,7 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) {RPCResult::Type::STR_AMOUNT, "unconfirmed_balance", "DEPRECATED. Identical to getbalances().mine.untrusted_pending"}, {RPCResult::Type::STR_AMOUNT, "immature_balance", "DEPRECATED. Identical to getbalances().mine.immature"}, {RPCResult::Type::NUM, "txcount", "the total number of transactions in the wallet"}, - {RPCResult::Type::NUM_TIME, "keypoololdest", "the " + UNIX_EPOCH_TIME + " of the oldest pre-generated key in the key pool"}, + {RPCResult::Type::NUM_TIME, "keypoololdest", "the " + UNIX_EPOCH_TIME + " of the oldest pre-generated key in the key pool. Legacy wallets only."}, {RPCResult::Type::NUM, "keypoolsize", "how many new keys are pre-generated (only counts external keys)"}, {RPCResult::Type::NUM, "keypoolsize_hd_internal", "how many new keys are pre-generated for internal use (used for change outputs, only appears if the wallet is using this feature, otherwise external keys are used)"}, {RPCResult::Type::NUM_TIME, "unlocked_until", "the " + UNIX_EPOCH_TIME + " until which the wallet is unlocked for transfers, or 0 if the wallet is locked"}, @@ -2457,6 +2452,7 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) {RPCResult::Type::NUM, "duration", "elapsed seconds since scan start"}, {RPCResult::Type::NUM, "progress", "scanning progress percentage [0.0, 1.0]"}, }}, + {RPCResult::Type::BOOL, "descriptors", "whether this wallet uses descriptors for scriptPubKey management"}, }}, }, RPCExamples{ @@ -2476,13 +2472,16 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) size_t kpExternalSize = pwallet->KeypoolCountExternalKeys(); const auto bal = pwallet->GetBalance(); + int64_t kp_oldest = pwallet->GetOldestKeyPoolTime(); obj.pushKV("walletname", pwallet->GetName()); obj.pushKV("walletversion", pwallet->GetVersion()); obj.pushKV("balance", ValueFromAmount(bal.m_mine_trusted)); obj.pushKV("unconfirmed_balance", ValueFromAmount(bal.m_mine_untrusted_pending)); obj.pushKV("immature_balance", ValueFromAmount(bal.m_mine_immature)); obj.pushKV("txcount", (int)pwallet->mapWallet.size()); - obj.pushKV("keypoololdest", pwallet->GetOldestKeyPoolTime()); + if (kp_oldest > 0) { + obj.pushKV("keypoololdest", kp_oldest); + } obj.pushKV("keypoolsize", (int64_t)kpExternalSize); LegacyScriptPubKeyMan* spk_man = pwallet->GetLegacyScriptPubKeyMan(); @@ -2510,6 +2509,7 @@ static UniValue getwalletinfo(const JSONRPCRequest& request) } else { obj.pushKV("scanning", false); } + obj.pushKV("descriptors", pwallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); return obj; } @@ -2586,7 +2586,7 @@ static UniValue loadwallet(const JSONRPCRequest& request) RPCHelpMan{"loadwallet", "\nLoads a wallet from a wallet file or directory." "\nNote that all wallet command-line options used when starting bitcoind will be" - "\napplied to the new wallet (eg -zapwallettxes, upgradewallet, rescan, etc).\n", + "\napplied to the new wallet (eg -zapwallettxes, rescan, etc).\n", { {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The wallet directory or .dat file."}, }, @@ -2706,6 +2706,7 @@ static UniValue createwallet(const JSONRPCRequest& request) {"blank", RPCArg::Type::BOOL, /* default */ "false", "Create a blank wallet. A blank wallet has no keys or HD seed. One can be set using sethdseed."}, {"passphrase", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "Encrypt the wallet with this passphrase."}, {"avoid_reuse", RPCArg::Type::BOOL, /* default */ "false", "Keep track of coin reuse, and treat dirty and clean coins differently with privacy considerations in mind."}, + {"descriptors", RPCArg::Type::BOOL, /* default */ "false", "Create a native descriptor wallet. The wallet will use descriptors internally to handle address creation"}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -2742,6 +2743,9 @@ static UniValue createwallet(const JSONRPCRequest& request) if (!request.params[4].isNull() && request.params[4].get_bool()) { flags |= WALLET_FLAG_AVOID_REUSE; } + if (!request.params[5].isNull() && request.params[5].get_bool()) { + flags |= WALLET_FLAG_DESCRIPTORS; + } std::string error; std::shared_ptr<CWallet> wallet; @@ -3535,28 +3539,29 @@ UniValue rescanblockchain(const JSONRPCRequest& request) }, }.Check(request); - WalletRescanReserver reserver(pwallet); + WalletRescanReserver reserver(*pwallet); if (!reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); } int start_height = 0; - uint256 start_block, stop_block; + Optional<int> stop_height; + uint256 start_block; { auto locked_chain = pwallet->chain().lock(); - Optional<int> tip_height = locked_chain->getHeight(); + LOCK(pwallet->cs_wallet); + int tip_height = pwallet->GetLastBlockHeight(); if (!request.params[0].isNull()) { start_height = request.params[0].get_int(); - if (start_height < 0 || !tip_height || start_height > *tip_height) { + if (start_height < 0 || start_height > tip_height) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid start_height"); } } - Optional<int> stop_height; if (!request.params[1].isNull()) { stop_height = request.params[1].get_int(); - if (*stop_height < 0 || !tip_height || *stop_height > *tip_height) { + if (*stop_height < 0 || *stop_height > tip_height) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid stop_height"); } else if (*stop_height < start_height) { @@ -3565,25 +3570,15 @@ UniValue rescanblockchain(const JSONRPCRequest& request) } // We can't rescan beyond non-pruned blocks, stop and throw an error - if (locked_chain->findPruned(start_height, stop_height)) { + if (!pwallet->chain().hasBlocks(pwallet->GetLastBlockHash(), start_height, stop_height)) { throw JSONRPCError(RPC_MISC_ERROR, "Can't rescan beyond pruned data. Use RPC call getblockchaininfo to determine your pruned height."); } - if (tip_height) { - start_block = locked_chain->getBlockHash(start_height); - // If called with a stop_height, set the stop_height here to - // trigger a rescan to that height. - // If called without a stop height, leave stop_height as null here - // so rescan continues to the tip (even if the tip advances during - // rescan). - if (stop_height) { - stop_block = locked_chain->getBlockHash(*stop_height); - } - } + CHECK_NONFATAL(pwallet->chain().findAncestorByHeight(pwallet->GetLastBlockHash(), start_height, FoundBlock().hash(start_block))); } CWallet::ScanResult result = - pwallet->ScanForWalletTransactions(start_block, stop_block, reserver, true /* fUpdate */); + pwallet->ScanForWalletTransactions(start_block, start_height, stop_height, reserver, true /* fUpdate */); switch (result.status) { case CWallet::ScanResult::SUCCESS: break; @@ -3829,7 +3824,7 @@ UniValue getaddressinfo(const JSONRPCRequest& request) ScriptPubKeyMan* spk_man = pwallet->GetScriptPubKeyMan(scriptPubKey); if (spk_man) { - if (const CKeyMetadata* meta = spk_man->GetMetadata(dest)) { + if (const std::unique_ptr<CKeyMetadata> meta = spk_man->GetMetadata(dest)) { ret.pushKV("timestamp", meta->nCreateTime); if (meta->has_key_origin) { ret.pushKV("hdkeypath", WriteHDKeypath(meta->key_origin.path)); @@ -4021,7 +4016,7 @@ UniValue sethdseed(const JSONRPCRequest& request) // Do not do anything to non-HD wallets if (!pwallet->CanSupportFeature(FEATURE_HD)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Cannot set a HD seed on a non-HD wallet. Start with -upgradewallet in order to upgrade a non-HD wallet to HD"); + throw JSONRPCError(RPC_WALLET_ERROR, "Cannot set a HD seed on a non-HD wallet. Use the upgradewallet RPC in order to upgrade a non-HD wallet to HD"); } EnsureWalletIsUnlocked(pwallet); @@ -4245,6 +4240,45 @@ UniValue walletcreatefundedpsbt(const JSONRPCRequest& request) return result; } +static UniValue upgradewallet(const JSONRPCRequest& request) +{ + std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); + CWallet* const pwallet = wallet.get(); + + if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) { + return NullUniValue; + } + + RPCHelpMan{"upgradewallet", + "\nUpgrade the wallet. Upgrades to the latest version if no version number is specified\n" + "New keys may be generated and a new wallet backup will need to be made.", + { + {"version", RPCArg::Type::NUM, /* default */ strprintf("%d", FEATURE_LATEST), "The version number to upgrade to. Default is the latest wallet version"} + }, + RPCResults{}, + RPCExamples{ + HelpExampleCli("upgradewallet", "169900") + + HelpExampleRpc("upgradewallet", "169900") + } + }.Check(request); + + RPCTypeCheck(request.params, {UniValue::VNUM}, true); + + EnsureWalletIsUnlocked(pwallet); + + int version = 0; + if (!request.params[0].isNull()) { + version = request.params[0].get_int(); + } + + std::string error; + std::vector<std::string> warnings; + if (!pwallet->UpgradeWallet(version, error, warnings)) { + throw JSONRPCError(RPC_WALLET_ERROR, error); + } + return error; +} + UniValue abortrescan(const JSONRPCRequest& request); // in rpcdump.cpp UniValue dumpprivkey(const JSONRPCRequest& request); // in rpcdump.cpp UniValue importprivkey(const JSONRPCRequest& request); @@ -4255,6 +4289,7 @@ UniValue importwallet(const JSONRPCRequest& request); UniValue importprunedfunds(const JSONRPCRequest& request); UniValue removeprunedfunds(const JSONRPCRequest& request); UniValue importmulti(const JSONRPCRequest& request); +UniValue importdescriptors(const JSONRPCRequest& request); void RegisterWalletRPCCommands(interfaces::Chain& chain, std::vector<std::unique_ptr<interfaces::Handler>>& handlers) { @@ -4268,7 +4303,7 @@ static const CRPCCommand commands[] = { "wallet", "addmultisigaddress", &addmultisigaddress, {"nrequired","keys","label","address_type"} }, { "wallet", "backupwallet", &backupwallet, {"destination"} }, { "wallet", "bumpfee", &bumpfee, {"txid", "options"} }, - { "wallet", "createwallet", &createwallet, {"wallet_name", "disable_private_keys", "blank", "passphrase", "avoid_reuse"} }, + { "wallet", "createwallet", &createwallet, {"wallet_name", "disable_private_keys", "blank", "passphrase", "avoid_reuse", "descriptors"} }, { "wallet", "dumpprivkey", &dumpprivkey, {"address"} }, { "wallet", "dumpwallet", &dumpwallet, {"filename"} }, { "wallet", "encryptwallet", &encryptwallet, {"passphrase"} }, @@ -4284,6 +4319,7 @@ static const CRPCCommand commands[] = { "wallet", "getbalances", &getbalances, {} }, { "wallet", "getwalletinfo", &getwalletinfo, {} }, { "wallet", "importaddress", &importaddress, {"address","label","rescan","p2sh"} }, + { "wallet", "importdescriptors", &importdescriptors, {"requests"} }, { "wallet", "importmulti", &importmulti, {"requests","options"} }, { "wallet", "importprivkey", &importprivkey, {"privkey","label","rescan"} }, { "wallet", "importprunedfunds", &importprunedfunds, {"rawtransaction","txoutproof"} }, @@ -4313,6 +4349,7 @@ static const CRPCCommand commands[] = { "wallet", "signmessage", &signmessage, {"address","message"} }, { "wallet", "signrawtransactionwithwallet", &signrawtransactionwithwallet, {"hexstring","prevtxs","sighashtype"} }, { "wallet", "unloadwallet", &unloadwallet, {"wallet_name"} }, + { "wallet", "upgradewallet", &upgradewallet, {"version"} }, { "wallet", "walletcreatefundedpsbt", &walletcreatefundedpsbt, {"inputs","outputs","locktime","options","bip32derivs"} }, { "wallet", "walletlock", &walletlock, {} }, { "wallet", "walletpassphrase", &walletpassphrase, {"passphrase","timeout"} }, diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index a7a29de9c6..8c149d455b 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index b96cb0aa1a..ecb95d599d 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019 The Bitcoin Core developers +// Copyright (c) 2019-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -20,7 +20,7 @@ bool LegacyScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestinat // Generate a new key that is added to wallet CPubKey new_key; if (!GetKeyFromPool(new_key, type)) { - error = "Error: Keypool ran out, please call keypoolrefill first"; + error = _("Error: Keypool ran out, please call keypoolrefill first").translated; return false; } LearnRelatedScripts(new_key, type); @@ -568,7 +568,7 @@ TransactionError LegacyScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psb return TransactionError::OK; } -const CKeyMetadata* LegacyScriptPubKeyMan::GetMetadata(const CTxDestination& dest) const +std::unique_ptr<CKeyMetadata> LegacyScriptPubKeyMan::GetMetadata(const CTxDestination& dest) const { LOCK(cs_KeyStore); @@ -576,14 +576,14 @@ const CKeyMetadata* LegacyScriptPubKeyMan::GetMetadata(const CTxDestination& des if (!key_id.IsNull()) { auto it = mapKeyMetadata.find(key_id); if (it != mapKeyMetadata.end()) { - return &it->second; + return MakeUnique<CKeyMetadata>(it->second); } } CScript scriptPubKey = GetScriptForDestination(dest); auto it = m_script_metadata.find(CScriptID(scriptPubKey)); if (it != m_script_metadata.end()) { - return &it->second; + return MakeUnique<CKeyMetadata>(it->second); } return nullptr; @@ -1497,3 +1497,668 @@ std::set<CKeyID> LegacyScriptPubKeyMan::GetKeys() const } return set_address; } + +void LegacyScriptPubKeyMan::SetType(OutputType type, bool internal) {} + +bool DescriptorScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) +{ + // Returns true if this descriptor supports getting new addresses. Conditions where we may be unable to fetch them (e.g. locked) are caught later + if (!CanGetAddresses(m_internal)) { + error = "No addresses available"; + return false; + } + { + LOCK(cs_desc_man); + assert(m_wallet_descriptor.descriptor->IsSingleType()); // This is a combo descriptor which should not be an active descriptor + if (type != m_address_type) { + throw std::runtime_error(std::string(__func__) + ": Types are inconsistent"); + } + + TopUp(); + + // Get the scriptPubKey from the descriptor + FlatSigningProvider out_keys; + std::vector<CScript> scripts_temp; + if (m_wallet_descriptor.range_end <= m_max_cached_index && !TopUp(1)) { + // We can't generate anymore keys + error = "Error: Keypool ran out, please call keypoolrefill first"; + return false; + } + if (!m_wallet_descriptor.descriptor->ExpandFromCache(m_wallet_descriptor.next_index, m_wallet_descriptor.cache, scripts_temp, out_keys)) { + // We can't generate anymore keys + error = "Error: Keypool ran out, please call keypoolrefill first"; + return false; + } + + Optional<OutputType> out_script_type = m_wallet_descriptor.descriptor->GetOutputType(); + if (out_script_type && out_script_type == type) { + ExtractDestination(scripts_temp[0], dest); + } else { + throw std::runtime_error(std::string(__func__) + ": Types are inconsistent. Stored type does not match type of newly generated address"); + } + m_wallet_descriptor.next_index++; + WalletBatch(m_storage.GetDatabase()).WriteDescriptor(GetID(), m_wallet_descriptor); + return true; + } +} + +isminetype DescriptorScriptPubKeyMan::IsMine(const CScript& script) const +{ + LOCK(cs_desc_man); + if (m_map_script_pub_keys.count(script) > 0) { + return ISMINE_SPENDABLE; + } + return ISMINE_NO; +} + +bool DescriptorScriptPubKeyMan::CheckDecryptionKey(const CKeyingMaterial& master_key, bool accept_no_keys) +{ + LOCK(cs_desc_man); + if (!m_map_keys.empty()) { + return false; + } + + bool keyPass = m_map_crypted_keys.empty(); // Always pass when there are no encrypted keys + bool keyFail = false; + for (const auto& mi : m_map_crypted_keys) { + const CPubKey &pubkey = mi.second.first; + const std::vector<unsigned char> &crypted_secret = mi.second.second; + CKey key; + if (!DecryptKey(master_key, crypted_secret, pubkey, key)) { + keyFail = true; + break; + } + keyPass = true; + if (m_decryption_thoroughly_checked) + break; + } + if (keyPass && keyFail) { + LogPrintf("The wallet is probably corrupted: Some keys decrypt but not all.\n"); + throw std::runtime_error("Error unlocking wallet: some keys decrypt but not all. Your wallet file may be corrupt."); + } + if (keyFail || (!keyPass && !accept_no_keys)) { + return false; + } + m_decryption_thoroughly_checked = true; + return true; +} + +bool DescriptorScriptPubKeyMan::Encrypt(const CKeyingMaterial& master_key, WalletBatch* batch) +{ + LOCK(cs_desc_man); + if (!m_map_crypted_keys.empty()) { + return false; + } + + for (const KeyMap::value_type& key_in : m_map_keys) + { + const CKey &key = key_in.second; + CPubKey pubkey = key.GetPubKey(); + CKeyingMaterial secret(key.begin(), key.end()); + std::vector<unsigned char> crypted_secret; + if (!EncryptSecret(master_key, secret, pubkey.GetHash(), crypted_secret)) { + return false; + } + m_map_crypted_keys[pubkey.GetID()] = make_pair(pubkey, crypted_secret); + batch->WriteCryptedDescriptorKey(GetID(), pubkey, crypted_secret); + } + m_map_keys.clear(); + return true; +} + +bool DescriptorScriptPubKeyMan::GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool) +{ + LOCK(cs_desc_man); + std::string error; + bool result = GetNewDestination(type, address, error); + index = m_wallet_descriptor.next_index - 1; + return result; +} + +void DescriptorScriptPubKeyMan::ReturnDestination(int64_t index, bool internal, const CTxDestination& addr) +{ + LOCK(cs_desc_man); + // Only return when the index was the most recent + if (m_wallet_descriptor.next_index - 1 == index) { + m_wallet_descriptor.next_index--; + } + WalletBatch(m_storage.GetDatabase()).WriteDescriptor(GetID(), m_wallet_descriptor); + NotifyCanGetAddressesChanged(); +} + +std::map<CKeyID, CKey> DescriptorScriptPubKeyMan::GetKeys() const +{ + AssertLockHeld(cs_desc_man); + if (m_storage.HasEncryptionKeys() && !m_storage.IsLocked()) { + KeyMap keys; + for (auto key_pair : m_map_crypted_keys) { + const CPubKey& pubkey = key_pair.second.first; + const std::vector<unsigned char>& crypted_secret = key_pair.second.second; + CKey key; + DecryptKey(m_storage.GetEncryptionKey(), crypted_secret, pubkey, key); + keys[pubkey.GetID()] = key; + } + return keys; + } + return m_map_keys; +} + +bool DescriptorScriptPubKeyMan::TopUp(unsigned int size) +{ + LOCK(cs_desc_man); + unsigned int target_size; + if (size > 0) { + target_size = size; + } else { + target_size = std::max(gArgs.GetArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 1); + } + + // Calculate the new range_end + int32_t new_range_end = std::max(m_wallet_descriptor.next_index + (int32_t)target_size, m_wallet_descriptor.range_end); + + // If the descriptor is not ranged, we actually just want to fill the first cache item + if (!m_wallet_descriptor.descriptor->IsRange()) { + new_range_end = 1; + m_wallet_descriptor.range_end = 1; + m_wallet_descriptor.range_start = 0; + } + + FlatSigningProvider provider; + provider.keys = GetKeys(); + + WalletBatch batch(m_storage.GetDatabase()); + uint256 id = GetID(); + for (int32_t i = m_max_cached_index + 1; i < new_range_end; ++i) { + FlatSigningProvider out_keys; + std::vector<CScript> scripts_temp; + DescriptorCache temp_cache; + // Maybe we have a cached xpub and we can expand from the cache first + if (!m_wallet_descriptor.descriptor->ExpandFromCache(i, m_wallet_descriptor.cache, scripts_temp, out_keys)) { + if (!m_wallet_descriptor.descriptor->Expand(i, provider, scripts_temp, out_keys, &temp_cache)) return false; + } + // Add all of the scriptPubKeys to the scriptPubKey set + for (const CScript& script : scripts_temp) { + m_map_script_pub_keys[script] = i; + } + for (const auto& pk_pair : out_keys.pubkeys) { + const CPubKey& pubkey = pk_pair.second; + if (m_map_pubkeys.count(pubkey) != 0) { + // We don't need to give an error here. + // It doesn't matter which of many valid indexes the pubkey has, we just need an index where we can derive it and it's private key + continue; + } + m_map_pubkeys[pubkey] = i; + } + // Write the cache + for (const auto& parent_xpub_pair : temp_cache.GetCachedParentExtPubKeys()) { + CExtPubKey xpub; + if (m_wallet_descriptor.cache.GetCachedParentExtPubKey(parent_xpub_pair.first, xpub)) { + if (xpub != parent_xpub_pair.second) { + throw std::runtime_error(std::string(__func__) + ": New cached parent xpub does not match already cached parent xpub"); + } + continue; + } + if (!batch.WriteDescriptorParentCache(parent_xpub_pair.second, id, parent_xpub_pair.first)) { + throw std::runtime_error(std::string(__func__) + ": writing cache item failed"); + } + m_wallet_descriptor.cache.CacheParentExtPubKey(parent_xpub_pair.first, parent_xpub_pair.second); + } + for (const auto& derived_xpub_map_pair : temp_cache.GetCachedDerivedExtPubKeys()) { + for (const auto& derived_xpub_pair : derived_xpub_map_pair.second) { + CExtPubKey xpub; + if (m_wallet_descriptor.cache.GetCachedDerivedExtPubKey(derived_xpub_map_pair.first, derived_xpub_pair.first, xpub)) { + if (xpub != derived_xpub_pair.second) { + throw std::runtime_error(std::string(__func__) + ": New cached derived xpub does not match already cached derived xpub"); + } + continue; + } + if (!batch.WriteDescriptorDerivedCache(derived_xpub_pair.second, id, derived_xpub_map_pair.first, derived_xpub_pair.first)) { + throw std::runtime_error(std::string(__func__) + ": writing cache item failed"); + } + m_wallet_descriptor.cache.CacheDerivedExtPubKey(derived_xpub_map_pair.first, derived_xpub_pair.first, derived_xpub_pair.second); + } + } + m_max_cached_index++; + } + m_wallet_descriptor.range_end = new_range_end; + batch.WriteDescriptor(GetID(), m_wallet_descriptor); + + // By this point, the cache size should be the size of the entire range + assert(m_wallet_descriptor.range_end - 1 == m_max_cached_index); + + NotifyCanGetAddressesChanged(); + return true; +} + +void DescriptorScriptPubKeyMan::MarkUnusedAddresses(const CScript& script) +{ + LOCK(cs_desc_man); + if (IsMine(script)) { + int32_t index = m_map_script_pub_keys[script]; + if (index >= m_wallet_descriptor.next_index) { + WalletLogPrintf("%s: Detected a used keypool item at index %d, mark all keypool items up to this item as used\n", __func__, index); + m_wallet_descriptor.next_index = index + 1; + } + if (!TopUp()) { + WalletLogPrintf("%s: Topping up keypool failed (locked wallet)\n", __func__); + } + } +} + +void DescriptorScriptPubKeyMan::AddDescriptorKey(const CKey& key, const CPubKey &pubkey) +{ + LOCK(cs_desc_man); + WalletBatch batch(m_storage.GetDatabase()); + if (!AddDescriptorKeyWithDB(batch, key, pubkey)) { + throw std::runtime_error(std::string(__func__) + ": writing descriptor private key failed"); + } +} + +bool DescriptorScriptPubKeyMan::AddDescriptorKeyWithDB(WalletBatch& batch, const CKey& key, const CPubKey &pubkey) +{ + AssertLockHeld(cs_desc_man); + assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); + + if (m_storage.HasEncryptionKeys()) { + if (m_storage.IsLocked()) { + return false; + } + + std::vector<unsigned char> crypted_secret; + CKeyingMaterial secret(key.begin(), key.end()); + if (!EncryptSecret(m_storage.GetEncryptionKey(), secret, pubkey.GetHash(), crypted_secret)) { + return false; + } + + m_map_crypted_keys[pubkey.GetID()] = make_pair(pubkey, crypted_secret); + return batch.WriteCryptedDescriptorKey(GetID(), pubkey, crypted_secret); + } else { + m_map_keys[pubkey.GetID()] = key; + return batch.WriteDescriptorKey(GetID(), pubkey, key.GetPrivKey()); + } +} + +bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_key) +{ + LOCK(cs_desc_man); + assert(m_storage.IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); + + // Ignore when there is already a descriptor + if (m_wallet_descriptor.descriptor) { + return false; + } + + int64_t creation_time = GetTime(); + + std::string xpub = EncodeExtPubKey(master_key.Neuter()); + + // Build descriptor string + std::string desc_prefix; + std::string desc_suffix = "/*)"; + switch (m_address_type) { + case OutputType::LEGACY: { + desc_prefix = "pkh(" + xpub + "/44'"; + break; + } + case OutputType::P2SH_SEGWIT: { + desc_prefix = "sh(wpkh(" + xpub + "/49'"; + desc_suffix += ")"; + break; + } + case OutputType::BECH32: { + desc_prefix = "wpkh(" + xpub + "/84'"; + break; + } + default: assert(false); + } + + // Mainnet derives at 0', testnet and regtest derive at 1' + if (Params().IsTestChain()) { + desc_prefix += "/1'"; + } else { + desc_prefix += "/0'"; + } + + std::string internal_path = m_internal ? "/1" : "/0"; + std::string desc_str = desc_prefix + "/0'" + internal_path + desc_suffix; + + // Make the descriptor + FlatSigningProvider keys; + std::string error; + std::unique_ptr<Descriptor> desc = Parse(desc_str, keys, error, false); + WalletDescriptor w_desc(std::move(desc), creation_time, 0, 0, 0); + m_wallet_descriptor = w_desc; + + // Store the master private key, and descriptor + WalletBatch batch(m_storage.GetDatabase()); + if (!AddDescriptorKeyWithDB(batch, master_key.key, master_key.key.GetPubKey())) { + throw std::runtime_error(std::string(__func__) + ": writing descriptor master private key failed"); + } + if (!batch.WriteDescriptor(GetID(), m_wallet_descriptor)) { + throw std::runtime_error(std::string(__func__) + ": writing descriptor failed"); + } + + // TopUp + TopUp(); + + m_storage.UnsetBlankWalletFlag(batch); + return true; +} + +bool DescriptorScriptPubKeyMan::IsHDEnabled() const +{ + LOCK(cs_desc_man); + return m_wallet_descriptor.descriptor->IsRange(); +} + +bool DescriptorScriptPubKeyMan::CanGetAddresses(bool internal) const +{ + // We can only give out addresses from descriptors that are single type (not combo), ranged, + // and either have cached keys or can generate more keys (ignoring encryption) + LOCK(cs_desc_man); + return m_wallet_descriptor.descriptor->IsSingleType() && + m_wallet_descriptor.descriptor->IsRange() && + (HavePrivateKeys() || m_wallet_descriptor.next_index < m_wallet_descriptor.range_end); +} + +bool DescriptorScriptPubKeyMan::HavePrivateKeys() const +{ + LOCK(cs_desc_man); + return m_map_keys.size() > 0 || m_map_crypted_keys.size() > 0; +} + +int64_t DescriptorScriptPubKeyMan::GetOldestKeyPoolTime() const +{ + // This is only used for getwalletinfo output and isn't relevant to descriptor wallets. + // The magic number 0 indicates that it shouldn't be displayed so that's what we return. + return 0; +} + +size_t DescriptorScriptPubKeyMan::KeypoolCountExternalKeys() const +{ + if (m_internal) { + return 0; + } + return GetKeyPoolSize(); +} + +unsigned int DescriptorScriptPubKeyMan::GetKeyPoolSize() const +{ + LOCK(cs_desc_man); + return m_wallet_descriptor.range_end - m_wallet_descriptor.next_index; +} + +int64_t DescriptorScriptPubKeyMan::GetTimeFirstKey() const +{ + LOCK(cs_desc_man); + return m_wallet_descriptor.creation_time; +} + +std::unique_ptr<FlatSigningProvider> DescriptorScriptPubKeyMan::GetSigningProvider(const CScript& script, bool include_private) const +{ + LOCK(cs_desc_man); + + // Find the index of the script + auto it = m_map_script_pub_keys.find(script); + if (it == m_map_script_pub_keys.end()) { + return nullptr; + } + int32_t index = it->second; + + return GetSigningProvider(index, include_private); +} + +std::unique_ptr<FlatSigningProvider> DescriptorScriptPubKeyMan::GetSigningProvider(const CPubKey& pubkey) const +{ + LOCK(cs_desc_man); + + // Find index of the pubkey + auto it = m_map_pubkeys.find(pubkey); + if (it == m_map_pubkeys.end()) { + return nullptr; + } + int32_t index = it->second; + + // Always try to get the signing provider with private keys. This function should only be called during signing anyways + return GetSigningProvider(index, true); +} + +std::unique_ptr<FlatSigningProvider> DescriptorScriptPubKeyMan::GetSigningProvider(int32_t index, bool include_private) const +{ + AssertLockHeld(cs_desc_man); + // Get the scripts, keys, and key origins for this script + std::unique_ptr<FlatSigningProvider> out_keys = MakeUnique<FlatSigningProvider>(); + std::vector<CScript> scripts_temp; + if (!m_wallet_descriptor.descriptor->ExpandFromCache(index, m_wallet_descriptor.cache, scripts_temp, *out_keys)) return nullptr; + + if (HavePrivateKeys() && include_private) { + FlatSigningProvider master_provider; + master_provider.keys = GetKeys(); + m_wallet_descriptor.descriptor->ExpandPrivate(index, master_provider, *out_keys); + } + + return out_keys; +} + +std::unique_ptr<SigningProvider> DescriptorScriptPubKeyMan::GetSolvingProvider(const CScript& script) const +{ + return GetSigningProvider(script, false); +} + +bool DescriptorScriptPubKeyMan::CanProvide(const CScript& script, SignatureData& sigdata) +{ + return IsMine(script); +} + +bool DescriptorScriptPubKeyMan::SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors) const +{ + std::unique_ptr<FlatSigningProvider> keys = MakeUnique<FlatSigningProvider>(); + for (const auto& coin_pair : coins) { + std::unique_ptr<FlatSigningProvider> coin_keys = GetSigningProvider(coin_pair.second.out.scriptPubKey, true); + if (!coin_keys) { + continue; + } + *keys = Merge(*keys, *coin_keys); + } + + return ::SignTransaction(tx, keys.get(), coins, sighash, input_errors); +} + +SigningResult DescriptorScriptPubKeyMan::SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const +{ + std::unique_ptr<FlatSigningProvider> keys = GetSigningProvider(GetScriptForDestination(pkhash), true); + if (!keys) { + return SigningResult::PRIVATE_KEY_NOT_AVAILABLE; + } + + CKeyID key_id(pkhash); + CKey key; + if (!keys->GetKey(key_id, key)) { + return SigningResult::PRIVATE_KEY_NOT_AVAILABLE; + } + + if (!MessageSign(key, message, str_sig)) { + return SigningResult::SIGNING_FAILED; + } + return SigningResult::OK; +} + +TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbtx, int sighash_type, bool sign, bool bip32derivs) const +{ + for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { + const CTxIn& txin = psbtx.tx->vin[i]; + PSBTInput& input = psbtx.inputs.at(i); + + if (PSBTInputSigned(input)) { + continue; + } + + // Verify input looks sane. This will check that we have at most one uxto, witness or non-witness. + if (!input.IsSane()) { + return TransactionError::INVALID_PSBT; + } + + // Get the Sighash type + if (sign && input.sighash_type > 0 && input.sighash_type != sighash_type) { + return TransactionError::SIGHASH_MISMATCH; + } + + // Get the scriptPubKey to know which SigningProvider to use + CScript script; + if (!input.witness_utxo.IsNull()) { + script = input.witness_utxo.scriptPubKey; + } else if (input.non_witness_utxo) { + if (txin.prevout.n >= input.non_witness_utxo->vout.size()) { + return TransactionError::MISSING_INPUTS; + } + script = input.non_witness_utxo->vout[txin.prevout.n].scriptPubKey; + } else { + // There's no UTXO so we can just skip this now + continue; + } + SignatureData sigdata; + input.FillSignatureData(sigdata); + + std::unique_ptr<FlatSigningProvider> keys = MakeUnique<FlatSigningProvider>(); + std::unique_ptr<FlatSigningProvider> script_keys = GetSigningProvider(script, sign); + if (script_keys) { + *keys = Merge(*keys, *script_keys); + } else { + // Maybe there are pubkeys listed that we can sign for + script_keys = MakeUnique<FlatSigningProvider>(); + for (const auto& pk_pair : input.hd_keypaths) { + const CPubKey& pubkey = pk_pair.first; + std::unique_ptr<FlatSigningProvider> pk_keys = GetSigningProvider(pubkey); + if (pk_keys) { + *keys = Merge(*keys, *pk_keys); + } + } + } + + SignPSBTInput(HidingSigningProvider(keys.get(), !sign, !bip32derivs), psbtx, i, sighash_type); + } + + // Fill in the bip32 keypaths and redeemscripts for the outputs so that hardware wallets can identify change + for (unsigned int i = 0; i < psbtx.tx->vout.size(); ++i) { + std::unique_ptr<SigningProvider> keys = GetSolvingProvider(psbtx.tx->vout.at(i).scriptPubKey); + if (!keys) { + continue; + } + UpdatePSBTOutput(HidingSigningProvider(keys.get(), true, !bip32derivs), psbtx, i); + } + + return TransactionError::OK; +} + +std::unique_ptr<CKeyMetadata> DescriptorScriptPubKeyMan::GetMetadata(const CTxDestination& dest) const +{ + std::unique_ptr<SigningProvider> provider = GetSigningProvider(GetScriptForDestination(dest)); + if (provider) { + KeyOriginInfo orig; + CKeyID key_id = GetKeyForDestination(*provider, dest); + if (provider->GetKeyOrigin(key_id, orig)) { + LOCK(cs_desc_man); + std::unique_ptr<CKeyMetadata> meta = MakeUnique<CKeyMetadata>(); + meta->key_origin = orig; + meta->has_key_origin = true; + meta->nCreateTime = m_wallet_descriptor.creation_time; + return meta; + } + } + return nullptr; +} + +uint256 DescriptorScriptPubKeyMan::GetID() const +{ + LOCK(cs_desc_man); + std::string desc_str = m_wallet_descriptor.descriptor->ToString(); + uint256 id; + CSHA256().Write((unsigned char*)desc_str.data(), desc_str.size()).Finalize(id.begin()); + return id; +} + +void DescriptorScriptPubKeyMan::SetType(OutputType type, bool internal) +{ + this->m_address_type = type; + this->m_internal = internal; +} + +void DescriptorScriptPubKeyMan::SetCache(const DescriptorCache& cache) +{ + LOCK(cs_desc_man); + m_wallet_descriptor.cache = cache; + for (int32_t i = m_wallet_descriptor.range_start; i < m_wallet_descriptor.range_end; ++i) { + FlatSigningProvider out_keys; + std::vector<CScript> scripts_temp; + if (!m_wallet_descriptor.descriptor->ExpandFromCache(i, m_wallet_descriptor.cache, scripts_temp, out_keys)) { + throw std::runtime_error("Error: Unable to expand wallet descriptor from cache"); + } + // Add all of the scriptPubKeys to the scriptPubKey set + for (const CScript& script : scripts_temp) { + if (m_map_script_pub_keys.count(script) != 0) { + throw std::runtime_error(strprintf("Error: Already loaded script at index %d as being at index %d", i, m_map_script_pub_keys[script])); + } + m_map_script_pub_keys[script] = i; + } + for (const auto& pk_pair : out_keys.pubkeys) { + const CPubKey& pubkey = pk_pair.second; + if (m_map_pubkeys.count(pubkey) != 0) { + // We don't need to give an error here. + // It doesn't matter which of many valid indexes the pubkey has, we just need an index where we can derive it and it's private key + continue; + } + m_map_pubkeys[pubkey] = i; + } + m_max_cached_index++; + } +} + +bool DescriptorScriptPubKeyMan::AddKey(const CKeyID& key_id, const CKey& key) +{ + LOCK(cs_desc_man); + m_map_keys[key_id] = key; + return true; +} + +bool DescriptorScriptPubKeyMan::AddCryptedKey(const CKeyID& key_id, const CPubKey& pubkey, const std::vector<unsigned char>& crypted_key) +{ + LOCK(cs_desc_man); + if (!m_map_keys.empty()) { + return false; + } + + m_map_crypted_keys[key_id] = make_pair(pubkey, crypted_key); + return true; +} + +bool DescriptorScriptPubKeyMan::HasWalletDescriptor(const WalletDescriptor& desc) const +{ + LOCK(cs_desc_man); + return m_wallet_descriptor.descriptor != nullptr && desc.descriptor != nullptr && m_wallet_descriptor.descriptor->ToString() == desc.descriptor->ToString(); +} + +void DescriptorScriptPubKeyMan::WriteDescriptor() +{ + LOCK(cs_desc_man); + WalletBatch batch(m_storage.GetDatabase()); + if (!batch.WriteDescriptor(GetID(), m_wallet_descriptor)) { + throw std::runtime_error(std::string(__func__) + ": writing descriptor failed"); + } +} + +const WalletDescriptor DescriptorScriptPubKeyMan::GetWalletDescriptor() const +{ + return m_wallet_descriptor; +} + +const std::vector<CScript> DescriptorScriptPubKeyMan::GetScriptPubKeys() const +{ + LOCK(cs_desc_man); + std::vector<CScript> script_pub_keys; + script_pub_keys.reserve(m_map_script_pub_keys.size()); + + for (auto const& script_pub_key: m_map_script_pub_keys) { + script_pub_keys.push_back(script_pub_key.first); + } + return script_pub_keys; +} diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 8512eadf31..3117b13d35 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -6,6 +6,7 @@ #define BITCOIN_WALLET_SCRIPTPUBKEYMAN_H #include <psbt.h> +#include <script/descriptor.h> #include <script/signingprovider.h> #include <script/standard.h> #include <util/error.h> @@ -204,7 +205,7 @@ public: virtual int64_t GetTimeFirstKey() const { return 0; } - virtual const CKeyMetadata* GetMetadata(const CTxDestination& dest) const { return nullptr; } + virtual std::unique_ptr<CKeyMetadata> GetMetadata(const CTxDestination& dest) const { return nullptr; } virtual std::unique_ptr<SigningProvider> GetSolvingProvider(const CScript& script) const { return nullptr; } @@ -222,6 +223,8 @@ public: virtual uint256 GetID() const { return uint256(); } + virtual void SetType(OutputType type, bool internal) {} + /** Prepends the wallet name in logging output to ease debugging in multi-wallet use cases */ template<typename... Params> void WalletLogPrintf(std::string fmt, Params... parameters) const { @@ -352,7 +355,7 @@ public: int64_t GetTimeFirstKey() const override; - const CKeyMetadata* GetMetadata(const CTxDestination& dest) const override; + std::unique_ptr<CKeyMetadata> GetMetadata(const CTxDestination& dest) const override; bool CanGetAddresses(bool internal = false) const override; @@ -366,6 +369,8 @@ public: uint256 GetID() const override; + void SetType(OutputType type, bool internal) override; + // Map from Key ID to key metadata. std::map<CKeyID, CKeyMetadata> mapKeyMetadata GUARDED_BY(cs_KeyStore); @@ -477,4 +482,111 @@ public: bool GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const override { return m_spk_man.GetKeyOrigin(keyid, info); } }; +class DescriptorScriptPubKeyMan : public ScriptPubKeyMan +{ +private: + WalletDescriptor m_wallet_descriptor GUARDED_BY(cs_desc_man); + + using ScriptPubKeyMap = std::map<CScript, int32_t>; // Map of scripts to descriptor range index + using PubKeyMap = std::map<CPubKey, int32_t>; // Map of pubkeys involved in scripts to descriptor range index + using CryptedKeyMap = std::map<CKeyID, std::pair<CPubKey, std::vector<unsigned char>>>; + using KeyMap = std::map<CKeyID, CKey>; + + ScriptPubKeyMap m_map_script_pub_keys GUARDED_BY(cs_desc_man); + PubKeyMap m_map_pubkeys GUARDED_BY(cs_desc_man); + int32_t m_max_cached_index = -1; + + OutputType m_address_type; + bool m_internal; + + KeyMap m_map_keys GUARDED_BY(cs_desc_man); + CryptedKeyMap m_map_crypted_keys GUARDED_BY(cs_desc_man); + + bool SetCrypted(); + + //! keeps track of whether Unlock has run a thorough check before + bool m_decryption_thoroughly_checked = false; + + bool AddDescriptorKeyWithDB(WalletBatch& batch, const CKey& key, const CPubKey &pubkey); + + KeyMap GetKeys() const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); + + // Fetch the SigningProvider for the given script and optionally include private keys + std::unique_ptr<FlatSigningProvider> GetSigningProvider(const CScript& script, bool include_private = false) const; + // Fetch the SigningProvider for the given pubkey and always include private keys. This should only be called by signing code. + std::unique_ptr<FlatSigningProvider> GetSigningProvider(const CPubKey& pubkey) const; + // Fetch the SigningProvider for a given index and optionally include private keys. Called by the above functions. + std::unique_ptr<FlatSigningProvider> GetSigningProvider(int32_t index, bool include_private = false) const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); + +public: + DescriptorScriptPubKeyMan(WalletStorage& storage, WalletDescriptor& descriptor) + : ScriptPubKeyMan(storage), + m_wallet_descriptor(descriptor) + {} + DescriptorScriptPubKeyMan(WalletStorage& storage, OutputType address_type, bool internal) + : ScriptPubKeyMan(storage), + m_address_type(address_type), m_internal(internal) + {} + + mutable RecursiveMutex cs_desc_man; + + bool GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) override; + isminetype IsMine(const CScript& script) const override; + + bool CheckDecryptionKey(const CKeyingMaterial& master_key, bool accept_no_keys = false) override; + bool Encrypt(const CKeyingMaterial& master_key, WalletBatch* batch) override; + + bool GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool) override; + void ReturnDestination(int64_t index, bool internal, const CTxDestination& addr) override; + + // Tops up the descriptor cache and m_map_script_pub_keys. The cache is stored in the wallet file + // and is used to expand the descriptor in GetNewDestination. DescriptorScriptPubKeyMan relies + // more on ephemeral data than LegacyScriptPubKeyMan. For wallets using unhardened derivation + // (with or without private keys), the "keypool" is a single xpub. + bool TopUp(unsigned int size = 0) override; + + void MarkUnusedAddresses(const CScript& script) override; + + bool IsHDEnabled() const override; + + //! Setup descriptors based on the given CExtkey + bool SetupDescriptorGeneration(const CExtKey& master_key); + + bool HavePrivateKeys() const override; + + int64_t GetOldestKeyPoolTime() const override; + size_t KeypoolCountExternalKeys() const override; + unsigned int GetKeyPoolSize() const override; + + int64_t GetTimeFirstKey() const override; + + std::unique_ptr<CKeyMetadata> GetMetadata(const CTxDestination& dest) const override; + + bool CanGetAddresses(bool internal = false) const override; + + std::unique_ptr<SigningProvider> GetSolvingProvider(const CScript& script) const override; + + bool CanProvide(const CScript& script, SignatureData& sigdata) override; + + bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors) const override; + SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const override; + TransactionError FillPSBT(PartiallySignedTransaction& psbt, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false) const override; + + uint256 GetID() const override; + + void SetType(OutputType type, bool internal) override; + + void SetCache(const DescriptorCache& cache); + + bool AddKey(const CKeyID& key_id, const CKey& key); + bool AddCryptedKey(const CKeyID& key_id, const CPubKey& pubkey, const std::vector<unsigned char>& crypted_key); + + bool HasWalletDescriptor(const WalletDescriptor& desc) const; + void AddDescriptorKey(const CKey& key, const CPubKey &pubkey); + void WriteDescriptor(); + + const WalletDescriptor GetWalletDescriptor() const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); + const std::vector<CScript> GetScriptPubKeys() const; +}; + #endif // BITCOIN_WALLET_SCRIPTPUBKEYMAN_H diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 21d57cb898..66f4542cf9 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -1,16 +1,16 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <node/context.h> -#include <wallet/wallet.h> -#include <wallet/coinselection.h> -#include <wallet/coincontrol.h> #include <amount.h> +#include <node/context.h> #include <primitives/transaction.h> #include <random.h> #include <test/util/setup_common.h> +#include <wallet/coincontrol.h> +#include <wallet/coinselection.h> #include <wallet/test/wallet_test_fixture.h> +#include <wallet/wallet.h> #include <boost/test/unit_test.hpp> #include <random> @@ -176,8 +176,8 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) selection.clear(); // Select 5 Cent - add_coin(3 * CENT, 3, actual_selection); - add_coin(2 * CENT, 2, actual_selection); + add_coin(4 * CENT, 4, actual_selection); + add_coin(1 * CENT, 1, actual_selection); BOOST_CHECK(SelectCoinsBnB(GroupCoins(utxo_pool), 5 * CENT, 0.5 * CENT, selection, value_ret, not_input_fees)); BOOST_CHECK(equal_sets(selection, actual_selection)); BOOST_CHECK_EQUAL(value_ret, 5 * CENT); @@ -204,9 +204,8 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) // Select 10 Cent add_coin(5 * CENT, 5, utxo_pool); + add_coin(5 * CENT, 5, actual_selection); add_coin(4 * CENT, 4, actual_selection); - add_coin(3 * CENT, 3, actual_selection); - add_coin(2 * CENT, 2, actual_selection); add_coin(1 * CENT, 1, actual_selection); BOOST_CHECK(SelectCoinsBnB(GroupCoins(utxo_pool), 10 * CENT, 0.5 * CENT, selection, value_ret, not_input_fees)); BOOST_CHECK(equal_sets(selection, actual_selection)); diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 8b7b7af21d..35860577cd 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2019 The Bitcoin Core developers +// Copyright (c) 2017-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index b9e714946d..7ba3148ff3 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index 81d8a60b8a..a294935b64 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index a487e9e2e0..2bdeb89f3c 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2019 The Bitcoin Core developers +// Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -46,7 +46,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) auto locked_chain = chain->lock(); LockAssertion lock(::cs_main); - // Verify ScanForWalletTransactions accommodates a null start block. + // Verify ScanForWalletTransactions fails to read an unknown start block. { CWallet wallet(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); { @@ -54,10 +54,10 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); - WalletRescanReserver reserver(&wallet); + WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions({} /* start_block */, {} /* stop_block */, reserver, false /* update */); - BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); + CWallet::ScanResult result = wallet.ScanForWalletTransactions({} /* start_block */, 0 /* start_height */, {} /* max_height */, reserver, false /* update */); + BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); BOOST_CHECK(result.last_failed_block.IsNull()); BOOST_CHECK(result.last_scanned_block.IsNull()); BOOST_CHECK(!result.last_scanned_height); @@ -73,9 +73,9 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); - WalletRescanReserver reserver(&wallet); + WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), {} /* stop_block */, reserver, false /* update */); + CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); BOOST_CHECK(result.last_failed_block.IsNull()); BOOST_CHECK_EQUAL(result.last_scanned_block, newTip->GetBlockHash()); @@ -96,9 +96,9 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); - WalletRescanReserver reserver(&wallet); + WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), {} /* stop_block */, reserver, false /* update */); + CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); BOOST_CHECK_EQUAL(result.last_failed_block, oldTip->GetBlockHash()); BOOST_CHECK_EQUAL(result.last_scanned_block, newTip->GetBlockHash()); @@ -118,9 +118,9 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) wallet.SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); - WalletRescanReserver reserver(&wallet); + WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), {} /* stop_block */, reserver, false /* update */); + CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); BOOST_CHECK_EQUAL(result.last_failed_block, newTip->GetBlockHash()); BOOST_CHECK(result.last_scanned_block.IsNull()); @@ -152,6 +152,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) { std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); wallet->SetupLegacyScriptPubKeyMan(); + WITH_LOCK(wallet->cs_wallet, wallet->SetLastBlockProcessed(newTip->nHeight, newTip->GetBlockHash())); AddWallet(wallet); UniValue keys; keys.setArray(); @@ -216,15 +217,19 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // Import key into wallet and call dumpwallet to create backup file. { std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); - auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); - LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); - spk_man->mapKeyMetadata[coinbaseKey.GetPubKey().GetID()].nCreateTime = KEY_TIME; - spk_man->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); + { + auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); + LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); + spk_man->mapKeyMetadata[coinbaseKey.GetPubKey().GetID()].nCreateTime = KEY_TIME; + spk_man->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); + AddWallet(wallet); + wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); + } JSONRPCRequest request; request.params.setArray(); request.params.push_back(backup_file); - AddWallet(wallet); + ::dumpwallet(request); RemoveWallet(wallet); } @@ -233,16 +238,17 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // were scanned, and no prior blocks were scanned. { std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(chain.get(), WalletLocation(), WalletDatabase::CreateDummy()); + LOCK(wallet->cs_wallet); wallet->SetupLegacyScriptPubKeyMan(); JSONRPCRequest request; request.params.setArray(); request.params.push_back(backup_file); AddWallet(wallet); + wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); ::importwallet(request); RemoveWallet(wallet); - LOCK(wallet->cs_wallet); BOOST_CHECK_EQUAL(wallet->mapWallet.size(), 3U); BOOST_CHECK_EQUAL(m_coinbase_txns.size(), 103U); for (size_t i = 0; i < m_coinbase_txns.size(); ++i) { @@ -454,15 +460,15 @@ public: CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), WalletDatabase::CreateMock()); { - LOCK(wallet->cs_wallet); + LOCK2(::cs_main, wallet->cs_wallet); wallet->SetLastBlockProcessed(::ChainActive().Height(), ::ChainActive().Tip()->GetBlockHash()); } bool firstRun; wallet->LoadWallet(firstRun); AddKey(*wallet, coinbaseKey); - WalletRescanReserver reserver(wallet.get()); + WalletRescanReserver reserver(*wallet); reserver.reserve(); - CWallet::ScanResult result = wallet->ScanForWalletTransactions(::ChainActive().Genesis()->GetBlockHash(), {} /* stop_block */, reserver, false /* update */); + CWallet::ScanResult result = wallet->ScanForWalletTransactions(::ChainActive().Genesis()->GetBlockHash(), 0 /* start_height */, {} /* max_height */, reserver, false /* update */); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); BOOST_CHECK_EQUAL(result.last_scanned_block, ::ChainActive().Tip()->GetBlockHash()); BOOST_CHECK_EQUAL(*result.last_scanned_height, ::ChainActive().Height()); @@ -631,4 +637,25 @@ BOOST_FIXTURE_TEST_CASE(dummy_input_size_test, TestChain100Setup) BOOST_CHECK_EQUAL(CalculateNestedKeyhashInputSize(true), DUMMY_NESTED_P2WPKH_INPUT_SIZE); } +bool malformed_descriptor(std::ios_base::failure e) +{ + std::string s(e.what()); + return s.find("Missing checksum") != std::string::npos; +} + +BOOST_FIXTURE_TEST_CASE(wallet_descriptor_test, BasicTestingSetup) +{ + std::vector<unsigned char> malformed_record; + CVectorWriter vw(0, 0, malformed_record, 0); + vw << std::string("notadescriptor"); + vw << (uint64_t)0; + vw << (int32_t)0; + vw << (int32_t)0; + vw << (int32_t)1; + + VectorReader vr(0, 0, malformed_record, 0); + WalletDescriptor w_desc; + BOOST_CHECK_EXCEPTION(vr >> w_desc, std::ios_base::failure, malformed_descriptor); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 45f5542cad..b6f25de64e 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -22,6 +22,7 @@ #include <script/script.h> #include <script/signingprovider.h> #include <util/bip32.h> +#include <util/check.h> #include <util/error.h> #include <util/fees.h> #include <util/moneystr.h> @@ -35,6 +36,8 @@ #include <boost/algorithm/string/replace.hpp> +using interfaces::FoundBlock; + const std::map<uint64_t,std::string> WALLET_FLAG_CAVEATS{ {WALLET_FLAG_AVOID_REUSE, "You need to rescan the blockchain in order to correctly mark used " @@ -225,10 +228,14 @@ WalletCreationStatus CreateWallet(interfaces::Chain& chain, const SecureString& // Set a seed for the wallet { LOCK(wallet->cs_wallet); - for (auto spk_man : wallet->GetActiveScriptPubKeyMans()) { - if (!spk_man->SetupGeneration()) { - error = "Unable to generate initial keys"; - return WalletCreationStatus::CREATION_FAILED; + if (wallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + wallet->SetupDescriptorScriptPubKeyMans(); + } else { + for (auto spk_man : wallet->GetActiveScriptPubKeyMans()) { + if (!spk_man->SetupGeneration()) { + error = "Unable to generate initial keys"; + return WalletCreationStatus::CREATION_FAILED; + } } } } @@ -585,8 +592,11 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) Lock(); Unlock(strWalletPassphrase); - // if we are using HD, replace the HD seed with a new one - if (auto spk_man = GetLegacyScriptPubKeyMan()) { + // If we are using descriptors, make new descriptors with a new seed + if (IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS) && !IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET)) { + SetupDescriptorScriptPubKeyMans(); + } else if (auto spk_man = GetLegacyScriptPubKeyMan()) { + // if we are using HD, replace the HD seed with a new one if (spk_man->IsHDEnabled()) { if (!spk_man->SetupGeneration(true)) { return false; @@ -744,22 +754,29 @@ bool CWallet::IsSpentKey(const uint256& hash, unsigned int n) const const CWalletTx* srctx = GetWalletTx(hash); if (srctx) { assert(srctx->tx->vout.size() > n); - LegacyScriptPubKeyMan* spk_man = GetLegacyScriptPubKeyMan(); - // When descriptor wallets arrive, these additional checks are - // likely superfluous and can be optimized out - assert(spk_man != nullptr); - for (const auto& keyid : GetAffectedKeys(srctx->tx->vout[n].scriptPubKey, *spk_man)) { - WitnessV0KeyHash wpkh_dest(keyid); - if (GetDestData(wpkh_dest, "used", nullptr)) { - return true; - } - ScriptHash sh_wpkh_dest(GetScriptForDestination(wpkh_dest)); - if (GetDestData(sh_wpkh_dest, "used", nullptr)) { - return true; - } - PKHash pkh_dest(keyid); - if (GetDestData(pkh_dest, "used", nullptr)) { - return true; + CTxDestination dest; + if (!ExtractDestination(srctx->tx->vout[n].scriptPubKey, dest)) { + return false; + } + if (GetDestData(dest, "used", nullptr)) { + return true; + } + if (IsLegacy()) { + LegacyScriptPubKeyMan* spk_man = GetLegacyScriptPubKeyMan(); + assert(spk_man != nullptr); + for (const auto& keyid : GetAffectedKeys(srctx->tx->vout[n].scriptPubKey, *spk_man)) { + WitnessV0KeyHash wpkh_dest(keyid); + if (GetDestData(wpkh_dest, "used", nullptr)) { + return true; + } + ScriptHash sh_wpkh_dest(GetScriptForDestination(wpkh_dest)); + if (GetDestData(sh_wpkh_dest, "used", nullptr)) { + return true; + } + PKHash pkh_dest(keyid); + if (GetDestData(pkh_dest, "used", nullptr)) { + return true; + } } } } @@ -1328,9 +1345,10 @@ CAmount CWallet::GetChange(const CTransaction& tx) const bool CWallet::IsHDEnabled() const { + // All Active ScriptPubKeyMans must be HD for this to be true bool result = true; - for (const auto& spk_man_pair : m_spk_managers) { - result &= spk_man_pair.second->IsHDEnabled(); + for (const auto& spk_man : GetActiveScriptPubKeyMans()) { + result &= spk_man->IsHDEnabled(); } return result; } @@ -1594,22 +1612,17 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r // Find starting block. May be null if nCreateTime is greater than the // highest blockchain timestamp, in which case there is nothing that needs // to be scanned. + int start_height = 0; uint256 start_block; - { - auto locked_chain = chain().lock(); - const Optional<int> start_height = locked_chain->findFirstBlockWithTimeAndHeight(startTime - TIMESTAMP_WINDOW, 0, &start_block); - const Optional<int> tip_height = locked_chain->getHeight(); - WalletLogPrintf("%s: Rescanning last %i blocks\n", __func__, tip_height && start_height ? *tip_height - *start_height + 1 : 0); - } + bool start = chain().findFirstBlockWithTimeAndHeight(startTime - TIMESTAMP_WINDOW, 0, FoundBlock().hash(start_block).height(start_height)); + WalletLogPrintf("%s: Rescanning last %i blocks\n", __func__, start ? WITH_LOCK(cs_wallet, return GetLastBlockHeight()) - start_height + 1 : 0); - if (!start_block.IsNull()) { + if (start) { // TODO: this should take into account failure by ScanResult::USER_ABORT - ScanResult result = ScanForWalletTransactions(start_block, {} /* stop_block */, reserver, update); + ScanResult result = ScanForWalletTransactions(start_block, start_height, {} /* max_height */, reserver, update); if (result.status == ScanResult::FAILURE) { int64_t time_max; - if (!chain().findBlock(result.last_failed_block, nullptr /* block */, nullptr /* time */, &time_max)) { - throw std::logic_error("ScanForWalletTransactions returned invalid block hash"); - } + CHECK_NONFATAL(chain().findBlock(result.last_failed_block, FoundBlock().maxTime(time_max))); return time_max + TIMESTAMP_WINDOW + 1; } } @@ -1623,9 +1636,9 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r * * @param[in] start_block Scan starting block. If block is not on the active * chain, the scan will return SUCCESS immediately. - * @param[in] stop_block Scan ending block. If block is not on the active - * chain, the scan will continue until it reaches the - * chain tip. + * @param[in] start_height Height of start_block + * @param[in] max_height Optional max scanning height. If unset there is + * no maximum and scanning can continue to the tip * * @return ScanResult returning scan information and indicating success or * failure. Return status will be set to SUCCESS if scan was @@ -1637,7 +1650,7 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r * the main chain after to the addition of any new keys you want to detect * transactions for. */ -CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_block, const uint256& stop_block, const WalletRescanReserver& reserver, bool fUpdate) +CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_block, int start_height, Optional<int> max_height, const WalletRescanReserver& reserver, bool fUpdate) { int64_t nNow = GetTime(); int64_t start_time = GetTimeMillis(); @@ -1651,36 +1664,32 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc fAbortRescan = false; ShowProgress(strprintf("%s " + _("Rescanning...").translated, GetDisplayName()), 0); // show rescan progress in GUI as dialog or on splashscreen, if -rescan on startup - uint256 tip_hash; - // The way the 'block_height' is initialized is just a workaround for the gcc bug #47679 since version 4.6.0. - Optional<int> block_height = MakeOptional(false, int()); - double progress_begin; - double progress_end; - { - auto locked_chain = chain().lock(); - if (Optional<int> tip_height = locked_chain->getHeight()) { - tip_hash = locked_chain->getBlockHash(*tip_height); - } - block_height = locked_chain->getBlockHeight(block_hash); - progress_begin = chain().guessVerificationProgress(block_hash); - progress_end = chain().guessVerificationProgress(stop_block.IsNull() ? tip_hash : stop_block); - } + uint256 tip_hash = WITH_LOCK(cs_wallet, return GetLastBlockHash()); + uint256 end_hash = tip_hash; + if (max_height) chain().findAncestorByHeight(tip_hash, *max_height, FoundBlock().hash(end_hash)); + double progress_begin = chain().guessVerificationProgress(block_hash); + double progress_end = chain().guessVerificationProgress(end_hash); double progress_current = progress_begin; - while (block_height && !fAbortRescan && !chain().shutdownRequested()) { + int block_height = start_height; + while (!fAbortRescan && !chain().shutdownRequested()) { m_scanning_progress = (progress_current - progress_begin) / (progress_end - progress_begin); - if (*block_height % 100 == 0 && progress_end - progress_begin > 0.0) { + if (block_height % 100 == 0 && progress_end - progress_begin > 0.0) { ShowProgress(strprintf("%s " + _("Rescanning...").translated, GetDisplayName()), std::max(1, std::min(99, (int)(m_scanning_progress * 100)))); } if (GetTime() >= nNow + 60) { nNow = GetTime(); - WalletLogPrintf("Still rescanning. At block %d. Progress=%f\n", *block_height, progress_current); + WalletLogPrintf("Still rescanning. At block %d. Progress=%f\n", block_height, progress_current); } CBlock block; - if (chain().findBlock(block_hash, &block) && !block.IsNull()) { + bool next_block; + uint256 next_block_hash; + bool reorg = false; + if (chain().findBlock(block_hash, FoundBlock().data(block)) && !block.IsNull()) { auto locked_chain = chain().lock(); LOCK(cs_wallet); - if (!locked_chain->getBlockHeight(block_hash)) { + next_block = chain().findNextBlock(block_hash, block_height, FoundBlock().hash(next_block_hash), &reorg); + if (reorg) { // Abort scan if current block is no longer active, to prevent // marking transactions as coming from the wrong block. // TODO: This should return success instead of failure, see @@ -1690,37 +1699,38 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc break; } for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { - CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, *block_height, block_hash, posInBlock); + CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, block_height, block_hash, posInBlock); SyncTransaction(block.vtx[posInBlock], confirm, fUpdate); } // scan succeeded, record block as most recent successfully scanned result.last_scanned_block = block_hash; - result.last_scanned_height = *block_height; + result.last_scanned_height = block_height; } else { // could not scan block, keep scanning but record this block as the most recent failure result.last_failed_block = block_hash; result.status = ScanResult::FAILURE; + next_block = chain().findNextBlock(block_hash, block_height, FoundBlock().hash(next_block_hash), &reorg); } - if (block_hash == stop_block) { + if (max_height && block_height >= *max_height) { break; } { auto locked_chain = chain().lock(); - Optional<int> tip_height = locked_chain->getHeight(); - if (!tip_height || *tip_height <= block_height || !locked_chain->getBlockHeight(block_hash)) { + if (!next_block || reorg) { // break successfully when rescan has reached the tip, or // previous block is no longer on the chain due to a reorg break; } // increment block and verification progress - block_hash = locked_chain->getBlockHash(++*block_height); + block_hash = next_block_hash; + ++block_height; progress_current = chain().guessVerificationProgress(block_hash); // handle updated tip hash const uint256 prev_tip_hash = tip_hash; - tip_hash = locked_chain->getBlockHash(*tip_height); - if (stop_block.IsNull() && prev_tip_hash != tip_hash) { + tip_hash = WITH_LOCK(cs_wallet, return GetLastBlockHash()); + if (!max_height && prev_tip_hash != tip_hash) { // in case the tip has changed, update progress max progress_end = chain().guessVerificationProgress(tip_hash); } @@ -1728,10 +1738,10 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc } ShowProgress(strprintf("%s " + _("Rescanning...").translated, GetDisplayName()), 100); // hide progress dialog in GUI if (block_height && fAbortRescan) { - WalletLogPrintf("Rescan aborted at block %d. Progress=%f\n", *block_height, progress_current); + WalletLogPrintf("Rescan aborted at block %d. Progress=%f\n", block_height, progress_current); result.status = ScanResult::USER_ABORT; } else if (block_height && chain().shutdownRequested()) { - WalletLogPrintf("Rescan interrupted by shutdown request at block %d. Progress=%f\n", *block_height, progress_current); + WalletLogPrintf("Rescan interrupted by shutdown request at block %d. Progress=%f\n", block_height, progress_current); result.status = ScanResult::USER_ABORT; } else { WalletLogPrintf("Rescan completed in %15dms\n", GetTimeMillis() - start_time); @@ -2377,6 +2387,13 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm ++it; } + unsigned int limit_ancestor_count = 0; + unsigned int limit_descendant_count = 0; + chain().getPackageLimits(limit_ancestor_count, limit_descendant_count); + size_t max_ancestors = (size_t)std::max<int64_t>(1, limit_ancestor_count); + size_t max_descendants = (size_t)std::max<int64_t>(1, limit_descendant_count); + bool fRejectLongChains = gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS); + // form groups from remaining coins; note that preset coins will not // automatically have their associated (same address) coins included if (coin_control.m_avoid_partial_spends && vCoins.size() > OUTPUT_GROUP_MAX_ENTRIES) { @@ -2385,14 +2402,7 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm // explicitly shuffling the outputs before processing Shuffle(vCoins.begin(), vCoins.end(), FastRandomContext()); } - std::vector<OutputGroup> groups = GroupOutputs(vCoins, !coin_control.m_avoid_partial_spends); - - unsigned int limit_ancestor_count; - unsigned int limit_descendant_count; - chain().getPackageLimits(limit_ancestor_count, limit_descendant_count); - size_t max_ancestors = (size_t)std::max<int64_t>(1, limit_ancestor_count); - size_t max_descendants = (size_t)std::max<int64_t>(1, limit_descendant_count); - bool fRejectLongChains = gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS); + std::vector<OutputGroup> groups = GroupOutputs(vCoins, !coin_control.m_avoid_partial_spends, max_ancestors); bool res = value_to_select <= 0 || SelectCoinsMinConf(value_to_select, CoinEligibilityFilter(1, 6, 0), groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used) || @@ -2432,11 +2442,17 @@ bool CWallet::SignTransaction(CMutableTransaction& tx) const bool CWallet::SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors) const { - // Sign the tx with ScriptPubKeyMans - // Because each ScriptPubKeyMan can sign more than one input, we need to keep track of each ScriptPubKeyMan that has signed this transaction. - // Each iteration, we may sign more txins than the txin that is specified in that iteration. - // We assume that each input is signed by only one ScriptPubKeyMan. - std::set<uint256> visited_spk_mans; + // Try to sign with all ScriptPubKeyMans + for (ScriptPubKeyMan* spk_man : GetAllScriptPubKeyMans()) { + // spk_man->SignTransaction will return true if the transaction is complete, + // so we can exit early and return true if that happens + if (spk_man->SignTransaction(tx, coins, sighash, input_errors)) { + return true; + } + } + + // At this point, one input was not fully signed otherwise we would have exited already + // Find that input and figure out what went wrong. for (unsigned int i = 0; i < tx.vin.size(); i++) { // Get the prevout CTxIn& txin = tx.vin[i]; @@ -2448,33 +2464,10 @@ bool CWallet::SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, // Check if this input is complete SignatureData sigdata = DataFromTransaction(tx, i, coin->second.out); - if (sigdata.complete) { - continue; - } - - // Input needs to be signed, find the right ScriptPubKeyMan - std::set<ScriptPubKeyMan*> spk_mans = GetScriptPubKeyMans(coin->second.out.scriptPubKey, sigdata); - if (spk_mans.size() == 0) { + if (!sigdata.complete) { input_errors[i] = "Unable to sign input, missing keys"; continue; } - - for (auto& spk_man : spk_mans) { - // If we've already been signed by this spk_man, skip it - if (visited_spk_mans.count(spk_man->GetID()) > 0) { - continue; - } - - // Sign the tx. - // spk_man->SignTransaction will return true if the transaction is complete, - // so we can exit early and return true if that happens. - if (spk_man->SignTransaction(tx, coins, sighash, input_errors)) { - return true; - } - - // Add this spk_man to visited_spk_mans so we can skip it later - visited_spk_mans.insert(spk_man->GetID()); - } } return false; } @@ -2510,52 +2503,10 @@ TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& comp } // Fill in information from ScriptPubKeyMans - // Because each ScriptPubKeyMan may be able to fill more than one input, we need to keep track of each ScriptPubKeyMan that has filled this psbt. - // Each iteration, we may fill more inputs than the input that is specified in that iteration. - // We assume that each input is filled by only one ScriptPubKeyMan - std::set<uint256> visited_spk_mans; - for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { - const CTxIn& txin = psbtx.tx->vin[i]; - PSBTInput& input = psbtx.inputs.at(i); - - if (PSBTInputSigned(input)) { - continue; - } - - // Get the scriptPubKey to know which ScriptPubKeyMan to use - CScript script; - if (!input.witness_utxo.IsNull()) { - script = input.witness_utxo.scriptPubKey; - } else if (input.non_witness_utxo) { - if (txin.prevout.n >= input.non_witness_utxo->vout.size()) { - return TransactionError::MISSING_INPUTS; - } - script = input.non_witness_utxo->vout[txin.prevout.n].scriptPubKey; - } else { - // There's no UTXO so we can just skip this now - continue; - } - SignatureData sigdata; - input.FillSignatureData(sigdata); - std::set<ScriptPubKeyMan*> spk_mans = GetScriptPubKeyMans(script, sigdata); - if (spk_mans.size() == 0) { - continue; - } - - for (auto& spk_man : spk_mans) { - // If we've already been signed by this spk_man, skip it - if (visited_spk_mans.count(spk_man->GetID()) > 0) { - continue; - } - - // Fill in the information from the spk_man - TransactionError res = spk_man->FillPSBT(psbtx, sighash_type, sign, bip32derivs); - if (res != TransactionError::OK) { - return res; - } - - // Add this spk_man to visited_spk_mans so we can skip it later - visited_spk_mans.insert(spk_man->GetID()); + for (ScriptPubKeyMan* spk_man : GetAllScriptPubKeyMans()) { + TransactionError res = spk_man->FillPSBT(psbtx, sighash_type, sign, bip32derivs); + if (res != TransactionError::OK) { + return res; } } @@ -2631,13 +2582,15 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nC return true; } -static bool IsCurrentForAntiFeeSniping(interfaces::Chain& chain, interfaces::Chain::Lock& locked_chain) +static bool IsCurrentForAntiFeeSniping(interfaces::Chain& chain, const uint256& block_hash) { if (chain.isInitialBlockDownload()) { return false; } constexpr int64_t MAX_ANTI_FEE_SNIPING_TIP_AGE = 8 * 60 * 60; // in seconds - if (locked_chain.getBlockTime(*locked_chain.getHeight()) < (GetTime() - MAX_ANTI_FEE_SNIPING_TIP_AGE)) { + int64_t block_time; + CHECK_NONFATAL(chain.findBlock(block_hash, FoundBlock().time(block_time))); + if (block_time < (GetTime() - MAX_ANTI_FEE_SNIPING_TIP_AGE)) { return false; } return true; @@ -2647,9 +2600,8 @@ static bool IsCurrentForAntiFeeSniping(interfaces::Chain& chain, interfaces::Cha * Return a height-based locktime for new transactions (uses the height of the * current chain tip unless we are not synced with the current chain */ -static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, interfaces::Chain::Lock& locked_chain) +static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, const uint256& block_hash, int block_height) { - uint32_t const height = locked_chain.getHeight().get_value_or(-1); uint32_t locktime; // Discourage fee sniping. // @@ -2671,8 +2623,8 @@ static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, interface // enough, that fee sniping isn't a problem yet, but by implementing a fix // now we ensure code won't be written that makes assumptions about // nLockTime that preclude a fix later. - if (IsCurrentForAntiFeeSniping(chain, locked_chain)) { - locktime = height; + if (IsCurrentForAntiFeeSniping(chain, block_hash)) { + locktime = block_height; // Secondly occasionally randomly pick a nLockTime even further back, so // that transactions that are delayed after signing for whatever reason, @@ -2686,7 +2638,6 @@ static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, interface // unique "nLockTime fingerprint", set nLockTime to a constant. locktime = 0; } - assert(locktime <= height); assert(locktime < LOCKTIME_THRESHOLD); return locktime; } @@ -2746,9 +2697,6 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std } CMutableTransaction txNew; - - txNew.nLockTime = GetLocktimeForNewTransaction(chain(), locked_chain); - FeeCalculation feeCalc; CAmount nFeeNeeded; int nBytes; @@ -2756,6 +2704,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std std::set<CInputCoin> setCoins; auto locked_chain = chain().lock(); LOCK(cs_wallet); + txNew.nLockTime = GetLocktimeForNewTransaction(chain(), GetLastBlockHash(), GetLastBlockHeight()); { std::vector<COutput> vAvailableCoins; AvailableCoins(*locked_chain, vAvailableCoins, true, &coin_control, 1, MAX_MONEY, MAX_MONEY, 0); @@ -2777,20 +2726,14 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std // rediscover unknown transactions that were written with keys of ours to recover // post-backup change. - // Reserve a new key pair from key pool - if (!CanGetAddresses(true)) { - strFailReason = _("Can't generate a change-address key. No keys in the internal keypool and can't generate any keys.").translated; - return false; - } + // Reserve a new key pair from key pool. If it fails, provide a dummy + // destination in case we don't need change. CTxDestination dest; - bool ret = reservedest.GetReservedDestination(dest, true); - if (!ret) - { - strFailReason = "Keypool ran out, please call keypoolrefill first"; - return false; + if (!reservedest.GetReservedDestination(dest, true)) { + strFailReason = _("Transaction needs a change address, but we can't generate it. Please call keypoolrefill first.").translated; } - scriptChange = GetScriptForDestination(dest); + assert(!dest.empty() || scriptChange.empty()); } CTxOut change_prototype_txout(0, scriptChange); coin_selection_params.change_output_size = GetSerializeSize(change_prototype_txout); @@ -3006,6 +2949,11 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std coin_selection_params.use_bnb = false; continue; } + + // Give up if change keypool ran out and we failed to find a solution without change: + if (scriptChange.empty() && nChangePosInOut != -1) { + return false; + } } // Shuffle selected coins and fill in final vin @@ -3286,6 +3234,8 @@ bool CWallet::GetNewDestination(const OutputType type, const std::string label, if (spk_man) { spk_man->TopUp(); result = spk_man->GetNewDestination(type, dest, error); + } else { + error = strprintf("Error: No %s addresses available.", FormatOutputType(type)); } if (result) { SetAddressBook(dest, label, "receive"); @@ -3301,7 +3251,7 @@ bool CWallet::GetNewChangeDestination(const OutputType type, CTxDestination& des ReserveDestination reservedest(this, type); if (!reservedest.GetReservedDestination(dest, true)) { - error = "Error: Keypool ran out, please call keypoolrefill first"; + error = _("Error: Keypool ran out, please call keypoolrefill first").translated; return false; } @@ -3576,12 +3526,13 @@ void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<C } // map in which we'll infer heights of other keys - const Optional<int> tip_height = locked_chain.getHeight(); - const int max_height = tip_height && *tip_height > 144 ? *tip_height - 144 : 0; // the tip can be reorganized; use a 144-block safety margin - std::map<CKeyID, int> mapKeyFirstBlock; + std::map<CKeyID, const CWalletTx::Confirmation*> mapKeyFirstBlock; + CWalletTx::Confirmation max_confirm; + max_confirm.block_height = GetLastBlockHeight() > 144 ? GetLastBlockHeight() - 144 : 0; // the tip can be reorganized; use a 144-block safety margin + CHECK_NONFATAL(chain().findAncestorByHeight(GetLastBlockHash(), max_confirm.block_height, FoundBlock().hash(max_confirm.hashBlock))); for (const CKeyID &keyid : spk_man->GetKeys()) { if (mapKeyBirth.count(keyid) == 0) - mapKeyFirstBlock[keyid] = max_height; + mapKeyFirstBlock[keyid] = &max_confirm; } // if there are no such keys, we're done @@ -3592,23 +3543,27 @@ void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<C for (const auto& entry : mapWallet) { // iterate over all wallet transactions... const CWalletTx &wtx = entry.second; - if (Optional<int> height = locked_chain.getBlockHeight(wtx.m_confirm.hashBlock)) { + if (wtx.m_confirm.status == CWalletTx::CONFIRMED) { // ... which are already in a block for (const CTxOut &txout : wtx.tx->vout) { // iterate over all their outputs for (const auto &keyid : GetAffectedKeys(txout.scriptPubKey, *spk_man)) { // ... and all their affected keys - std::map<CKeyID, int>::iterator rit = mapKeyFirstBlock.find(keyid); - if (rit != mapKeyFirstBlock.end() && *height < rit->second) - rit->second = *height; + auto rit = mapKeyFirstBlock.find(keyid); + if (rit != mapKeyFirstBlock.end() && wtx.m_confirm.block_height < rit->second->block_height) { + rit->second = &wtx.m_confirm; + } } } } } // Extract block timestamps for those keys - for (const auto& entry : mapKeyFirstBlock) - mapKeyBirth[entry.first] = locked_chain.getBlockTime(entry.second) - TIMESTAMP_WINDOW; // block times can be 2h off + for (const auto& entry : mapKeyFirstBlock) { + int64_t block_time; + CHECK_NONFATAL(chain().findBlock(entry.second->hashBlock, FoundBlock().time(block_time))); + mapKeyBirth[entry.first] = block_time - TIMESTAMP_WINDOW; // block times can be 2h off + } } /** @@ -3637,7 +3592,7 @@ unsigned int CWallet::ComputeTimeSmart(const CWalletTx& wtx) const unsigned int nTimeSmart = wtx.nTimeReceived; if (!wtx.isUnconfirmed() && !wtx.isAbandoned()) { int64_t blocktime; - if (chain().findBlock(wtx.m_confirm.hashBlock, nullptr /* block */, &blocktime)) { + if (chain().findBlock(wtx.m_confirm.hashBlock, FoundBlock().time(blocktime))) { int64_t latestNow = wtx.nTimeReceived; int64_t latestEntry = 0; @@ -3830,44 +3785,6 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, } } - int prev_version = walletInstance->GetVersion(); - if (gArgs.GetBoolArg("-upgradewallet", fFirstRun)) - { - int nMaxVersion = gArgs.GetArg("-upgradewallet", 0); - if (nMaxVersion == 0) // the -upgradewallet without argument case - { - walletInstance->WalletLogPrintf("Performing wallet upgrade to %i\n", FEATURE_LATEST); - nMaxVersion = FEATURE_LATEST; - walletInstance->SetMinVersion(FEATURE_LATEST); // permanently upgrade the wallet immediately - } - else - walletInstance->WalletLogPrintf("Allowing wallet upgrade up to %i\n", nMaxVersion); - if (nMaxVersion < walletInstance->GetVersion()) - { - error = _("Cannot downgrade wallet").translated; - return nullptr; - } - walletInstance->SetMaxVersion(nMaxVersion); - } - - // Upgrade to HD if explicit upgrade - if (gArgs.GetBoolArg("-upgradewallet", false)) { - LOCK(walletInstance->cs_wallet); - - // Do not upgrade versions to any version between HD_SPLIT and FEATURE_PRE_SPLIT_KEYPOOL unless already supporting HD_SPLIT - int max_version = walletInstance->GetVersion(); - if (!walletInstance->CanSupportFeature(FEATURE_HD_SPLIT) && max_version >= FEATURE_HD_SPLIT && max_version < FEATURE_PRE_SPLIT_KEYPOOL) { - error = _("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use -upgradewallet=169900 or -upgradewallet with no version specified.").translated; - return nullptr; - } - - for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { - if (!spk_man->Upgrade(prev_version, error)) { - return nullptr; - } - } - } - if (fFirstRun) { // ensure this wallet.dat can only be opened by clients supporting HD with chain split and expects no default key @@ -3875,15 +3792,23 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, walletInstance->SetWalletFlags(wallet_creation_flags, false); - // Always create LegacyScriptPubKeyMan for now - walletInstance->SetupLegacyScriptPubKeyMan(); + // Only create LegacyScriptPubKeyMan when not descriptor wallet + if (!walletInstance->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + walletInstance->SetupLegacyScriptPubKeyMan(); + } if (!(wallet_creation_flags & (WALLET_FLAG_DISABLE_PRIVATE_KEYS | WALLET_FLAG_BLANK_WALLET))) { LOCK(walletInstance->cs_wallet); - for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { - if (!spk_man->SetupGeneration()) { - error = _("Unable to generate initial keys").translated; - return nullptr; + if (walletInstance->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + walletInstance->SetupDescriptorScriptPubKeyMans(); + // SetupDescriptorScriptPubKeyMans already calls SetupGeneration for us so we don't need to call SetupGeneration separately + } else { + // Legacy wallets need SetupGeneration here. + for (auto spk_man : walletInstance->GetActiveScriptPubKeyMans()) { + if (!spk_man->SetupGeneration()) { + error = _("Unable to generate initial keys").translated; + return nullptr; + } } } } @@ -4064,8 +3989,8 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, } { - WalletRescanReserver reserver(walletInstance.get()); - if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(locked_chain->getBlockHash(rescan_height), {} /* stop block */, reserver, true /* update */).status)) { + WalletRescanReserver reserver(*walletInstance); + if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(locked_chain->getBlockHash(rescan_height), rescan_height, {} /* max height */, reserver, true /* update */).status)) { error = _("Failed to rescan the wallet during initialization").translated; return nullptr; } @@ -4129,6 +4054,42 @@ const CAddressBookData* CWallet::FindAddressBookEntry(const CTxDestination& dest return &address_book_it->second; } +bool CWallet::UpgradeWallet(int version, std::string& error, std::vector<std::string>& warnings) +{ + int prev_version = GetVersion(); + int nMaxVersion = version; + if (nMaxVersion == 0) // the -upgradewallet without argument case + { + WalletLogPrintf("Performing wallet upgrade to %i\n", FEATURE_LATEST); + nMaxVersion = FEATURE_LATEST; + SetMinVersion(FEATURE_LATEST); // permanently upgrade the wallet immediately + } + else + WalletLogPrintf("Allowing wallet upgrade up to %i\n", nMaxVersion); + if (nMaxVersion < GetVersion()) + { + error = _("Cannot downgrade wallet").translated; + return false; + } + SetMaxVersion(nMaxVersion); + + LOCK(cs_wallet); + + // Do not upgrade versions to any version between HD_SPLIT and FEATURE_PRE_SPLIT_KEYPOOL unless already supporting HD_SPLIT + int max_version = GetVersion(); + if (!CanSupportFeature(FEATURE_HD_SPLIT) && max_version >= FEATURE_HD_SPLIT && max_version < FEATURE_PRE_SPLIT_KEYPOOL) { + error = _("Cannot upgrade a non HD split wallet without upgrading to support pre split keypool. Please use version 169900 or no version specified.").translated; + return false; + } + + for (auto spk_man : GetActiveScriptPubKeyMans()) { + if (!spk_man->Upgrade(prev_version, error)) { + return false; + } + } + return true; +} + void CWallet::postInitProcess() { auto locked_chain = chain().lock(); @@ -4186,32 +4147,49 @@ bool CWalletTx::IsImmatureCoinBase() const return GetBlocksToMaturity() > 0; } -std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outputs, bool single_coin) const { +std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outputs, bool single_coin, const size_t max_ancestors) const { std::vector<OutputGroup> groups; std::map<CTxDestination, OutputGroup> gmap; - CTxDestination dst; + std::set<CTxDestination> full_groups; + for (const auto& output : outputs) { if (output.fSpendable) { + CTxDestination dst; CInputCoin input_coin = output.GetInputCoin(); size_t ancestors, descendants; chain().getTransactionAncestry(output.tx->GetHash(), ancestors, descendants); if (!single_coin && ExtractDestination(output.tx->tx->vout[output.i].scriptPubKey, dst)) { - // Limit output groups to no more than 10 entries, to protect - // against inadvertently creating a too-large transaction - // when using -avoidpartialspends - if (gmap[dst].m_outputs.size() >= OUTPUT_GROUP_MAX_ENTRIES) { - groups.push_back(gmap[dst]); - gmap.erase(dst); + auto it = gmap.find(dst); + if (it != gmap.end()) { + // Limit output groups to no more than OUTPUT_GROUP_MAX_ENTRIES + // number of entries, to protect against inadvertently creating + // a too-large transaction when using -avoidpartialspends to + // prevent breaking consensus or surprising users with a very + // high amount of fees. + if (it->second.m_outputs.size() >= OUTPUT_GROUP_MAX_ENTRIES) { + groups.push_back(it->second); + it->second = OutputGroup{}; + full_groups.insert(dst); + } + it->second.Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); + } else { + gmap[dst].Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); } - gmap[dst].Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); } else { groups.emplace_back(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants); } } } if (!single_coin) { - for (const auto& it : gmap) groups.push_back(it.second); + for (auto& it : gmap) { + auto& group = it.second; + if (full_groups.count(it.first) > 0) { + // Make this unattractive as we want coin selection to avoid it if possible + group.m_ancestors = max_ancestors - 1; + } + groups.push_back(group); + } } return groups; } @@ -4341,6 +4319,9 @@ std::unique_ptr<SigningProvider> CWallet::GetSolvingProvider(const CScript& scri LegacyScriptPubKeyMan* CWallet::GetLegacyScriptPubKeyMan() const { + if (IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + return nullptr; + } // Legacy wallets only have one ScriptPubKeyMan which is a LegacyScriptPubKeyMan. // Everything in m_internal_spk_managers and m_external_spk_managers point to the same legacyScriptPubKeyMan. auto it = m_internal_spk_managers.find(OutputType::LEGACY); @@ -4356,7 +4337,7 @@ LegacyScriptPubKeyMan* CWallet::GetOrCreateLegacyScriptPubKeyMan() void CWallet::SetupLegacyScriptPubKeyMan() { - if (!m_internal_spk_managers.empty() || !m_external_spk_managers.empty() || !m_spk_managers.empty()) { + if (!m_internal_spk_managers.empty() || !m_external_spk_managers.empty() || !m_spk_managers.empty() || IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { return; } @@ -4385,3 +4366,153 @@ void CWallet::ConnectScriptPubKeyManNotifiers() spk_man->NotifyCanGetAddressesChanged.connect(NotifyCanGetAddressesChanged); } } + +void CWallet::LoadDescriptorScriptPubKeyMan(uint256 id, WalletDescriptor& desc) +{ + auto spk_manager = std::unique_ptr<ScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, desc)); + m_spk_managers[id] = std::move(spk_manager); +} + +void CWallet::SetupDescriptorScriptPubKeyMans() +{ + AssertLockHeld(cs_wallet); + + // Make a seed + CKey seed_key; + seed_key.MakeNewKey(true); + CPubKey seed = seed_key.GetPubKey(); + assert(seed_key.VerifyPubKey(seed)); + + // Get the extended key + CExtKey master_key; + master_key.SetSeed(seed_key.begin(), seed_key.size()); + + for (bool internal : {false, true}) { + for (OutputType t : OUTPUT_TYPES) { + auto spk_manager = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, t, internal)); + if (IsCrypted()) { + if (IsLocked()) { + throw std::runtime_error(std::string(__func__) + ": Wallet is locked, cannot setup new descriptors"); + } + if (!spk_manager->CheckDecryptionKey(vMasterKey) && !spk_manager->Encrypt(vMasterKey, nullptr)) { + throw std::runtime_error(std::string(__func__) + ": Could not encrypt new descriptors"); + } + } + spk_manager->SetupDescriptorGeneration(master_key); + uint256 id = spk_manager->GetID(); + m_spk_managers[id] = std::move(spk_manager); + SetActiveScriptPubKeyMan(id, t, internal); + } + } +} + +void CWallet::SetActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal, bool memonly) +{ + WalletLogPrintf("Setting spkMan to active: id = %s, type = %d, internal = %d\n", id.ToString(), static_cast<int>(type), static_cast<int>(internal)); + auto& spk_mans = internal ? m_internal_spk_managers : m_external_spk_managers; + auto spk_man = m_spk_managers.at(id).get(); + spk_man->SetType(type, internal); + spk_mans[type] = spk_man; + + if (!memonly) { + WalletBatch batch(*database); + if (!batch.WriteActiveScriptPubKeyMan(static_cast<uint8_t>(type), id, internal)) { + throw std::runtime_error(std::string(__func__) + ": writing active ScriptPubKeyMan id failed"); + } + } + NotifyCanGetAddressesChanged(); +} + +bool CWallet::IsLegacy() const +{ + if (m_internal_spk_managers.count(OutputType::LEGACY) == 0) { + return false; + } + auto spk_man = dynamic_cast<LegacyScriptPubKeyMan*>(m_internal_spk_managers.at(OutputType::LEGACY)); + return spk_man != nullptr; +} + +DescriptorScriptPubKeyMan* CWallet::GetDescriptorScriptPubKeyMan(const WalletDescriptor& desc) const +{ + for (auto& spk_man_pair : m_spk_managers) { + // Try to downcast to DescriptorScriptPubKeyMan then check if the descriptors match + DescriptorScriptPubKeyMan* spk_manager = dynamic_cast<DescriptorScriptPubKeyMan*>(spk_man_pair.second.get()); + if (spk_manager != nullptr && spk_manager->HasWalletDescriptor(desc)) { + return spk_manager; + } + } + + return nullptr; +} + +ScriptPubKeyMan* CWallet::AddWalletDescriptor(WalletDescriptor& desc, const FlatSigningProvider& signing_provider, const std::string& label) +{ + if (!IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { + WalletLogPrintf("Cannot add WalletDescriptor to a non-descriptor wallet\n"); + return nullptr; + } + + LOCK(cs_wallet); + auto new_spk_man = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, desc)); + + // If we already have this descriptor, remove it from the maps but add the existing cache to desc + auto old_spk_man = GetDescriptorScriptPubKeyMan(desc); + if (old_spk_man) { + WalletLogPrintf("Update existing descriptor: %s\n", desc.descriptor->ToString()); + + { + LOCK(old_spk_man->cs_desc_man); + new_spk_man->SetCache(old_spk_man->GetWalletDescriptor().cache); + } + + // Remove from maps of active spkMans + auto old_spk_man_id = old_spk_man->GetID(); + for (bool internal : {false, true}) { + for (OutputType t : OUTPUT_TYPES) { + auto active_spk_man = GetScriptPubKeyMan(t, internal); + if (active_spk_man && active_spk_man->GetID() == old_spk_man_id) { + if (internal) { + m_internal_spk_managers.erase(t); + } else { + m_external_spk_managers.erase(t); + } + break; + } + } + } + m_spk_managers.erase(old_spk_man_id); + } + + // Add the private keys to the descriptor + for (const auto& entry : signing_provider.keys) { + const CKey& key = entry.second; + new_spk_man->AddDescriptorKey(key, key.GetPubKey()); + } + + // Top up key pool, the manager will generate new scriptPubKeys internally + new_spk_man->TopUp(); + + // Apply the label if necessary + // Note: we disable labels for ranged descriptors + if (!desc.descriptor->IsRange()) { + auto script_pub_keys = new_spk_man->GetScriptPubKeys(); + if (script_pub_keys.empty()) { + WalletLogPrintf("Could not generate scriptPubKeys (cache is empty)\n"); + return nullptr; + } + + CTxDestination dest; + if (ExtractDestination(script_pub_keys.at(0), dest)) { + SetAddressBook(dest, label, "receive"); + } + } + + // Save the descriptor to memory + auto ret = new_spk_man.get(); + m_spk_managers[new_spk_man->GetID()] = std::move(new_spk_man); + + // Save the descriptor to DB + ret->WriteDescriptor(); + + return ret; +} diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 6c54c72e76..24c78fceb3 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -111,7 +111,8 @@ static constexpr uint64_t KNOWN_WALLET_FLAGS = WALLET_FLAG_AVOID_REUSE | WALLET_FLAG_BLANK_WALLET | WALLET_FLAG_KEY_ORIGIN_METADATA - | WALLET_FLAG_DISABLE_PRIVATE_KEYS; + | WALLET_FLAG_DISABLE_PRIVATE_KEYS + | WALLET_FLAG_DESCRIPTORS; static constexpr uint64_t MUTABLE_WALLET_FLAGS = WALLET_FLAG_AVOID_REUSE; @@ -121,6 +122,7 @@ static const std::map<std::string,WalletFlags> WALLET_FLAG_MAP{ {"blank", WALLET_FLAG_BLANK_WALLET}, {"key_origin_metadata", WALLET_FLAG_KEY_ORIGIN_METADATA}, {"disable_private_keys", WALLET_FLAG_DISABLE_PRIVATE_KEYS}, + {"descriptor_wallet", WALLET_FLAG_DESCRIPTORS}, }; extern const std::map<uint64_t,std::string> WALLET_FLAG_CAVEATS; @@ -830,7 +832,7 @@ public: bool IsSpentKey(const uint256& hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void SetSpentKeyState(WalletBatch& batch, const uint256& hash, unsigned int n, bool used, std::set<CTxDestination>& tx_destinations) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - std::vector<OutputGroup> GroupOutputs(const std::vector<COutput>& outputs, bool single_coin) const; + std::vector<OutputGroup> GroupOutputs(const std::vector<COutput>& outputs, bool single_coin, const size_t max_ancestors) const; bool IsLockedCoin(uint256 hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void LockCoin(const COutPoint& output) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -907,7 +909,7 @@ public: //! USER_ABORT. uint256 last_failed_block; }; - ScanResult ScanForWalletTransactions(const uint256& first_block, const uint256& last_block, const WalletRescanReserver& reserver, bool fUpdate); + ScanResult ScanForWalletTransactions(const uint256& start_block, int start_height, Optional<int> max_height, const WalletRescanReserver& reserver, bool fUpdate); void transactionRemovedFromMempool(const CTransactionRef &ptx) override; void ReacceptWalletTransactions() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void ResendWalletTransactions(); @@ -1163,6 +1165,9 @@ public: returns false if unknown, non-tolerable flags are present */ bool SetWalletFlags(uint64_t overwriteFlags, bool memOnly); + /** Determine if we are a legacy wallet */ + bool IsLegacy() const; + /** Returns a bracketed wallet name for displaying in logs, will return [default wallet] if the wallet has no name */ const std::string GetDisplayName() const override { std::string wallet_name = GetName().length() == 0 ? "default wallet" : GetName(); @@ -1175,6 +1180,9 @@ public: LogPrintf(("%s " + fmt).c_str(), GetDisplayName(), parameters...); }; + /** Upgrade the wallet */ + bool UpgradeWallet(int version, std::string& error, std::vector<std::string>& warnings); + //! Returns all unique ScriptPubKeyMans in m_internal_spk_managers and m_external_spk_managers std::set<ScriptPubKeyMan*> GetActiveScriptPubKeyMans() const; @@ -1213,6 +1221,12 @@ public: assert(m_last_block_processed_height >= 0); return m_last_block_processed_height; }; + uint256 GetLastBlockHash() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) + { + AssertLockHeld(cs_wallet); + assert(m_last_block_processed_height >= 0); + return m_last_block_processed; + } /** Set last block processed height, currently only use in unit test */ void SetLastBlockProcessed(int block_height, uint256 block_hash) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { @@ -1223,6 +1237,25 @@ public: //! Connect the signals from ScriptPubKeyMans to the signals in CWallet void ConnectScriptPubKeyManNotifiers(); + + //! Instantiate a descriptor ScriptPubKeyMan from the WalletDescriptor and load it + void LoadDescriptorScriptPubKeyMan(uint256 id, WalletDescriptor& desc); + + //! Sets the active ScriptPubKeyMan for the specified type and internal + //! @param[in] id The unique id for the ScriptPubKeyMan + //! @param[in] type The OutputType this ScriptPubKeyMan provides addresses for + //! @param[in] internal Whether this ScriptPubKeyMan provides change addresses + //! @param[in] memonly Whether to record this update to the database. Set to true for wallet loading, normally false when actually updating the wallet. + void SetActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal, bool memonly = false); + + //! Create new DescriptorScriptPubKeyMans and add them to the wallet + void SetupDescriptorScriptPubKeyMans(); + + //! Return the DescriptorScriptPubKeyMan for a WalletDescriptor if it is already in the wallet + DescriptorScriptPubKeyMan* GetDescriptorScriptPubKeyMan(const WalletDescriptor& desc) const; + + //! Add a descriptor to the wallet, return a ScriptPubKeyMan & associated output type + ScriptPubKeyMan* AddWalletDescriptor(WalletDescriptor& desc, const FlatSigningProvider& signing_provider, const std::string& label); }; /** @@ -1235,35 +1268,35 @@ void MaybeResendWalletTxs(); class WalletRescanReserver { private: - CWallet* m_wallet; + CWallet& m_wallet; bool m_could_reserve; public: - explicit WalletRescanReserver(CWallet* w) : m_wallet(w), m_could_reserve(false) {} + explicit WalletRescanReserver(CWallet& w) : m_wallet(w), m_could_reserve(false) {} bool reserve() { assert(!m_could_reserve); - std::lock_guard<std::mutex> lock(m_wallet->mutexScanning); - if (m_wallet->fScanningWallet) { + std::lock_guard<std::mutex> lock(m_wallet.mutexScanning); + if (m_wallet.fScanningWallet) { return false; } - m_wallet->m_scanning_start = GetTimeMillis(); - m_wallet->m_scanning_progress = 0; - m_wallet->fScanningWallet = true; + m_wallet.m_scanning_start = GetTimeMillis(); + m_wallet.m_scanning_progress = 0; + m_wallet.fScanningWallet = true; m_could_reserve = true; return true; } bool isReserved() const { - return (m_could_reserve && m_wallet->fScanningWallet); + return (m_could_reserve && m_wallet.fScanningWallet); } ~WalletRescanReserver() { - std::lock_guard<std::mutex> lock(m_wallet->mutexScanning); + std::lock_guard<std::mutex> lock(m_wallet.mutexScanning); if (m_could_reserve) { - m_wallet->fScanningWallet = false; + m_wallet.fScanningWallet = false; } } }; diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 568b21ed00..79316ca3e7 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -21,6 +21,8 @@ namespace DBKeys { const std::string ACENTRY{"acentry"}; +const std::string ACTIVEEXTERNALSPK{"activeexternalspk"}; +const std::string ACTIVEINTERNALSPK{"activeinternalspk"}; const std::string BESTBLOCK_NOMERKLE{"bestblock_nomerkle"}; const std::string BESTBLOCK{"bestblock"}; const std::string CRYPTED_KEY{"ckey"}; @@ -41,6 +43,10 @@ const std::string PURPOSE{"purpose"}; const std::string SETTINGS{"settings"}; const std::string TX{"tx"}; const std::string VERSION{"version"}; +const std::string WALLETDESCRIPTOR{"walletdescriptor"}; +const std::string WALLETDESCRIPTORCACHE{"walletdescriptorcache"}; +const std::string WALLETDESCRIPTORCKEY{"walletdescriptorckey"}; +const std::string WALLETDESCRIPTORKEY{"walletdescriptorkey"}; const std::string WATCHMETA{"watchmeta"}; const std::string WATCHS{"watchs"}; } // namespace DBKeys @@ -179,6 +185,51 @@ bool WalletBatch::WriteMinVersion(int nVersion) return WriteIC(DBKeys::MINVERSION, nVersion); } +bool WalletBatch::WriteActiveScriptPubKeyMan(uint8_t type, const uint256& id, bool internal) +{ + std::string key = internal ? DBKeys::ACTIVEINTERNALSPK : DBKeys::ACTIVEEXTERNALSPK; + return WriteIC(make_pair(key, type), id); +} + +bool WalletBatch::WriteDescriptorKey(const uint256& desc_id, const CPubKey& pubkey, const CPrivKey& privkey) +{ + // hash pubkey/privkey to accelerate wallet load + std::vector<unsigned char> key; + key.reserve(pubkey.size() + privkey.size()); + key.insert(key.end(), pubkey.begin(), pubkey.end()); + key.insert(key.end(), privkey.begin(), privkey.end()); + + return WriteIC(std::make_pair(DBKeys::WALLETDESCRIPTORKEY, std::make_pair(desc_id, pubkey)), std::make_pair(privkey, Hash(key.begin(), key.end())), false); +} + +bool WalletBatch::WriteCryptedDescriptorKey(const uint256& desc_id, const CPubKey& pubkey, const std::vector<unsigned char>& secret) +{ + if (!WriteIC(std::make_pair(DBKeys::WALLETDESCRIPTORCKEY, std::make_pair(desc_id, pubkey)), secret, false)) { + return false; + } + EraseIC(std::make_pair(DBKeys::WALLETDESCRIPTORKEY, std::make_pair(desc_id, pubkey))); + return true; +} + +bool WalletBatch::WriteDescriptor(const uint256& desc_id, const WalletDescriptor& descriptor) +{ + return WriteIC(make_pair(DBKeys::WALLETDESCRIPTOR, desc_id), descriptor); +} + +bool WalletBatch::WriteDescriptorDerivedCache(const CExtPubKey& xpub, const uint256& desc_id, uint32_t key_exp_index, uint32_t der_index) +{ + std::vector<unsigned char> ser_xpub(BIP32_EXTKEY_SIZE); + xpub.Encode(ser_xpub.data()); + return WriteIC(std::make_pair(std::make_pair(DBKeys::WALLETDESCRIPTORCACHE, desc_id), std::make_pair(key_exp_index, der_index)), ser_xpub); +} + +bool WalletBatch::WriteDescriptorParentCache(const CExtPubKey& xpub, const uint256& desc_id, uint32_t key_exp_index) +{ + std::vector<unsigned char> ser_xpub(BIP32_EXTKEY_SIZE); + xpub.Encode(ser_xpub.data()); + return WriteIC(std::make_pair(std::make_pair(DBKeys::WALLETDESCRIPTORCACHE, desc_id), key_exp_index), ser_xpub); +} + class CWalletScanState { public: unsigned int nKeys{0}; @@ -189,6 +240,11 @@ public: bool fIsEncrypted{false}; bool fAnyUnordered{false}; std::vector<uint256> vWalletUpgrade; + std::map<OutputType, uint256> m_active_external_spks; + std::map<OutputType, uint256> m_active_internal_spks; + std::map<uint256, DescriptorCache> m_descriptor_caches; + std::map<std::pair<uint256, CKeyID>, CKey> m_descriptor_keys; + std::map<std::pair<uint256, CKeyID>, std::pair<CPubKey, std::vector<unsigned char>>> m_descriptor_crypt_keys; CWalletScanState() { } @@ -404,6 +460,108 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, } else if (strType == DBKeys::OLD_KEY) { strErr = "Found unsupported 'wkey' record, try loading with version 0.18"; return false; + } else if (strType == DBKeys::ACTIVEEXTERNALSPK || strType == DBKeys::ACTIVEINTERNALSPK) { + uint8_t type; + ssKey >> type; + uint256 id; + ssValue >> id; + + bool internal = strType == DBKeys::ACTIVEINTERNALSPK; + auto& spk_mans = internal ? wss.m_active_internal_spks : wss.m_active_external_spks; + if (spk_mans.count(static_cast<OutputType>(type)) > 0) { + strErr = "Multiple ScriptPubKeyMans specified for a single type"; + return false; + } + spk_mans[static_cast<OutputType>(type)] = id; + } else if (strType == DBKeys::WALLETDESCRIPTOR) { + uint256 id; + ssKey >> id; + WalletDescriptor desc; + ssValue >> desc; + if (wss.m_descriptor_caches.count(id) == 0) { + wss.m_descriptor_caches[id] = DescriptorCache(); + } + pwallet->LoadDescriptorScriptPubKeyMan(id, desc); + } else if (strType == DBKeys::WALLETDESCRIPTORCACHE) { + bool parent = true; + uint256 desc_id; + uint32_t key_exp_index; + uint32_t der_index; + ssKey >> desc_id; + ssKey >> key_exp_index; + + // if the der_index exists, it's a derived xpub + try + { + ssKey >> der_index; + parent = false; + } + catch (...) {} + + std::vector<unsigned char> ser_xpub(BIP32_EXTKEY_SIZE); + ssValue >> ser_xpub; + CExtPubKey xpub; + xpub.Decode(ser_xpub.data()); + if (wss.m_descriptor_caches.count(desc_id)) { + wss.m_descriptor_caches[desc_id] = DescriptorCache(); + } + if (parent) { + wss.m_descriptor_caches[desc_id].CacheParentExtPubKey(key_exp_index, xpub); + } else { + wss.m_descriptor_caches[desc_id].CacheDerivedExtPubKey(key_exp_index, der_index, xpub); + } + } else if (strType == DBKeys::WALLETDESCRIPTORKEY) { + uint256 desc_id; + CPubKey pubkey; + ssKey >> desc_id; + ssKey >> pubkey; + if (!pubkey.IsValid()) + { + strErr = "Error reading wallet database: CPubKey corrupt"; + return false; + } + CKey key; + CPrivKey pkey; + uint256 hash; + + wss.nKeys++; + ssValue >> pkey; + ssValue >> hash; + + // hash pubkey/privkey to accelerate wallet load + std::vector<unsigned char> to_hash; + to_hash.reserve(pubkey.size() + pkey.size()); + to_hash.insert(to_hash.end(), pubkey.begin(), pubkey.end()); + to_hash.insert(to_hash.end(), pkey.begin(), pkey.end()); + + if (Hash(to_hash.begin(), to_hash.end()) != hash) + { + strErr = "Error reading wallet database: CPubKey/CPrivKey corrupt"; + return false; + } + + if (!key.Load(pkey, pubkey, true)) + { + strErr = "Error reading wallet database: CPrivKey corrupt"; + return false; + } + wss.m_descriptor_keys.insert(std::make_pair(std::make_pair(desc_id, pubkey.GetID()), key)); + } else if (strType == DBKeys::WALLETDESCRIPTORCKEY) { + uint256 desc_id; + CPubKey pubkey; + ssKey >> desc_id; + ssKey >> pubkey; + if (!pubkey.IsValid()) + { + strErr = "Error reading wallet database: CPubKey corrupt"; + return false; + } + std::vector<unsigned char> privkey; + ssValue >> privkey; + wss.nCKeys++; + + wss.m_descriptor_crypt_keys.insert(std::make_pair(std::make_pair(desc_id, pubkey.GetID()), std::make_pair(pubkey, privkey))); + wss.fIsEncrypted = true; } else if (strType != DBKeys::BESTBLOCK && strType != DBKeys::BESTBLOCK_NOMERKLE && strType != DBKeys::MINVERSION && strType != DBKeys::ACENTRY && strType != DBKeys::VERSION && strType != DBKeys::SETTINGS) { @@ -497,6 +655,31 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) result = DBErrors::CORRUPT; } + // Set the active ScriptPubKeyMans + for (auto spk_man_pair : wss.m_active_external_spks) { + pwallet->SetActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /* internal */ false, /* memonly */ true); + } + for (auto spk_man_pair : wss.m_active_internal_spks) { + pwallet->SetActiveScriptPubKeyMan(spk_man_pair.second, spk_man_pair.first, /* internal */ true, /* memonly */ true); + } + + // Set the descriptor caches + for (auto desc_cache_pair : wss.m_descriptor_caches) { + auto spk_man = pwallet->GetScriptPubKeyMan(desc_cache_pair.first); + assert(spk_man); + ((DescriptorScriptPubKeyMan*)spk_man)->SetCache(desc_cache_pair.second); + } + + // Set the descriptor keys + for (auto desc_key_pair : wss.m_descriptor_keys) { + auto spk_man = pwallet->GetScriptPubKeyMan(desc_key_pair.first.first); + ((DescriptorScriptPubKeyMan*)spk_man)->AddKey(desc_key_pair.first.second, desc_key_pair.second); + } + for (auto desc_key_pair : wss.m_descriptor_crypt_keys) { + auto spk_man = pwallet->GetScriptPubKeyMan(desc_key_pair.first.first); + ((DescriptorScriptPubKeyMan*)spk_man)->AddCryptedKey(desc_key_pair.first.second, desc_key_pair.second.first, desc_key_pair.second.second); + } + if (fNoncriticalErrors && result == DBErrors::LOAD_OK) result = DBErrors::NONCRITICAL_ERROR; @@ -516,7 +699,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) wss.nKeys, wss.nCKeys, wss.nKeyMeta, wss.nKeys + wss.nCKeys, wss.m_unknown_records); // nTimeFirstKey is only reliable if all keys have metadata - if ((wss.nKeys + wss.nCKeys + wss.nWatchKeys) != wss.nKeyMeta) { + if (pwallet->IsLegacy() && (wss.nKeys + wss.nCKeys + wss.nWatchKeys) != wss.nKeyMeta) { auto spk_man = pwallet->GetOrCreateLegacyScriptPubKeyMan(); if (spk_man) { LOCK(spk_man->cs_KeyStore); diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 1a65125480..2701481c58 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,6 +9,7 @@ #include <amount.h> #include <script/sign.h> #include <wallet/db.h> +#include <wallet/walletutil.h> #include <key.h> #include <stdint.h> @@ -54,6 +55,8 @@ enum class DBErrors namespace DBKeys { extern const std::string ACENTRY; +extern const std::string ACTIVEEXTERNALSPK; +extern const std::string ACTIVEINTERNALSPK; extern const std::string BESTBLOCK; extern const std::string BESTBLOCK_NOMERKLE; extern const std::string CRYPTED_KEY; @@ -74,6 +77,9 @@ extern const std::string PURPOSE; extern const std::string SETTINGS; extern const std::string TX; extern const std::string VERSION; +extern const std::string WALLETDESCRIPTOR; +extern const std::string WALLETDESCRIPTORCKEY; +extern const std::string WALLETDESCRIPTORKEY; extern const std::string WATCHMETA; extern const std::string WATCHS; } // namespace DBKeys @@ -240,11 +246,19 @@ public: bool WriteMinVersion(int nVersion); + bool WriteDescriptorKey(const uint256& desc_id, const CPubKey& pubkey, const CPrivKey& privkey); + bool WriteCryptedDescriptorKey(const uint256& desc_id, const CPubKey& pubkey, const std::vector<unsigned char>& secret); + bool WriteDescriptor(const uint256& desc_id, const WalletDescriptor& descriptor); + bool WriteDescriptorDerivedCache(const CExtPubKey& xpub, const uint256& desc_id, uint32_t key_exp_index, uint32_t der_index); + bool WriteDescriptorParentCache(const CExtPubKey& xpub, const uint256& desc_id, uint32_t key_exp_index); + /// Write destination data key,value tuple to database bool WriteDestData(const std::string &address, const std::string &key, const std::string &value); /// Erase destination data tuple from wallet database bool EraseDestData(const std::string &address, const std::string &key); + bool WriteActiveScriptPubKeyMan(uint8_t type, const uint256& id, bool internal); + DBErrors LoadWallet(CWallet* pwallet); DBErrors FindWalletTx(std::vector<uint256>& vTxHash, std::vector<CWalletTx>& vWtx); DBErrors ZapWalletTx(std::vector<CWalletTx>& vWtx); diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 8c918f4eb0..65643669c2 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2019 The Bitcoin Core developers +// Copyright (c) 2016-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp index 04c2407a89..8bac0608a9 100644 --- a/src/wallet/walletutil.cpp +++ b/src/wallet/walletutil.cpp @@ -100,5 +100,10 @@ WalletLocation::WalletLocation(const std::string& name) bool WalletLocation::Exists() const { - return fs::symlink_status(m_path).type() != fs::file_not_found; + fs::path path = m_path; + // For the default wallet, check specifically for the wallet.dat file + if (m_name.empty()) { + path = fs::absolute("wallet.dat", m_path); + } + return fs::symlink_status(path).type() != fs::file_not_found; } diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index c91c9aca96..d7e07ed04c 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -6,6 +6,7 @@ #define BITCOIN_WALLET_WALLETUTIL_H #include <fs.h> +#include <script/descriptor.h> #include <vector> @@ -55,6 +56,9 @@ enum WalletFlags : uint64_t { //! bitcoin from opening the wallet, thinking it was newly created, and //! then improperly reinitializing it. WALLET_FLAG_BLANK_WALLET = (1ULL << 33), + + //! Indicate that this wallet supports DescriptorScriptPubKeyMan + WALLET_FLAG_DESCRIPTORS = (1ULL << 34), }; //! Get the path of the wallet directory. @@ -83,4 +87,41 @@ public: bool Exists() const; }; +/** Descriptor with some wallet metadata */ +class WalletDescriptor +{ +public: + std::shared_ptr<Descriptor> descriptor; + uint64_t creation_time; + int32_t range_start; // First item in range; start of range, inclusive, i.e. [range_start, range_end). This never changes. + int32_t range_end; // Item after the last; end of range, exclusive, i.e. [range_start, range_end). This will increment with each TopUp() + int32_t next_index; // Position of the next item to generate + DescriptorCache cache; + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + if (ser_action.ForRead()) { + std::string desc; + std::string error; + READWRITE(desc); + FlatSigningProvider keys; + descriptor = Parse(desc, keys, error, true); + if (!descriptor) { + throw std::ios_base::failure("Invalid descriptor: " + error); + } + } else { + READWRITE(descriptor->ToString()); + } + READWRITE(creation_time); + READWRITE(next_index); + READWRITE(range_start); + READWRITE(range_end); + } + + WalletDescriptor() {} + WalletDescriptor(std::shared_ptr<Descriptor> descriptor, uint64_t creation_time, int32_t range_start, int32_t range_end, int32_t next_index) : descriptor(descriptor), creation_time(creation_time), range_start(range_start), range_end(range_end), next_index(next_index) {} +}; + #endif // BITCOIN_WALLET_WALLETUTIL_H diff --git a/src/zmq/zmqrpc.cpp b/src/zmq/zmqrpc.cpp index 0fbefb6023..cce6210129 100644 --- a/src/zmq/zmqrpc.cpp +++ b/src/zmq/zmqrpc.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019 The Bitcoin Core developers +// Copyright (c) 2018-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. |