diff options
102 files changed, 2715 insertions, 857 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index ac39fe235b..0c43e61592 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -39,6 +39,7 @@ after_build: - ps: clcache -z before_test: - ps: ${conf_ini} = (Get-Content([IO.Path]::Combine(${env:APPVEYOR_BUILD_FOLDER}, "test", "config.ini.in"))) +- ps: ${conf_ini} = ${conf_ini}.Replace("@PACKAGE_NAME@", "Bitcoin Core") - ps: ${conf_ini} = ${conf_ini}.Replace("@abs_top_srcdir@", ${env:APPVEYOR_BUILD_FOLDER}) - ps: ${conf_ini} = ${conf_ini}.Replace("@abs_top_builddir@", ${env:APPVEYOR_BUILD_FOLDER}) - ps: ${conf_ini} = ${conf_ini}.Replace("@EXEEXT@", ".exe") diff --git a/Makefile.am b/Makefile.am index 85674f819a..ec0743c3fa 100644 --- a/Makefile.am +++ b/Makefile.am @@ -43,6 +43,7 @@ DIST_DOCS = $(wildcard doc/*.md) $(wildcard doc/release-notes/*.md) DIST_CONTRIB = $(top_srcdir)/contrib/bitcoin-cli.bash-completion \ $(top_srcdir)/contrib/bitcoin-tx.bash-completion \ $(top_srcdir)/contrib/bitcoind.bash-completion \ + $(top_srcdir)/contrib/debian/copyright \ $(top_srcdir)/contrib/init \ $(top_srcdir)/contrib/install_db4.sh DIST_SHARE = \ diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h index 4ac27dae3f..b5a05e2629 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h @@ -11,10 +11,10 @@ #define CLIENT_VERSION_IS_RELEASE false /* Major version */ -#define CLIENT_VERSION_MAJOR 1 +#define CLIENT_VERSION_MAJOR 0 /* Minor version */ -#define CLIENT_VERSION_MINOR 17 +#define CLIENT_VERSION_MINOR 18 /* Build revision */ #define CLIENT_VERSION_REVISION 99 @@ -29,7 +29,7 @@ #define COPYRIGHT_HOLDERS_SUBSTITUTION "Bitcoin Core" /* Copyright year */ -#define COPYRIGHT_YEAR 2018 +#define COPYRIGHT_YEAR 2019 /* Define to 1 to enable wallet functions */ #define ENABLE_WALLET 1 diff --git a/configure.ac b/configure.ac index a3ba8ce808..854d6b1d49 100644 --- a/configure.ac +++ b/configure.ac @@ -195,8 +195,8 @@ AC_ARG_ENABLE([glibc-back-compat], [use_glibc_compat=no]) AC_ARG_ENABLE([asm], - [AS_HELP_STRING([--enable-asm], - [Enable assembly routines (default is yes)])], + [AS_HELP_STRING([--disable-asm], + [disable assembly routines (enabled by default)])], [use_asm=$enableval], [use_asm=yes]) @@ -339,6 +339,13 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-implicit-fallthrough"],,[[$CXXFLAG_WERROR]]) fi +enable_hwcrc32=no +enable_sse41=no +enable_avx2=no +enable_shani=no + +if test "x$use_asm" = "xyes"; then + # Check for optional instruction set support. Enabling these does _not_ imply that all code will # be compiled with them, rather that specific objects/libs may use them after checking for runtime # compatibility. @@ -416,6 +423,8 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ) CXXFLAGS="$TEMP_CXXFLAGS" +fi + CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO -D__STDC_FORMAT_MACROS" AC_ARG_WITH([utils], @@ -827,8 +836,23 @@ AC_LINK_IFELSE([AC_LANG_SOURCE([ } ])], [ - AC_DEFINE(HAVE_THREAD_LOCAL,1,[Define if thread_local is supported.]) - AC_MSG_RESULT(yes) + case $host in + *mingw*) + # mingw32's implementation of thread_local has also been shown to behave + # erroneously under concurrent usage; see: + # https://gist.github.com/jamesob/fe9a872051a88b2025b1aa37bfa98605 + AC_MSG_RESULT(no) + ;; + *darwin*) + # TODO enable thread_local on later versions of Darwin where it is + # supported (per https://stackoverflow.com/a/29929949) + AC_MSG_RESULT(no) + ;; + *) + AC_DEFINE(HAVE_THREAD_LOCAL,1,[Define if thread_local is supported.]) + AC_MSG_RESULT(yes) + ;; + esac ], [ AC_MSG_RESULT(no) diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index 41a640b464..a57ecf9818 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -47,17 +47,36 @@ def git_config_get(option, default=None): except subprocess.CalledProcessError: return default -def retrieve_json(req, ghtoken): +def get_response(req_url, ghtoken): + req = Request(req_url) + if ghtoken is not None: + req.add_header('Authorization', 'token ' + ghtoken) + return urlopen(req) + +def retrieve_json(req_url, ghtoken, use_pagination=False): ''' Retrieve json from github. Return None if an error happens. ''' try: - if ghtoken is not None: - req.add_header('Authorization', 'token ' + ghtoken) - result = urlopen(req) reader = codecs.getreader('utf-8') - obj = json.load(reader(result)) + if not use_pagination: + return json.load(reader(get_response(req_url, ghtoken))) + + obj = [] + page_num = 1 + while True: + req_url_page = '{}?page={}'.format(req_url, page_num) + result = get_response(req_url_page, ghtoken) + obj.extend(json.load(reader(result))) + + link = result.headers.get('link', None) + if link is not None: + link_next = [l for l in link.split(',') if 'rel="next"' in l] + if len(link_next) > 0: + page_num = int(link_next[0][link_next[0].find("page=")+5:link_next[0].find(">")]) + continue + break return obj except HTTPError as e: error_message = e.read() @@ -69,16 +88,16 @@ def retrieve_json(req, ghtoken): return None def retrieve_pr_info(repo,pull,ghtoken): - req = Request("https://api.github.com/repos/"+repo+"/pulls/"+pull) - return retrieve_json(req,ghtoken) + req_url = "https://api.github.com/repos/"+repo+"/pulls/"+pull + return retrieve_json(req_url,ghtoken) def retrieve_pr_comments(repo,pull,ghtoken): - req = Request("https://api.github.com/repos/"+repo+"/issues/"+pull+"/comments") - return retrieve_json(req,ghtoken) + req_url = "https://api.github.com/repos/"+repo+"/issues/"+pull+"/comments" + return retrieve_json(req_url,ghtoken,use_pagination=True) def retrieve_pr_reviews(repo,pull,ghtoken): - req = Request("https://api.github.com/repos/"+repo+"/pulls/"+pull+"/reviews") - return retrieve_json(req,ghtoken) + req_url = "https://api.github.com/repos/"+repo+"/pulls/"+pull+"/reviews" + return retrieve_json(req_url,ghtoken,use_pagination=True) def ask_prompt(text): print(text,end=" ",file=stderr) diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 62c764bb31..cf071167c4 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -620,8 +620,8 @@ class AddressBookPage Mode m_mode; } -AddressBookPage::AddressBookPage(Mode _mode) : - m_mode(_mode) +AddressBookPage::AddressBookPage(Mode _mode) + : m_mode(_mode) ... ``` diff --git a/doc/release-notes-15849.md b/doc/release-notes-15849.md new file mode 100644 index 0000000000..a1df31f250 --- /dev/null +++ b/doc/release-notes-15849.md @@ -0,0 +1,6 @@ +Thread names in logs +-------------------- + +On platforms supporting `thread_local`, log lines can be prefixed with the name +of the thread that caused the log. To enable this behavior, use +`-logthreadnames=1`. diff --git a/doc/release-notes.md b/doc/release-notes.md index d4acbb9c88..0de0f563b1 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -24,14 +24,9 @@ shut down (which might take a few minutes for older versions), then run the installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) or `bitcoind`/`bitcoin-qt` (on Linux). -The first time you run version 0.15.0, your chainstate database will be converted to a -new format, which will take anywhere from a few minutes to half an hour, -depending on the speed of your machine. - -Note that the block database format also changed in version 0.8.0 and there is no -automatic upgrade code from before version 0.8 to version 0.15.0. Upgrading -directly from 0.7.x and earlier without redownloading the blockchain is not supported. -However, as usual, old wallet versions are still supported. +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but might take some time if the datadir needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. Downgrading warning ------------------- diff --git a/doc/release-process.md b/doc/release-process.md index eb1f5ad222..7522310ce2 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -4,15 +4,14 @@ Release Process Before every release candidate: * Update translations (ping wumpus on IRC) see [translation_process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md#synchronising-translations). - * Update manpages, see [gen-manpages.sh](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-manpagessh). -* Update release candidate version in `configure.ac` (`CLIENT_VERSION_RC`) +* Update release candidate version in `configure.ac` (`CLIENT_VERSION_RC`). Before every minor and major release: * Update [bips.md](bips.md) to account for changes since the last release. -* Update version in `configure.ac` (don't forget to set `CLIENT_VERSION_IS_RELEASE` to `true`) (don't forget to set `CLIENT_VERSION_RC` to `0`) -* Write release notes (see below) +* Update version in `configure.ac` (don't forget to set `CLIENT_VERSION_RC` to `0`). +* Write release notes (see below). * Update `src/chainparams.cpp` nMinimumChainWork with information from the getblockchaininfo rpc. * Update `src/chainparams.cpp` defaultAssumeValid with information from the getblockhash rpc. - The selected value must not be orphaned so it may be useful to set the value two blocks back from the tip. @@ -26,7 +25,13 @@ Before every major release: * Update [`src/chainparams.cpp`](/src/chainparams.cpp) m_assumed_blockchain_size and m_assumed_chain_state_size with the current size plus some overhead. * Update `src/chainparams.cpp` chainTxData with statistics about the transaction count and rate. Use the output of the RPC `getchaintxstats`, see [this pull request](https://github.com/bitcoin/bitcoin/pull/12270) for an example. Reviewers can verify the results by running `getchaintxstats <window_block_count> <window_last_block_hash>` with the `window_block_count` and `window_last_block_hash` from your output. -* Update version of `contrib/gitian-descriptors/*.yml`: usually one'd want to do this on master after branching off the release - but be sure to at least do it before a new major release +* Update version of `contrib/gitian-descriptors/*.yml`: usually one'd want to do this on master after branching off the release - but be sure to at least do it before a new major release. +* In `configure.ac` and `build_msvc/bitcoin_config.h` on _the master branch_: + - update `CLIENT_VERSION_MINOR` version +* In `configure.ac` and `build_msvc/bitcoin_config.h` on _a new release branch_ (see [this commit](https://github.com/bitcoin/bitcoin/commit/742f7dd972fca3dd4a33cfff90bf901b71a687e7)): + - update `CLIENT_VERSION_MINOR` version + - set `CLIENT_VERSION_REVISION` to `0` + - set `CLIENT_VERSION_IS_RELEASE` to `true` ### First time / New builders diff --git a/src/Makefile.am b/src/Makefile.am index 859c17afbd..ed5cab7f04 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -113,7 +113,6 @@ BITCOIN_CORE_H = \ chainparams.h \ chainparamsbase.h \ chainparamsseeds.h \ - checkpoints.h \ checkqueue.h \ clientversion.h \ coins.h \ @@ -134,6 +133,7 @@ BITCOIN_CORE_H = \ httprpc.h \ httpserver.h \ index/base.h \ + index/blockfilterindex.h \ index/txindex.h \ indirectmap.h \ init.h \ @@ -209,6 +209,7 @@ BITCOIN_CORE_H = \ util/memory.h \ util/moneystr.h \ util/rbf.h \ + util/threadnames.h \ util/time.h \ util/url.h \ util/validation.h \ @@ -257,12 +258,12 @@ libbitcoin_server_a_SOURCES = \ blockencodings.cpp \ blockfilter.cpp \ chain.cpp \ - checkpoints.cpp \ consensus/tx_verify.cpp \ flatfile.cpp \ httprpc.cpp \ httpserver.cpp \ index/base.cpp \ + index/blockfilterindex.cpp \ index/txindex.cpp \ interfaces/chain.cpp \ interfaces/node.cpp \ @@ -489,6 +490,7 @@ libbitcoin_util_a_SOURCES = \ util/system.cpp \ util/moneystr.cpp \ util/rbf.cpp \ + util/threadnames.cpp \ util/strencodings.cpp \ util/time.cpp \ util/url.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index e248ef970f..1144ca8a78 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -92,6 +92,7 @@ BITCOIN_TESTS =\ test/blockchain_tests.cpp \ test/blockencodings_tests.cpp \ test/blockfilter_tests.cpp \ + test/blockfilter_index_tests.cpp \ test/bloom_tests.cpp \ test/bswap_tests.cpp \ test/checkqueue_tests.cpp \ @@ -137,6 +138,7 @@ BITCOIN_TESTS =\ test/skiplist_tests.cpp \ test/streams_tests.cpp \ test/sync_tests.cpp \ + test/util_threadnames_tests.cpp \ test/timedata_tests.cpp \ test/torcontrol_tests.cpp \ test/transaction_tests.cpp \ diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index dde75c1b12..b31f86cdd9 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -19,6 +19,7 @@ #include <util/system.h> #include <httpserver.h> #include <httprpc.h> +#include <util/threadnames.h> #include <util/strencodings.h> #include <walletinitinterface.h> @@ -64,6 +65,8 @@ static bool AppInit(int argc, char* argv[]) bool fRet = false; + util::ThreadRename("init"); + // // Parameters // diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index e15213c552..787390be31 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -2,6 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <mutex> +#include <sstream> + #include <blockfilter.h> #include <crypto/siphash.h> #include <hash.h> @@ -15,6 +18,10 @@ static constexpr int GCS_SER_TYPE = SER_NETWORK; /// Protocol version used to serialize parameters in GCS filter encoding. static constexpr int GCS_SER_VERSION = 0; +static const std::map<BlockFilterType, std::string> g_filter_types = { + {BlockFilterType::BASIC, "basic"}, +}; + template <typename OStream> static void GolombRiceEncode(BitStreamWriter<OStream>& bitwriter, uint8_t P, uint64_t x) { @@ -197,6 +204,57 @@ bool GCSFilter::MatchAny(const ElementSet& elements) const return MatchInternal(queries.data(), queries.size()); } +const std::string& BlockFilterTypeName(BlockFilterType filter_type) +{ + static std::string unknown_retval = ""; + auto it = g_filter_types.find(filter_type); + return it != g_filter_types.end() ? it->second : unknown_retval; +} + +bool BlockFilterTypeByName(const std::string& name, BlockFilterType& filter_type) { + for (const auto& entry : g_filter_types) { + if (entry.second == name) { + filter_type = entry.first; + return true; + } + } + return false; +} + +const std::vector<BlockFilterType>& AllBlockFilterTypes() +{ + static std::vector<BlockFilterType> types; + + static std::once_flag flag; + std::call_once(flag, []() { + types.reserve(g_filter_types.size()); + for (auto entry : g_filter_types) { + types.push_back(entry.first); + } + }); + + return types; +} + +const std::string& ListBlockFilterTypes() +{ + static std::string type_list; + + static std::once_flag flag; + std::call_once(flag, []() { + std::stringstream ret; + bool first = true; + for (auto entry : g_filter_types) { + if (!first) ret << ", "; + ret << entry.second; + first = false; + } + type_list = ret.str(); + }); + + return type_list; +} + static GCSFilter::ElementSet BasicFilterElements(const CBlock& block, const CBlockUndo& block_undo) { diff --git a/src/blockfilter.h b/src/blockfilter.h index e5e087ed5a..914b94fec1 100644 --- a/src/blockfilter.h +++ b/src/blockfilter.h @@ -6,6 +6,7 @@ #define BITCOIN_BLOCKFILTER_H #include <stdint.h> +#include <string> #include <unordered_set> #include <vector> @@ -89,6 +90,18 @@ enum class BlockFilterType : uint8_t INVALID = 255, }; +/** Get the human-readable name for a filter type. Returns empty string for unknown types. */ +const std::string& BlockFilterTypeName(BlockFilterType filter_type); + +/** Find a filter type by its human-readable name. */ +bool BlockFilterTypeByName(const std::string& name, BlockFilterType& filter_type); + +/** Get a list of known filter types. */ +const std::vector<BlockFilterType>& AllBlockFilterTypes(); + +/** Get a comma-separated list of known filter type names. */ +const std::string& ListBlockFilterTypes(); + /** * Complete block filter struct as defined in BIP 157. Serialization matches * payload of "cfilter" messages. diff --git a/src/chain.cpp b/src/chain.cpp index d462f94ab5..5520d8149a 100644 --- a/src/chain.cpp +++ b/src/chain.cpp @@ -59,10 +59,11 @@ const CBlockIndex *CChain::FindFork(const CBlockIndex *pindex) const { return pindex; } -CBlockIndex* CChain::FindEarliestAtLeast(int64_t nTime) const +CBlockIndex* CChain::FindEarliestAtLeast(int64_t nTime, int height) const { - std::vector<CBlockIndex*>::const_iterator lower = std::lower_bound(vChain.begin(), vChain.end(), nTime, - [](CBlockIndex* pBlock, const int64_t& time) -> bool { return pBlock->GetBlockTimeMax() < time; }); + std::pair<int64_t, int> blockparams = std::make_pair(nTime, height); + std::vector<CBlockIndex*>::const_iterator lower = std::lower_bound(vChain.begin(), vChain.end(), blockparams, + [](CBlockIndex* pBlock, const std::pair<int64_t, int>& blockparams) -> bool { return pBlock->GetBlockTimeMax() < blockparams.first || pBlock->nHeight < blockparams.second; }); return (lower == vChain.end() ? nullptr : *lower); } diff --git a/src/chain.h b/src/chain.h index 2b6d2d082c..dd9cc2a598 100644 --- a/src/chain.h +++ b/src/chain.h @@ -465,8 +465,8 @@ public: /** Find the last common block between this chain and a block index entry. */ const CBlockIndex *FindFork(const CBlockIndex *pindex) const; - /** Find the earliest block with timestamp equal or greater than the given. */ - CBlockIndex* FindEarliestAtLeast(int64_t nTime) const; + /** Find the earliest block with timestamp equal or greater than the given time and height equal or greater than the given height. */ + CBlockIndex* FindEarliestAtLeast(int64_t nTime, int height) const; }; #endif // BITCOIN_CHAIN_H diff --git a/src/checkpoints.cpp b/src/checkpoints.cpp deleted file mode 100644 index ad5edfeb39..0000000000 --- a/src/checkpoints.cpp +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <checkpoints.h> - -#include <chain.h> -#include <chainparams.h> -#include <reverse_iterator.h> -#include <validation.h> - -#include <stdint.h> - - -namespace Checkpoints { - - CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) - { - const MapCheckpoints& checkpoints = data.mapCheckpoints; - - for (const MapCheckpoints::value_type& i : reverse_iterate(checkpoints)) - { - const uint256& hash = i.second; - CBlockIndex* pindex = LookupBlockIndex(hash); - if (pindex) { - return pindex; - } - } - return nullptr; - } - -} // namespace Checkpoints diff --git a/src/checkpoints.h b/src/checkpoints.h deleted file mode 100644 index a25e97e469..0000000000 --- a/src/checkpoints.h +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) 2009-2018 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_CHECKPOINTS_H -#define BITCOIN_CHECKPOINTS_H - -#include <uint256.h> - -#include <map> - -class CBlockIndex; -struct CCheckpointData; - -/** - * Block-chain checkpoints are compiled-in sanity checks. - * They are updated every release or three. - */ -namespace Checkpoints -{ - -//! Returns last CBlockIndex* that is a checkpoint -CBlockIndex* GetLastCheckpoint(const CCheckpointData& data); - -} //namespace Checkpoints - -#endif // BITCOIN_CHECKPOINTS_H diff --git a/src/coins.h b/src/coins.h index d39ebf9062..482e233e8c 100644 --- a/src/coins.h +++ b/src/coins.h @@ -294,6 +294,10 @@ public: bool HaveInputs(const CTransaction& tx) const; private: + /** + * @note this is marked const, but may actually append to `cacheCoins`, increasing + * memory usage. + */ CCoinsMap::iterator FetchCoin(const COutPoint &outpoint) const; }; diff --git a/src/dummywallet.cpp b/src/dummywallet.cpp index 8a76021a5b..eeec6dec25 100644 --- a/src/dummywallet.cpp +++ b/src/dummywallet.cpp @@ -23,11 +23,33 @@ public: void DummyWalletInit::AddWalletOptions() const { - std::vector<std::string> opts = {"-addresstype", "-changetype", "-disablewallet", "-discardfee=<amt>", "-fallbackfee=<amt>", - "-keypool=<n>", "-mintxfee=<amt>", "-paytxfee=<amt>", "-rescan", "-salvagewallet", "-spendzeroconfchange", "-txconfirmtarget=<n>", - "-upgradewallet", "-wallet=<path>", "-walletbroadcast", "-walletdir=<dir>", "-walletnotify=<cmd>", "-walletrbf", "-zapwallettxes=<mode>", - "-dblogsize=<n>", "-flushwallet", "-privdb", "-walletrejectlongchains"}; - gArgs.AddHiddenArgs(opts); + gArgs.AddHiddenArgs({ + "-addresstype", + "-avoidpartialspends", + "-changetype", + "-disablewallet", + "-discardfee=<amt>", + "-fallbackfee=<amt>", + "-keypool=<n>", + "-maxtxfee=<amt>", + "-mintxfee=<amt>", + "-paytxfee=<amt>", + "-rescan", + "-salvagewallet", + "-spendzeroconfchange", + "-txconfirmtarget=<n>", + "-upgradewallet", + "-wallet=<path>", + "-walletbroadcast", + "-walletdir=<dir>", + "-walletnotify=<cmd>", + "-walletrbf", + "-zapwallettxes=<mode>", + "-dblogsize=<n>", + "-flushwallet", + "-privdb", + "-walletrejectlongchains", + }); } const WalletInitInterface& g_wallet_init_interface = DummyWalletInit(); diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 5d9c3d2c1a..63639fa3e0 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -6,6 +6,7 @@ #include <chainparamsbase.h> #include <compat.h> +#include <util/threadnames.h> #include <util/system.h> #include <util/strencodings.h> #include <netbase.h> @@ -17,7 +18,7 @@ #include <memory> #include <stdio.h> #include <stdlib.h> -#include <string.h> +#include <string> #include <sys/types.h> #include <sys/stat.h> @@ -284,7 +285,7 @@ static void http_reject_request_cb(struct evhttp_request* req, void*) /** Event dispatcher thread */ static bool ThreadHTTP(struct event_base* base) { - RenameThread("bitcoin-http"); + util::ThreadRename("http"); LogPrint(BCLog::HTTP, "Entering http event loop\n"); event_base_dispatch(base); // Event loop will be interrupted by InterruptHTTPServer() @@ -335,9 +336,9 @@ static bool HTTPBindAddresses(struct evhttp* http) } /** Simple wrapper to set thread name and run work queue */ -static void HTTPWorkQueueRun(WorkQueue<HTTPClosure>* queue) +static void HTTPWorkQueueRun(WorkQueue<HTTPClosure>* queue, int worker_num) { - RenameThread("bitcoin-httpworker"); + util::ThreadRename(strprintf("httpworker.%i", worker_num)); queue->Run(); } @@ -430,7 +431,7 @@ void StartHTTPServer() threadHTTP = std::thread(ThreadHTTP, eventBase); for (int i = 0; i < rpcThreads; i++) { - g_thread_http_workers.emplace_back(HTTPWorkQueueRun, workQueue); + g_thread_http_workers.emplace_back(HTTPWorkQueueRun, workQueue, i); } } diff --git a/src/index/base.cpp b/src/index/base.cpp index f6f59572ce..9e48f0bd27 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -41,9 +41,9 @@ bool BaseIndex::DB::ReadBestBlock(CBlockLocator& locator) const return success; } -bool BaseIndex::DB::WriteBestBlock(const CBlockLocator& locator) +void BaseIndex::DB::WriteBestBlock(CDBBatch& batch, const CBlockLocator& locator) { - return Write(DB_BEST_BLOCK, locator); + batch.Write(DB_BEST_BLOCK, locator); } BaseIndex::~BaseIndex() @@ -95,7 +95,11 @@ void BaseIndex::ThreadSync() int64_t last_locator_write_time = 0; while (true) { if (m_interrupt) { - WriteBestBlock(pindex); + m_best_block_index = pindex; + // No need to handle errors in Commit. If it fails, the error will be already be + // logged. The best way to recover is to continue, as index cannot be corrupted by + // a missed commit to disk for an advanced index state. + Commit(); return; } @@ -103,11 +107,17 @@ void BaseIndex::ThreadSync() LOCK(cs_main); const CBlockIndex* pindex_next = NextSyncBlock(pindex); if (!pindex_next) { - WriteBestBlock(pindex); m_best_block_index = pindex; m_synced = true; + // No need to handle errors in Commit. See rationale above. + Commit(); break; } + if (pindex_next->pprev != pindex && !Rewind(pindex, pindex_next->pprev)) { + FatalError("%s: Failed to rewind index %s to a previous chain tip", + __func__, GetName()); + return; + } pindex = pindex_next; } @@ -119,8 +129,10 @@ void BaseIndex::ThreadSync() } if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) { - WriteBestBlock(pindex); + m_best_block_index = pindex; last_locator_write_time = current_time; + // No need to handle errors in Commit. See rationale above. + Commit(); } CBlock block; @@ -144,12 +156,35 @@ void BaseIndex::ThreadSync() } } -bool BaseIndex::WriteBestBlock(const CBlockIndex* block_index) +bool BaseIndex::Commit() +{ + CDBBatch batch(GetDB()); + if (!CommitInternal(batch) || !GetDB().WriteBatch(batch)) { + return error("%s: Failed to commit latest %s state", __func__, GetName()); + } + return true; +} + +bool BaseIndex::CommitInternal(CDBBatch& batch) { LOCK(cs_main); - if (!GetDB().WriteBestBlock(chainActive.GetLocator(block_index))) { - return error("%s: Failed to write locator to disk", __func__); + GetDB().WriteBestBlock(batch, chainActive.GetLocator(m_best_block_index)); + return true; +} + +bool BaseIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) +{ + assert(current_tip == m_best_block_index); + assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); + + // In the case of a reorg, ensure persisted block locator is not stale. + m_best_block_index = new_tip; + if (!Commit()) { + // If commit fails, revert the best block index to avoid corruption. + m_best_block_index = current_tip; + return false; } + return true; } @@ -180,6 +215,11 @@ void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const best_block_index->GetBlockHash().ToString()); return; } + if (best_block_index != pindex->pprev && !Rewind(best_block_index, pindex->pprev)) { + FatalError("%s: Failed to rewind index %s to a previous chain tip", + __func__, GetName()); + return; + } } if (WriteBlock(*block, pindex)) { @@ -224,9 +264,10 @@ void BaseIndex::ChainStateFlushed(const CBlockLocator& locator) return; } - if (!GetDB().WriteBestBlock(locator)) { - error("%s: Failed to write locator to disk", __func__); - } + // No need to handle errors in Commit. If it fails, the error will be already be logged. The + // best way to recover is to continue, as index cannot be corrupted by a missed commit to disk + // for an advanced index state. + Commit(); } bool BaseIndex::BlockUntilSyncedToCurrentChain() diff --git a/src/index/base.h b/src/index/base.h index 04ee6e6cc2..31acbed0c1 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -32,7 +32,7 @@ protected: bool ReadBestBlock(CBlockLocator& locator) const; /// Write block locator of the chain that the txindex is in sync with. - bool WriteBestBlock(const CBlockLocator& locator); + void WriteBestBlock(CDBBatch& batch, const CBlockLocator& locator); }; private: @@ -54,8 +54,15 @@ private: /// over and the sync thread exits. void ThreadSync(); - /// Write the current chain block locator to the DB. - bool WriteBestBlock(const CBlockIndex* block_index); + /// Write the current index state (eg. chain block locator and subclass-specific items) to disk. + /// + /// Recommendations for error handling: + /// If called on a successor of the previous committed best block in the index, the index can + /// continue processing without risk of corruption, though the index state will need to catch up + /// from further behind on reboot. If the new state is not a successor of the previous state (due + /// to a chain reorganization), the index must halt until Commit succeeds or else it could end up + /// getting corrupted. + bool Commit(); protected: void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex, @@ -69,6 +76,14 @@ protected: /// Write update index entries for a newly connected block. virtual bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) { return true; } + /// Virtual method called internally by Commit that can be overridden to atomically + /// commit more index state. + virtual bool CommitInternal(CDBBatch& batch); + + /// Rewind index to an earlier chain tip during a chain reorg. The tip must + /// be an ancestor of the current best block. + virtual bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip); + virtual DB& GetDB() const = 0; /// Get the name of the index for display in logs. diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp new file mode 100644 index 0000000000..20f33baf2c --- /dev/null +++ b/src/index/blockfilterindex.cpp @@ -0,0 +1,467 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <map> + +#include <dbwrapper.h> +#include <index/blockfilterindex.h> +#include <util/system.h> +#include <validation.h> + +/* The index database stores three items for each block: the disk location of the encoded filter, + * its dSHA256 hash, and the header. Those belonging to blocks on the active chain are indexed by + * height, and those belonging to blocks that have been reorganized out of the active chain are + * indexed by block hash. This ensures that filter data for any block that becomes part of the + * active chain can always be retrieved, alleviating timing concerns. + * + * The filters themselves are stored in flat files and referenced by the LevelDB entries. This + * minimizes the amount of data written to LevelDB and keeps the database values constant size. The + * disk location of the next block filter to be written (represented as a FlatFilePos) is stored + * under the DB_FILTER_POS key. + * + * Keys for the height index have the type [DB_BLOCK_HEIGHT, uint32 (BE)]. The height is represented + * as big-endian so that sequential reads of filters by height are fast. + * Keys for the hash index have the type [DB_BLOCK_HASH, uint256]. + */ +constexpr char DB_BLOCK_HASH = 's'; +constexpr char DB_BLOCK_HEIGHT = 't'; +constexpr char DB_FILTER_POS = 'P'; + +constexpr unsigned int MAX_FLTR_FILE_SIZE = 0x1000000; // 16 MiB +/** The pre-allocation chunk size for fltr?????.dat files */ +constexpr unsigned int FLTR_FILE_CHUNK_SIZE = 0x100000; // 1 MiB + +namespace { + +struct DBVal { + uint256 hash; + uint256 header; + FlatFilePos pos; + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITE(hash); + READWRITE(header); + READWRITE(pos); + } +}; + +struct DBHeightKey { + int height; + + DBHeightKey() : height(0) {} + DBHeightKey(int height_in) : height(height_in) {} + + template<typename Stream> + void Serialize(Stream& s) const + { + ser_writedata8(s, DB_BLOCK_HEIGHT); + ser_writedata32be(s, height); + } + + template<typename Stream> + void Unserialize(Stream& s) + { + char prefix = ser_readdata8(s); + if (prefix != DB_BLOCK_HEIGHT) { + throw std::ios_base::failure("Invalid format for block filter index DB height key"); + } + height = ser_readdata32be(s); + } +}; + +struct DBHashKey { + uint256 hash; + + DBHashKey(const uint256& hash_in) : hash(hash_in) {} + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + char prefix = DB_BLOCK_HASH; + READWRITE(prefix); + if (prefix != DB_BLOCK_HASH) { + throw std::ios_base::failure("Invalid format for block filter index DB hash key"); + } + + READWRITE(hash); + } +}; + +}; // namespace + +static std::map<BlockFilterType, BlockFilterIndex> g_filter_indexes; + +BlockFilterIndex::BlockFilterIndex(BlockFilterType filter_type, + size_t n_cache_size, bool f_memory, bool f_wipe) + : m_filter_type(filter_type) +{ + const std::string& filter_name = BlockFilterTypeName(filter_type); + if (filter_name.empty()) throw std::invalid_argument("unknown filter_type"); + + fs::path path = GetDataDir() / "indexes" / "blockfilter" / filter_name; + fs::create_directories(path); + + m_name = filter_name + " block filter index"; + m_db = MakeUnique<BaseIndex::DB>(path / "db", n_cache_size, f_memory, f_wipe); + m_filter_fileseq = MakeUnique<FlatFileSeq>(std::move(path), "fltr", FLTR_FILE_CHUNK_SIZE); +} + +bool BlockFilterIndex::Init() +{ + if (!m_db->Read(DB_FILTER_POS, m_next_filter_pos)) { + // Check that the cause of the read failure is that the key does not exist. Any other errors + // indicate database corruption or a disk failure, and starting the index would cause + // further corruption. + if (m_db->Exists(DB_FILTER_POS)) { + return error("%s: Cannot read current %s state; index may be corrupted", + __func__, GetName()); + } + + // If the DB_FILTER_POS is not set, then initialize to the first location. + m_next_filter_pos.nFile = 0; + m_next_filter_pos.nPos = 0; + } + return BaseIndex::Init(); +} + +bool BlockFilterIndex::CommitInternal(CDBBatch& batch) +{ + const FlatFilePos& pos = m_next_filter_pos; + + // Flush current filter file to disk. + CAutoFile file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + return error("%s: Failed to open filter file %d", __func__, pos.nFile); + } + if (!FileCommit(file.Get())) { + return error("%s: Failed to commit filter file %d", __func__, pos.nFile); + } + + batch.Write(DB_FILTER_POS, pos); + return BaseIndex::CommitInternal(batch); +} + +bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const +{ + CAutoFile filein(m_filter_fileseq->Open(pos, true), SER_DISK, CLIENT_VERSION); + if (filein.IsNull()) { + return false; + } + + uint256 block_hash; + std::vector<unsigned char> encoded_filter; + try { + filein >> block_hash >> encoded_filter; + filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter)); + } + catch (const std::exception& e) { + return error("%s: Failed to deserialize block filter from disk: %s", __func__, e.what()); + } + + return true; +} + +size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter) +{ + assert(filter.GetFilterType() == GetFilterType()); + + size_t data_size = + GetSerializeSize(filter.GetBlockHash(), CLIENT_VERSION) + + GetSerializeSize(filter.GetEncodedFilter(), CLIENT_VERSION); + + // If writing the filter would overflow the file, flush and move to the next one. + if (pos.nPos + data_size > MAX_FLTR_FILE_SIZE) { + CAutoFile last_file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION); + if (last_file.IsNull()) { + LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile); + return 0; + } + if (!TruncateFile(last_file.Get(), pos.nPos)) { + LogPrintf("%s: Failed to truncate filter file %d\n", __func__, pos.nFile); + return 0; + } + if (!FileCommit(last_file.Get())) { + LogPrintf("%s: Failed to commit filter file %d\n", __func__, pos.nFile); + return 0; + } + + pos.nFile++; + pos.nPos = 0; + } + + // Pre-allocate sufficient space for filter data. + bool out_of_space; + m_filter_fileseq->Allocate(pos, data_size, out_of_space); + if (out_of_space) { + LogPrintf("%s: out of disk space\n", __func__); + return 0; + } + + CAutoFile fileout(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION); + if (fileout.IsNull()) { + LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile); + return 0; + } + + fileout << filter.GetBlockHash() << filter.GetEncodedFilter(); + return data_size; +} + +bool BlockFilterIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) +{ + CBlockUndo block_undo; + uint256 prev_header; + + if (pindex->nHeight > 0) { + if (!UndoReadFromDisk(block_undo, pindex)) { + return false; + } + + std::pair<uint256, DBVal> read_out; + if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { + return false; + } + + uint256 expected_block_hash = pindex->pprev->GetBlockHash(); + if (read_out.first != expected_block_hash) { + return error("%s: previous block header belongs to unexpected block %s; expected %s", + __func__, read_out.first.ToString(), expected_block_hash.ToString()); + } + + prev_header = read_out.second.header; + } + + BlockFilter filter(m_filter_type, block, block_undo); + + size_t bytes_written = WriteFilterToDisk(m_next_filter_pos, filter); + if (bytes_written == 0) return false; + + std::pair<uint256, DBVal> value; + value.first = pindex->GetBlockHash(); + value.second.hash = filter.GetHash(); + value.second.header = filter.ComputeHeader(prev_header); + value.second.pos = m_next_filter_pos; + + if (!m_db->Write(DBHeightKey(pindex->nHeight), value)) { + return false; + } + + m_next_filter_pos.nPos += bytes_written; + return true; +} + +static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch, + const std::string& index_name, + int start_height, int stop_height) +{ + DBHeightKey key(start_height); + db_it.Seek(key); + + for (int height = start_height; height <= stop_height; ++height) { + if (!db_it.GetKey(key) || key.height != height) { + return error("%s: unexpected key in %s: expected (%c, %d)", + __func__, index_name, DB_BLOCK_HEIGHT, height); + } + + std::pair<uint256, DBVal> value; + if (!db_it.GetValue(value)) { + return error("%s: unable to read value in %s at key (%c, %d)", + __func__, index_name, DB_BLOCK_HEIGHT, height); + } + + batch.Write(DBHashKey(value.first), std::move(value.second)); + + db_it.Next(); + } + return true; +} + +bool BlockFilterIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) +{ + assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); + + CDBBatch batch(*m_db); + std::unique_ptr<CDBIterator> db_it(m_db->NewIterator()); + + // During a reorg, we need to copy all filters for blocks that are getting disconnected from the + // height index to the hash index so we can still find them when the height index entries are + // overwritten. + if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight, current_tip->nHeight)) { + return false; + } + + // The latest filter position gets written in Commit by the call to the BaseIndex::Rewind. + // But since this creates new references to the filter, the position should get updated here + // atomically as well in case Commit fails. + batch.Write(DB_FILTER_POS, m_next_filter_pos); + if (!m_db->WriteBatch(batch)) return false; + + return BaseIndex::Rewind(current_tip, new_tip); +} + +static bool LookupOne(const CDBWrapper& db, const CBlockIndex* block_index, DBVal& result) +{ + // First check if the result is stored under the height index and the value there matches the + // block hash. This should be the case if the block is on the active chain. + std::pair<uint256, DBVal> read_out; + if (!db.Read(DBHeightKey(block_index->nHeight), read_out)) { + return false; + } + if (read_out.first == block_index->GetBlockHash()) { + result = std::move(read_out.second); + return true; + } + + // If value at the height index corresponds to an different block, the result will be stored in + // the hash index. + return db.Read(DBHashKey(block_index->GetBlockHash()), result); +} + +static bool LookupRange(CDBWrapper& db, const std::string& index_name, int start_height, + const CBlockIndex* stop_index, std::vector<DBVal>& results) +{ + if (start_height < 0) { + return error("%s: start height (%d) is negative", __func__, start_height); + } + if (start_height > stop_index->nHeight) { + return error("%s: start height (%d) is greater than stop height (%d)", + __func__, start_height, stop_index->nHeight); + } + + size_t results_size = static_cast<size_t>(stop_index->nHeight - start_height + 1); + std::vector<std::pair<uint256, DBVal>> values(results_size); + + DBHeightKey key(start_height); + std::unique_ptr<CDBIterator> db_it(db.NewIterator()); + db_it->Seek(DBHeightKey(start_height)); + for (int height = start_height; height <= stop_index->nHeight; ++height) { + if (!db_it->Valid() || !db_it->GetKey(key) || key.height != height) { + return false; + } + + size_t i = static_cast<size_t>(height - start_height); + if (!db_it->GetValue(values[i])) { + return error("%s: unable to read value in %s at key (%c, %d)", + __func__, index_name, DB_BLOCK_HEIGHT, height); + } + + db_it->Next(); + } + + results.resize(results_size); + + // Iterate backwards through block indexes collecting results in order to access the block hash + // of each entry in case we need to look it up in the hash index. + for (const CBlockIndex* block_index = stop_index; + block_index && block_index->nHeight >= start_height; + block_index = block_index->pprev) { + uint256 block_hash = block_index->GetBlockHash(); + + size_t i = static_cast<size_t>(block_index->nHeight - start_height); + if (block_hash == values[i].first) { + results[i] = std::move(values[i].second); + continue; + } + + if (!db.Read(DBHashKey(block_hash), results[i])) { + return error("%s: unable to read value in %s at key (%c, %s)", + __func__, index_name, DB_BLOCK_HASH, block_hash.ToString()); + } + } + + return true; +} + +bool BlockFilterIndex::LookupFilter(const CBlockIndex* block_index, BlockFilter& filter_out) const +{ + DBVal entry; + if (!LookupOne(*m_db, block_index, entry)) { + return false; + } + + return ReadFilterFromDisk(entry.pos, filter_out); +} + +bool BlockFilterIndex::LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) const +{ + DBVal entry; + if (!LookupOne(*m_db, block_index, entry)) { + return false; + } + + header_out = entry.header; + return true; +} + +bool BlockFilterIndex::LookupFilterRange(int start_height, const CBlockIndex* stop_index, + std::vector<BlockFilter>& filters_out) const +{ + std::vector<DBVal> entries; + if (!LookupRange(*m_db, m_name, start_height, stop_index, entries)) { + return false; + } + + filters_out.resize(entries.size()); + auto filter_pos_it = filters_out.begin(); + for (const auto& entry : entries) { + if (!ReadFilterFromDisk(entry.pos, *filter_pos_it)) { + return false; + } + ++filter_pos_it; + } + + return true; +} + +bool BlockFilterIndex::LookupFilterHashRange(int start_height, const CBlockIndex* stop_index, + std::vector<uint256>& hashes_out) const + +{ + std::vector<DBVal> entries; + if (!LookupRange(*m_db, m_name, start_height, stop_index, entries)) { + return false; + } + + hashes_out.clear(); + hashes_out.reserve(entries.size()); + for (const auto& entry : entries) { + hashes_out.push_back(entry.hash); + } + return true; +} + +BlockFilterIndex* GetBlockFilterIndex(BlockFilterType filter_type) +{ + auto it = g_filter_indexes.find(filter_type); + return it != g_filter_indexes.end() ? &it->second : nullptr; +} + +void ForEachBlockFilterIndex(std::function<void (BlockFilterIndex&)> fn) +{ + for (auto& entry : g_filter_indexes) fn(entry.second); +} + +bool InitBlockFilterIndex(BlockFilterType filter_type, + size_t n_cache_size, bool f_memory, bool f_wipe) +{ + auto result = g_filter_indexes.emplace(std::piecewise_construct, + std::forward_as_tuple(filter_type), + std::forward_as_tuple(filter_type, + n_cache_size, f_memory, f_wipe)); + return result.second; +} + +bool DestroyBlockFilterIndex(BlockFilterType filter_type) +{ + return g_filter_indexes.erase(filter_type); +} + +void DestroyAllBlockFilterIndexes() +{ + g_filter_indexes.clear(); +} diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h new file mode 100644 index 0000000000..436d52515f --- /dev/null +++ b/src/index/blockfilterindex.h @@ -0,0 +1,94 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_INDEX_BLOCKFILTERINDEX_H +#define BITCOIN_INDEX_BLOCKFILTERINDEX_H + +#include <blockfilter.h> +#include <chain.h> +#include <flatfile.h> +#include <index/base.h> + +/** + * BlockFilterIndex is used to store and retrieve block filters, hashes, and headers for a range of + * blocks by height. An index is constructed for each supported filter type with its own database + * (ie. filter data for different types are stored in separate databases). + * + * This index is used to serve BIP 157 net requests. + */ +class BlockFilterIndex final : public BaseIndex +{ +private: + BlockFilterType m_filter_type; + std::string m_name; + std::unique_ptr<BaseIndex::DB> m_db; + + FlatFilePos m_next_filter_pos; + std::unique_ptr<FlatFileSeq> m_filter_fileseq; + + bool ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const; + size_t WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter); + +protected: + bool Init() override; + + bool CommitInternal(CDBBatch& batch) override; + + bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; + + bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) override; + + BaseIndex::DB& GetDB() const override { return *m_db; } + + const char* GetName() const override { return m_name.c_str(); } + +public: + /** Constructs the index, which becomes available to be queried. */ + explicit BlockFilterIndex(BlockFilterType filter_type, + size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + + BlockFilterType GetFilterType() const { return m_filter_type; } + + /** Get a single filter by block. */ + bool LookupFilter(const CBlockIndex* block_index, BlockFilter& filter_out) const; + + /** Get a single filter header by block. */ + bool LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) const; + + /** Get a range of filters between two heights on a chain. */ + bool LookupFilterRange(int start_height, const CBlockIndex* stop_index, + std::vector<BlockFilter>& filters_out) const; + + /** Get a range of filter hashes between two heights on a chain. */ + bool LookupFilterHashRange(int start_height, const CBlockIndex* stop_index, + std::vector<uint256>& hashes_out) const; +}; + +/** + * Get a block filter index by type. Returns nullptr if index has not been initialized or was + * already destroyed. + */ +BlockFilterIndex* GetBlockFilterIndex(BlockFilterType filter_type); + +/** Iterate over all running block filter indexes, invoking fn on each. */ +void ForEachBlockFilterIndex(std::function<void (BlockFilterIndex&)> fn); + +/** + * Initialize a block filter index for the given type if one does not already exist. Returns true if + * a new index is created and false if one has already been initialized. + */ +bool InitBlockFilterIndex(BlockFilterType filter_type, + size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + +/** + * Destroy the block filter index with the given type. Returns false if no such index exists. This + * just releases the allocated memory and closes the database connection, it does not delete the + * index data. + */ +bool DestroyBlockFilterIndex(BlockFilterType filter_type); + +/** Destroy all open block filter indexes. */ +void DestroyAllBlockFilterIndexes(); + +#endif // BITCOIN_INDEX_BLOCKFILTERINDEX_H diff --git a/src/init.cpp b/src/init.cpp index 2b538fb836..2272624f8a 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -12,14 +12,15 @@ #include <addrman.h> #include <amount.h> #include <banman.h> +#include <blockfilter.h> #include <chain.h> #include <chainparams.h> -#include <checkpoints.h> #include <compat/sanity.h> #include <consensus/validation.h> #include <fs.h> #include <httpserver.h> #include <httprpc.h> +#include <index/blockfilterindex.h> #include <interfaces/chain.h> #include <index/txindex.h> #include <key.h> @@ -40,6 +41,7 @@ #include <script/sigcache.h> #include <scheduler.h> #include <shutdown.h> +#include <util/threadnames.h> #include <timedata.h> #include <txdb.h> #include <txmempool.h> @@ -65,7 +67,6 @@ #include <boost/algorithm/string/replace.hpp> #include <boost/algorithm/string/split.hpp> #include <boost/thread.hpp> -#include <openssl/crypto.h> #if ENABLE_ZMQ #include <zmq/zmqabstractnotifier.h> @@ -191,6 +192,7 @@ void Interrupt() if (g_txindex) { g_txindex->Interrupt(); } + ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Interrupt(); }); } void Shutdown(InitInterfaces& interfaces) @@ -205,7 +207,7 @@ void Shutdown(InitInterfaces& interfaces) /// for example if the data directory was found to be locked. /// Be sure that anything that writes files or flushes caches only does this if the respective /// module was initialized. - RenameThread("bitcoin-shutoff"); + util::ThreadRename("shutoff"); mempool.AddTransactionsUpdated(1); StopHTTPRPC(); @@ -222,6 +224,7 @@ void Shutdown(InitInterfaces& interfaces) if (peerLogic) UnregisterValidationInterface(peerLogic.get()); if (g_connman) g_connman->Stop(); if (g_txindex) g_txindex->Stop(); + ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Stop(); }); StopTorControl(); @@ -236,6 +239,7 @@ void Shutdown(InitInterfaces& interfaces) g_connman.reset(); g_banman.reset(); g_txindex.reset(); + DestroyAllBlockFilterIndexes(); if (g_is_mempool_loaded && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { DumpMempool(); @@ -341,14 +345,15 @@ static void registerSignalHandler(int signal, void(*handler)(int)) } #endif +static boost::signals2::connection rpc_notify_block_change_connection; static void OnRPCStarted() { - uiInterface.NotifyBlockTip_connect(&RPCNotifyBlockChange); + rpc_notify_block_change_connection = uiInterface.NotifyBlockTip_connect(&RPCNotifyBlockChange); } static void OnRPCStopped() { - uiInterface.NotifyBlockTip_disconnect(&RPCNotifyBlockChange); + rpc_notify_block_change_connection.disconnect(); RPCNotifyBlockChange(false, nullptr); g_best_block_cv.notify_all(); LogPrint(BCLog::RPC, "RPC stopped.\n"); @@ -406,6 +411,10 @@ void SetupServerArgs() hidden_args.emplace_back("-sysperms"); #endif gArgs.AddArg("-txindex", strprintf("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)", DEFAULT_TXINDEX), false, OptionsCategory::OPTIONS); + gArgs.AddArg("-blockfilterindex=<type>", + strprintf("Maintain an index of compact filters by block (default: %s, values: %s).", DEFAULT_BLOCKFILTERINDEX, ListBlockFilterTypes()) + + " If <type> is not supplied or if <type> = 1, indexes for all known types are enabled.", + false, OptionsCategory::OPTIONS); gArgs.AddArg("-addnode=<ip>", "Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info). This option can be specified multiple times to add multiple nodes.", false, OptionsCategory::CONNECTION); gArgs.AddArg("-banscore=<n>", strprintf("Threshold for disconnecting misbehaving peers (default: %u)", DEFAULT_BANSCORE_THRESHOLD), false, OptionsCategory::CONNECTION); @@ -498,12 +507,11 @@ void SetupServerArgs() gArgs.AddArg("-debugexclude=<category>", strprintf("Exclude debugging information for a category. Can be used in conjunction with -debug=1 to output debug logs for all categories except one or more specified categories."), false, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-logips", strprintf("Include IP addresses in debug output (default: %u)", DEFAULT_LOGIPS), false, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-logtimestamps", strprintf("Prepend debug output with timestamp (default: %u)", DEFAULT_LOGTIMESTAMPS), false, OptionsCategory::DEBUG_TEST); + gArgs.AddArg("-logthreadnames", strprintf("Prepend debug output with name of the originating thread (only available on platforms supporting thread_local) (default: %u)", DEFAULT_LOGTHREADNAMES), false, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-logtimemicros", strprintf("Add microsecond precision to debug timestamps (default: %u)", DEFAULT_LOGTIMEMICROS), true, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-mocktime=<n>", "Replace actual time with <n> seconds since epoch (default: 0)", true, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-maxsigcachesize=<n>", strprintf("Limit sum of signature cache and script execution cache sizes to <n> MiB (default: %u)", DEFAULT_MAX_SIG_CACHE_SIZE), true, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-maxtipage=<n>", strprintf("Maximum tip age in seconds to consider node in initial block download (default: %u)", DEFAULT_MAX_TIP_AGE), true, OptionsCategory::DEBUG_TEST); - gArgs.AddArg("-maxtxfee=<amt>", strprintf("Maximum total fees (in %s) to use in a single wallet transaction; setting this too low may abort large transactions (default: %s)", // TODO move setting to wallet - CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MAXFEE)), false, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-printpriority", strprintf("Log transaction fee per kB when mining blocks (default: %u)", DEFAULT_PRINTPRIORITY), true, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -daemon. To disable logging to file, set -nodebuglogfile)", false, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-shrinkdebugfile", "Shrink debug.log file on client startup (default: 1 when no -debug)", false, OptionsCategory::DEBUG_TEST); @@ -513,7 +521,7 @@ void SetupServerArgs() gArgs.AddArg("-acceptnonstdtxn", strprintf("Relay and mine \"non-standard\" transactions (%sdefault: %u)", "testnet/regtest only; ", !testnetChainParams->RequireStandard()), true, OptionsCategory::NODE_RELAY); gArgs.AddArg("-incrementalrelayfee=<amt>", strprintf("Fee rate (in %s/kB) used to define cost of relay, used for mempool limiting and BIP 125 replacement. (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_INCREMENTAL_RELAY_FEE)), true, OptionsCategory::NODE_RELAY); - gArgs.AddArg("-dustrelayfee=<amt>", strprintf("Fee rate (in %s/kB) used to defined dust, the value of an output such that it will cost more than its value in fees at this fee rate to spend it. (default: %s)", CURRENCY_UNIT, FormatMoney(DUST_RELAY_TX_FEE)), true, OptionsCategory::NODE_RELAY); + gArgs.AddArg("-dustrelayfee=<amt>", strprintf("Fee rate (in %s/kB) used to define dust, the value of an output such that it will cost more than its value in fees at this fee rate to spend it. (default: %s)", CURRENCY_UNIT, FormatMoney(DUST_RELAY_TX_FEE)), true, OptionsCategory::NODE_RELAY); gArgs.AddArg("-bytespersigop", strprintf("Equivalent bytes per sigop in transactions for relay and mining (default: %u)", DEFAULT_BYTES_PER_SIGOP), false, OptionsCategory::NODE_RELAY); gArgs.AddArg("-datacarrier", strprintf("Relay and mine data carrier transactions (default: %u)", DEFAULT_ACCEPT_DATACARRIER), false, OptionsCategory::NODE_RELAY); gArgs.AddArg("-datacarriersize", strprintf("Maximum size of data in data carrier transactions we relay and mine (default: %u)", MAX_OP_RETURN_RELAY), false, OptionsCategory::NODE_RELAY); @@ -660,7 +668,7 @@ static void CleanupBlockRevFiles() static void ThreadImport(std::vector<fs::path> vImportFiles) { const CChainParams& chainparams = Params(); - RenameThread("bitcoin-loadblk"); + util::ThreadRename("loadblk"); ScheduleBatchPriority(); { @@ -856,6 +864,7 @@ void InitLogging() LogInstance().m_print_to_console = gArgs.GetBoolArg("-printtoconsole", !gArgs.GetBoolArg("-daemon", false)); LogInstance().m_log_timestamps = gArgs.GetBoolArg("-logtimestamps", DEFAULT_LOGTIMESTAMPS); LogInstance().m_log_time_micros = gArgs.GetBoolArg("-logtimemicros", DEFAULT_LOGTIMEMICROS); + LogInstance().m_log_threadnames = gArgs.GetBoolArg("-logthreadnames", DEFAULT_LOGTHREADNAMES); fLogIPs = gArgs.GetBoolArg("-logips", DEFAULT_LOGIPS); @@ -875,6 +884,7 @@ int nUserMaxConnections; int nFD; ServiceFlags nLocalServices = ServiceFlags(NODE_NETWORK | NODE_NETWORK_LIMITED); int64_t peer_connect_timeout; +std::vector<BlockFilterType> g_enabled_filter_types; } // namespace @@ -956,10 +966,29 @@ bool AppInitParameterInteraction() return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist."), gArgs.GetArg("-blocksdir", "").c_str())); } + // parse and validate enabled filter types + std::string blockfilterindex_value = gArgs.GetArg("-blockfilterindex", DEFAULT_BLOCKFILTERINDEX); + if (blockfilterindex_value == "" || blockfilterindex_value == "1") { + g_enabled_filter_types = AllBlockFilterTypes(); + } else if (blockfilterindex_value != "0") { + const std::vector<std::string> names = gArgs.GetArgs("-blockfilterindex"); + g_enabled_filter_types.reserve(names.size()); + for (const auto& name : names) { + BlockFilterType filter_type; + if (!BlockFilterTypeByName(name, filter_type)) { + return InitError(strprintf(_("Unknown -blockfilterindex value %s."), name)); + } + g_enabled_filter_types.push_back(filter_type); + } + } + // if using block pruning, then disallow txindex if (gArgs.GetArg("-prune", 0)) { if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) return InitError(_("Prune mode is incompatible with -txindex.")); + if (!g_enabled_filter_types.empty()) { + return InitError(_("Prune mode is incompatible with -blockfilterindex.")); + } } // -bind and -whitebind can't be set when not listening @@ -1123,22 +1152,6 @@ bool AppInitParameterInteraction() dustRelayFee = CFeeRate(n); } - // This is required by both the wallet and node - if (gArgs.IsArgSet("-maxtxfee")) - { - CAmount nMaxFee = 0; - if (!ParseMoney(gArgs.GetArg("-maxtxfee", ""), nMaxFee)) - return InitError(AmountErrMsg("maxtxfee", gArgs.GetArg("-maxtxfee", ""))); - if (nMaxFee > HIGH_MAX_TX_FEE) - InitWarning(_("-maxtxfee is set very high! Fees this large could be paid on a single transaction.")); - maxTxFee = nMaxFee; - if (CFeeRate(maxTxFee, 1000) < ::minRelayTxFee) - { - return InitError(strprintf(_("Invalid amount for -maxtxfee=<amount>: '%s' (must be at least the minrelay fee of %s to prevent stuck transactions)"), - gArgs.GetArg("-maxtxfee", ""), ::minRelayTxFee.ToString())); - } - } - fRequireStandard = !gArgs.GetBoolArg("-acceptnonstdtxn", !chainparams.RequireStandard()); if (chainparams.RequireStandard() && !fRequireStandard) return InitError(strprintf("acceptnonstdtxn is not currently supported for %s chain", chainparams.NetworkIDString())); @@ -1276,7 +1289,7 @@ bool AppInitMain(InitInterfaces& interfaces) LogPrintf("Using %u threads for script verification\n", nScriptCheckThreads); if (nScriptCheckThreads) { for (int i=0; i<nScriptCheckThreads-1; i++) - threadGroup.create_thread(&ThreadScriptCheck); + threadGroup.create_thread([i]() { return ThreadScriptCheck(i); }); } // Start the lightweight task scheduler thread @@ -1450,6 +1463,13 @@ bool AppInitMain(InitInterfaces& interfaces) nTotalCache -= nBlockTreeDBCache; int64_t nTxIndexCache = std::min(nTotalCache / 8, gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0); nTotalCache -= nTxIndexCache; + int64_t filter_index_cache = 0; + if (!g_enabled_filter_types.empty()) { + size_t n_indexes = g_enabled_filter_types.size(); + int64_t max_cache = std::min(nTotalCache / 8, max_filter_index_cache << 20); + filter_index_cache = max_cache / n_indexes; + nTotalCache -= filter_index_cache * n_indexes; + } int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache nTotalCache -= nCoinDBCache; @@ -1460,6 +1480,10 @@ bool AppInitMain(InitInterfaces& interfaces) if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { LogPrintf("* Using %.1f MiB for transaction index database\n", nTxIndexCache * (1.0 / 1024 / 1024)); } + for (BlockFilterType filter_type : g_enabled_filter_types) { + LogPrintf("* Using %.1f MiB for %s block filter index database\n", + filter_index_cache * (1.0 / 1024 / 1024), BlockFilterTypeName(filter_type)); + } LogPrintf("* Using %.1f MiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024)); @@ -1647,6 +1671,11 @@ bool AppInitMain(InitInterfaces& interfaces) g_txindex->Start(); } + for (const auto& filter_type : g_enabled_filter_types) { + InitBlockFilterIndex(filter_type, filter_index_cache, false, fReindex); + GetBlockFilterIndex(filter_type)->Start(); + } + // ********************************************************* Step 9: load wallet for (const auto& client : interfaces.chain_clients) { if (!client->load()) { @@ -1689,8 +1718,9 @@ bool AppInitMain(InitInterfaces& interfaces) // Either install a handler to notify us when genesis activates, or set fHaveGenesis directly. // No locking, as this happens before any background thread is started. + boost::signals2::connection block_notify_genesis_wait_connection; if (chainActive.Tip() == nullptr) { - uiInterface.NotifyBlockTip_connect(BlockNotifyGenesisWait); + block_notify_genesis_wait_connection = uiInterface.NotifyBlockTip_connect(BlockNotifyGenesisWait); } else { fHaveGenesis = true; } @@ -1714,7 +1744,7 @@ bool AppInitMain(InitInterfaces& interfaces) while (!fHaveGenesis && !ShutdownRequested()) { g_genesis_wait_cv.wait_for(lock, std::chrono::milliseconds(500)); } - uiInterface.NotifyBlockTip_disconnect(BlockNotifyGenesisWait); + block_notify_genesis_wait_connection.disconnect(); } if (ShutdownRequested()) { diff --git a/src/interfaces/chain.cpp b/src/interfaces/chain.cpp index b61a51b235..839af650bb 100644 --- a/src/interfaces/chain.cpp +++ b/src/interfaces/chain.cpp @@ -84,29 +84,15 @@ class LockImpl : public Chain::Lock CBlockIndex* block = ::chainActive[height]; return block && ((block->nStatus & BLOCK_HAVE_DATA) != 0) && block->nTx > 0; } - Optional<int> findFirstBlockWithTime(int64_t time, uint256* hash) override + Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height, uint256* hash) override { - CBlockIndex* block = ::chainActive.FindEarliestAtLeast(time); + CBlockIndex* block = ::chainActive.FindEarliestAtLeast(time, height); if (block) { if (hash) *hash = block->GetBlockHash(); return block->nHeight; } return nullopt; } - Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height) override - { - // TODO: Could update CChain::FindEarliestAtLeast() to take a height - // parameter and use it with std::lower_bound() to make this - // implementation more efficient and allow combining - // findFirstBlockWithTime and findFirstBlockWithTimeAndHeight into one - // method. - for (CBlockIndex* block = ::chainActive[height]; block; block = ::chainActive.Next(block)) { - if (block->GetBlockTime() >= time) { - return block->nHeight; - } - } - return nullopt; - } Optional<int> findPruned(int start_height, Optional<int> stop_height) override { if (::fPruneMode) { @@ -339,7 +325,6 @@ public: CFeeRate relayMinFee() override { return ::minRelayTxFee; } CFeeRate relayIncrementalFee() override { return ::incrementalRelayFee; } CFeeRate relayDustFee() override { return ::dustRelayFee; } - CAmount maxTxFee() override { return ::maxTxFee; } bool getPruneMode() override { return ::fPruneMode; } bool p2pEnabled() override { return g_connman != nullptr; } bool isReadyToBroadcast() override { return !::fImporting && !::fReindex && !IsInitialBlockDownload(); } diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 17d7b6d8f1..7564ad26ac 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -105,20 +105,11 @@ public: virtual bool haveBlockOnDisk(int height) = 0; //! Return height of the first block in the chain with timestamp equal - //! or greater than the given time, or nullopt if there is no block with - //! a high enough timestamp. Also return the block hash as an optional - //! output parameter (to avoid the cost of a second lookup in case this - //! information is needed.) - virtual Optional<int> findFirstBlockWithTime(int64_t time, uint256* hash) = 0; - - //! Return height of the first block in the chain with timestamp equal //! or greater than the given time and height equal or greater than the - //! given height, or nullopt if there is no such block. - //! - //! Calling this with height 0 is equivalent to calling - //! findFirstBlockWithTime, but less efficient because it requires a - //! linear instead of a binary search. - virtual Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height) = 0; + //! given height, or nullopt if there is no block with a high enough + //! timestamp and height. Also return the block hash as an optional output parameter + //! (to avoid the cost of a second lookup in case this information is needed.) + virtual Optional<int> findFirstBlockWithTimeAndHeight(int64_t time, int height, uint256* hash) = 0; //! Return height of last block in the specified range which is pruned, or //! nullopt if no block in the range is pruned. Range is inclusive. @@ -216,12 +207,6 @@ public: //! Relay dust fee setting (-dustrelayfee), reflecting lowest rate it's economical to spend. virtual CFeeRate relayDustFee() = 0; - //! Node max tx fee setting (-maxtxfee). - //! This could be replaced by a per-wallet max fee, as proposed at - //! https://github.com/bitcoin/bitcoin/issues/15355 - //! But for the time being, wallets call this to access the node setting. - virtual CAmount maxTxFee() = 0; - //! Check if pruning is enabled. virtual bool getPruneMode() = 0; diff --git a/src/interfaces/node.cpp b/src/interfaces/node.cpp index 73a5074133..f3ee8fe364 100644 --- a/src/interfaces/node.cpp +++ b/src/interfaces/node.cpp @@ -207,7 +207,6 @@ public: } } bool getNetworkActive() override { return g_connman && g_connman->GetNetworkActive(); } - CAmount getMaxTxFee() override { return ::maxTxFee; } CFeeRate estimateSmartFee(int num_blocks, bool conservative, int* returned_target = nullptr) override { FeeCalculation fee_calc; diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 76b93af234..1ccd2a31b7 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -159,9 +159,6 @@ public: //! Get network active. virtual bool getNetworkActive() = 0; - //! Get max tx fee. - virtual CAmount getMaxTxFee() = 0; - //! Estimate smart fee. virtual CFeeRate estimateSmartFee(int num_blocks, bool conservative, int* returned_target = nullptr) = 0; diff --git a/src/interfaces/wallet.cpp b/src/interfaces/wallet.cpp index ed73a71354..b57299d78d 100644 --- a/src/interfaces/wallet.cpp +++ b/src/interfaces/wallet.cpp @@ -469,6 +469,7 @@ public: bool IsWalletFlagSet(uint64_t flag) override { return m_wallet->IsWalletFlagSet(flag); } OutputType getDefaultAddressType() override { return m_wallet->m_default_address_type; } OutputType getDefaultChangeType() override { return m_wallet->m_default_change_type; } + CAmount getDefaultMaxTxFee() override { return m_wallet->m_default_max_tx_fee; } void remove() override { RemoveWallet(m_wallet); diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index cabc455b1f..7096f54047 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -247,6 +247,9 @@ public: // Get default change type. virtual OutputType getDefaultChangeType() = 0; + //! Get max tx fee. + virtual CAmount getDefaultMaxTxFee() = 0; + // Remove wallet. virtual void remove() = 0; diff --git a/src/logging.cpp b/src/logging.cpp index 36cad6573a..3eda4995db 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -4,8 +4,11 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <logging.h> +#include <util/threadnames.h> #include <util/time.h> +#include <mutex> + const char * const DEFAULT_DEBUGLOGFILE = "debug.log"; BCLog::Logger& LogInstance() @@ -174,7 +177,7 @@ std::vector<CLogCategoryActive> ListActiveLogCategories() return ret; } -std::string BCLog::Logger::LogTimestampStr(const std::string &str) +std::string BCLog::Logger::LogTimestampStr(const std::string& str) { std::string strStamped; @@ -196,21 +199,24 @@ std::string BCLog::Logger::LogTimestampStr(const std::string &str) } else strStamped = str; - if (!str.empty() && str[str.size()-1] == '\n') - m_started_new_line = true; - else - m_started_new_line = false; - return strStamped; } void BCLog::Logger::LogPrintStr(const std::string &str) { - std::string strTimestamped = LogTimestampStr(str); + std::string str_prefixed = str; + + if (m_log_threadnames && m_started_new_line) { + str_prefixed.insert(0, "[" + util::ThreadGetInternalName() + "] "); + } + + str_prefixed = LogTimestampStr(str_prefixed); + + m_started_new_line = !str.empty() && str[str.size()-1] == '\n'; if (m_print_to_console) { // print to console - fwrite(strTimestamped.data(), 1, strTimestamped.size(), stdout); + fwrite(str_prefixed.data(), 1, str_prefixed.size(), stdout); fflush(stdout); } if (m_print_to_file) { @@ -218,7 +224,7 @@ void BCLog::Logger::LogPrintStr(const std::string &str) // buffer if we haven't opened the log yet if (m_fileout == nullptr) { - m_msgs_before_open.push_back(strTimestamped); + m_msgs_before_open.push_back(str_prefixed); } else { @@ -232,7 +238,7 @@ void BCLog::Logger::LogPrintStr(const std::string &str) m_fileout = new_fileout; } } - FileWriteStr(strTimestamped, m_fileout); + FileWriteStr(str_prefixed, m_fileout); } } } diff --git a/src/logging.h b/src/logging.h index ac9d0dc0c7..e399d4c307 100644 --- a/src/logging.h +++ b/src/logging.h @@ -19,6 +19,7 @@ static const bool DEFAULT_LOGTIMEMICROS = false; static const bool DEFAULT_LOGIPS = false; static const bool DEFAULT_LOGTIMESTAMPS = true; +static const bool DEFAULT_LOGTHREADNAMES = false; extern const char * const DEFAULT_DEBUGLOGFILE; extern bool fLogIPs; @@ -81,6 +82,7 @@ namespace BCLog { bool m_log_timestamps = DEFAULT_LOGTIMESTAMPS; bool m_log_time_micros = DEFAULT_LOGTIMEMICROS; + bool m_log_threadnames = DEFAULT_LOGTHREADNAMES; fs::path m_file_path; std::atomic<bool> m_reopen_file{false}; diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index 6f8542123d..63a3d06267 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -59,7 +59,7 @@ bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType) std::vector<std::vector<unsigned char> > vSolutions; whichType = Solver(scriptPubKey, vSolutions); - if (whichType == TX_NONSTANDARD || whichType == TX_WITNESS_UNKNOWN) { + if (whichType == TX_NONSTANDARD) { return false; } else if (whichType == TX_MULTISIG) { unsigned char m = vSolutions.front()[0]; diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index f6f8e31363..aad991e2f1 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -222,6 +222,10 @@ inline void UnserializeTransaction(TxType& tx, Stream& s) { for (size_t i = 0; i < tx.vin.size(); i++) { s >> tx.vin[i].scriptWitness.stack; } + if (!tx.HasWitness()) { + /* It's illegal to encode witnesses when all witness stacks are empty. */ + throw std::ios_base::failure("Superfluous witness record"); + } } if (flags) { /* Unknown flag in the serialization */ diff --git a/src/qt/addressbookpage.cpp b/src/qt/addressbookpage.cpp index 726dafccb0..50d6afabcd 100644 --- a/src/qt/addressbookpage.cpp +++ b/src/qt/addressbookpage.cpp @@ -107,7 +107,7 @@ AddressBookPage::AddressBookPage(const PlatformStyle *platformStyle, Mode _mode, ui->newAddress->setVisible(true); break; case ReceivingTab: - ui->labelExplanation->setText(tr("These are your Bitcoin addresses for receiving payments. It is recommended to use a new receiving address for each transaction.")); + ui->labelExplanation->setText(tr("These are your Bitcoin addresses for receiving payments. Use the 'Create new receiving address' button in the receive tab to create new addresses.")); ui->deleteAddress->setVisible(false); ui->newAddress->setVisible(false); break; diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 1b063771ef..81255aaae9 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -30,6 +30,7 @@ #include <interfaces/handler.h> #include <interfaces/node.h> #include <noui.h> +#include <util/threadnames.h> #include <rpc/server.h> #include <ui_interface.h> #include <uint256.h> @@ -149,6 +150,7 @@ void BitcoinCore::initialize() try { qDebug() << __func__ << ": Running initialization in thread"; + util::ThreadRename("qt-init"); bool rv = m_node.appInitMain(); Q_EMIT initializeResult(rv); } catch (const std::exception& e) { @@ -423,6 +425,7 @@ int GuiMain(int argc, char* argv[]) std::tie(argc, argv) = winArgs.get(); #endif SetupEnvironment(); + util::ThreadRename("main"); std::unique_ptr<interfaces::Node> node = interfaces::MakeNode(); diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index 27b4c182f9..88a35534c2 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -11,7 +11,6 @@ #include <chain.h> #include <chainparams.h> -#include <checkpoints.h> #include <clientversion.h> #include <interfaces/handler.h> #include <interfaces/node.h> diff --git a/src/qt/forms/receivecoinsdialog.ui b/src/qt/forms/receivecoinsdialog.ui index 8876ea1337..0d280f2993 100644 --- a/src/qt/forms/receivecoinsdialog.ui +++ b/src/qt/forms/receivecoinsdialog.ui @@ -108,7 +108,7 @@ </size> </property> <property name="text"> - <string>&Request payment</string> + <string>&Create new receiving address</string> </property> <property name="icon"> <iconset resource="../bitcoin.qrc"> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index ba0a5abdf3..45f21d50fc 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -175,7 +175,9 @@ bool parseBitcoinURI(QString uri, SendCoinsRecipient *out) QString formatBitcoinURI(const SendCoinsRecipient &info) { - QString ret = QString("bitcoin:%1").arg(info.address); + bool bech_32 = info.address.startsWith(QString::fromStdString(Params().Bech32HRP() + "1")); + + QString ret = QString("bitcoin:%1").arg(bech_32 ? info.address.toUpper() : info.address); int paramCount = 0; if (info.amount) @@ -244,6 +246,11 @@ QList<QModelIndex> getEntryData(QAbstractItemView *view, int column) return view->selectionModel()->selectedRows(column); } +QString getDefaultDataDirectory() +{ + return boostPathToQString(GetDefaultDataDir()); +} + QString getSaveFileName(QWidget *parent, const QString &caption, const QString &dir, const QString &filter, QString *selectedSuffixOut) diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index cbec73a882..bea4a83494 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -79,6 +79,11 @@ namespace GUIUtil void setClipboard(const QString& str); + /** + * Determine default data directory for operating system. + */ + QString getDefaultDataDirectory(); + /** Get save filename, mimics QFileDialog::getSaveFileName, except that it appends a default suffix when no suffix is provided by the user. diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index 499af9fa07..c595361934 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -168,7 +168,7 @@ QString Intro::getDataDirectory() void Intro::setDataDirectory(const QString &dataDir) { ui->dataDirectory->setText(dataDir); - if(dataDir == getDefaultDataDirectory()) + if(dataDir == GUIUtil::getDefaultDataDirectory()) { ui->dataDirDefault->setChecked(true); ui->dataDirectory->setEnabled(false); @@ -180,11 +180,6 @@ void Intro::setDataDirectory(const QString &dataDir) } } -QString Intro::getDefaultDataDirectory() -{ - return GUIUtil::boostPathToQString(GetDefaultDataDir()); -} - bool Intro::pickDataDirectory(interfaces::Node& node) { QSettings settings; @@ -193,7 +188,7 @@ bool Intro::pickDataDirectory(interfaces::Node& node) if(!gArgs.GetArg("-datadir", "").empty()) return true; /* 1) Default data directory for operating system */ - QString dataDir = getDefaultDataDirectory(); + QString dataDir = GUIUtil::getDefaultDataDirectory(); /* 2) Allow QSettings to override default dir */ dataDir = settings.value("strDataDir", dataDir).toString(); @@ -239,7 +234,7 @@ bool Intro::pickDataDirectory(interfaces::Node& node) * override -datadir in the bitcoin.conf file in the default data directory * (to be consistent with bitcoind behavior) */ - if(dataDir != getDefaultDataDirectory()) { + if(dataDir != GUIUtil::getDefaultDataDirectory()) { node.softSetArg("-datadir", GUIUtil::qstringToBoostPath(dataDir).string()); // use OS locale for path setting } return true; @@ -293,7 +288,7 @@ void Intro::on_ellipsisButton_clicked() void Intro::on_dataDirDefault_clicked() { - setDataDirectory(getDefaultDataDirectory()); + setDataDirectory(GUIUtil::getDefaultDataDirectory()); } void Intro::on_dataDirCustom_clicked() diff --git a/src/qt/intro.h b/src/qt/intro.h index b537c94f7d..c3b26808d4 100644 --- a/src/qt/intro.h +++ b/src/qt/intro.h @@ -48,11 +48,6 @@ public: */ static bool pickDataDirectory(interfaces::Node& node); - /** - * Determine default data directory for operating system. - */ - static QString getDefaultDataDirectory(); - Q_SIGNALS: void requestCheck(); diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 9094aeff56..40dc7bf400 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -154,6 +154,10 @@ void OptionsDialog::setModel(OptionsModel *_model) if (_model->isRestartRequired()) showRestartWarning(true); + // Prune values are in GB to be consistent with intro.cpp + static constexpr uint64_t nMinDiskSpace = (MIN_DISK_SPACE_FOR_BLOCK_FILES / GB_BYTES) + (MIN_DISK_SPACE_FOR_BLOCK_FILES % GB_BYTES) ? 1 : 0; + ui->pruneSize->setRange(nMinDiskSpace, std::numeric_limits<int>::max()); + QString strLabel = _model->getOverriddenByCommandLine(); if (strLabel.isEmpty()) strLabel = tr("none"); @@ -164,10 +168,6 @@ void OptionsDialog::setModel(OptionsModel *_model) mapper->toFirst(); updateDefaultProxyNets(); - - // Prune values are in GB to be consistent with intro.cpp - static constexpr uint64_t nMinDiskSpace = (MIN_DISK_SPACE_FOR_BLOCK_FILES / GB_BYTES) + (MIN_DISK_SPACE_FOR_BLOCK_FILES % GB_BYTES) ? 1 : 0; - ui->pruneSize->setRange(nMinDiskSpace, _model->node().getAssumedBlockchainSize()); } /* warn when one of the following settings changes by user action (placed here so init via mapper doesn't trigger them) */ diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index 62496a57f4..5b4fb4cc18 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -17,7 +17,6 @@ #include <net.h> #include <netbase.h> #include <txdb.h> // for -dbcache defaults -#include <qt/intro.h> #include <QNetworkProxy> #include <QSettings> @@ -110,7 +109,7 @@ void OptionsModel::Init(bool resetSettings) addOverriddenOption("-par"); if (!settings.contains("strDataDir")) - settings.setValue("strDataDir", Intro::getDefaultDataDirectory()); + settings.setValue("strDataDir", GUIUtil::getDefaultDataDirectory()); // Wallet #ifdef ENABLE_WALLET @@ -187,7 +186,7 @@ void OptionsModel::Reset() BackupSettings(GetDataDir(true) / "guisettings.ini.bak", settings); // Save the strDataDir setting - QString dataDir = Intro::getDefaultDataDirectory(); + QString dataDir = GUIUtil::getDefaultDataDirectory(); dataDir = settings.value("strDataDir", dataDir).toString(); // Remove all entries from our QSettings object diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index c7ced3c106..8b6dcf0445 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -21,8 +21,6 @@ #include <util/strencodings.h> #include <util/system.h> -#include <openssl/crypto.h> - #include <univalue.h> #ifdef ENABLE_WALLET diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 6e00ab755c..8a0b265834 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -578,7 +578,7 @@ void SendCoinsDialog::processSendCoinsReturn(const WalletModel::SendCoinsReturn msgParams.second = CClientUIInterface::MSG_ERROR; break; case WalletModel::AbsurdFee: - msgParams.first = tr("A fee higher than %1 is considered an absurdly high fee.").arg(BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), model->node().getMaxTxFee())); + msgParams.first = tr("A fee higher than %1 is considered an absurdly high fee.").arg(BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), model->wallet().getDefaultMaxTxFee())); break; case WalletModel::PaymentRequestExpired: msgParams.first = tr("Payment request expired."); diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index f4f3be8f43..fd392b7cf7 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -222,9 +222,9 @@ WalletModel::SendCoinsReturn WalletModel::prepareTransaction(WalletModelTransact } // reject absurdly high fee. (This can never happen because the - // wallet caps the fee at maxTxFee. This merely serves as a + // wallet caps the fee at m_default_max_tx_fee. This merely serves as a // belt-and-suspenders check) - if (nFeeRequired > m_node.getMaxTxFee()) + if (nFeeRequired > m_wallet->getDefaultMaxTxFee()) return AbsurdFee; } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 1cc496c733..672fc69673 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -7,13 +7,14 @@ #include <amount.h> #include <base58.h> +#include <blockfilter.h> #include <chain.h> #include <chainparams.h> -#include <checkpoints.h> #include <coins.h> #include <consensus/validation.h> #include <core_io.h> #include <hash.h> +#include <index/blockfilterindex.h> #include <index/txindex.h> #include <key_io.h> #include <policy/feerate.h> @@ -1011,7 +1012,7 @@ static UniValue pruneblockchain(const JSONRPCRequest& request) // too low to be a block time (corresponds to timestamp from Sep 2001). if (heightParam > 1000000000) { // Add a 2 hour buffer to include blocks which might have had old timestamps - CBlockIndex* pindex = chainActive.FindEarliestAtLeast(heightParam - TIMESTAMP_WINDOW); + CBlockIndex* pindex = chainActive.FindEarliestAtLeast(heightParam - TIMESTAMP_WINDOW, 0); if (!pindex) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Could not find block with at least the specified timestamp."); } @@ -2299,6 +2300,85 @@ UniValue scantxoutset(const JSONRPCRequest& request) return result; } +static UniValue getblockfilter(const JSONRPCRequest& request) +{ + if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) { + throw std::runtime_error( + RPCHelpMan{"getblockfilter", + "\nRetrieve a BIP 157 content filter for a particular block.\n", + { + {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hash of the block"}, + {"filtertype", RPCArg::Type::STR, /*default*/ "basic", "The type name of the filter"}, + }, + RPCResult{ + "{\n" + " \"filter\" : (string) the hex-encoded filter data\n" + " \"header\" : (string) the hex-encoded filter header\n" + "}\n" + }, + RPCExamples{ + HelpExampleCli("getblockfilter", "\"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09\" \"basic\"") + } + }.ToString() + ); + } + + uint256 block_hash = ParseHashV(request.params[0], "blockhash"); + std::string filtertype_name = "basic"; + if (!request.params[1].isNull()) { + filtertype_name = request.params[1].get_str(); + } + + BlockFilterType filtertype; + if (!BlockFilterTypeByName(filtertype_name, filtertype)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown filtertype"); + } + + BlockFilterIndex* index = GetBlockFilterIndex(filtertype); + if (!index) { + throw JSONRPCError(RPC_MISC_ERROR, "Index is not enabled for filtertype " + filtertype_name); + } + + const CBlockIndex* block_index; + bool block_was_connected; + { + LOCK(cs_main); + block_index = LookupBlockIndex(block_hash); + if (!block_index) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); + } + block_was_connected = block_index->IsValid(BLOCK_VALID_SCRIPTS); + } + + bool index_ready = index->BlockUntilSyncedToCurrentChain(); + + BlockFilter filter; + uint256 filter_header; + if (!index->LookupFilter(block_index, filter) || + !index->LookupFilterHeader(block_index, filter_header)) { + int err_code; + std::string errmsg = "Filter not found."; + + if (!block_was_connected) { + err_code = RPC_INVALID_ADDRESS_OR_KEY; + errmsg += " Block was not connected to active chain."; + } else if (!index_ready) { + err_code = RPC_MISC_ERROR; + errmsg += " Block filters are still in the process of being indexed."; + } else { + err_code = RPC_INTERNAL_ERROR; + errmsg += " This error is unexpected and indicates index corruption."; + } + + throw JSONRPCError(err_code, errmsg); + } + + UniValue ret(UniValue::VOBJ); + ret.pushKV("filter", HexStr(filter.GetEncodedFilter())); + ret.pushKV("header", filter_header.GetHex()); + return ret; +} + // clang-format off static const CRPCCommand commands[] = { // category name actor (function) argNames @@ -2326,6 +2406,7 @@ static const CRPCCommand commands[] = { "blockchain", "preciousblock", &preciousblock, {"blockhash"} }, { "blockchain", "scantxoutset", &scantxoutset, {"action", "scanobjects"} }, + { "blockchain", "getblockfilter", &getblockfilter, {"blockhash", "filtertype"} }, /* Not shown in help */ { "hidden", "invalidateblock", &invalidateblock, {"blockhash"} }, diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 1a7216ceeb..78d7bbc80c 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -15,6 +15,7 @@ #include <key_io.h> #include <keystore.h> #include <merkleblock.h> +#include <node/coin.h> #include <node/psbt.h> #include <node/transaction.h> #include <policy/policy.h> @@ -770,7 +771,14 @@ static UniValue signrawtransactionwithkey(const JSONRPCRequest& request) keystore.AddKey(key); } - return SignTransaction(*g_rpc_interfaces->chain, mtx, request.params[2], &keystore, true, request.params[3]); + // Fetch previous transactions (inputs): + std::map<COutPoint, Coin> coins; + for (const CTxIn& txin : mtx.vin) { + coins[txin.prevout]; // Create empty map entry keyed by prevout. + } + FindCoins(coins); + + return SignTransaction(mtx, request.params[2], &keystore, coins, true, request.params[3]); } static UniValue sendrawtransaction(const JSONRPCRequest& request) diff --git a/src/rpc/rawtransaction_util.cpp b/src/rpc/rawtransaction_util.cpp index 728fc62e25..8af491977c 100644 --- a/src/rpc/rawtransaction_util.cpp +++ b/src/rpc/rawtransaction_util.cpp @@ -149,18 +149,8 @@ static void TxInErrorToJSON(const CTxIn& txin, UniValue& vErrorsRet, const std:: vErrorsRet.push_back(entry); } -// TODO(https://github.com/bitcoin/bitcoin/pull/10973#discussion_r267084237): -// The dependency on interfaces::Chain should be removed, so -// signrawtransactionwithkey doesn't need access to a Chain instance. -UniValue SignTransaction(interfaces::Chain& chain, CMutableTransaction& mtx, const UniValue& prevTxsUnival, CBasicKeyStore *keystore, bool is_temp_keystore, const UniValue& hashType) +UniValue SignTransaction(CMutableTransaction& mtx, const UniValue& prevTxsUnival, CBasicKeyStore* keystore, std::map<COutPoint, Coin>& coins, bool is_temp_keystore, const UniValue& hashType) { - // Fetch previous transactions (inputs): - std::map<COutPoint, Coin> coins; - for (const CTxIn& txin : mtx.vin) { - coins[txin.prevout]; // Create empty map entry keyed by prevout. - } - chain.findCoins(coins); - // Add previous txouts given in the RPC call: if (!prevTxsUnival.isNull()) { UniValue prevTxs = prevTxsUnival.get_array(); diff --git a/src/rpc/rawtransaction_util.h b/src/rpc/rawtransaction_util.h index 5529dedbd4..c115d33a77 100644 --- a/src/rpc/rawtransaction_util.h +++ b/src/rpc/rawtransaction_util.h @@ -5,16 +5,26 @@ #ifndef BITCOIN_RPC_RAWTRANSACTION_UTIL_H #define BITCOIN_RPC_RAWTRANSACTION_UTIL_H +#include <map> + class CBasicKeyStore; class UniValue; struct CMutableTransaction; +class Coin; +class COutPoint; -namespace interfaces { -class Chain; -} // namespace interfaces - -/** Sign a transaction with the given keystore and previous transactions */ -UniValue SignTransaction(interfaces::Chain& chain, CMutableTransaction& mtx, const UniValue& prevTxs, CBasicKeyStore *keystore, bool tempKeystore, const UniValue& hashType); +/** + * Sign a transaction with the given keystore and previous transactions + * + * @param mtx The transaction to-be-signed + * @param prevTxs Array of previous txns outputs that tx depends on but may not yet be in the block chain + * @param keystore Temporary keystore containing signing keys + * @param coins Map of unspent outputs - coins in mempool and current chain UTXO set, may be extended by previous txns outputs after call + * @param tempKeystore Whether to use temporary keystore + * @param hashType The signature hash type + * @returns JSON object with details of signed transaction + */ +UniValue SignTransaction(CMutableTransaction& mtx, const UniValue& prevTxs, CBasicKeyStore* keystore, std::map<COutPoint, Coin>& coins, bool tempKeystore, const UniValue& hashType); /** Create a transaction from univalue parameters */ CMutableTransaction ConstructTransaction(const UniValue& inputs_in, const UniValue& outputs_in, const UniValue& locktime, const UniValue& rbf); diff --git a/src/script/ismine.h b/src/script/ismine.h index 601e70f709..55e28e225a 100644 --- a/src/script/ismine.h +++ b/src/script/ismine.h @@ -9,6 +9,7 @@ #include <script/standard.h> #include <stdint.h> +#include <bitset> class CKeyStore; class CScript; @@ -16,10 +17,11 @@ class CScript; /** IsMine() return codes */ enum isminetype { - ISMINE_NO = 0, - ISMINE_WATCH_ONLY = 1, - ISMINE_SPENDABLE = 2, - ISMINE_ALL = ISMINE_WATCH_ONLY | ISMINE_SPENDABLE + ISMINE_NO = 0, + ISMINE_WATCH_ONLY = 1 << 0, + ISMINE_SPENDABLE = 1 << 1, + ISMINE_ALL = ISMINE_WATCH_ONLY | ISMINE_SPENDABLE, + ISMINE_ENUM_ELEMENTS, }; /** used for bitflags of isminetype */ typedef uint8_t isminefilter; @@ -27,4 +29,23 @@ typedef uint8_t isminefilter; isminetype IsMine(const CKeyStore& keystore, const CScript& scriptPubKey); isminetype IsMine(const CKeyStore& keystore, const CTxDestination& dest); +/** + * Cachable amount subdivided into watchonly and spendable parts. + */ +struct CachableAmount +{ + // NO and ALL are never (supposed to be) cached + std::bitset<ISMINE_ENUM_ELEMENTS> m_cached; + CAmount m_value[ISMINE_ENUM_ELEMENTS]; + inline void Reset() + { + m_cached.reset(); + } + void Set(isminefilter filter, CAmount value) + { + m_cached.set(filter); + m_value[filter] = value; + } +}; + #endif // BITCOIN_SCRIPT_ISMINE_H diff --git a/src/script/standard.h b/src/script/standard.h index fc20fb6a08..f16068c413 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -153,8 +153,7 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) * multisig scripts, this populates the addressRet vector with the pubkey IDs * and nRequiredRet with the n required to spend. For other destinations, * addressRet is populated with a single value and nRequiredRet is set to 1. - * Returns true if successful. Currently does not extract address from - * pay-to-witness scripts. + * Returns true if successful. * * Note: this function confuses destinations (a subset of CScripts that are * encodable as an address) with key identifiers (of keys involved in a diff --git a/src/serialize.h b/src/serialize.h index 2d0cfbbbf0..b001ee1324 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -89,6 +89,11 @@ template<typename Stream> inline void ser_writedata32(Stream &s, uint32_t obj) obj = htole32(obj); s.write((char*)&obj, 4); } +template<typename Stream> inline void ser_writedata32be(Stream &s, uint32_t obj) +{ + obj = htobe32(obj); + s.write((char*)&obj, 4); +} template<typename Stream> inline void ser_writedata64(Stream &s, uint64_t obj) { obj = htole64(obj); @@ -118,6 +123,12 @@ template<typename Stream> inline uint32_t ser_readdata32(Stream &s) s.read((char*)&obj, 4); return le32toh(obj); } +template<typename Stream> inline uint32_t ser_readdata32be(Stream &s) +{ + uint32_t obj; + s.read((char*)&obj, 4); + return be32toh(obj); +} template<typename Stream> inline uint64_t ser_readdata64(Stream &s) { uint64_t obj; diff --git a/src/sync.cpp b/src/sync.cpp index 23ca866e53..e7c0a6f9bc 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -3,9 +3,11 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <sync.h> +#include <tinyformat.h> #include <logging.h> #include <util/strencodings.h> +#include <util/threadnames.h> #include <stdio.h> @@ -37,23 +39,30 @@ void PrintLockContention(const char* pszName, const char* pszFile, int nLine) // struct CLockLocation { - CLockLocation(const char* pszName, const char* pszFile, int nLine, bool fTryIn) - { - mutexName = pszName; - sourceFile = pszFile; - sourceLine = nLine; - fTry = fTryIn; - } + CLockLocation( + const char* pszName, + const char* pszFile, + int nLine, + bool fTryIn, + const std::string& thread_name) + : fTry(fTryIn), + mutexName(pszName), + sourceFile(pszFile), + m_thread_name(thread_name), + sourceLine(nLine) {} std::string ToString() const { - return mutexName + " " + sourceFile + ":" + itostr(sourceLine) + (fTry ? " (TRY)" : ""); + return tfm::format( + "%s %s:%s%s (in thread %s)", + mutexName, sourceFile, itostr(sourceLine), (fTry ? " (TRY)" : ""), m_thread_name); } private: bool fTry; std::string mutexName; std::string sourceFile; + const std::string& m_thread_name; int sourceLine; }; @@ -125,7 +134,7 @@ static void push_lock(void* c, const CLockLocation& locklocation) std::pair<void*, void*> p1 = std::make_pair(i.first, c); if (lockdata.lockorders.count(p1)) continue; - lockdata.lockorders[p1] = g_lockstack; + lockdata.lockorders.emplace(p1, g_lockstack); std::pair<void*, void*> p2 = std::make_pair(c, i.first); lockdata.invlockorders.insert(p2); @@ -141,7 +150,7 @@ static void pop_lock() void EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs, bool fTry) { - push_lock(cs, CLockLocation(pszName, pszFile, nLine, fTry)); + push_lock(cs, CLockLocation(pszName, pszFile, nLine, fTry, util::ThreadGetInternalName())); } void LeaveCritical() diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp new file mode 100644 index 0000000000..db0b973463 --- /dev/null +++ b/src/test/blockfilter_index_tests.cpp @@ -0,0 +1,307 @@ +// Copyright (c) 2017-2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <blockfilter.h> +#include <chainparams.h> +#include <consensus/validation.h> +#include <index/blockfilterindex.h> +#include <miner.h> +#include <pow.h> +#include <test/setup_common.h> +#include <script/standard.h> +#include <validation.h> + +#include <boost/test/unit_test.hpp> + +BOOST_AUTO_TEST_SUITE(blockfilter_index_tests) + +static bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex* block_index, + BlockFilter& filter) +{ + CBlock block; + if (!ReadBlockFromDisk(block, block_index->GetBlockPos(), Params().GetConsensus())) { + return false; + } + + CBlockUndo block_undo; + if (block_index->nHeight > 0 && !UndoReadFromDisk(block_undo, block_index)) { + return false; + } + + filter = BlockFilter(filter_type, block, block_undo); + return true; +} + +static bool CheckFilterLookups(BlockFilterIndex& filter_index, const CBlockIndex* block_index, + uint256& last_header) +{ + BlockFilter expected_filter; + if (!ComputeFilter(filter_index.GetFilterType(), block_index, expected_filter)) { + BOOST_ERROR("ComputeFilter failed on block " << block_index->nHeight); + return false; + } + + BlockFilter filter; + uint256 filter_header; + std::vector<BlockFilter> filters; + std::vector<uint256> filter_hashes; + + BOOST_CHECK(filter_index.LookupFilter(block_index, filter)); + BOOST_CHECK(filter_index.LookupFilterHeader(block_index, filter_header)); + BOOST_CHECK(filter_index.LookupFilterRange(block_index->nHeight, block_index, filters)); + BOOST_CHECK(filter_index.LookupFilterHashRange(block_index->nHeight, block_index, + filter_hashes)); + + BOOST_CHECK_EQUAL(filters.size(), 1); + BOOST_CHECK_EQUAL(filter_hashes.size(), 1); + + BOOST_CHECK_EQUAL(filter.GetHash(), expected_filter.GetHash()); + BOOST_CHECK_EQUAL(filter_header, expected_filter.ComputeHeader(last_header)); + BOOST_CHECK_EQUAL(filters[0].GetHash(), expected_filter.GetHash()); + BOOST_CHECK_EQUAL(filter_hashes[0], expected_filter.GetHash()); + + filters.clear(); + filter_hashes.clear(); + last_header = filter_header; + return true; +} + +static CBlock CreateBlock(const CBlockIndex* prev, + const std::vector<CMutableTransaction>& txns, + const CScript& scriptPubKey) +{ + const CChainParams& chainparams = Params(); + std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey); + CBlock& block = pblocktemplate->block; + block.hashPrevBlock = prev->GetBlockHash(); + block.nTime = prev->nTime + 1; + + // Replace mempool-selected txns with just coinbase plus passed-in txns: + block.vtx.resize(1); + for (const CMutableTransaction& tx : txns) { + block.vtx.push_back(MakeTransactionRef(tx)); + } + // IncrementExtraNonce creates a valid coinbase and merkleRoot + unsigned int extraNonce = 0; + IncrementExtraNonce(&block, prev, extraNonce); + + while (!CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce; + + return block; +} + +static bool BuildChain(const CBlockIndex* pindex, const CScript& coinbase_script_pub_key, + size_t length, std::vector<std::shared_ptr<CBlock>>& chain) +{ + std::vector<CMutableTransaction> no_txns; + + chain.resize(length); + for (auto& block : chain) { + block = std::make_shared<CBlock>(CreateBlock(pindex, no_txns, coinbase_script_pub_key)); + CBlockHeader header = block->GetBlockHeader(); + + CValidationState state; + if (!ProcessNewBlockHeaders({header}, state, Params(), &pindex, nullptr)) { + return false; + } + } + + return true; +} + +BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, TestChain100Setup) +{ + BlockFilterIndex filter_index(BlockFilterType::BASIC, 1 << 20, true); + + uint256 last_header; + + // Filter should not be found in the index before it is started. + { + LOCK(cs_main); + + BlockFilter filter; + uint256 filter_header; + std::vector<BlockFilter> filters; + std::vector<uint256> filter_hashes; + + for (const CBlockIndex* block_index = chainActive.Genesis(); + block_index != nullptr; + block_index = chainActive.Next(block_index)) { + BOOST_CHECK(!filter_index.LookupFilter(block_index, filter)); + BOOST_CHECK(!filter_index.LookupFilterHeader(block_index, filter_header)); + BOOST_CHECK(!filter_index.LookupFilterRange(block_index->nHeight, block_index, filters)); + BOOST_CHECK(!filter_index.LookupFilterHashRange(block_index->nHeight, block_index, + filter_hashes)); + } + } + + // BlockUntilSyncedToCurrentChain should return false before index is started. + BOOST_CHECK(!filter_index.BlockUntilSyncedToCurrentChain()); + + filter_index.Start(); + + // Allow filter index to catch up with the block index. + constexpr int64_t timeout_ms = 10 * 1000; + int64_t time_start = GetTimeMillis(); + while (!filter_index.BlockUntilSyncedToCurrentChain()) { + BOOST_REQUIRE(time_start + timeout_ms > GetTimeMillis()); + MilliSleep(100); + } + + // Check that filter index has all blocks that were in the chain before it started. + { + LOCK(cs_main); + const CBlockIndex* block_index; + for (block_index = chainActive.Genesis(); + block_index != nullptr; + block_index = chainActive.Next(block_index)) { + CheckFilterLookups(filter_index, block_index, last_header); + } + } + + // Create two forks. + const CBlockIndex* tip; + { + LOCK(cs_main); + tip = chainActive.Tip(); + } + CScript coinbase_script_pub_key = GetScriptForDestination(coinbaseKey.GetPubKey().GetID()); + std::vector<std::shared_ptr<CBlock>> chainA, chainB; + BOOST_REQUIRE(BuildChain(tip, coinbase_script_pub_key, 10, chainA)); + BOOST_REQUIRE(BuildChain(tip, coinbase_script_pub_key, 10, chainB)); + + // Check that new blocks on chain A get indexed. + uint256 chainA_last_header = last_header; + for (size_t i = 0; i < 2; i++) { + const auto& block = chainA[i]; + BOOST_REQUIRE(ProcessNewBlock(Params(), block, true, nullptr)); + + const CBlockIndex* block_index; + { + LOCK(cs_main); + block_index = LookupBlockIndex(block->GetHash()); + } + + BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); + CheckFilterLookups(filter_index, block_index, chainA_last_header); + } + + // Reorg to chain B. + uint256 chainB_last_header = last_header; + for (size_t i = 0; i < 3; i++) { + const auto& block = chainB[i]; + BOOST_REQUIRE(ProcessNewBlock(Params(), block, true, nullptr)); + + const CBlockIndex* block_index; + { + LOCK(cs_main); + block_index = LookupBlockIndex(block->GetHash()); + } + + BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); + CheckFilterLookups(filter_index, block_index, chainB_last_header); + } + + // Check that filters for stale blocks on A can be retrieved. + chainA_last_header = last_header; + for (size_t i = 0; i < 2; i++) { + const auto& block = chainA[i]; + const CBlockIndex* block_index; + { + LOCK(cs_main); + block_index = LookupBlockIndex(block->GetHash()); + } + + BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); + CheckFilterLookups(filter_index, block_index, chainA_last_header); + } + + // Reorg back to chain A. + for (size_t i = 2; i < 4; i++) { + const auto& block = chainA[i]; + BOOST_REQUIRE(ProcessNewBlock(Params(), block, true, nullptr)); + } + + // Check that chain A and B blocks can be retrieved. + chainA_last_header = last_header; + chainB_last_header = last_header; + for (size_t i = 0; i < 3; i++) { + const CBlockIndex* block_index; + + { + LOCK(cs_main); + block_index = LookupBlockIndex(chainA[i]->GetHash()); + } + BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); + CheckFilterLookups(filter_index, block_index, chainA_last_header); + + { + LOCK(cs_main); + block_index = LookupBlockIndex(chainB[i]->GetHash()); + } + BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); + CheckFilterLookups(filter_index, block_index, chainB_last_header); + } + + // Test lookups for a range of filters/hashes. + std::vector<BlockFilter> filters; + std::vector<uint256> filter_hashes; + + { + LOCK(cs_main); + tip = chainActive.Tip(); + } + BOOST_CHECK(filter_index.LookupFilterRange(0, tip, filters)); + BOOST_CHECK(filter_index.LookupFilterHashRange(0, tip, filter_hashes)); + + BOOST_CHECK_EQUAL(filters.size(), tip->nHeight + 1); + BOOST_CHECK_EQUAL(filter_hashes.size(), tip->nHeight + 1); + + filters.clear(); + filter_hashes.clear(); + + filter_index.Interrupt(); + filter_index.Stop(); +} + +BOOST_FIXTURE_TEST_CASE(blockfilter_index_init_destroy, BasicTestingSetup) +{ + SetDataDir("tempdir"); + + BlockFilterIndex* filter_index; + + filter_index = GetBlockFilterIndex(BlockFilterType::BASIC); + BOOST_CHECK(filter_index == nullptr); + + BOOST_CHECK(InitBlockFilterIndex(BlockFilterType::BASIC, 1 << 20, true, false)); + + filter_index = GetBlockFilterIndex(BlockFilterType::BASIC); + BOOST_CHECK(filter_index != nullptr); + BOOST_CHECK(filter_index->GetFilterType() == BlockFilterType::BASIC); + + // Initialize returns false if index already exists. + BOOST_CHECK(!InitBlockFilterIndex(BlockFilterType::BASIC, 1 << 20, true, false)); + + int iter_count = 0; + ForEachBlockFilterIndex([&iter_count](BlockFilterIndex& _index) { iter_count++; }); + BOOST_CHECK_EQUAL(iter_count, 1); + + BOOST_CHECK(DestroyBlockFilterIndex(BlockFilterType::BASIC)); + + // Destroy returns false because index was already destroyed. + BOOST_CHECK(!DestroyBlockFilterIndex(BlockFilterType::BASIC)); + + filter_index = GetBlockFilterIndex(BlockFilterType::BASIC); + BOOST_CHECK(filter_index == nullptr); + + // Reinitialize index. + BOOST_CHECK(InitBlockFilterIndex(BlockFilterType::BASIC, 1 << 20, true, false)); + + DestroyAllBlockFilterIndexes(); + + filter_index = GetBlockFilterIndex(BlockFilterType::BASIC); + BOOST_CHECK(filter_index == nullptr); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/blockfilter_tests.cpp b/src/test/blockfilter_tests.cpp index f4c74af098..df0a041e0e 100644 --- a/src/test/blockfilter_tests.cpp +++ b/src/test/blockfilter_tests.cpp @@ -54,7 +54,7 @@ BOOST_AUTO_TEST_CASE(gcsfilter_default_constructor) BOOST_AUTO_TEST_CASE(blockfilter_basic_test) { - CScript included_scripts[5], excluded_scripts[3]; + CScript included_scripts[5], excluded_scripts[4]; // First two are outputs on a single transaction. included_scripts[0] << std::vector<unsigned char>(0, 65) << OP_CHECKSIG; @@ -73,14 +73,19 @@ BOOST_AUTO_TEST_CASE(blockfilter_basic_test) // This script is not related to the block at all. excluded_scripts[1] << std::vector<unsigned char>(5, 33) << OP_CHECKSIG; + // OP_RETURN is non-standard since it's not followed by a data push, but is still excluded from + // filter. + excluded_scripts[2] << OP_RETURN << OP_4 << OP_ADD << OP_8 << OP_EQUAL; + CMutableTransaction tx_1; tx_1.vout.emplace_back(100, included_scripts[0]); tx_1.vout.emplace_back(200, included_scripts[1]); + tx_1.vout.emplace_back(0, excluded_scripts[0]); CMutableTransaction tx_2; tx_2.vout.emplace_back(300, included_scripts[2]); - tx_2.vout.emplace_back(0, excluded_scripts[0]); - tx_2.vout.emplace_back(400, excluded_scripts[2]); // Script is empty + tx_2.vout.emplace_back(0, excluded_scripts[2]); + tx_2.vout.emplace_back(400, excluded_scripts[3]); // Script is empty CBlock block; block.vtx.push_back(MakeTransactionRef(tx_1)); @@ -90,7 +95,7 @@ BOOST_AUTO_TEST_CASE(blockfilter_basic_test) block_undo.vtxundo.emplace_back(); block_undo.vtxundo.back().vprevout.emplace_back(CTxOut(500, included_scripts[3]), 1000, true); block_undo.vtxundo.back().vprevout.emplace_back(CTxOut(600, included_scripts[4]), 10000, false); - block_undo.vtxundo.back().vprevout.emplace_back(CTxOut(700, excluded_scripts[2]), 100000, false); + block_undo.vtxundo.back().vprevout.emplace_back(CTxOut(700, excluded_scripts[3]), 100000, false); BlockFilter block_filter(BlockFilterType::BASIC, block, block_undo); const GCSFilter& filter = block_filter.GetFilter(); @@ -174,4 +179,16 @@ BOOST_AUTO_TEST_CASE(blockfilters_json_test) } } +BOOST_AUTO_TEST_CASE(blockfilter_type_names) +{ + BOOST_CHECK_EQUAL(BlockFilterTypeName(BlockFilterType::BASIC), "basic"); + BOOST_CHECK_EQUAL(BlockFilterTypeName(static_cast<BlockFilterType>(255)), ""); + + BlockFilterType filter_type; + BOOST_CHECK(BlockFilterTypeByName("basic", filter_type)); + BOOST_CHECK_EQUAL(filter_type, BlockFilterType::BASIC); + + BOOST_CHECK(!BlockFilterTypeByName("unknown", filter_type)); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 8a219a8284..0d05b6514f 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -18,8 +18,6 @@ #include <vector> #include <boost/test/unit_test.hpp> -#include <openssl/aes.h> -#include <openssl/evp.h> BOOST_FIXTURE_TEST_SUITE(crypto_tests, BasicTestingSetup) diff --git a/src/test/data/blockfilters.json b/src/test/data/blockfilters.json index 134b788eed..8945296a07 100644 --- a/src/test/data/blockfilters.json +++ b/src/test/data/blockfilters.json @@ -3,9 +3,11 @@ [0,"000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943","0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4adae5494dffff001d1aa4ae180101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000",[],"0000000000000000000000000000000000000000000000000000000000000000","019dfca8","21584579b7eb08997773e5aeff3a7f932700042d0ed2a6129012b7d7ae81b750","Genesis block"], [2,"000000006c02c8ea6e4ff69651f7fcde348fb9d557a06e6957b65552002a7820","0100000006128e87be8b1b4dea47a7247d5528d2702c96826c7a648497e773b800000000e241352e3bec0a95a6217e10c3abb54adfa05abb12c126695595580fb92e222032e7494dffff001d00d235340101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0432e7494d010e062f503253482fffffffff0100f2052a010000002321038a7f6ef1c8ca0c588aa53fa860128077c9e6c11e6830f4d7ee4e763a56b7718fac00000000",[],"d7bdac13a59d745b1add0d2ce852f1a0442e8945fc1bf3848d3cbffd88c24fe1","0174a170","186afd11ef2b5e7e3504f2e8cbf8df28a1fd251fe53d60dff8b1467d1b386cf0",""], [3,"000000008b896e272758da5297bcd98fdc6d97c9b765ecec401e286dc1fdbe10","0100000020782a005255b657696ea057d5b98f34defcf75196f64f6eeac8026c0000000041ba5afc532aae03151b8aa87b65e1594f97504a768e010c98c0add79216247186e7494dffff001d058dc2b60101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0486e7494d0151062f503253482fffffffff0100f2052a01000000232103f6d9ff4c12959445ca5549c811683bf9c88e637b222dd2e0311154c4c85cf423ac00000000",[],"186afd11ef2b5e7e3504f2e8cbf8df28a1fd251fe53d60dff8b1467d1b386cf0","016cf7a0","8d63aadf5ab7257cb6d2316a57b16f517bff1c6388f124ec4c04af1212729d2a",""], +[15007,"0000000038c44c703bae0f98cdd6bf30922326340a5996cc692aaae8bacf47ad","0100000002394092aa378fe35d7e9ac79c869b975c4de4374cd75eb5484b0e1e00000000eb9b8670abd44ad6c55cee18e3020fb0c6519e7004b01a16e9164867531b67afc33bc94fffff001d123f10050101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e04c33bc94f0115062f503253482fffffffff0100f2052a01000000232103f268e9ae07e0f8cb2f6e901d87c510d650b97230c0365b021df8f467363cafb1ac00000000",[],"18b5c2b0146d2d09d24fb00ff5b52bd0742f36c9e65527abdb9de30c027a4748","013c3710","07384b01311867949e0c046607c66b7a766d338474bb67f66c8ae9dbd454b20e","Tx has non-standard OP_RETURN output followed by opcodes"], [49291,"0000000018b07dca1b28b4b5a119f6d6e71698ce1ed96f143f54179ce177a19c","02000000abfaf47274223ca2fea22797e44498240e482cb4c2f2baea088962f800000000604b5b52c32305b15d7542071d8b04e750a547500005d4010727694b6e72a776e55d0d51ffff001d211806480201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0d038bc0000102062f503253482fffffffff01a078072a01000000232102971dd6034ed0cf52450b608d196c07d6345184fcb14deb277a6b82d526a6163dac0000000001000000081cefd96060ecb1c4fbe675ad8a4f8bdc61d634c52b3a1c4116dee23749fe80ff000000009300493046022100866859c21f306538152e83f115bcfbf59ab4bb34887a88c03483a5dff9895f96022100a6dfd83caa609bf0516debc2bf65c3df91813a4842650a1858b3f61cfa8af249014730440220296d4b818bb037d0f83f9f7111665f49532dfdcbec1e6b784526e9ac4046eaa602204acf3a5cb2695e8404d80bf49ab04828bcbe6fc31d25a2844ced7a8d24afbdff01ffffffff1cefd96060ecb1c4fbe675ad8a4f8bdc61d634c52b3a1c4116dee23749fe80ff020000009400483045022100e87899175991aa008176cb553c6f2badbb5b741f328c9845fcab89f8b18cae2302200acce689896dc82933015e7230e5230d5cff8a1ffe82d334d60162ac2c5b0c9601493046022100994ad29d1e7b03e41731a4316e5f4992f0d9b6e2efc40a1ccd2c949b461175c502210099b69fdc2db00fbba214f16e286f6a49e2d8a0d5ffc6409d87796add475478d601ffffffff1e4a6d2d280ea06680d6cf8788ac90344a9c67cca9b06005bbd6d3f6945c8272010000009500493046022100a27400ba52fd842ce07398a1de102f710a10c5599545e6c95798934352c2e4df022100f6383b0b14c9f64b6718139f55b6b9494374755b86bae7d63f5d3e583b57255a01493046022100fdf543292f34e1eeb1703b264965339ec4a450ec47585009c606b3edbc5b617b022100a5fbb1c8de8aaaa582988cdb23622838e38de90bebcaab3928d949aa502a65d401ffffffff1e4a6d2d280ea06680d6cf8788ac90344a9c67cca9b06005bbd6d3f6945c8272020000009400493046022100ac626ac3051f875145b4fe4cfe089ea895aac73f65ab837b1ac30f5d875874fa022100bc03e79fa4b7eb707fb735b95ff6613ca33adeaf3a0607cdcead4cfd3b51729801483045022100b720b04a5c5e2f61b7df0fcf334ab6fea167b7aaede5695d3f7c6973496adbf1022043328c4cc1cdc3e5db7bb895ccc37133e960b2fd3ece98350f774596badb387201ffffffff23a8733e349c97d6cd90f520fdd084ba15ce0a395aad03cd51370602bb9e5db3010000004a00483045022100e8556b72c5e9c0da7371913a45861a61c5df434dfd962de7b23848e1a28c86ca02205d41ceda00136267281be0974be132ac4cda1459fe2090ce455619d8b91045e901ffffffff6856d609b881e875a5ee141c235e2a82f6b039f2b9babe82333677a5570285a6000000006a473044022040a1c631554b8b210fbdf2a73f191b2851afb51d5171fb53502a3a040a38d2c0022040d11cf6e7b41fe1b66c3d08f6ada1aee07a047cb77f242b8ecc63812c832c9a012102bcfad931b502761e452962a5976c79158a0f6d307ad31b739611dac6a297c256ffffffff6856d609b881e875a5ee141c235e2a82f6b039f2b9babe82333677a5570285a601000000930048304502205b109df098f7e932fbf71a45869c3f80323974a826ee2770789eae178a21bfc8022100c0e75615e53ee4b6e32b9bb5faa36ac539e9c05fa2ae6b6de5d09c08455c8b9601483045022009fb7d27375c47bea23b24818634df6a54ecf72d52e0c1268fb2a2c84f1885de022100e0ed4f15d62e7f537da0d0f1863498f9c7c0c0a4e00e4679588c8d1a9eb20bb801ffffffffa563c3722b7b39481836d5edfc1461f97335d5d1e9a23ade13680d0e2c1c371f030000006c493046022100ecc38ae2b1565643dc3c0dad5e961a5f0ea09cab28d024f92fa05c922924157e022100ebc166edf6fbe4004c72bfe8cf40130263f98ddff728c8e67b113dbd621906a601210211a4ed241174708c07206601b44a4c1c29e5ad8b1f731c50ca7e1d4b2a06dc1fffffffff02d0223a00000000001976a91445db0b779c0b9fa207f12a8218c94fc77aff504588ac80f0fa02000000000000000000",["5221033423007d8f263819a2e42becaaf5b06f34cb09919e06304349d950668209eaed21021d69e2b68c3960903b702af7829fadcd80bd89b158150c85c4a75b2c8cb9c39452ae","52210279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f8179821021d69e2b68c3960903b702af7829fadcd80bd89b158150c85c4a75b2c8cb9c39452ae","522102a7ae1e0971fc1689bd66d2a7296da3a1662fd21a53c9e38979e0f090a375c12d21022adb62335f41eb4e27056ac37d462cda5ad783fa8e0e526ed79c752475db285d52ae","52210279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f8179821022adb62335f41eb4e27056ac37d462cda5ad783fa8e0e526ed79c752475db285d52ae","512103b9d1d0e2b4355ec3cdef7c11a5c0beff9e8b8d8372ab4b4e0aaf30e80173001951ae","76a9149144761ebaccd5b4bbdc2a35453585b5637b2f8588ac","522103f1848b40621c5d48471d9784c8174ca060555891ace6d2b03c58eece946b1a9121020ee5d32b54d429c152fdc7b1db84f2074b0564d35400d89d11870f9273ec140c52ae","76a914f4fa1cc7de742d135ea82c17adf0bb9cf5f4fb8388ac"],"ed47705334f4643892ca46396eb3f4196a5e30880589e4009ef38eae895d4a13","0afbc2920af1b027f31f87b592276eb4c32094bb4d3697021b4c6380","b6d98692cec5145f67585f3434ec3c2b3030182e1cb3ec58b855c5c164dfaaa3","Tx pays to empty output script"], -[180480,"00000000fd3ceb2404ff07a785c7fdcc76619edc8ed61bd25134eaa22084366a","020000006058aa080a655aa991a444bd7d1f2defd9a3bbe68aabb69030cf3b4e00000000d2e826bfd7ef0beaa891a7eedbc92cd6a544a6cb61c7bdaa436762eb2123ef9790f5f552ffff001d0002c90f0501000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0300c102024608062f503253482fffffffff01c0c6072a01000000232102e769e60137a4df6b0df8ebd387cca44c4c57ae74cc0114a8e8317c8f3bfd85e9ac00000000010000000381a0802911a01ffb025c4dea0bc77963e8c1bb46313b71164c53f72f37fe5248010000000151ffffffffc904b267833d215e2128bd9575242232ac2bc311550c7fc1f0ef6f264b40d14c010000000151ffffffffdf0915666649dba81886519c531649b7b02180b4af67d6885e871299e9d5f775000000000151ffffffff0180817dcb00000000232103bb52138972c48a132fc1f637858c5189607dd0f7fe40c4f20f6ad65f2d389ba4ac0000000001000000018da38b434fba82d66052af74fc5e4e94301b114d9bc03f819dc876398404c8b4010000006c493046022100fe738b7580dc5fb5168e51fc61b5aed211125eb71068031009a22d9bbad752c5022100be5086baa384d40bcab0fa586e4f728397388d86e18b66cc417dc4f7fa4f9878012103f233299455134caa2687bdf15cb0becdfb03bd0ff2ff38e65ec6b7834295c34fffffffff022ebc1400000000001976a9147779b7fba1c1e06b717069b80ca170e8b04458a488ac9879c40f000000001976a9142a0307cd925dbb66b534c4db33003dd18c57015788ac0000000001000000026139a62e3422a602de36c873a225c1d3ca5aeee598539ceecb9f0dc8d1ad0f83010000006b483045022100ad9f32b4a0a2ddc19b5a74eba78123e57616f1b3cfd72ce68c03ea35a3dda1f002200dbd22aa6da17213df5e70dfc3b2611d40f70c98ed9626aa5e2cde9d97461f0a012103ddb295d2f1e8319187738fb4b230fdd9aa29d0e01647f69f6d770b9ab24eea90ffffffff983c82c87cf020040d671956525014d5c2b28c6d948c85e1a522362c0059eeae010000006b4830450221009ca544274c786d30a5d5d25e17759201ea16d3aedddf0b9e9721246f7ef6b32e02202cfa5564b6e87dfd9fd98957820e4d4e6238baeb0f65fe305d91506bb13f5f4f012103c99113deac0d5d044e3ac0346abc02501542af8c8d3759f1382c72ff84e704f7ffffffff02c0c62d00000000001976a914ae19d27efe12f5a886dc79af37ad6805db6f922d88ac70ce2000000000001976a9143b8d051d37a07ea1042067e93efe63dbf73920b988ac000000000100000002be566e8cd9933f0c75c4a82c027f7d0c544d5c101d0607ef6ae5d07b98e7f1dc000000006b483045022036a8cdfd5ea7ebc06c2bfb6e4f942bbf9a1caeded41680d11a3a9f5d8284abad022100cacb92a5be3f39e8bc14db1710910ef7b395fa1e18f45d41c28d914fcdde33be012102bf59abf110b5131fae0a3ce1ec379329b4c896a6ae5d443edb68529cc2bc7816ffffffff96cf67645b76ceb23fe922874847456a15feee1655082ff32d25a6bf2c0dfc90000000006a47304402203471ca2001784a5ac0abab583581f2613523da47ec5f53df833c117b5abd81500220618a2847723d57324f2984678db556dbca1a72230fc7e39df04c2239942ba942012102925c9794fd7bb9f8b29e207d5fc491b1150135a21f505041858889fa4edf436fffffffff026c840f00000000001976a914797fb8777d7991d8284d88bfd421ce520f0f843188ac00ca9a3b000000001976a9146d10f3f592699265d10b106eda37c3ce793f7a8588ac00000000",["","","","76a9142903b138c24be9e070b3e73ec495d77a204615e788ac","76a91433a1941fd9a37b9821d376f5a51bd4b52fa50e2888ac","76a914e4374e8155d0865742ca12b8d4d14d41b57d682f88ac","76a914001fa7459a6cfc64bdc178ba7e7a21603bb2568f88ac","76a914f6039952bc2b307aeec5371bfb96b66078ec17f688ac"],"b109139671dbedc2b6fcd499a5480a7461ae458af8ff9411d819aa64ba6995d1","0db414c859a07e8205876354a210a75042d0463404913d61a8e068e58a3ae2aa080026","a0af77e0a7ed20ea78d2def3200cc24f08217dcd51755c7c7feb0e2ba8316c2d","Tx spends from empty output script"], -[926485,"000000000000015d6077a411a8f5cc95caf775ccf11c54e27df75ce58d187313","0000002060bbab0edbf3ef8a49608ee326f8fd75c473b7e3982095e2d100000000000000c30134f8c9b6d2470488d7a67a888f6fa12f8692e0c3411fbfb92f0f68f67eedae03ca57ef13021acc22dc4105010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff2f0315230e0004ae03ca57043e3d1e1d0c8796bf579aef0c0000000000122f4e696e6a61506f6f6c2f5345475749542fffffffff038427a112000000001976a914876fbb82ec05caa6af7a3b5e5a983aae6c6cc6d688ac0000000000000000266a24aa21a9ed5c748e121c0fe146d973a4ac26fa4a68b0549d46ee22d25f50a5e46fe1b377ee00000000000000002952534b424c4f434b3acd16772ad61a3c5f00287480b720f6035d5e54c9efc71be94bb5e3727f10909001200000000000000000000000000000000000000000000000000000000000000000000000000100000000010145310e878941a1b2bc2d33797ee4d89d95eaaf2e13488063a2aa9a74490f510a0100000023220020b6744de4f6ec63cc92f7c220cdefeeb1b1bed2b66c8e5706d80ec247d37e65a1ffffffff01002d3101000000001976a9143ebc40e411ed3c76f86711507ab952300890397288ac0400473044022001dd489a5d4e2fbd8a3ade27177f6b49296ba7695c40dbbe650ea83f106415fd02200b23a0602d8ff1bdf79dee118205fc7e9b40672bf31563e5741feb53fb86388501483045022100f88f040e90cc5dc6c6189d04718376ac19ed996bf9e4a3c29c3718d90ffd27180220761711f16c9e3a44f71aab55cbc0634907a1fa8bb635d971a9a01d368727bea10169522103b3623117e988b76aaabe3d63f56a4fc88b228a71e64c4cc551d1204822fe85cb2103dd823066e096f72ed617a41d3ca56717db335b1ea47a1b4c5c9dbdd0963acba621033d7c89bd9da29fa8d44db7906a9778b53121f72191184a9fee785c39180e4be153ae00000000010000000120925534261de4dcebb1ed5ab1b62bfe7a3ef968fb111dc2c910adfebc6e3bdf010000006b483045022100f50198f5ae66211a4f485190abe4dc7accdabe3bc214ebc9ea7069b97097d46e0220316a70a03014887086e335fc1b48358d46cd6bdc9af3b57c109c94af76fc915101210316cff587a01a2736d5e12e53551b18d73780b83c3bfb4fcf209c869b11b6415effffffff0220a10700000000001976a91450333046115eaa0ac9e0216565f945070e44573988ac2e7cd01a000000001976a914c01a7ca16b47be50cbdbc60724f701d52d75156688ac00000000010000000203a25f58630d7a1ea52550365fd2156683f56daf6ca73a4b4bbd097e66516322010000006a47304402204efc3d70e4ca3049c2a425025edf22d5ca355f9ec899dbfbbeeb2268533a0f2b02204780d3739653035af4814ea52e1396d021953f948c29754edd0ee537364603dc012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff03a25f58630d7a1ea52550365fd2156683f56daf6ca73a4b4bbd097e66516322000000006a47304402202d96defdc5b4af71d6ba28c9a6042c2d5ee7bc6de565d4db84ef517445626e03022022da80320e9e489c8f41b74833dfb6a54a4eb5087cdb46eb663eef0b25caa526012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff0200e1f5050000000017a914b7e6f7ff8658b2d1fb107e3d7be7af4742e6b1b3876f88fc00000000001976a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac0000000001000000043ffd60d3818431c495b89be84afac205d5d1ed663009291c560758bbd0a66df5010000006b483045022100f344607de9df42049688dcae8ff1db34c0c7cd25ec05516e30d2bc8f12ac9b2f022060b648f6a21745ea6d9782e17bcc4277b5808326488a1f40d41e125879723d3a012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffffa5379401cce30f84731ef1ba65ce27edf2cc7ce57704507ebe8714aa16a96b92010000006a473044022020c37a63bf4d7f564c2192528709b6a38ab8271bd96898c6c2e335e5208661580220435c6f1ad4d9305d2c0a818b2feb5e45d443f2f162c0f61953a14d097fd07064012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff70e731e193235ff12c3184510895731a099112ffca4b00246c60003c40f843ce000000006a473044022053760f74c29a879e30a17b5f03a5bb057a5751a39f86fa6ecdedc36a1b7db04c022041d41c9b95f00d2d10a0373322a9025dba66c942196bc9d8adeb0e12d3024728012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff66b7a71b3e50379c8e85fc18fe3f1a408fc985f257036c34702ba205cef09f6f000000006a4730440220499bf9e2db3db6e930228d0661395f65431acae466634d098612fd80b08459ee022040e069fc9e3c60009f521cef54c38aadbd1251aee37940e6018aadb10f194d6a012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff0200e1f5050000000017a9148fc37ad460fdfbd2b44fe446f6e3071a4f64faa6878f447f0b000000001976a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac00000000",["a914feb8a29635c56d9cd913122f90678756bf23887687","76a914c01a7ca16b47be50cbdbc60724f701d52d75156688ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac"],"da49977ba1ee0d620a2c4f8f646b03cd0d230f5c6c994722e3ba884889f0be1a","09027acea61b6cc3fb33f5d52f7d088a6b2f75d234e89ca800","4cd9dd007a325199102f1fc0b7d77ca25ee3c84d46018c4353ecfcb56c0d3e7a","Duplicate pushdata 913bcc2be49cb534c20474c4dee1e9c4c317e7eb"], -[987876,"0000000000000c00901f2049055e2a437c819d79a3d54fd63e6af796cd7b8a79","000000202694f74969fdb542090e95a56bc8aa2d646e27033850e32f1c5f000000000000f7e53676b3f12d5beb524ed617f2d25f5a93b5f4f52c1ba2678260d72712f8dd0a6dfe5740257e1a4b1768960101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff1603e4120ff9c30a1c216900002f424d4920546573742fffffff0001205fa012000000001e76a914c486de584a735ec2f22da7cd9681614681f92173d83d0aa68688ac00000000",[],"e9d729b72d533c29abe5276d5cf6c152f3723f10efe000b1e0c9ca5265a8beb6","010c0b40","e6137ae5a8424c40da1e5023c16975cc97b09300b4c050e6b1c713add3836c40","Coinbase tx has unparseable output script"], -[1263442,"000000006f27ddfe1dd680044a34548f41bed47eba9e6f0b310da21423bc5f33","000000201c8d1a529c39a396db2db234d5ec152fa651a2872966daccbde028b400000000083f14492679151dbfaa1a825ef4c18518e780c1f91044180280a7d33f4a98ff5f45765aaddc001d38333b9a02010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff230352471300fe5f45765afe94690a000963676d696e6572343208000000000000000000ffffffff024423a804000000001976a914f2c25ac3d59f3d674b1d1d0a25c27339aaac0ba688ac0000000000000000266a24aa21a9edcb26cb3052426b9ebb4d19c819ef87c19677bbf3a7c46ef0855bd1b2abe83491012000000000000000000000000000000000000000000000000000000000000000000000000002000000000101d20978463906ba4ff5e7192494b88dd5eb0de85d900ab253af909106faa22cc5010000000004000000014777ff000000000016001446c29eabe8208a33aa1023c741fa79aa92e881ff0347304402207d7ca96134f2bcfdd6b536536fdd39ad17793632016936f777ebb32c22943fda02206014d2fb8a6aa58279797f861042ba604ebd2f8f61e5bddbd9d3be5a245047b201004b632103eeaeba7ce5dc2470221e9517fb498e8d6bd4e73b85b8be655196972eb9ccd5566754b2752103a40b74d43df244799d041f32ce1ad515a6cd99501701540e38750d883ae21d3a68ac00000000",["002027a5000c7917f785d8fc6e5a55adfca8717ecb973ebb7743849ff956d896a7ed"],"a4a4d6c6034da8aa06f01fe71f1fffbd79e032006b07f6c7a2c60a66aa310c01","0385acb4f0fe889ef0","3588f34fbbc11640f9ed40b2a66a4e096215d50389691309c1dac74d4268aa81","Includes witness data"] +[180480,"00000000fd3ceb2404ff07a785c7fdcc76619edc8ed61bd25134eaa22084366a","020000006058aa080a655aa991a444bd7d1f2defd9a3bbe68aabb69030cf3b4e00000000d2e826bfd7ef0beaa891a7eedbc92cd6a544a6cb61c7bdaa436762eb2123ef9790f5f552ffff001d0002c90f0501000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0300c102024608062f503253482fffffffff01c0c6072a01000000232102e769e60137a4df6b0df8ebd387cca44c4c57ae74cc0114a8e8317c8f3bfd85e9ac00000000010000000381a0802911a01ffb025c4dea0bc77963e8c1bb46313b71164c53f72f37fe5248010000000151ffffffffc904b267833d215e2128bd9575242232ac2bc311550c7fc1f0ef6f264b40d14c010000000151ffffffffdf0915666649dba81886519c531649b7b02180b4af67d6885e871299e9d5f775000000000151ffffffff0180817dcb00000000232103bb52138972c48a132fc1f637858c5189607dd0f7fe40c4f20f6ad65f2d389ba4ac0000000001000000018da38b434fba82d66052af74fc5e4e94301b114d9bc03f819dc876398404c8b4010000006c493046022100fe738b7580dc5fb5168e51fc61b5aed211125eb71068031009a22d9bbad752c5022100be5086baa384d40bcab0fa586e4f728397388d86e18b66cc417dc4f7fa4f9878012103f233299455134caa2687bdf15cb0becdfb03bd0ff2ff38e65ec6b7834295c34fffffffff022ebc1400000000001976a9147779b7fba1c1e06b717069b80ca170e8b04458a488ac9879c40f000000001976a9142a0307cd925dbb66b534c4db33003dd18c57015788ac0000000001000000026139a62e3422a602de36c873a225c1d3ca5aeee598539ceecb9f0dc8d1ad0f83010000006b483045022100ad9f32b4a0a2ddc19b5a74eba78123e57616f1b3cfd72ce68c03ea35a3dda1f002200dbd22aa6da17213df5e70dfc3b2611d40f70c98ed9626aa5e2cde9d97461f0a012103ddb295d2f1e8319187738fb4b230fdd9aa29d0e01647f69f6d770b9ab24eea90ffffffff983c82c87cf020040d671956525014d5c2b28c6d948c85e1a522362c0059eeae010000006b4830450221009ca544274c786d30a5d5d25e17759201ea16d3aedddf0b9e9721246f7ef6b32e02202cfa5564b6e87dfd9fd98957820e4d4e6238baeb0f65fe305d91506bb13f5f4f012103c99113deac0d5d044e3ac0346abc02501542af8c8d3759f1382c72ff84e704f7ffffffff02c0c62d00000000001976a914ae19d27efe12f5a886dc79af37ad6805db6f922d88ac70ce2000000000001976a9143b8d051d37a07ea1042067e93efe63dbf73920b988ac000000000100000002be566e8cd9933f0c75c4a82c027f7d0c544d5c101d0607ef6ae5d07b98e7f1dc000000006b483045022036a8cdfd5ea7ebc06c2bfb6e4f942bbf9a1caeded41680d11a3a9f5d8284abad022100cacb92a5be3f39e8bc14db1710910ef7b395fa1e18f45d41c28d914fcdde33be012102bf59abf110b5131fae0a3ce1ec379329b4c896a6ae5d443edb68529cc2bc7816ffffffff96cf67645b76ceb23fe922874847456a15feee1655082ff32d25a6bf2c0dfc90000000006a47304402203471ca2001784a5ac0abab583581f2613523da47ec5f53df833c117b5abd81500220618a2847723d57324f2984678db556dbca1a72230fc7e39df04c2239942ba942012102925c9794fd7bb9f8b29e207d5fc491b1150135a21f505041858889fa4edf436fffffffff026c840f00000000001976a914797fb8777d7991d8284d88bfd421ce520f0f843188ac00ca9a3b000000001976a9146d10f3f592699265d10b106eda37c3ce793f7a8588ac00000000",["","","","76a9142903b138c24be9e070b3e73ec495d77a204615e788ac","76a91433a1941fd9a37b9821d376f5a51bd4b52fa50e2888ac","76a914e4374e8155d0865742ca12b8d4d14d41b57d682f88ac","76a914001fa7459a6cfc64bdc178ba7e7a21603bb2568f88ac","76a914f6039952bc2b307aeec5371bfb96b66078ec17f688ac"],"d34ef98386f413769502808d4bac5f20f8dfd5bffc9eedafaa71de0eb1f01489","0db414c859a07e8205876354a210a75042d0463404913d61a8e068e58a3ae2aa080026","c582d51c0ca365e3fcf36c51cb646d7f83a67e867cb4743fd2128e3e022b700c","Tx spends from empty output script"], +[926485,"000000000000015d6077a411a8f5cc95caf775ccf11c54e27df75ce58d187313","0000002060bbab0edbf3ef8a49608ee326f8fd75c473b7e3982095e2d100000000000000c30134f8c9b6d2470488d7a67a888f6fa12f8692e0c3411fbfb92f0f68f67eedae03ca57ef13021acc22dc4105010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff2f0315230e0004ae03ca57043e3d1e1d0c8796bf579aef0c0000000000122f4e696e6a61506f6f6c2f5345475749542fffffffff038427a112000000001976a914876fbb82ec05caa6af7a3b5e5a983aae6c6cc6d688ac0000000000000000266a24aa21a9ed5c748e121c0fe146d973a4ac26fa4a68b0549d46ee22d25f50a5e46fe1b377ee00000000000000002952534b424c4f434b3acd16772ad61a3c5f00287480b720f6035d5e54c9efc71be94bb5e3727f10909001200000000000000000000000000000000000000000000000000000000000000000000000000100000000010145310e878941a1b2bc2d33797ee4d89d95eaaf2e13488063a2aa9a74490f510a0100000023220020b6744de4f6ec63cc92f7c220cdefeeb1b1bed2b66c8e5706d80ec247d37e65a1ffffffff01002d3101000000001976a9143ebc40e411ed3c76f86711507ab952300890397288ac0400473044022001dd489a5d4e2fbd8a3ade27177f6b49296ba7695c40dbbe650ea83f106415fd02200b23a0602d8ff1bdf79dee118205fc7e9b40672bf31563e5741feb53fb86388501483045022100f88f040e90cc5dc6c6189d04718376ac19ed996bf9e4a3c29c3718d90ffd27180220761711f16c9e3a44f71aab55cbc0634907a1fa8bb635d971a9a01d368727bea10169522103b3623117e988b76aaabe3d63f56a4fc88b228a71e64c4cc551d1204822fe85cb2103dd823066e096f72ed617a41d3ca56717db335b1ea47a1b4c5c9dbdd0963acba621033d7c89bd9da29fa8d44db7906a9778b53121f72191184a9fee785c39180e4be153ae00000000010000000120925534261de4dcebb1ed5ab1b62bfe7a3ef968fb111dc2c910adfebc6e3bdf010000006b483045022100f50198f5ae66211a4f485190abe4dc7accdabe3bc214ebc9ea7069b97097d46e0220316a70a03014887086e335fc1b48358d46cd6bdc9af3b57c109c94af76fc915101210316cff587a01a2736d5e12e53551b18d73780b83c3bfb4fcf209c869b11b6415effffffff0220a10700000000001976a91450333046115eaa0ac9e0216565f945070e44573988ac2e7cd01a000000001976a914c01a7ca16b47be50cbdbc60724f701d52d75156688ac00000000010000000203a25f58630d7a1ea52550365fd2156683f56daf6ca73a4b4bbd097e66516322010000006a47304402204efc3d70e4ca3049c2a425025edf22d5ca355f9ec899dbfbbeeb2268533a0f2b02204780d3739653035af4814ea52e1396d021953f948c29754edd0ee537364603dc012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff03a25f58630d7a1ea52550365fd2156683f56daf6ca73a4b4bbd097e66516322000000006a47304402202d96defdc5b4af71d6ba28c9a6042c2d5ee7bc6de565d4db84ef517445626e03022022da80320e9e489c8f41b74833dfb6a54a4eb5087cdb46eb663eef0b25caa526012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff0200e1f5050000000017a914b7e6f7ff8658b2d1fb107e3d7be7af4742e6b1b3876f88fc00000000001976a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac0000000001000000043ffd60d3818431c495b89be84afac205d5d1ed663009291c560758bbd0a66df5010000006b483045022100f344607de9df42049688dcae8ff1db34c0c7cd25ec05516e30d2bc8f12ac9b2f022060b648f6a21745ea6d9782e17bcc4277b5808326488a1f40d41e125879723d3a012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffffa5379401cce30f84731ef1ba65ce27edf2cc7ce57704507ebe8714aa16a96b92010000006a473044022020c37a63bf4d7f564c2192528709b6a38ab8271bd96898c6c2e335e5208661580220435c6f1ad4d9305d2c0a818b2feb5e45d443f2f162c0f61953a14d097fd07064012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff70e731e193235ff12c3184510895731a099112ffca4b00246c60003c40f843ce000000006a473044022053760f74c29a879e30a17b5f03a5bb057a5751a39f86fa6ecdedc36a1b7db04c022041d41c9b95f00d2d10a0373322a9025dba66c942196bc9d8adeb0e12d3024728012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff66b7a71b3e50379c8e85fc18fe3f1a408fc985f257036c34702ba205cef09f6f000000006a4730440220499bf9e2db3db6e930228d0661395f65431acae466634d098612fd80b08459ee022040e069fc9e3c60009f521cef54c38aadbd1251aee37940e6018aadb10f194d6a012103f7a897e4dbecab2264b21917f90664ea8256189ea725d28740cf7ba5d85b5763ffffffff0200e1f5050000000017a9148fc37ad460fdfbd2b44fe446f6e3071a4f64faa6878f447f0b000000001976a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac00000000",["a914feb8a29635c56d9cd913122f90678756bf23887687","76a914c01a7ca16b47be50cbdbc60724f701d52d75156688ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac","76a914913bcc2be49cb534c20474c4dee1e9c4c317e7eb88ac"],"8f13b9a9c85611635b47906c3053ac53cfcec7211455d4cb0d63dc9acc13d472","09027acea61b6cc3fb33f5d52f7d088a6b2f75d234e89ca800","546c574a0472144bcaf9b6aeabf26372ad87c7af7d1ee0dbfae5e099abeae49c","Duplicate pushdata 913bcc2be49cb534c20474c4dee1e9c4c317e7eb"], +[987876,"0000000000000c00901f2049055e2a437c819d79a3d54fd63e6af796cd7b8a79","000000202694f74969fdb542090e95a56bc8aa2d646e27033850e32f1c5f000000000000f7e53676b3f12d5beb524ed617f2d25f5a93b5f4f52c1ba2678260d72712f8dd0a6dfe5740257e1a4b1768960101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff1603e4120ff9c30a1c216900002f424d4920546573742fffffff0001205fa012000000001e76a914c486de584a735ec2f22da7cd9681614681f92173d83d0aa68688ac00000000",[],"fe4d230dbb0f4fec9bed23a5283e08baf996e3f32b93f52c7de1f641ddfd04ad","010c0b40","0965a544743bbfa36f254446e75630c09404b3d164a261892372977538928ed5","Coinbase tx has unparseable output script"], +[1263442,"000000006f27ddfe1dd680044a34548f41bed47eba9e6f0b310da21423bc5f33","000000201c8d1a529c39a396db2db234d5ec152fa651a2872966daccbde028b400000000083f14492679151dbfaa1a825ef4c18518e780c1f91044180280a7d33f4a98ff5f45765aaddc001d38333b9a02010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff230352471300fe5f45765afe94690a000963676d696e6572343208000000000000000000ffffffff024423a804000000001976a914f2c25ac3d59f3d674b1d1d0a25c27339aaac0ba688ac0000000000000000266a24aa21a9edcb26cb3052426b9ebb4d19c819ef87c19677bbf3a7c46ef0855bd1b2abe83491012000000000000000000000000000000000000000000000000000000000000000000000000002000000000101d20978463906ba4ff5e7192494b88dd5eb0de85d900ab253af909106faa22cc5010000000004000000014777ff000000000016001446c29eabe8208a33aa1023c741fa79aa92e881ff0347304402207d7ca96134f2bcfdd6b536536fdd39ad17793632016936f777ebb32c22943fda02206014d2fb8a6aa58279797f861042ba604ebd2f8f61e5bddbd9d3be5a245047b201004b632103eeaeba7ce5dc2470221e9517fb498e8d6bd4e73b85b8be655196972eb9ccd5566754b2752103a40b74d43df244799d041f32ce1ad515a6cd99501701540e38750d883ae21d3a68ac00000000",["002027a5000c7917f785d8fc6e5a55adfca8717ecb973ebb7743849ff956d896a7ed"],"31d66d516a9eda7de865df29f6ef6cb8e4bf9309e5dac899968a9a62a5df61e3","0385acb4f0fe889ef0","4e6d564c2a2452065c205dd7eb2791124e0c4e0dbb064c410c24968572589dec","Includes witness data"], +[1414221,"0000000000000027b2b3b3381f114f674f481544ff2be37ae3788d7e078383b1","000000204ea88307a7959d8207968f152bedca5a93aefab253f1fb2cfb032a400000000070cebb14ec6dbc27a9dfd066d9849a4d3bac5f674665f73a5fe1de01a022a0c851fda85bf05f4c19a779d1450102000000010000000000000000000000000000000000000000000000000000000000000000ffffffff18034d94154d696e6572476174653030310d000000f238f401ffffffff01c817a804000000000000000000",[],"5e5e12d90693c8e936f01847859404c67482439681928353ca1296982042864e","00","021e8882ef5a0ed932edeebbecfeda1d7ce528ec7b3daa27641acf1189d7b5dc","Empty data"] ] diff --git a/src/test/setup_common.cpp b/src/test/setup_common.cpp index 29633cc7bf..6b671b2400 100644 --- a/src/test/setup_common.cpp +++ b/src/test/setup_common.cpp @@ -94,7 +94,7 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha nScriptCheckThreads = 3; for (int i = 0; i < nScriptCheckThreads - 1; i++) - threadGroup.create_thread(&ThreadScriptCheck); + threadGroup.create_thread([i]() { return ThreadScriptCheck(i); }); g_banman = MakeUnique<BanMan>(GetDataDir() / "banlist.dat", nullptr, DEFAULT_MISBEHAVING_BANTIME); g_connman = MakeUnique<CConnman>(0x1337, 0x1337); // Deterministic randomness for tests. diff --git a/src/test/skiplist_tests.cpp b/src/test/skiplist_tests.cpp index fcd92da8ca..3d39dfdb75 100644 --- a/src/test/skiplist_tests.cpp +++ b/src/test/skiplist_tests.cpp @@ -136,7 +136,7 @@ BOOST_AUTO_TEST_CASE(findearliestatleast_test) // Pick a random element in vBlocksMain. int r = InsecureRandRange(vBlocksMain.size()); int64_t test_time = vBlocksMain[r].nTime; - CBlockIndex *ret = chain.FindEarliestAtLeast(test_time); + CBlockIndex* ret = chain.FindEarliestAtLeast(test_time, 0); BOOST_CHECK(ret->nTimeMax >= test_time); BOOST_CHECK((ret->pprev==nullptr) || ret->pprev->nTimeMax < test_time); BOOST_CHECK(vBlocksMain[r].GetAncestor(ret->nHeight) == ret); @@ -158,22 +158,34 @@ BOOST_AUTO_TEST_CASE(findearliestatleast_edge_test) CChain chain; chain.SetTip(&blocks.back()); - BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(50)->nHeight, 0); - BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(100)->nHeight, 0); - BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(150)->nHeight, 3); - BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(200)->nHeight, 3); - BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(250)->nHeight, 6); - BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(300)->nHeight, 6); - BOOST_CHECK(!chain.FindEarliestAtLeast(350)); - - BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(0)->nHeight, 0); - BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(-1)->nHeight, 0); - - BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(std::numeric_limits<int64_t>::min())->nHeight, 0); - BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(-int64_t(std::numeric_limits<unsigned int>::max()) - 1)->nHeight, 0); - BOOST_CHECK(!chain.FindEarliestAtLeast(std::numeric_limits<int64_t>::max())); - BOOST_CHECK(!chain.FindEarliestAtLeast(std::numeric_limits<unsigned int>::max())); - BOOST_CHECK(!chain.FindEarliestAtLeast(int64_t(std::numeric_limits<unsigned int>::max()) + 1)); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(50, 0)->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(100, 0)->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(150, 0)->nHeight, 3); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(200, 0)->nHeight, 3); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(250, 0)->nHeight, 6); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(300, 0)->nHeight, 6); + BOOST_CHECK(!chain.FindEarliestAtLeast(350, 0)); + + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(0, 0)->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(-1, 0)->nHeight, 0); + + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(std::numeric_limits<int64_t>::min(), 0)->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(-int64_t(std::numeric_limits<unsigned int>::max()) - 1, 0)->nHeight, 0); + BOOST_CHECK(!chain.FindEarliestAtLeast(std::numeric_limits<int64_t>::max(), 0)); + BOOST_CHECK(!chain.FindEarliestAtLeast(std::numeric_limits<unsigned int>::max(), 0)); + BOOST_CHECK(!chain.FindEarliestAtLeast(int64_t(std::numeric_limits<unsigned int>::max()) + 1, 0)); + + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(0, -1)->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(0, 0)->nHeight, 0); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(0, 3)->nHeight, 3); + BOOST_CHECK_EQUAL(chain.FindEarliestAtLeast(0, 8)->nHeight, 8); + BOOST_CHECK(!chain.FindEarliestAtLeast(0, 9)); + + CBlockIndex* ret1 = chain.FindEarliestAtLeast(100, 2); + BOOST_CHECK(ret1->nTimeMax >= 100 && ret1->nHeight == 2); + BOOST_CHECK(!chain.FindEarliestAtLeast(300, 9)); + CBlockIndex* ret2 = chain.FindEarliestAtLeast(200, 4); + BOOST_CHECK(ret2->nTimeMax >= 200 && ret2->nHeight == 4); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 6795308e83..0f1834240d 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -158,6 +158,9 @@ struct TestArgsManager : public ArgsManager AddArg(args[i], "", false, OptionsCategory::OPTIONS); } } + using ArgsManager::ReadConfigStream; + using ArgsManager::cs_args; + using ArgsManager::m_network; }; BOOST_AUTO_TEST_CASE(util_ParseParameters) @@ -575,6 +578,232 @@ BOOST_AUTO_TEST_CASE(util_GetChainName) BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); } +// Test different ways settings can be merged, and verify results. This test can +// be used to confirm that updates to settings code don't change behavior +// unintentially. +// +// The test covers: +// +// - Combining different setting actions. Possible actions are: configuring a +// setting, negating a setting (adding "-no" prefix), and configuring/negating +// settings in a network section (adding "main." or "test." prefixes). +// +// - Combining settings from command line arguments and a config file. +// +// - Combining SoftSet and ForceSet calls. +// +// - Testing "main" and "test" network values to make sure settings from network +// sections are applied and to check for mainnet-specific behaviors like +// inheriting settings from the default section. +// +// - Testing network-specific settings like "-wallet", that may be ignored +// outside a network section, and non-network specific settings like "-server" +// that aren't sensitive to the network. +// +struct SettingsMergeTestingSetup : public BasicTestingSetup { + //! Max number of actions to sequence together. Can decrease this when + //! debugging to make test results easier to understand. + static constexpr int MAX_ACTIONS = 3; + + enum Action { SET = 0, NEGATE, SECTION_SET, SECTION_NEGATE, END }; + using ActionList = Action[MAX_ACTIONS]; + + //! Enumerate all possible test configurations. + template <typename Fn> + void ForEachMergeSetup(Fn&& fn) + { + ForEachActionList([&](const ActionList& arg_actions) { + ForEachActionList([&](const ActionList& conf_actions) { + for (bool soft_set : {false, true}) { + for (bool force_set : {false, true}) { + for (const std::string& section : {CBaseChainParams::MAIN, CBaseChainParams::TESTNET}) { + for (const std::string& network : {CBaseChainParams::MAIN, CBaseChainParams::TESTNET}) { + for (bool net_specific : {false, true}) { + fn(arg_actions, conf_actions, soft_set, force_set, section, network, net_specific); + } + } + } + } + } + }); + }); + } + + //! Enumerate interesting combinations of actions. + template <typename Fn> + void ForEachActionList(Fn&& fn) + { + ActionList actions = {SET}; + for (bool done = false; !done;) { + int prev_action = -1; + bool skip_actions = false; + for (Action action : actions) { + if ((prev_action == END && action != END) || (prev_action != END && action == prev_action)) { + // To cut down list of enumerated settings, skip enumerating + // settings with ignored actions after an END, and settings that + // repeat the same action twice in a row. + skip_actions = true; + break; + } + prev_action = action; + } + if (!skip_actions) fn(actions); + done = true; + for (Action& action : actions) { + action = Action(action < END ? action + 1 : 0); + if (action) { + done = false; + break; + } + } + } + } + + //! Translate actions into a list of <key>=<value> setting strings. + std::vector<std::string> GetValues(const ActionList& actions, + const std::string& section, + const std::string& name, + const std::string& value_prefix) + { + std::vector<std::string> values; + int suffix = 0; + for (Action action : actions) { + if (action == END) break; + std::string prefix; + if (action == SECTION_SET || action == SECTION_NEGATE) prefix = section + "."; + if (action == SET || action == SECTION_SET) { + for (int i = 0; i < 2; ++i) { + values.push_back(prefix + name + "=" + value_prefix + std::to_string(++suffix)); + } + } + if (action == NEGATE || action == SECTION_NEGATE) { + values.push_back(prefix + "no" + name + "=1"); + } + } + return values; + } +}; + +// Regression test covering different ways config settings can be merged. The +// test parses and merges settings, representing the results as strings that get +// compared against an expected hash. To debug, the result strings can be dumped +// to a file (see below). +BOOST_FIXTURE_TEST_CASE(util_SettingsMerge, SettingsMergeTestingSetup) +{ + CHash256 out_sha; + FILE* out_file = nullptr; + if (const char* out_path = getenv("SETTINGS_MERGE_TEST_OUT")) { + out_file = fsbridge::fopen(out_path, "w"); + if (!out_file) throw std::system_error(errno, std::generic_category(), "fopen failed"); + } + + ForEachMergeSetup([&](const ActionList& arg_actions, const ActionList& conf_actions, bool soft_set, bool force_set, + const std::string& section, const std::string& network, bool net_specific) { + TestArgsManager parser; + LOCK(parser.cs_args); + + std::string desc = "net="; + desc += network; + parser.m_network = network; + + const std::string& name = net_specific ? "server" : "wallet"; + const std::string key = "-" + name; + parser.AddArg(key, name, false, OptionsCategory::OPTIONS); + if (net_specific) parser.SetNetworkOnlyArg(key); + + auto args = GetValues(arg_actions, section, name, "a"); + std::vector<const char*> argv = {"ignored"}; + for (auto& arg : args) { + arg.insert(0, "-"); + desc += " "; + desc += arg; + argv.push_back(arg.c_str()); + } + std::string error; + BOOST_CHECK(parser.ParseParameters(argv.size(), argv.data(), error)); + BOOST_CHECK_EQUAL(error, ""); + + std::string conf; + for (auto& conf_val : GetValues(conf_actions, section, name, "c")) { + desc += " "; + desc += conf_val; + conf += conf_val; + conf += "\n"; + } + std::istringstream conf_stream(conf); + BOOST_CHECK(parser.ReadConfigStream(conf_stream, "filepath", error)); + BOOST_CHECK_EQUAL(error, ""); + + if (soft_set) { + desc += " soft"; + parser.SoftSetArg(key, "soft1"); + parser.SoftSetArg(key, "soft2"); + } + + if (force_set) { + desc += " force"; + parser.ForceSetArg(key, "force1"); + parser.ForceSetArg(key, "force2"); + } + + desc += " || "; + + if (!parser.IsArgSet(key)) { + desc += "unset"; + BOOST_CHECK(!parser.IsArgNegated(key)); + BOOST_CHECK_EQUAL(parser.GetArg(key, "default"), "default"); + BOOST_CHECK(parser.GetArgs(key).empty()); + } else if (parser.IsArgNegated(key)) { + desc += "negated"; + BOOST_CHECK_EQUAL(parser.GetArg(key, "default"), "0"); + BOOST_CHECK(parser.GetArgs(key).empty()); + } else { + desc += parser.GetArg(key, "default"); + desc += " |"; + for (const auto& arg : parser.GetArgs(key)) { + desc += " "; + desc += arg; + } + } + + std::set<std::string> ignored = parser.GetUnsuitableSectionOnlyArgs(); + if (!ignored.empty()) { + desc += " | ignored"; + for (const auto& arg : ignored) { + desc += " "; + desc += arg; + } + } + + desc += "\n"; + + out_sha.Write((const unsigned char*)desc.data(), desc.size()); + if (out_file) { + BOOST_REQUIRE(fwrite(desc.data(), 1, desc.size(), out_file) == desc.size()); + } + }); + + if (out_file) { + if (fclose(out_file)) throw std::system_error(errno, std::generic_category(), "fclose failed"); + out_file = nullptr; + } + + unsigned char out_sha_bytes[CSHA256::OUTPUT_SIZE]; + out_sha.Finalize(out_sha_bytes); + std::string out_sha_hex = HexStr(std::begin(out_sha_bytes), std::end(out_sha_bytes)); + + // If check below fails, should manually dump the results with: + // + // SETTINGS_MERGE_TEST_OUT=results.txt ./test_bitcoin --run_test=util_tests/util_SettingsMerge + // + // And verify diff against previous results to make sure the changes are expected. + // + // Results file is formatted like: + // + // <input> || <IsArgSet/IsArgNegated/GetArg output> | <GetArgs output> | <GetUnsuitable output> + BOOST_CHECK_EQUAL(out_sha_hex, "80964e17fbd3c5569d3c824d032e28e2d319ef57494735b0e76eb7aad9957f2c"); +} + BOOST_AUTO_TEST_CASE(util_FormatMoney) { BOOST_CHECK_EQUAL(FormatMoney(0), "0.00"); diff --git a/src/test/util_threadnames_tests.cpp b/src/test/util_threadnames_tests.cpp new file mode 100644 index 0000000000..71c0168ca3 --- /dev/null +++ b/src/test/util_threadnames_tests.cpp @@ -0,0 +1,73 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/threadnames.h> +#include <test/setup_common.h> + +#include <thread> +#include <vector> +#include <set> +#include <mutex> + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(util_threadnames_tests, BasicTestingSetup) + +const std::string TEST_THREAD_NAME_BASE = "test_thread."; + +/** + * Run a bunch of threads to all call util::ThreadRename. + * + * @return the set of name each thread has after attempted renaming. + */ +std::set<std::string> RenameEnMasse(int num_threads) +{ + std::vector<std::thread> threads; + std::set<std::string> names; + std::mutex lock; + + auto RenameThisThread = [&](int i) { + util::ThreadRename(TEST_THREAD_NAME_BASE + std::to_string(i)); + std::lock_guard<std::mutex> guard(lock); + names.insert(util::ThreadGetInternalName()); + }; + + for (int i = 0; i < num_threads; ++i) { + threads.push_back(std::thread(RenameThisThread, i)); + } + + for (std::thread& thread : threads) thread.join(); + + return names; +} + +/** + * Rename a bunch of threads with the same basename (expect_multiple=true), ensuring suffixes are + * applied properly. + */ +BOOST_AUTO_TEST_CASE(util_threadnames_test_rename_threaded) +{ + BOOST_CHECK_EQUAL(util::ThreadGetInternalName(), ""); + +#if !defined(HAVE_THREAD_LOCAL) + // This test doesn't apply to platforms where we don't have thread_local. + return; +#endif + + std::set<std::string> names = RenameEnMasse(100); + + BOOST_CHECK_EQUAL(names.size(), 100); + + // Names "test_thread.[n]" should exist for n = [0, 99] + for (int i = 0; i < 100; ++i) { + BOOST_CHECK(names.find(TEST_THREAD_NAME_BASE + std::to_string(i)) != names.end()); + } + +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txdb.h b/src/txdb.h index d5a4bfaeb1..c4ece11503 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -37,6 +37,8 @@ static const int64_t nMaxBlockDBCache = 2; // Unlike for the UTXO database, for the txindex scenario the leveldb cache make // a meaningful difference: https://github.com/bitcoin/bitcoin/pull/8273#issuecomment-229601991 static const int64_t nMaxTxIndexCache = 1024; +//! Max memory allocated to all block filter index caches combined in MiB. +static const int64_t max_filter_index_cache = 1024; //! Max memory allocated to coin DB specific cache (MiB) static const int64_t nMaxCoinsDBCache = 8; diff --git a/src/ui_interface.cpp b/src/ui_interface.cpp index c084c4e0e2..31a95486d7 100644 --- a/src/ui_interface.cpp +++ b/src/ui_interface.cpp @@ -28,10 +28,6 @@ struct UISignals { boost::signals2::connection CClientUIInterface::signal_name##_connect(std::function<signal_name##Sig> fn) \ { \ return g_ui_signals.signal_name.connect(fn); \ - } \ - void CClientUIInterface::signal_name##_disconnect(std::function<signal_name##Sig> fn) \ - { \ - return g_ui_signals.signal_name.disconnect(&fn); \ } ADD_SIGNALS_IMPL_WRAPPER(ThreadSafeMessageBox); diff --git a/src/ui_interface.h b/src/ui_interface.h index 60d85bc142..d408f6f889 100644 --- a/src/ui_interface.h +++ b/src/ui_interface.h @@ -81,8 +81,7 @@ public: #define ADD_SIGNALS_DECL_WRAPPER(signal_name, rtype, ...) \ rtype signal_name(__VA_ARGS__); \ using signal_name##Sig = rtype(__VA_ARGS__); \ - boost::signals2::connection signal_name##_connect(std::function<signal_name##Sig> fn); \ - void signal_name##_disconnect(std::function<signal_name##Sig> fn); + boost::signals2::connection signal_name##_connect(std::function<signal_name##Sig> fn); /** Show message box. */ ADD_SIGNALS_DECL_WRAPPER(ThreadSafeMessageBox, bool, const std::string& message, const std::string& caption, unsigned int style); diff --git a/src/util/system.cpp b/src/util/system.cpp index 9594dd81bf..efd35bed55 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -60,10 +60,6 @@ #include <shlobj.h> #endif -#ifdef HAVE_SYS_PRCTL_H -#include <sys/prctl.h> -#endif - #ifdef HAVE_MALLOPT_ARENA_MAX #include <malloc.h> #endif @@ -1137,22 +1133,6 @@ void runCommand(const std::string& strCommand) LogPrintf("runCommand error: system(%s) returned %d\n", strCommand, nErr); } -void RenameThread(const char* name) -{ -#if defined(PR_SET_NAME) - // Only the first 15 characters are used (16 - NUL terminator) - ::prctl(PR_SET_NAME, name, 0, 0, 0); -#elif (defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)) - pthread_set_name_np(pthread_self(), name); - -#elif defined(MAC_OSX) - pthread_setname_np(name); -#else - // Prevent warnings for unused parameters... - (void)name; -#endif -} - void SetupEnvironment() { #ifdef HAVE_MALLOPT_ARENA_MAX diff --git a/src/util/system.h b/src/util/system.h index 54eb88e261..1a83cb67b1 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -20,6 +20,7 @@ #include <fs.h> #include <logging.h> #include <sync.h> +#include <util/threadnames.h> #include <tinyformat.h> #include <util/memory.h> #include <util/time.h> @@ -325,15 +326,12 @@ std::string HelpMessageOpt(const std::string& option, const std::string& message */ int GetNumCores(); -void RenameThread(const char* name); - /** * .. and a wrapper that just calls func once */ template <typename Callable> void TraceThread(const char* name, Callable func) { - std::string s = strprintf("bitcoin-%s", name); - RenameThread(s.c_str()); + util::ThreadRename(name); try { LogPrintf("%s thread start\n", name); diff --git a/src/util/threadnames.cpp b/src/util/threadnames.cpp new file mode 100644 index 0000000000..7b0d744aec --- /dev/null +++ b/src/util/threadnames.cpp @@ -0,0 +1,57 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <atomic> +#include <thread> + +#include <util/threadnames.h> + +#ifdef HAVE_SYS_PRCTL_H +#include <sys/prctl.h> // For prctl, PR_SET_NAME, PR_GET_NAME +#endif + +//! Set the thread's name at the process level. Does not affect the +//! internal name. +static void SetThreadName(const char* name) +{ +#if defined(PR_SET_NAME) + // Only the first 15 characters are used (16 - NUL terminator) + ::prctl(PR_SET_NAME, name, 0, 0, 0); +#elif (defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)) + pthread_set_name_np(pthread_self(), name); +#elif defined(MAC_OSX) + pthread_setname_np(name); +#else + // Prevent warnings for unused parameters... + (void)name; +#endif +} + +// If we have thread_local, just keep thread ID and name in a thread_local +// global. +#if defined(HAVE_THREAD_LOCAL) + +static thread_local std::string g_thread_name; +const std::string& util::ThreadGetInternalName() { return g_thread_name; } +//! Set the in-memory internal name for this thread. Does not affect the process +//! name. +static void SetInternalName(std::string name) { g_thread_name = std::move(name); } + +// Without thread_local available, don't handle internal name at all. +#else + +static const std::string empty_string; +const std::string& util::ThreadGetInternalName() { return empty_string; } +static void SetInternalName(std::string name) { } +#endif + +void util::ThreadRename(std::string&& name) +{ + SetThreadName(("bitcoin-" + name).c_str()); + SetInternalName(std::move(name)); +} diff --git a/src/util/threadnames.h b/src/util/threadnames.h new file mode 100644 index 0000000000..aaf07b9bf8 --- /dev/null +++ b/src/util/threadnames.h @@ -0,0 +1,21 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_THREADNAMES_H +#define BITCOIN_UTIL_THREADNAMES_H + +#include <string> + +namespace util { +//! Rename a thread both in terms of an internal (in-memory) name as well +//! as its system thread name. +void ThreadRename(std::string&&); + +//! Get the thread's internal (in-memory) name; used e.g. for identification in +//! logging. +const std::string& ThreadGetInternalName(); + +} // namespace util + +#endif // BITCOIN_UTIL_THREADNAMES_H diff --git a/src/validation.cpp b/src/validation.cpp index be6257ea28..d8c8e638ce 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -8,7 +8,6 @@ #include <arith_uint256.h> #include <chain.h> #include <chainparams.h> -#include <checkpoints.h> #include <checkqueue.h> #include <consensus/consensus.h> #include <consensus/merkle.h> @@ -37,6 +36,7 @@ #include <txdb.h> #include <txmempool.h> #include <ui_interface.h> +#include <uint256.h> #include <undo.h> #include <util/moneystr.h> #include <util/rbf.h> @@ -48,6 +48,7 @@ #include <future> #include <sstream> +#include <string> #include <boost/algorithm/string/replace.hpp> #include <boost/thread.hpp> @@ -252,7 +253,6 @@ uint256 hashAssumeValid; arith_uint256 nMinimumChainWork; CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); -CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; CBlockPolicyEstimator feeEstimator; CTxMemPool mempool(&feeEstimator); @@ -562,6 +562,13 @@ static bool CheckInputsFromMempoolAndCache(const CTransaction& tx, CValidationSt return CheckInputs(tx, state, view, true, flags, cacheSigStore, true, txdata); } +/** + * @param[out] coins_to_uncache Return any outpoints which were not previously present in the + * coins cache, but were added as a result of validating the tx + * for mempool acceptance. This allows the caller to optionally + * remove the cache additions if the associated transaction ends + * up being rejected by the mempool. + */ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool& pool, CValidationState& state, const CTransactionRef& ptx, bool* pfMissingInputs, int64_t nAcceptTime, std::list<CTransactionRef>* plTxnReplaced, bool bypass_limits, const CAmount& nAbsurdFee, std::vector<COutPoint>& coins_to_uncache, bool test_accept) EXCLUSIVE_LOCKS_REQUIRED(cs_main) @@ -657,6 +664,10 @@ static bool AcceptToMemoryPoolWorker(const CChainParams& chainparams, CTxMemPool if (!pcoinsTip->HaveCoinInCache(txin.prevout)) { coins_to_uncache.push_back(txin.prevout); } + + // Note: this call may add txin.prevout to the coins cache + // (pcoinsTip.cacheCoins) by way of FetchCoin(). It should be removed + // later (via coins_to_uncache) if this tx turns out to be invalid. if (!view.HaveCoin(txin.prevout)) { // Are inputs missing because we already have the tx? for (size_t out = 0; out < tx.vout.size(); out++) { @@ -978,6 +989,11 @@ static bool AcceptToMemoryPoolWithTime(const CChainParams& chainparams, CTxMemPo std::vector<COutPoint> coins_to_uncache; bool res = AcceptToMemoryPoolWorker(chainparams, pool, state, tx, pfMissingInputs, nAcceptTime, plTxnReplaced, bypass_limits, nAbsurdFee, coins_to_uncache, test_accept); if (!res) { + // Remove coins that were not present in the coins cache before calling ATMPW; + // this is to prevent memory DoS in case we receive a large number of + // invalid transactions that attempt to overrun the in-memory coins cache + // (`CCoinsViewCache::cacheCoins`). + for (const COutPoint& hashTx : coins_to_uncache) pcoinsTip->Uncache(hashTx); } @@ -1654,8 +1670,8 @@ static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, CValidationState& static CCheckQueue<CScriptCheck> scriptcheckqueue(128); -void ThreadScriptCheck() { - RenameThread("bitcoin-scriptch"); +void ThreadScriptCheck(int worker_num) { + util::ThreadRename(strprintf("scriptch.%i", worker_num)); scriptcheckqueue.Thread(); } @@ -3188,6 +3204,22 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc return commitment; } +//! Returns last CBlockIndex* that is a checkpoint +static CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) +{ + const MapCheckpoints& checkpoints = data.mapCheckpoints; + + for (const MapCheckpoints::value_type& i : reverse_iterate(checkpoints)) + { + const uint256& hash = i.second; + CBlockIndex* pindex = LookupBlockIndex(hash); + if (pindex) { + return pindex; + } + } + return nullptr; +} + /** Context-dependent validity checks. * By "context", we mean only the previous block headers, but not the UTXO * set; UTXO-related validity checks are done in ConnectBlock(). @@ -3212,7 +3244,7 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationSta // Don't accept any forks from the main chain prior to last checkpoint. // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in our // MapBlockIndex. - CBlockIndex* pcheckpoint = Checkpoints::GetLastCheckpoint(params.Checkpoints()); + CBlockIndex* pcheckpoint = GetLastCheckpoint(params.Checkpoints()); if (pcheckpoint && nHeight < pcheckpoint->nHeight) return state.DoS(100, error("%s: forked chain older than last checkpoint (height %d)", __func__, nHeight), REJECT_CHECKPOINT, "bad-fork-prior-to-checkpoint"); } diff --git a/src/validation.h b/src/validation.h index 9f00cab49e..61a7a270fc 100644 --- a/src/validation.h +++ b/src/validation.h @@ -53,12 +53,6 @@ static const bool DEFAULT_WHITELISTRELAY = true; static const bool DEFAULT_WHITELISTFORCERELAY = false; /** Default for -minrelaytxfee, minimum relay fee for transactions */ static const unsigned int DEFAULT_MIN_RELAY_TX_FEE = 1000; -//! -maxtxfee default -static const CAmount DEFAULT_TRANSACTION_MAXFEE = COIN / 10; -//! Discourage users to set fees higher than this amount (in satoshis) per kB -static const CAmount HIGH_TX_FEE_PER_KB = COIN / 100; -//! -maxtxfee will warn if called with a higher fee than this amount (in satoshis) -static const CAmount HIGH_MAX_TX_FEE = 100 * HIGH_TX_FEE_PER_KB; /** Default for -limitancestorcount, max number of in-mempool ancestors */ static const unsigned int DEFAULT_ANCESTOR_LIMIT = 25; /** Default for -limitancestorsize, maximum kilobytes of tx + all in-mempool ancestors */ @@ -116,6 +110,7 @@ static const int64_t MAX_FEE_ESTIMATION_TIP_AGE = 3 * 60 * 60; static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; +static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; /** Default for -persistmempool */ static const bool DEFAULT_PERSIST_MEMPOOL = true; @@ -162,8 +157,6 @@ extern bool fCheckpointsEnabled; extern size_t nCoinCacheUsage; /** A fee rate smaller than this is considered zero fee (for relaying, mining and transaction creation) */ extern CFeeRate minRelayTxFee; -/** Absolute maximum transaction fee (in satoshis) used by wallet and mempool (rejects high fee in sendrawtransaction) */ -extern CAmount maxTxFee; /** If the tip is older than this (in seconds), the node is considered to be in initial block download. */ extern int64_t nMaxTipAge; extern bool fEnableReplacement; @@ -254,7 +247,7 @@ bool LoadChainTip(const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_m /** Unload database information */ void UnloadBlockIndex(); /** Run an instance of the script checking thread */ -void ThreadScriptCheck(); +void ThreadScriptCheck(int worker_num); /** Check whether we are doing an initial block download (synchronizing from disk or network) */ bool IsInitialBlockDownload(); /** Retrieve a transaction (from memory pool, or from disk, if possible) */ diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 4ec9dca420..78db4df5e5 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -146,9 +146,9 @@ Result CreateTotalBumpTransaction(const CWallet* wallet, const uint256& txid, co } // Check that in all cases the new fee doesn't violate maxTxFee - const CAmount max_tx_fee = wallet->chain().maxTxFee(); + const CAmount max_tx_fee = wallet->m_default_max_tx_fee; if (new_fee > max_tx_fee) { - errors.push_back(strprintf("Specified or calculated fee %s is too high (cannot be higher than maxTxFee %s)", + errors.push_back(strprintf("Specified or calculated fee %s is too high (cannot be higher than -maxtxfee %s)", FormatMoney(new_fee), FormatMoney(max_tx_fee))); return Result::WALLET_ERROR; } diff --git a/src/wallet/fees.cpp b/src/wallet/fees.cpp index 560c86a70a..d9ae18ed60 100644 --- a/src/wallet/fees.cpp +++ b/src/wallet/fees.cpp @@ -22,7 +22,7 @@ CAmount GetMinimumFee(const CWallet& wallet, unsigned int nTxBytes, const CCoinC { CAmount fee_needed = GetMinimumFeeRate(wallet, coin_control, feeCalc).GetFee(nTxBytes); // Always obey the maximum - const CAmount max_tx_fee = wallet.chain().maxTxFee(); + const CAmount max_tx_fee = wallet.m_default_max_tx_fee; if (fee_needed > max_tx_fee) { fee_needed = max_tx_fee; if (feeCalc) feeCalc->reason = FeeReason::MAXTXFEE; diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index e7eea94e06..47ef01bfd1 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -48,6 +48,8 @@ void WalletInit::AddWalletOptions() const gArgs.AddArg("-fallbackfee=<amt>", strprintf("A fee rate (in %s/kB) that will be used when fee estimation has insufficient data (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_FALLBACK_FEE)), false, OptionsCategory::WALLET); gArgs.AddArg("-keypool=<n>", strprintf("Set key pool size to <n> (default: %u)", DEFAULT_KEYPOOL_SIZE), false, OptionsCategory::WALLET); + gArgs.AddArg("-maxtxfee=<amt>", strprintf("Maximum total fees (in %s) to use in a single wallet transaction; setting this too low may abort large transactions (default: %s)", + CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MAXFEE)), false, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-mintxfee=<amt>", strprintf("Fees (in %s/kB) smaller than this are considered zero fee for transaction creation (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MINFEE)), false, OptionsCategory::WALLET); gArgs.AddArg("-paytxfee=<amt>", strprintf("Fee (in %s/kB) to add to transactions you send (default: %s)", @@ -124,10 +126,6 @@ bool WalletInit::ParameterInteraction() const if (gArgs.GetArg("-prune", 0) && gArgs.GetBoolArg("-rescan", false)) return InitError(_("Rescans are not possible in pruned mode. You will need to use -reindex which will download the whole blockchain again.")); - if (::minRelayTxFee.GetFeePerK() > HIGH_TX_FEE_PER_KB) - InitWarning(AmountHighWarn("-minrelaytxfee") + " " + - _("The wallet will avoid paying less than the minimum relay fee.")); - return true; } diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 0d804dcc10..5a22508c4b 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -3150,7 +3150,14 @@ UniValue signrawtransactionwithwallet(const JSONRPCRequest& request) LOCK(pwallet->cs_wallet); EnsureWalletIsUnlocked(pwallet); - return SignTransaction(pwallet->chain(), mtx, request.params[1], pwallet, false, request.params[2]); + // Fetch previous transactions (inputs): + std::map<COutPoint, Coin> coins; + for (const CTxIn& txin : mtx.vin) { + coins[txin.prevout]; // Create empty map entry keyed by prevout. + } + pwallet->chain().findCoins(coins); + + return SignTransaction(mtx, request.params[1], pwallet, coins, false, request.params[2]); } static UniValue bumpfee(const JSONRPCRequest& request) @@ -3683,9 +3690,20 @@ static UniValue getaddressesbylabel(const JSONRPCRequest& request) // Find all addresses that have the given label UniValue ret(UniValue::VOBJ); + std::set<std::string> addresses; for (const std::pair<const CTxDestination, CAddressBookData>& item : pwallet->mapAddressBook) { if (item.second.name == label) { - ret.pushKV(EncodeDestination(item.first), AddressBookDataToJSON(item.second, false)); + std::string address = EncodeDestination(item.first); + // CWallet::mapAddressBook is not expected to contain duplicate + // address strings, but build a separate set as a precaution just in + // case it does. + bool unique = addresses.emplace(address).second; + assert(unique); + // UniValue::pushKV checks if the key exists in O(N) + // and since duplicate addresses are unexpected (checked with + // std::set in O(log(N))), UniValue::__pushKV is used instead, + // which currently is O(1). + ret.__pushKV(address, AddressBookDataToJSON(item.second, false)); } } diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index b7ca746f8b..34b9770e8b 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -69,8 +69,7 @@ static void add_coin(const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = fa std::unique_ptr<CWalletTx> wtx = MakeUnique<CWalletTx>(&testWallet, MakeTransactionRef(std::move(tx))); if (fIsFromMe) { - wtx->fDebitCached = true; - wtx->nDebitCached = 1; + wtx->m_amounts[CWalletTx::DEBIT].Set(ISMINE_SPENDABLE, 1); } COutput output(wtx.get(), nInput, nAge, true /* spendable */, true /* solvable */, true /* safe */); vCoins.push_back(output); @@ -115,7 +114,7 @@ inline std::vector<OutputGroup>& GroupCoins(const std::vector<COutput>& coins) { static std::vector<OutputGroup> static_groups; static_groups.clear(); - for (auto& coin : coins) static_groups.emplace_back(coin.GetInputCoin(), coin.nDepth, coin.tx->fDebitCached && coin.tx->nDebitCached == 1 /* HACK: we can't figure out the is_me flag so we use the conditions defined above; perhaps set safe to false for !fIsFromMe in add_coin() */, 0, 0); + for (auto& coin : coins) static_groups.emplace_back(coin.GetInputCoin(), coin.nDepth, coin.tx->m_amounts[CWalletTx::DEBIT].m_cached[ISMINE_SPENDABLE] && coin.tx->m_amounts[CWalletTx::DEBIT].m_value[ISMINE_SPENDABLE] == 1 /* HACK: we can't figure out the is_me flag so we use the conditions defined above; perhaps set safe to false for !fIsFromMe in add_coin() */, 0, 0); return static_groups; } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 4737e2f6f4..4ab94f0c2c 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1,12 +1,11 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2018 The Bitcoin Core developers +// Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <wallet/wallet.h> #include <chain.h> -#include <checkpoints.h> #include <consensus/consensus.h> #include <consensus/validation.h> #include <fs.h> @@ -1729,7 +1728,7 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r uint256 start_block; { auto locked_chain = chain().lock(); - const Optional<int> start_height = locked_chain->findFirstBlockWithTime(startTime - TIMESTAMP_WINDOW, &start_block); + const Optional<int> start_height = locked_chain->findFirstBlockWithTimeAndHeight(startTime - TIMESTAMP_WINDOW, 0, &start_block); const Optional<int> tip_height = locked_chain->getHeight(); WalletLogPrintf("%s: Rescanning last %i blocks\n", __func__, tip_height && start_height ? *tip_height - *start_height + 1 : 0); } @@ -1772,6 +1771,7 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_block, const uint256& stop_block, const WalletRescanReserver& reserver, bool fUpdate) { int64_t nNow = GetTime(); + int64_t start_time = GetTimeMillis(); assert(reserver.isReserved()); @@ -1780,90 +1780,90 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc WalletLogPrintf("Rescan started from block %s...\n", start_block.ToString()); + fAbortRescan = false; + ShowProgress(strprintf("%s " + _("Rescanning..."), GetDisplayName()), 0); // show rescan progress in GUI as dialog or on splashscreen, if -rescan on startup + uint256 tip_hash; + // The way the 'block_height' is initialized is just a workaround for the gcc bug #47679 since version 4.6.0. + Optional<int> block_height = MakeOptional(false, int()); + double progress_begin; + double progress_end; { - fAbortRescan = false; - ShowProgress(strprintf("%s " + _("Rescanning..."), GetDisplayName()), 0); // show rescan progress in GUI as dialog or on splashscreen, if -rescan on startup - uint256 tip_hash; - // The way the 'block_height' is initialized is just a workaround for the gcc bug #47679 since version 4.6.0. - Optional<int> block_height = MakeOptional(false, int()); - double progress_begin; - double progress_end; - { - auto locked_chain = chain().lock(); - if (Optional<int> tip_height = locked_chain->getHeight()) { - tip_hash = locked_chain->getBlockHash(*tip_height); - } - block_height = locked_chain->getBlockHeight(block_hash); - progress_begin = chain().guessVerificationProgress(block_hash); - progress_end = chain().guessVerificationProgress(stop_block.IsNull() ? tip_hash : stop_block); - } - double progress_current = progress_begin; - while (block_height && !fAbortRescan && !chain().shutdownRequested()) { - if (*block_height % 100 == 0 && progress_end - progress_begin > 0.0) { - ShowProgress(strprintf("%s " + _("Rescanning..."), GetDisplayName()), std::max(1, std::min(99, (int)((progress_current - progress_begin) / (progress_end - progress_begin) * 100)))); - } - if (GetTime() >= nNow + 60) { - nNow = GetTime(); - WalletLogPrintf("Still rescanning. At block %d. Progress=%f\n", *block_height, progress_current); - } + auto locked_chain = chain().lock(); + if (Optional<int> tip_height = locked_chain->getHeight()) { + tip_hash = locked_chain->getBlockHash(*tip_height); + } + block_height = locked_chain->getBlockHeight(block_hash); + progress_begin = chain().guessVerificationProgress(block_hash); + progress_end = chain().guessVerificationProgress(stop_block.IsNull() ? tip_hash : stop_block); + } + double progress_current = progress_begin; + while (block_height && !fAbortRescan && !chain().shutdownRequested()) { + if (*block_height % 100 == 0 && progress_end - progress_begin > 0.0) { + ShowProgress(strprintf("%s " + _("Rescanning..."), GetDisplayName()), std::max(1, std::min(99, (int)((progress_current - progress_begin) / (progress_end - progress_begin) * 100)))); + } + if (GetTime() >= nNow + 60) { + nNow = GetTime(); + WalletLogPrintf("Still rescanning. At block %d. Progress=%f\n", *block_height, progress_current); + } - CBlock block; - if (chain().findBlock(block_hash, &block) && !block.IsNull()) { - auto locked_chain = chain().lock(); - LOCK(cs_wallet); - if (!locked_chain->getBlockHeight(block_hash)) { - // Abort scan if current block is no longer active, to prevent - // marking transactions as coming from the wrong block. - // TODO: This should return success instead of failure, see - // https://github.com/bitcoin/bitcoin/pull/14711#issuecomment-458342518 - result.last_failed_block = block_hash; - result.status = ScanResult::FAILURE; - break; - } - for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { - SyncTransaction(block.vtx[posInBlock], block_hash, posInBlock, fUpdate); - } - // scan succeeded, record block as most recent successfully scanned - result.last_scanned_block = block_hash; - result.last_scanned_height = *block_height; - } else { - // could not scan block, keep scanning but record this block as the most recent failure + CBlock block; + if (chain().findBlock(block_hash, &block) && !block.IsNull()) { + auto locked_chain = chain().lock(); + LOCK(cs_wallet); + if (!locked_chain->getBlockHeight(block_hash)) { + // Abort scan if current block is no longer active, to prevent + // marking transactions as coming from the wrong block. + // TODO: This should return success instead of failure, see + // https://github.com/bitcoin/bitcoin/pull/14711#issuecomment-458342518 result.last_failed_block = block_hash; result.status = ScanResult::FAILURE; + break; } - if (block_hash == stop_block) { + for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { + SyncTransaction(block.vtx[posInBlock], block_hash, posInBlock, fUpdate); + } + // scan succeeded, record block as most recent successfully scanned + result.last_scanned_block = block_hash; + result.last_scanned_height = *block_height; + } else { + // could not scan block, keep scanning but record this block as the most recent failure + result.last_failed_block = block_hash; + result.status = ScanResult::FAILURE; + } + if (block_hash == stop_block) { + break; + } + { + auto locked_chain = chain().lock(); + Optional<int> tip_height = locked_chain->getHeight(); + if (!tip_height || *tip_height <= block_height || !locked_chain->getBlockHeight(block_hash)) { + // break successfully when rescan has reached the tip, or + // previous block is no longer on the chain due to a reorg break; } - { - auto locked_chain = chain().lock(); - Optional<int> tip_height = locked_chain->getHeight(); - if (!tip_height || *tip_height <= block_height || !locked_chain->getBlockHeight(block_hash)) { - // break successfully when rescan has reached the tip, or - // previous block is no longer on the chain due to a reorg - break; - } - // increment block and verification progress - block_hash = locked_chain->getBlockHash(++*block_height); - progress_current = chain().guessVerificationProgress(block_hash); + // increment block and verification progress + block_hash = locked_chain->getBlockHash(++*block_height); + progress_current = chain().guessVerificationProgress(block_hash); - // handle updated tip hash - const uint256 prev_tip_hash = tip_hash; - tip_hash = locked_chain->getBlockHash(*tip_height); - if (stop_block.IsNull() && prev_tip_hash != tip_hash) { - // in case the tip has changed, update progress max - progress_end = chain().guessVerificationProgress(tip_hash); - } + // handle updated tip hash + const uint256 prev_tip_hash = tip_hash; + tip_hash = locked_chain->getBlockHash(*tip_height); + if (stop_block.IsNull() && prev_tip_hash != tip_hash) { + // in case the tip has changed, update progress max + progress_end = chain().guessVerificationProgress(tip_hash); } } - ShowProgress(strprintf("%s " + _("Rescanning..."), GetDisplayName()), 100); // hide progress dialog in GUI - if (block_height && fAbortRescan) { - WalletLogPrintf("Rescan aborted at block %d. Progress=%f\n", *block_height, progress_current); - result.status = ScanResult::USER_ABORT; - } else if (block_height && chain().shutdownRequested()) { - WalletLogPrintf("Rescan interrupted by shutdown request at block %d. Progress=%f\n", *block_height, progress_current); - result.status = ScanResult::USER_ABORT; - } + } + ShowProgress(strprintf("%s " + _("Rescanning..."), GetDisplayName()), 100); // hide progress dialog in GUI + if (block_height && fAbortRescan) { + WalletLogPrintf("Rescan aborted at block %d. Progress=%f\n", *block_height, progress_current); + result.status = ScanResult::USER_ABORT; + } else if (block_height && chain().shutdownRequested()) { + WalletLogPrintf("Rescan interrupted by shutdown request at block %d. Progress=%f\n", *block_height, progress_current); + result.status = ScanResult::USER_ABORT; + } else { + WalletLogPrintf("Rescan completed in %15dms\n", GetTimeMillis() - start_time); } return result; } @@ -1932,33 +1932,26 @@ std::set<uint256> CWalletTx::GetConflicts() const return result; } +CAmount CWalletTx::GetCachableAmount(AmountType type, const isminefilter& filter, bool recalculate) const +{ + auto& amount = m_amounts[type]; + if (recalculate || !amount.m_cached[filter]) { + amount.Set(filter, type == DEBIT ? pwallet->GetDebit(*tx, filter) : pwallet->GetCredit(*tx, filter)); + } + return amount.m_value[filter]; +} + CAmount CWalletTx::GetDebit(const isminefilter& filter) const { if (tx->vin.empty()) return 0; CAmount debit = 0; - if(filter & ISMINE_SPENDABLE) - { - if (fDebitCached) - debit += nDebitCached; - else - { - nDebitCached = pwallet->GetDebit(*tx, ISMINE_SPENDABLE); - fDebitCached = true; - debit += nDebitCached; - } + if (filter & ISMINE_SPENDABLE) { + debit += GetCachableAmount(DEBIT, ISMINE_SPENDABLE); } - if(filter & ISMINE_WATCH_ONLY) - { - if(fWatchDebitCached) - debit += nWatchDebitCached; - else - { - nWatchDebitCached = pwallet->GetDebit(*tx, ISMINE_WATCH_ONLY); - fWatchDebitCached = true; - debit += nWatchDebitCached; - } + if (filter & ISMINE_WATCH_ONLY) { + debit += GetCachableAmount(DEBIT, ISMINE_WATCH_ONLY); } return debit; } @@ -1970,28 +1963,12 @@ CAmount CWalletTx::GetCredit(interfaces::Chain::Lock& locked_chain, const ismine return 0; CAmount credit = 0; - if (filter & ISMINE_SPENDABLE) - { + if (filter & ISMINE_SPENDABLE) { // GetBalance can assume transactions in mapWallet won't change - if (fCreditCached) - credit += nCreditCached; - else - { - nCreditCached = pwallet->GetCredit(*tx, ISMINE_SPENDABLE); - fCreditCached = true; - credit += nCreditCached; - } + credit += GetCachableAmount(CREDIT, ISMINE_SPENDABLE); } - if (filter & ISMINE_WATCH_ONLY) - { - if (fWatchCreditCached) - credit += nWatchCreditCached; - else - { - nWatchCreditCached = pwallet->GetCredit(*tx, ISMINE_WATCH_ONLY); - fWatchCreditCached = true; - credit += nWatchCreditCached; - } + if (filter & ISMINE_WATCH_ONLY) { + credit += GetCachableAmount(CREDIT, ISMINE_WATCH_ONLY); } return credit; } @@ -1999,11 +1976,7 @@ CAmount CWalletTx::GetCredit(interfaces::Chain::Lock& locked_chain, const ismine CAmount CWalletTx::GetImmatureCredit(interfaces::Chain::Lock& locked_chain, bool fUseCache) const { if (IsImmatureCoinBase(locked_chain) && IsInMainChain(locked_chain)) { - if (fUseCache && fImmatureCreditCached) - return nImmatureCreditCached; - nImmatureCreditCached = pwallet->GetCredit(*tx, ISMINE_SPENDABLE); - fImmatureCreditCached = true; - return nImmatureCreditCached; + return GetCachableAmount(IMMATURE_CREDIT, ISMINE_SPENDABLE, !fUseCache); } return 0; @@ -2014,23 +1987,15 @@ CAmount CWalletTx::GetAvailableCredit(interfaces::Chain::Lock& locked_chain, boo if (pwallet == nullptr) return 0; + // Avoid caching ismine for NO or ALL cases (could remove this check and simplify in the future). + bool allow_cache = filter == ISMINE_SPENDABLE || filter == ISMINE_WATCH_ONLY; + // Must wait until coinbase is safely deep enough in the chain before valuing it if (IsImmatureCoinBase(locked_chain)) return 0; - CAmount* cache = nullptr; - bool* cache_used = nullptr; - - if (filter == ISMINE_SPENDABLE) { - cache = &nAvailableCreditCached; - cache_used = &fAvailableCreditCached; - } else if (filter == ISMINE_WATCH_ONLY) { - cache = &nAvailableWatchCreditCached; - cache_used = &fAvailableWatchCreditCached; - } - - if (fUseCache && cache_used && *cache_used) { - return *cache; + if (fUseCache && allow_cache && m_amounts[AVAILABLE_CREDIT].m_cached[filter]) { + return m_amounts[AVAILABLE_CREDIT].m_value[filter]; } CAmount nCredit = 0; @@ -2046,22 +2011,17 @@ CAmount CWalletTx::GetAvailableCredit(interfaces::Chain::Lock& locked_chain, boo } } - if (cache) { - *cache = nCredit; - assert(cache_used); - *cache_used = true; + if (allow_cache) { + m_amounts[AVAILABLE_CREDIT].Set(filter, nCredit); } + return nCredit; } CAmount CWalletTx::GetImmatureWatchOnlyCredit(interfaces::Chain::Lock& locked_chain, const bool fUseCache) const { if (IsImmatureCoinBase(locked_chain) && IsInMainChain(locked_chain)) { - if (fUseCache && fImmatureWatchCreditCached) - return nImmatureWatchCreditCached; - nImmatureWatchCreditCached = pwallet->GetCredit(*tx, ISMINE_WATCH_ONLY); - fImmatureWatchCreditCached = true; - return nImmatureWatchCreditCached; + return GetCachableAmount(IMMATURE_CREDIT, ISMINE_WATCH_ONLY, !fUseCache); } return 0; @@ -4204,6 +4164,29 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, return nullptr; } } + + if (gArgs.IsArgSet("-maxtxfee")) + { + CAmount nMaxFee = 0; + if (!ParseMoney(gArgs.GetArg("-maxtxfee", ""), nMaxFee)) { + chain.initError(AmountErrMsg("maxtxfee", gArgs.GetArg("-maxtxfee", ""))); + return nullptr; + } + if (nMaxFee > HIGH_MAX_TX_FEE) { + chain.initWarning(_("-maxtxfee is set very high! Fees this large could be paid on a single transaction.")); + } + if (CFeeRate(nMaxFee, 1000) < chain.relayMinFee()) { + chain.initError(strprintf(_("Invalid amount for -maxtxfee=<amount>: '%s' (must be at least the minrelay fee of %s to prevent stuck transactions)"), + gArgs.GetArg("-maxtxfee", ""), chain.relayMinFee().ToString())); + return nullptr; + } + walletInstance->m_default_max_tx_fee = nMaxFee; + } + + if (chain.relayMinFee().GetFeePerK() > HIGH_TX_FEE_PER_KB) + chain.initWarning(AmountHighWarn("-minrelaytxfee") + " " + + _("The wallet will avoid paying less than the minimum relay fee.")); + walletInstance->m_confirm_target = gArgs.GetArg("-txconfirmtarget", DEFAULT_TX_CONFIRM_TARGET); walletInstance->m_spend_zero_conf_change = gArgs.GetBoolArg("-spendzeroconfchange", DEFAULT_SPEND_ZEROCONF_CHANGE); walletInstance->m_signal_rbf = gArgs.GetBoolArg("-walletrbf", DEFAULT_WALLET_RBF); @@ -4258,12 +4241,11 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, // No need to read and scan block if block was created before // our wallet birthday (as adjusted for block time variability) if (walletInstance->nTimeFirstKey) { - if (Optional<int> first_block = locked_chain->findFirstBlockWithTimeAndHeight(walletInstance->nTimeFirstKey - TIMESTAMP_WINDOW, rescan_height)) { + if (Optional<int> first_block = locked_chain->findFirstBlockWithTimeAndHeight(walletInstance->nTimeFirstKey - TIMESTAMP_WINDOW, rescan_height, nullptr)) { rescan_height = *first_block; } } - nStart = GetTimeMillis(); { WalletRescanReserver reserver(walletInstance.get()); if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(locked_chain->getBlockHash(rescan_height), {} /* stop block */, reserver, true /* update */).status)) { @@ -4271,7 +4253,6 @@ std::shared_ptr<CWallet> CWallet::CreateWalletFromFile(interfaces::Chain& chain, return nullptr; } } - walletInstance->WalletLogPrintf("Rescan completed in %15dms\n", GetTimeMillis() - nStart); walletInstance->ChainStateFlushed(locked_chain->getTipLocator()); walletInstance->database->IncrementUpdateCounter(); @@ -4399,7 +4380,7 @@ bool CWalletTx::AcceptToMemoryPool(interfaces::Chain::Lock& locked_chain, CValid // user could call sendmoney in a loop and hit spurious out of funds errors // because we think that this newly generated transaction's change is // unavailable as we're not yet aware that it is in the mempool. - bool ret = locked_chain.submitToMemoryPool(tx, pwallet->chain().maxTxFee(), state); + bool ret = locked_chain.submitToMemoryPool(tx, pwallet->m_default_max_tx_fee, state); fInMempool |= ret; return ret; } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 62ba0aa962..900af75f4f 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -73,6 +73,12 @@ static const unsigned int DEFAULT_TX_CONFIRM_TARGET = 6; static const bool DEFAULT_WALLET_RBF = false; static const bool DEFAULT_WALLETBROADCAST = true; static const bool DEFAULT_DISABLE_WALLET = false; +//! -maxtxfee default +constexpr CAmount DEFAULT_TRANSACTION_MAXFEE{COIN / 10}; +//! Discourage users to set fees higher than this amount (in satoshis) per kB +constexpr CAmount HIGH_TX_FEE_PER_KB{COIN / 100}; +//! -maxtxfee will warn if called with a higher fee than this amount (in satoshis) +constexpr CAmount HIGH_MAX_TX_FEE{100 * HIGH_TX_FEE_PER_KB}; //! Pre-calculated constants for input size estimation in *virtual size* static constexpr size_t DUMMY_NESTED_P2WPKH_INPUT_SIZE = 91; @@ -369,24 +375,11 @@ public: std::multimap<int64_t, CWalletTx*>::const_iterator m_it_wtxOrdered; // memory only - mutable bool fDebitCached; - mutable bool fCreditCached; - mutable bool fImmatureCreditCached; - mutable bool fAvailableCreditCached; - mutable bool fWatchDebitCached; - mutable bool fWatchCreditCached; - mutable bool fImmatureWatchCreditCached; - mutable bool fAvailableWatchCreditCached; + enum AmountType { DEBIT, CREDIT, IMMATURE_CREDIT, AVAILABLE_CREDIT, AMOUNTTYPE_ENUM_ELEMENTS }; + CAmount GetCachableAmount(AmountType type, const isminefilter& filter, bool recalculate = false) const; + mutable CachableAmount m_amounts[AMOUNTTYPE_ENUM_ELEMENTS]; mutable bool fChangeCached; mutable bool fInMempool; - mutable CAmount nDebitCached; - mutable CAmount nCreditCached; - mutable CAmount nImmatureCreditCached; - mutable CAmount nAvailableCreditCached; - mutable CAmount nWatchDebitCached; - mutable CAmount nWatchCreditCached; - mutable CAmount nImmatureWatchCreditCached; - mutable CAmount nAvailableWatchCreditCached; mutable CAmount nChangeCached; CWalletTx(const CWallet* pwalletIn, CTransactionRef arg) : CMerkleTx(std::move(arg)) @@ -403,24 +396,8 @@ public: nTimeReceived = 0; nTimeSmart = 0; fFromMe = false; - fDebitCached = false; - fCreditCached = false; - fImmatureCreditCached = false; - fAvailableCreditCached = false; - fWatchDebitCached = false; - fWatchCreditCached = false; - fImmatureWatchCreditCached = false; - fAvailableWatchCreditCached = false; fChangeCached = false; fInMempool = false; - nDebitCached = 0; - nCreditCached = 0; - nImmatureCreditCached = 0; - nAvailableCreditCached = 0; - nWatchDebitCached = 0; - nWatchCreditCached = 0; - nAvailableWatchCreditCached = 0; - nImmatureWatchCreditCached = 0; nChangeCached = 0; nOrderPos = -1; } @@ -464,14 +441,10 @@ public: //! make sure balances are recalculated void MarkDirty() { - fCreditCached = false; - fAvailableCreditCached = false; - fImmatureCreditCached = false; - fWatchDebitCached = false; - fWatchCreditCached = false; - fAvailableWatchCreditCached = false; - fImmatureWatchCreditCached = false; - fDebitCached = false; + m_amounts[DEBIT].Reset(); + m_amounts[CREDIT].Reset(); + m_amounts[IMMATURE_CREDIT].Reset(); + m_amounts[AVAILABLE_CREDIT].Reset(); fChangeCached = false; } @@ -989,6 +962,8 @@ public: CFeeRate m_discard_rate{DEFAULT_DISCARD_FEE}; OutputType m_default_address_type{DEFAULT_ADDRESS_TYPE}; OutputType m_default_change_type{DEFAULT_CHANGE_TYPE}; + /** Absolute maximum transaction fee (in satoshis) used by default for the wallet */ + CAmount m_default_max_tx_fee{DEFAULT_TRANSACTION_MAXFEE}; bool NewKeyPool(); size_t KeypoolCountExternalKeys() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp index 2d8adf8ba6..b227a15556 100644 --- a/src/wallet/walletutil.cpp +++ b/src/wallet/walletutil.cpp @@ -31,6 +31,8 @@ fs::path GetWalletDir() static bool IsBerkeleyBtree(const fs::path& path) { + if (!fs::exists(path)) return false; + // A Berkeley DB Btree file has at least 4K. // This check also prevents opening lock files. boost::system::error_code ec; diff --git a/test/config.ini.in b/test/config.ini.in index 6b7ef70659..060c553da2 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -6,6 +6,7 @@ # test/functional/test_runner.py and test/util/bitcoin-util-test.py [environment] +PACKAGE_NAME=@PACKAGE_NAME@ SRCDIR=@abs_top_srcdir@ BUILDDIR=@abs_top_builddir@ EXEEXT=@EXEEXT@ diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py index 02deae92f3..d262dae5aa 100644 --- a/test/functional/data/invalid_txs.py +++ b/test/functional/data/invalid_txs.py @@ -71,9 +71,13 @@ class InputMissing(BadTxTemplate): reject_reason = "bad-txns-vin-empty" expect_disconnect = False + # We use a blank transaction here to make sure + # it is interpreted as a non-witness transaction. + # Otherwise the transaction will fail the + # "surpufluous witness" check during deserialization + # rather than the input count check. def get_tx(self): tx = CTransaction() - tx.vout.append(CTxOut(0, sc.CScript([sc.OP_TRUE] * 100))) tx.calc_sha256() return tx diff --git a/test/functional/feature_assumevalid.py b/test/functional/feature_assumevalid.py index e7e4f84ad9..b7814bf33e 100755 --- a/test/functional/feature_assumevalid.py +++ b/test/functional/feature_assumevalid.py @@ -32,7 +32,7 @@ Start three nodes: import time from test_framework.blocktools import (create_block, create_coinbase) -from test_framework.key import CECKey +from test_framework.key import ECKey from test_framework.messages import ( CBlockHeader, COutPoint, @@ -104,9 +104,9 @@ class AssumeValidTest(BitcoinTestFramework): self.blocks = [] # Get a pubkey for the coinbase TXO - coinbase_key = CECKey() - coinbase_key.set_secretbytes(b"horsebattery") - coinbase_pubkey = coinbase_key.get_pubkey() + coinbase_key = ECKey() + coinbase_key.generate() + coinbase_pubkey = coinbase_key.get_pubkey().get_bytes() # Create the first block with a coinbase output to our key height = 1 diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index e7a888c329..72eb4f804f 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -14,7 +14,7 @@ from test_framework.blocktools import ( get_legacy_sigopcount_block, MAX_BLOCK_SIGOPS, ) -from test_framework.key import CECKey +from test_framework.key import ECKey from test_framework.messages import ( CBlock, COIN, @@ -86,9 +86,9 @@ class FullBlockTest(BitcoinTestFramework): self.bootstrap_p2p() # Add one p2p connection to the node self.block_heights = {} - self.coinbase_key = CECKey() - self.coinbase_key.set_secretbytes(b"horsebattery") - self.coinbase_pubkey = self.coinbase_key.get_pubkey() + self.coinbase_key = ECKey() + self.coinbase_key.generate() + self.coinbase_pubkey = self.coinbase_key.get_pubkey().get_bytes() self.tip = None self.blocks = {} self.genesis_hash = int(self.nodes[0].getbestblockhash(), 16) @@ -146,20 +146,6 @@ class FullBlockTest(BitcoinTestFramework): badtx = template.get_tx() if TxTemplate != invalid_txs.InputMissing: self.sign_tx(badtx, attempt_spend_tx) - else: - # Segwit is active in regtest at this point, so to deserialize a - # transaction without any inputs correctly, we set the outputs - # to an empty list. This is a hack, as the serialization of an - # empty list of outputs is deserialized as flags==0 and thus - # deserialization of the outputs is skipped. - # A policy check requires "loose" txs to be of a minimum size, - # so vtx is not set to be empty in the TxTemplate class and we - # only apply the workaround where txs are not "loose", i.e. in - # blocks. - # - # The workaround has the purpose that both sides calculate - # the same tx hash in the merkle tree - badtx.vout = [] badtx.rehash() badblock = self.update_block(blockname, [badtx]) self.send_blocks( @@ -528,7 +514,7 @@ class FullBlockTest(BitcoinTestFramework): tx.vin.append(CTxIn(COutPoint(b39.vtx[i].sha256, 0), b'')) # Note: must pass the redeem_script (not p2sh_script) to the signature hash function (sighash, err) = SignatureHash(redeem_script, tx, 1, SIGHASH_ALL) - sig = self.coinbase_key.sign(sighash) + bytes(bytearray([SIGHASH_ALL])) + sig = self.coinbase_key.sign_ecdsa(sighash) + bytes(bytearray([SIGHASH_ALL])) scriptSig = CScript([sig, redeem_script]) tx.vin[1].scriptSig = scriptSig @@ -1284,7 +1270,7 @@ class FullBlockTest(BitcoinTestFramework): tx.vin[0].scriptSig = CScript() return (sighash, err) = SignatureHash(spend_tx.vout[0].scriptPubKey, tx, 0, SIGHASH_ALL) - tx.vin[0].scriptSig = CScript([self.coinbase_key.sign(sighash) + bytes(bytearray([SIGHASH_ALL]))]) + tx.vin[0].scriptSig = CScript([self.coinbase_key.sign_ecdsa(sighash) + bytes(bytearray([SIGHASH_ALL]))]) def create_and_sign_transaction(self, spend_tx, value, script=CScript([OP_TRUE])): tx = self.create_tx(spend_tx, 0, value, script) diff --git a/test/functional/feature_filelock.py b/test/functional/feature_filelock.py index 9fb0d35a68..ba116e41f5 100755 --- a/test/functional/feature_filelock.py +++ b/test/functional/feature_filelock.py @@ -23,7 +23,7 @@ class FilelockTest(BitcoinTestFramework): self.log.info("Using datadir {}".format(datadir)) self.log.info("Check that we can't start a second bitcoind instance using the same datadir") - expected_msg = "Error: Cannot obtain a lock on data directory {}. Bitcoin Core is probably already running.".format(datadir) + expected_msg = "Error: Cannot obtain a lock on data directory {0}. {1} is probably already running.".format(datadir, self.config['environment']['PACKAGE_NAME']) self.nodes[1].assert_start_raises_init_error(extra_args=['-datadir={}'.format(self.nodes[0].datadir), '-noserver'], expected_msg=expected_msg) if self.is_wallet_compiled(): diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index 8fb7c49640..e2b3b2d544 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -35,16 +35,17 @@ def mine_large_blocks(node, n): # followed by 950k of OP_NOP. This would be non-standard in a non-coinbase # transaction but is consensus valid. + # Set the nTime if this is the first time this function has been called. + # A static variable ensures that time is monotonicly increasing and is therefore + # different for each block created => blockhash is unique. + if "nTimes" not in mine_large_blocks.__dict__: + mine_large_blocks.nTime = 0 + # Get the block parameters for the first block big_script = CScript([OP_RETURN] + [OP_NOP] * 950000) best_block = node.getblock(node.getbestblockhash()) height = int(best_block["height"]) + 1 - try: - # Static variable ensures that time is monotonicly increasing and is therefore - # different for each block created => blockhash is unique. - mine_large_blocks.nTime = min(mine_large_blocks.nTime, int(best_block["time"])) + 1 - except AttributeError: - mine_large_blocks.nTime = int(best_block["time"]) + 1 + mine_large_blocks.nTime = max(mine_large_blocks.nTime, int(best_block["time"])) + 1 previousblockhash = int(best_block["hash"], 16) for _ in range(n): diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 15b7ba9ae1..7bb7044cc0 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -16,7 +16,7 @@ class TestBitcoinCli(BitcoinTestFramework): """Main test logic""" cli_response = self.nodes[0].cli("-version").send_cli() - assert "Bitcoin Core RPC client version" in cli_response + assert "{} RPC client version".format(self.config['environment']['PACKAGE_NAME']) in cli_response self.log.info("Compare responses from getwalletinfo RPC and `bitcoin-cli getwalletinfo`") if self.is_wallet_compiled(): diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 700fdf6e04..481d697e63 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -3,11 +3,12 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid network messages.""" +import asyncio import os import struct from test_framework import messages -from test_framework.mininode import P2PDataStore +from test_framework.mininode import P2PDataStore, NetworkThread from test_framework.test_framework import BitcoinTestFramework @@ -143,8 +144,15 @@ class InvalidMessagesTest(BitcoinTestFramework): def test_magic_bytes(self): conn = self.nodes[0].add_p2p_connection(P2PDataStore()) - conn._on_data = lambda: None # Need to ignore all incoming messages from now, since they come with "invalid" magic bytes - conn.magic_bytes = b'\x00\x11\x22\x32' + + def swap_magic_bytes(): + conn._on_data = lambda: None # Need to ignore all incoming messages from now, since they come with "invalid" magic bytes + conn.magic_bytes = b'\x00\x11\x22\x32' + + # Call .result() to block until the atomic swap is complete, otherwise + # we might run into races later on + asyncio.run_coroutine_threadsafe(asyncio.coroutine(swap_magic_bytes)(), NetworkThread.network_event_loop).result() + with self.nodes[0].assert_debug_log(['PROCESSMESSAGE: INVALID MESSAGESTART ping']): conn.send_message(messages.msg_ping(nonce=0xff)) conn.wait_for_disconnect(timeout=1) diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index a901e11536..000c30646a 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -9,7 +9,7 @@ import struct import time from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment, get_witness_script, WITNESS_COMMITMENT_HEADER -from test_framework.key import CECKey, CPubKey +from test_framework.key import ECKey from test_framework.messages import ( BIP125_SEQUENCE_NUMBER, CBlock, @@ -36,6 +36,7 @@ from test_framework.messages import ( ser_vector, sha256, uint256_from_str, + FromHex, ) from test_framework.mininode import ( P2PInterface, @@ -77,6 +78,7 @@ from test_framework.util import ( disconnect_nodes, get_bip9_status, hex_str_to_bytes, + assert_raises_rpc_error, ) # The versionbit bit used to signal activation of SegWit @@ -100,7 +102,7 @@ def get_p2pkh_script(pubkeyhash): def sign_p2pk_witness_input(script, tx_to, in_idx, hashtype, value, key): """Add signature for a P2PK witness program.""" tx_hash = SegwitVersion1SignatureHash(script, tx_to, in_idx, hashtype, value) - signature = key.sign(tx_hash) + chr(hashtype).encode('latin-1') + signature = key.sign_ecdsa(tx_hash) + chr(hashtype).encode('latin-1') tx_to.wit.vtxinwit[in_idx].scriptWitness.stack = [signature, script] tx_to.rehash() @@ -269,6 +271,7 @@ class SegWitTest(BitcoinTestFramework): self.test_non_standard_witness() self.test_upgrade_after_activation() self.test_witness_sigops() + self.test_superfluous_witness() # Individual tests @@ -1357,7 +1360,8 @@ class SegWitTest(BitcoinTestFramework): def test_segwit_versions(self): """Test validity of future segwit version transactions. - Future segwit version transactions are non-standard, but valid in blocks. + Future segwit versions are non-standard to spend, but valid in blocks. + Sending to future segwit versions is always allowed. Can run this before and after segwit activation.""" NUM_SEGWIT_VERSIONS = 17 # will test OP_0, OP1, ..., OP_16 @@ -1397,7 +1401,7 @@ class SegWitTest(BitcoinTestFramework): assert len(self.nodes[0].getrawmempool()) == 0 # Finally, verify that version 0 -> version 1 transactions - # are non-standard + # are standard script_pubkey = CScript([CScriptOp(OP_1), witness_hash]) tx2 = CTransaction() tx2.vin = [CTxIn(COutPoint(tx.sha256, 0), b"")] @@ -1405,10 +1409,9 @@ class SegWitTest(BitcoinTestFramework): tx2.wit.vtxinwit.append(CTxInWitness()) tx2.wit.vtxinwit[0].scriptWitness.stack = [witness_program] tx2.rehash() - # Gets accepted to test_node, because standardness of outputs isn't - # checked with fRequireStandard + # Gets accepted to both policy-enforcing nodes and others. test_transaction_acceptance(self.nodes[0], self.test_node, tx2, with_witness=True, accepted=True) - test_transaction_acceptance(self.nodes[1], self.std_node, tx2, with_witness=True, accepted=False) + test_transaction_acceptance(self.nodes[1], self.std_node, tx2, with_witness=True, accepted=True) temp_utxo.pop() # last entry in temp_utxo was the output we just spent temp_utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) @@ -1479,10 +1482,9 @@ class SegWitTest(BitcoinTestFramework): # Segwit transactions using uncompressed pubkeys are not accepted # under default policy, but should still pass consensus. - key = CECKey() - key.set_secretbytes(b"9") - key.set_compressed(False) - pubkey = CPubKey(key.get_pubkey()) + key = ECKey() + key.generate(False) + pubkey = key.get_pubkey().get_bytes() assert_equal(len(pubkey), 65) # This should be an uncompressed pubkey utxo = self.utxo.pop(0) @@ -1512,7 +1514,7 @@ class SegWitTest(BitcoinTestFramework): tx2.vout.append(CTxOut(tx.vout[0].nValue - 1000, script_wsh)) script = get_p2pkh_script(pubkeyhash) sig_hash = SegwitVersion1SignatureHash(script, tx2, 0, SIGHASH_ALL, tx.vout[0].nValue) - signature = key.sign(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL + signature = key.sign_ecdsa(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL tx2.wit.vtxinwit.append(CTxInWitness()) tx2.wit.vtxinwit[0].scriptWitness.stack = [signature, pubkey] tx2.rehash() @@ -1566,7 +1568,7 @@ class SegWitTest(BitcoinTestFramework): tx5.vin.append(CTxIn(COutPoint(tx4.sha256, 0), b"")) tx5.vout.append(CTxOut(tx4.vout[0].nValue - 1000, CScript([OP_TRUE]))) (sig_hash, err) = SignatureHash(script_pubkey, tx5, 0, SIGHASH_ALL) - signature = key.sign(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL + signature = key.sign_ecdsa(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL tx5.vin[0].scriptSig = CScript([signature, pubkey]) tx5.rehash() # Should pass policy and consensus. @@ -1579,9 +1581,9 @@ class SegWitTest(BitcoinTestFramework): @subtest def test_signature_version_1(self): - key = CECKey() - key.set_secretbytes(b"9") - pubkey = CPubKey(key.get_pubkey()) + key = ECKey() + key.generate() + pubkey = key.get_pubkey().get_bytes() witness_program = CScript([pubkey, CScriptOp(OP_CHECKSIG)]) witness_hash = sha256(witness_program) @@ -1716,7 +1718,7 @@ class SegWitTest(BitcoinTestFramework): script = get_p2pkh_script(pubkeyhash) sig_hash = SegwitVersion1SignatureHash(script, tx2, 0, SIGHASH_ALL, tx.vout[0].nValue) - signature = key.sign(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL + signature = key.sign_ecdsa(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL # Check that we can't have a scriptSig tx2.vin[0].scriptSig = CScript([signature, pubkey]) @@ -2035,5 +2037,31 @@ class SegWitTest(BitcoinTestFramework): # TODO: test p2sh sigop counting + def test_superfluous_witness(self): + # Serialization of tx that puts witness flag to 1 always + def serialize_with_bogus_witness(tx): + flags = 1 + r = b"" + r += struct.pack("<i", tx.nVersion) + if flags: + dummy = [] + r += ser_vector(dummy) + r += struct.pack("<B", flags) + r += ser_vector(tx.vin) + r += ser_vector(tx.vout) + if flags & 1: + if (len(tx.wit.vtxinwit) != len(tx.vin)): + # vtxinwit must have the same length as vin + tx.wit.vtxinwit = tx.wit.vtxinwit[:len(tx.vin)] + for i in range(len(tx.wit.vtxinwit), len(tx.vin)): + tx.wit.vtxinwit.append(CTxInWitness()) + r += tx.wit.serialize() + r += struct.pack("<I", tx.nLockTime) + return r + + raw = self.nodes[0].createrawtransaction([{"txid":"00"*32, "vout":0}], {self.nodes[0].getnewaddress():1}) + tx = FromHex(CTransaction(), raw) + assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].decoderawtransaction, serialize_with_bogus_witness(tx).hex()) + if __name__ == '__main__': SegWitTest().main() diff --git a/test/functional/rpc_getblockfilter.py b/test/functional/rpc_getblockfilter.py new file mode 100755 index 0000000000..bd93b6f7a4 --- /dev/null +++ b/test/functional/rpc_getblockfilter.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test the getblockfilter RPC.""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, assert_is_hex_string, assert_raises_rpc_error, + connect_nodes, disconnect_nodes, sync_blocks + ) + +FILTER_TYPES = ["basic"] + +class GetBlockFilterTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 2 + self.extra_args = [["-blockfilterindex"], []] + + def run_test(self): + # Create two chains by disconnecting nodes 0 & 1, mining, then reconnecting + disconnect_nodes(self.nodes[0], 1) + + self.nodes[0].generate(3) + self.nodes[1].generate(4) + + assert_equal(self.nodes[0].getblockcount(), 3) + chain0_hashes = [self.nodes[0].getblockhash(block_height) for block_height in range(4)] + + # Reorg node 0 to a new chain + connect_nodes(self.nodes[0], 1) + sync_blocks(self.nodes) + + assert_equal(self.nodes[0].getblockcount(), 4) + chain1_hashes = [self.nodes[0].getblockhash(block_height) for block_height in range(4)] + + # Test getblockfilter returns a filter for all blocks and filter types on active chain + for block_hash in chain1_hashes: + for filter_type in FILTER_TYPES: + result = self.nodes[0].getblockfilter(block_hash, filter_type) + assert_is_hex_string(result['filter']) + + # Test getblockfilter returns a filter for all blocks and filter types on stale chain + for block_hash in chain0_hashes: + for filter_type in FILTER_TYPES: + result = self.nodes[0].getblockfilter(block_hash, filter_type) + assert_is_hex_string(result['filter']) + + # Test getblockfilter with unknown block + bad_block_hash = "0123456789abcdef" * 4 + assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblockfilter, bad_block_hash, "basic") + + # Test getblockfilter with undefined filter type + genesis_hash = self.nodes[0].getblockhash(0) + assert_raises_rpc_error(-5, "Unknown filtertype", self.nodes[0].getblockfilter, genesis_hash, "unknown") + +if __name__ == '__main__': + GetBlockFilterTest().main() diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index fb68f79bbd..8a7ea7aa58 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -150,10 +150,11 @@ class PSBTTest(BitcoinTestFramework): new_psbt = self.nodes[0].converttopsbt(rawtx['hex']) self.nodes[0].decodepsbt(new_psbt) - # Make sure that a psbt with signatures cannot be converted + # Make sure that a non-psbt with signatures cannot be converted + # Error could be either "TX decode failed" (segwit inputs causes parsing to fail) or "Inputs must not have scriptSigs and scriptWitnesses" signedtx = self.nodes[0].signrawtransactionwithwallet(rawtx['hex']) - assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].converttopsbt, signedtx['hex']) - assert_raises_rpc_error(-22, "TX decode failed", self.nodes[0].converttopsbt, signedtx['hex'], False) + assert_raises_rpc_error(-22, "", self.nodes[0].converttopsbt, signedtx['hex']) + assert_raises_rpc_error(-22, "", self.nodes[0].converttopsbt, signedtx['hex'], False) # Unless we allow it to convert and strip signatures self.nodes[0].converttopsbt(signedtx['hex'], True) diff --git a/test/functional/test_framework/key.py b/test/functional/test_framework/key.py index 1b3e510dc4..912c0ca978 100644 --- a/test/functional/test_framework/key.py +++ b/test/functional/test_framework/key.py @@ -1,226 +1,386 @@ -# Copyright (c) 2011 Sam Rushing -"""ECC secp256k1 OpenSSL wrapper. +# Copyright (c) 2019 Pieter Wuille +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test-only secp256k1 elliptic curve implementation -WARNING: This module does not mlock() secrets; your private keys may end up on -disk in swap! Use with caution! +WARNING: This code is slow, uses bad randomness, does not properly protect +keys, and is trivially vulnerable to side channel attacks. Do not use for +anything but tests.""" +import random -This file is modified from python-bitcoinlib. -""" - -import ctypes -import ctypes.util -import hashlib - -ssl = ctypes.cdll.LoadLibrary(ctypes.util.find_library ('ssl') or 'libeay32') - -ssl.BN_new.restype = ctypes.c_void_p -ssl.BN_new.argtypes = [] - -ssl.BN_bin2bn.restype = ctypes.c_void_p -ssl.BN_bin2bn.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.c_void_p] - -ssl.BN_CTX_free.restype = None -ssl.BN_CTX_free.argtypes = [ctypes.c_void_p] - -ssl.BN_CTX_new.restype = ctypes.c_void_p -ssl.BN_CTX_new.argtypes = [] - -ssl.ECDH_compute_key.restype = ctypes.c_int -ssl.ECDH_compute_key.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p] - -ssl.ECDSA_sign.restype = ctypes.c_int -ssl.ECDSA_sign.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] - -ssl.ECDSA_verify.restype = ctypes.c_int -ssl.ECDSA_verify.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p] - -ssl.EC_KEY_free.restype = None -ssl.EC_KEY_free.argtypes = [ctypes.c_void_p] - -ssl.EC_KEY_new_by_curve_name.restype = ctypes.c_void_p -ssl.EC_KEY_new_by_curve_name.argtypes = [ctypes.c_int] - -ssl.EC_KEY_get0_group.restype = ctypes.c_void_p -ssl.EC_KEY_get0_group.argtypes = [ctypes.c_void_p] - -ssl.EC_KEY_get0_public_key.restype = ctypes.c_void_p -ssl.EC_KEY_get0_public_key.argtypes = [ctypes.c_void_p] - -ssl.EC_KEY_set_private_key.restype = ctypes.c_int -ssl.EC_KEY_set_private_key.argtypes = [ctypes.c_void_p, ctypes.c_void_p] - -ssl.EC_KEY_set_conv_form.restype = None -ssl.EC_KEY_set_conv_form.argtypes = [ctypes.c_void_p, ctypes.c_int] - -ssl.EC_KEY_set_public_key.restype = ctypes.c_int -ssl.EC_KEY_set_public_key.argtypes = [ctypes.c_void_p, ctypes.c_void_p] - -ssl.i2o_ECPublicKey.restype = ctypes.c_void_p -ssl.i2o_ECPublicKey.argtypes = [ctypes.c_void_p, ctypes.c_void_p] - -ssl.EC_POINT_new.restype = ctypes.c_void_p -ssl.EC_POINT_new.argtypes = [ctypes.c_void_p] - -ssl.EC_POINT_free.restype = None -ssl.EC_POINT_free.argtypes = [ctypes.c_void_p] - -ssl.EC_POINT_mul.restype = ctypes.c_int -ssl.EC_POINT_mul.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p] - -# this specifies the curve used with ECDSA. -NID_secp256k1 = 714 # from openssl/obj_mac.h +def modinv(a, n): + """Compute the modular inverse of a modulo n + See https://en.wikipedia.org/wiki/Extended_Euclidean_algorithm#Modular_integers. + """ + t1, t2 = 0, 1 + r1, r2 = n, a + while r2 != 0: + q = r1 // r2 + t1, t2 = t2, t1 - q * t2 + r1, r2 = r2, r1 - q * r2 + if r1 > 1: + return None + if t1 < 0: + t1 += n + return t1 + +def jacobi_symbol(n, k): + """Compute the Jacobi symbol of n modulo k + + See http://en.wikipedia.org/wiki/Jacobi_symbol + + For our application k is always prime, so this is the same as the Legendre symbol.""" + assert k > 0 and k & 1, "jacobi symbol is only defined for positive odd k" + n %= k + t = 0 + while n != 0: + while n & 1 == 0: + n >>= 1 + r = k & 7 + t ^= (r == 3 or r == 5) + n, k = k, n + t ^= (n & k & 3 == 3) + n = n % k + if k == 1: + return -1 if t else 1 + return 0 + +def modsqrt(a, p): + """Compute the square root of a modulo p when p % 4 = 3. + + The Tonelli-Shanks algorithm can be used. See https://en.wikipedia.org/wiki/Tonelli-Shanks_algorithm + + Limiting this function to only work for p % 4 = 3 means we don't need to + iterate through the loop. The highest n such that p - 1 = 2^n Q with Q odd + is n = 1. Therefore Q = (p-1)/2 and sqrt = a^((Q+1)/2) = a^((p+1)/4) + + secp256k1's is defined over field of size 2**256 - 2**32 - 977, which is 3 mod 4. + """ + if p % 4 != 3: + raise NotImplementedError("modsqrt only implemented for p % 4 = 3") + sqrt = pow(a, (p + 1)//4, p) + if pow(sqrt, 2, p) == a % p: + return sqrt + return None + +class EllipticCurve: + def __init__(self, p, a, b): + """Initialize elliptic curve y^2 = x^3 + a*x + b over GF(p).""" + self.p = p + self.a = a % p + self.b = b % p + + def affine(self, p1): + """Convert a Jacobian point tuple p1 to affine form, or None if at infinity. + + An affine point is represented as the Jacobian (x, y, 1)""" + x1, y1, z1 = p1 + if z1 == 0: + return None + inv = modinv(z1, self.p) + inv_2 = (inv**2) % self.p + inv_3 = (inv_2 * inv) % self.p + return ((inv_2 * x1) % self.p, (inv_3 * y1) % self.p, 1) + + def negate(self, p1): + """Negate a Jacobian point tuple p1.""" + x1, y1, z1 = p1 + return (x1, (self.p - y1) % self.p, z1) + + def on_curve(self, p1): + """Determine whether a Jacobian tuple p is on the curve (and not infinity)""" + x1, y1, z1 = p1 + z2 = pow(z1, 2, self.p) + z4 = pow(z2, 2, self.p) + return z1 != 0 and (pow(x1, 3, self.p) + self.a * x1 * z4 + self.b * z2 * z4 - pow(y1, 2, self.p)) % self.p == 0 + + def is_x_coord(self, x): + """Test whether x is a valid X coordinate on the curve.""" + x_3 = pow(x, 3, self.p) + return jacobi_symbol(x_3 + self.a * x + self.b, self.p) != -1 + + def lift_x(self, x): + """Given an X coordinate on the curve, return a corresponding affine point.""" + x_3 = pow(x, 3, self.p) + v = x_3 + self.a * x + self.b + y = modsqrt(v, self.p) + if y is None: + return None + return (x, y, 1) + + def double(self, p1): + """Double a Jacobian tuple p1 + + See https://en.wikibooks.org/wiki/Cryptography/Prime_Curve/Jacobian_Coordinates - Point Doubling""" + x1, y1, z1 = p1 + if z1 == 0: + return (0, 1, 0) + y1_2 = (y1**2) % self.p + y1_4 = (y1_2**2) % self.p + x1_2 = (x1**2) % self.p + s = (4*x1*y1_2) % self.p + m = 3*x1_2 + if self.a: + m += self.a * pow(z1, 4, self.p) + m = m % self.p + x2 = (m**2 - 2*s) % self.p + y2 = (m*(s - x2) - 8*y1_4) % self.p + z2 = (2*y1*z1) % self.p + return (x2, y2, z2) + + def add_mixed(self, p1, p2): + """Add a Jacobian tuple p1 and an affine tuple p2 + + See https://en.wikibooks.org/wiki/Cryptography/Prime_Curve/Jacobian_Coordinates - Point Addition (with affine point)""" + x1, y1, z1 = p1 + x2, y2, z2 = p2 + assert(z2 == 1) + # Adding to the point at infinity is a no-op + if z1 == 0: + return p2 + z1_2 = (z1**2) % self.p + z1_3 = (z1_2 * z1) % self.p + u2 = (x2 * z1_2) % self.p + s2 = (y2 * z1_3) % self.p + if x1 == u2: + if (y1 != s2): + # p1 and p2 are inverses. Return the point at infinity. + return (0, 1, 0) + # p1 == p2. The formulas below fail when the two points are equal. + return self.double(p1) + h = u2 - x1 + r = s2 - y1 + h_2 = (h**2) % self.p + h_3 = (h_2 * h) % self.p + u1_h_2 = (x1 * h_2) % self.p + x3 = (r**2 - h_3 - 2*u1_h_2) % self.p + y3 = (r*(u1_h_2 - x3) - y1*h_3) % self.p + z3 = (h*z1) % self.p + return (x3, y3, z3) + + def add(self, p1, p2): + """Add two Jacobian tuples p1 and p2 + + See https://en.wikibooks.org/wiki/Cryptography/Prime_Curve/Jacobian_Coordinates - Point Addition""" + x1, y1, z1 = p1 + x2, y2, z2 = p2 + # Adding the point at infinity is a no-op + if z1 == 0: + return p2 + if z2 == 0: + return p1 + # Adding an Affine to a Jacobian is more efficient since we save field multiplications and squarings when z = 1 + if z1 == 1: + return self.add_mixed(p2, p1) + if z2 == 1: + return self.add_mixed(p1, p2) + z1_2 = (z1**2) % self.p + z1_3 = (z1_2 * z1) % self.p + z2_2 = (z2**2) % self.p + z2_3 = (z2_2 * z2) % self.p + u1 = (x1 * z2_2) % self.p + u2 = (x2 * z1_2) % self.p + s1 = (y1 * z2_3) % self.p + s2 = (y2 * z1_3) % self.p + if u1 == u2: + if (s1 != s2): + # p1 and p2 are inverses. Return the point at infinity. + return (0, 1, 0) + # p1 == p2. The formulas below fail when the two points are equal. + return self.double(p1) + h = u2 - u1 + r = s2 - s1 + h_2 = (h**2) % self.p + h_3 = (h_2 * h) % self.p + u1_h_2 = (u1 * h_2) % self.p + x3 = (r**2 - h_3 - 2*u1_h_2) % self.p + y3 = (r*(u1_h_2 - x3) - s1*h_3) % self.p + z3 = (h*z1*z2) % self.p + return (x3, y3, z3) + + def mul(self, ps): + """Compute a (multi) point multiplication + + ps is a list of (Jacobian tuple, scalar) pairs. + """ + r = (0, 1, 0) + for i in range(255, -1, -1): + r = self.double(r) + for (p, n) in ps: + if ((n >> i) & 1): + r = self.add(r, p) + return r + +SECP256K1 = EllipticCurve(2**256 - 2**32 - 977, 0, 7) +SECP256K1_G = (0x79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798, 0x483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8, 1) SECP256K1_ORDER = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141 SECP256K1_ORDER_HALF = SECP256K1_ORDER // 2 -# Thx to Sam Devlin for the ctypes magic 64-bit fix. -def _check_result(val, func, args): - if val == 0: - raise ValueError - else: - return ctypes.c_void_p (val) - -ssl.EC_KEY_new_by_curve_name.restype = ctypes.c_void_p -ssl.EC_KEY_new_by_curve_name.errcheck = _check_result - -class CECKey(): - """Wrapper around OpenSSL's EC_KEY""" - - POINT_CONVERSION_COMPRESSED = 2 - POINT_CONVERSION_UNCOMPRESSED = 4 +class ECPubKey(): + """A secp256k1 public key""" def __init__(self): - self.k = ssl.EC_KEY_new_by_curve_name(NID_secp256k1) - - def __del__(self): - if ssl: - ssl.EC_KEY_free(self.k) - self.k = None - - def set_secretbytes(self, secret): - priv_key = ssl.BN_bin2bn(secret, 32, ssl.BN_new()) - group = ssl.EC_KEY_get0_group(self.k) - pub_key = ssl.EC_POINT_new(group) - ctx = ssl.BN_CTX_new() - if not ssl.EC_POINT_mul(group, pub_key, priv_key, None, None, ctx): - raise ValueError("Could not derive public key from the supplied secret.") - ssl.EC_POINT_mul(group, pub_key, priv_key, None, None, ctx) - ssl.EC_KEY_set_private_key(self.k, priv_key) - ssl.EC_KEY_set_public_key(self.k, pub_key) - ssl.EC_POINT_free(pub_key) - ssl.BN_CTX_free(ctx) - return self.k - - def set_privkey(self, key): - self.mb = ctypes.create_string_buffer(key) - return ssl.d2i_ECPrivateKey(ctypes.byref(self.k), ctypes.byref(ctypes.pointer(self.mb)), len(key)) - - def set_pubkey(self, key): - self.mb = ctypes.create_string_buffer(key) - return ssl.o2i_ECPublicKey(ctypes.byref(self.k), ctypes.byref(ctypes.pointer(self.mb)), len(key)) - - def get_privkey(self): - size = ssl.i2d_ECPrivateKey(self.k, 0) - mb_pri = ctypes.create_string_buffer(size) - ssl.i2d_ECPrivateKey(self.k, ctypes.byref(ctypes.pointer(mb_pri))) - return mb_pri.raw - - def get_pubkey(self): - size = ssl.i2o_ECPublicKey(self.k, 0) - mb = ctypes.create_string_buffer(size) - ssl.i2o_ECPublicKey(self.k, ctypes.byref(ctypes.pointer(mb))) - return mb.raw - - def get_raw_ecdh_key(self, other_pubkey): - ecdh_keybuffer = ctypes.create_string_buffer(32) - r = ssl.ECDH_compute_key(ctypes.pointer(ecdh_keybuffer), 32, - ssl.EC_KEY_get0_public_key(other_pubkey.k), - self.k, 0) - if r != 32: - raise Exception('CKey.get_ecdh_key(): ECDH_compute_key() failed') - return ecdh_keybuffer.raw - - def get_ecdh_key(self, other_pubkey, kdf=lambda k: hashlib.sha256(k).digest()): - # FIXME: be warned it's not clear what the kdf should be as a default - r = self.get_raw_ecdh_key(other_pubkey) - return kdf(r) - - def sign(self, hash, low_s = True): - # FIXME: need unit tests for below cases - if not isinstance(hash, bytes): - raise TypeError('Hash must be bytes instance; got %r' % hash.__class__) - if len(hash) != 32: - raise ValueError('Hash must be exactly 32 bytes long') - - sig_size0 = ctypes.c_uint32() - sig_size0.value = ssl.ECDSA_size(self.k) - mb_sig = ctypes.create_string_buffer(sig_size0.value) - result = ssl.ECDSA_sign(0, hash, len(hash), mb_sig, ctypes.byref(sig_size0), self.k) - assert 1 == result - assert mb_sig.raw[0] == 0x30 - assert mb_sig.raw[1] == sig_size0.value - 2 - total_size = mb_sig.raw[1] - assert mb_sig.raw[2] == 2 - r_size = mb_sig.raw[3] - assert mb_sig.raw[4 + r_size] == 2 - s_size = mb_sig.raw[5 + r_size] - s_value = int.from_bytes(mb_sig.raw[6+r_size:6+r_size+s_size], byteorder='big') - if (not low_s) or s_value <= SECP256K1_ORDER_HALF: - return mb_sig.raw[:sig_size0.value] - else: - low_s_value = SECP256K1_ORDER - s_value - low_s_bytes = (low_s_value).to_bytes(33, byteorder='big') - while len(low_s_bytes) > 1 and low_s_bytes[0] == 0 and low_s_bytes[1] < 0x80: - low_s_bytes = low_s_bytes[1:] - new_s_size = len(low_s_bytes) - new_total_size_byte = (total_size + new_s_size - s_size).to_bytes(1,byteorder='big') - new_s_size_byte = (new_s_size).to_bytes(1,byteorder='big') - return b'\x30' + new_total_size_byte + mb_sig.raw[2:5+r_size] + new_s_size_byte + low_s_bytes - - def verify(self, hash, sig): - """Verify a DER signature""" - return ssl.ECDSA_verify(0, hash, len(hash), sig, len(sig), self.k) == 1 - - def set_compressed(self, compressed): - if compressed: - form = self.POINT_CONVERSION_COMPRESSED + """Construct an uninitialized public key""" + self.valid = False + + def set(self, data): + """Construct a public key from a serialization in compressed or uncompressed format""" + if (len(data) == 65 and data[0] == 0x04): + p = (int.from_bytes(data[1:33], 'big'), int.from_bytes(data[33:65], 'big'), 1) + self.valid = SECP256K1.on_curve(p) + if self.valid: + self.p = p + self.compressed = False + elif (len(data) == 33 and (data[0] == 0x02 or data[0] == 0x03)): + x = int.from_bytes(data[1:33], 'big') + if SECP256K1.is_x_coord(x): + p = SECP256K1.lift_x(x) + # if the oddness of the y co-ord isn't correct, find the other + # valid y + if (p[1] & 1) != (data[0] & 1): + p = SECP256K1.negate(p) + self.p = p + self.valid = True + self.compressed = True + else: + self.valid = False else: - form = self.POINT_CONVERSION_UNCOMPRESSED - ssl.EC_KEY_set_conv_form(self.k, form) - + self.valid = False -class CPubKey(bytes): - """An encapsulated public key - - Attributes: + @property + def is_compressed(self): + return self.compressed - is_valid - Corresponds to CPubKey.IsValid() - is_fullyvalid - Corresponds to CPubKey.IsFullyValid() - is_compressed - Corresponds to CPubKey.IsCompressed() - """ + @property + def is_valid(self): + return self.valid + + def get_bytes(self): + assert(self.valid) + p = SECP256K1.affine(self.p) + if p is None: + return None + if self.compressed: + return bytes([0x02 + (p[1] & 1)]) + p[0].to_bytes(32, 'big') + else: + return bytes([0x04]) + p[0].to_bytes(32, 'big') + p[1].to_bytes(32, 'big') + + def verify_ecdsa(self, sig, msg, low_s=True): + """Verify a strictly DER-encoded ECDSA signature against this pubkey. + + See https://en.wikipedia.org/wiki/Elliptic_Curve_Digital_Signature_Algorithm for the + ECDSA verifier algorithm""" + assert(self.valid) + + # Extract r and s from the DER formatted signature. Return false for + # any DER encoding errors. + if (sig[1] + 2 != len(sig)): + return False + if (len(sig) < 4): + return False + if (sig[0] != 0x30): + return False + if (sig[2] != 0x02): + return False + rlen = sig[3] + if (len(sig) < 6 + rlen): + return False + if rlen < 1 or rlen > 33: + return False + if sig[4] >= 0x80: + return False + if (rlen > 1 and (sig[4] == 0) and not (sig[5] & 0x80)): + return False + r = int.from_bytes(sig[4:4+rlen], 'big') + if (sig[4+rlen] != 0x02): + return False + slen = sig[5+rlen] + if slen < 1 or slen > 33: + return False + if (len(sig) != 6 + rlen + slen): + return False + if sig[6+rlen] >= 0x80: + return False + if (slen > 1 and (sig[6+rlen] == 0) and not (sig[7+rlen] & 0x80)): + return False + s = int.from_bytes(sig[6+rlen:6+rlen+slen], 'big') + + # Verify that r and s are within the group order + if r < 1 or s < 1 or r >= SECP256K1_ORDER or s >= SECP256K1_ORDER: + return False + if low_s and s >= SECP256K1_ORDER_HALF: + return False + z = int.from_bytes(msg, 'big') + + # Run verifier algorithm on r, s + w = modinv(s, SECP256K1_ORDER) + u1 = z*w % SECP256K1_ORDER + u2 = r*w % SECP256K1_ORDER + R = SECP256K1.affine(SECP256K1.mul([(SECP256K1_G, u1), (self.p, u2)])) + if R is None or R[0] != r: + return False + return True + +class ECKey(): + """A secp256k1 private key""" - def __new__(cls, buf, _cec_key=None): - self = super(CPubKey, cls).__new__(cls, buf) - if _cec_key is None: - _cec_key = CECKey() - self._cec_key = _cec_key - self.is_fullyvalid = _cec_key.set_pubkey(self) != 0 - return self + def __init__(self): + self.valid = False + + def set(self, secret, compressed): + """Construct a private key object with given 32-byte secret and compressed flag.""" + assert(len(secret) == 32) + secret = int.from_bytes(secret, 'big') + self.valid = (secret > 0 and secret < SECP256K1_ORDER) + if self.valid: + self.secret = secret + self.compressed = compressed + + def generate(self, compressed=True): + """Generate a random private key (compressed or uncompressed).""" + self.set(random.randrange(1, SECP256K1_ORDER).to_bytes(32, 'big'), compressed) + + def get_bytes(self): + """Retrieve the 32-byte representation of this key.""" + assert(self.valid) + return self.secret.to_bytes(32, 'big') @property def is_valid(self): - return len(self) > 0 + return self.valid @property def is_compressed(self): - return len(self) == 33 - - def verify(self, hash, sig): - return self._cec_key.verify(hash, sig) - - def __str__(self): - return repr(self) - - def __repr__(self): - return '%s(%s)' % (self.__class__.__name__, super(CPubKey, self).__repr__()) + return self.compressed + def get_pubkey(self): + """Compute an ECPubKey object for this secret key.""" + assert(self.valid) + ret = ECPubKey() + p = SECP256K1.mul([(SECP256K1_G, self.secret)]) + ret.p = p + ret.valid = True + ret.compressed = self.compressed + return ret + + def sign_ecdsa(self, msg, low_s=True): + """Construct a DER-encoded ECDSA signature with this key. + + See https://en.wikipedia.org/wiki/Elliptic_Curve_Digital_Signature_Algorithm for the + ECDSA signer algorithm.""" + assert(self.valid) + z = int.from_bytes(msg, 'big') + # Note: no RFC6979, but a simple random nonce (some tests rely on distinct transactions for the same operation) + k = random.randrange(1, SECP256K1_ORDER) + R = SECP256K1.affine(SECP256K1.mul([(SECP256K1_G, k)])) + r = R[0] % SECP256K1_ORDER + s = (modinv(k, SECP256K1_ORDER) * (z + self.secret * r)) % SECP256K1_ORDER + if low_s and s > SECP256K1_ORDER_HALF: + s = SECP256K1_ORDER - s + # Represent in DER format. The byte representations of r and s have + # length rounded up (255 bits becomes 32 bytes and 256 bits becomes 33 + # bytes). + rb = r.to_bytes((r.bit_length() + 8) // 8, 'big') + sb = s.to_bytes((s.bit_length() + 8) // 8, 'big') + return b'\x30' + bytes([4 + len(rb) + len(sb), 2, len(rb)]) + rb + bytes([2, len(sb)]) + sb diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index 7a063ac526..11ea968257 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -531,7 +531,7 @@ class P2PDataStore(P2PInterface): for b in blocks: self.send_message(msg_block(block=b)) else: - self.send_message(msg_headers([CBlockHeader(blocks[-1])])) + self.send_message(msg_headers([CBlockHeader(block) for block in blocks])) wait_until(lambda: blocks[-1].sha256 in self.getdata_requests, timeout=timeout, lock=mininode_lock) if expect_disconnect: diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 4aeff24d12..555d55d97f 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -556,21 +556,12 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def is_cli_compiled(self): """Checks whether bitcoin-cli was compiled.""" - config = configparser.ConfigParser() - config.read_file(open(self.options.configfile)) - - return config["components"].getboolean("ENABLE_CLI") + return self.config["components"].getboolean("ENABLE_CLI") def is_wallet_compiled(self): """Checks whether the wallet module was compiled.""" - config = configparser.ConfigParser() - config.read_file(open(self.options.configfile)) - - return config["components"].getboolean("ENABLE_WALLET") + return self.config["components"].getboolean("ENABLE_WALLET") def is_zmq_compiled(self): """Checks whether the zmq module was compiled.""" - config = configparser.ConfigParser() - config.read_file(open(self.options.configfile)) - - return config["components"].getboolean("ENABLE_ZMQ") + return self.config["components"].getboolean("ENABLE_ZMQ") diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 61c5c9aefb..d406ee3229 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -69,6 +69,7 @@ TEST_EXIT_SKIPPED = 77 BASE_SCRIPTS = [ # Scripts that are run by the travis build process. # Longest test should go first, to favor running tests in parallel + 'feature_pruning.py', 'feature_fee_estimation.py', 'wallet_hd.py', 'wallet_backup.py', @@ -144,6 +145,7 @@ BASE_SCRIPTS = [ 'wallet_txn_doublespend.py', 'wallet_txn_clone.py --mineblock', 'feature_notifications.py', + 'rpc_getblockfilter.py', 'rpc_invalidateblock.py', 'feature_rbf.py', 'mempool_packages.py', @@ -198,7 +200,6 @@ BASE_SCRIPTS = [ EXTENDED_SCRIPTS = [ # These tests are not run by the travis build process. # Longest test should go first, to favor running tests in parallel - 'feature_pruning.py', 'feature_dbcrash.py', ] diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index e2a20beec5..c02d7422b9 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -4,20 +4,23 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet balance RPC methods.""" from decimal import Decimal +import struct +from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE as ADDRESS_WATCHONLY from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, + connect_nodes_bi, + sync_blocks, ) -RANDOM_COINBASE_ADDRESS = 'mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ' def create_transactions(node, address, amt, fees): # Create and sign raw transactions from node to address for amt. # Creates a transaction for each fee and returns an array # of the raw transactions. - utxos = node.listunspent(0) + utxos = [u for u in node.listunspent(0) if u['spendable']] # Create transactions inputs = [] @@ -25,7 +28,7 @@ def create_transactions(node, address, amt, fees): for utxo in utxos: inputs.append({"txid": utxo["txid"], "vout": utxo["vout"]}) ins_total += utxo['amount'] - if ins_total > amt: + if ins_total + max(fees) > amt: break txs = [] @@ -33,6 +36,7 @@ def create_transactions(node, address, amt, fees): outputs = {address: amt, node.getrawchangeaddress(): ins_total - amt - fee} raw_tx = node.createrawtransaction(inputs, outputs, 0, True) raw_tx = node.signrawtransactionwithwallet(raw_tx) + assert_equal(raw_tx['complete'], True) txs.append(raw_tx) return txs @@ -41,21 +45,26 @@ class WalletTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True + self.extra_args = [ + ['-limitdescendantcount=3'], # Limit mempool descendants as a hack to have wallet txs rejected from the mempool + [], + ] def skip_test_if_missing_module(self): self.skip_if_no_wallet() def run_test(self): + self.nodes[0].importaddress(ADDRESS_WATCHONLY) # Check that nodes don't own any UTXOs assert_equal(len(self.nodes[0].listunspent()), 0) assert_equal(len(self.nodes[1].listunspent()), 0) - self.log.info("Mining one block for each node") + self.log.info("Mining blocks ...") self.nodes[0].generate(1) self.sync_all() self.nodes[1].generate(1) - self.nodes[1].generatetoaddress(100, RANDOM_COINBASE_ADDRESS) + self.nodes[1].generatetoaddress(101, ADDRESS_WATCHONLY) self.sync_all() assert_equal(self.nodes[0].getbalance(), 50) @@ -64,8 +73,10 @@ class WalletTest(BitcoinTestFramework): self.log.info("Test getbalance with different arguments") assert_equal(self.nodes[0].getbalance("*"), 50) assert_equal(self.nodes[0].getbalance("*", 1), 50) - assert_equal(self.nodes[0].getbalance("*", 1, True), 50) + assert_equal(self.nodes[0].getbalance("*", 1, True), 100) assert_equal(self.nodes[0].getbalance(minconf=1), 50) + assert_equal(self.nodes[0].getbalance(minconf=0, include_watchonly=True), 100) + assert_equal(self.nodes[1].getbalance(minconf=0, include_watchonly=True), 50) # Send 40 BTC from 0 to 1 and 60 BTC from 1 to 0. txs = create_transactions(self.nodes[0], self.nodes[1].getnewaddress(), 40, [Decimal('0.01')]) @@ -83,32 +94,34 @@ class WalletTest(BitcoinTestFramework): self.log.info("Test getbalance and getunconfirmedbalance with unconfirmed inputs") - # getbalance without any arguments includes unconfirmed transactions, but not untrusted transactions - assert_equal(self.nodes[0].getbalance(), Decimal('9.99')) # change from node 0's send - assert_equal(self.nodes[1].getbalance(), Decimal('29.99')) # change from node 1's send - # Same with minconf=0 - assert_equal(self.nodes[0].getbalance(minconf=0), Decimal('9.99')) - assert_equal(self.nodes[1].getbalance(minconf=0), Decimal('29.99')) - # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago - # TODO: fix getbalance tracking of coin spentness depth - assert_equal(self.nodes[0].getbalance(minconf=1), Decimal('0')) - assert_equal(self.nodes[1].getbalance(minconf=1), Decimal('0')) - # getunconfirmedbalance - assert_equal(self.nodes[0].getunconfirmedbalance(), Decimal('60')) # output of node 1's spend - assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('0')) # Doesn't include output of node 0's send since it was spent + def test_balances(*, fee_node_1=0): + # getbalance without any arguments includes unconfirmed transactions, but not untrusted transactions + assert_equal(self.nodes[0].getbalance(), Decimal('9.99')) # change from node 0's send + assert_equal(self.nodes[1].getbalance(), Decimal('30') - fee_node_1) # change from node 1's send + # Same with minconf=0 + assert_equal(self.nodes[0].getbalance(minconf=0), Decimal('9.99')) + assert_equal(self.nodes[1].getbalance(minconf=0), Decimal('30') - fee_node_1) + # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago + # TODO: fix getbalance tracking of coin spentness depth + assert_equal(self.nodes[0].getbalance(minconf=1), Decimal('0')) + assert_equal(self.nodes[1].getbalance(minconf=1), Decimal('0')) + # getunconfirmedbalance + assert_equal(self.nodes[0].getunconfirmedbalance(), Decimal('60')) # output of node 1's spend + assert_equal(self.nodes[0].getwalletinfo()["unconfirmed_balance"], Decimal('60')) + assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('0')) # Doesn't include output of node 0's send since it was spent + assert_equal(self.nodes[1].getwalletinfo()["unconfirmed_balance"], Decimal('0')) + + test_balances(fee_node_1=Decimal('0.01')) # Node 1 bumps the transaction fee and resends self.nodes[1].sendrawtransaction(txs[1]['hex']) + self.nodes[0].sendrawtransaction(txs[1]['hex']) # sending on both nodes is faster than waiting for propagation self.sync_all() self.log.info("Test getbalance and getunconfirmedbalance with conflicted unconfirmed inputs") + test_balances(fee_node_1=Decimal('0.02')) - assert_equal(self.nodes[0].getwalletinfo()["unconfirmed_balance"], Decimal('60')) # output of node 1's send - assert_equal(self.nodes[0].getunconfirmedbalance(), Decimal('60')) - assert_equal(self.nodes[1].getwalletinfo()["unconfirmed_balance"], Decimal('0')) # Doesn't include output of node 0's send since it was spent - assert_equal(self.nodes[1].getunconfirmedbalance(), Decimal('0')) - - self.nodes[1].generatetoaddress(1, RANDOM_COINBASE_ADDRESS) + self.nodes[1].generatetoaddress(1, ADDRESS_WATCHONLY) self.sync_all() # balances are correct after the transactions are confirmed @@ -118,7 +131,7 @@ class WalletTest(BitcoinTestFramework): # Send total balance away from node 1 txs = create_transactions(self.nodes[1], self.nodes[0].getnewaddress(), Decimal('29.97'), [Decimal('0.01')]) self.nodes[1].sendrawtransaction(txs[0]['hex']) - self.nodes[1].generatetoaddress(2, RANDOM_COINBASE_ADDRESS) + self.nodes[1].generatetoaddress(2, ADDRESS_WATCHONLY) self.sync_all() # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago @@ -140,6 +153,51 @@ class WalletTest(BitcoinTestFramework): after = self.nodes[1].getunconfirmedbalance() assert_equal(before + Decimal('0.1'), after) + # Create 3 more wallet txs, where the last is not accepted to the + # mempool because it is the third descendant of the tx above + for _ in range(3): + # Set amount high enough such that all coins are spent by each tx + txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 99) + + self.log.info('Check that wallet txs not in the mempool are untrusted') + assert txid not in self.nodes[0].getrawmempool() + assert_equal(self.nodes[0].gettransaction(txid)['trusted'], False) + assert_equal(self.nodes[0].getbalance(minconf=0), 0) + + self.log.info("Test replacement and reorg of non-mempool tx") + tx_orig = self.nodes[0].gettransaction(txid)['hex'] + # Increase fee by 1 coin + tx_replace = tx_orig.replace( + struct.pack("<q", 99 * 10**8).hex(), + struct.pack("<q", 98 * 10**8).hex(), + ) + tx_replace = self.nodes[0].signrawtransactionwithwallet(tx_replace)['hex'] + # Total balance is given by the sum of outputs of the tx + total_amount = sum([o['value'] for o in self.nodes[0].decoderawtransaction(tx_replace)['vout']]) + self.sync_all() + self.nodes[1].sendrawtransaction(hexstring=tx_replace, maxfeerate=0) + + # Now confirm tx_replace + block_reorg = self.nodes[1].generatetoaddress(1, ADDRESS_WATCHONLY)[0] + self.sync_all() + assert_equal(self.nodes[0].getbalance(minconf=0), total_amount) + + self.log.info('Put txs back into mempool of node 1 (not node 0)') + self.nodes[0].invalidateblock(block_reorg) + self.nodes[1].invalidateblock(block_reorg) + assert_equal(self.nodes[0].getbalance(minconf=0), 0) # wallet txs not in the mempool are untrusted + self.nodes[0].generatetoaddress(1, ADDRESS_WATCHONLY) + assert_equal(self.nodes[0].getbalance(minconf=0), 0) # wallet txs not in the mempool are untrusted + + # Now confirm tx_orig + self.restart_node(1, ['-persistmempool=0']) + connect_nodes_bi(self.nodes, 0, 1) + sync_blocks(self.nodes) + self.nodes[1].sendrawtransaction(tx_orig) + self.nodes[1].generatetoaddress(1, ADDRESS_WATCHONLY) + self.sync_all() + assert_equal(self.nodes[0].getbalance(minconf=0), total_amount + 1) # The reorg recovered our fee of 1 coin + if __name__ == '__main__': WalletTest().main() diff --git a/test/functional/wallet_import_rescan.py b/test/functional/wallet_import_rescan.py index 72df714d80..9de30d0374 100755 --- a/test/functional/wallet_import_rescan.py +++ b/test/functional/wallet_import_rescan.py @@ -166,11 +166,12 @@ class ImportRescanTest(BitcoinTestFramework): timestamp = self.nodes[0].getblockheader(self.nodes[0].getbestblockhash())["time"] set_node_times(self.nodes, timestamp + TIMESTAMP_WINDOW + 1) self.nodes[0].generate(1) - self.sync_blocks() + self.sync_all() # For each variation of wallet key import, invoke the import RPC and # check the results from getbalance and listtransactions. for variant in IMPORT_VARIANTS: + self.log.info('Run import for variant {}'.format(variant)) variant.expect_disabled = variant.rescan == Rescan.yes and variant.prune and variant.call == Call.single expect_rescan = variant.rescan == Rescan.yes and not variant.expect_disabled variant.node = self.nodes[2 + IMPORT_NODES.index(ImportNode(variant.prune, expect_rescan))] @@ -192,10 +193,11 @@ class ImportRescanTest(BitcoinTestFramework): # Generate a block containing the new transactions. self.nodes[0].generate(1) assert_equal(self.nodes[0].getrawmempool(), []) - self.sync_blocks() + self.sync_all() # Check the latest results from getbalance and listtransactions. for variant in IMPORT_VARIANTS: + self.log.info('Run check for variant {}'.format(variant)) if not variant.expect_disabled: variant.expected_balance += variant.sent_amount variant.expected_txs += 1 diff --git a/test/lint/check-doc.py b/test/lint/check-doc.py index 3b05d5055c..1d6122a13d 100755 --- a/test/lint/check-doc.py +++ b/test/lint/check-doc.py @@ -12,26 +12,23 @@ Author: @MarcoFalke from subprocess import check_output import re -import sys FOLDER_GREP = 'src' FOLDER_TEST = 'src/test/' REGEX_ARG = '(?:ForceSet|SoftSet|Get|Is)(?:Bool)?Args?(?:Set)?\("(-[^"]+)"' REGEX_DOC = 'AddArg\("(-[^"=]+?)(?:=|")' -CMD_ROOT_DIR = '`git rev-parse --show-toplevel`/{}'.format(FOLDER_GREP) +CMD_ROOT_DIR = '$(git rev-parse --show-toplevel)/{}'.format(FOLDER_GREP) CMD_GREP_ARGS = r"git grep --perl-regexp '{}' -- {} ':(exclude){}'".format(REGEX_ARG, CMD_ROOT_DIR, FOLDER_TEST) +CMD_GREP_WALLET_ARGS = r"git grep --function-context 'void WalletInit::AddWalletOptions' -- {} | grep AddArg".format(CMD_ROOT_DIR) +CMD_GREP_WALLET_HIDDEN_ARGS = r"git grep --function-context 'void DummyWalletInit::AddWalletOptions' -- {}".format(CMD_ROOT_DIR) CMD_GREP_DOCS = r"git grep --perl-regexp '{}' {}".format(REGEX_DOC, CMD_ROOT_DIR) # list unsupported, deprecated and duplicate args as they need no documentation SET_DOC_OPTIONAL = set(['-h', '-help', '-dbcrashratio', '-forcecompactdb']) -def main(): - if sys.version_info >= (3, 6): - used = check_output(CMD_GREP_ARGS, shell=True, universal_newlines=True, encoding='utf8') - docd = check_output(CMD_GREP_DOCS, shell=True, universal_newlines=True, encoding='utf8') - else: - used = check_output(CMD_GREP_ARGS, shell=True).decode('utf8').strip() - docd = check_output(CMD_GREP_DOCS, shell=True).decode('utf8').strip() +def lint_missing_argument_documentation(): + used = check_output(CMD_GREP_ARGS, shell=True).decode('utf8').strip() + docd = check_output(CMD_GREP_DOCS, shell=True).decode('utf8').strip() args_used = set(re.findall(re.compile(REGEX_ARG), used)) args_docd = set(re.findall(re.compile(REGEX_DOC), docd)).union(SET_DOC_OPTIONAL) @@ -45,7 +42,24 @@ def main(): print("Args unknown : {}".format(len(args_unknown))) print(args_unknown) - sys.exit(len(args_need_doc)) + assert 0 == len(args_need_doc), "Please document the following arguments: {}".format(args_need_doc) + + +def lint_missing_hidden_wallet_args(): + wallet_args = check_output(CMD_GREP_WALLET_ARGS, shell=True).decode('utf8').strip() + wallet_hidden_args = check_output(CMD_GREP_WALLET_HIDDEN_ARGS, shell=True).decode('utf8').strip() + + wallet_args = set(re.findall(re.compile(REGEX_DOC), wallet_args)) + wallet_hidden_args = set(re.findall(re.compile(r' "([^"=]+)'), wallet_hidden_args)) + + hidden_missing = wallet_args.difference(wallet_hidden_args) + if hidden_missing: + assert 0, "Please add {} to the hidden args in DummyWalletInit::AddWalletOptions".format(hidden_missing) + + +def main(): + lint_missing_argument_documentation() + lint_missing_hidden_wallet_args() if __name__ == "__main__": diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh index 87b451dbbd..e1a99abc49 100755 --- a/test/lint/lint-circular-dependencies.sh +++ b/test/lint/lint-circular-dependencies.sh @@ -10,7 +10,6 @@ export LC_ALL=C EXPECTED_CIRCULAR_DEPENDENCIES=( "chainparamsbase -> util/system -> chainparamsbase" - "checkpoints -> validation -> checkpoints" "index/txindex -> validation -> index/txindex" "policy/fees -> txmempool -> policy/fees" "policy/policy -> policy/settings -> policy/policy" @@ -37,7 +36,6 @@ EXPECTED_CIRCULAR_DEPENDENCIES=( "txmempool -> validation -> validationinterface -> txmempool" "qt/addressbookpage -> qt/bitcoingui -> qt/walletview -> qt/receivecoinsdialog -> qt/addressbookpage" "qt/addressbookpage -> qt/bitcoingui -> qt/walletview -> qt/signverifymessagedialog -> qt/addressbookpage" - "qt/guiutil -> qt/walletmodel -> qt/optionsmodel -> qt/intro -> qt/guiutil" "qt/addressbookpage -> qt/bitcoingui -> qt/walletview -> qt/sendcoinsdialog -> qt/sendcoinsentry -> qt/addressbookpage" ) diff --git a/test/lint/lint-python-dead-code.sh b/test/lint/lint-python-dead-code.sh index 863caa9d5c..588ba428d7 100755 --- a/test/lint/lint-python-dead-code.sh +++ b/test/lint/lint-python-dead-code.sh @@ -15,5 +15,5 @@ fi vulture \ --min-confidence 60 \ - --ignore-names "argtypes,connection_lost,connection_made,converter,data_received,daemon,errcheck,get_ecdh_key,get_privkey,is_compressed,is_fullyvalid,msg_generic,on_*,optionxform,restype,set_privkey,profile_with_perf" \ + --ignore-names "argtypes,connection_lost,connection_made,converter,data_received,daemon,errcheck,is_compressed,is_valid,verify_ecdsa,msg_generic,on_*,optionxform,restype,profile_with_perf" \ $(git ls-files -- "*.py" ":(exclude)contrib/" ":(exclude)test/functional/data/invalid_txs.py") |