diff options
42 files changed, 306 insertions, 232 deletions
diff --git a/.python-version b/.python-version new file mode 100644 index 0000000000..7bcbb3808b --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.4.9 diff --git a/.travis.yml b/.travis.yml index dec517f0a2..d051f37d98 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,13 +26,13 @@ env: - WINEDEBUG=fixme-all - DOCKER_PACKAGES="build-essential libtool autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache" before_install: - - set -o errexit; if ! source .travis/test_03_before_install.sh; then set +o errexit; false; fi + - set -o errexit; source .travis/test_03_before_install.sh install: - - set -o errexit; if ! source .travis/test_04_install.sh; then set +o errexit; false; fi + - set -o errexit; source .travis/test_04_install.sh before_script: - - set -o errexit; if ! source .travis/test_05_before_script.sh; then set +o errexit; false; fi + - set -o errexit; source .travis/test_05_before_script.sh script: - - if [ $SECONDS -gt 1200 ]; then set +o errexit; echo "Travis early exit to cache current state"; false; else set -o errexit; if ! source .travis/test_06_script.sh; then set +o errexit; false; fi; fi + - if [ $SECONDS -gt 1200 ]; then set +o errexit; echo "Travis early exit to cache current state"; false; else set -o errexit; source .travis/test_06_script.sh; fi after_script: - echo $TRAVIS_COMMIT_RANGE - echo $TRAVIS_COMMIT_LOG @@ -43,7 +43,7 @@ jobs: env: cache: false language: python - python: '3.6' + python: '3.4' # Oldest supported version according to doc/dependencies.md install: - set -o errexit; source .travis/lint_04_install.sh before_script: diff --git a/contrib/devtools/clang-format-diff.py b/contrib/devtools/clang-format-diff.py index 77e845a9b4..f322b3a880 100755 --- a/contrib/devtools/clang-format-diff.py +++ b/contrib/devtools/clang-format-diff.py @@ -109,7 +109,7 @@ def main(): match = re.search('^\+\+\+\ (.*?/){%s}(\S*)' % args.p, line) if match: filename = match.group(2) - if filename == None: + if filename is None: continue if args.regex is not None: diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index 2d539ffe43..6d7a592f01 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -491,7 +491,7 @@ def get_git_change_year_range(filename): def file_already_has_core_copyright(file_lines): index, _ = get_updatable_copyright_line(file_lines) - return index != None + return index is not None ################################################################################ # insert header execution diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index 4e90f85f50..4f827401fb 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -14,7 +14,6 @@ # In case of a clean merge that is accepted by the user, the local branch with # name $BRANCH is overwritten with the merged result, and optionally pushed. -from __future__ import division,print_function,unicode_literals import os from sys import stdin,stdout,stderr import argparse @@ -23,10 +22,7 @@ import subprocess import sys import json import codecs -try: - from urllib.request import Request,urlopen -except: - from urllib2 import Request,urlopen +from urllib.request import Request, urlopen # External tools (can be overridden using environment) GIT = os.getenv('GIT','git') diff --git a/contrib/devtools/update-translations.py b/contrib/devtools/update-translations.py index f0098cfcdf..1b9d3a4c27 100755 --- a/contrib/devtools/update-translations.py +++ b/contrib/devtools/update-translations.py @@ -125,7 +125,7 @@ def escape_cdata(text): return text def contains_bitcoin_addr(text, errors): - if text != None and ADDRESS_REGEXP.search(text) != None: + if text is not None and ADDRESS_REGEXP.search(text) is not None: errors.append('Translation "%s" contains a bitcoin address. This will be removed.' % (text)) return True return False diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index b6ead4a166..56c1fbfc92 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -7,7 +7,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. # -from __future__ import print_function, division import struct import re import os diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py index 911c3c959d..e10b46d831 100755 --- a/contrib/linearize/linearize-hashes.py +++ b/contrib/linearize/linearize-hashes.py @@ -7,11 +7,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. # -from __future__ import print_function -try: # Python 3 - import http.client as httplib -except ImportError: # Python 2 - import httplib +from http.client import HttpConnection import json import re import base64 @@ -31,7 +27,7 @@ class BitcoinRPC: authpair = "%s:%s" % (username, password) authpair = authpair.encode('utf-8') self.authhdr = b"Basic " + base64.b64encode(authpair) - self.conn = httplib.HTTPConnection(host, port=port, timeout=30) + self.conn = HttpConnection(host, port=port, timeout=30) def execute(self, obj): try: diff --git a/depends/packages/expat.mk b/depends/packages/expat.mk index acbc60eea3..8d06882cdb 100644 --- a/depends/packages/expat.mk +++ b/depends/packages/expat.mk @@ -1,11 +1,11 @@ package=expat -$(package)_version=2.2.5 -$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_2_2_5/ +$(package)_version=2.2.6 +$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_2_2_6/ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=d9dc32efba7e74f788fcc4f212a43216fc37cf5f23f4c2339664d473353aedf6 +$(package)_sha256_hash=17b43c2716d521369f82fc2dc70f359860e90fa440bea65b3b85f0b246ea81f2 define $(package)_set_vars -$(package)_config_opts=--disable-static +$(package)_config_opts=--disable-static --without-docbook endef define $(package)_config_cmds diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index dc1d17cd57..deebf13e98 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -1,9 +1,9 @@ PACKAGE=qt -$(package)_version=5.9.6 +$(package)_version=5.9.7 $(package)_download_path=https://download.qt.io/official_releases/qt/5.9/$($(package)_version)/submodules $(package)_suffix=opensource-src-$($(package)_version).tar.xz $(package)_file_name=qtbase-$($(package)_suffix) -$(package)_sha256_hash=eed620cb268b199bd83b3fc6a471c51d51e1dc2dbb5374fc97a0cc75facbe36f +$(package)_sha256_hash=36dd9574f006eaa1e5af780e4b33d11fe39d09fd7c12f3b9d83294174bd28f00 $(package)_dependencies=openssl zlib $(package)_linux_dependencies=freetype fontconfig libxcb libX11 xproto libXext $(package)_build_subdir=qtbase @@ -11,10 +11,10 @@ $(package)_qt_libs=corelib network widgets gui plugins testlib $(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_configure_mac.patch fix_no_printer.patch fix_rcc_determinism.patch fix_riscv64_arch.patch xkb-default.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) -$(package)_qttranslations_sha256_hash=9822084f8e2d2939ba39f4af4c0c2320e45d5996762a9423f833055607604ed8 +$(package)_qttranslations_sha256_hash=b36da7d93c3ab6fca56b32053bb73bc619c8b192bb89b74e3bcde2705f1c2a14 $(package)_qttools_file_name=qttools-$($(package)_suffix) -$(package)_qttools_sha256_hash=50e75417ec0c74bb8b1989d1d8e981ee83690dce7dfc0c2169f7c00f397e5117 +$(package)_qttools_sha256_hash=d62e0f70d99645d6704dbb8976fb2222443061743689943d40970c52c49367a1 $(package)_extra_sources = $($(package)_qttranslations_file_name) $(package)_extra_sources += $($(package)_qttools_file_name) @@ -27,6 +27,7 @@ $(package)_config_opts += -c++std c++11 $(package)_config_opts += -confirm-license $(package)_config_opts += -dbus-runtime $(package)_config_opts += -hostprefix $(build_prefix) +$(package)_config_opts += -no-compile-examples $(package)_config_opts += -no-cups $(package)_config_opts += -no-egl $(package)_config_opts += -no-eglfs @@ -69,9 +70,20 @@ $(package)_config_opts += -system-zlib $(package)_config_opts += -static $(package)_config_opts += -silent $(package)_config_opts += -v +$(package)_config_opts += -no-feature-dial +$(package)_config_opts += -no-feature-ftp +$(package)_config_opts += -no-feature-lcdnumber +$(package)_config_opts += -no-feature-pdf $(package)_config_opts += -no-feature-printer $(package)_config_opts += -no-feature-printdialog $(package)_config_opts += -no-feature-concurrent +$(package)_config_opts += -no-feature-sql +$(package)_config_opts += -no-feature-statemachine +$(package)_config_opts += -no-feature-syntaxhighlighter +$(package)_config_opts += -no-feature-textbrowser +$(package)_config_opts += -no-feature-textodfwriter +$(package)_config_opts += -no-feature-udpsocket +$(package)_config_opts += -no-feature-wizard $(package)_config_opts += -no-feature-xml ifneq ($(build_os),darwin) diff --git a/doc/dependencies.md b/doc/dependencies.md index 85e93bee2b..50dde02fad 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -9,7 +9,7 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | Boost | [1.64.0](https://www.boost.org/users/download/) | [1.47.0](https://github.com/bitcoin/bitcoin/pull/8920) | No | | | | Clang | | [3.3+](https://llvm.org/releases/download.html) (C++11 support) | | | | | D-Bus | [1.10.18](https://cgit.freedesktop.org/dbus/dbus/tree/NEWS?h=dbus-1.10) | | No | Yes | | -| Expat | [2.2.5](https://libexpat.github.io/) | | No | Yes | | +| Expat | [2.2.6](https://libexpat.github.io/) | | No | Yes | | | fontconfig | [2.12.1](https://www.freedesktop.org/software/fontconfig/release/) | | No | Yes | | | FreeType | [2.7.1](https://download.savannah.gnu.org/releases/freetype) | | No | | | | GCC | | [4.8+](https://gcc.gnu.org/) (C++11 support) | | | | @@ -23,7 +23,7 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | protobuf | [2.6.1](https://github.com/google/protobuf/releases) | | No | | | | Python (tests) | | [3.4](https://www.python.org/downloads) | | | | | qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | -| Qt | [5.9.6](https://download.qt.io/official_releases/qt/) | [5.2](https://github.com/bitcoin/bitcoin/pull/14725) | No | | | +| Qt | [5.9.7](https://download.qt.io/official_releases/qt/) | [5.2](https://github.com/bitcoin/bitcoin/pull/14725) | No | | | | XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L87) (Linux only) | | xkbcommon | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk#L86) (Linux only) | | ZeroMQ | [4.2.5](https://github.com/zeromq/libzmq/releases) | 4.0.0 | No | | | diff --git a/doc/psbt.md b/doc/psbt.md index 95e2f7fa01..7e6a93714d 100644 --- a/doc/psbt.md +++ b/doc/psbt.md @@ -90,7 +90,7 @@ the command line in case `bitcoin-cli` is used. Setup: - All three call `getnewaddress` to create a new address; call these addresses *Aalice*, *Abob*, and *Acarol*. -- All three call `getaddressinfo X`, with *X* their respective address, and +- All three call `getaddressinfo "X"`, with *X* their respective address, and remember the corresponding public keys. Call these public keys *Kalice*, *Kbob*, and *Kcarol*. - All three now run `addmultisigaddress 2 ["Kalice","Kbob","Kcarol"]` to teach @@ -105,28 +105,28 @@ Setup: output. Again, it may be necessary to explicitly specify the addresstype in order to get a result that matches. This command won't enable them to initiate transactions later, however. -- They can now give out *D* as address others can pay to. +- They can now give out *Amulti* as address others can pay to. Later, when *V* BTC has been received on *Amulti*, and Bob and Carol want to move the coins in their entirety to address *Asend*, with no change. Alice does not need to be involved. - One of them - let's assume Carol here - initiates the creation. She runs - `walletcreatefundedpsbt [] {"Asend":V} 0 false {"subtractFeeFromOutputs":[0], "includeWatching":true}`. - We call the resulting PSBT *P*. P does not contain any signatures. + `walletcreatefundedpsbt [] {"Asend":V} 0 {"subtractFeeFromOutputs":[0], "includeWatching":true}`. + We call the resulting PSBT *P*. *P* does not contain any signatures. - Carol needs to sign the transaction herself. In order to do so, she runs - `walletprocesspsbt P`, and gives the resulting PSBT *P2* to Bob. + `walletprocesspsbt "P"`, and gives the resulting PSBT *P2* to Bob. - Bob inspects the PSBT using `decodepsbt "P2"` to determine if the transaction has indeed just the expected input, and an output to *Asend*, and the fee is reasonable. If he agrees, he calls `walletprocesspsbt "P2"` to sign. The resulting PSBT *P3* contains both Carol's and Bob's signature. -- Now anyone can call `finalizepsbt "P2"` to extract a fully signed transaction +- Now anyone can call `finalizepsbt "P3"` to extract a fully signed transaction *T*. - Finally anyone can broadcast the transaction using `sendrawtransaction "T"`. In case there are more signers, it may be advantageous to let them all sign in parallel, rather passing the PSBT from one signer to the next one. In the above example this would translate to Carol handing a copy of *P* to each signer -separately. They can then all invoke `walletprocesspsbt P`, and end up with +separately. They can then all invoke `walletprocesspsbt "P"`, and end up with their individually-signed PSBT structures. They then all send those back to Carol (or anyone) who can combine them using `combinepsbt`. The last two steps (`finalizepsbt` and `sendrawtransaction`) remain unchanged. diff --git a/src/addrman.cpp b/src/addrman.cpp index 44328c3056..06c342ba73 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -217,7 +217,7 @@ void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime return; // find a bucket it is in now - int nRnd = RandomInt(ADDRMAN_NEW_BUCKET_COUNT); + int nRnd = insecure_rand.randrange(ADDRMAN_NEW_BUCKET_COUNT); int nUBucket = -1; for (unsigned int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; n++) { int nB = (n + nRnd) % ADDRMAN_NEW_BUCKET_COUNT; @@ -291,7 +291,7 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP int nFactor = 1; for (int n = 0; n < pinfo->nRefCount; n++) nFactor *= 2; - if (nFactor > 1 && (RandomInt(nFactor) != 0)) + if (nFactor > 1 && (insecure_rand.randrange(nFactor) != 0)) return false; } else { pinfo = Create(addr, source, &nId); @@ -356,12 +356,12 @@ CAddrInfo CAddrMan::Select_(bool newOnly) // Use a 50% chance for choosing between tried and new table entries. if (!newOnly && - (nTried > 0 && (nNew == 0 || RandomInt(2) == 0))) { + (nTried > 0 && (nNew == 0 || insecure_rand.randbool() == 0))) { // use a tried node double fChanceFactor = 1.0; while (1) { - int nKBucket = RandomInt(ADDRMAN_TRIED_BUCKET_COUNT); - int nKBucketPos = RandomInt(ADDRMAN_BUCKET_SIZE); + int nKBucket = insecure_rand.randrange(ADDRMAN_TRIED_BUCKET_COUNT); + int nKBucketPos = insecure_rand.randrange(ADDRMAN_BUCKET_SIZE); while (vvTried[nKBucket][nKBucketPos] == -1) { nKBucket = (nKBucket + insecure_rand.randbits(ADDRMAN_TRIED_BUCKET_COUNT_LOG2)) % ADDRMAN_TRIED_BUCKET_COUNT; nKBucketPos = (nKBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; @@ -369,7 +369,7 @@ CAddrInfo CAddrMan::Select_(bool newOnly) int nId = vvTried[nKBucket][nKBucketPos]; assert(mapInfo.count(nId) == 1); CAddrInfo& info = mapInfo[nId]; - if (RandomInt(1 << 30) < fChanceFactor * info.GetChance() * (1 << 30)) + if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) return info; fChanceFactor *= 1.2; } @@ -377,8 +377,8 @@ CAddrInfo CAddrMan::Select_(bool newOnly) // use a new node double fChanceFactor = 1.0; while (1) { - int nUBucket = RandomInt(ADDRMAN_NEW_BUCKET_COUNT); - int nUBucketPos = RandomInt(ADDRMAN_BUCKET_SIZE); + int nUBucket = insecure_rand.randrange(ADDRMAN_NEW_BUCKET_COUNT); + int nUBucketPos = insecure_rand.randrange(ADDRMAN_BUCKET_SIZE); while (vvNew[nUBucket][nUBucketPos] == -1) { nUBucket = (nUBucket + insecure_rand.randbits(ADDRMAN_NEW_BUCKET_COUNT_LOG2)) % ADDRMAN_NEW_BUCKET_COUNT; nUBucketPos = (nUBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; @@ -386,7 +386,7 @@ CAddrInfo CAddrMan::Select_(bool newOnly) int nId = vvNew[nUBucket][nUBucketPos]; assert(mapInfo.count(nId) == 1); CAddrInfo& info = mapInfo[nId]; - if (RandomInt(1 << 30) < fChanceFactor * info.GetChance() * (1 << 30)) + if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) return info; fChanceFactor *= 1.2; } @@ -482,7 +482,7 @@ void CAddrMan::GetAddr_(std::vector<CAddress>& vAddr) if (vAddr.size() >= nNodes) break; - int nRndPos = RandomInt(vRandom.size() - n) + n; + int nRndPos = insecure_rand.randrange(vRandom.size() - n) + n; SwapRandom(n, nRndPos); assert(mapInfo.count(vRandom[n]) == 1); @@ -530,10 +530,6 @@ void CAddrMan::SetServices_(const CService& addr, ServiceFlags nServices) info.nServices = nServices; } -int CAddrMan::RandomInt(int nMax){ - return GetRandInt(nMax); -} - void CAddrMan::ResolveCollisions_() { for (std::set<int>::iterator it = m_tried_collisions.begin(); it != m_tried_collisions.end();) { @@ -593,7 +589,7 @@ CAddrInfo CAddrMan::SelectTriedCollision_() std::set<int>::iterator it = m_tried_collisions.begin(); // Selects a random element from m_tried_collisions - std::advance(it, GetRandInt(m_tried_collisions.size())); + std::advance(it, insecure_rand.randrange(m_tried_collisions.size())); int id_new = *it; // If id_new not found in mapInfo remove it from m_tried_collisions diff --git a/src/addrman.h b/src/addrman.h index b97feb6f08..af5a1d3b23 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -266,9 +266,6 @@ protected: //! Return a random to-be-evicted tried table address. CAddrInfo SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs); - //! Wraps GetRandInt to allow tests to override RandomInt and make it determinismistic. - virtual int RandomInt(int nMax); - #ifdef DEBUG_ADDRMAN //! Perform consistency check. Returns an error code or zero. int Check_() EXCLUSIVE_LOCKS_REQUIRED(cs); @@ -473,7 +470,7 @@ public: { LOCK(cs); std::vector<int>().swap(vRandom); - nKey = GetRandHash(); + nKey = insecure_rand.rand256(); for (size_t bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; bucket++) { for (size_t entry = 0; entry < ADDRMAN_BUCKET_SIZE; entry++) { vvNew[bucket][entry] = -1; diff --git a/src/init.cpp b/src/init.cpp index b1fa8fc695..8ecd79197f 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -500,7 +500,7 @@ void SetupServerArgs() gArgs.AddArg("-rest", strprintf("Accept public REST requests (default: %u)", DEFAULT_REST_ENABLE), false, OptionsCategory::RPC); gArgs.AddArg("-rpcallowip=<ip>", "Allow JSON-RPC connections from specified source. Valid for <ip> are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24). This option can be specified multiple times", false, OptionsCategory::RPC); - gArgs.AddArg("-rpcauth=<userpw>", "Username and hashed password for JSON-RPC connections. The field <userpw> comes in the format: <USERNAME>:<SALT>$<HASH>. A canonical python script is included in share/rpcauth. The client then connects normally using the rpcuser=<USERNAME>/rpcpassword=<PASSWORD> pair of arguments. This option can be specified multiple times", false, OptionsCategory::RPC); + gArgs.AddArg("-rpcauth=<userpw>", "Username and HMAC-SHA-256 hashed password for JSON-RPC connections. The field <userpw> comes in the format: <USERNAME>:<SALT>$<HASH>. A canonical python script is included in share/rpcauth. The client then connects normally using the rpcuser=<USERNAME>/rpcpassword=<PASSWORD> pair of arguments. This option can be specified multiple times", false, OptionsCategory::RPC); gArgs.AddArg("-rpcbind=<addr>[:port]", "Bind to given address to listen for JSON-RPC connections. Do not expose the RPC server to untrusted networks such as the public internet! This option is ignored unless -rpcallowip is also passed. Port is optional and overrides -rpcport. Use [host]:port notation for IPv6. This option can be specified multiple times (default: 127.0.0.1 and ::1 i.e., localhost)", false, OptionsCategory::RPC); gArgs.AddArg("-rpccookiefile=<loc>", "Location of the auth cookie. Relative paths will be prefixed by a net-specific datadir location. (default: data dir)", false, OptionsCategory::RPC); gArgs.AddArg("-rpcpassword=<pw>", "Password for JSON-RPC connections", false, OptionsCategory::RPC); diff --git a/src/net.cpp b/src/net.cpp index fde85b0f2a..e595fb0b0b 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -134,11 +134,12 @@ static std::vector<CAddress> convertSeed6(const std::vector<SeedSpec6> &vSeedsIn const int64_t nOneWeek = 7*24*60*60; std::vector<CAddress> vSeedsOut; vSeedsOut.reserve(vSeedsIn.size()); + FastRandomContext rng; for (const auto& seed_in : vSeedsIn) { struct in6_addr ip; memcpy(&ip, seed_in.addr, sizeof(ip)); CAddress addr(CService(ip, seed_in.port), GetDesirableServiceFlags(NODE_NONE)); - addr.nTime = GetTime() - GetRand(nOneWeek) - nOneWeek; + addr.nTime = GetTime() - rng.randrange(nOneWeek) - nOneWeek; vSeedsOut.push_back(addr); } return vSeedsOut; @@ -189,16 +190,16 @@ void AdvertiseLocal(CNode *pnode) // If discovery is enabled, sometimes give our peer the address it // tells us that it sees us as in case it has a better idea of our // address than we do. + FastRandomContext rng; if (IsPeerAddrLocalGood(pnode) && (!addrLocal.IsRoutable() || - GetRand((GetnScore(addrLocal) > LOCAL_MANUAL) ? 8:2) == 0)) + rng.randbits((GetnScore(addrLocal) > LOCAL_MANUAL) ? 3 : 1) == 0)) { addrLocal.SetIP(pnode->GetAddrLocal()); } if (addrLocal.IsRoutable() || gArgs.GetBoolArg("-addrmantest", false)) { LogPrint(BCLog::NET, "AdvertiseLocal: advertising address %s\n", addrLocal.ToString()); - FastRandomContext insecure_rand; - pnode->PushAddress(addrLocal, insecure_rand); + pnode->PushAddress(addrLocal, rng); } } } diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 40c5112578..0e222bdfa4 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -779,10 +779,11 @@ unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) nNextSweep = nMinExpTime + ORPHAN_TX_EXPIRE_INTERVAL; if (nErased > 0) LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx due to expiration\n", nErased); } + FastRandomContext rng; while (mapOrphanTransactions.size() > nMaxOrphans) { // Evict a random orphan: - uint256 randomhash = GetRandHash(); + uint256 randomhash = rng.rand256(); std::map<uint256, COrphanTx>::iterator it = mapOrphanTransactions.lower_bound(randomhash); if (it == mapOrphanTransactions.end()) it = mapOrphanTransactions.begin(); diff --git a/src/random.cpp b/src/random.cpp index a34c70e1d5..f8ffda136d 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -398,6 +398,7 @@ uint256 FastRandomContext::rand256() std::vector<unsigned char> FastRandomContext::randbytes(size_t len) { + if (requires_seed) RandomSeed(); std::vector<unsigned char> ret(len); if (len > 0) { rng.Output(&ret[0], len); @@ -463,6 +464,20 @@ FastRandomContext::FastRandomContext(bool fDeterministic) : requires_seed(!fDete rng.SetKey(seed.begin(), 32); } +FastRandomContext& FastRandomContext::operator=(FastRandomContext&& from) noexcept +{ + requires_seed = from.requires_seed; + rng = from.rng; + std::copy(std::begin(from.bytebuf), std::end(from.bytebuf), std::begin(bytebuf)); + bytebuf_size = from.bytebuf_size; + bitbuf = from.bitbuf; + bitbuf_size = from.bitbuf_size; + from.requires_seed = true; + from.bytebuf_size = 0; + from.bitbuf_size = 0; + return *this; +} + void RandomInit() { RDRandInit(); diff --git a/src/random.h b/src/random.h index 3d5421eb3e..00e90abbc5 100644 --- a/src/random.h +++ b/src/random.h @@ -76,6 +76,14 @@ public: /** Initialize with explicit seed (only for testing) */ explicit FastRandomContext(const uint256& seed); + // Do not permit copying a FastRandomContext (move it, or create a new one to get reseeded). + FastRandomContext(const FastRandomContext&) = delete; + FastRandomContext(FastRandomContext&&) = delete; + FastRandomContext& operator=(const FastRandomContext&) = delete; + + /** Move a FastRandomContext. If the original one is used again, it will be reseeded. */ + FastRandomContext& operator=(FastRandomContext&& from) noexcept; + /** Generate a random 64-bit integer. */ uint64_t rand64() { @@ -130,6 +138,29 @@ public: inline uint64_t operator()() { return rand64(); } }; +/** More efficient than using std::shuffle on a FastRandomContext. + * + * This is more efficient as std::shuffle will consume entropy in groups of + * 64 bits at the time and throw away most. + * + * This also works around a bug in libstdc++ std::shuffle that may cause + * type::operator=(type&&) to be invoked on itself, which the library's + * debug mode detects and panics on. This is a known issue, see + * https://stackoverflow.com/questions/22915325/avoiding-self-assignment-in-stdshuffle + */ +template<typename I, typename R> +void Shuffle(I first, I last, R&& rng) +{ + while (first != last) { + size_t j = rng.randrange(last - first); + if (j) { + using std::swap; + swap(*first, *(first + j)); + } + ++first; + } +} + /* Number of random bytes returned by GetOSRand. * When changing this constant make sure to change all call sites, and make * sure that the underlying OS APIs for all platforms support the number. diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 77545650d5..234da5ae4d 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -32,12 +32,6 @@ public: insecure_rand = FastRandomContext(true); } - int RandomInt(int nMax) override - { - state = (CHashWriter(SER_GETHASH, 0) << state).GetCheapHash(); - return (unsigned int)(state % nMax); - } - CAddrInfo* Find(const CNetAddr& addr, int* pnId = nullptr) { LOCK(cs); diff --git a/src/test/cuckoocache_tests.cpp b/src/test/cuckoocache_tests.cpp index ad6169ca61..a99e17693a 100644 --- a/src/test/cuckoocache_tests.cpp +++ b/src/test/cuckoocache_tests.cpp @@ -21,40 +21,23 @@ * using BOOST_CHECK_CLOSE to fail. * */ -FastRandomContext local_rand_ctx(true); - BOOST_AUTO_TEST_SUITE(cuckoocache_tests); - -/** insecure_GetRandHash fills in a uint256 from local_rand_ctx - */ -static void insecure_GetRandHash(uint256& t) -{ - uint32_t* ptr = (uint32_t*)t.begin(); - for (uint8_t j = 0; j < 8; ++j) - *(ptr++) = local_rand_ctx.rand32(); -} - - - /* Test that no values not inserted into the cache are read out of it. * * There are no repeats in the first 200000 insecure_GetRandHash calls */ BOOST_AUTO_TEST_CASE(test_cuckoocache_no_fakes) { - local_rand_ctx = FastRandomContext(true); + SeedInsecureRand(true); CuckooCache::cache<uint256, SignatureCacheHasher> cc{}; size_t megabytes = 4; cc.setup_bytes(megabytes << 20); - uint256 v; for (int x = 0; x < 100000; ++x) { - insecure_GetRandHash(v); - cc.insert(v); + cc.insert(InsecureRand256()); } for (int x = 0; x < 100000; ++x) { - insecure_GetRandHash(v); - BOOST_CHECK(!cc.contains(v, false)); + BOOST_CHECK(!cc.contains(InsecureRand256(), false)); } }; @@ -64,7 +47,7 @@ BOOST_AUTO_TEST_CASE(test_cuckoocache_no_fakes) template <typename Cache> static double test_cache(size_t megabytes, double load) { - local_rand_ctx = FastRandomContext(true); + SeedInsecureRand(true); std::vector<uint256> hashes; Cache set{}; size_t bytes = megabytes * (1 << 20); @@ -74,7 +57,7 @@ static double test_cache(size_t megabytes, double load) for (uint32_t i = 0; i < n_insert; ++i) { uint32_t* ptr = (uint32_t*)hashes[i].begin(); for (uint8_t j = 0; j < 8; ++j) - *(ptr++) = local_rand_ctx.rand32(); + *(ptr++) = InsecureRand32(); } /** We make a copy of the hashes because future optimizations of the * cuckoocache may overwrite the inserted element, so the test is @@ -135,7 +118,7 @@ template <typename Cache> static void test_cache_erase(size_t megabytes) { double load = 1; - local_rand_ctx = FastRandomContext(true); + SeedInsecureRand(true); std::vector<uint256> hashes; Cache set{}; size_t bytes = megabytes * (1 << 20); @@ -145,7 +128,7 @@ static void test_cache_erase(size_t megabytes) for (uint32_t i = 0; i < n_insert; ++i) { uint32_t* ptr = (uint32_t*)hashes[i].begin(); for (uint8_t j = 0; j < 8; ++j) - *(ptr++) = local_rand_ctx.rand32(); + *(ptr++) = InsecureRand32(); } /** We make a copy of the hashes because future optimizations of the * cuckoocache may overwrite the inserted element, so the test is @@ -198,7 +181,7 @@ template <typename Cache> static void test_cache_erase_parallel(size_t megabytes) { double load = 1; - local_rand_ctx = FastRandomContext(true); + SeedInsecureRand(true); std::vector<uint256> hashes; Cache set{}; size_t bytes = megabytes * (1 << 20); @@ -208,7 +191,7 @@ static void test_cache_erase_parallel(size_t megabytes) for (uint32_t i = 0; i < n_insert; ++i) { uint32_t* ptr = (uint32_t*)hashes[i].begin(); for (uint8_t j = 0; j < 8; ++j) - *(ptr++) = local_rand_ctx.rand32(); + *(ptr++) = InsecureRand32(); } /** We make a copy of the hashes because future optimizations of the * cuckoocache may overwrite the inserted element, so the test is @@ -300,7 +283,7 @@ static void test_cache_generations() // iterations with non-deterministic values, so it isn't "overfit" to the // specific entropy in FastRandomContext(true) and implementation of the // cache. - local_rand_ctx = FastRandomContext(true); + SeedInsecureRand(true); // block_activity models a chunk of network activity. n_insert elements are // added to the cache. The first and last n/4 are stored for removal later @@ -317,7 +300,7 @@ static void test_cache_generations() for (uint32_t i = 0; i < n_insert; ++i) { uint32_t* ptr = (uint32_t*)inserts[i].begin(); for (uint8_t j = 0; j < 8; ++j) - *(ptr++) = local_rand_ctx.rand32(); + *(ptr++) = InsecureRand32(); } for (uint32_t i = 0; i < n_insert / 4; ++i) reads.push_back(inserts[i]); diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index b49f447c33..8cf614bc8d 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -111,7 +111,7 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) static void AddRandomOutboundPeer(std::vector<CNode *> &vNodes, PeerLogicValidation &peerLogic) { - CAddress addr(ip(GetRandInt(0xffffffff)), NODE_NONE); + CAddress addr(ip(g_insecure_rand_ctx.randbits(32)), NODE_NONE); vNodes.emplace_back(new CNode(id++, ServiceFlags(NODE_NETWORK|NODE_WITNESS), 0, INVALID_SOCKET, addr, 0, 0, CAddress(), "", /*fInboundIn=*/ false)); CNode &node = *vNodes.back(); node.SetSendVersion(PROTOCOL_VERSION); diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index c488d3edcf..7341389208 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -189,8 +189,8 @@ public: prevector_tester() { SeedInsecureRand(); - rand_seed = insecure_rand_seed; - rand_cache = insecure_rand_ctx; + rand_seed = InsecureRand256(); + rand_cache = FastRandomContext(rand_seed); } }; diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index 679e857ce6..1057d09471 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -38,11 +38,18 @@ BOOST_AUTO_TEST_CASE(fastrandom_tests) BOOST_CHECK(ctx1.randbytes(50) == ctx2.randbytes(50)); // Check that a nondeterministic ones are not - FastRandomContext ctx3; - FastRandomContext ctx4; - BOOST_CHECK(ctx3.rand64() != ctx4.rand64()); // extremely unlikely to be equal - BOOST_CHECK(ctx3.rand256() != ctx4.rand256()); - BOOST_CHECK(ctx3.randbytes(7) != ctx4.randbytes(7)); + { + FastRandomContext ctx3, ctx4; + BOOST_CHECK(ctx3.rand64() != ctx4.rand64()); // extremely unlikely to be equal + } + { + FastRandomContext ctx3, ctx4; + BOOST_CHECK(ctx3.rand256() != ctx4.rand256()); + } + { + FastRandomContext ctx3, ctx4; + BOOST_CHECK(ctx3.randbytes(7) != ctx4.randbytes(7)); + } } BOOST_AUTO_TEST_CASE(fastrandom_randbits) @@ -75,8 +82,42 @@ BOOST_AUTO_TEST_CASE(stdrandom_test) for (int j = 1; j <= 10; ++j) { BOOST_CHECK(std::find(test.begin(), test.end(), j) != test.end()); } + Shuffle(test.begin(), test.end(), ctx); + for (int j = 1; j <= 10; ++j) { + BOOST_CHECK(std::find(test.begin(), test.end(), j) != test.end()); + } } } +/** Test that Shuffle reaches every permutation with equal probability. */ +BOOST_AUTO_TEST_CASE(shuffle_stat_test) +{ + FastRandomContext ctx(true); + uint32_t counts[5 * 5 * 5 * 5 * 5] = {0}; + for (int i = 0; i < 12000; ++i) { + int data[5] = {0, 1, 2, 3, 4}; + Shuffle(std::begin(data), std::end(data), ctx); + int pos = data[0] + data[1] * 5 + data[2] * 25 + data[3] * 125 + data[4] * 625; + ++counts[pos]; + } + unsigned int sum = 0; + double chi_score = 0.0; + for (int i = 0; i < 5 * 5 * 5 * 5 * 5; ++i) { + int i1 = i % 5, i2 = (i / 5) % 5, i3 = (i / 25) % 5, i4 = (i / 125) % 5, i5 = i / 625; + uint32_t count = counts[i]; + if (i1 == i2 || i1 == i3 || i1 == i4 || i1 == i5 || i2 == i3 || i2 == i4 || i2 == i5 || i3 == i4 || i3 == i5 || i4 == i5) { + BOOST_CHECK(count == 0); + } else { + chi_score += ((count - 100.0) * (count - 100.0)) / 100.0; + BOOST_CHECK(count > 50); + BOOST_CHECK(count < 150); + sum += count; + } + } + BOOST_CHECK(chi_score > 58.1411); // 99.9999% confidence interval + BOOST_CHECK(chi_score < 210.275); + BOOST_CHECK_EQUAL(sum, 12000); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 0fd271c758..1b394753ef 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -1030,6 +1030,28 @@ BOOST_AUTO_TEST_CASE(script_PushData) BOOST_CHECK(EvalScript(pushdata4Stack, CScript(pushdata4, pushdata4 + sizeof(pushdata4)), SCRIPT_VERIFY_P2SH, BaseSignatureChecker(), SigVersion::BASE, &err)); BOOST_CHECK(pushdata4Stack == directStack); BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_OK, ScriptErrorString(err)); + + const std::vector<unsigned char> pushdata1_trunc{OP_PUSHDATA1, 1}; + const std::vector<unsigned char> pushdata2_trunc{OP_PUSHDATA2, 1, 0}; + const std::vector<unsigned char> pushdata4_trunc{OP_PUSHDATA4, 1, 0, 0, 0}; + + std::vector<std::vector<unsigned char>> stack_ignore; + BOOST_CHECK(!EvalScript(stack_ignore, CScript(pushdata1_trunc.begin(), pushdata1_trunc.end()), SCRIPT_VERIFY_P2SH, BaseSignatureChecker(), SigVersion::BASE, &err)); + BOOST_CHECK_EQUAL(err, SCRIPT_ERR_BAD_OPCODE); + BOOST_CHECK(!EvalScript(stack_ignore, CScript(pushdata2_trunc.begin(), pushdata2_trunc.end()), SCRIPT_VERIFY_P2SH, BaseSignatureChecker(), SigVersion::BASE, &err)); + BOOST_CHECK_EQUAL(err, SCRIPT_ERR_BAD_OPCODE); + BOOST_CHECK(!EvalScript(stack_ignore, CScript(pushdata4_trunc.begin(), pushdata4_trunc.end()), SCRIPT_VERIFY_P2SH, BaseSignatureChecker(), SigVersion::BASE, &err)); + BOOST_CHECK_EQUAL(err, SCRIPT_ERR_BAD_OPCODE); +} + +BOOST_AUTO_TEST_CASE(script_cltv_truncated) +{ + const auto script_cltv_trunc = CScript() << OP_CHECKLOCKTIMEVERIFY; + + std::vector<std::vector<unsigned char>> stack_ignore; + ScriptError err; + BOOST_CHECK(!EvalScript(stack_ignore, script_cltv_trunc, SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY, BaseSignatureChecker(), SigVersion::BASE, &err)); + BOOST_CHECK_EQUAL(err, SCRIPT_ERR_INVALID_STACK_OPERATION); } static CScript diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index f7874e6882..bb8db9fa8d 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -11,6 +11,7 @@ #include <crypto/sha256.h> #include <miner.h> #include <net_processing.h> +#include <noui.h> #include <pow.h> #include <rpc/register.h> #include <rpc/server.h> @@ -36,11 +37,7 @@ void CConnmanTest::ClearNodes() g_connman->vNodes.clear(); } -uint256 insecure_rand_seed = GetRandHash(); -FastRandomContext insecure_rand_ctx(insecure_rand_seed); - -extern bool fPrintToConsole; -extern void noui_connect(); +thread_local FastRandomContext g_insecure_rand_ctx; std::ostream& operator<<(std::ostream& os, const uint256& num) { @@ -118,16 +115,16 @@ TestingSetup::TestingSetup(const std::string& chainName) : BasicTestingSetup(cha TestingSetup::~TestingSetup() { - threadGroup.interrupt_all(); - threadGroup.join_all(); - GetMainSignals().FlushBackgroundCallbacks(); - GetMainSignals().UnregisterBackgroundSignalScheduler(); - g_connman.reset(); - peerLogic.reset(); - UnloadBlockIndex(); - pcoinsTip.reset(); - pcoinsdbview.reset(); - pblocktree.reset(); + threadGroup.interrupt_all(); + threadGroup.join_all(); + GetMainSignals().FlushBackgroundCallbacks(); + GetMainSignals().UnregisterBackgroundSignalScheduler(); + g_connman.reset(); + peerLogic.reset(); + UnloadBlockIndex(); + pcoinsTip.reset(); + pcoinsdbview.reset(); + pblocktree.reset(); } TestChain100Setup::TestChain100Setup() : TestingSetup(CBaseChainParams::REGTEST) diff --git a/src/test/test_bitcoin.h b/src/test/test_bitcoin.h index 182571b004..71c0379eac 100644 --- a/src/test/test_bitcoin.h +++ b/src/test/test_bitcoin.h @@ -26,24 +26,18 @@ std::ostream& operator<<(typename std::enable_if<std::is_enum<T>::value, std::os return stream << static_cast<typename std::underlying_type<T>::type>(e); } -extern uint256 insecure_rand_seed; -extern FastRandomContext insecure_rand_ctx; +thread_local extern FastRandomContext g_insecure_rand_ctx; -static inline void SeedInsecureRand(bool fDeterministic = false) +static inline void SeedInsecureRand(bool deterministic = false) { - if (fDeterministic) { - insecure_rand_seed = uint256(); - } else { - insecure_rand_seed = GetRandHash(); - } - insecure_rand_ctx = FastRandomContext(insecure_rand_seed); + g_insecure_rand_ctx = FastRandomContext(deterministic); } -static inline uint32_t InsecureRand32() { return insecure_rand_ctx.rand32(); } -static inline uint256 InsecureRand256() { return insecure_rand_ctx.rand256(); } -static inline uint64_t InsecureRandBits(int bits) { return insecure_rand_ctx.randbits(bits); } -static inline uint64_t InsecureRandRange(uint64_t range) { return insecure_rand_ctx.randrange(range); } -static inline bool InsecureRandBool() { return insecure_rand_ctx.randbool(); } +static inline uint32_t InsecureRand32() { return g_insecure_rand_ctx.rand32(); } +static inline uint256 InsecureRand256() { return g_insecure_rand_ctx.rand256(); } +static inline uint64_t InsecureRandBits(int bits) { return g_insecure_rand_ctx.randbits(bits); } +static inline uint64_t InsecureRandRange(uint64_t range) { return g_insecure_rand_ctx.randrange(range); } +static inline bool InsecureRandBool() { return g_insecure_rand_ctx.randbool(); } static constexpr CAmount CENT{1000000}; @@ -73,7 +67,7 @@ struct CConnmanTest { }; class PeerLogicValidation; -struct TestingSetup: public BasicTestingSetup { +struct TestingSetup : public BasicTestingSetup { boost::thread_group threadGroup; CConnman* connman; CScheduler scheduler; diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 935df34d79..7dd176b25d 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -104,8 +104,8 @@ void BuildChain(const uint256& root, int height, const unsigned int invalid_rate { if (height <= 0 || blocks.size() >= max_size) return; - bool gen_invalid = GetRand(100) < invalid_rate; - bool gen_fork = GetRand(100) < branch_rate; + bool gen_invalid = InsecureRandRange(100) < invalid_rate; + bool gen_fork = InsecureRandRange(100) < branch_rate; const std::shared_ptr<const CBlock> pblock = gen_invalid ? BadBlock(root) : GoodBlock(root); blocks.push_back(pblock); @@ -157,7 +157,7 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) threads.create_thread([&blocks]() { bool ignored; for (int i = 0; i < 1000; i++) { - auto block = blocks[GetRand(blocks.size() - 1)]; + auto block = blocks[InsecureRandRange(blocks.size() - 1)]; ProcessNewBlock(Params(), block, true, &ignored); } diff --git a/src/validation.cpp b/src/validation.cpp index e92cbf1b8d..5696684ed6 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -3267,6 +3267,7 @@ static bool ContextualCheckBlock(const CBlock& block, CValidationState& state, c // Start enforcing BIP113 (Median Time Past) using versionbits logic. int nLockTimeFlags = 0; if (VersionBitsState(pindexPrev, consensusParams, Consensus::DEPLOYMENT_CSV, versionbitscache) == ThresholdState::ACTIVE) { + assert(pindexPrev != nullptr); nLockTimeFlags |= LOCKTIME_MEDIAN_TIME_PAST; } diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 5e955b8495..8a37f374a1 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -223,7 +223,7 @@ bool KnapsackSolver(const CAmount& nTargetValue, std::vector<OutputGroup>& group std::vector<OutputGroup> applicable_groups; CAmount nTotalLower = 0; - random_shuffle(groups.begin(), groups.end(), GetRandInt); + Shuffle(groups.begin(), groups.end(), FastRandomContext()); for (const OutputGroup& group : groups) { if (group.m_value == nTargetValue) { diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 829a1a2478..fea0378839 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -22,6 +22,7 @@ #include <policy/rbf.h> #include <primitives/block.h> #include <primitives/transaction.h> +#include <script/descriptor.h> #include <script/script.h> #include <shutdown.h> #include <timedata.h> @@ -104,67 +105,17 @@ std::string COutput::ToString() const return strprintf("COutput(%s, %d, %d) [%s]", tx->GetHash().ToString(), i, nDepth, FormatMoney(tx->tx->vout[i].nValue)); } -/** A class to identify which pubkeys a script and a keystore have in common. */ -class CAffectedKeysVisitor : public boost::static_visitor<void> { -private: - const CKeyStore &keystore; - std::vector<CKeyID> &vKeys; - -public: - /** - * @param[in] keystoreIn The CKeyStore that is queried for the presence of a pubkey. - * @param[out] vKeysIn A vector to which a script's pubkey identifiers are appended if they are in the keystore. - */ - CAffectedKeysVisitor(const CKeyStore &keystoreIn, std::vector<CKeyID> &vKeysIn) : keystore(keystoreIn), vKeys(vKeysIn) {} - - /** - * Apply the visitor to each destination in a script, recursively to the redeemscript - * in the case of p2sh destinations. - * @param[in] script The CScript from which destinations are extracted. - * @post Any CKeyIDs that script and keystore have in common are appended to the visitor's vKeys. - */ - void Process(const CScript &script) { - txnouttype type; - std::vector<CTxDestination> vDest; - int nRequired; - if (ExtractDestinations(script, type, vDest, nRequired)) { - for (const CTxDestination &dest : vDest) - boost::apply_visitor(*this, dest); - } - } - - void operator()(const CKeyID &keyId) { - if (keystore.HaveKey(keyId)) - vKeys.push_back(keyId); - } - - void operator()(const CScriptID &scriptId) { - CScript script; - if (keystore.GetCScript(scriptId, script)) - Process(script); - } - - void operator()(const WitnessV0ScriptHash& scriptID) - { - CScriptID id; - CRIPEMD160().Write(scriptID.begin(), 32).Finalize(id.begin()); - CScript script; - if (keystore.GetCScript(id, script)) { - Process(script); - } - } - - void operator()(const WitnessV0KeyHash& keyid) - { - CKeyID id(keyid); - if (keystore.HaveKey(id)) { - vKeys.push_back(id); - } +std::vector<CKeyID> GetAffectedKeys(const CScript& spk, const SigningProvider& provider) +{ + std::vector<CScript> dummy; + FlatSigningProvider out; + InferDescriptor(spk, provider)->Expand(0, DUMMY_SIGNING_PROVIDER, dummy, out); + std::vector<CKeyID> ret; + for (const auto& entry : out.pubkeys) { + ret.push_back(entry.first); } - - template<typename X> - void operator()(const X &none) {} -}; + return ret; +} const CWalletTx* CWallet::GetWalletTx(const uint256& hash) const { @@ -977,9 +928,7 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, const CBlockI // loop though all outputs for (const CTxOut& txout: tx.vout) { // extract addresses and check if they match with an unused keypool key - std::vector<CKeyID> vAffected; - CAffectedKeysVisitor(*this, vAffected).Process(txout.scriptPubKey); - for (const CKeyID &keyid : vAffected) { + for (const auto& keyid : GetAffectedKeys(txout.scriptPubKey, *this)) { std::map<CKeyID, int64_t>::const_iterator mi = m_pool_key_to_index.find(keyid); if (mi != m_pool_key_to_index.end()) { WalletLogPrintf("%s: Detected a used keypool key, mark all keypool key up to this key as used\n", __func__); @@ -2462,7 +2411,7 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm // Cases where we have 11+ outputs all pointing to the same destination may result in // privacy leaks as they will potentially be deterministically sorted. We solve that by // explicitly shuffling the outputs before processing - std::shuffle(vCoins.begin(), vCoins.end(), FastRandomContext()); + Shuffle(vCoins.begin(), vCoins.end(), FastRandomContext()); } std::vector<OutputGroup> groups = GroupOutputs(vCoins, !coin_control.m_avoid_partial_spends); @@ -2922,7 +2871,7 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std // Shuffle selected coins and fill in final vin txNew.vin.clear(); std::vector<CInputCoin> selected_coins(setCoins.begin(), setCoins.end()); - std::shuffle(selected_coins.begin(), selected_coins.end(), FastRandomContext()); + Shuffle(selected_coins.begin(), selected_coins.end(), FastRandomContext()); // Note how the sequence number is set to non-maxint so that // the nLockTime set above actually works. @@ -3711,7 +3660,6 @@ void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<C return; // find first block that affects those keys, if there are any left - std::vector<CKeyID> vAffected; for (const auto& entry : mapWallet) { // iterate over all wallet transactions... const CWalletTx &wtx = entry.second; @@ -3721,14 +3669,12 @@ void CWallet::GetKeyBirthTimes(interfaces::Chain::Lock& locked_chain, std::map<C int nHeight = pindex->nHeight; for (const CTxOut &txout : wtx.tx->vout) { // iterate over all their outputs - CAffectedKeysVisitor(*this, vAffected).Process(txout.scriptPubKey); - for (const CKeyID &keyid : vAffected) { + for (const auto &keyid : GetAffectedKeys(txout.scriptPubKey, *this)) { // ... and all their affected keys std::map<CKeyID, CBlockIndex*>::iterator rit = mapKeyFirstBlock.find(keyid); if (rit != mapKeyFirstBlock.end() && nHeight < rit->second->nHeight) rit->second = pindex; } - vAffected.clear(); } } } diff --git a/test/functional/interface_http.py b/test/functional/interface_http.py index e4b86f9e1e..20889366e5 100755 --- a/test/functional/interface_http.py +++ b/test/functional/interface_http.py @@ -31,13 +31,13 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() assert(b'"error":null' in out1) - assert(conn.sock!=None) #according to http/1.1 connection must still be open! + assert(conn.sock is not None) #according to http/1.1 connection must still be open! #send 2nd request without closing connection conn.request('POST', '/', '{"method": "getchaintips"}', headers) out1 = conn.getresponse().read() assert(b'"error":null' in out1) #must also response with a correct json-rpc message - assert(conn.sock!=None) #according to http/1.1 connection must still be open! + assert(conn.sock is not None) #according to http/1.1 connection must still be open! conn.close() #same should be if we add keep-alive because this should be the std. behaviour @@ -48,13 +48,13 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() assert(b'"error":null' in out1) - assert(conn.sock!=None) #according to http/1.1 connection must still be open! + assert(conn.sock is not None) #according to http/1.1 connection must still be open! #send 2nd request without closing connection conn.request('POST', '/', '{"method": "getchaintips"}', headers) out1 = conn.getresponse().read() assert(b'"error":null' in out1) #must also response with a correct json-rpc message - assert(conn.sock!=None) #according to http/1.1 connection must still be open! + assert(conn.sock is not None) #according to http/1.1 connection must still be open! conn.close() #now do the same with "Connection: close" @@ -65,7 +65,7 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() assert(b'"error":null' in out1) - assert(conn.sock==None) #now the connection must be closed after the response + assert(conn.sock is None) #now the connection must be closed after the response #node1 (2nd node) is running with disabled keep-alive option urlNode1 = urllib.parse.urlparse(self.nodes[1].url) @@ -88,7 +88,7 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read() assert(b'"error":null' in out1) - assert(conn.sock!=None) #connection must be closed because bitcoind should use keep-alive by default + assert(conn.sock is not None) #connection must be closed because bitcoind should use keep-alive by default # Check excessive request size conn = http.client.HTTPConnection(urlNode2.hostname, urlNode2.port) diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index bec6a0050a..442aa8e99a 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -6,6 +6,7 @@ from io import BytesIO import math + from test_framework.test_framework import BitcoinTestFramework from test_framework.messages import ( BIP125_SEQUENCE_NUMBER, @@ -71,6 +72,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): ))['hex'] txid_in_block = node.sendrawtransaction(hexstring=raw_tx_in_block, allowhighfees=True) node.generate(1) + self.mempool_size = 0 self.check_mempool_result( result_expected=[{'txid': txid_in_block, 'allowed': False, 'reject-reason': '18: txn-already-known'}], rawtxs=[raw_tx_in_block], @@ -90,9 +92,25 @@ class MempoolAcceptanceTest(BitcoinTestFramework): rawtxs=[raw_tx_0], ) + self.log.info('A final transaction not in the mempool') + coin = node.listunspent()[0] # Pick a random coin(base) to spend + raw_tx_final = node.signrawtransactionwithwallet(node.createrawtransaction( + inputs=[{'txid': coin['txid'], 'vout': coin['vout'], "sequence": 0xffffffff}], # SEQUENCE_FINAL + outputs=[{node.getnewaddress(): 0.025}], + locktime=node.getblockcount() + 2000, # Can be anything + ))['hex'] + tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx_final))) + self.check_mempool_result( + result_expected=[{'txid': tx.rehash(), 'allowed': True}], + rawtxs=[bytes_to_hex_str(tx.serialize())], + allowhighfees=True, + ) + node.sendrawtransaction(hexstring=raw_tx_final, allowhighfees=True) + self.mempool_size += 1 + self.log.info('A transaction in the mempool') node.sendrawtransaction(hexstring=raw_tx_0) - self.mempool_size = 1 + self.mempool_size += 1 self.check_mempool_result( result_expected=[{'txid': txid_0, 'allowed': False, 'reject-reason': '18: txn-already-in-mempool'}], rawtxs=[raw_tx_0], diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index b4e9d967fd..d74d4eaaf1 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -42,6 +42,7 @@ import time from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error, wait_until + class MempoolPersistTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 3 @@ -60,7 +61,7 @@ class MempoolPersistTest(BitcoinTestFramework): self.log.debug("Send 5 transactions from node2 (to its own address)") for i in range(5): - self.nodes[2].sendtoaddress(self.nodes[2].getnewaddress(), Decimal("10")) + last_txid = self.nodes[2].sendtoaddress(self.nodes[2].getnewaddress(), Decimal("10")) node2_balance = self.nodes[2].getbalance() self.sync_all() @@ -68,6 +69,13 @@ class MempoolPersistTest(BitcoinTestFramework): assert_equal(len(self.nodes[0].getrawmempool()), 5) assert_equal(len(self.nodes[1].getrawmempool()), 5) + self.log.debug("Prioritize a transaction on node0") + fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] + assert_equal(fees['base'], fees['modified']) + self.nodes[0].prioritisetransaction(txid=last_txid, fee_delta=1000) + fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] + assert_equal(fees['base'] + Decimal('0.00001000'), fees['modified']) + self.log.debug("Stop-start the nodes. Verify that node0 has the transactions in its mempool and node1 does not. Verify that node2 calculates its balance correctly after loading wallet transactions.") self.stop_nodes() # Give this node a head-start, so we can be "extra-sure" that it didn't load anything later @@ -81,6 +89,10 @@ class MempoolPersistTest(BitcoinTestFramework): # The others have loaded their mempool. If node_1 loaded anything, we'd probably notice by now: assert_equal(len(self.nodes[1].getrawmempool()), 0) + self.log.debug('Verify prioritization is loaded correctly') + fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] + assert_equal(fees['base'] + Decimal('0.00001000'), fees['modified']) + # Verify accounting of mempool transactions after restart is correct self.nodes[2].syncwithvalidationinterfacequeue() # Flush mempool to wallet assert_equal(node2_balance, self.nodes[2].getbalance()) diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index c5ddee56f1..da16bfbbfb 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -84,7 +84,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): high_fee_tx = x # Something high-fee should have been mined! - assert(high_fee_tx != None) + assert(high_fee_tx is not None) # Add a prioritisation before a tx is in the mempool (de-prioritising a # high-fee transaction so that it's now low fee). diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index c72cb8835c..356a45d6d0 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -450,6 +450,8 @@ class CTransaction: if flags != 0: self.wit.vtxinwit = [CTxInWitness() for i in range(len(self.vin))] self.wit.deserialize(f) + else: + self.wit = CTxWitness() self.nLockTime = struct.unpack("<I", f.read(4))[0] self.sha256 = None self.hash = None @@ -764,7 +766,7 @@ class HeaderAndShortIDs: self.prefilled_txn = [] self.use_witness = False - if p2pheaders_and_shortids != None: + if p2pheaders_and_shortids is not None: self.header = p2pheaders_and_shortids.header self.nonce = p2pheaders_and_shortids.nonce self.shortids = p2pheaders_and_shortids.shortids @@ -822,7 +824,7 @@ class BlockTransactionsRequest: def __init__(self, blockhash=0, indexes = None): self.blockhash = blockhash - self.indexes = indexes if indexes != None else [] + self.indexes = indexes if indexes is not None else [] def deserialize(self, f): self.blockhash = deser_uint256(f) @@ -863,7 +865,7 @@ class BlockTransactions: def __init__(self, blockhash=0, transactions = None): self.blockhash = blockhash - self.transactions = transactions if transactions != None else [] + self.transactions = transactions if transactions is not None else [] def deserialize(self, f): self.blockhash = deser_uint256(f) @@ -1052,7 +1054,7 @@ class msg_getdata: command = b"getdata" def __init__(self, inv=None): - self.inv = inv if inv != None else [] + self.inv = inv if inv is not None else [] def deserialize(self, f): self.inv = deser_vector(f, CInv) diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 2c5ba24a6a..012c80a1be 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -450,6 +450,10 @@ class CScript(bytes): # join makes no sense for a CScript() raise NotImplementedError + # Python 3.4 compatibility + def hex(self): + return hexlify(self).decode('ascii') + def __new__(cls, value=b''): if isinstance(value, bytes) or isinstance(value, bytearray): return super(CScript, cls).__new__(cls, value) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 10d5c659ad..21bf35597e 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -281,7 +281,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # Public helper methods. These can be accessed by the subclass test scripts. def add_nodes(self, num_nodes, extra_args=None, *, rpchost=None, binary=None): - """Instantiate TestNode objects""" + """Instantiate TestNode objects. + + Should only be called once after the nodes have been specified in + set_test_params().""" if self.bind_to_localhost_only: extra_confs = [["bind=127.0.0.1"]] * num_nodes else: @@ -294,7 +297,19 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): assert_equal(len(extra_args), num_nodes) assert_equal(len(binary), num_nodes) for i in range(num_nodes): - self.nodes.append(TestNode(i, get_datadir_path(self.options.tmpdir, i), rpchost=rpchost, timewait=self.rpc_timewait, bitcoind=binary[i], bitcoin_cli=self.options.bitcoincli, mocktime=self.mocktime, coverage_dir=self.options.coveragedir, extra_conf=extra_confs[i], extra_args=extra_args[i], use_cli=self.options.usecli)) + self.nodes.append(TestNode( + i, + get_datadir_path(self.options.tmpdir, i), + rpchost=rpchost, + timewait=self.rpc_timewait, + bitcoind=binary[i], + bitcoin_cli=self.options.bitcoincli, + mocktime=self.mocktime, + coverage_dir=self.options.coveragedir, + extra_conf=extra_confs[i], + extra_args=extra_args[i], + use_cli=self.options.usecli, + )) def start_node(self, i, *args, **kwargs): """Start a bitcoind""" diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index ebae3faffc..031a8824b1 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -68,7 +68,7 @@ class TestNode(): self.rpc_timeout = timewait self.binary = bitcoind self.coverage_dir = coverage_dir - if extra_conf != None: + if extra_conf is not None: append_config(datadir, extra_conf) # Most callers will just need to add extra args to the standard list below. # For those callers that need more flexibility, they can just set the args property directly. diff --git a/test/lint/check-doc.py b/test/lint/check-doc.py index b0d9f87958..4facd6c334 100755 --- a/test/lint/check-doc.py +++ b/test/lint/check-doc.py @@ -26,8 +26,12 @@ SET_DOC_OPTIONAL = set(['-h', '-help', '-dbcrashratio', '-forcecompactdb']) def main(): - used = check_output(CMD_GREP_ARGS, shell=True, universal_newlines=True, encoding='utf8') - docd = check_output(CMD_GREP_DOCS, shell=True, universal_newlines=True, encoding='utf8') + if sys.version_info >= (3, 6): + used = check_output(CMD_GREP_ARGS, shell=True, universal_newlines=True, encoding='utf8') + docd = check_output(CMD_GREP_DOCS, shell=True, universal_newlines=True, encoding='utf8') + else: + used = check_output(CMD_GREP_ARGS, shell=True, universal_newlines=True) # encoding='utf8' + docd = check_output(CMD_GREP_DOCS, shell=True, universal_newlines=True) # encoding='utf8' args_used = set(re.findall(re.compile(REGEX_ARG), used)) args_docd = set(re.findall(re.compile(REGEX_DOC), docd)).union(SET_DOC_OPTIONAL) diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh index d44a585294..3dbb9fff28 100755 --- a/test/lint/lint-python.sh +++ b/test/lint/lint-python.sh @@ -36,6 +36,7 @@ export LC_ALL=C # E701 multiple statements on one line (colon) # E702 multiple statements on one line (semicolon) # E703 statement ends with a semicolon +# E711 comparison to None should be 'if cond is None:' # E714 test for object identity should be "is not" # E721 do not compare types, use "isinstance()" # E741 do not use variables named "l", "O", or "I" @@ -87,4 +88,4 @@ elif PYTHONWARNINGS="ignore" flake8 --version | grep -q "Python 2"; then exit 0 fi -PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=E101,E112,E113,E115,E116,E125,E129,E131,E133,E223,E224,E242,E266,E271,E272,E273,E274,E275,E304,E306,E401,E402,E502,E701,E702,E703,E714,E721,E741,E742,E743,E901,E902,F401,F402,F403,F404,F405,F406,F407,F601,F602,F621,F622,F631,F701,F702,F703,F704,F705,F706,F707,F811,F812,F821,F822,F823,F831,F841,W191,W291,W292,W293,W504,W601,W602,W603,W604,W605,W606 "${@:-.}" +PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=E101,E112,E113,E115,E116,E125,E129,E131,E133,E223,E224,E242,E266,E271,E272,E273,E274,E275,E304,E306,E401,E402,E502,E701,E702,E703,E711,E714,E721,E741,E742,E743,E901,E902,F401,F402,F403,F404,F405,F406,F407,F601,F602,F621,F622,F631,F701,F702,F703,F704,F705,F706,F707,F811,F812,F821,F822,F823,F831,F841,W191,W291,W292,W293,W504,W601,W602,W603,W604,W605,W606 "${@:-.}" diff --git a/test/util/bitcoin-util-test.py b/test/util/bitcoin-util-test.py index 92fef30e13..7b1cc2b031 100755 --- a/test/util/bitcoin-util-test.py +++ b/test/util/bitcoin-util-test.py @@ -9,14 +9,9 @@ Runs automatically during `make check`. Can also be run manually.""" -from __future__ import division,print_function,unicode_literals - import argparse import binascii -try: - import configparser -except ImportError: - import ConfigParser as configparser +import configparser import difflib import json import logging |