diff options
94 files changed, 1490 insertions, 878 deletions
diff --git a/configure.ac b/configure.ac index 496a6bcf9f..3672700488 100644 --- a/configure.ac +++ b/configure.ac @@ -549,6 +549,8 @@ AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64],,, #include <byteswap.h> #endif]) +AC_CHECK_DECLS([__builtin_clz, __builtin_clzl, __builtin_clzll]) + dnl Check for MSG_NOSIGNAL AC_MSG_CHECKING(for MSG_NOSIGNAL) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/socket.h>]], diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index 3fee39143d..03ccf5b624 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2017 Bitcoin Core Developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,7 +15,7 @@ # In case of a clean merge that is accepted by the user, the local branch with # name $BRANCH is overwritten with the merged result, and optionally pushed. from __future__ import division,print_function,unicode_literals -import os +import os,sys from sys import stdin,stdout,stderr import argparse import hashlib @@ -127,6 +127,9 @@ def tree_sha512sum(commit='HEAD'): raise IOError('Non-zero return value executing git cat-file') return overall.hexdigest() +def print_merge_details(pull, title, branch, base_branch, head_branch): + print('%s#%s%s %s %sinto %s%s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title,ATTR_RESET+ATTR_PR,branch,ATTR_RESET)) + subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch]) def parse_arguments(): epilog = ''' @@ -171,7 +174,7 @@ def main(): info = retrieve_pr_info(repo,pull) if info is None: exit(1) - title = info['title'] + title = info['title'].strip() # precedence order for destination branch argument: # - command line argument # - githubmerge.branch setting @@ -256,8 +259,7 @@ def main(): printf("ERROR: Cannot update message.",file=stderr) exit(4) - print('%s#%s%s %s %sinto %s%s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title,ATTR_RESET+ATTR_PR,branch,ATTR_RESET)) - subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch]) + print_merge_details(pull, title, branch, base_branch, head_branch) print() # Run test command if configured. @@ -276,12 +278,6 @@ def main(): print("Difference with github ignored.",file=stderr) else: exit(6) - reply = ask_prompt("Press 'd' to accept the diff.") - if reply.lower() == 'd': - print("Diff accepted.",file=stderr) - else: - print("ERROR: Diff rejected.",file=stderr) - exit(6) else: # Verify the result manually. print("Dropping you on a shell so you can try building/testing the merged source.",file=stderr) @@ -290,12 +286,6 @@ def main(): if os.path.isfile('/etc/debian_version'): # Show pull number on Debian default prompt os.putenv('debian_chroot',pull) subprocess.call([BASH,'-i']) - reply = ask_prompt("Type 'm' to accept the merge.") - if reply.lower() == 'm': - print("Merge accepted.",file=stderr) - else: - print("ERROR: Merge rejected.",file=stderr) - exit(7) second_sha512 = tree_sha512sum() if first_sha512 != second_sha512: @@ -303,16 +293,19 @@ def main(): exit(8) # Sign the merge commit. - reply = ask_prompt("Type 's' to sign off on the merge.") - if reply == 's': - try: - subprocess.check_call([GIT,'commit','-q','--gpg-sign','--amend','--no-edit']) - except subprocess.CalledProcessError as e: - print("Error signing, exiting.",file=stderr) + print_merge_details(pull, title, branch, base_branch, head_branch) + while True: + reply = ask_prompt("Type 's' to sign off on the above merge, or 'x' to reject and exit.").lower() + if reply == 's': + try: + subprocess.check_call([GIT,'commit','-q','--gpg-sign','--amend','--no-edit']) + break + except subprocess.CalledProcessError as e: + print("Error signing, exiting.",file=stderr) + exit(1) + elif reply == 'x': + print("Not signing off on merge, exiting.",file=stderr) exit(1) - else: - print("Not signing off on merge, exiting.",file=stderr) - exit(1) # Put the result in branch. subprocess.check_call([GIT,'checkout','-q',branch]) @@ -326,9 +319,13 @@ def main(): subprocess.call([GIT,'branch','-q','-D',local_merge_branch],stderr=devnull) # Push the result. - reply = ask_prompt("Type 'push' to push the result to %s, branch %s." % (host_repo,branch)) - if reply.lower() == 'push': - subprocess.check_call([GIT,'push',host_repo,'refs/heads/'+branch]) + while True: + reply = ask_prompt("Type 'push' to push the result to %s, branch %s, or 'x' to exit without pushing." % (host_repo,branch)).lower() + if reply == 'push': + subprocess.check_call([GIT,'push',host_repo,'refs/heads/'+branch]) + break + elif reply == 'x': + exit(1) if __name__ == '__main__': main() diff --git a/contrib/macdeploy/macdeployqtplus b/contrib/macdeploy/macdeployqtplus index 5995f9f438..23a568ad13 100755 --- a/contrib/macdeploy/macdeployqtplus +++ b/contrib/macdeploy/macdeployqtplus @@ -302,7 +302,6 @@ def copyFramework(framework, path, verbose): if os.path.exists(fromContentsDir): toContentsDir = os.path.join(path, framework.destinationVersionContentsDirectory) shutil.copytree(fromContentsDir, toContentsDir, symlinks=True) - contentslinkfrom = os.path.join(path, framework.destinationContentsDirectory) if verbose >= 3: print("Copied Contents:", fromContentsDir) print(" to:", toContentsDir) @@ -675,9 +674,8 @@ else: if verbose >= 2: print("+ Installing qt.conf +") -f = open(os.path.join(applicationBundle.resourcesPath, "qt.conf"), "wb") -f.write(qt_conf.encode()) -f.close() +with open(os.path.join(applicationBundle.resourcesPath, "qt.conf"), "wb") as f: + f.write(qt_conf.encode()) # ------------------------------------------------ diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 797507cd3e..fd75ada79f 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -495,3 +495,76 @@ Git and GitHub tips This will add an `upstream-pull` remote to your git repository, which can be fetched using `git fetch --all` or `git fetch upstream-pull`. Afterwards, you can use `upstream-pull/NUMBER/head` in arguments to `git show`, `git checkout` and anywhere a commit id would be acceptable to see the changes from pull request NUMBER. + +RPC interface guidelines +-------------------------- + +A few guidelines for introducing and reviewing new RPC interfaces: + +- Method naming: use consecutive lower-case names such as `getrawtransaction` and `submitblock` + + - *Rationale*: Consistency with existing interface. + +- Argument naming: use snake case `fee_delta` (and not, e.g. camel case `feeDelta`) + + - *Rationale*: Consistency with existing interface. + +- Use the JSON parser for parsing, don't manually parse integers or strings from + arguments unless absolutely necessary. + + - *Rationale*: Introduces hand-rolled string manipulation code at both the caller and callee sites, + which is error prone, and it is easy to get things such as escaping wrong. + JSON already supports nested data structures, no need to re-invent the wheel. + + - *Exception*: AmountToValue can parse amounts as string. This was introduced because many JSON + parsers and formatters hard-code handling decimal numbers as floating point + values, resulting in potential loss of precision. This is unacceptable for + monetary values. **Always** use `AmountToValue` and `ValueToAmount` when + inputting or outputting monetary values. The only exceptions to this are + `prioritisetransaction` and `getblocktemplate` because their interface + is specified as-is in BIP22. + +- Missing arguments and 'null' should be treated the same: as default values. If there is no + default value, both cases should fail in the same way. + + - *Rationale*: Avoids surprises when switching to name-based arguments. Missing name-based arguments + are passed as 'null'. + + - *Exception*: Many legacy exceptions to this exist, one of the worst ones is + `getbalance` which follows a completely different code path based on the + number of arguments. We are still in the process of cleaning these up. Do not introduce + new ones. + +- Try not to overload methods on argument type. E.g. don't make `getblock(true)` and `getblock("hash")` + do different things. + + - *Rationale*: This is impossible to use with `bitcoin-cli`, and can be surprising to users. + + - *Exception*: Some RPC calls can take both an `int` and `bool`, most notably when a bool was switched + to a multi-value, or due to other historical reasons. **Always** have false map to 0 and + true to 1 in this case. + +- Don't forget to fill in the argument names correctly in the RPC command table. + + - *Rationale*: If not, the call can not be used with name-based arguments. + +- Set okSafeMode in the RPC command table to a sensible value: safe mode is when the + blockchain is regarded to be in a confused state, and the client deems it unsafe to + do anything irreversible such as send. Anything that just queries should be permitted. + + - *Rationale*: Troubleshooting a node in safe mode is difficult if half the + RPCs don't work. + +- Add every non-string RPC argument `(method, idx, name)` to the table `vRPCConvertParams` in `rpc/client.cpp`. + + - *Rationale*: `bitcoin-cli` and the GUI debug console use this table to determine how to + convert a plaintext command line to JSON. If the types don't match, the method can be unusable + from there. + +- A RPC method must either be a wallet method or a non-wallet method. Do not + introduce new methods such as `getinfo` and `signrawtransaction` that differ + in behavior based on presence of a wallet. + + - *Rationale*: as well as complicating the implementation and interfering + with the introduction of multi-wallet, wallet and non-wallet code should be + separated to avoid introducing circular dependencies between code units. diff --git a/src/Makefile.am b/src/Makefile.am index 96b9a27f7b..cb88171348 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -246,6 +246,8 @@ crypto_libbitcoin_crypto_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) crypto_libbitcoin_crypto_a_SOURCES = \ crypto/aes.cpp \ crypto/aes.h \ + crypto/chacha20.h \ + crypto/chacha20.cpp \ crypto/common.h \ crypto/hmac_sha256.cpp \ crypto/hmac_sha256.h \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index d08c8bde5b..10cb7e775a 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -57,8 +57,8 @@ BITCOIN_TESTS =\ test/policyestimator_tests.cpp \ test/pow_tests.cpp \ test/prevector_tests.cpp \ - test/random_tests.cpp \ test/raii_event_tests.cpp \ + test/random_tests.cpp \ test/reverselock_tests.cpp \ test/rpc_tests.cpp \ test/sanity_tests.cpp \ diff --git a/src/addrman.cpp b/src/addrman.cpp index ed9c128eb2..4a408b9beb 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -351,8 +351,8 @@ CAddrInfo CAddrMan::Select_(bool newOnly) int nKBucket = RandomInt(ADDRMAN_TRIED_BUCKET_COUNT); int nKBucketPos = RandomInt(ADDRMAN_BUCKET_SIZE); while (vvTried[nKBucket][nKBucketPos] == -1) { - nKBucket = (nKBucket + insecure_rand.rand32()) % ADDRMAN_TRIED_BUCKET_COUNT; - nKBucketPos = (nKBucketPos + insecure_rand.rand32()) % ADDRMAN_BUCKET_SIZE; + nKBucket = (nKBucket + insecure_rand.randbits(ADDRMAN_TRIED_BUCKET_COUNT_LOG2)) % ADDRMAN_TRIED_BUCKET_COUNT; + nKBucketPos = (nKBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; } int nId = vvTried[nKBucket][nKBucketPos]; assert(mapInfo.count(nId) == 1); @@ -368,8 +368,8 @@ CAddrInfo CAddrMan::Select_(bool newOnly) int nUBucket = RandomInt(ADDRMAN_NEW_BUCKET_COUNT); int nUBucketPos = RandomInt(ADDRMAN_BUCKET_SIZE); while (vvNew[nUBucket][nUBucketPos] == -1) { - nUBucket = (nUBucket + insecure_rand.rand32()) % ADDRMAN_NEW_BUCKET_COUNT; - nUBucketPos = (nUBucketPos + insecure_rand.rand32()) % ADDRMAN_BUCKET_SIZE; + nUBucket = (nUBucket + insecure_rand.randbits(ADDRMAN_NEW_BUCKET_COUNT_LOG2)) % ADDRMAN_NEW_BUCKET_COUNT; + nUBucketPos = (nUBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; } int nId = vvNew[nUBucket][nUBucketPos]; assert(mapInfo.count(nId) == 1); diff --git a/src/addrman.h b/src/addrman.h index f123b20b1b..70d907488f 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -136,13 +136,13 @@ public: */ //! total number of buckets for tried addresses -#define ADDRMAN_TRIED_BUCKET_COUNT 256 +#define ADDRMAN_TRIED_BUCKET_COUNT_LOG2 8 //! total number of buckets for new addresses -#define ADDRMAN_NEW_BUCKET_COUNT 1024 +#define ADDRMAN_NEW_BUCKET_COUNT_LOG2 10 //! maximum allowed number of entries in buckets for new and tried addresses -#define ADDRMAN_BUCKET_SIZE 64 +#define ADDRMAN_BUCKET_SIZE_LOG2 6 //! over how many buckets entries with tried addresses from a single group (/16 for IPv4) are spread #define ADDRMAN_TRIED_BUCKETS_PER_GROUP 8 @@ -171,6 +171,11 @@ public: //! the maximum number of nodes to return in a getaddr call #define ADDRMAN_GETADDR_MAX 2500 +//! Convenience +#define ADDRMAN_TRIED_BUCKET_COUNT (1 << ADDRMAN_TRIED_BUCKET_COUNT_LOG2) +#define ADDRMAN_NEW_BUCKET_COUNT (1 << ADDRMAN_NEW_BUCKET_COUNT_LOG2) +#define ADDRMAN_BUCKET_SIZE (1 << ADDRMAN_BUCKET_SIZE_LOG2) + /** * Stochastical (IP) address manager */ diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index 6fa9fe4fe8..88a2a570f9 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -68,7 +68,7 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::State& state) PrevectorJob(){ } PrevectorJob(FastRandomContext& insecure_rand){ - p.resize(insecure_rand.rand32() % (PREVECTOR_SIZE*2)); + p.resize(insecure_rand.randrange(PREVECTOR_SIZE*2)); } bool operator()() { diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index 737d3572ae..2914a36c7b 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -7,6 +7,7 @@ #include "bench.h" #include "bloom.h" #include "hash.h" +#include "random.h" #include "uint256.h" #include "utiltime.h" #include "crypto/ripemd160.h" @@ -69,6 +70,28 @@ static void SipHash_32b(benchmark::State& state) } } +static void FastRandom_32bit(benchmark::State& state) +{ + FastRandomContext rng(true); + uint32_t x = 0; + while (state.KeepRunning()) { + for (int i = 0; i < 1000000; i++) { + x += rng.rand32(); + } + } +} + +static void FastRandom_1bit(benchmark::State& state) +{ + FastRandomContext rng(true); + uint32_t x = 0; + while (state.KeepRunning()) { + for (int i = 0; i < 1000000; i++) { + x += rng.randbool(); + } + } +} + BENCHMARK(RIPEMD160); BENCHMARK(SHA1); BENCHMARK(SHA256); @@ -76,3 +99,5 @@ BENCHMARK(SHA512); BENCHMARK(SHA256_32b); BENCHMARK(SipHash_32b); +BENCHMARK(FastRandom_32bit); +BENCHMARK(FastRandom_1bit); diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 83b855cbcf..45738b5df8 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -657,11 +657,13 @@ static void MutateTx(CMutableTransaction& tx, const std::string& command, MutateTxDelOutput(tx, commandVal); else if (command == "outaddr") MutateTxAddOutAddr(tx, commandVal); - else if (command == "outpubkey") + else if (command == "outpubkey") { + if (!ecc) { ecc.reset(new Secp256k1Init()); } MutateTxAddOutPubKey(tx, commandVal); - else if (command == "outmultisig") + } else if (command == "outmultisig") { + if (!ecc) { ecc.reset(new Secp256k1Init()); } MutateTxAddOutMultiSig(tx, commandVal); - else if (command == "outscript") + } else if (command == "outscript") MutateTxAddOutScript(tx, commandVal); else if (command == "outdata") MutateTxAddOutData(tx, commandVal); diff --git a/src/chainparamsbase.cpp b/src/chainparamsbase.cpp index cb71a8b550..d013cc1450 100644 --- a/src/chainparamsbase.cpp +++ b/src/chainparamsbase.cpp @@ -103,8 +103,3 @@ std::string ChainNameFromCommandLine() return CBaseChainParams::TESTNET; return CBaseChainParams::MAIN; } - -bool AreBaseParamsConfigured() -{ - return pCurrentBaseParams != NULL; -} diff --git a/src/chainparamsbase.h b/src/chainparamsbase.h index 59493afb9b..84350cf65b 100644 --- a/src/chainparamsbase.h +++ b/src/chainparamsbase.h @@ -53,10 +53,4 @@ void SelectBaseParams(const std::string& chain); */ std::string ChainNameFromCommandLine(); -/** - * Return true if SelectBaseParamsFromCommandLine() has been called to select - * a network. - */ -bool AreBaseParamsConfigured(); - #endif // BITCOIN_CHAINPARAMSBASE_H diff --git a/src/coins.h b/src/coins.h index 0cb8519df8..065bae56e9 100644 --- a/src/coins.h +++ b/src/coins.h @@ -18,7 +18,7 @@ #include <stdint.h> #include <boost/foreach.hpp> -#include <boost/unordered_map.hpp> +#include <unordered_map> /** * Pruned version of CTransaction: only retains metadata and unspent transaction outputs @@ -280,7 +280,7 @@ struct CCoinsCacheEntry CCoinsCacheEntry() : coins(), flags(0) {} }; -typedef boost::unordered_map<uint256, CCoinsCacheEntry, SaltedTxidHasher> CCoinsMap; +typedef std::unordered_map<uint256, CCoinsCacheEntry, SaltedTxidHasher> CCoinsMap; /** Cursor for iterating over CoinsView state */ class CCoinsViewCursor diff --git a/src/core_write.cpp b/src/core_write.cpp index a3ca87c8b5..d116e617ee 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -151,6 +151,8 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry) entry.pushKV("txid", tx.GetHash().GetHex()); entry.pushKV("hash", tx.GetWitnessHash().GetHex()); entry.pushKV("version", tx.nVersion); + entry.pushKV("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION)); + entry.pushKV("vsize", (GetTransactionWeight(tx) + WITNESS_SCALE_FACTOR - 1) / WITNESS_SCALE_FACTOR); entry.pushKV("locktime", (int64_t)tx.nLockTime); UniValue vin(UniValue::VARR); diff --git a/src/crypto/chacha20.cpp b/src/crypto/chacha20.cpp new file mode 100644 index 0000000000..816ae870e1 --- /dev/null +++ b/src/crypto/chacha20.cpp @@ -0,0 +1,180 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +// Based on the public domain implementation 'merged' by D. J. Bernstein +// See https://cr.yp.to/chacha.html. + +#include "crypto/common.h" +#include "crypto/chacha20.h" + +#include <string.h> + +constexpr static inline uint32_t rotl32(uint32_t v, int c) { return (v << c) | (v >> (32 - c)); } + +#define QUARTERROUND(a,b,c,d) \ + a += b; d = rotl32(d ^ a, 16); \ + c += d; b = rotl32(b ^ c, 12); \ + a += b; d = rotl32(d ^ a, 8); \ + c += d; b = rotl32(b ^ c, 7); + +static const unsigned char sigma[] = "expand 32-byte k"; +static const unsigned char tau[] = "expand 16-byte k"; + +void ChaCha20::SetKey(const unsigned char* k, size_t keylen) +{ + const unsigned char *constants; + + input[4] = ReadLE32(k + 0); + input[5] = ReadLE32(k + 4); + input[6] = ReadLE32(k + 8); + input[7] = ReadLE32(k + 12); + if (keylen == 32) { /* recommended */ + k += 16; + constants = sigma; + } else { /* keylen == 16 */ + constants = tau; + } + input[8] = ReadLE32(k + 0); + input[9] = ReadLE32(k + 4); + input[10] = ReadLE32(k + 8); + input[11] = ReadLE32(k + 12); + input[0] = ReadLE32(constants + 0); + input[1] = ReadLE32(constants + 4); + input[2] = ReadLE32(constants + 8); + input[3] = ReadLE32(constants + 12); + input[12] = 0; + input[13] = 0; + input[14] = 0; + input[15] = 0; +} + +ChaCha20::ChaCha20() +{ + memset(input, 0, sizeof(input)); +} + +ChaCha20::ChaCha20(const unsigned char* k, size_t keylen) +{ + SetKey(k, keylen); +} + +void ChaCha20::SetIV(uint64_t iv) +{ + input[14] = iv; + input[15] = iv >> 32; +} + +void ChaCha20::Seek(uint64_t pos) +{ + input[12] = pos; + input[13] = pos >> 32; +} + +void ChaCha20::Output(unsigned char* c, size_t bytes) +{ + uint32_t x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15; + uint32_t j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15; + unsigned char *ctarget = NULL; + unsigned char tmp[64]; + unsigned int i; + + if (!bytes) return; + + j0 = input[0]; + j1 = input[1]; + j2 = input[2]; + j3 = input[3]; + j4 = input[4]; + j5 = input[5]; + j6 = input[6]; + j7 = input[7]; + j8 = input[8]; + j9 = input[9]; + j10 = input[10]; + j11 = input[11]; + j12 = input[12]; + j13 = input[13]; + j14 = input[14]; + j15 = input[15]; + + for (;;) { + if (bytes < 64) { + ctarget = c; + c = tmp; + } + x0 = j0; + x1 = j1; + x2 = j2; + x3 = j3; + x4 = j4; + x5 = j5; + x6 = j6; + x7 = j7; + x8 = j8; + x9 = j9; + x10 = j10; + x11 = j11; + x12 = j12; + x13 = j13; + x14 = j14; + x15 = j15; + for (i = 20;i > 0;i -= 2) { + QUARTERROUND( x0, x4, x8,x12) + QUARTERROUND( x1, x5, x9,x13) + QUARTERROUND( x2, x6,x10,x14) + QUARTERROUND( x3, x7,x11,x15) + QUARTERROUND( x0, x5,x10,x15) + QUARTERROUND( x1, x6,x11,x12) + QUARTERROUND( x2, x7, x8,x13) + QUARTERROUND( x3, x4, x9,x14) + } + x0 += j0; + x1 += j1; + x2 += j2; + x3 += j3; + x4 += j4; + x5 += j5; + x6 += j6; + x7 += j7; + x8 += j8; + x9 += j9; + x10 += j10; + x11 += j11; + x12 += j12; + x13 += j13; + x14 += j14; + x15 += j15; + + ++j12; + if (!j12) ++j13; + + WriteLE32(c + 0, x0); + WriteLE32(c + 4, x1); + WriteLE32(c + 8, x2); + WriteLE32(c + 12, x3); + WriteLE32(c + 16, x4); + WriteLE32(c + 20, x5); + WriteLE32(c + 24, x6); + WriteLE32(c + 28, x7); + WriteLE32(c + 32, x8); + WriteLE32(c + 36, x9); + WriteLE32(c + 40, x10); + WriteLE32(c + 44, x11); + WriteLE32(c + 48, x12); + WriteLE32(c + 52, x13); + WriteLE32(c + 56, x14); + WriteLE32(c + 60, x15); + + if (bytes <= 64) { + if (bytes < 64) { + for (i = 0;i < bytes;++i) ctarget[i] = c[i]; + } + input[12] = j12; + input[13] = j13; + return; + } + bytes -= 64; + c += 64; + } +} diff --git a/src/crypto/chacha20.h b/src/crypto/chacha20.h new file mode 100644 index 0000000000..a305977bcd --- /dev/null +++ b/src/crypto/chacha20.h @@ -0,0 +1,26 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CRYPTO_CHACHA20_H +#define BITCOIN_CRYPTO_CHACHA20_H + +#include <stdint.h> +#include <stdlib.h> + +/** A PRNG class for ChaCha20. */ +class ChaCha20 +{ +private: + uint32_t input[16]; + +public: + ChaCha20(); + ChaCha20(const unsigned char* key, size_t keylen); + void SetKey(const unsigned char* key, size_t keylen); + void SetIV(uint64_t iv); + void Seek(uint64_t pos); + void Output(unsigned char* output, size_t bytes); +}; + +#endif // BITCOIN_CRYPTO_CHACHA20_H diff --git a/src/crypto/common.h b/src/crypto/common.h index 4a9d1150b6..bcca3d30ea 100644 --- a/src/crypto/common.h +++ b/src/crypto/common.h @@ -79,4 +79,25 @@ void static inline WriteBE64(unsigned char* ptr, uint64_t x) memcpy(ptr, (char*)&v, 8); } +/** Return the smallest number n such that (x >> n) == 0 (or 64 if the highest bit in x is set. */ +uint64_t static inline CountBits(uint64_t x) +{ +#ifdef HAVE_DECL___BUILTIN_CLZL + if (sizeof(unsigned long) >= sizeof(uint64_t)) { + return x ? 8 * sizeof(unsigned long) - __builtin_clzl(x) : 0; + } +#endif +#ifdef HAVE_DECL___BUILTIN_CLZLL + if (sizeof(unsigned long long) >= sizeof(uint64_t)) { + return x ? 8 * sizeof(unsigned long long) - __builtin_clzll(x) : 0; + } +#endif + int ret = 0; + while (x) { + x >>= 1; + ++ret; + } + return ret; +} + #endif // BITCOIN_CRYPTO_COMMON_H diff --git a/src/init.cpp b/src/init.cpp index f06c9e1100..64f571f284 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -436,6 +436,8 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-dropmessagestest=<n>", "Randomly drop 1 of every <n> network messages"); strUsage += HelpMessageOpt("-fuzzmessagestest=<n>", "Randomly fuzz 1 of every <n> network messages"); strUsage += HelpMessageOpt("-stopafterblockimport", strprintf("Stop running after importing blocks from disk (default: %u)", DEFAULT_STOPAFTERBLOCKIMPORT)); + strUsage += HelpMessageOpt("-stopatheight", strprintf("Stop running after reaching the given height in the main chain (default: %u)", DEFAULT_STOPATHEIGHT)); + strUsage += HelpMessageOpt("-limitancestorcount=<n>", strprintf("Do not accept transactions if number of in-mempool ancestors is <n> or more (default: %u)", DEFAULT_ANCESTOR_LIMIT)); strUsage += HelpMessageOpt("-limitancestorsize=<n>", strprintf("Do not accept transactions whose size with all in-mempool ancestors exceeds <n> kilobytes (default: %u)", DEFAULT_ANCESTOR_SIZE_LIMIT)); strUsage += HelpMessageOpt("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT)); diff --git a/src/key.cpp b/src/key.cpp index b4f0dc8202..5a75647f1a 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -131,14 +131,6 @@ void CKey::MakeNewKey(bool fCompressedIn) { fCompressed = fCompressedIn; } -bool CKey::SetPrivKey(const CPrivKey &privkey, bool fCompressedIn) { - if (!ec_privkey_import_der(secp256k1_context_sign, (unsigned char*)begin(), &privkey[0], privkey.size())) - return false; - fCompressed = fCompressedIn; - fValid = true; - return true; -} - CPrivKey CKey::GetPrivKey() const { assert(fValid); CPrivKey privkey; @@ -94,9 +94,6 @@ public: //! Check whether the public key corresponding to this private key is (to be) compressed. bool IsCompressed() const { return fCompressed; } - //! Initialize from a CPrivKey (serialized OpenSSL private key data). - bool SetPrivKey(const CPrivKey& vchPrivKey, bool fCompressed); - //! Generate a new private key using a cryptographic PRNG. void MakeNewKey(bool fCompressed); diff --git a/src/memusage.h b/src/memusage.h index 81e8702954..b69acafffd 100644 --- a/src/memusage.h +++ b/src/memusage.h @@ -12,6 +12,8 @@ #include <map> #include <set> #include <vector> +#include <unordered_map> +#include <unordered_set> #include <boost/foreach.hpp> #include <boost/unordered_set.hpp> @@ -149,7 +151,7 @@ static inline size_t DynamicUsage(const std::shared_ptr<X>& p) // Boost data structures template<typename X> -struct boost_unordered_node : private X +struct unordered_node : private X { private: void* ptr; @@ -158,13 +160,25 @@ private: template<typename X, typename Y> static inline size_t DynamicUsage(const boost::unordered_set<X, Y>& s) { - return MallocUsage(sizeof(boost_unordered_node<X>)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count()); + return MallocUsage(sizeof(unordered_node<X>)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count()); } template<typename X, typename Y, typename Z> static inline size_t DynamicUsage(const boost::unordered_map<X, Y, Z>& m) { - return MallocUsage(sizeof(boost_unordered_node<std::pair<const X, Y> >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); + return MallocUsage(sizeof(unordered_node<std::pair<const X, Y> >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); +} + +template<typename X, typename Y> +static inline size_t DynamicUsage(const std::unordered_set<X, Y>& s) +{ + return MallocUsage(sizeof(unordered_node<X>)) * s.size() + MallocUsage(sizeof(void*) * s.bucket_count()); +} + +template<typename X, typename Y, typename Z> +static inline size_t DynamicUsage(const std::unordered_map<X, Y, Z>& m) +{ + return MallocUsage(sizeof(unordered_node<std::pair<const X, Y> >)) * m.size() + MallocUsage(sizeof(void*) * m.bucket_count()); } } diff --git a/src/merkleblock.cpp b/src/merkleblock.cpp index e3f3e4621a..78d7cd6001 100644 --- a/src/merkleblock.cpp +++ b/src/merkleblock.cpp @@ -65,7 +65,7 @@ uint256 CPartialMerkleTree::CalcHash(int height, unsigned int pos, const std::ve } else { // calculate left hash uint256 left = CalcHash(height-1, pos*2, vTxid), right; - // calculate right hash if not beyond the end of the array - copy left hash otherwise1 + // calculate right hash if not beyond the end of the array - copy left hash otherwise if (pos*2+1 < CalcTreeWidth(height-1)) right = CalcHash(height-1, pos*2+1, vTxid); else diff --git a/src/merkleblock.h b/src/merkleblock.h index 73cbf670ee..de4c5c8d29 100644 --- a/src/merkleblock.h +++ b/src/merkleblock.h @@ -23,7 +23,7 @@ * storing a bit for each traversed node, signifying whether the node is the * parent of at least one matched leaf txid (or a matched txid itself). In * case we are at the leaf level, or this bit is 0, its merkle node hash is - * stored, and its children are not explorer further. Otherwise, no hash is + * stored, and its children are not explored further. Otherwise, no hash is * stored, but we recurse into both (or the only) child branch. During * decoding, the same depth-first traversal is performed, consuming bits and * hashes as they written during encoding. diff --git a/src/miner.cpp b/src/miner.cpp index 386d75c4be..69a89bd617 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -317,9 +317,7 @@ int BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& already bool BlockAssembler::SkipMapTxEntry(CTxMemPool::txiter it, indexed_modified_transaction_set &mapModifiedTx, CTxMemPool::setEntries &failedTx) { assert (it != mempool.mapTx.end()); - if (mapModifiedTx.count(it) || inBlock.count(it) || failedTx.count(it)) - return true; - return false; + return mapModifiedTx.count(it) || inBlock.count(it) || failedTx.count(it); } void BlockAssembler::SortForBlock(const CTxMemPool::setEntries& package, CTxMemPool::txiter entry, std::vector<CTxMemPool::txiter>& sortedEntries) diff --git a/src/net.cpp b/src/net.cpp index 27389d6e0c..ed4c752606 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -418,10 +418,10 @@ void CConnman::DumpBanlist() CBanDB bandb; banmap_t banmap; - SetBannedSetDirty(false); GetBanned(banmap); - if (!bandb.Write(banmap)) - SetBannedSetDirty(true); + if (bandb.Write(banmap)) { + SetBannedSetDirty(false); + } LogPrint(BCLog::NET, "Flushed %d banned node ips/subnets to banlist.dat %dms\n", banmap.size(), GetTimeMillis() - nStart); @@ -541,6 +541,8 @@ bool CConnman::Unban(const CSubNet &subNet) { void CConnman::GetBanned(banmap_t &banMap) { LOCK(cs_setBanned); + // Sweep the banlist so expired bans are not returned + SweepBanned(); banMap = setBanned; //create a thread safe copy } @@ -708,7 +710,7 @@ bool CNode::ReceiveMsgBytes(const char *pch, unsigned int nBytes, bool& complete handled = msg.readData(pch, nBytes); if (handled < 0) - return false; + return false; if (msg.in_data && msg.hdr.nMessageSize > MAX_PROTOCOL_MESSAGE_LENGTH) { LogPrint(BCLog::NET, "Oversized message from peer=%i, disconnecting\n", GetId()); @@ -786,7 +788,7 @@ int CNetMessage::readHeader(const char *pch, unsigned int nBytes) // reject messages larger than MAX_SIZE if (hdr.nMessageSize > MAX_SIZE) - return -1; + return -1; // switch state to reading message data in_data = true; @@ -1299,59 +1301,55 @@ void CConnman::ThreadSocketHandler() } if (recvSet || errorSet) { + // typical socket buffer is 8K-64K + char pchBuf[0x10000]; + int nBytes = 0; { - { - // typical socket buffer is 8K-64K - char pchBuf[0x10000]; - int nBytes = 0; - { - LOCK(pnode->cs_hSocket); - if (pnode->hSocket == INVALID_SOCKET) - continue; - nBytes = recv(pnode->hSocket, pchBuf, sizeof(pchBuf), MSG_DONTWAIT); - } - if (nBytes > 0) - { - bool notify = false; - if (!pnode->ReceiveMsgBytes(pchBuf, nBytes, notify)) - pnode->CloseSocketDisconnect(); - RecordBytesRecv(nBytes); - if (notify) { - size_t nSizeAdded = 0; - auto it(pnode->vRecvMsg.begin()); - for (; it != pnode->vRecvMsg.end(); ++it) { - if (!it->complete()) - break; - nSizeAdded += it->vRecv.size() + CMessageHeader::HEADER_SIZE; - } - { - LOCK(pnode->cs_vProcessMsg); - pnode->vProcessMsg.splice(pnode->vProcessMsg.end(), pnode->vRecvMsg, pnode->vRecvMsg.begin(), it); - pnode->nProcessQueueSize += nSizeAdded; - pnode->fPauseRecv = pnode->nProcessQueueSize > nReceiveFloodSize; - } - WakeMessageHandler(); - } - } - else if (nBytes == 0) - { - // socket closed gracefully - if (!pnode->fDisconnect) { - LogPrint(BCLog::NET, "socket closed\n"); - } - pnode->CloseSocketDisconnect(); + LOCK(pnode->cs_hSocket); + if (pnode->hSocket == INVALID_SOCKET) + continue; + nBytes = recv(pnode->hSocket, pchBuf, sizeof(pchBuf), MSG_DONTWAIT); + } + if (nBytes > 0) + { + bool notify = false; + if (!pnode->ReceiveMsgBytes(pchBuf, nBytes, notify)) + pnode->CloseSocketDisconnect(); + RecordBytesRecv(nBytes); + if (notify) { + size_t nSizeAdded = 0; + auto it(pnode->vRecvMsg.begin()); + for (; it != pnode->vRecvMsg.end(); ++it) { + if (!it->complete()) + break; + nSizeAdded += it->vRecv.size() + CMessageHeader::HEADER_SIZE; } - else if (nBytes < 0) { - // error - int nErr = WSAGetLastError(); - if (nErr != WSAEWOULDBLOCK && nErr != WSAEMSGSIZE && nErr != WSAEINTR && nErr != WSAEINPROGRESS) - { - if (!pnode->fDisconnect) - LogPrintf("socket recv error %s\n", NetworkErrorString(nErr)); - pnode->CloseSocketDisconnect(); - } + LOCK(pnode->cs_vProcessMsg); + pnode->vProcessMsg.splice(pnode->vProcessMsg.end(), pnode->vRecvMsg, pnode->vRecvMsg.begin(), it); + pnode->nProcessQueueSize += nSizeAdded; + pnode->fPauseRecv = pnode->nProcessQueueSize > nReceiveFloodSize; } + WakeMessageHandler(); + } + } + else if (nBytes == 0) + { + // socket closed gracefully + if (!pnode->fDisconnect) { + LogPrint(BCLog::NET, "socket closed\n"); + } + pnode->CloseSocketDisconnect(); + } + else if (nBytes < 0) + { + // error + int nErr = WSAGetLastError(); + if (nErr != WSAEWOULDBLOCK && nErr != WSAEMSGSIZE && nErr != WSAEINTR && nErr != WSAEINPROGRESS) + { + if (!pnode->fDisconnect) + LogPrintf("socket recv error %s\n", NetworkErrorString(nErr)); + pnode->CloseSocketDisconnect(); } } } @@ -699,15 +699,15 @@ private: public: NodeId GetId() const { - return id; + return id; } uint64_t GetLocalNonce() const { - return nLocalHostNonce; + return nLocalHostNonce; } int GetMyStartingHeight() const { - return nMyStartingHeight; + return nMyStartingHeight; } int GetRefCount() @@ -758,7 +758,7 @@ public: // after addresses were pushed. if (_addr.IsValid() && !addrKnown.contains(_addr.GetKey())) { if (vAddrToSend.size() >= MAX_ADDR_TO_SEND) { - vAddrToSend[insecure_rand.rand32() % vAddrToSend.size()] = _addr; + vAddrToSend[insecure_rand.randrange(vAddrToSend.size())] = _addr; } else { vAddrToSend.push_back(_addr); } diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 2e7b99baa4..718a7de031 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2681,100 +2681,100 @@ bool ProcessMessages(CNode* pfrom, CConnman& connman, const std::atomic<bool>& i // this maintains the order of responses if (!pfrom->vRecvGetData.empty()) return true; - // Don't bother if send buffer is too full to respond anyway - if (pfrom->fPauseSend) - return false; + // Don't bother if send buffer is too full to respond anyway + if (pfrom->fPauseSend) + return false; - std::list<CNetMessage> msgs; - { - LOCK(pfrom->cs_vProcessMsg); - if (pfrom->vProcessMsg.empty()) - return false; - // Just take one message - msgs.splice(msgs.begin(), pfrom->vProcessMsg, pfrom->vProcessMsg.begin()); - pfrom->nProcessQueueSize -= msgs.front().vRecv.size() + CMessageHeader::HEADER_SIZE; - pfrom->fPauseRecv = pfrom->nProcessQueueSize > connman.GetReceiveFloodSize(); - fMoreWork = !pfrom->vProcessMsg.empty(); - } - CNetMessage& msg(msgs.front()); - - msg.SetVersion(pfrom->GetRecvVersion()); - // Scan for message start - if (memcmp(msg.hdr.pchMessageStart, chainparams.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { - LogPrintf("PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.hdr.GetCommand()), pfrom->id); - pfrom->fDisconnect = true; + std::list<CNetMessage> msgs; + { + LOCK(pfrom->cs_vProcessMsg); + if (pfrom->vProcessMsg.empty()) return false; - } + // Just take one message + msgs.splice(msgs.begin(), pfrom->vProcessMsg, pfrom->vProcessMsg.begin()); + pfrom->nProcessQueueSize -= msgs.front().vRecv.size() + CMessageHeader::HEADER_SIZE; + pfrom->fPauseRecv = pfrom->nProcessQueueSize > connman.GetReceiveFloodSize(); + fMoreWork = !pfrom->vProcessMsg.empty(); + } + CNetMessage& msg(msgs.front()); + + msg.SetVersion(pfrom->GetRecvVersion()); + // Scan for message start + if (memcmp(msg.hdr.pchMessageStart, chainparams.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { + LogPrintf("PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.hdr.GetCommand()), pfrom->id); + pfrom->fDisconnect = true; + return false; + } - // Read header - CMessageHeader& hdr = msg.hdr; - if (!hdr.IsValid(chainparams.MessageStart())) - { - LogPrintf("PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(hdr.GetCommand()), pfrom->id); - return fMoreWork; - } - std::string strCommand = hdr.GetCommand(); + // Read header + CMessageHeader& hdr = msg.hdr; + if (!hdr.IsValid(chainparams.MessageStart())) + { + LogPrintf("PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(hdr.GetCommand()), pfrom->id); + return fMoreWork; + } + std::string strCommand = hdr.GetCommand(); + + // Message size + unsigned int nMessageSize = hdr.nMessageSize; - // Message size - unsigned int nMessageSize = hdr.nMessageSize; + // Checksum + CDataStream& vRecv = msg.vRecv; + const uint256& hash = msg.GetMessageHash(); + if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) + { + LogPrintf("%s(%s, %u bytes): CHECKSUM ERROR expected %s was %s\n", __func__, + SanitizeString(strCommand), nMessageSize, + HexStr(hash.begin(), hash.begin()+CMessageHeader::CHECKSUM_SIZE), + HexStr(hdr.pchChecksum, hdr.pchChecksum+CMessageHeader::CHECKSUM_SIZE)); + return fMoreWork; + } - // Checksum - CDataStream& vRecv = msg.vRecv; - const uint256& hash = msg.GetMessageHash(); - if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) + // Process message + bool fRet = false; + try + { + fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.nTime, chainparams, connman, interruptMsgProc); + if (interruptMsgProc) + return false; + if (!pfrom->vRecvGetData.empty()) + fMoreWork = true; + } + catch (const std::ios_base::failure& e) + { + connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_MALFORMED, std::string("error parsing message"))); + if (strstr(e.what(), "end of data")) { - LogPrintf("%s(%s, %u bytes): CHECKSUM ERROR expected %s was %s\n", __func__, - SanitizeString(strCommand), nMessageSize, - HexStr(hash.begin(), hash.begin()+CMessageHeader::CHECKSUM_SIZE), - HexStr(hdr.pchChecksum, hdr.pchChecksum+CMessageHeader::CHECKSUM_SIZE)); - return fMoreWork; + // Allow exceptions from under-length message on vRecv + LogPrintf("%s(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } - - // Process message - bool fRet = false; - try + else if (strstr(e.what(), "size too large")) { - fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.nTime, chainparams, connman, interruptMsgProc); - if (interruptMsgProc) - return false; - if (!pfrom->vRecvGetData.empty()) - fMoreWork = true; + // Allow exceptions from over-long size + LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } - catch (const std::ios_base::failure& e) + else if (strstr(e.what(), "non-canonical ReadCompactSize()")) { - connman.PushMessage(pfrom, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::REJECT, strCommand, REJECT_MALFORMED, std::string("error parsing message"))); - if (strstr(e.what(), "end of data")) - { - // Allow exceptions from under-length message on vRecv - LogPrintf("%s(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } - else if (strstr(e.what(), "size too large")) - { - // Allow exceptions from over-long size - LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } - else if (strstr(e.what(), "non-canonical ReadCompactSize()")) - { - // Allow exceptions from non-canonical encoding - LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); - } - else - { - PrintExceptionContinue(&e, "ProcessMessages()"); - } + // Allow exceptions from non-canonical encoding + LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } - catch (const std::exception& e) { + else + { PrintExceptionContinue(&e, "ProcessMessages()"); - } catch (...) { - PrintExceptionContinue(NULL, "ProcessMessages()"); } + } + catch (const std::exception& e) { + PrintExceptionContinue(&e, "ProcessMessages()"); + } catch (...) { + PrintExceptionContinue(NULL, "ProcessMessages()"); + } - if (!fRet) { - LogPrintf("%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->id); - } + if (!fRet) { + LogPrintf("%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->id); + } - LOCK(cs_main); - SendRejectsAndCheckIfBanned(pfrom, connman); + LOCK(cs_main); + SendRejectsAndCheckIfBanned(pfrom, connman); return fMoreWork; } diff --git a/src/netaddress.cpp b/src/netaddress.cpp index ab07270f3f..34a7029862 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -179,12 +179,6 @@ bool CNetAddr::IsLocal() const return false; } -bool CNetAddr::IsMulticast() const -{ - return (IsIPv4() && (GetByte(3) & 0xF0) == 0xE0) - || (GetByte(15) == 0xFF); -} - bool CNetAddr::IsValid() const { // Cleanup 3-byte shifted addresses caused by garbage in size field diff --git a/src/netaddress.h b/src/netaddress.h index a85c2b7452..fbc4d1a65f 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -65,7 +65,6 @@ class CNetAddr bool IsLocal() const; bool IsRoutable() const; bool IsValid() const; - bool IsMulticast() const; enum Network GetNetwork() const; std::string ToString() const; std::string ToStringIP() const; diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index f3f7f8378e..bd169f875a 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -586,7 +586,7 @@ bool CBlockPolicyEstimator::Write(CAutoFile& fileout) const feeStats->Write(fileout); } catch (const std::exception&) { - LogPrintf("CBlockPolicyEstimator::Write(): unable to read policy estimator data (non-fatal)\n"); + LogPrintf("CBlockPolicyEstimator::Write(): unable to write policy estimator data (non-fatal)\n"); return false; } return true; diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index d413e8b087..5059030309 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -22,8 +22,8 @@ public: uint256 hash; uint32_t n; - COutPoint() { SetNull(); } - COutPoint(uint256 hashIn, uint32_t nIn) { hash = hashIn; n = nIn; } + COutPoint(): n((uint32_t) -1) { } + COutPoint(const uint256& hashIn, uint32_t nIn): hash(hashIn), n(nIn) { } ADD_SERIALIZE_METHODS; diff --git a/src/qt/bantablemodel.cpp b/src/qt/bantablemodel.cpp index 4b34e73eb7..f8a99506c1 100644 --- a/src/qt/bantablemodel.cpp +++ b/src/qt/bantablemodel.cpp @@ -181,7 +181,5 @@ void BanTableModel::sort(int column, Qt::SortOrder order) bool BanTableModel::shouldShow() { - if (priv->size() > 0) - return true; - return false; + return priv->size() > 0; } diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index bb8aa23de8..7f2f83d9f7 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -829,7 +829,7 @@ void RPCConsole::on_lineEdit_returnPressed() cmdBeforeBrowsing = QString(); - message(CMD_REQUEST, cmd); + message(CMD_REQUEST, QString::fromStdString(strFilteredCmd)); Q_EMIT cmdRequest(cmd); cmd = QString::fromStdString(strFilteredCmd); diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index f794b6b382..a0dce3d997 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -86,7 +86,8 @@ void WalletTests::walletTests() TestChain100Setup test; test.CreateAndProcessBlock({}, GetScriptForRawPubKey(test.coinbaseKey.GetPubKey())); bitdb.MakeMock(); - CWallet wallet("wallet_test.dat"); + std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, "wallet_test.dat")); + CWallet wallet(std::move(dbw)); bool firstRun; wallet.LoadWallet(firstRun); { diff --git a/src/random.cpp b/src/random.cpp index 6bcd0a70ba..6187f16290 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -240,22 +240,16 @@ uint256 GetRandHash() return hash; } -FastRandomContext::FastRandomContext(bool fDeterministic) +void FastRandomContext::RandomSeed() { - // The seed values have some unlikely fixed points which we avoid. - if (fDeterministic) { - Rz = Rw = 11; - } else { - uint32_t tmp; - do { - GetRandBytes((unsigned char*)&tmp, 4); - } while (tmp == 0 || tmp == 0x9068ffffU); - Rz = tmp; - do { - GetRandBytes((unsigned char*)&tmp, 4); - } while (tmp == 0 || tmp == 0x464fffffU); - Rw = tmp; - } + uint256 seed = GetRandHash(); + rng.SetKey(seed.begin(), 32); + requires_seed = false; +} + +FastRandomContext::FastRandomContext(const uint256& seed) : requires_seed(false), bytebuf_size(0), bitbuf_size(0) +{ + rng.SetKey(seed.begin(), 32); } bool Random_SanityCheck() @@ -288,3 +282,12 @@ bool Random_SanityCheck() } while (num_overwritten < NUM_OS_RANDOM_BYTES && tries < MAX_TRIES); return (num_overwritten == NUM_OS_RANDOM_BYTES); /* If this failed, bailed out after too many tries */ } + +FastRandomContext::FastRandomContext(bool fDeterministic) : requires_seed(!fDeterministic), bytebuf_size(0), bitbuf_size(0) +{ + if (!fDeterministic) { + return; + } + uint256 seed; + rng.SetKey(seed.begin(), 32); +} diff --git a/src/random.h b/src/random.h index 0464bdce14..9551e1c461 100644 --- a/src/random.h +++ b/src/random.h @@ -6,6 +6,8 @@ #ifndef BITCOIN_RANDOM_H #define BITCOIN_RANDOM_H +#include "crypto/chacha20.h" +#include "crypto/common.h" #include "uint256.h" #include <stdint.h> @@ -33,17 +35,79 @@ void GetStrongRandBytes(unsigned char* buf, int num); * This class is not thread-safe. */ class FastRandomContext { +private: + bool requires_seed; + ChaCha20 rng; + + unsigned char bytebuf[64]; + int bytebuf_size; + + uint64_t bitbuf; + int bitbuf_size; + + void RandomSeed(); + + void FillByteBuffer() + { + if (requires_seed) { + RandomSeed(); + } + rng.Output(bytebuf, sizeof(bytebuf)); + bytebuf_size = sizeof(bytebuf); + } + + void FillBitBuffer() + { + bitbuf = rand64(); + bitbuf_size = 64; + } + public: - explicit FastRandomContext(bool fDeterministic=false); + explicit FastRandomContext(bool fDeterministic = false); + + /** Initialize with explicit seed (only for testing) */ + explicit FastRandomContext(const uint256& seed); + + /** Generate a random 64-bit integer. */ + uint64_t rand64() + { + if (bytebuf_size < 8) FillByteBuffer(); + uint64_t ret = ReadLE64(bytebuf + 64 - bytebuf_size); + bytebuf_size -= 8; + return ret; + } - uint32_t rand32() { - Rz = 36969 * (Rz & 65535) + (Rz >> 16); - Rw = 18000 * (Rw & 65535) + (Rw >> 16); - return (Rw << 16) + Rz; + /** Generate a random (bits)-bit integer. */ + uint64_t randbits(int bits) { + if (bits == 0) { + return 0; + } else if (bits > 32) { + return rand64() >> (64 - bits); + } else { + if (bitbuf_size < bits) FillBitBuffer(); + uint64_t ret = bitbuf & (~(uint64_t)0 >> (64 - bits)); + bitbuf >>= bits; + bitbuf_size -= bits; + return ret; + } } - uint32_t Rz; - uint32_t Rw; + /** Generate a random integer in the range [0..range). */ + uint64_t randrange(uint64_t range) + { + --range; + int bits = CountBits(range); + while (true) { + uint64_t ret = randbits(bits); + if (ret <= range) return ret; + } + } + + /** Generate a random 32-bit integer. */ + uint32_t rand32() { return randbits(32); } + + /** Generate a random boolean. */ + bool randbool() { return randbits(1); } }; /* Number of random bytes returned by GetOSRand. diff --git a/src/rest.cpp b/src/rest.cpp index 9dcaf269d6..7537ed4502 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -5,6 +5,7 @@ #include "chain.h" #include "chainparams.h" +#include "core_io.h" #include "primitives/block.h" #include "primitives/transaction.h" #include "validation.h" @@ -56,10 +57,6 @@ struct CCoin { } }; -/* Defined in rawtransaction.cpp */ -void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); -void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); - static bool RESTERR(HTTPRequest* req, enum HTTPStatusCode status, std::string message) { req->WriteHeader("Content-Type", "text/plain"); @@ -383,7 +380,7 @@ static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) case RF_JSON: { UniValue objTx(UniValue::VOBJ); - TxToJSON(*tx, hashBlock, objTx); + TxToUniv(*tx, hashBlock, objTx); std::string strJSON = objTx.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); @@ -577,7 +574,7 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) // include the script in a json output UniValue o(UniValue::VOBJ); - ScriptPubKeyToJSON(coin.out.scriptPubKey, o, true); + ScriptPubKeyToUniv(coin.out.scriptPubKey, o, true); utxo.push_back(Pair("scriptPubKey", o)); utxos.push_back(utxo); } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 01066d0eb2..8f446aee65 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -12,6 +12,7 @@ #include "coins.h" #include "consensus/validation.h" #include "validation.h" +#include "core_io.h" #include "policy/policy.h" #include "primitives/transaction.h" #include "rpc/server.h" @@ -123,7 +124,7 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool tx if(txDetails) { UniValue objTx(UniValue::VOBJ); - TxToJSON(*tx, uint256(), objTx); + TxToUniv(*tx, uint256(), objTx); txs.push_back(objTx); } else @@ -975,7 +976,7 @@ UniValue gettxout(const JSONRPCRequest& request) ret.push_back(Pair("confirmations", pindex->nHeight - coins.nHeight + 1)); ret.push_back(Pair("value", ValueFromAmount(coins.vout[n].nValue))); UniValue o(UniValue::VOBJ); - ScriptPubKeyToJSON(coins.vout[n].scriptPubKey, o, true); + ScriptPubKeyToUniv(coins.vout[n].scriptPubKey, o, true); ret.push_back(Pair("scriptPubKey", o)); ret.push_back(Pair("version", coins.nVersion)); ret.push_back(Pair("coinbase", coins.fCoinBase)); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 717e9d75f3..3947fb3f7d 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -34,77 +34,15 @@ #include <univalue.h> -void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex) -{ - txnouttype type; - std::vector<CTxDestination> addresses; - int nRequired; - - out.push_back(Pair("asm", ScriptToAsmStr(scriptPubKey))); - if (fIncludeHex) - out.push_back(Pair("hex", HexStr(scriptPubKey.begin(), scriptPubKey.end()))); - - if (!ExtractDestinations(scriptPubKey, type, addresses, nRequired)) { - out.push_back(Pair("type", GetTxnOutputType(type))); - return; - } - - out.push_back(Pair("reqSigs", nRequired)); - out.push_back(Pair("type", GetTxnOutputType(type))); - - UniValue a(UniValue::VARR); - BOOST_FOREACH(const CTxDestination& addr, addresses) - a.push_back(CBitcoinAddress(addr).ToString()); - out.push_back(Pair("addresses", a)); -} void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) { - entry.push_back(Pair("txid", tx.GetHash().GetHex())); - entry.push_back(Pair("hash", tx.GetWitnessHash().GetHex())); - entry.push_back(Pair("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION))); - entry.push_back(Pair("vsize", (int)::GetVirtualTransactionSize(tx))); - entry.push_back(Pair("version", tx.nVersion)); - entry.push_back(Pair("locktime", (int64_t)tx.nLockTime)); - - UniValue vin(UniValue::VARR); - for (unsigned int i = 0; i < tx.vin.size(); i++) { - const CTxIn& txin = tx.vin[i]; - UniValue in(UniValue::VOBJ); - if (tx.IsCoinBase()) - in.push_back(Pair("coinbase", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()))); - else { - in.push_back(Pair("txid", txin.prevout.hash.GetHex())); - in.push_back(Pair("vout", (int64_t)txin.prevout.n)); - UniValue o(UniValue::VOBJ); - o.push_back(Pair("asm", ScriptToAsmStr(txin.scriptSig, true))); - o.push_back(Pair("hex", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()))); - in.push_back(Pair("scriptSig", o)); - } - if (tx.HasWitness()) { - UniValue txinwitness(UniValue::VARR); - for (unsigned int j = 0; j < tx.vin[i].scriptWitness.stack.size(); j++) { - std::vector<unsigned char> item = tx.vin[i].scriptWitness.stack[j]; - txinwitness.push_back(HexStr(item.begin(), item.end())); - } - in.push_back(Pair("txinwitness", txinwitness)); - } - in.push_back(Pair("sequence", (int64_t)txin.nSequence)); - vin.push_back(in); - } - entry.push_back(Pair("vin", vin)); - UniValue vout(UniValue::VARR); - for (unsigned int i = 0; i < tx.vout.size(); i++) { - const CTxOut& txout = tx.vout[i]; - UniValue out(UniValue::VOBJ); - out.push_back(Pair("value", ValueFromAmount(txout.nValue))); - out.push_back(Pair("n", (int64_t)i)); - UniValue o(UniValue::VOBJ); - ScriptPubKeyToJSON(txout.scriptPubKey, o, true); - out.push_back(Pair("scriptPubKey", o)); - vout.push_back(out); - } - entry.push_back(Pair("vout", vout)); + // Call into TxToUniv() in bitcoin-common to decode the transaction hex. + // + // Blockchain contextual information (confirmations and blocktime) is not + // available to code in bitcoin-common, so we query them here and push the + // data into the returned UniValue. + TxToUniv(tx, uint256(), entry); if (!hashBlock.IsNull()) { entry.push_back(Pair("blockhash", hashBlock.GetHex())); @@ -525,7 +463,7 @@ UniValue decoderawtransaction(const JSONRPCRequest& request) throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); UniValue result(UniValue::VOBJ); - TxToJSON(CTransaction(std::move(mtx)), uint256(), result); + TxToUniv(CTransaction(std::move(mtx)), uint256(), result); return result; } @@ -565,7 +503,7 @@ UniValue decodescript(const JSONRPCRequest& request) } else { // Empty scripts are valid } - ScriptPubKeyToJSON(script, r, false); + ScriptPubKeyToUniv(script, r, false); UniValue type; type = find_value(r, "type"); diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 8ecf0bbdac..f4e5313a78 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -247,10 +247,10 @@ bool EvalScript(std::vector<std::vector<unsigned char> >& stack, const CScript& { static const CScriptNum bnZero(0); static const CScriptNum bnOne(1); - static const CScriptNum bnFalse(0); - static const CScriptNum bnTrue(1); + // static const CScriptNum bnFalse(0); + // static const CScriptNum bnTrue(1); static const valtype vchFalse(0); - static const valtype vchZero(0); + // static const valtype vchZero(0); static const valtype vchTrue(1, 1); CScript::const_iterator pc = script.begin(); diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 5d1c5b78d1..39fa381dd0 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -203,10 +203,11 @@ BOOST_AUTO_TEST_CASE(addrman_select) BOOST_CHECK(addrman.size() == 7); // Test 12: Select pulls from new and tried regardless of port number. - BOOST_CHECK(addrman.Select().ToString() == "250.4.6.6:8333"); - BOOST_CHECK(addrman.Select().ToString() == "250.3.2.2:9999"); - BOOST_CHECK(addrman.Select().ToString() == "250.3.3.3:9999"); - BOOST_CHECK(addrman.Select().ToString() == "250.4.4.4:8333"); + std::set<uint16_t> ports; + for (int i = 0; i < 20; ++i) { + ports.insert(addrman.Select().GetPort()); + } + BOOST_CHECK_EQUAL(ports.size(), 3); } BOOST_AUTO_TEST_CASE(addrman_new_collisions) diff --git a/src/test/amount_tests.cpp b/src/test/amount_tests.cpp index fd6f88b366..c95def5e87 100644 --- a/src/test/amount_tests.cpp +++ b/src/test/amount_tests.cpp @@ -9,9 +9,16 @@ BOOST_FIXTURE_TEST_SUITE(amount_tests, BasicTestingSetup) +BOOST_AUTO_TEST_CASE(MoneyRangeTest) +{ + BOOST_CHECK_EQUAL(MoneyRange(CAmount(-1)), false); + BOOST_CHECK_EQUAL(MoneyRange(MAX_MONEY + CAmount(1)), false); + BOOST_CHECK_EQUAL(MoneyRange(CAmount(1)), true); +} + BOOST_AUTO_TEST_CASE(GetFeeTest) { - CFeeRate feeRate; + CFeeRate feeRate, altFeeRate; feeRate = CFeeRate(0); // Must always return 0 @@ -53,6 +60,11 @@ BOOST_AUTO_TEST_CASE(GetFeeTest) BOOST_CHECK_EQUAL(feeRate.GetFee(8), -1); // Special case: returns -1 instead of 0 BOOST_CHECK_EQUAL(feeRate.GetFee(9), -1); + // check alternate constructor + feeRate = CFeeRate(1000); + altFeeRate = CFeeRate(feeRate); + BOOST_CHECK_EQUAL(feeRate.GetFee(100), altFeeRate.GetFee(100)); + // Check full constructor // default value BOOST_CHECK(CFeeRate(CAmount(-1), 1000) == CFeeRate(-1)); @@ -68,4 +80,28 @@ BOOST_AUTO_TEST_CASE(GetFeeTest) CFeeRate(MAX_MONEY, std::numeric_limits<size_t>::max() >> 1).GetFeePerK(); } +BOOST_AUTO_TEST_CASE(BinaryOperatorTest) +{ + CFeeRate a, b; + a = CFeeRate(1); + b = CFeeRate(2); + BOOST_CHECK(a < b); + BOOST_CHECK(b > a); + BOOST_CHECK(a == a); + BOOST_CHECK(a <= b); + BOOST_CHECK(a <= a); + BOOST_CHECK(b >= a); + BOOST_CHECK(b >= b); + // a should be 0.00000002 BTC/kB now + a += a; + BOOST_CHECK(a == b); +} + +BOOST_AUTO_TEST_CASE(ToStringTest) +{ + CFeeRate feeRate; + feeRate = CFeeRate(1); + BOOST_CHECK_EQUAL(feeRate.ToString(), "0.00000001 BTC/kB"); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 4d17417179..72e562808a 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -3,12 +3,14 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "crypto/aes.h" +#include "crypto/chacha20.h" #include "crypto/ripemd160.h" #include "crypto/sha1.h" #include "crypto/sha256.h" #include "crypto/sha512.h" #include "crypto/hmac_sha256.h" #include "crypto/hmac_sha512.h" +#include "random.h" #include "utilstrencodings.h" #include "test/test_bitcoin.h" #include "test/test_random.h" @@ -187,6 +189,19 @@ void TestAES256CBC(const std::string &hexkey, const std::string &hexiv, bool pad } } +void TestChaCha20(const std::string &hexkey, uint64_t nonce, uint64_t seek, const std::string& hexout) +{ + std::vector<unsigned char> key = ParseHex(hexkey); + ChaCha20 rng(key.data(), key.size()); + rng.SetIV(nonce); + rng.Seek(seek); + std::vector<unsigned char> out = ParseHex(hexout); + std::vector<unsigned char> outres; + outres.resize(out.size()); + rng.Output(outres.data(), outres.size()); + BOOST_CHECK(out == outres); +} + std::string LongTestString(void) { std::string ret; for (int i=0; i<200000; i++) { @@ -439,4 +454,57 @@ BOOST_AUTO_TEST_CASE(aes_cbc_testvectors) { "b2eb05e2c39be9fcda6c19078c6a9d1b3f461796d6b0d6b2e0c2a72b4d80e644"); } + +BOOST_AUTO_TEST_CASE(chacha20_testvector) +{ + // Test vector from RFC 7539 + TestChaCha20("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 0x4a000000UL, 1, + "224f51f3401bd9e12fde276fb8631ded8c131f823d2c06e27e4fcaec9ef3cf788a3b0aa372600a92b57974cded2b9334794cb" + "a40c63e34cdea212c4cf07d41b769a6749f3f630f4122cafe28ec4dc47e26d4346d70b98c73f3e9c53ac40c5945398b6eda1a" + "832c89c167eacd901d7e2bf363"); + + // Test vectors from https://tools.ietf.org/html/draft-agl-tls-chacha20poly1305-04#section-7 + TestChaCha20("0000000000000000000000000000000000000000000000000000000000000000", 0, 0, + "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7da41597c5157488d7724e03fb8d84a376a43b" + "8f41518a11cc387b669b2ee6586"); + TestChaCha20("0000000000000000000000000000000000000000000000000000000000000001", 0, 0, + "4540f05a9f1fb296d7736e7b208e3c96eb4fe1834688d2604f450952ed432d41bbe2a0b6ea7566d2a5d1e7e20d42af2c53d79" + "2b1c43fea817e9ad275ae546963"); + TestChaCha20("0000000000000000000000000000000000000000000000000000000000000000", 0x0100000000000000ULL, 0, + "de9cba7bf3d69ef5e786dc63973f653a0b49e015adbff7134fcb7df137821031e85a050278a7084527214f73efc7fa5b52770" + "62eb7a0433e445f41e3"); + TestChaCha20("0000000000000000000000000000000000000000000000000000000000000000", 1, 0, + "ef3fdfd6c61578fbf5cf35bd3dd33b8009631634d21e42ac33960bd138e50d32111e4caf237ee53ca8ad6426194a88545ddc4" + "97a0b466e7d6bbdb0041b2f586b"); + TestChaCha20("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", 0x0706050403020100ULL, 0, + "f798a189f195e66982105ffb640bb7757f579da31602fc93ec01ac56f85ac3c134a4547b733b46413042c9440049176905d3b" + "e59ea1c53f15916155c2be8241a38008b9a26bc35941e2444177c8ade6689de95264986d95889fb60e84629c9bd9a5acb1cc1" + "18be563eb9b3a4a472f82e09a7e778492b562ef7130e88dfe031c79db9d4f7c7a899151b9a475032b63fc385245fe054e3dd5" + "a97a5f576fe064025d3ce042c566ab2c507b138db853e3d6959660996546cc9c4a6eafdc777c040d70eaf46f76dad3979e5c5" + "360c3317166a1c894c94a371876a94df7628fe4eaaf2ccb27d5aaae0ad7ad0f9d4b6ad3b54098746d4524d38407a6deb3ab78" + "fab78c9"); +} + +BOOST_AUTO_TEST_CASE(countbits_tests) +{ + FastRandomContext ctx; + for (int i = 0; i <= 64; ++i) { + if (i == 0) { + // Check handling of zero. + BOOST_CHECK_EQUAL(CountBits(0), 0); + } else if (i < 10) { + for (uint64_t j = 1 << (i - 1); (j >> i) == 0; ++j) { + // Exhaustively test up to 10 bits + BOOST_CHECK_EQUAL(CountBits(j), i); + } + } else { + for (int k = 0; k < 1000; k++) { + // Randomly test 1000 samples of each length above 10 bits. + uint64_t j = ((uint64_t)1) << (i - 1) | ctx.randbits(i - 1); + BOOST_CHECK_EQUAL(CountBits(j), i); + } + } + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index b9ed4952bb..0c7f3e5e23 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -11,6 +11,7 @@ #include "net.h" #include "netbase.h" #include "chainparams.h" +#include "util.h" class CAddrManSerializationMock : public CAddrMan { @@ -72,6 +73,18 @@ CDataStream AddrmanToStream(CAddrManSerializationMock& _addrman) BOOST_FIXTURE_TEST_SUITE(net_tests, BasicTestingSetup) +BOOST_AUTO_TEST_CASE(cnode_listen_port) +{ + // test default + unsigned short port = GetListenPort(); + BOOST_CHECK(port == Params().GetDefaultPort()); + // test set port + unsigned short altPort = 12345; + SoftSetArg("-port", std::to_string(altPort)); + port = GetListenPort(); + BOOST_CHECK(port == altPort); +} + BOOST_AUTO_TEST_CASE(caddrdb_read) { CAddrManUncorrupted addrmanUncorrupted; diff --git a/src/test/prevector_tests.cpp b/src/test/prevector_tests.cpp index bd8a7819a4..cfed5e347e 100644 --- a/src/test/prevector_tests.cpp +++ b/src/test/prevector_tests.cpp @@ -28,6 +28,7 @@ class prevector_tester { typedef typename pretype::size_type Size; bool passed = true; FastRandomContext rand_cache; + uint256 rand_seed; template <typename A, typename B> @@ -183,13 +184,12 @@ public: } ~prevector_tester() { - BOOST_CHECK_MESSAGE(passed, "insecure_rand_Rz: " - << rand_cache.Rz - << ", insecure_rand_Rw: " - << rand_cache.Rw); + BOOST_CHECK_MESSAGE(passed, "insecure_rand: " + rand_seed.ToString()); } + prevector_tester() { seed_insecure_rand(); + rand_seed = insecure_rand_seed; rand_cache = insecure_rand_ctx; } }; diff --git a/src/test/random_tests.cpp b/src/test/random_tests.cpp index d2c46c0daa..8596734226 100644 --- a/src/test/random_tests.cpp +++ b/src/test/random_tests.cpp @@ -15,5 +15,39 @@ BOOST_AUTO_TEST_CASE(osrandom_tests) BOOST_CHECK(Random_SanityCheck()); } -BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_CASE(fastrandom_tests) +{ + // Check that deterministic FastRandomContexts are deterministic + FastRandomContext ctx1(true); + FastRandomContext ctx2(true); + + BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); + BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); + BOOST_CHECK_EQUAL(ctx1.rand64(), ctx2.rand64()); + BOOST_CHECK_EQUAL(ctx1.randbits(3), ctx2.randbits(3)); + BOOST_CHECK_EQUAL(ctx1.randbits(7), ctx2.randbits(7)); + BOOST_CHECK_EQUAL(ctx1.rand32(), ctx2.rand32()); + BOOST_CHECK_EQUAL(ctx1.randbits(3), ctx2.randbits(3)); + + // Check that a nondeterministic ones are not + FastRandomContext ctx3; + FastRandomContext ctx4; + BOOST_CHECK(ctx3.rand64() != ctx4.rand64()); // extremely unlikely to be equal +} +BOOST_AUTO_TEST_CASE(fastrandom_randbits) +{ + FastRandomContext ctx1; + FastRandomContext ctx2; + for (int bits = 0; bits < 63; ++bits) { + for (int j = 0; j < 1000; ++j) { + uint64_t rangebits = ctx1.randbits(bits); + BOOST_CHECK_EQUAL(rangebits >> bits, 0); + uint64_t range = ((uint64_t)1) << bits | rangebits; + uint64_t rand = ctx2.randrange(range); + BOOST_CHECK(rand < range); + } + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index bda3819662..cb625bda11 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -27,7 +27,8 @@ #include <boost/thread.hpp> -FastRandomContext insecure_rand_ctx(true); +uint256 insecure_rand_seed = GetRandHash(); +FastRandomContext insecure_rand_ctx(insecure_rand_seed); extern bool fPrintToConsole; extern void noui_connect(); diff --git a/src/test/test_random.h b/src/test/test_random.h index 4a1637ac72..318c44df4d 100644 --- a/src/test/test_random.h +++ b/src/test/test_random.h @@ -8,11 +8,17 @@ #include "random.h" +extern uint256 insecure_rand_seed; extern FastRandomContext insecure_rand_ctx; static inline void seed_insecure_rand(bool fDeterministic = false) { - insecure_rand_ctx = FastRandomContext(fDeterministic); + if (fDeterministic) { + insecure_rand_seed = uint256(); + } else { + insecure_rand_seed = GetRandHash(); + } + insecure_rand_ctx = FastRandomContext(insecure_rand_seed); } static inline uint32_t insecure_rand(void) diff --git a/src/util.cpp b/src/util.cpp index 0dc203cba5..cf10ee4aa5 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -88,19 +88,6 @@ #include <openssl/rand.h> #include <openssl/conf.h> -// Work around clang compilation problem in Boost 1.46: -// /usr/include/boost/program_options/detail/config_file.hpp:163:17: error: call to function 'to_internal' that is neither visible in the template definition nor found by argument-dependent lookup -// See also: http://stackoverflow.com/questions/10020179/compilation-fail-in-boost-librairies-program-options -// http://clang.debian.net/status.php?version=3.0&key=CANNOT_FIND_FUNCTION -namespace boost { - - namespace program_options { - std::string to_internal(const std::string&); - } - -} // namespace boost - - const char * const BITCOIN_CONF_FILENAME = "bitcoin.conf"; const char * const BITCOIN_PID_FILENAME = "bitcoind.pid"; diff --git a/src/utiltime.cpp b/src/utiltime.cpp index a9936a645a..510f540b1d 100644 --- a/src/utiltime.cpp +++ b/src/utiltime.cpp @@ -9,14 +9,17 @@ #include "utiltime.h" +#include <atomic> + #include <boost/date_time/posix_time/posix_time.hpp> #include <boost/thread.hpp> -static int64_t nMockTime = 0; //!< For unit testing +static std::atomic<int64_t> nMockTime(0); //!< For unit testing int64_t GetTime() { - if (nMockTime) return nMockTime; + int64_t mocktime = nMockTime.load(std::memory_order_relaxed); + if (mocktime) return mocktime; time_t now = time(NULL); assert(now > 0); @@ -25,7 +28,7 @@ int64_t GetTime() void SetMockTime(int64_t nMockTimeIn) { - nMockTime = nMockTimeIn; + nMockTime.store(nMockTimeIn, std::memory_order_relaxed); } int64_t GetTimeMillis() @@ -52,7 +55,8 @@ int64_t GetSystemTimeInSeconds() /** Return a time useful for the debug log */ int64_t GetLogTimeMicros() { - if (nMockTime) return nMockTime*1000000; + int64_t mocktime = nMockTime.load(std::memory_order_relaxed); + if (mocktime) return mocktime*1000000; return GetTimeMicros(); } diff --git a/src/validation.cpp b/src/validation.cpp index 6c60be45a1..f189b741cd 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1299,10 +1299,7 @@ void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight) bool CScriptCheck::operator()() { const CScript &scriptSig = ptxTo->vin[nIn].scriptSig; const CScriptWitness *witness = &ptxTo->vin[nIn].scriptWitness; - if (!VerifyScript(scriptSig, scriptPubKey, witness, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, amount, cacheStore, *txdata), &error)) { - return false; - } - return true; + return VerifyScript(scriptSig, scriptPubKey, witness, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, amount, cacheStore, *txdata), &error); } int GetSpendHeight(const CCoinsViewCache& inputs) @@ -2537,6 +2534,9 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, return false; } + int nStopAtHeight = GetArg("-stopatheight", DEFAULT_STOPATHEIGHT); + if (nStopAtHeight && pindexNewTip && pindexNewTip->nHeight >= nStopAtHeight) StartShutdown(); + return true; } diff --git a/src/validation.h b/src/validation.h index c0f9b6d513..24ebf238df 100644 --- a/src/validation.h +++ b/src/validation.h @@ -145,6 +145,9 @@ static const int MAX_UNCONNECTING_HEADERS = 10; static const bool DEFAULT_PEERBLOOMFILTERS = true; +/** Default for -stopatheight */ +static const int DEFAULT_STOPATHEIGHT = 0; + struct BlockHasher { size_t operator()(const uint256& hash) const { return hash.GetCheapHash(); } diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index f47fc92b57..25f6bdd9d9 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -359,13 +359,16 @@ void CDBEnv::CheckpointLSN(const std::string& strFile) } -CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnCloseIn) : pdb(NULL), activeTxn(NULL) +CDB::CDB(CWalletDBWrapper& dbw, const char* pszMode, bool fFlushOnCloseIn) : pdb(NULL), activeTxn(NULL) { int ret; fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w')); fFlushOnClose = fFlushOnCloseIn; - if (strFilename.empty()) + env = dbw.env; + if (dbw.IsDummy()) { return; + } + const std::string &strFilename = dbw.strFile; bool fCreate = strchr(pszMode, 'c') != NULL; unsigned int nFlags = DB_THREAD; @@ -373,17 +376,17 @@ CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnClose nFlags |= DB_CREATE; { - LOCK(bitdb.cs_db); - if (!bitdb.Open(GetDataDir())) + LOCK(env->cs_db); + if (!env->Open(GetDataDir())) throw std::runtime_error("CDB: Failed to open database environment."); strFile = strFilename; - ++bitdb.mapFileUseCount[strFile]; - pdb = bitdb.mapDb[strFile]; + ++env->mapFileUseCount[strFile]; + pdb = env->mapDb[strFile]; if (pdb == NULL) { - pdb = new Db(bitdb.dbenv, 0); + pdb = new Db(env->dbenv, 0); - bool fMockDb = bitdb.IsMock(); + bool fMockDb = env->IsMock(); if (fMockDb) { DbMpoolFile* mpf = pdb->get_mpf(); ret = mpf->set_flags(DB_MPOOL_NOFILE, 1); @@ -401,7 +404,7 @@ CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnClose if (ret != 0) { delete pdb; pdb = NULL; - --bitdb.mapFileUseCount[strFile]; + --env->mapFileUseCount[strFile]; strFile = ""; throw std::runtime_error(strprintf("CDB: Error %d, can't open database %s", ret, strFilename)); } @@ -413,7 +416,7 @@ CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnClose fReadOnly = fTmp; } - bitdb.mapDb[strFile] = pdb; + env->mapDb[strFile] = pdb; } } } @@ -428,7 +431,7 @@ void CDB::Flush() if (fReadOnly) nMinutes = 1; - bitdb.dbenv->txn_checkpoint(nMinutes ? GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); + env->dbenv->txn_checkpoint(nMinutes ? GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); } void CDB::Close() @@ -444,8 +447,8 @@ void CDB::Close() Flush(); { - LOCK(bitdb.cs_db); - --bitdb.mapFileUseCount[strFile]; + LOCK(env->cs_db); + --env->mapFileUseCount[strFile]; } } @@ -463,32 +466,28 @@ void CDBEnv::CloseDb(const std::string& strFile) } } -bool CDBEnv::RemoveDb(const std::string& strFile) -{ - this->CloseDb(strFile); - - LOCK(cs_db); - int rc = dbenv->dbremove(NULL, strFile.c_str(), NULL, DB_AUTO_COMMIT); - return (rc == 0); -} - -bool CDB::Rewrite(const std::string& strFile, const char* pszSkip) +bool CDB::Rewrite(CWalletDBWrapper& dbw, const char* pszSkip) { + if (dbw.IsDummy()) { + return true; + } + CDBEnv *env = dbw.env; + const std::string& strFile = dbw.strFile; while (true) { { - LOCK(bitdb.cs_db); - if (!bitdb.mapFileUseCount.count(strFile) || bitdb.mapFileUseCount[strFile] == 0) { + LOCK(env->cs_db); + if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) { // Flush log data to the dat file - bitdb.CloseDb(strFile); - bitdb.CheckpointLSN(strFile); - bitdb.mapFileUseCount.erase(strFile); + env->CloseDb(strFile); + env->CheckpointLSN(strFile); + env->mapFileUseCount.erase(strFile); bool fSuccess = true; LogPrintf("CDB::Rewrite: Rewriting %s...\n", strFile); std::string strFileRes = strFile + ".rewrite"; { // surround usage of db with extra {} - CDB db(strFile.c_str(), "r"); - Db* pdbCopy = new Db(bitdb.dbenv, 0); + CDB db(dbw, "r"); + Db* pdbCopy = new Db(env->dbenv, 0); int ret = pdbCopy->open(NULL, // Txn pointer strFileRes.c_str(), // Filename @@ -531,17 +530,17 @@ bool CDB::Rewrite(const std::string& strFile, const char* pszSkip) } if (fSuccess) { db.Close(); - bitdb.CloseDb(strFile); + env->CloseDb(strFile); if (pdbCopy->close(0)) fSuccess = false; delete pdbCopy; } } if (fSuccess) { - Db dbA(bitdb.dbenv, 0); + Db dbA(env->dbenv, 0); if (dbA.remove(strFile.c_str(), NULL, 0)) fSuccess = false; - Db dbB(bitdb.dbenv, 0); + Db dbB(env->dbenv, 0); if (dbB.rename(strFileRes.c_str(), NULL, strFile.c_str(), 0)) fSuccess = false; } @@ -596,16 +595,21 @@ void CDBEnv::Flush(bool fShutdown) } } -bool CDB::PeriodicFlush(std::string strFile) +bool CDB::PeriodicFlush(CWalletDBWrapper& dbw) { + if (dbw.IsDummy()) { + return true; + } bool ret = false; + CDBEnv *env = dbw.env; + const std::string& strFile = dbw.strFile; TRY_LOCK(bitdb.cs_db,lockDb); if (lockDb) { // Don't do this if any databases are in use int nRefCount = 0; - std::map<std::string, int>::iterator mit = bitdb.mapFileUseCount.begin(); - while (mit != bitdb.mapFileUseCount.end()) + std::map<std::string, int>::iterator mit = env->mapFileUseCount.begin(); + while (mit != env->mapFileUseCount.end()) { nRefCount += (*mit).second; mit++; @@ -614,17 +618,17 @@ bool CDB::PeriodicFlush(std::string strFile) if (nRefCount == 0) { boost::this_thread::interruption_point(); - std::map<std::string, int>::iterator mi = bitdb.mapFileUseCount.find(strFile); - if (mi != bitdb.mapFileUseCount.end()) + std::map<std::string, int>::iterator mi = env->mapFileUseCount.find(strFile); + if (mi != env->mapFileUseCount.end()) { LogPrint(BCLog::DB, "Flushing %s\n", strFile); int64_t nStart = GetTimeMillis(); // Flush wallet file so it's self contained - bitdb.CloseDb(strFile); - bitdb.CheckpointLSN(strFile); + env->CloseDb(strFile); + env->CheckpointLSN(strFile); - bitdb.mapFileUseCount.erase(mi++); + env->mapFileUseCount.erase(mi++); LogPrint(BCLog::DB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); ret = true; } @@ -633,3 +637,52 @@ bool CDB::PeriodicFlush(std::string strFile) return ret; } + +bool CWalletDBWrapper::Rewrite(const char* pszSkip) +{ + return CDB::Rewrite(*this, pszSkip); +} + +bool CWalletDBWrapper::Backup(const std::string& strDest) +{ + if (IsDummy()) { + return false; + } + while (true) + { + { + LOCK(env->cs_db); + if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) + { + // Flush log data to the dat file + env->CloseDb(strFile); + env->CheckpointLSN(strFile); + env->mapFileUseCount.erase(strFile); + + // Copy wallet file + fs::path pathSrc = GetDataDir() / strFile; + fs::path pathDest(strDest); + if (fs::is_directory(pathDest)) + pathDest /= strFile; + + try { + fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); + LogPrintf("copied %s to %s\n", strFile, pathDest.string()); + return true; + } catch (const fs::filesystem_error& e) { + LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), e.what()); + return false; + } + } + } + MilliSleep(100); + } + return false; +} + +void CWalletDBWrapper::Flush(bool shutdown) +{ + if (!IsDummy()) { + env->Flush(shutdown); + } +} diff --git a/src/wallet/db.h b/src/wallet/db.h index 9f912f9a1a..1a46448cc7 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -72,7 +72,6 @@ public: void CheckpointLSN(const std::string& strFile); void CloseDb(const std::string& strFile); - bool RemoveDb(const std::string& strFile); DbTxn* TxnBegin(int flags = DB_TXN_WRITE_NOSYNC) { @@ -86,6 +85,52 @@ public: extern CDBEnv bitdb; +/** An instance of this class represents one database. + * For BerkeleyDB this is just a (env, strFile) tuple. + **/ +class CWalletDBWrapper +{ + friend class CDB; +public: + /** Create dummy DB handle */ + CWalletDBWrapper(): env(nullptr) + { + } + + /** Create DB handle to real database */ + CWalletDBWrapper(CDBEnv *env_in, const std::string &strFile_in): + env(env_in), strFile(strFile_in) + { + } + + /** Rewrite the entire database on disk, with the exception of key pszSkip if non-zero + */ + bool Rewrite(const char* pszSkip=nullptr); + + /** Back up the entire database to a file. + */ + bool Backup(const std::string& strDest); + + /** Get a name for this database, for debugging etc. + */ + std::string GetName() const { return strFile; } + + /** Make sure all changes are flushed to disk. + */ + void Flush(bool shutdown); + +private: + /** BerkeleyDB specific */ + CDBEnv *env; + std::string strFile; + + /** Return whether this database handle is a dummy for testing. + * Only to be used at a low level, application should ideally not care + * about this. + */ + bool IsDummy() { return env == nullptr; } +}; + /** RAII class that provides access to a Berkeley database */ class CDB @@ -96,18 +141,19 @@ protected: DbTxn* activeTxn; bool fReadOnly; bool fFlushOnClose; + CDBEnv *env; - explicit CDB(const std::string& strFilename, const char* pszMode = "r+", bool fFlushOnCloseIn=true); +public: + explicit CDB(CWalletDBWrapper& dbw, const char* pszMode = "r+", bool fFlushOnCloseIn=true); ~CDB() { Close(); } -public: void Flush(); void Close(); static bool Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue)); /* flush the wallet passively (TRY_LOCK) ideal to be called periodically */ - static bool PeriodicFlush(std::string strFile); + static bool PeriodicFlush(CWalletDBWrapper& dbw); /* verifies the database environment */ static bool VerifyEnvironment(const std::string& walletFile, const fs::path& dataDir, std::string& errorStr); /* verifies the database file */ @@ -117,7 +163,7 @@ private: CDB(const CDB&); void operator=(const CDB&); -protected: +public: template <typename K, typename T> bool Read(const K& key, T& value) { @@ -156,7 +202,7 @@ protected: bool Write(const K& key, const T& value, bool fOverwrite = true) { if (!pdb) - return false; + return true; if (fReadOnly) assert(!"Write called on database in read-only mode"); @@ -310,7 +356,7 @@ public: return Write(std::string("version"), nVersion); } - bool static Rewrite(const std::string& strFile, const char* pszSkip = NULL); + bool static Rewrite(CWalletDBWrapper& dbw, const char* pszSkip = NULL); }; #endif // BITCOIN_WALLET_DB_H diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 82e5974065..f5b63c1ecd 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -160,7 +160,7 @@ CFeeBumper::CFeeBumper(const CWallet *pWallet, const uint256 txidIn, int newConf } else { // if user specified a confirm target then don't consider any global payTxFee if (specifiedConfirmTarget) { - nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool, ::feeEstimator, CAmount(0)); + nNewFee = CWallet::GetMinimumFee(maxNewTxSize, newConfirmTarget, mempool, ::feeEstimator, true); } // otherwise use the regular wallet logic to select payTxFee or default confirm target else { diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 5cbeeab4eb..d1e7485d04 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -729,6 +729,8 @@ UniValue getbalance(const JSONRPCRequest& request) if (request.params.size() == 0) return ValueFromAmount(pwallet->GetBalance()); + const std::string* account = request.params[0].get_str() != "*" ? &request.params[0].get_str() : nullptr; + int nMinDepth = 1; if (request.params.size() > 1) nMinDepth = request.params[1].get_int(); @@ -737,41 +739,7 @@ UniValue getbalance(const JSONRPCRequest& request) if(request.params[2].get_bool()) filter = filter | ISMINE_WATCH_ONLY; - if (request.params[0].get_str() == "*") { - // Calculate total balance in a very different way from GetBalance(). - // The biggest difference is that GetBalance() sums up all unspent - // TxOuts paying to the wallet, while this sums up both spent and - // unspent TxOuts paying to the wallet, and then subtracts the values of - // TxIns spending from the wallet. This also has fewer restrictions on - // which unconfirmed transactions are considered trusted. - CAmount nBalance = 0; - for (const std::pair<uint256, CWalletTx>& pairWtx : pwallet->mapWallet) { - const CWalletTx& wtx = pairWtx.second; - if (!CheckFinalTx(wtx) || wtx.GetBlocksToMaturity() > 0 || wtx.GetDepthInMainChain() < 0) - continue; - - CAmount allFee; - std::string strSentAccount; - std::list<COutputEntry> listReceived; - std::list<COutputEntry> listSent; - wtx.GetAmounts(listReceived, listSent, allFee, strSentAccount, filter); - if (wtx.GetDepthInMainChain() >= nMinDepth) - { - BOOST_FOREACH(const COutputEntry& r, listReceived) - nBalance += r.amount; - } - BOOST_FOREACH(const COutputEntry& s, listSent) - nBalance -= s.amount; - nBalance -= allFee; - } - return ValueFromAmount(nBalance); - } - - std::string strAccount = AccountFromValue(request.params[0]); - - CAmount nBalance = pwallet->GetAccountBalance(strAccount, nMinDepth, filter); - - return ValueFromAmount(nBalance); + return ValueFromAmount(pwallet->GetLegacyBalance(filter, nMinDepth, account)); } UniValue getunconfirmedbalance(const JSONRPCRequest &request) @@ -901,7 +869,7 @@ UniValue sendfrom(const JSONRPCRequest& request) EnsureWalletIsUnlocked(pwallet); // Check funds - CAmount nBalance = pwallet->GetAccountBalance(strAccount, nMinDepth, ISMINE_SPENDABLE); + CAmount nBalance = pwallet->GetLegacyBalance(ISMINE_SPENDABLE, nMinDepth, &strAccount); if (nAmount > nBalance) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Account has insufficient funds"); @@ -1010,7 +978,7 @@ UniValue sendmany(const JSONRPCRequest& request) EnsureWalletIsUnlocked(pwallet); // Check funds - CAmount nBalance = pwallet->GetAccountBalance(strAccount, nMinDepth, ISMINE_SPENDABLE); + CAmount nBalance = pwallet->GetLegacyBalance(ISMINE_SPENDABLE, nMinDepth, &strAccount); if (totalAmount > nBalance) throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, "Account has insufficient funds"); @@ -2078,7 +2046,7 @@ UniValue walletpassphrase(const JSONRPCRequest& request) int64_t nSleepTime = request.params[1].get_int64(); pwallet->nRelockTime = GetTime() + nSleepTime; - RPCRunLater(strprintf("lockwallet(%s)", pwallet->strWalletFile), boost::bind(LockWallet, pwallet), nSleepTime); + RPCRunLater(strprintf("lockwallet(%s)", pwallet->GetName()), boost::bind(LockWallet, pwallet), nSleepTime); return NullUniValue; } diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index a76db37617..1989bf8d9b 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -14,7 +14,8 @@ WalletTestingSetup::WalletTestingSetup(const std::string& chainName): bitdb.MakeMock(); bool fFirstRun; - pwalletMain = new CWallet("wallet_test.dat"); + std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, "wallet_test.dat")); + pwalletMain = new CWallet(std::move(dbw)); pwalletMain->LoadWallet(fFirstRun); RegisterValidationInterface(pwalletMain); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 33dd30a795..a8f818a494 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -162,7 +162,7 @@ void CWallet::DeriveNewChildKey(CKeyMetadata& metadata, CKey& secret, bool inter secret = childKey.key; metadata.hdMasterKeyID = hdChain.masterKeyID; // update the chain model in the database - if (!CWalletDB(strWalletFile).WriteHDChain(hdChain)) + if (!CWalletDB(*dbw).WriteHDChain(hdChain)) throw std::runtime_error(std::string(__func__) + ": Writing HD chain model failed"); } @@ -181,10 +181,8 @@ bool CWallet::AddKeyPubKey(const CKey& secret, const CPubKey &pubkey) if (HaveWatchOnly(script)) RemoveWatchOnly(script); - if (!fFileBacked) - return true; if (!IsCrypted()) { - return CWalletDB(strWalletFile).WriteKey(pubkey, + return CWalletDB(*dbw).WriteKey(pubkey, secret.GetPrivKey(), mapKeyMetadata[pubkey.GetID()]); } @@ -196,8 +194,6 @@ bool CWallet::AddCryptedKey(const CPubKey &vchPubKey, { if (!CCryptoKeyStore::AddCryptedKey(vchPubKey, vchCryptedSecret)) return false; - if (!fFileBacked) - return true; { LOCK(cs_wallet); if (pwalletdbEncryption) @@ -205,7 +201,7 @@ bool CWallet::AddCryptedKey(const CPubKey &vchPubKey, vchCryptedSecret, mapKeyMetadata[vchPubKey.GetID()]); else - return CWalletDB(strWalletFile).WriteCryptedKey(vchPubKey, + return CWalletDB(*dbw).WriteCryptedKey(vchPubKey, vchCryptedSecret, mapKeyMetadata[vchPubKey.GetID()]); } @@ -241,9 +237,7 @@ bool CWallet::AddCScript(const CScript& redeemScript) { if (!CCryptoKeyStore::AddCScript(redeemScript)) return false; - if (!fFileBacked) - return true; - return CWalletDB(strWalletFile).WriteCScript(Hash160(redeemScript), redeemScript); + return CWalletDB(*dbw).WriteCScript(Hash160(redeemScript), redeemScript); } bool CWallet::LoadCScript(const CScript& redeemScript) @@ -269,9 +263,7 @@ bool CWallet::AddWatchOnly(const CScript& dest) const CKeyMetadata& meta = mapKeyMetadata[CScriptID(dest)]; UpdateTimeFirstKey(meta.nCreateTime); NotifyWatchonlyChanged(true); - if (!fFileBacked) - return true; - return CWalletDB(strWalletFile).WriteWatchOnly(dest, meta); + return CWalletDB(*dbw).WriteWatchOnly(dest, meta); } bool CWallet::AddWatchOnly(const CScript& dest, int64_t nCreateTime) @@ -287,9 +279,8 @@ bool CWallet::RemoveWatchOnly(const CScript &dest) return false; if (!HaveWatchOnly()) NotifyWatchonlyChanged(false); - if (fFileBacked) - if (!CWalletDB(strWalletFile).EraseWatchOnly(dest)) - return false; + if (!CWalletDB(*dbw).EraseWatchOnly(dest)) + return false; return true; } @@ -354,7 +345,7 @@ bool CWallet::ChangeWalletPassphrase(const SecureString& strOldWalletPassphrase, return false; if (!crypter.Encrypt(_vMasterKey, pMasterKey.second.vchCryptedKey)) return false; - CWalletDB(strWalletFile).WriteMasterKey(pMasterKey.first, pMasterKey.second); + CWalletDB(*dbw).WriteMasterKey(pMasterKey.first, pMasterKey.second); if (fWasLocked) Lock(); return true; @@ -367,7 +358,7 @@ bool CWallet::ChangeWalletPassphrase(const SecureString& strOldWalletPassphrase, void CWallet::SetBestChain(const CBlockLocator& loc) { - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); walletdb.WriteBestBlock(loc); } @@ -386,9 +377,8 @@ bool CWallet::SetMinVersion(enum WalletFeature nVersion, CWalletDB* pwalletdbIn, if (nVersion > nWalletMaxVersion) nWalletMaxVersion = nVersion; - if (fFileBacked) { - CWalletDB* pwalletdb = pwalletdbIn ? pwalletdbIn : new CWalletDB(strWalletFile); + CWalletDB* pwalletdb = pwalletdbIn ? pwalletdbIn : new CWalletDB(*dbw); if (nWalletVersion > 40000) pwalletdb->WriteMinVersion(nWalletVersion); if (!pwalletdbIn) @@ -442,7 +432,7 @@ bool CWallet::HasWalletSpend(const uint256& txid) const void CWallet::Flush(bool shutdown) { - bitdb.Flush(shutdown); + dbw->Flush(shutdown); } bool CWallet::Verify() @@ -595,24 +585,19 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) { LOCK(cs_wallet); mapMasterKeys[++nMasterKeyMaxID] = kMasterKey; - if (fFileBacked) - { - assert(!pwalletdbEncryption); - pwalletdbEncryption = new CWalletDB(strWalletFile); - if (!pwalletdbEncryption->TxnBegin()) { - delete pwalletdbEncryption; - pwalletdbEncryption = NULL; - return false; - } - pwalletdbEncryption->WriteMasterKey(nMasterKeyMaxID, kMasterKey); + assert(!pwalletdbEncryption); + pwalletdbEncryption = new CWalletDB(*dbw); + if (!pwalletdbEncryption->TxnBegin()) { + delete pwalletdbEncryption; + pwalletdbEncryption = NULL; + return false; } + pwalletdbEncryption->WriteMasterKey(nMasterKeyMaxID, kMasterKey); if (!EncryptKeys(_vMasterKey)) { - if (fFileBacked) { - pwalletdbEncryption->TxnAbort(); - delete pwalletdbEncryption; - } + pwalletdbEncryption->TxnAbort(); + delete pwalletdbEncryption; // We now probably have half of our keys encrypted in memory, and half not... // die and let the user reload the unencrypted wallet. assert(false); @@ -621,19 +606,16 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) // Encryption was introduced in version 0.4.0 SetMinVersion(FEATURE_WALLETCRYPT, pwalletdbEncryption, true); - if (fFileBacked) - { - if (!pwalletdbEncryption->TxnCommit()) { - delete pwalletdbEncryption; - // We now have keys encrypted in memory, but not on disk... - // die to avoid confusion and let the user reload the unencrypted wallet. - assert(false); - } - + if (!pwalletdbEncryption->TxnCommit()) { delete pwalletdbEncryption; - pwalletdbEncryption = NULL; + // We now have keys encrypted in memory, but not on disk... + // die to avoid confusion and let the user reload the unencrypted wallet. + assert(false); } + delete pwalletdbEncryption; + pwalletdbEncryption = NULL; + Lock(); Unlock(strWalletPassphrase); @@ -652,7 +634,7 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) // Need to completely rewrite the wallet file; if we don't, bdb might keep // bits of the unencrypted private key in slack space in the database file. - CDB::Rewrite(strWalletFile); + dbw->Rewrite(); } NotifyStatusChanged(this); @@ -663,7 +645,7 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) DBErrors CWallet::ReorderTransactions() { LOCK(cs_wallet); - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); // Old wallets didn't have any defined order for transactions // Probably a bad idea to change the output of this @@ -744,14 +726,14 @@ int64_t CWallet::IncOrderPosNext(CWalletDB *pwalletdb) if (pwalletdb) { pwalletdb->WriteOrderPosNext(nOrderPosNext); } else { - CWalletDB(strWalletFile).WriteOrderPosNext(nOrderPosNext); + CWalletDB(*dbw).WriteOrderPosNext(nOrderPosNext); } return nRet; } bool CWallet::AccountMove(std::string strFrom, std::string strTo, CAmount nAmount, std::string strComment) { - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); if (!walletdb.TxnBegin()) return false; @@ -785,7 +767,7 @@ bool CWallet::AccountMove(std::string strFrom, std::string strTo, CAmount nAmoun bool CWallet::GetAccountPubkey(CPubKey &pubKey, std::string strAccount, bool bForceNew) { - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); CAccount account; walletdb.ReadAccount(strAccount, account); @@ -846,7 +828,7 @@ bool CWallet::MarkReplaced(const uint256& originalHash, const uint256& newHash) wtx.mapValue["replaced_by_txid"] = newHash.ToString(); - CWalletDB walletdb(strWalletFile, "r+"); + CWalletDB walletdb(*dbw, "r+"); bool success = true; if (!walletdb.WriteTx(wtx)) { @@ -863,7 +845,7 @@ bool CWallet::AddToWallet(const CWalletTx& wtxIn, bool fFlushOnClose) { LOCK(cs_wallet); - CWalletDB walletdb(strWalletFile, "r+", fFlushOnClose); + CWalletDB walletdb(*dbw, "r+", fFlushOnClose); uint256 hash = wtxIn.GetHash(); @@ -1007,7 +989,7 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) { LOCK2(cs_main, cs_wallet); - CWalletDB walletdb(strWalletFile, "r+"); + CWalletDB walletdb(*dbw, "r+"); std::set<uint256> todo; std::set<uint256> done; @@ -1079,7 +1061,7 @@ void CWallet::MarkConflicted(const uint256& hashBlock, const uint256& hashTx) return; // Do not flush the wallet here for performance reasons - CWalletDB walletdb(strWalletFile, "r+", false); + CWalletDB walletdb(*dbw, "r+", false); std::set<uint256> todo; std::set<uint256> done; @@ -1142,12 +1124,12 @@ void CWallet::TransactionAddedToMempool(const CTransactionRef& ptx) { void CWallet::BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex *pindex, const std::vector<CTransactionRef>& vtxConflicted) { LOCK2(cs_main, cs_wallet); - // TODO: Tempoarily ensure that mempool removals are notified before + // TODO: Temporarily ensure that mempool removals are notified before // connected transactions. This shouldn't matter, but the abandoned // state of transactions in our wallet is currently cleared when we // receive another notification and there is a race condition where // notification of a connected conflict might cause an outside process - // to abandon a transaction and then have it inadvertantly cleared by + // to abandon a transaction and then have it inadvertently cleared by // the notification that the conflicted transaction was evicted. for (const CTransactionRef& ptx : vtxConflicted) { @@ -1362,7 +1344,7 @@ bool CWallet::SetHDMasterKey(const CPubKey& pubkey, CHDChain *possibleOldChain) bool CWallet::SetHDChain(const CHDChain& chain, bool memonly) { LOCK(cs_wallet); - if (!memonly && !CWalletDB(strWalletFile).WriteHDChain(chain)) + if (!memonly && !CWalletDB(*dbw).WriteHDChain(chain)) throw std::runtime_error(std::string(__func__) + ": writing chain failed"); hdChain = chain; @@ -1475,41 +1457,6 @@ void CWalletTx::GetAmounts(std::list<COutputEntry>& listReceived, } -void CWalletTx::GetAccountAmounts(const std::string& strAccount, CAmount& nReceived, - CAmount& nSent, CAmount& nFee, const isminefilter& filter) const -{ - nReceived = nSent = nFee = 0; - - CAmount allFee; - std::string strSentAccount; - std::list<COutputEntry> listReceived; - std::list<COutputEntry> listSent; - GetAmounts(listReceived, listSent, allFee, strSentAccount, filter); - - if (strAccount == strSentAccount) - { - BOOST_FOREACH(const COutputEntry& s, listSent) - nSent += s.amount; - nFee = allFee; - } - { - LOCK(pwallet->cs_wallet); - BOOST_FOREACH(const COutputEntry& r, listReceived) - { - if (pwallet->mapAddressBook.count(r.destination)) - { - std::map<CTxDestination, CAddressBookData>::const_iterator mi = pwallet->mapAddressBook.find(r.destination); - if (mi != pwallet->mapAddressBook.end() && (*mi).second.name == strAccount) - nReceived += r.amount; - } - else if (strAccount.empty()) - { - nReceived += r.amount; - } - } - } -} - /** * Scan the block chain (starting in pindexStart) for transactions * from or to us. If fUpdate is true, found transactions that already @@ -1544,6 +1491,10 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool f { if (pindex->nHeight % 100 == 0 && dProgressTip - dProgressStart > 0.0) ShowProgress(_("Rescanning..."), std::max(1, std::min(99, (int)((GuessVerificationProgress(chainParams.TxData(), pindex) - dProgressStart) / (dProgressTip - dProgressStart) * 100)))); + if (GetTime() >= nNow + 60) { + nNow = GetTime(); + LogPrintf("Still rescanning. At block %d. Progress=%f\n", pindex->nHeight, GuessVerificationProgress(chainParams.TxData(), pindex)); + } CBlock block; if (ReadBlockFromDisk(block, pindex, Params().GetConsensus())) { @@ -1557,10 +1508,6 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool f ret = nullptr; } pindex = chainActive.Next(pindex); - if (GetTime() >= nNow + 60) { - nNow = GetTime(); - LogPrintf("Still rescanning. At block %d. Progress=%f\n", pindex->nHeight, GuessVerificationProgress(chainParams.TxData(), pindex)); - } } if (pindex && fAbortRescan) { LogPrintf("Rescan aborted at block %d. Progress=%f\n", pindex->nHeight, GuessVerificationProgress(chainParams.TxData(), pindex)); @@ -1993,6 +1940,49 @@ CAmount CWallet::GetImmatureWatchOnlyBalance() const return nTotal; } +// Calculate total balance in a different way from GetBalance. The biggest +// difference is that GetBalance sums up all unspent TxOuts paying to the +// wallet, while this sums up both spent and unspent TxOuts paying to the +// wallet, and then subtracts the values of TxIns spending from the wallet. This +// also has fewer restrictions on which unconfirmed transactions are considered +// trusted. +CAmount CWallet::GetLegacyBalance(const isminefilter& filter, int minDepth, const std::string* account) const +{ + LOCK2(cs_main, cs_wallet); + + CAmount balance = 0; + for (const auto& entry : mapWallet) { + const CWalletTx& wtx = entry.second; + const int depth = wtx.GetDepthInMainChain(); + if (depth < 0 || !CheckFinalTx(*wtx.tx) || wtx.GetBlocksToMaturity() > 0) { + continue; + } + + // Loop through tx outputs and add incoming payments. For outgoing txs, + // treat change outputs specially, as part of the amount debited. + CAmount debit = wtx.GetDebit(filter); + const bool outgoing = debit > 0; + for (const CTxOut& out : wtx.tx->vout) { + if (outgoing && IsChange(out)) { + debit -= out.nValue; + } else if (IsMine(out) & filter && depth >= minDepth && (!account || *account == GetAccountName(out.scriptPubKey))) { + balance += out.nValue; + } + } + + // For outgoing txs, subtract amount debited. + if (outgoing && (!account || *account == wtx.strFromAccount)) { + balance -= debit; + } + } + + if (account) { + balance += CWalletDB(*dbw).GetAccountCreditDebit(*account); + } + + return balance; +} + void CWallet::AvailableCoins(std::vector<COutput>& vCoins, bool fOnlySafe, const CCoinControl *coinControl, bool fIncludeZeroValue) const { vCoins.clear(); @@ -2095,7 +2085,7 @@ static void ApproximateBestSubset(const std::vector<CInputCoin>& vValue, const C //that the rng is fast. We do not use a constant random sequence, //because there may be some privacy improvement by making //the selection random. - if (nPass == 0 ? insecure_rand.rand32()&1 : !vfIncluded[i]) + if (nPass == 0 ? insecure_rand.randbool() : !vfIncluded[i]) { nTotal += vValue[i].txout.nValue; vfIncluded[i] = true; @@ -2573,9 +2563,10 @@ bool CWallet::CreateTransaction(const std::vector<CRecipient>& vecSend, CWalletT std::vector<CTxOut>::iterator position = txNew.vout.begin()+nChangePosInOut; txNew.vout.insert(position, newTxOut); } - } - else + } else { reservekey.ReturnKey(); + nChangePosInOut = -1; + } // Fill vin // @@ -2759,13 +2750,13 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CCon } void CWallet::ListAccountCreditDebit(const std::string& strAccount, std::list<CAccountingEntry>& entries) { - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); return walletdb.ListAccountCreditDebit(strAccount, entries); } bool CWallet::AddAccountingEntry(const CAccountingEntry& acentry) { - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); return AddAccountingEntry(acentry, &walletdb); } @@ -2787,17 +2778,12 @@ CAmount CWallet::GetRequiredFee(unsigned int nTxBytes) return std::max(minTxFee.GetFee(nTxBytes), ::minRelayTxFee.GetFee(nTxBytes)); } -CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator) +CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, bool ignoreUserSetFee) { // payTxFee is the user-set global for desired feerate - return GetMinimumFee(nTxBytes, nConfirmTarget, pool, estimator, payTxFee.GetFee(nTxBytes)); -} - -CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, CAmount targetFee) -{ - CAmount nFeeNeeded = targetFee; + CAmount nFeeNeeded = payTxFee.GetFee(nTxBytes); // User didn't set: use -txconfirmtarget to estimate... - if (nFeeNeeded == 0) { + if (nFeeNeeded == 0 || ignoreUserSetFee) { int estimateFoundTarget = nConfirmTarget; nFeeNeeded = estimator.estimateSmartFee(nConfirmTarget, &estimateFoundTarget, pool).GetFee(nTxBytes); // ... unless we don't have enough mempool data for estimatefee, then use fallbackFee @@ -2817,13 +2803,11 @@ CAmount CWallet::GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarge DBErrors CWallet::LoadWallet(bool& fFirstRunRet) { - if (!fFileBacked) - return DB_LOAD_OK; fFirstRunRet = false; - DBErrors nLoadWalletRet = CWalletDB(strWalletFile,"cr+").LoadWallet(this); + DBErrors nLoadWalletRet = CWalletDB(*dbw,"cr+").LoadWallet(this); if (nLoadWalletRet == DB_NEED_REWRITE) { - if (CDB::Rewrite(strWalletFile, "\x04pool")) + if (dbw->Rewrite("\x04pool")) { LOCK(cs_wallet); setKeyPool.clear(); @@ -2844,17 +2828,15 @@ DBErrors CWallet::LoadWallet(bool& fFirstRunRet) DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut) { - if (!fFileBacked) - return DB_LOAD_OK; AssertLockHeld(cs_wallet); // mapWallet vchDefaultKey = CPubKey(); - DBErrors nZapSelectTxRet = CWalletDB(strWalletFile,"cr+").ZapSelectTx(vHashIn, vHashOut); + DBErrors nZapSelectTxRet = CWalletDB(*dbw,"cr+").ZapSelectTx(vHashIn, vHashOut); for (uint256 hash : vHashOut) mapWallet.erase(hash); if (nZapSelectTxRet == DB_NEED_REWRITE) { - if (CDB::Rewrite(strWalletFile, "\x04pool")) + if (dbw->Rewrite("\x04pool")) { setKeyPool.clear(); // Note: can't top-up keypool here, because wallet is locked. @@ -2874,13 +2856,11 @@ DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256 DBErrors CWallet::ZapWalletTx(std::vector<CWalletTx>& vWtx) { - if (!fFileBacked) - return DB_LOAD_OK; vchDefaultKey = CPubKey(); - DBErrors nZapWalletTxRet = CWalletDB(strWalletFile,"cr+").ZapWalletTx(vWtx); + DBErrors nZapWalletTxRet = CWalletDB(*dbw,"cr+").ZapWalletTx(vWtx); if (nZapWalletTxRet == DB_NEED_REWRITE) { - if (CDB::Rewrite(strWalletFile, "\x04pool")) + if (dbw->Rewrite("\x04pool")) { LOCK(cs_wallet); setKeyPool.clear(); @@ -2910,11 +2890,9 @@ bool CWallet::SetAddressBook(const CTxDestination& address, const std::string& s } NotifyAddressBookChanged(this, address, strName, ::IsMine(*this, address) != ISMINE_NO, strPurpose, (fUpdated ? CT_UPDATED : CT_NEW) ); - if (!fFileBacked) - return false; - if (!strPurpose.empty() && !CWalletDB(strWalletFile).WritePurpose(CBitcoinAddress(address).ToString(), strPurpose)) + if (!strPurpose.empty() && !CWalletDB(*dbw).WritePurpose(CBitcoinAddress(address).ToString(), strPurpose)) return false; - return CWalletDB(strWalletFile).WriteName(CBitcoinAddress(address).ToString(), strName); + return CWalletDB(*dbw).WriteName(CBitcoinAddress(address).ToString(), strName); } bool CWallet::DelAddressBook(const CTxDestination& address) @@ -2922,33 +2900,40 @@ bool CWallet::DelAddressBook(const CTxDestination& address) { LOCK(cs_wallet); // mapAddressBook - if(fFileBacked) + // Delete destdata tuples associated with address + std::string strAddress = CBitcoinAddress(address).ToString(); + BOOST_FOREACH(const PAIRTYPE(std::string, std::string) &item, mapAddressBook[address].destdata) { - // Delete destdata tuples associated with address - std::string strAddress = CBitcoinAddress(address).ToString(); - BOOST_FOREACH(const PAIRTYPE(std::string, std::string) &item, mapAddressBook[address].destdata) - { - CWalletDB(strWalletFile).EraseDestData(strAddress, item.first); - } + CWalletDB(*dbw).EraseDestData(strAddress, item.first); } mapAddressBook.erase(address); } NotifyAddressBookChanged(this, address, "", ::IsMine(*this, address) != ISMINE_NO, "", CT_DELETED); - if (!fFileBacked) - return false; - CWalletDB(strWalletFile).ErasePurpose(CBitcoinAddress(address).ToString()); - return CWalletDB(strWalletFile).EraseName(CBitcoinAddress(address).ToString()); + CWalletDB(*dbw).ErasePurpose(CBitcoinAddress(address).ToString()); + return CWalletDB(*dbw).EraseName(CBitcoinAddress(address).ToString()); } -bool CWallet::SetDefaultKey(const CPubKey &vchPubKey) +const std::string& CWallet::GetAccountName(const CScript& scriptPubKey) const { - if (fFileBacked) - { - if (!CWalletDB(strWalletFile).WriteDefaultKey(vchPubKey)) - return false; + CTxDestination address; + if (ExtractDestination(scriptPubKey, address) && !scriptPubKey.IsUnspendable()) { + auto mi = mapAddressBook.find(address); + if (mi != mapAddressBook.end()) { + return mi->second.name; + } } + // A scriptPubKey that doesn't have an entry in the address book is + // associated with the default account (""). + const static std::string DEFAULT_ACCOUNT_NAME; + return DEFAULT_ACCOUNT_NAME; +} + +bool CWallet::SetDefaultKey(const CPubKey &vchPubKey) +{ + if (!CWalletDB(*dbw).WriteDefaultKey(vchPubKey)) + return false; vchDefaultKey = vchPubKey; return true; } @@ -2961,7 +2946,7 @@ bool CWallet::NewKeyPool() { { LOCK(cs_wallet); - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); BOOST_FOREACH(int64_t nIndex, setKeyPool) walletdb.ErasePool(nIndex); setKeyPool.clear(); @@ -2982,7 +2967,7 @@ size_t CWallet::KeypoolCountExternalKeys() if (!IsHDEnabled() || !CanSupportFeature(FEATURE_HD_SPLIT)) return setKeyPool.size(); - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); // count amount of external keys size_t amountE = 0; @@ -3025,7 +3010,7 @@ bool CWallet::TopUpKeyPool(unsigned int kpSize) missingInternal = 0; } bool internal = false; - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); for (int64_t i = missingInternal + missingExternal; i--;) { int64_t nEnd = 1; @@ -3056,7 +3041,7 @@ void CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool int if(setKeyPool.empty()) return; - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); // try to find a key that matches the internal/external filter for(const int64_t& id : setKeyPool) @@ -3082,11 +3067,8 @@ void CWallet::ReserveKeyFromKeyPool(int64_t& nIndex, CKeyPool& keypool, bool int void CWallet::KeepKey(int64_t nIndex) { // Remove from key pool - if (fFileBacked) - { - CWalletDB walletdb(strWalletFile); - walletdb.ErasePool(nIndex); - } + CWalletDB walletdb(*dbw); + walletdb.ErasePool(nIndex); LogPrintf("keypool keep %d\n", nIndex); } @@ -3128,7 +3110,7 @@ int64_t CWallet::GetOldestKeyPoolTime() return GetTime(); CKeyPool keypool; - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); if (IsHDEnabled() && CanSupportFeature(FEATURE_HD_SPLIT)) { @@ -3294,37 +3276,6 @@ std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() return ret; } -CAmount CWallet::GetAccountBalance(const std::string& strAccount, int nMinDepth, const isminefilter& filter) -{ - CWalletDB walletdb(strWalletFile); - return GetAccountBalance(walletdb, strAccount, nMinDepth, filter); -} - -CAmount CWallet::GetAccountBalance(CWalletDB& walletdb, const std::string& strAccount, int nMinDepth, const isminefilter& filter) -{ - CAmount nBalance = 0; - - // Tally wallet transactions - for (std::map<uint256, CWalletTx>::iterator it = mapWallet.begin(); it != mapWallet.end(); ++it) - { - const CWalletTx& wtx = (*it).second; - if (!CheckFinalTx(wtx) || wtx.GetBlocksToMaturity() > 0 || wtx.GetDepthInMainChain() < 0) - continue; - - CAmount nReceived, nSent, nFee; - wtx.GetAccountAmounts(strAccount, nReceived, nSent, nFee, filter); - - if (nReceived != 0 && wtx.GetDepthInMainChain() >= nMinDepth) - nBalance += nReceived; - nBalance -= nSent + nFee; - } - - // Tally internal accounting entries - nBalance += walletdb.GetAccountCreditDebit(strAccount); - - return nBalance; -} - std::set<CTxDestination> CWallet::GetAccountAddresses(const std::string& strAccount) const { LOCK(cs_wallet); @@ -3376,7 +3327,7 @@ void CWallet::GetAllReserveKeys(std::set<CKeyID>& setAddress) const { setAddress.clear(); - CWalletDB walletdb(strWalletFile); + CWalletDB walletdb(*dbw); LOCK2(cs_main, cs_wallet); BOOST_FOREACH(const int64_t& id, setKeyPool) @@ -3598,18 +3549,14 @@ bool CWallet::AddDestData(const CTxDestination &dest, const std::string &key, co return false; mapAddressBook[dest].destdata.insert(std::make_pair(key, value)); - if (!fFileBacked) - return true; - return CWalletDB(strWalletFile).WriteDestData(CBitcoinAddress(dest).ToString(), key, value); + return CWalletDB(*dbw).WriteDestData(CBitcoinAddress(dest).ToString(), key, value); } bool CWallet::EraseDestData(const CTxDestination &dest, const std::string &key) { if (!mapAddressBook[dest].destdata.erase(key)) return false; - if (!fFileBacked) - return true; - return CWalletDB(strWalletFile).EraseDestData(CBitcoinAddress(dest).ToString(), key); + return CWalletDB(*dbw).EraseDestData(CBitcoinAddress(dest).ToString(), key); } bool CWallet::LoadDestData(const CTxDestination &dest, const std::string &key, const std::string &value) @@ -3679,7 +3626,8 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) if (GetBoolArg("-zapwallettxes", false)) { uiInterface.InitMessage(_("Zapping all transactions from wallet...")); - CWallet *tempWallet = new CWallet(walletFile); + std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, walletFile)); + CWallet *tempWallet = new CWallet(std::move(dbw)); DBErrors nZapWalletRet = tempWallet->ZapWalletTx(vWtx); if (nZapWalletRet != DB_LOAD_OK) { InitError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile)); @@ -3694,7 +3642,8 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) int64_t nStart = GetTimeMillis(); bool fFirstRun = true; - CWallet *walletInstance = new CWallet(walletFile); + std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, walletFile)); + CWallet *walletInstance = new CWallet(std::move(dbw)); DBErrors nLoadWalletRet = walletInstance->LoadWallet(fFirstRun); if (nLoadWalletRet != DB_LOAD_OK) { @@ -3785,7 +3734,7 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) CBlockIndex *pindexRescan = chainActive.Genesis(); if (!GetBoolArg("-rescan", false)) { - CWalletDB walletdb(walletFile); + CWalletDB walletdb(*walletInstance->dbw); CBlockLocator locator; if (walletdb.ReadBestBlock(locator)) pindexRescan = FindForkInGlobalIndex(chainActive, locator); @@ -3818,7 +3767,7 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) // Restore wallet transaction metadata after -zapwallettxes=1 if (GetBoolArg("-zapwallettxes", false) && GetArg("-zapwallettxes", "1") != "2") { - CWalletDB walletdb(walletFile); + CWalletDB walletdb(*walletInstance->dbw); BOOST_FOREACH(const CWalletTx& wtxOld, vWtx) { @@ -3978,38 +3927,7 @@ bool CWallet::ParameterInteraction() bool CWallet::BackupWallet(const std::string& strDest) { - if (!fFileBacked) - return false; - while (true) - { - { - LOCK(bitdb.cs_db); - if (!bitdb.mapFileUseCount.count(strWalletFile) || bitdb.mapFileUseCount[strWalletFile] == 0) - { - // Flush log data to the dat file - bitdb.CloseDb(strWalletFile); - bitdb.CheckpointLSN(strWalletFile); - bitdb.mapFileUseCount.erase(strWalletFile); - - // Copy wallet file - fs::path pathSrc = GetDataDir() / strWalletFile; - fs::path pathDest(strDest); - if (fs::is_directory(pathDest)) - pathDest /= strWalletFile; - - try { - fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); - LogPrintf("copied %s to %s\n", strWalletFile, pathDest.string()); - return true; - } catch (const fs::filesystem_error& e) { - LogPrintf("error copying %s to %s - %s\n", strWalletFile, pathDest.string(), e.what()); - return false; - } - } - } - MilliSleep(100); - } - return false; + return dbw->Backup(strDest); } CKeyPool::CKeyPool() diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index c0ed44377f..8015cc8492 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -451,9 +451,6 @@ public: void GetAmounts(std::list<COutputEntry>& listReceived, std::list<COutputEntry>& listSent, CAmount& nFee, std::string& strSentAccount, const isminefilter& filter) const; - void GetAccountAmounts(const std::string& strAccount, CAmount& nReceived, - CAmount& nSent, CAmount& nFee, const isminefilter& filter) const; - bool IsFromMe(const isminefilter& filter) const { return (GetDebit(filter) > 0); @@ -699,8 +696,6 @@ private: /* HD derive new child key (on internal or external chain) */ void DeriveNewChildKey(CKeyMetadata& metadata, CKey& secret, bool internal = false); - bool fFileBacked; - std::set<int64_t> setKeyPool; int64_t nTimeFirstKey; @@ -716,17 +711,33 @@ private: */ bool AddWatchOnly(const CScript& dest) override; + std::unique_ptr<CWalletDBWrapper> dbw; + public: /* * Main wallet lock. - * This lock protects all the fields added by CWallet - * except for: - * fFileBacked (immutable after instantiation) - * strWalletFile (immutable after instantiation) + * This lock protects all the fields added by CWallet. */ mutable CCriticalSection cs_wallet; - const std::string strWalletFile; + /** Get database handle used by this wallet. Ideally this function would + * not be necessary. + */ + CWalletDBWrapper& GetDBHandle() + { + return *dbw; + } + + /** Get a name for this wallet for logging/debugging purposes. + */ + std::string GetName() const + { + if (dbw) { + return dbw->GetName(); + } else { + return "dummy"; + } + } void LoadKeyPool(int nIndex, const CKeyPool &keypool) { @@ -748,15 +759,16 @@ public: MasterKeyMap mapMasterKeys; unsigned int nMasterKeyMaxID; - CWallet() + // Create wallet with dummy database handle + CWallet(): dbw(new CWalletDBWrapper()) { SetNull(); } - CWallet(const std::string& strWalletFileIn) : strWalletFile(strWalletFileIn) + // Create wallet with passed-in database handle + CWallet(std::unique_ptr<CWalletDBWrapper> dbw_in) : dbw(std::move(dbw_in)) { SetNull(); - fFileBacked = true; } ~CWallet() @@ -769,7 +781,6 @@ public: { nWalletVersion = FEATURE_BASE; nWalletMaxVersion = FEATURE_BASE; - fFileBacked = false; nMasterKeyMaxID = 0; pwalletdbEncryption = NULL; nOrderPosNext = 0; @@ -904,6 +915,7 @@ public: CAmount GetWatchOnlyBalance() const; CAmount GetUnconfirmedWatchOnlyBalance() const; CAmount GetImmatureWatchOnlyBalance() const; + CAmount GetLegacyBalance(const isminefilter& filter, int minDepth, const std::string* account) const; /** * Insert additional inputs into the transaction by @@ -933,12 +945,7 @@ public: * Estimate the minimum fee considering user set parameters * and the required fee */ - static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator); - /** - * Estimate the minimum fee considering required fee and targetFee or if 0 - * then fee estimation for nConfirmTarget - */ - static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, CAmount targetFee); + static CAmount GetMinimumFee(unsigned int nTxBytes, unsigned int nConfirmTarget, const CTxMemPool& pool, const CBlockPolicyEstimator& estimator, bool ignoreUserSetFee = false); /** * Return the minimum required fee taking into account the * floating relay fee and user set minimum transaction fee @@ -958,8 +965,6 @@ public: std::set< std::set<CTxDestination> > GetAddressGroupings(); std::map<CTxDestination, CAmount> GetAddressBalances(); - CAmount GetAccountBalance(const std::string& strAccount, int nMinDepth, const isminefilter& filter); - CAmount GetAccountBalance(CWalletDB& walletdb, const std::string& strAccount, int nMinDepth, const isminefilter& filter); std::set<CTxDestination> GetAccountAddresses(const std::string& strAccount) const; isminetype IsMine(const CTxIn& txin) const; @@ -990,6 +995,8 @@ public: bool DelAddressBook(const CTxDestination& address); + const std::string& GetAccountName(const CScript& scriptPubKey) const; + void Inventory(const uint256 &hash) override { { diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index ceff2d36e3..a90fa6dbbd 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -33,7 +33,7 @@ static std::atomic<unsigned int> nWalletDBUpdateCounter; bool CWalletDB::WriteName(const std::string& strAddress, const std::string& strName) { nWalletDBUpdateCounter++; - return Write(make_pair(std::string("name"), strAddress), strName); + return batch.Write(std::make_pair(std::string("name"), strAddress), strName); } bool CWalletDB::EraseName(const std::string& strAddress) @@ -41,38 +41,38 @@ bool CWalletDB::EraseName(const std::string& strAddress) // This should only be used for sending addresses, never for receiving addresses, // receiving addresses must always have an address book entry if they're not change return. nWalletDBUpdateCounter++; - return Erase(make_pair(std::string("name"), strAddress)); + return batch.Erase(std::make_pair(std::string("name"), strAddress)); } bool CWalletDB::WritePurpose(const std::string& strAddress, const std::string& strPurpose) { nWalletDBUpdateCounter++; - return Write(make_pair(std::string("purpose"), strAddress), strPurpose); + return batch.Write(std::make_pair(std::string("purpose"), strAddress), strPurpose); } bool CWalletDB::ErasePurpose(const std::string& strPurpose) { nWalletDBUpdateCounter++; - return Erase(make_pair(std::string("purpose"), strPurpose)); + return batch.Erase(std::make_pair(std::string("purpose"), strPurpose)); } bool CWalletDB::WriteTx(const CWalletTx& wtx) { nWalletDBUpdateCounter++; - return Write(std::make_pair(std::string("tx"), wtx.GetHash()), wtx); + return batch.Write(std::make_pair(std::string("tx"), wtx.GetHash()), wtx); } bool CWalletDB::EraseTx(uint256 hash) { nWalletDBUpdateCounter++; - return Erase(std::make_pair(std::string("tx"), hash)); + return batch.Erase(std::make_pair(std::string("tx"), hash)); } bool CWalletDB::WriteKey(const CPubKey& vchPubKey, const CPrivKey& vchPrivKey, const CKeyMetadata& keyMeta) { nWalletDBUpdateCounter++; - if (!Write(std::make_pair(std::string("keymeta"), vchPubKey), + if (!batch.Write(std::make_pair(std::string("keymeta"), vchPubKey), keyMeta, false)) return false; @@ -82,7 +82,7 @@ bool CWalletDB::WriteKey(const CPubKey& vchPubKey, const CPrivKey& vchPrivKey, c vchKey.insert(vchKey.end(), vchPubKey.begin(), vchPubKey.end()); vchKey.insert(vchKey.end(), vchPrivKey.begin(), vchPrivKey.end()); - return Write(std::make_pair(std::string("key"), vchPubKey), std::make_pair(vchPrivKey, Hash(vchKey.begin(), vchKey.end())), false); + return batch.Write(std::make_pair(std::string("key"), vchPubKey), std::make_pair(vchPrivKey, Hash(vchKey.begin(), vchKey.end())), false); } bool CWalletDB::WriteCryptedKey(const CPubKey& vchPubKey, @@ -92,16 +92,16 @@ bool CWalletDB::WriteCryptedKey(const CPubKey& vchPubKey, const bool fEraseUnencryptedKey = true; nWalletDBUpdateCounter++; - if (!Write(std::make_pair(std::string("keymeta"), vchPubKey), + if (!batch.Write(std::make_pair(std::string("keymeta"), vchPubKey), keyMeta)) return false; - if (!Write(std::make_pair(std::string("ckey"), vchPubKey), vchCryptedSecret, false)) + if (!batch.Write(std::make_pair(std::string("ckey"), vchPubKey), vchCryptedSecret, false)) return false; if (fEraseUnencryptedKey) { - Erase(std::make_pair(std::string("key"), vchPubKey)); - Erase(std::make_pair(std::string("wkey"), vchPubKey)); + batch.Erase(std::make_pair(std::string("key"), vchPubKey)); + batch.Erase(std::make_pair(std::string("wkey"), vchPubKey)); } return true; } @@ -109,92 +109,92 @@ bool CWalletDB::WriteCryptedKey(const CPubKey& vchPubKey, bool CWalletDB::WriteMasterKey(unsigned int nID, const CMasterKey& kMasterKey) { nWalletDBUpdateCounter++; - return Write(std::make_pair(std::string("mkey"), nID), kMasterKey, true); + return batch.Write(std::make_pair(std::string("mkey"), nID), kMasterKey, true); } bool CWalletDB::WriteCScript(const uint160& hash, const CScript& redeemScript) { nWalletDBUpdateCounter++; - return Write(std::make_pair(std::string("cscript"), hash), *(const CScriptBase*)(&redeemScript), false); + return batch.Write(std::make_pair(std::string("cscript"), hash), *(const CScriptBase*)(&redeemScript), false); } bool CWalletDB::WriteWatchOnly(const CScript &dest, const CKeyMetadata& keyMeta) { nWalletDBUpdateCounter++; - if (!Write(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)), keyMeta)) + if (!batch.Write(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)), keyMeta)) return false; - return Write(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest)), '1'); + return batch.Write(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest)), '1'); } bool CWalletDB::EraseWatchOnly(const CScript &dest) { nWalletDBUpdateCounter++; - if (!Erase(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)))) + if (!batch.Erase(std::make_pair(std::string("watchmeta"), *(const CScriptBase*)(&dest)))) return false; - return Erase(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest))); + return batch.Erase(std::make_pair(std::string("watchs"), *(const CScriptBase*)(&dest))); } bool CWalletDB::WriteBestBlock(const CBlockLocator& locator) { nWalletDBUpdateCounter++; - Write(std::string("bestblock"), CBlockLocator()); // Write empty block locator so versions that require a merkle branch automatically rescan - return Write(std::string("bestblock_nomerkle"), locator); + batch.Write(std::string("bestblock"), CBlockLocator()); // Write empty block locator so versions that require a merkle branch automatically rescan + return batch.Write(std::string("bestblock_nomerkle"), locator); } bool CWalletDB::ReadBestBlock(CBlockLocator& locator) { - if (Read(std::string("bestblock"), locator) && !locator.vHave.empty()) return true; - return Read(std::string("bestblock_nomerkle"), locator); + if (batch.Read(std::string("bestblock"), locator) && !locator.vHave.empty()) return true; + return batch.Read(std::string("bestblock_nomerkle"), locator); } bool CWalletDB::WriteOrderPosNext(int64_t nOrderPosNext) { nWalletDBUpdateCounter++; - return Write(std::string("orderposnext"), nOrderPosNext); + return batch.Write(std::string("orderposnext"), nOrderPosNext); } bool CWalletDB::WriteDefaultKey(const CPubKey& vchPubKey) { nWalletDBUpdateCounter++; - return Write(std::string("defaultkey"), vchPubKey); + return batch.Write(std::string("defaultkey"), vchPubKey); } bool CWalletDB::ReadPool(int64_t nPool, CKeyPool& keypool) { - return Read(std::make_pair(std::string("pool"), nPool), keypool); + return batch.Read(std::make_pair(std::string("pool"), nPool), keypool); } bool CWalletDB::WritePool(int64_t nPool, const CKeyPool& keypool) { nWalletDBUpdateCounter++; - return Write(std::make_pair(std::string("pool"), nPool), keypool); + return batch.Write(std::make_pair(std::string("pool"), nPool), keypool); } bool CWalletDB::ErasePool(int64_t nPool) { nWalletDBUpdateCounter++; - return Erase(std::make_pair(std::string("pool"), nPool)); + return batch.Erase(std::make_pair(std::string("pool"), nPool)); } bool CWalletDB::WriteMinVersion(int nVersion) { - return Write(std::string("minversion"), nVersion); + return batch.Write(std::string("minversion"), nVersion); } bool CWalletDB::ReadAccount(const std::string& strAccount, CAccount& account) { account.SetNull(); - return Read(make_pair(std::string("acc"), strAccount), account); + return batch.Read(std::make_pair(std::string("acc"), strAccount), account); } bool CWalletDB::WriteAccount(const std::string& strAccount, const CAccount& account) { - return Write(make_pair(std::string("acc"), strAccount), account); + return batch.Write(std::make_pair(std::string("acc"), strAccount), account); } bool CWalletDB::WriteAccountingEntry(const uint64_t nAccEntryNum, const CAccountingEntry& acentry) { - return Write(std::make_pair(std::string("acentry"), std::make_pair(acentry.strAccount, nAccEntryNum)), acentry); + return batch.Write(std::make_pair(std::string("acentry"), std::make_pair(acentry.strAccount, nAccEntryNum)), acentry); } bool CWalletDB::WriteAccountingEntry_Backend(const CAccountingEntry& acentry) @@ -218,7 +218,7 @@ void CWalletDB::ListAccountCreditDebit(const std::string& strAccount, std::list< { bool fAllAccounts = (strAccount == "*"); - Dbc* pcursor = GetCursor(); + Dbc* pcursor = batch.GetCursor(); if (!pcursor) throw std::runtime_error(std::string(__func__) + ": cannot create DB cursor"); bool setRange = true; @@ -229,7 +229,7 @@ void CWalletDB::ListAccountCreditDebit(const std::string& strAccount, std::list< if (setRange) ssKey << std::make_pair(std::string("acentry"), std::make_pair((fAllAccounts ? std::string("") : strAccount), uint64_t(0))); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = ReadAtCursor(pcursor, ssKey, ssValue, setRange); + int ret = batch.ReadAtCursor(pcursor, ssKey, ssValue, setRange); setRange = false; if (ret == DB_NOTFOUND) break; @@ -560,7 +560,7 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) LOCK(pwallet->cs_wallet); try { int nMinVersion = 0; - if (Read((std::string)"minversion", nMinVersion)) + if (batch.Read((std::string)"minversion", nMinVersion)) { if (nMinVersion > CLIENT_VERSION) return DB_TOO_NEW; @@ -568,7 +568,7 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) } // Get cursor - Dbc* pcursor = GetCursor(); + Dbc* pcursor = batch.GetCursor(); if (!pcursor) { LogPrintf("Error getting wallet database cursor\n"); @@ -580,7 +580,7 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet) // Read next record CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = ReadAtCursor(pcursor, ssKey, ssValue); + int ret = batch.ReadAtCursor(pcursor, ssKey, ssValue); if (ret == DB_NOTFOUND) break; else if (ret != 0) @@ -664,14 +664,14 @@ DBErrors CWalletDB::FindWalletTx(std::vector<uint256>& vTxHash, std::vector<CWal try { int nMinVersion = 0; - if (Read((std::string)"minversion", nMinVersion)) + if (batch.Read((std::string)"minversion", nMinVersion)) { if (nMinVersion > CLIENT_VERSION) return DB_TOO_NEW; } // Get cursor - Dbc* pcursor = GetCursor(); + Dbc* pcursor = batch.GetCursor(); if (!pcursor) { LogPrintf("Error getting wallet database cursor\n"); @@ -683,7 +683,7 @@ DBErrors CWalletDB::FindWalletTx(std::vector<uint256>& vTxHash, std::vector<CWal // Read next record CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); - int ret = ReadAtCursor(pcursor, ssKey, ssValue); + int ret = batch.ReadAtCursor(pcursor, ssKey, ssValue); if (ret == DB_NOTFOUND) break; else if (ret != 0) @@ -797,9 +797,9 @@ void MaybeCompactWalletDB() if (nLastFlushed != CWalletDB::GetUpdateCounter() && GetTime() - nLastWalletUpdate >= 2) { - const std::string& strFile = pwalletMain->strWalletFile; - if (CDB::PeriodicFlush(strFile)) + if (CDB::PeriodicFlush(pwalletMain->GetDBHandle())) { nLastFlushed = CWalletDB::GetUpdateCounter(); + } } fOneThread = false; } @@ -855,20 +855,20 @@ bool CWalletDB::VerifyDatabaseFile(const std::string& walletFile, const fs::path bool CWalletDB::WriteDestData(const std::string &address, const std::string &key, const std::string &value) { nWalletDBUpdateCounter++; - return Write(std::make_pair(std::string("destdata"), std::make_pair(address, key)), value); + return batch.Write(std::make_pair(std::string("destdata"), std::make_pair(address, key)), value); } bool CWalletDB::EraseDestData(const std::string &address, const std::string &key) { nWalletDBUpdateCounter++; - return Erase(std::make_pair(std::string("destdata"), std::make_pair(address, key))); + return batch.Erase(std::make_pair(std::string("destdata"), std::make_pair(address, key))); } bool CWalletDB::WriteHDChain(const CHDChain& chain) { nWalletDBUpdateCounter++; - return Write(std::string("hdchain"), chain); + return batch.Write(std::string("hdchain"), chain); } void CWalletDB::IncrementUpdateCounter() @@ -880,3 +880,28 @@ unsigned int CWalletDB::GetUpdateCounter() { return nWalletDBUpdateCounter; } + +bool CWalletDB::TxnBegin() +{ + return batch.TxnBegin(); +} + +bool CWalletDB::TxnCommit() +{ + return batch.TxnCommit(); +} + +bool CWalletDB::TxnAbort() +{ + return batch.TxnAbort(); +} + +bool CWalletDB::ReadVersion(int& nVersion) +{ + return batch.ReadVersion(nVersion); +} + +bool CWalletDB::WriteVersion(int nVersion) +{ + return batch.WriteVersion(nVersion); +} diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index b94f341b2e..cd9fe279c5 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -17,6 +17,21 @@ #include <utility> #include <vector> +/** + * Overview of wallet database classes: + * + * - CDBEnv is an environment in which the database exists (has no analog in dbwrapper.h) + * - CWalletDBWrapper represents a wallet database (similar to CDBWrapper in dbwrapper.h) + * - CDB is a low-level database transaction (similar to CDBBatch in dbwrapper.h) + * - CWalletDB is a modifier object for the wallet, and encapsulates a database + * transaction as well as methods to act on the database (no analog in + * dbwrapper.h) + * + * The latter two are named confusingly, in contrast to what the names CDB + * and CWalletDB suggest they are transient transaction objects and don't + * represent the database itself. + */ + static const bool DEFAULT_FLUSHWALLET = true; class CAccount; @@ -118,11 +133,16 @@ public: } }; -/** Access to the wallet database */ -class CWalletDB : public CDB +/** Access to the wallet database. + * This should really be named CWalletDBBatch, as it represents a single transaction at the + * database. It will be committed when the object goes out of scope. + * Optionally (on by default) it will flush to disk as well. + */ +class CWalletDB { public: - CWalletDB(const std::string& strFilename, const char* pszMode = "r+", bool _fFlushOnClose = true) : CDB(strFilename, pszMode, _fFlushOnClose) + CWalletDB(CWalletDBWrapper& dbw, const char* pszMode = "r+", bool _fFlushOnClose = true) : + batch(dbw, pszMode, _fFlushOnClose) { } @@ -194,7 +214,20 @@ public: static void IncrementUpdateCounter(); static unsigned int GetUpdateCounter(); + + //! Begin a new transaction + bool TxnBegin(); + //! Commit current transaction + bool TxnCommit(); + //! Abort current transaction + bool TxnAbort(); + //! Read wallet version + bool ReadVersion(int& nVersion); + //! Write wallet version + bool WriteVersion(int nVersion); private: + CDB batch; + CWalletDB(const CWalletDB&); void operator=(const CWalletDB&); }; diff --git a/test/functional/README.md b/test/functional/README.md index 651b01f18a..e6c4849702 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -59,7 +59,7 @@ thread.) * RPC calls are available in p2p tests. * Can be used to write free-form tests, where specific p2p-protocol behavior -is tested. Examples: ```p2p-accept-block.py```, ```maxblocksinflight.py```. +is tested. Examples: ```p2p-accept-block.py```, ```p2p-compactblocks.py```. ## Comptool diff --git a/test/functional/disconnect_ban.py b/test/functional/disconnect_ban.py index 3f451d49d2..6ec42c8aba 100755 --- a/test/functional/disconnect_ban.py +++ b/test/functional/disconnect_ban.py @@ -30,7 +30,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.log.info("setban: successfully ban single IP address") assert_equal(len(self.nodes[1].getpeerinfo()), 2) # node1 should have 2 connections to node0 at this point self.nodes[1].setban("127.0.0.1", "add") - wait_until(lambda: len(self.nodes[1].getpeerinfo()) == 0) + assert wait_until(lambda: len(self.nodes[1].getpeerinfo()) == 0, timeout=10) assert_equal(len(self.nodes[1].getpeerinfo()), 0) # all nodes must be disconnected at this point assert_equal(len(self.nodes[1].listbanned()), 1) @@ -64,7 +64,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.nodes[1].setban("2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/19", "add", 1000) # ban for 1000 seconds listBeforeShutdown = self.nodes[1].listbanned() assert_equal("192.168.0.1/32", listBeforeShutdown[2]['address']) - wait_until(lambda: len(self.nodes[1].listbanned()) == 3) + assert wait_until(lambda: len(self.nodes[1].listbanned()) == 3, timeout=10) stop_node(self.nodes[1], 1) @@ -78,7 +78,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.nodes[1].clearbanned() connect_nodes_bi(self.nodes, 0, 1) - self.log.info("Test disconnectrnode RPCs") + self.log.info("Test disconnectnode RPCs") self.log.info("disconnectnode: fail to disconnect when calling with address and nodeid") address1 = self.nodes[0].getpeerinfo()[0]['addr'] @@ -91,7 +91,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.log.info("disconnectnode: successfully disconnect node by address") address1 = self.nodes[0].getpeerinfo()[0]['addr'] self.nodes[0].disconnectnode(address=address1) - wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1) + assert wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) assert not [node for node in self.nodes[0].getpeerinfo() if node['addr'] == address1] self.log.info("disconnectnode: successfully reconnect node") @@ -102,7 +102,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.log.info("disconnectnode: successfully disconnect node by node id") id1 = self.nodes[0].getpeerinfo()[0]['id'] self.nodes[0].disconnectnode(nodeid=id1) - wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1) + assert wait_until(lambda: len(self.nodes[0].getpeerinfo()) == 1, timeout=10) assert not [node for node in self.nodes[0].getpeerinfo() if node['id'] == id1] if __name__ == '__main__': diff --git a/test/functional/fundrawtransaction.py b/test/functional/fundrawtransaction.py index b86ea2d877..e0ca94a6b3 100755 --- a/test/functional/fundrawtransaction.py +++ b/test/functional/fundrawtransaction.py @@ -53,6 +53,11 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[0].generate(121) self.sync_all() + # ensure that setting changePosition in fundraw with an exact match is handled properly + rawmatch = self.nodes[2].createrawtransaction([], {self.nodes[2].getnewaddress():50}) + rawmatch = self.nodes[2].fundrawtransaction(rawmatch, {"changePosition":1, "subtractFeeFromOutputs":[0]}) + assert_equal(rawmatch["changepos"], -1) + watchonly_address = self.nodes[0].getnewaddress() watchonly_pubkey = self.nodes[0].validateaddress(watchonly_address)["pubkey"] watchonly_amount = Decimal(200) diff --git a/test/functional/import-abort-rescan.py b/test/functional/import-abort-rescan.py new file mode 100755 index 0000000000..ffe45bbb1d --- /dev/null +++ b/test/functional/import-abort-rescan.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test wallet import RPCs. + +Test rescan behavior of importprivkey when aborted. The test ensures that: +1. The abortrescan command indeed stops the rescan process. +2. Subsequent rescan catches the aborted address UTXO +""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import (assert_equal, get_rpc_proxy) +from decimal import Decimal +import threading # for bg importprivkey +import time # for sleep + +class ImportAbortRescanTest(BitcoinTestFramework): + def __init__(self): + super().__init__() + self.setup_clean_chain = True + + def run_test(self): + # Generate for BTC + assert_equal(self.nodes[0].getbalance(), 0) + assert_equal(self.nodes[1].getbalance(), 0) + self.nodes[0].generate(300) + assert_equal(self.nodes[1].getbalance(), 0) + # Make blocks with spam to cause rescan delay + for i in range(5): + for j in range(5): + self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 0.1) + self.nodes[0].generate(10) + addr = self.nodes[0].getnewaddress() + privkey = self.nodes[0].dumpprivkey(addr) + self.nodes[0].sendtoaddress(addr, 0.123) + self.nodes[0].generate(10) # mature tx + self.sync_all() + + # Import this address in the background ... + node1ref = get_rpc_proxy(self.nodes[1].url, 1, timeout=600) + importthread = threading.Thread(target=node1ref.importprivkey, args=[privkey]) + importthread.start() + # ... then abort rescan; try a bunch until abortres becomes true, + # because we will start checking before above thread starts processing + for i in range(2000): + time.sleep(0.001) + abortres = self.nodes[1].abortrescan() + if abortres: break + assert abortres # if false, we failed to abort + # import should die soon + for i in range(10): + time.sleep(0.1) + deadres = not importthread.isAlive() + if deadres: break + + assert deadres # if false, importthread did not die soon enough + assert_equal(self.nodes[1].getbalance(), 0.0) + + # Import a different address and let it run + self.nodes[1].importprivkey(self.nodes[0].dumpprivkey(self.nodes[0].getnewaddress())) + # Expect original privkey to now also be discovered and added to balance + assert_equal(self.nodes[1].getbalance(), Decimal("0.123")) + +if __name__ == "__main__": + ImportAbortRescanTest().main() diff --git a/test/functional/maxblocksinflight.py b/test/functional/maxblocksinflight.py deleted file mode 100755 index 4ef2a35a44..0000000000 --- a/test/functional/maxblocksinflight.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2015-2016 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test nodes responses to having many blocks in flight. - -In this test we connect to one node over p2p, send it numerous inv's, and -compare the resulting number of getdata requests to a max allowed value. We -test for exceeding 128 blocks in flight, which was the limit an 0.9 client will -reach. [0.10 clients shouldn't request more than 16 from a single peer.] -""" - -from test_framework.mininode import * -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import * - -MAX_REQUESTS = 128 - -class TestManager(NodeConnCB): - def on_getdata(self, conn, message): - self.log.debug("got getdata %s" % repr(message)) - # Log the requests - for inv in message.inv: - if inv.hash not in self.blockReqCounts: - self.blockReqCounts[inv.hash] = 0 - self.blockReqCounts[inv.hash] += 1 - - def on_close(self, conn): - if not self.disconnectOkay: - raise EarlyDisconnectError(0) - - def add_new_connection(self, connection): - super().add_connection(connection) - self.blockReqCounts = {} - self.disconnectOkay = False - - def run(self): - self.connection.rpc.generate(1) # Leave IBD - - numBlocksToGenerate = [8, 16, 128, 1024] - for count in range(len(numBlocksToGenerate)): - current_invs = [] - for i in range(numBlocksToGenerate[count]): - current_invs.append(CInv(2, random.randrange(0, 1 << 256))) - if len(current_invs) >= 50000: - self.connection.send_message(msg_inv(current_invs)) - current_invs = [] - if len(current_invs) > 0: - self.connection.send_message(msg_inv(current_invs)) - - # Wait and see how many blocks were requested - time.sleep(2) - - total_requests = 0 - with mininode_lock: - for key in self.blockReqCounts: - total_requests += self.blockReqCounts[key] - if self.blockReqCounts[key] > 1: - raise AssertionError("Error, test failed: block %064x requested more than once" % key) - if total_requests > MAX_REQUESTS: - raise AssertionError("Error, too many blocks (%d) requested" % total_requests) - self.log.info("Round %d: success (total requests: %d)" % (count, total_requests)) - - self.disconnectOkay = True - self.connection.disconnect_node() - - -class MaxBlocksInFlightTest(BitcoinTestFramework): - def add_options(self, parser): - parser.add_option("--testbinary", dest="testbinary", - default=os.getenv("BITCOIND", "bitcoind"), - help="Binary to test max block requests behavior") - - def __init__(self): - super().__init__() - self.setup_clean_chain = True - self.num_nodes = 1 - - def setup_network(self): - self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, - extra_args=[['-whitelist=127.0.0.1']], - binary=[self.options.testbinary]) - - def run_test(self): - test = TestManager() - # pass log handler through to the test manager object - test.log = self.log - test.add_new_connection(NodeConn('127.0.0.1', p2p_port(0), self.nodes[0], test)) - NetworkThread().start() # Start up network handling in another thread - test.run() - -if __name__ == '__main__': - MaxBlocksInFlightTest().main() diff --git a/test/functional/net.py b/test/functional/net.py index e9463c7dc7..9eae140455 100755 --- a/test/functional/net.py +++ b/test/functional/net.py @@ -7,15 +7,15 @@ Tests correspond to code in rpc/net.cpp. """ -from decimal import Decimal import time from test_framework.test_framework import BitcoinTestFramework -from test_framework.authproxy import JSONRPCException from test_framework.util import ( assert_equal, - start_nodes, + assert_raises_jsonrpc, connect_nodes_bi, + p2p_port, + start_nodes, ) @@ -32,8 +32,41 @@ class NetTest(BitcoinTestFramework): self.sync_all() def run_test(self): + self._test_connection_count() + self._test_getnettotals() + self._test_getnetworkinginfo() + self._test_getaddednodeinfo() + + def _test_connection_count(self): + # connect_nodes_bi connects each node to the other + assert_equal(self.nodes[0].getconnectioncount(), 2) + + def _test_getnettotals(self): + # check that getnettotals totalbytesrecv and totalbytessent + # are consistent with getpeerinfo + peer_info = self.nodes[0].getpeerinfo() + assert_equal(len(peer_info), 2) + net_totals = self.nodes[0].getnettotals() + assert_equal(sum([peer['bytesrecv'] for peer in peer_info]), + net_totals['totalbytesrecv']) + assert_equal(sum([peer['bytessent'] for peer in peer_info]), + net_totals['totalbytessent']) + # test getnettotals and getpeerinfo by doing a ping + # the bytes sent/received should change + # note ping and pong are 32 bytes each + self.nodes[0].ping() + time.sleep(0.1) + peer_info_after_ping = self.nodes[0].getpeerinfo() + net_totals_after_ping = self.nodes[0].getnettotals() + for before, after in zip(peer_info, peer_info_after_ping): + assert_equal(before['bytesrecv_per_msg']['pong'] + 32, after['bytesrecv_per_msg']['pong']) + assert_equal(before['bytessent_per_msg']['ping'] + 32, after['bytessent_per_msg']['ping']) + assert_equal(net_totals['totalbytesrecv'] + 32*2, net_totals_after_ping['totalbytesrecv']) + assert_equal(net_totals['totalbytessent'] + 32*2, net_totals_after_ping['totalbytessent']) + + def _test_getnetworkinginfo(self): assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], True) - assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2) # bilateral connection + assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2) self.nodes[0].setnetworkactive(False) assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], False) @@ -49,6 +82,19 @@ class NetTest(BitcoinTestFramework): assert_equal(self.nodes[0].getnetworkinfo()['networkactive'], True) assert_equal(self.nodes[0].getnetworkinfo()['connections'], 2) + def _test_getaddednodeinfo(self): + assert_equal(self.nodes[0].getaddednodeinfo(), []) + # add a node (node2) to node0 + ip_port = "127.0.0.1:{}".format(p2p_port(2)) + self.nodes[0].addnode(ip_port, 'add') + # check that the node has indeed been added + added_nodes = self.nodes[0].getaddednodeinfo(ip_port) + assert_equal(len(added_nodes), 1) + assert_equal(added_nodes[0]['addednode'], ip_port) + # check that a non-existant node returns an error + assert_raises_jsonrpc(-24, "Node has not been added", + self.nodes[0].getaddednodeinfo, '1.1.1.1') + if __name__ == '__main__': NetTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index f7a501ab76..0996b1bc20 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -77,6 +77,7 @@ BASE_SCRIPTS= [ 'rawtransactions.py', 'reindex.py', # vv Tests less than 30s vv + "zmq_test.py", 'mempool_resurrect_test.py', 'txn_doublespend.py --mineblock', 'txn_clone.py', @@ -108,13 +109,9 @@ BASE_SCRIPTS= [ 'rpcnamedargs.py', 'listsinceblock.py', 'p2p-leaktests.py', + 'import-abort-rescan.py', ] -ZMQ_SCRIPTS = [ - # ZMQ test can only be run if bitcoin was built with zmq-enabled. - # call test_runner.py with -nozmq to explicitly exclude these tests. - 'zmq_test.py'] - EXTENDED_SCRIPTS = [ # These tests are not run by the travis build process. # Longest test should go first, to favor running tests in parallel @@ -143,13 +140,12 @@ EXTENDED_SCRIPTS = [ 'txn_clone.py --mineblock', 'forknotify.py', 'invalidateblock.py', - 'maxblocksinflight.py', 'p2p-acceptblock.py', 'replace-by-fee.py', ] # Place EXTENDED_SCRIPTS first since it has the 3 longest running tests -ALL_SCRIPTS = EXTENDED_SCRIPTS + BASE_SCRIPTS + ZMQ_SCRIPTS +ALL_SCRIPTS = EXTENDED_SCRIPTS + BASE_SCRIPTS NON_SCRIPTS = [ # These are python files that live in the functional tests directory, but are not test scripts. @@ -174,7 +170,6 @@ def main(): parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.') parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.') parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs') - parser.add_argument('--nozmq', action='store_true', help='do not run the zmq tests') args, unknown_args = parser.parse_known_args() # Create a set to store arguments and create the passon string @@ -192,7 +187,6 @@ def main(): enable_wallet = config["components"].getboolean("ENABLE_WALLET") enable_utils = config["components"].getboolean("ENABLE_UTILS") enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND") - enable_zmq = config["components"].getboolean("ENABLE_ZMQ") and not args.nozmq if config["environment"]["EXEEXT"] == ".exe" and not args.force: # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9 @@ -205,15 +199,6 @@ def main(): print("Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make") sys.exit(0) - # python3-zmq may not be installed. Handle this gracefully and with some helpful info - if enable_zmq: - try: - import zmq - except ImportError: - print("ERROR: \"import zmq\" failed. Use -nozmq to run without the ZMQ tests." - "To run zmq tests, see dependency info in /test/README.md.") - raise - # Build list of tests if tests: # Individual tests have been specified. Run specified tests that exist @@ -221,11 +206,9 @@ def main(): test_list = [t for t in ALL_SCRIPTS if (t in tests or re.sub(".py$", "", t) in tests)] else: - # No individual tests have been specified. Run base tests, and - # optionally ZMQ tests and extended tests. + # No individual tests have been specified. + # Run all base tests, and optionally run extended tests. test_list = BASE_SCRIPTS - if enable_zmq: - test_list += ZMQ_SCRIPTS if args.extended: # place the EXTENDED_SCRIPTS first since the three longest ones # are there and the list is shorter diff --git a/test/functional/wallet.py b/test/functional/wallet.py index 80f74fa108..558ffadfd4 100755 --- a/test/functional/wallet.py +++ b/test/functional/wallet.py @@ -6,7 +6,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -class WalletTest (BitcoinTestFramework): +class WalletTest(BitcoinTestFramework): def check_fee_amount(self, curr_balance, balance_with_fee, fee_per_byte, tx_size): """Return curr_balance after asserting the fee was in range""" @@ -28,7 +28,7 @@ class WalletTest (BitcoinTestFramework): self.is_network_split=False self.sync_all() - def run_test (self): + def run_test(self): # Check that there's no UTXO on none of the nodes assert_equal(len(self.nodes[0].listunspent()), 0) @@ -52,13 +52,33 @@ class WalletTest (BitcoinTestFramework): assert_equal(self.nodes[2].getbalance(), 0) # Check that only first and second nodes have UTXOs - assert_equal(len(self.nodes[0].listunspent()), 1) + utxos = self.nodes[0].listunspent() + assert_equal(len(utxos), 1) assert_equal(len(self.nodes[1].listunspent()), 1) assert_equal(len(self.nodes[2].listunspent()), 0) # Send 21 BTC from 0 to 2 using sendtoaddress call. self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 11) - self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 10) + mempool_txid = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 10) + + self.log.info("test gettxout") + # utxo spent in mempool should be visible if you exclude mempool + # but invisible if you include mempool + confirmed_txid, confirmed_index = utxos[0]["txid"], utxos[0]["vout"] + txout = self.nodes[0].gettxout(confirmed_txid, confirmed_index, False) + assert_equal(txout['value'], 50) + txout = self.nodes[0].gettxout(confirmed_txid, confirmed_index, True) + assert txout is None + # new utxo from mempool should be invisible if you exclude mempool + # but visible if you include mempool + txout = self.nodes[0].gettxout(mempool_txid, 0, False) + assert txout is None + txout1 = self.nodes[0].gettxout(mempool_txid, 0, True) + txout2 = self.nodes[0].gettxout(mempool_txid, 1, True) + # note the mempool tx will have randomly assigned indices + # but 10 will go to node2 and the rest will go to node0 + balance = self.nodes[0].getbalance() + assert_equal(set([txout1['value'], txout2['value']]), set([10, balance])) walletinfo = self.nodes[0].getwalletinfo() assert_equal(walletinfo['immature_balance'], 0) diff --git a/test/functional/zmq_test.py b/test/functional/zmq_test.py index 9e27b46381..891b609ffa 100755 --- a/test/functional/zmq_test.py +++ b/test/functional/zmq_test.py @@ -3,11 +3,13 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the ZMQ API.""" +import configparser +import os +import struct +import sys from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * -import zmq -import struct class ZMQTest (BitcoinTestFramework): @@ -18,6 +20,21 @@ class ZMQTest (BitcoinTestFramework): port = 28332 def setup_nodes(self): + # Try to import python3-zmq. Skip this test if the import fails. + try: + import zmq + except ImportError: + self.log.warning("python3-zmq module not available. Skipping zmq tests!") + sys.exit(self.TEST_EXIT_SKIPPED) + + # Check that bitcoin has been built with ZMQ enabled + config = configparser.ConfigParser() + config.read_file(open(os.path.dirname(__file__) + "/config.ini")) + + if not config["components"].getboolean("ENABLE_ZMQ"): + self.log.warning("bitcoind has not been built with zmq enabled. Skipping zmq tests!") + sys.exit(self.TEST_EXIT_SKIPPED) + self.zmqContext = zmq.Context() self.zmqSubSocket = self.zmqContext.socket(zmq.SUB) self.zmqSubSocket.setsockopt(zmq.SUBSCRIBE, b"hashblock") diff --git a/test/util/data/blanktxv1.json b/test/util/data/blanktxv1.json index 51c25a5a98..9fe2de649b 100644 --- a/test/util/data/blanktxv1.json +++ b/test/util/data/blanktxv1.json @@ -2,6 +2,8 @@ "txid": "d21633ba23f70118185227be58a63527675641ad37967e2aa461559f577aec43", "hash": "d21633ba23f70118185227be58a63527675641ad37967e2aa461559f577aec43", "version": 1, + "size": 10, + "vsize": 10, "locktime": 0, "vin": [ ], diff --git a/test/util/data/blanktxv2.json b/test/util/data/blanktxv2.json index 266919f445..e97626e421 100644 --- a/test/util/data/blanktxv2.json +++ b/test/util/data/blanktxv2.json @@ -2,6 +2,8 @@ "txid": "4ebd325a4b394cff8c57e8317ccf5a8d0e2bdf1b8526f8aad6c8e43d8240621a", "hash": "4ebd325a4b394cff8c57e8317ccf5a8d0e2bdf1b8526f8aad6c8e43d8240621a", "version": 2, + "size": 10, + "vsize": 10, "locktime": 0, "vin": [ ], diff --git a/test/util/data/tt-delin1-out.json b/test/util/data/tt-delin1-out.json index 712a2c27f8..f6dfbb51cc 100644 --- a/test/util/data/tt-delin1-out.json +++ b/test/util/data/tt-delin1-out.json @@ -2,6 +2,8 @@ "txid": "81b2035be1da1abe745c6141174a73d151009ec17b3d5ebffa2e177408c50dfd", "hash": "81b2035be1da1abe745c6141174a73d151009ec17b3d5ebffa2e177408c50dfd", "version": 1, + "size": 3040, + "vsize": 3040, "locktime": 0, "vin": [ { diff --git a/test/util/data/tt-delout1-out.json b/test/util/data/tt-delout1-out.json index afc4e95762..6769ed79ff 100644 --- a/test/util/data/tt-delout1-out.json +++ b/test/util/data/tt-delout1-out.json @@ -2,6 +2,8 @@ "txid": "c46ccd75b5050e942b2e86a3648f843f525fe6fc000bf0534ba5973063354493", "hash": "c46ccd75b5050e942b2e86a3648f843f525fe6fc000bf0534ba5973063354493", "version": 1, + "size": 3155, + "vsize": 3155, "locktime": 0, "vin": [ { diff --git a/test/util/data/tt-locktime317000-out.json b/test/util/data/tt-locktime317000-out.json index 2b9075f8ac..82b64df075 100644 --- a/test/util/data/tt-locktime317000-out.json +++ b/test/util/data/tt-locktime317000-out.json @@ -2,6 +2,8 @@ "txid": "aded538f642c17e15f4d3306b8be7e1a4d1ae0c4616d641ab51ea09ba65e5cb5", "hash": "aded538f642c17e15f4d3306b8be7e1a4d1ae0c4616d641ab51ea09ba65e5cb5", "version": 1, + "size": 3189, + "vsize": 3189, "locktime": 317000, "vin": [ { diff --git a/test/util/data/txcreate1.json b/test/util/data/txcreate1.json index f83e036f33..36741044c9 100644 --- a/test/util/data/txcreate1.json +++ b/test/util/data/txcreate1.json @@ -2,6 +2,8 @@ "txid": "fe7d174f42dce0cffa7a527e9bc8368956057619ec817648f6138b98f2533e8f", "hash": "fe7d174f42dce0cffa7a527e9bc8368956057619ec817648f6138b98f2533e8f", "version": 2, + "size": 201, + "vsize": 201, "locktime": 0, "vin": [ { diff --git a/test/util/data/txcreate2.json b/test/util/data/txcreate2.json index fb5e177db7..23fe7ace67 100644 --- a/test/util/data/txcreate2.json +++ b/test/util/data/txcreate2.json @@ -2,6 +2,8 @@ "txid": "0481afb29931341d0d7861d8a2f6f26456fa042abf54a23e96440ed7946e0715", "hash": "0481afb29931341d0d7861d8a2f6f26456fa042abf54a23e96440ed7946e0715", "version": 2, + "size": 19, + "vsize": 19, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreatedata1.json b/test/util/data/txcreatedata1.json index 760518d30a..e65a1859eb 100644 --- a/test/util/data/txcreatedata1.json +++ b/test/util/data/txcreatedata1.json @@ -2,6 +2,8 @@ "txid": "07894b4d12fe7853dd911402db1620920d261b9627c447f931417d330c25f06e", "hash": "07894b4d12fe7853dd911402db1620920d261b9627c447f931417d330c25f06e", "version": 1, + "size": 176, + "vsize": 176, "locktime": 0, "vin": [ { diff --git a/test/util/data/txcreatedata2.json b/test/util/data/txcreatedata2.json index 3c6da40f90..8f1544e1c0 100644 --- a/test/util/data/txcreatedata2.json +++ b/test/util/data/txcreatedata2.json @@ -2,6 +2,8 @@ "txid": "c14b007fa3a6c1e7765919c1d14c1cfc2b8642c3a5d3be4b1fa8c4ccfec98bb0", "hash": "c14b007fa3a6c1e7765919c1d14c1cfc2b8642c3a5d3be4b1fa8c4ccfec98bb0", "version": 2, + "size": 176, + "vsize": 176, "locktime": 0, "vin": [ { diff --git a/test/util/data/txcreatedata_seq0.json b/test/util/data/txcreatedata_seq0.json index d272a4c447..e52401f418 100644 --- a/test/util/data/txcreatedata_seq0.json +++ b/test/util/data/txcreatedata_seq0.json @@ -2,6 +2,8 @@ "txid": "8df6ed527472542dd5e137c242a7c5a9f337ac34f7b257ae4af886aeaebb51b0", "hash": "8df6ed527472542dd5e137c242a7c5a9f337ac34f7b257ae4af886aeaebb51b0", "version": 2, + "size": 85, + "vsize": 85, "locktime": 0, "vin": [ { diff --git a/test/util/data/txcreatedata_seq1.json b/test/util/data/txcreatedata_seq1.json index d323255418..093ff4a56b 100644 --- a/test/util/data/txcreatedata_seq1.json +++ b/test/util/data/txcreatedata_seq1.json @@ -2,6 +2,8 @@ "txid": "c4dea671b0d7b48f8ab10bc46650e8329d3c5766931f548f513847a19f5ba75b", "hash": "c4dea671b0d7b48f8ab10bc46650e8329d3c5766931f548f513847a19f5ba75b", "version": 1, + "size": 126, + "vsize": 126, "locktime": 0, "vin": [ { diff --git a/test/util/data/txcreatemultisig1.json b/test/util/data/txcreatemultisig1.json index f6ce43c202..0cc530836a 100644 --- a/test/util/data/txcreatemultisig1.json +++ b/test/util/data/txcreatemultisig1.json @@ -2,6 +2,8 @@ "txid": "0d1d4edfc217d9db3ab6a9298f26a52eae3c52f55a6cb8ccbc14f7c727572894", "hash": "0d1d4edfc217d9db3ab6a9298f26a52eae3c52f55a6cb8ccbc14f7c727572894", "version": 1, + "size": 124, + "vsize": 124, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreatemultisig2.json b/test/util/data/txcreatemultisig2.json index e09d22060f..8ad2ffdc65 100644 --- a/test/util/data/txcreatemultisig2.json +++ b/test/util/data/txcreatemultisig2.json @@ -2,6 +2,8 @@ "txid": "0d861f278a3b7bce7cb5a88d71e6e6a903336f95ad5a2c29b295b63835b6eee3", "hash": "0d861f278a3b7bce7cb5a88d71e6e6a903336f95ad5a2c29b295b63835b6eee3", "version": 1, + "size": 42, + "vsize": 42, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreatemultisig3.json b/test/util/data/txcreatemultisig3.json index 88e32bd310..086bf44b8a 100644 --- a/test/util/data/txcreatemultisig3.json +++ b/test/util/data/txcreatemultisig3.json @@ -2,6 +2,8 @@ "txid": "ccc552220b46a3b5140048b03395987ce4f0fa1ddf8c635bba1fa44e0f8c1d7f", "hash": "ccc552220b46a3b5140048b03395987ce4f0fa1ddf8c635bba1fa44e0f8c1d7f", "version": 1, + "size": 53, + "vsize": 53, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreatemultisig4.json b/test/util/data/txcreatemultisig4.json index fc69c7269c..d23ccc045e 100644 --- a/test/util/data/txcreatemultisig4.json +++ b/test/util/data/txcreatemultisig4.json @@ -2,6 +2,8 @@ "txid": "5e8b1cc73234e208d4b7ca9075f136b908c34101be7a048df4ba9ac758b61567", "hash": "5e8b1cc73234e208d4b7ca9075f136b908c34101be7a048df4ba9ac758b61567", "version": 1, + "size": 42, + "vsize": 42, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreateoutpubkey1.json b/test/util/data/txcreateoutpubkey1.json index 6019fa2dcd..f10aaecf7a 100644 --- a/test/util/data/txcreateoutpubkey1.json +++ b/test/util/data/txcreateoutpubkey1.json @@ -2,6 +2,8 @@ "txid": "f42b38ac12e3fafc96ba1a9ba70cbfe326744aef75df5fb9db5d6e2855ca415f", "hash": "f42b38ac12e3fafc96ba1a9ba70cbfe326744aef75df5fb9db5d6e2855ca415f", "version": 1, + "size": 54, + "vsize": 54, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreateoutpubkey2.json b/test/util/data/txcreateoutpubkey2.json index 6fc3d57527..5a473b76c3 100644 --- a/test/util/data/txcreateoutpubkey2.json +++ b/test/util/data/txcreateoutpubkey2.json @@ -2,6 +2,8 @@ "txid": "70f2a088cde460e677415fa1fb71895e90c231e6ed38ed203a35b6f848e9cc73", "hash": "70f2a088cde460e677415fa1fb71895e90c231e6ed38ed203a35b6f848e9cc73", "version": 1, + "size": 41, + "vsize": 41, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreateoutpubkey3.json b/test/util/data/txcreateoutpubkey3.json index a1a25fc834..b8389b8f7e 100644 --- a/test/util/data/txcreateoutpubkey3.json +++ b/test/util/data/txcreateoutpubkey3.json @@ -2,6 +2,8 @@ "txid": "bfc7e898ee9f6a9652d7b8cca147e2da134502e2ada0f279ed634fc8cf833f8c", "hash": "bfc7e898ee9f6a9652d7b8cca147e2da134502e2ada0f279ed634fc8cf833f8c", "version": 1, + "size": 42, + "vsize": 42, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreatescript1.json b/test/util/data/txcreatescript1.json index 8ffecba411..823168e9fb 100644 --- a/test/util/data/txcreatescript1.json +++ b/test/util/data/txcreatescript1.json @@ -2,6 +2,8 @@ "txid": "f0851b68202f736b792649cfc960259c2374badcb644ab20cac726b5f72f61c9", "hash": "f0851b68202f736b792649cfc960259c2374badcb644ab20cac726b5f72f61c9", "version": 1, + "size": 20, + "vsize": 20, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreatescript2.json b/test/util/data/txcreatescript2.json index 41eb69f1af..d4c7e10c78 100644 --- a/test/util/data/txcreatescript2.json +++ b/test/util/data/txcreatescript2.json @@ -2,6 +2,8 @@ "txid": "6e07a7cc075e0703f32ee8c4e5373fe654bfbc315148fda364e1be286ff290d0", "hash": "6e07a7cc075e0703f32ee8c4e5373fe654bfbc315148fda364e1be286ff290d0", "version": 1, + "size": 42, + "vsize": 42, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreatescript3.json b/test/util/data/txcreatescript3.json index 90e7e27f9f..001e69511f 100644 --- a/test/util/data/txcreatescript3.json +++ b/test/util/data/txcreatescript3.json @@ -2,6 +2,8 @@ "txid": "8a234037b088e987c877030efc83374a07441c321bf9dc6dd2f206bc26507df8", "hash": "8a234037b088e987c877030efc83374a07441c321bf9dc6dd2f206bc26507df8", "version": 1, + "size": 53, + "vsize": 53, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreatescript4.json b/test/util/data/txcreatescript4.json index 11783751a4..20094bcd44 100644 --- a/test/util/data/txcreatescript4.json +++ b/test/util/data/txcreatescript4.json @@ -2,6 +2,8 @@ "txid": "24225cf5e9391100d6b218134b9f03383ca4c880a1f634ac12990cf28b66adbc", "hash": "24225cf5e9391100d6b218134b9f03383ca4c880a1f634ac12990cf28b66adbc", "version": 1, + "size": 42, + "vsize": 42, "locktime": 0, "vin": [ ], diff --git a/test/util/data/txcreatesignv1.json b/test/util/data/txcreatesignv1.json index ff39e71b40..519d3ab066 100644 --- a/test/util/data/txcreatesignv1.json +++ b/test/util/data/txcreatesignv1.json @@ -2,6 +2,8 @@ "txid": "977e7cd286cb72cd470d539ba6cb48400f8f387d97451d45cdb8819437a303af", "hash": "977e7cd286cb72cd470d539ba6cb48400f8f387d97451d45cdb8819437a303af", "version": 1, + "size": 224, + "vsize": 224, "locktime": 0, "vin": [ { |