diff options
29 files changed, 500 insertions, 282 deletions
diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index bb6ffb0253..f1b6a12fd0 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -70,6 +70,14 @@ def ask_prompt(text): print("",file=stderr) return reply +def get_symlink_files(): + files = sorted(subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', 'HEAD']).splitlines()) + ret = [] + for f in files: + if (int(f.decode('utf-8').split(" ")[0], 8) & 0o170000) == 0o120000: + ret.append(f.decode('utf-8').split("\t")[1]) + return ret + def tree_sha512sum(): files = sorted(subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', '--name-only', 'HEAD']).splitlines()) overall = hashlib.sha512() @@ -200,6 +208,12 @@ def main(): print("ERROR: Creating merge failed (already merged?).",file=stderr) exit(4) + symlink_files = get_symlink_files() + for f in symlink_files: + print("ERROR: File %s was a symlink" % f) + if len(symlink_files) > 0: + exit(4) + # Put tree SHA512 into the message try: first_sha512 = tree_sha512sum() @@ -212,10 +226,6 @@ def main(): except subprocess.CalledProcessError as e: printf("ERROR: Cannot update message.",file=stderr) exit(4) - second_sha512 = tree_sha512sum() - if first_sha512 != second_sha512: - print("ERROR: Tree hash changed unexpectedly",file=stderr) - exit(4) print('%s#%s%s %s %sinto %s%s' % (ATTR_RESET+ATTR_PR,pull,ATTR_RESET,title,ATTR_RESET+ATTR_PR,branch,ATTR_RESET)) subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch]) @@ -258,6 +268,11 @@ def main(): print("ERROR: Merge rejected.",file=stderr) exit(7) + second_sha512 = tree_sha512sum() + if first_sha512 != second_sha512: + print("ERROR: Tree hash changed unexpectedly",file=stderr) + exit(8) + # Sign the merge commit. reply = ask_prompt("Type 's' to sign off on the merge.") if reply == 's': diff --git a/contrib/verify-commits/gpg.sh b/contrib/verify-commits/gpg.sh index 09ff237544..b01e2a6d39 100755 --- a/contrib/verify-commits/gpg.sh +++ b/contrib/verify-commits/gpg.sh @@ -8,21 +8,43 @@ VALID=false REVSIG=false IFS=' ' -for LINE in $(echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null); do +if [ "$BITCOIN_VERIFY_COMMITS_ALLOW_SHA1" = 1 ]; then + GPG_RES="$(echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null)" +else + # Note how we've disabled SHA1 with the --weak-digest option, disabling + # signatures - including selfsigs - that use SHA1. While you might think that + # collision attacks shouldn't be an issue as they'd be an attack on yourself, + # in fact because what's being signed is a commit object that's + # semi-deterministically generated by untrusted input (the pull-req) in theory + # an attacker could construct a pull-req that results in a commit object that + # they've created a collision for. Not the most likely attack, but preventing + # it is pretty easy so we do so as a "belt-and-suspenders" measure. + GPG_RES="" + for LINE in "$(gpg --version)"; do + case "$LINE" in + "gpg (GnuPG) 1.4.1"*|"gpg (GnuPG) 2.0."*) + echo "Please upgrade to at least gpg 2.1.10 to check for weak signatures" > /dev/stderr + GPG_RES="$(echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null)" + ;; + # We assume if you're running 2.1+, you're probably running 2.1.10+ + # gpg will fail otherwise + # We assume if you're running 1.X, it is either 1.4.1X or 1.4.20+ + # gpg will fail otherwise + esac + done + [ "$GPG_RES" = "" ] && GPG_RES="$(echo "$INPUT" | gpg --trust-model always --weak-digest sha1 "$@" 2>/dev/null)" +fi +for LINE in $(echo "$GPG_RES"); do case "$LINE" in "[GNUPG:] VALIDSIG "*) while read KEY; do - case "$LINE" in "[GNUPG:] VALIDSIG $KEY "*) VALID=true;; esac + [ "${LINE#?GNUPG:? VALIDSIG * * * * * * * * * }" = "$KEY" ] && VALID=true done < ./contrib/verify-commits/trusted-keys ;; "[GNUPG:] REVKEYSIG "*) [ "$BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG" != 1 ] && exit 1 - while read KEY; do - case "$LINE" in "[GNUPG:] REVKEYSIG ${KEY#????????????????????????} "*) - REVSIG=true - GOODREVSIG="[GNUPG:] GOODSIG ${KEY#????????????????????????} " - esac - done < ./contrib/verify-commits/trusted-keys + REVSIG=true + GOODREVSIG="[GNUPG:] GOODSIG ${LINE#* * *}" ;; esac done @@ -30,7 +52,7 @@ if ! $VALID; then exit 1 fi if $VALID && $REVSIG; then - echo "$INPUT" | gpg --trust-model always "$@" | grep "\[GNUPG:\] \(NEWSIG\|SIG_ID\|VALIDSIG\)" 2>/dev/null + echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null | grep "\[GNUPG:\] \(NEWSIG\|SIG_ID\|VALIDSIG\)" echo "$GOODREVSIG" else echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null diff --git a/contrib/verify-commits/trusted-keys b/contrib/verify-commits/trusted-keys index 75242c2a97..5610692616 100644 --- a/contrib/verify-commits/trusted-keys +++ b/contrib/verify-commits/trusted-keys @@ -1,4 +1,4 @@ 71A3B16735405025D447E8F274810B012346C9A6 -3F1888C6DCA92A6499C4911FDBA1A67379A1A931 +133EAC179436F14A5CF1B794860FEB804E669320 32EE5C4C3FA15CCADB46ABE529D4BCB6416F53EC -FE09B823E6D83A3BC7983EAA2D7F2372E50FE137 +B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B diff --git a/contrib/verify-commits/trusted-sha512-root-commit b/contrib/verify-commits/trusted-sha512-root-commit new file mode 100644 index 0000000000..c28f50ff78 --- /dev/null +++ b/contrib/verify-commits/trusted-sha512-root-commit @@ -0,0 +1 @@ +f7ec7cfd38b543ba81ac7bed5b77f9a19739460b diff --git a/contrib/verify-commits/verify-commits.sh b/contrib/verify-commits/verify-commits.sh index b2cebdf1a0..40c9341445 100755 --- a/contrib/verify-commits/verify-commits.sh +++ b/contrib/verify-commits/verify-commits.sh @@ -9,7 +9,10 @@ DIR=$(dirname "$0") [ "/${DIR#/}" != "$DIR" ] && DIR=$(dirname "$(pwd)/$0") +echo "Using verify-commits data from ${DIR}" + VERIFIED_ROOT=$(cat "${DIR}/trusted-git-root") +VERIFIED_SHA512_ROOT=$(cat "${DIR}/trusted-sha512-root-commit") REVSIG_ALLOWED=$(cat "${DIR}/allow-revsig-commits") HAVE_FAILED=false @@ -17,18 +20,75 @@ IS_SIGNED () { if [ $1 = $VERIFIED_ROOT ]; then return 0; fi + + VERIFY_TREE=$2 + NO_SHA1=$3 + if [ $1 = $VERIFIED_SHA512_ROOT ]; then + if [ "$VERIFY_TREE" = "1" ]; then + echo "All Tree-SHA512s matched up to $VERIFIED_SHA512_ROOT" > /dev/stderr + fi + VERIFY_TREE=0 + NO_SHA1=0 + fi + + if [ "$NO_SHA1" = "1" ]; then + export BITCOIN_VERIFY_COMMITS_ALLOW_SHA1=0 + else + export BITCOIN_VERIFY_COMMITS_ALLOW_SHA1=1 + fi + if [ "${REVSIG_ALLOWED#*$1}" != "$REVSIG_ALLOWED" ]; then export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=1 else export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=0 fi - if ! git -c "gpg.program=${DIR}/gpg.sh" verify-commit $1 > /dev/null 2>&1; then + + if ! git -c "gpg.program=${DIR}/gpg.sh" verify-commit $1 > /dev/null; then return 1; fi + + # We set $4 to 1 on the first call, always verifying the top of the tree + if [ "$VERIFY_TREE" = 1 -o "$4" = "1" ]; then + IFS_CACHE="$IFS" + IFS=' +' + for LINE in $(git ls-tree --full-tree -r $1); do + case "$LINE" in + "12"*) + echo "Repo contains symlinks" > /dev/stderr + IFS="$IFS_CACHE" + return 1 + ;; + esac + done + IFS="$IFS_CACHE" + + FILE_HASHES="" + for FILE in $(git ls-tree --full-tree -r --name-only $1 | LC_ALL=C sort); do + HASH=$(git cat-file blob $1:"$FILE" | sha512sum | { read FIRST OTHER; echo $FIRST; } ) + [ "$FILE_HASHES" != "" ] && FILE_HASHES="$FILE_HASHES"' +' + FILE_HASHES="$FILE_HASHES$HASH $FILE" + done + HASH_MATCHES=0 + MSG="$(git show -s --format=format:%B $1 | tail -n1)" + + case "$MSG -" in + "Tree-SHA512: $(echo "$FILE_HASHES" | sha512sum)") + HASH_MATCHES=1;; + esac + + if [ "$HASH_MATCHES" = "0" ]; then + echo "Tree-SHA512 did not match for commit $1" > /dev/stderr + HAVE_FAILED=true + return 1 + fi + fi + local PARENTS PARENTS=$(git show -s --format=format:%P $1) for PARENT in $PARENTS; do - if IS_SIGNED $PARENT; then + if IS_SIGNED $PARENT $VERIFY_TREE $NO_SHA1 0; then return 0; fi break @@ -50,7 +110,12 @@ else TEST_COMMIT="$1" fi -IS_SIGNED "$TEST_COMMIT" +DO_CHECKOUT_TEST=0 +if [ x"$2" = "x--tree-checks" ]; then + DO_CHECKOUT_TEST=1 +fi + +IS_SIGNED "$TEST_COMMIT" "$DO_CHECKOUT_TEST" 1 1 RES=$? if [ "$RES" = 1 ]; then if ! "$HAVE_FAILED"; then diff --git a/qa/rpc-tests/bip68-sequence.py b/qa/rpc-tests/bip68-sequence.py index 74ac393fe9..e516ed2ec3 100755 --- a/qa/rpc-tests/bip68-sequence.py +++ b/qa/rpc-tests/bip68-sequence.py @@ -46,14 +46,12 @@ class BIP68Test(BitcoinTestFramework): print("Running test BIP68 not consensus before versionbits activation") self.test_bip68_not_consensus() - print("Verifying nVersion=2 transactions aren't standard") - self.test_version2_relay(before_activation=True) - print("Activating BIP68 (and 112/113)") self.activateCSV() - print("Verifying nVersion=2 transactions are now standard") - self.test_version2_relay(before_activation=False) + print("Verifying nVersion=2 transactions are standard.") + print("Note that with current versions of bitcoin software, nVersion=2 transactions are always standard (independent of BIP68 activation status).") + self.test_version2_relay() print("Passed\n") @@ -403,8 +401,8 @@ class BIP68Test(BitcoinTestFramework): assert(get_bip9_status(self.nodes[0], 'csv')['status'] == 'active') sync_blocks(self.nodes) - # Use self.nodes[1] to test standardness relay policy - def test_version2_relay(self, before_activation): + # Use self.nodes[1] to test that version 2 transactions are standard. + def test_version2_relay(self): inputs = [ ] outputs = { self.nodes[1].getnewaddress() : 1.0 } rawtx = self.nodes[1].createrawtransaction(inputs, outputs) @@ -412,12 +410,7 @@ class BIP68Test(BitcoinTestFramework): tx = FromHex(CTransaction(), rawtxfund) tx.nVersion = 2 tx_signed = self.nodes[1].signrawtransaction(ToHex(tx))["hex"] - try: - tx_id = self.nodes[1].sendrawtransaction(tx_signed) - assert(before_activation == False) - except: - assert(before_activation) - + tx_id = self.nodes[1].sendrawtransaction(tx_signed) if __name__ == '__main__': BIP68Test().main() diff --git a/qa/rpc-tests/import-rescan.py b/qa/rpc-tests/import-rescan.py index 7ca61824c9..64e0eec61e 100755 --- a/qa/rpc-tests/import-rescan.py +++ b/qa/rpc-tests/import-rescan.py @@ -56,7 +56,7 @@ class Variant(collections.namedtuple("Variant", "call data rescan prune")): "scriptPubKey": { "address": self.address["address"] }, - "timestamp": timestamp + RESCAN_WINDOW + (1 if self.rescan == Rescan.late_timestamp else 0), + "timestamp": timestamp + TIMESTAMP_WINDOW + (1 if self.rescan == Rescan.late_timestamp else 0), "pubkeys": [self.address["pubkey"]] if self.data == Data.pub else [], "keys": [self.key] if self.data == Data.priv else [], "label": self.label, @@ -108,7 +108,7 @@ ImportNode = collections.namedtuple("ImportNode", "prune rescan") IMPORT_NODES = [ImportNode(*fields) for fields in itertools.product((False, True), repeat=2)] # Rescans start at the earliest block up to 2 hours before the key timestamp. -RESCAN_WINDOW = 2 * 60 * 60 +TIMESTAMP_WINDOW = 2 * 60 * 60 class ImportRescanTest(BitcoinTestFramework): @@ -141,7 +141,7 @@ class ImportRescanTest(BitcoinTestFramework): self.nodes[0].generate(1) assert_equal(self.nodes[0].getrawmempool(), []) timestamp = self.nodes[0].getblockheader(self.nodes[0].getbestblockhash())["time"] - set_node_times(self.nodes, timestamp + RESCAN_WINDOW + 1) + set_node_times(self.nodes, timestamp + TIMESTAMP_WINDOW + 1) self.nodes[0].generate(1) sync_blocks(self.nodes) diff --git a/qa/rpc-tests/pruning.py b/qa/rpc-tests/pruning.py index ace8ced422..d4924d058c 100755 --- a/qa/rpc-tests/pruning.py +++ b/qa/rpc-tests/pruning.py @@ -19,7 +19,7 @@ MIN_BLOCKS_TO_KEEP = 288 # Rescans start at the earliest block up to 2 hours before a key timestamp, so # the manual prune RPC avoids pruning blocks in the same window to be # compatible with pruning based on key creation time. -RESCAN_WINDOW = 2 * 60 * 60 +TIMESTAMP_WINDOW = 2 * 60 * 60 def calc_usage(blockdir): @@ -242,7 +242,7 @@ class PruneTest(BitcoinTestFramework): def height(index): if use_timestamp: - return node.getblockheader(node.getblockhash(index))["time"] + RESCAN_WINDOW + return node.getblockheader(node.getblockhash(index))["time"] + TIMESTAMP_WINDOW else: return index diff --git a/qa/rpc-tests/segwit.py b/qa/rpc-tests/segwit.py index f475427842..761c00dc01 100755 --- a/qa/rpc-tests/segwit.py +++ b/qa/rpc-tests/segwit.py @@ -248,20 +248,9 @@ class SegWitTest(BitcoinTestFramework): assert(tmpl['transactions'][0]['txid'] == txid) assert(tmpl['transactions'][0]['sigops'] == 8) - print("Verify non-segwit miners get a valid GBT response after the fork") + print("Non-segwit miners are not able to use GBT response after activation.") send_to_witness(1, self.nodes[0], find_unspent(self.nodes[0], 50), self.pubkey[0], False, Decimal("49.998")) - try: - tmpl = self.nodes[0].getblocktemplate({}) - assert(len(tmpl['transactions']) == 1) # Doesn't include witness tx - assert(tmpl['sizelimit'] == 1000000) - assert('weightlimit' not in tmpl) - assert(tmpl['sigoplimit'] == 20000) - assert(tmpl['transactions'][0]['hash'] == txid) - assert(tmpl['transactions'][0]['sigops'] == 2) - assert(('!segwit' in tmpl['rules']) or ('segwit' not in tmpl['rules'])) - except JSONRPCException: - # This is an acceptable outcome - pass + assert_raises_jsonrpc(-8, "Support for 'segwit' rule requires explicit client support", self.nodes[0].getblocktemplate, {}) print("Verify behaviour of importaddress, addwitnessaddress and listunspent") diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py index e838a40582..95619dda26 100644 --- a/qa/rpc-tests/test_framework/util.py +++ b/qa/rpc-tests/test_framework/util.py @@ -15,6 +15,7 @@ import http.client import random import shutil import subprocess +import tempfile import time import re import errno @@ -305,27 +306,7 @@ def initialize_chain_clean(test_dir, num_nodes): datadir=initialize_datadir(test_dir, i) -def _rpchost_to_args(rpchost): - '''Convert optional IP:port spec to rpcconnect/rpcport args''' - if rpchost is None: - return [] - - match = re.match('(\[[0-9a-fA-f:]+\]|[^:]+)(?::([0-9]+))?$', rpchost) - if not match: - raise ValueError('Invalid RPC host spec ' + rpchost) - - rpcconnect = match.group(1) - rpcport = match.group(2) - - if rpcconnect.startswith('['): # remove IPv6 [...] wrapping - rpcconnect = rpcconnect[1:-1] - - rv = ['-rpcconnect=' + rpcconnect] - if rpcport: - rv += ['-rpcport=' + rpcport] - return rv - -def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None): +def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=None, stderr=None): """ Start a bitcoind and return RPC connection to it """ @@ -334,7 +315,7 @@ def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary= binary = os.getenv("BITCOIND", "bitcoind") args = [ binary, "-datadir="+datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-mocktime="+str(get_mocktime()) ] if extra_args is not None: args.extend(extra_args) - bitcoind_processes[i] = subprocess.Popen(args) + bitcoind_processes[i] = subprocess.Popen(args, stderr=stderr) if os.getenv("PYTHON_DEBUG", ""): print("start_node: bitcoind started, waiting for RPC to come up") url = rpc_url(i, rpchost) @@ -348,6 +329,25 @@ def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary= return proxy +def assert_start_raises_init_error(i, dirname, extra_args=None, expected_msg=None): + with tempfile.SpooledTemporaryFile(max_size=2**16) as log_stderr: + try: + node = start_node(i, dirname, extra_args, stderr=log_stderr) + stop_node(node, i) + except Exception as e: + assert 'bitcoind exited' in str(e) #node must have shutdown + if expected_msg is not None: + log_stderr.seek(0) + stderr = log_stderr.read().decode('utf-8') + if expected_msg not in stderr: + raise AssertionError("Expected error \"" + expected_msg + "\" not found in:\n" + stderr) + else: + if expected_msg is None: + assert_msg = "bitcoind should have exited with an error" + else: + assert_msg = "bitcoind should have exited with expected error " + expected_msg + raise AssertionError(assert_msg) + def start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, timewait=None, binary=None): """ Start multiple bitcoinds, return RPC connections to them diff --git a/qa/rpc-tests/wallet-hd.py b/qa/rpc-tests/wallet-hd.py index bf07590098..1dcfe5300f 100755 --- a/qa/rpc-tests/wallet-hd.py +++ b/qa/rpc-tests/wallet-hd.py @@ -10,6 +10,7 @@ from test_framework.util import ( start_node, assert_equal, connect_nodes_bi, + assert_start_raises_init_error ) import os import shutil @@ -31,6 +32,12 @@ class WalletHDTest(BitcoinTestFramework): def run_test (self): tmpdir = self.options.tmpdir + # Make sure can't switch off usehd after wallet creation + self.stop_node(1) + assert_start_raises_init_error(1, self.options.tmpdir, ['-usehd=0'], 'already existing HD wallet') + self.nodes[1] = start_node(1, self.options.tmpdir, self.node_args[1]) + connect_nodes_bi(self.nodes, 0, 1) + # Make sure we use hd, keep masterkeyid masterkeyid = self.nodes[1].getwalletinfo()['hdmasterkeyid'] assert_equal(len(masterkeyid), 40) diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 3c9df4f713..b0df3d2b04 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -92,6 +92,8 @@ bool benchmark::State::KeepRunning() --count; + assert(count != 0 && "count == 0 => (now == 0 && beginTime == 0) => return above"); + // Output results double average = (now-beginTime)/count; int64_t averageCycles = (nowCycles-beginCycles)/count; diff --git a/src/chain.h b/src/chain.h index 4fd5aed93b..de120d2d75 100644 --- a/src/chain.h +++ b/src/chain.h @@ -14,6 +14,20 @@ #include <vector> +/** + * Maximum amount of time that a block timestamp is allowed to exceed the + * current network-adjusted time before the block will be accepted. + */ +static const int64_t MAX_FUTURE_BLOCK_TIME = 2 * 60 * 60; + +/** + * Timestamp window used as a grace period by code that compares external + * timestamps (such as timestamps passed to RPCs, or wallet key creation times) + * to block timestamps. This should be set at least as high as + * MAX_FUTURE_BLOCK_TIME. + */ +static const int64_t TIMESTAMP_WINDOW = MAX_FUTURE_BLOCK_TIME; + class CBlockFileInfo { public: diff --git a/src/init.cpp b/src/init.cpp index a311ee7d8c..280793c72b 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1639,7 +1639,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler) #ifdef ENABLE_WALLET if (pwalletMain) - pwalletMain->postInitProcess(threadGroup); + pwalletMain->postInitProcess(scheduler); #endif return !fRequestShutdown; diff --git a/src/net.cpp b/src/net.cpp index e38d3b344e..6ff63d4730 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -2288,7 +2288,7 @@ bool CConnman::Start(CScheduler& scheduler, std::string& strNodeError, Options c threadMessageHandler = std::thread(&TraceThread<std::function<void()> >, "msghand", std::function<void()>(std::bind(&CConnman::ThreadMessageHandler, this))); // Dump network addresses - scheduler.scheduleEvery(boost::bind(&CConnman::DumpData, this), DUMP_ADDRESSES_INTERVAL); + scheduler.scheduleEvery(std::bind(&CConnman::DumpData, this), DUMP_ADDRESSES_INTERVAL * 1000); return true; } diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index 790bc71d14..28ef1fb464 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -69,6 +69,9 @@ uint256 CTransaction::ComputeHash() const uint256 CTransaction::GetWitnessHash() const { + if (!HasWitness()) { + return GetHash(); + } return SerializeHash(*this, SER_GETHASH, 0); } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 6826ce4a79..dd46a3c3ba 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -846,7 +846,7 @@ UniValue pruneblockchain(const JSONRPCRequest& request) // too low to be a block time (corresponds to timestamp from Sep 2001). if (heightParam > 1000000000) { // Add a 2 hour buffer to include blocks which might have had old timestamps - CBlockIndex* pindex = chainActive.FindEarliestAtLeast(heightParam - 7200); + CBlockIndex* pindex = chainActive.FindEarliestAtLeast(heightParam - TIMESTAMP_WINDOW); if (!pindex) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Could not find block with at least the specified timestamp."); } diff --git a/src/scheduler.cpp b/src/scheduler.cpp index b01170074d..861c1a0220 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -104,20 +104,20 @@ void CScheduler::schedule(CScheduler::Function f, boost::chrono::system_clock::t newTaskScheduled.notify_one(); } -void CScheduler::scheduleFromNow(CScheduler::Function f, int64_t deltaSeconds) +void CScheduler::scheduleFromNow(CScheduler::Function f, int64_t deltaMilliSeconds) { - schedule(f, boost::chrono::system_clock::now() + boost::chrono::seconds(deltaSeconds)); + schedule(f, boost::chrono::system_clock::now() + boost::chrono::milliseconds(deltaMilliSeconds)); } -static void Repeat(CScheduler* s, CScheduler::Function f, int64_t deltaSeconds) +static void Repeat(CScheduler* s, CScheduler::Function f, int64_t deltaMilliSeconds) { f(); - s->scheduleFromNow(boost::bind(&Repeat, s, f, deltaSeconds), deltaSeconds); + s->scheduleFromNow(boost::bind(&Repeat, s, f, deltaMilliSeconds), deltaMilliSeconds); } -void CScheduler::scheduleEvery(CScheduler::Function f, int64_t deltaSeconds) +void CScheduler::scheduleEvery(CScheduler::Function f, int64_t deltaMilliSeconds) { - scheduleFromNow(boost::bind(&Repeat, this, f, deltaSeconds), deltaSeconds); + scheduleFromNow(boost::bind(&Repeat, this, f, deltaMilliSeconds), deltaMilliSeconds); } size_t CScheduler::getQueueInfo(boost::chrono::system_clock::time_point &first, diff --git a/src/scheduler.h b/src/scheduler.h index 436659e58b..613fc1653f 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -10,7 +10,6 @@ // boost::thread / boost::function / boost::chrono should be ported to // std::thread / std::function / std::chrono when we support C++11. // -#include <boost/function.hpp> #include <boost/chrono/chrono.hpp> #include <boost/thread.hpp> #include <map> @@ -23,7 +22,7 @@ // // CScheduler* s = new CScheduler(); // s->scheduleFromNow(doSomething, 11); // Assuming a: void doSomething() { } -// s->scheduleFromNow(boost::bind(Class::func, this, argument), 3); +// s->scheduleFromNow(std::bind(Class::func, this, argument), 3); // boost::thread* t = new boost::thread(boost::bind(CScheduler::serviceQueue, s)); // // ... then at program shutdown, clean up the thread running serviceQueue: @@ -39,20 +38,20 @@ public: CScheduler(); ~CScheduler(); - typedef boost::function<void(void)> Function; + typedef std::function<void(void)> Function; // Call func at/after time t void schedule(Function f, boost::chrono::system_clock::time_point t); // Convenience method: call f once deltaSeconds from now - void scheduleFromNow(Function f, int64_t deltaSeconds); + void scheduleFromNow(Function f, int64_t deltaMilliSeconds); // Another convenience method: call f approximately // every deltaSeconds forever, starting deltaSeconds from now. // To be more precise: every time f is finished, it // is rescheduled to run deltaSeconds later. If you // need more accurate scheduling, don't use this method. - void scheduleEvery(Function f, int64_t deltaSeconds); + void scheduleEvery(Function f, int64_t deltaMilliSeconds); // To keep things as simple as possible, there is no unschedule. diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 374423179c..3b5da4980b 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -189,7 +189,9 @@ BOOST_AUTO_TEST_CASE(tx_invalid) // verifyFlags is a comma separated list of script verification flags to apply, or "NONE" UniValue tests = read_json(std::string(json_tests::tx_invalid, json_tests::tx_invalid + sizeof(json_tests::tx_invalid))); - ScriptError err; + // Initialize to SCRIPT_ERR_OK. The tests expect err to be changed to a + // value other than SCRIPT_ERR_OK. + ScriptError err = SCRIPT_ERR_OK; for (unsigned int idx = 0; idx < tests.size(); idx++) { UniValue test = tests[idx]; std::string strTest = test.write(); diff --git a/src/validation.cpp b/src/validation.cpp index e84b1a7281..e09eba1aca 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2983,7 +2983,7 @@ bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& sta return state.Invalid(false, REJECT_INVALID, "time-too-old", "block's timestamp is too early"); // Check timestamp - if (block.GetBlockTime() > nAdjustedTime + 2 * 60 * 60) + if (block.GetBlockTime() > nAdjustedTime + MAX_FUTURE_BLOCK_TIME) return state.Invalid(false, REJECT_INVALID, "time-too-new", "block timestamp too far in the future"); // Reject outdated version blocks when 95% (75% on testnet) of the network has upgraded: diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 7d1b429b30..74d87f9d15 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -18,6 +18,7 @@ #endif #include <boost/filesystem.hpp> +#include <boost/foreach.hpp> #include <boost/thread.hpp> #include <boost/version.hpp> @@ -145,7 +146,7 @@ void CDBEnv::MakeMock() fMockDb = true; } -CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, bool (*recoverFunc)(CDBEnv& dbenv, const std::string& strFile)) +CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, bool (*recoverFunc)(const std::string& strFile)) { LOCK(cs_db); assert(mapFileUseCount.count(strFile) == 0); @@ -158,10 +159,134 @@ CDBEnv::VerifyResult CDBEnv::Verify(const std::string& strFile, bool (*recoverFu return RECOVER_FAIL; // Try to recover: - bool fRecovered = (*recoverFunc)(*this, strFile); + bool fRecovered = (*recoverFunc)(strFile); return (fRecovered ? RECOVER_OK : RECOVER_FAIL); } +bool CDB::Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue)) +{ + // Recovery procedure: + // move wallet file to wallet.timestamp.bak + // Call Salvage with fAggressive=true to + // get as much data as possible. + // Rewrite salvaged data to fresh wallet file + // Set -rescan so any missing transactions will be + // found. + int64_t now = GetTime(); + std::string newFilename = strprintf("wallet.%d.bak", now); + + int result = bitdb.dbenv->dbrename(NULL, filename.c_str(), NULL, + newFilename.c_str(), DB_AUTO_COMMIT); + if (result == 0) + LogPrintf("Renamed %s to %s\n", filename, newFilename); + else + { + LogPrintf("Failed to rename %s to %s\n", filename, newFilename); + return false; + } + + std::vector<CDBEnv::KeyValPair> salvagedData; + bool fSuccess = bitdb.Salvage(newFilename, true, salvagedData); + if (salvagedData.empty()) + { + LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename); + return false; + } + LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size()); + + std::unique_ptr<Db> pdbCopy(new Db(bitdb.dbenv, 0)); + int ret = pdbCopy->open(NULL, // Txn pointer + filename.c_str(), // Filename + "main", // Logical db name + DB_BTREE, // Database type + DB_CREATE, // Flags + 0); + if (ret > 0) + { + LogPrintf("Cannot create database file %s\n", filename); + return false; + } + + DbTxn* ptxn = bitdb.TxnBegin(); + BOOST_FOREACH(CDBEnv::KeyValPair& row, salvagedData) + { + if (recoverKVcallback) + { + CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION); + CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION); + string strType, strErr; + if (!(*recoverKVcallback)(callbackDataIn, ssKey, ssValue)) + continue; + } + Dbt datKey(&row.first[0], row.first.size()); + Dbt datValue(&row.second[0], row.second.size()); + int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE); + if (ret2 > 0) + fSuccess = false; + } + ptxn->commit(0); + pdbCopy->close(0); + + return fSuccess; +} + +bool CDB::VerifyEnvironment(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& errorStr) +{ + LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0)); + LogPrintf("Using wallet %s\n", walletFile); + + // Wallet file must be a plain filename without a directory + if (walletFile != boost::filesystem::basename(walletFile) + boost::filesystem::extension(walletFile)) + { + errorStr = strprintf(_("Wallet %s resides outside data directory %s"), walletFile, dataDir.string()); + return false; + } + + if (!bitdb.Open(dataDir)) + { + // try moving the database env out of the way + boost::filesystem::path pathDatabase = dataDir / "database"; + boost::filesystem::path pathDatabaseBak = dataDir / strprintf("database.%d.bak", GetTime()); + try { + boost::filesystem::rename(pathDatabase, pathDatabaseBak); + LogPrintf("Moved old %s to %s. Retrying.\n", pathDatabase.string(), pathDatabaseBak.string()); + } catch (const boost::filesystem::filesystem_error&) { + // failure is ok (well, not really, but it's not worse than what we started with) + } + + // try again + if (!bitdb.Open(dataDir)) { + // if it still fails, it probably means we can't even create the database env + errorStr = strprintf(_("Error initializing wallet database environment %s!"), GetDataDir()); + return false; + } + } + return true; +} + +bool CDB::VerifyDatabaseFile(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& warningStr, std::string& errorStr, bool (*recoverFunc)(const std::string& strFile)) +{ + if (boost::filesystem::exists(dataDir / walletFile)) + { + CDBEnv::VerifyResult r = bitdb.Verify(walletFile, recoverFunc); + if (r == CDBEnv::RECOVER_OK) + { + warningStr = strprintf(_("Warning: Wallet file corrupt, data salvaged!" + " Original %s saved as %s in %s; if" + " your balance or transactions are incorrect you should" + " restore from a backup."), + walletFile, "wallet.{timestamp}.bak", dataDir); + } + if (r == CDBEnv::RECOVER_FAIL) + { + errorStr = strprintf(_("%s corrupt, salvage failed"), walletFile); + return false; + } + } + // also return true if files does not exists + return true; +} + /* End of headers, beginning of key/value data */ static const char *HEADER_END = "HEADER=END"; /* End of key/value data */ @@ -473,3 +598,41 @@ void CDBEnv::Flush(bool fShutdown) } } } + +bool CDB::PeriodicFlush(std::string strFile) +{ + bool ret = false; + TRY_LOCK(bitdb.cs_db,lockDb); + if (lockDb) + { + // Don't do this if any databases are in use + int nRefCount = 0; + map<string, int>::iterator mi = bitdb.mapFileUseCount.begin(); + while (mi != bitdb.mapFileUseCount.end()) + { + nRefCount += (*mi).second; + mi++; + } + + if (nRefCount == 0) + { + boost::this_thread::interruption_point(); + map<string, int>::iterator mi = bitdb.mapFileUseCount.find(strFile); + if (mi != bitdb.mapFileUseCount.end()) + { + LogPrint("db", "Flushing %s\n", strFile); + int64_t nStart = GetTimeMillis(); + + // Flush wallet file so it's self contained + bitdb.CloseDb(strFile); + bitdb.CheckpointLSN(strFile); + + bitdb.mapFileUseCount.erase(mi++); + LogPrint("db", "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); + ret = true; + } + } + } + + return ret; +} diff --git a/src/wallet/db.h b/src/wallet/db.h index b4ce044e7f..19c54e314c 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -56,7 +56,7 @@ public: enum VerifyResult { VERIFY_OK, RECOVER_OK, RECOVER_FAIL }; - VerifyResult Verify(const std::string& strFile, bool (*recoverFunc)(CDBEnv& dbenv, const std::string& strFile)); + VerifyResult Verify(const std::string& strFile, bool (*recoverFunc)(const std::string& strFile)); /** * Salvage data from a file that Verify says is bad. * fAggressive sets the DB_AGGRESSIVE flag (see berkeley DB->verify() method documentation). @@ -104,6 +104,15 @@ protected: public: void Flush(); void Close(); + static bool Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue)); + + /* flush the wallet passively (TRY_LOCK) + ideal to be called periodically */ + static bool PeriodicFlush(std::string strFile); + /* verifies the database environment */ + static bool VerifyEnvironment(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& errorStr); + /* verifies the database file */ + static bool VerifyDatabaseFile(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& warningStr, std::string& errorStr, bool (*recoverFunc)(const std::string& strFile)); private: CDB(const CDB&); diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 182a62cbb1..e02c6513e4 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -513,7 +513,7 @@ UniValue importwallet(const JSONRPCRequest& request) pwallet->ShowProgress("", 100); // hide progress dialog in GUI CBlockIndex *pindex = chainActive.Tip(); - while (pindex && pindex->pprev && pindex->GetBlockTime() > nTimeBegin - 7200) + while (pindex && pindex->pprev && pindex->GetBlockTime() > nTimeBegin - TIMESTAMP_WINDOW) pindex = pindex->pprev; pwallet->UpdateTimeFirstKey(nTimeBegin); @@ -1095,7 +1095,7 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) } if (fRescan && fRunScan && requests.size()) { - CBlockIndex* pindex = nLowestTimestamp > minimumTimestamp ? chainActive.FindEarliestAtLeast(std::max<int64_t>(nLowestTimestamp - 7200, 0)) : chainActive.Genesis(); + CBlockIndex* pindex = nLowestTimestamp > minimumTimestamp ? chainActive.FindEarliestAtLeast(std::max<int64_t>(nLowestTimestamp - TIMESTAMP_WINDOW, 0)) : chainActive.Genesis(); CBlockIndex* scannedRange = nullptr; if (pindex) { scannedRange = pwallet->ScanForWalletTransactions(pindex, true); @@ -1112,7 +1112,7 @@ UniValue importmulti(const JSONRPCRequest& mainRequest) // range, or if the import result already has an error set, let // the result stand unmodified. Otherwise replace the result // with an error message. - if (GetImportTimestamp(request, now) - 7200 >= scannedRange->GetBlockTimeMax() || results.at(i).exists("error")) { + if (GetImportTimestamp(request, now) - TIMESTAMP_WINDOW >= scannedRange->GetBlockTimeMax() || results.at(i).exists("error")) { response.push_back(results.at(i)); } else { UniValue result = UniValue(UniValue::VOBJ); diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 19c1e9f291..42b67fac39 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -415,7 +415,7 @@ BOOST_FIXTURE_TEST_CASE(rescan, TestChain100Setup) CKey futureKey; futureKey.MakeNewKey(true); key.pushKV("scriptPubKey", HexStr(GetScriptForRawPubKey(futureKey.GetPubKey()))); - key.pushKV("timestamp", newTip->GetBlockTimeMax() + 7200); + key.pushKV("timestamp", newTip->GetBlockTimeMax() + TIMESTAMP_WINDOW); key.pushKV("internal", UniValue(true)); keys.push_back(key); JSONRPCRequest request; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 0e95699bbd..b7c2d8ea8d 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -20,6 +20,7 @@ #include "primitives/transaction.h" #include "script/script.h" #include "script/sign.h" +#include "scheduler.h" #include "timedata.h" #include "txmempool.h" #include "util.h" @@ -444,57 +445,30 @@ bool CWallet::Verify() if (GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET)) return true; - LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(0, 0, 0)); - std::string walletFile = GetArg("-wallet", DEFAULT_WALLET_DAT); - - LogPrintf("Using wallet %s\n", walletFile); uiInterface.InitMessage(_("Verifying wallet...")); + std::string walletFile = GetArg("-wallet", DEFAULT_WALLET_DAT); - // Wallet file must be a plain filename without a directory - if (walletFile != boost::filesystem::basename(walletFile) + boost::filesystem::extension(walletFile)) - return InitError(strprintf(_("Wallet %s resides outside data directory %s"), walletFile, GetDataDir().string())); + std::string strError; + if (!CWalletDB::VerifyEnvironment(walletFile, GetDataDir().string(), strError)) + return InitError(strError); - if (!bitdb.Open(GetDataDir())) - { - // try moving the database env out of the way - boost::filesystem::path pathDatabase = GetDataDir() / "database"; - boost::filesystem::path pathDatabaseBak = GetDataDir() / strprintf("database.%d.bak", GetTime()); - try { - boost::filesystem::rename(pathDatabase, pathDatabaseBak); - LogPrintf("Moved old %s to %s. Retrying.\n", pathDatabase.string(), pathDatabaseBak.string()); - } catch (const boost::filesystem::filesystem_error&) { - // failure is ok (well, not really, but it's not worse than what we started with) - } - - // try again - if (!bitdb.Open(GetDataDir())) { - // if it still fails, it probably means we can't even create the database env - return InitError(strprintf(_("Error initializing wallet database environment %s!"), GetDataDir())); - } - } - if (GetBoolArg("-salvagewallet", false)) { // Recover readable keypairs: - if (!CWalletDB::Recover(bitdb, walletFile, true)) + CWallet dummyWallet; + if (!CWalletDB::Recover(walletFile, (void *)&dummyWallet, CWalletDB::RecoverKeysOnlyFilter)) return false; } - - if (boost::filesystem::exists(GetDataDir() / walletFile)) + + std::string strWarning; + bool dbV = CWalletDB::VerifyDatabaseFile(walletFile, GetDataDir().string(), strWarning, strError); + if (!strWarning.empty()) + InitWarning(strWarning); + if (!dbV) { - CDBEnv::VerifyResult r = bitdb.Verify(walletFile, CWalletDB::Recover); - if (r == CDBEnv::RECOVER_OK) - { - InitWarning(strprintf(_("Warning: Wallet file corrupt, data salvaged!" - " Original %s saved as %s in %s; if" - " your balance or transactions are incorrect you should" - " restore from a backup."), - walletFile, "wallet.{timestamp}.bak", GetDataDir())); - } - if (r == CDBEnv::RECOVER_FAIL) - return InitError(strprintf(_("%s corrupt, salvage failed"), walletFile)); + InitError(strError); + return false; } - return true; } @@ -1518,7 +1492,7 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool f // no need to read and scan block, if block was created before // our wallet birthday (as adjusted for block time variability) - while (pindex && nTimeFirstKey && (pindex->GetBlockTime() < (nTimeFirstKey - 7200))) + while (pindex && nTimeFirstKey && (pindex->GetBlockTime() < (nTimeFirstKey - TIMESTAMP_WINDOW))) pindex = chainActive.Next(pindex); ShowProgress(_("Rescanning..."), 0); // show rescan progress in GUI as dialog or on splashscreen, if -rescan on startup @@ -3451,7 +3425,7 @@ void CWallet::GetKeyBirthTimes(std::map<CTxDestination, int64_t> &mapKeyBirth) c // Extract block timestamps for those keys for (std::map<CKeyID, CBlockIndex*>::const_iterator it = mapKeyFirstBlock.begin(); it != mapKeyFirstBlock.end(); it++) - mapKeyBirth[it->first] = it->second->GetBlockTime() - 7200; // block times can be 2h off + mapKeyBirth[it->first] = it->second->GetBlockTime() - TIMESTAMP_WINDOW; // block times can be 2h off } /** @@ -3707,17 +3681,13 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile) RegisterValidationInterface(walletInstance); - CBlockIndex *pindexRescan = chainActive.Tip(); - if (GetBoolArg("-rescan", false)) - pindexRescan = chainActive.Genesis(); - else + CBlockIndex *pindexRescan = chainActive.Genesis(); + if (!GetBoolArg("-rescan", false)) { CWalletDB walletdb(walletFile); CBlockLocator locator; if (walletdb.ReadBestBlock(locator)) pindexRescan = FindForkInGlobalIndex(chainActive, locator); - else - pindexRescan = chainActive.Genesis(); } if (chainActive.Tip() && chainActive.Tip() != pindexRescan) { @@ -3806,17 +3776,17 @@ bool CWallet::InitLoadWallet() return true; } -std::atomic<bool> CWallet::fFlushThreadRunning(false); +std::atomic<bool> CWallet::fFlushScheduled(false); -void CWallet::postInitProcess(boost::thread_group& threadGroup) +void CWallet::postInitProcess(CScheduler& scheduler) { // Add wallet transactions that aren't already in a block to mempool // Do this here as mempool requires genesis block to be loaded ReacceptWalletTransactions(); // Run a thread to flush wallet periodically - if (!CWallet::fFlushThreadRunning.exchange(true)) { - threadGroup.create_thread(ThreadFlushWalletDB); + if (!CWallet::fFlushScheduled.exchange(true)) { + scheduler.scheduleEvery(MaybeCompactWalletDB, 500); } } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 99f6f43a47..02c1b18769 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -29,7 +29,6 @@ #include <vector> #include <boost/shared_ptr.hpp> -#include <boost/thread.hpp> extern CWallet* pwalletMain; @@ -79,6 +78,7 @@ class CCoinControl; class COutput; class CReserveKey; class CScript; +class CScheduler; class CTxMemPool; class CWalletTx; @@ -256,6 +256,31 @@ private: const CWallet* pwallet; public: + /** + * Key/value map with information about the transaction. + * + * The following keys can be read and written through the map and are + * serialized in the wallet database: + * + * "comment", "to" - comment strings provided to sendtoaddress, + * sendfrom, sendmany wallet RPCs + * "replaces_txid" - txid (as HexStr) of transaction replaced by + * bumpfee on transaction created by bumpfee + * "replaced_by_txid" - txid (as HexStr) of transaction created by + * bumpfee on transaction replaced by bumpfee + * "from", "message" - obsolete fields that could be set in UI prior to + * 2011 (removed in commit 4d9b223) + * + * The following keys are serialized in the wallet database, but shouldn't + * be read or written through the map (they will be temporarily added and + * removed from the map during serialization): + * + * "fromaccount" - serialized strFromAccount value + * "n" - serialized nOrderPos value + * "timesmart" - serialized nTimeSmart value + * "spent" - serialized vfSpent value that existed prior to + * 2014 (removed in commit 93a18a3) + */ mapValue_t mapValue; std::vector<std::pair<std::string, std::string> > vOrderForm; unsigned int fTimeReceivedIsTxTime; @@ -378,7 +403,6 @@ public: } mapValue.erase("fromaccount"); - mapValue.erase("version"); mapValue.erase("spent"); mapValue.erase("n"); mapValue.erase("timesmart"); @@ -578,7 +602,7 @@ private: class CWallet : public CCryptoKeyStore, public CValidationInterface { private: - static std::atomic<bool> fFlushThreadRunning; + static std::atomic<bool> fFlushScheduled; /** * Select a set of coins such that nValueRet >= nTargetValue and at least @@ -987,7 +1011,7 @@ public: * Wallet post-init setup * Gives the wallet a chance to register repetitive tasks and complete post-init tasks */ - void postInitProcess(boost::thread_group& threadGroup); + void postInitProcess(CScheduler& scheduler); /* Wallets parameter interaction */ static bool ParameterInteraction(); @@ -1022,6 +1046,10 @@ public: pwallet = pwalletIn; } + CReserveKey() = default; + CReserveKey(const CReserveKey&) = delete; + CReserveKey& operator=(const CReserveKey&) = delete; + ~CReserveKey() { ReturnKey(); diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 44a01d4a36..5ba9f150a8 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -546,7 +546,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, return true; } -static bool IsKeyType(string strType) +bool CWalletDB::IsKeyType(const std::string& strType) { return (strType== "key" || strType == "wkey" || strType == "mkey" || strType == "ckey"); @@ -777,156 +777,81 @@ DBErrors CWalletDB::ZapWalletTx(vector<CWalletTx>& vWtx) return DB_LOAD_OK; } -void ThreadFlushWalletDB() +void MaybeCompactWalletDB() { - // Make this thread recognisable as the wallet flushing thread - RenameThread("bitcoin-wallet"); - - static bool fOneThread; - if (fOneThread) + static std::atomic<bool> fOneThread; + if (fOneThread.exchange(true)) { return; - fOneThread = true; - if (!GetBoolArg("-flushwallet", DEFAULT_FLUSHWALLET)) + } + if (!GetBoolArg("-flushwallet", DEFAULT_FLUSHWALLET)) { return; + } - unsigned int nLastSeen = CWalletDB::GetUpdateCounter(); - unsigned int nLastFlushed = CWalletDB::GetUpdateCounter(); - int64_t nLastWalletUpdate = GetTime(); - while (true) - { - MilliSleep(500); + static unsigned int nLastSeen = CWalletDB::GetUpdateCounter(); + static unsigned int nLastFlushed = CWalletDB::GetUpdateCounter(); + static int64_t nLastWalletUpdate = GetTime(); - if (nLastSeen != CWalletDB::GetUpdateCounter()) - { - nLastSeen = CWalletDB::GetUpdateCounter(); - nLastWalletUpdate = GetTime(); - } - - if (nLastFlushed != CWalletDB::GetUpdateCounter() && GetTime() - nLastWalletUpdate >= 2) - { - TRY_LOCK(bitdb.cs_db,lockDb); - if (lockDb) - { - // Don't do this if any databases are in use - int nRefCount = 0; - map<string, int>::iterator mi = bitdb.mapFileUseCount.begin(); - while (mi != bitdb.mapFileUseCount.end()) - { - nRefCount += (*mi).second; - mi++; - } + if (nLastSeen != CWalletDB::GetUpdateCounter()) + { + nLastSeen = CWalletDB::GetUpdateCounter(); + nLastWalletUpdate = GetTime(); + } - if (nRefCount == 0) - { - boost::this_thread::interruption_point(); - const std::string& strFile = pwalletMain->strWalletFile; - map<string, int>::iterator _mi = bitdb.mapFileUseCount.find(strFile); - if (_mi != bitdb.mapFileUseCount.end()) - { - LogPrint("db", "Flushing %s\n", strFile); - nLastFlushed = CWalletDB::GetUpdateCounter(); - int64_t nStart = GetTimeMillis(); - - // Flush wallet file so it's self contained - bitdb.CloseDb(strFile); - bitdb.CheckpointLSN(strFile); - - bitdb.mapFileUseCount.erase(_mi++); - LogPrint("db", "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart); - } - } - } - } + if (nLastFlushed != CWalletDB::GetUpdateCounter() && GetTime() - nLastWalletUpdate >= 2) + { + const std::string& strFile = pwalletMain->strWalletFile; + if (CDB::PeriodicFlush(strFile)) + nLastFlushed = CWalletDB::GetUpdateCounter(); } + fOneThread = false; } // // Try to (very carefully!) recover wallet file if there is a problem. // -bool CWalletDB::Recover(CDBEnv& dbenv, const std::string& filename, bool fOnlyKeys) -{ - // Recovery procedure: - // move wallet file to wallet.timestamp.bak - // Call Salvage with fAggressive=true to - // get as much data as possible. - // Rewrite salvaged data to fresh wallet file - // Set -rescan so any missing transactions will be - // found. - int64_t now = GetTime(); - std::string newFilename = strprintf("wallet.%d.bak", now); - - int result = dbenv.dbenv->dbrename(NULL, filename.c_str(), NULL, - newFilename.c_str(), DB_AUTO_COMMIT); - if (result == 0) - LogPrintf("Renamed %s to %s\n", filename, newFilename); - else - { - LogPrintf("Failed to rename %s to %s\n", filename, newFilename); - return false; - } +bool CWalletDB::Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue)) +{ + return CDB::Recover(filename, callbackDataIn, recoverKVcallback); +} + +bool CWalletDB::Recover(const std::string& filename) +{ + // recover without a key filter callback + // results in recovering all record types + return CWalletDB::Recover(filename, NULL, NULL); +} - std::vector<CDBEnv::KeyValPair> salvagedData; - bool fSuccess = dbenv.Salvage(newFilename, true, salvagedData); - if (salvagedData.empty()) +bool CWalletDB::RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, CDataStream ssValue) +{ + CWallet *dummyWallet = reinterpret_cast<CWallet*>(callbackData); + CWalletScanState dummyWss; + std::string strType, strErr; + bool fReadOK; { - LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename); - return false; + // Required in LoadKeyMetadata(): + LOCK(dummyWallet->cs_wallet); + fReadOK = ReadKeyValue(dummyWallet, ssKey, ssValue, + dummyWss, strType, strErr); } - LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size()); - - std::unique_ptr<Db> pdbCopy(new Db(dbenv.dbenv, 0)); - int ret = pdbCopy->open(NULL, // Txn pointer - filename.c_str(), // Filename - "main", // Logical db name - DB_BTREE, // Database type - DB_CREATE, // Flags - 0); - if (ret > 0) + if (!IsKeyType(strType) && strType != "hdchain") + return false; + if (!fReadOK) { - LogPrintf("Cannot create database file %s\n", filename); + LogPrintf("WARNING: CWalletDB::Recover skipping %s: %s\n", strType, strErr); return false; } - CWallet dummyWallet; - CWalletScanState wss; - DbTxn* ptxn = dbenv.TxnBegin(); - BOOST_FOREACH(CDBEnv::KeyValPair& row, salvagedData) - { - if (fOnlyKeys) - { - CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION); - CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION); - string strType, strErr; - bool fReadOK; - { - // Required in LoadKeyMetadata(): - LOCK(dummyWallet.cs_wallet); - fReadOK = ReadKeyValue(&dummyWallet, ssKey, ssValue, - wss, strType, strErr); - } - if (!IsKeyType(strType) && strType != "hdchain") - continue; - if (!fReadOK) - { - LogPrintf("WARNING: CWalletDB::Recover skipping %s: %s\n", strType, strErr); - continue; - } - } - Dbt datKey(&row.first[0], row.first.size()); - Dbt datValue(&row.second[0], row.second.size()); - int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE); - if (ret2 > 0) - fSuccess = false; - } - ptxn->commit(0); - pdbCopy->close(0); + return true; +} - return fSuccess; +bool CWalletDB::VerifyEnvironment(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& errorStr) +{ + return CDB::VerifyEnvironment(walletFile, dataDir, errorStr); } -bool CWalletDB::Recover(CDBEnv& dbenv, const std::string& filename) +bool CWalletDB::VerifyDatabaseFile(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& warningStr, std::string& errorStr) { - return CWalletDB::Recover(dbenv, filename, false); + return CDB::VerifyDatabaseFile(walletFile, dataDir, errorStr, warningStr, CWalletDB::Recover); } bool CWalletDB::WriteDestData(const std::string &address, const std::string &key, const std::string &value) diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 2d95df91da..2e9899acc6 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -170,8 +170,18 @@ public: DBErrors FindWalletTx(std::vector<uint256>& vTxHash, std::vector<CWalletTx>& vWtx); DBErrors ZapWalletTx(std::vector<CWalletTx>& vWtx); DBErrors ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut); - static bool Recover(CDBEnv& dbenv, const std::string& filename, bool fOnlyKeys); - static bool Recover(CDBEnv& dbenv, const std::string& filename); + /* Try to (very carefully!) recover wallet database (with a possible key type filter) */ + static bool Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue)); + /* Recover convenience-function to bypass the key filter callback, called when verify failes, recoveres everything */ + static bool Recover(const std::string& filename); + /* Recover filter (used as callback), will only let keys (cryptographical keys) as KV/key-type pass through */ + static bool RecoverKeysOnlyFilter(void *callbackData, CDataStream ssKey, CDataStream ssValue); + /* Function to determin if a certain KV/key-type is a key (cryptographical key) type */ + static bool IsKeyType(const std::string& strType); + /* verifies the database environment */ + static bool VerifyEnvironment(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& errorStr); + /* verifies the database file */ + static bool VerifyDatabaseFile(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& warningStr, std::string& errorStr); //! write the hdchain model (external chain child index counter) bool WriteHDChain(const CHDChain& chain); @@ -183,6 +193,7 @@ private: void operator=(const CWalletDB&); }; -void ThreadFlushWalletDB(); +//! Compacts BDB state so that wallet.dat is self-contained (if there are changes) +void MaybeCompactWalletDB(); #endif // BITCOIN_WALLET_WALLETDB_H |