diff options
63 files changed, 1402 insertions, 255 deletions
diff --git a/contrib/debian/copyright b/contrib/debian/copyright index a6ee201991..3741031f9c 100644 --- a/contrib/debian/copyright +++ b/contrib/debian/copyright @@ -15,10 +15,6 @@ Files: src/json/* Copyright: 2007-2009, John W. Wilkinson License: Expat -Files: src/strlcpy.h -Copyright: 1998, Todd C. Miller <Todd.Miller@courtesan.com> -License: ISC - Files: debian/* Copyright: 2010-2011, Jonas Smedegaard <dr@jones.dk> 2011, Matt Corallo <matt@bluematt.me> diff --git a/depends/.gitignore b/depends/.gitignore index 82c48638b0..1f163897b9 100644 --- a/depends/.gitignore +++ b/depends/.gitignore @@ -3,3 +3,7 @@ work/ built/ sources/ config.site +x86_64* +i686* +mips* +arm* diff --git a/depends/config.guess b/depends/config.guess index dbfb9786cb..f7eb141e75 100755 --- a/depends/config.guess +++ b/depends/config.guess @@ -2,7 +2,7 @@ # Attempt to guess a canonical system name. # Copyright 1992-2015 Free Software Foundation, Inc. -timestamp='2015-01-01' +timestamp='2015-03-04' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -168,20 +168,27 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # Note: NetBSD doesn't particularly care about the vendor # portion of the name. We always set it to "unknown". sysctl="sysctl -n hw.machine_arch" - UNAME_MACHINE_ARCH=`(/sbin/$sysctl 2>/dev/null || \ - /usr/sbin/$sysctl 2>/dev/null || echo unknown)` + UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \ + /sbin/$sysctl 2>/dev/null || \ + /usr/sbin/$sysctl 2>/dev/null || \ + echo unknown)` case "${UNAME_MACHINE_ARCH}" in armeb) machine=armeb-unknown ;; arm*) machine=arm-unknown ;; sh3el) machine=shl-unknown ;; sh3eb) machine=sh-unknown ;; sh5el) machine=sh5le-unknown ;; + earmv*) + arch=`echo ${UNAME_MACHINE_ARCH} | sed -e 's,^e\(armv[0-9]\).*$,\1,'` + endian=`echo ${UNAME_MACHINE_ARCH} | sed -ne 's,^.*\(eb\)$,\1,p'` + machine=${arch}${endian}-unknown + ;; *) machine=${UNAME_MACHINE_ARCH}-unknown ;; esac # The Operating System including object format, if it has switched # to ELF recently, or will in the future. case "${UNAME_MACHINE_ARCH}" in - arm*|i386|m68k|ns32k|sh3*|sparc|vax) + arm*|earm*|i386|m68k|ns32k|sh3*|sparc|vax) eval $set_cc_for_build if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ELF__ @@ -197,6 +204,13 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in os=netbsd ;; esac + # Determine ABI tags. + case "${UNAME_MACHINE_ARCH}" in + earm*) + expr='s/^earmv[0-9]/-eabi/;s/eb$//' + abi=`echo ${UNAME_MACHINE_ARCH} | sed -e "$expr"` + ;; + esac # The OS release # Debian GNU/NetBSD machines have a different userland, and # thus, need a distinct triplet. However, they do not need @@ -213,7 +227,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: # contains redundant information, the shorter form: # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. - echo "${machine}-${os}${release}" + echo "${machine}-${os}${release}${abi}" exit ;; *:Bitrig:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` @@ -933,6 +947,9 @@ EOF crisv32:Linux:*:*) echo ${UNAME_MACHINE}-axis-linux-${LIBC} exit ;; + e2k:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; frv:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; diff --git a/depends/config.sub b/depends/config.sub index 6d2e94c8bf..8f1229c6f7 100755 --- a/depends/config.sub +++ b/depends/config.sub @@ -2,7 +2,7 @@ # Configuration validation subroutine script. # Copyright 1992-2015 Free Software Foundation, Inc. -timestamp='2015-01-01' +timestamp='2015-03-08' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -117,7 +117,7 @@ maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` case $maybe_os in nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \ linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \ - knetbsd*-gnu* | netbsd*-gnu* | \ + knetbsd*-gnu* | netbsd*-gnu* | netbsd*-eabi* | \ kopensolaris*-gnu* | \ storm-chaos* | os2-emx* | rtmk-nova*) os=-$maybe_os @@ -259,7 +259,7 @@ case $basic_machine in | bfin \ | c4x | c8051 | clipper \ | d10v | d30v | dlx | dsp16xx \ - | epiphany \ + | e2k | epiphany \ | fido | fr30 | frv | ft32 \ | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ | hexagon \ @@ -381,7 +381,7 @@ case $basic_machine in | c[123]* | c30-* | [cjt]90-* | c4x-* \ | c8051-* | clipper-* | craynv-* | cydra-* \ | d10v-* | d30v-* | dlx-* \ - | elxsi-* \ + | e2k-* | elxsi-* \ | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \ | h8300-* | h8500-* \ | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ @@ -518,6 +518,9 @@ case $basic_machine in basic_machine=i386-pc os=-aros ;; + asmjs) + basic_machine=asmjs-unknown + ;; aux) basic_machine=m68k-apple os=-aux @@ -1373,7 +1376,7 @@ case $os in | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \ | -sym* | -kopensolaris* | -plan9* \ | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ - | -aos* | -aros* \ + | -aos* | -aros* | -cloudabi* \ | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \ diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 23154ee903..f14aed7287 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -36,4 +36,4 @@ Only supports JSON as output format. Risks ------------- -Running a webbrowser on the same node with a REST enabled bitcoind can be a risk. Accessing prepared XSS websites could read out tx/block data of your node by placing links like `<script src="http://127.0.0.1:1234/tx/json/1234567890">` which might break the nodes privacy.
\ No newline at end of file +Running a webbrowser on the same node with a REST enabled bitcoind can be a risk. Accessing prepared XSS websites could read out tx/block data of your node by placing links like `<script src="http://127.0.0.1:8332/rest/tx/1234567890.json">` which might break the nodes privacy. diff --git a/doc/dnsseed-policy.md b/doc/dnsseed-policy.md index 66a1757ac5..f15ff44e5d 100644 --- a/doc/dnsseed-policy.md +++ b/doc/dnsseed-policy.md @@ -7,9 +7,8 @@ As such, DNS seeds must be run by entities which have some minimum level of trust within the Bitcoin community. Other implementations of Bitcoin software may also use the same -seeds and may be more exposed. In light of this exposure this -document establishes some basic expectations for the expectations -for the operation of dnsseeds. +seeds and may be more exposed. In light of this exposure, this +document establishes some basic expectations for operating dnsseeds. 0. A DNS seed operating organization or person is expected to follow good host security practices and maintain control of diff --git a/doc/gitian-building.md b/doc/gitian-building.md index 25d3b8390c..d285fffdbb 100644 --- a/doc/gitian-building.md +++ b/doc/gitian-building.md @@ -74,11 +74,11 @@ In the VirtualBox GUI click "Create" and choose the following parameters in the - Disk size: at least 40GB; as low as 20GB *may* be possible, but better to err on the safe side - Push the `Create` button -Get the [Debian 7.7 net installer](http://cdimage.debian.org/debian-cd/7.7.0/amd64/iso-cd/debian-7.7.0-amd64-netinst.iso) (a more recent minor version should also work, see also [Debian Network installation](https://www.debian.org/CD/netinst/)). +Get the [Debian 7.8 net installer](http://cdimage.debian.org/debian-cd/7.8.0/amd64/iso-cd/debian-7.8.0-amd64-netinst.iso) (a more recent minor version should also work, see also [Debian Network installation](https://www.debian.org/CD/netinst/)). This DVD image can be validated using a SHA256 hashing tool, for example on Unixy OSes by entering the following in a terminal: - echo "d440e85b4121f94608748139f25dbce1ad36771348b002fe07d4d44b9d9e623f debian-7.7.0-amd64-netinst.iso" | sha256sum -c + echo "e39c36d6adc0fd86c6edb0e03e22919086c883b37ca194d063b8e3e8f6ff6a3a debian-7.8.0-amd64-netinst.iso" | sha256sum -c # (must return OK) After creating the VM, we need to configure it. diff --git a/doc/img/bootstrap1.png b/doc/img/bootstrap1.png Binary files differdeleted file mode 100644 index 075930791b..0000000000 --- a/doc/img/bootstrap1.png +++ /dev/null diff --git a/doc/img/bootstrap2.png b/doc/img/bootstrap2.png Binary files differdeleted file mode 100644 index 6461f81810..0000000000 --- a/doc/img/bootstrap2.png +++ /dev/null diff --git a/doc/img/bootstrap4.png b/doc/img/bootstrap4.png Binary files differdeleted file mode 100644 index ad69737922..0000000000 --- a/doc/img/bootstrap4.png +++ /dev/null diff --git a/doc/img/bootstrap5.png b/doc/img/bootstrap5.png Binary files differdeleted file mode 100644 index d8d9baaf37..0000000000 --- a/doc/img/bootstrap5.png +++ /dev/null diff --git a/qa/pull-tester/rpc-tests.sh b/qa/pull-tester/rpc-tests.sh index efeee45530..053e8b8a7f 100755 --- a/qa/pull-tester/rpc-tests.sh +++ b/qa/pull-tester/rpc-tests.sh @@ -27,6 +27,7 @@ testScripts=( 'mempool_coinbase_spends.py' 'httpbasics.py' 'zapwallettxes.py' + 'proxy_test.py' # 'forknotify.py' ); if [ "x${ENABLE_BITCOIND}${ENABLE_UTILS}${ENABLE_WALLET}" = "x111" ]; then diff --git a/qa/rpc-tests/proxy_test.py b/qa/rpc-tests/proxy_test.py new file mode 100755 index 0000000000..d6d9e6725b --- /dev/null +++ b/qa/rpc-tests/proxy_test.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python2 +# Copyright (c) 2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +import socket +import traceback, sys +from binascii import hexlify +import time, os + +from socks5 import Socks5Configuration, Socks5Command, Socks5Server, AddressType +from test_framework import BitcoinTestFramework +from util import * +''' +Test plan: +- Start bitcoind's with different proxy configurations +- Use addnode to initiate connections +- Verify that proxies are connected to, and the right connection command is given +- Proxy configurations to test on bitcoind side: + - `-proxy` (proxy everything) + - `-onion` (proxy just onions) + - `-proxyrandomize` Circuit randomization +- Proxy configurations to test on proxy side, + - support no authentication (other proxy) + - support no authentication + user/pass authentication (Tor) + - proxy on IPv6 + +- Create various proxies (as threads) +- Create bitcoinds that connect to them +- Manipulate the bitcoinds using addnode (onetry) an observe effects + +addnode connect to IPv4 +addnode connect to IPv6 +addnode connect to onion +addnode connect to generic DNS name +''' + +class ProxyTest(BitcoinTestFramework): + def __init__(self): + # Create two proxies on different ports + # ... one unauthenticated + self.conf1 = Socks5Configuration() + self.conf1.addr = ('127.0.0.1', 13000 + (os.getpid() % 1000)) + self.conf1.unauth = True + self.conf1.auth = False + # ... one supporting authenticated and unauthenticated (Tor) + self.conf2 = Socks5Configuration() + self.conf2.addr = ('127.0.0.1', 14000 + (os.getpid() % 1000)) + self.conf2.unauth = True + self.conf2.auth = True + # ... one on IPv6 with similar configuration + self.conf3 = Socks5Configuration() + self.conf3.af = socket.AF_INET6 + self.conf3.addr = ('::1', 15000 + (os.getpid() % 1000)) + self.conf3.unauth = True + self.conf3.auth = True + + self.serv1 = Socks5Server(self.conf1) + self.serv1.start() + self.serv2 = Socks5Server(self.conf2) + self.serv2.start() + self.serv3 = Socks5Server(self.conf3) + self.serv3.start() + + def setup_nodes(self): + # Note: proxies are not used to connect to local nodes + # this is because the proxy to use is based on CService.GetNetwork(), which return NET_UNROUTABLE for localhost + return start_nodes(4, self.options.tmpdir, extra_args=[ + ['-listen', '-debug=net', '-debug=proxy', '-proxy=%s:%i' % (self.conf1.addr),'-proxyrandomize=1'], + ['-listen', '-debug=net', '-debug=proxy', '-proxy=%s:%i' % (self.conf1.addr),'-onion=%s:%i' % (self.conf2.addr),'-proxyrandomize=0'], + ['-listen', '-debug=net', '-debug=proxy', '-proxy=%s:%i' % (self.conf2.addr),'-proxyrandomize=1'], + ['-listen', '-debug=net', '-debug=proxy', '-proxy=[%s]:%i' % (self.conf3.addr),'-proxyrandomize=0'] + ]) + + def node_test(self, node, proxies, auth): + rv = [] + # Test: outgoing IPv4 connection through node + node.addnode("15.61.23.23:1234", "onetry") + cmd = proxies[0].queue.get() + assert(isinstance(cmd, Socks5Command)) + # Note: bitcoind's SOCKS5 implementation only sends atyp DOMAINNAME, even if connecting directly to IPv4/IPv6 + assert_equal(cmd.atyp, AddressType.DOMAINNAME) + assert_equal(cmd.addr, "15.61.23.23") + assert_equal(cmd.port, 1234) + if not auth: + assert_equal(cmd.username, None) + assert_equal(cmd.password, None) + rv.append(cmd) + + # Test: outgoing IPv6 connection through node + node.addnode("[1233:3432:2434:2343:3234:2345:6546:4534]:5443", "onetry") + cmd = proxies[1].queue.get() + assert(isinstance(cmd, Socks5Command)) + # Note: bitcoind's SOCKS5 implementation only sends atyp DOMAINNAME, even if connecting directly to IPv4/IPv6 + assert_equal(cmd.atyp, AddressType.DOMAINNAME) + assert_equal(cmd.addr, "1233:3432:2434:2343:3234:2345:6546:4534") + assert_equal(cmd.port, 5443) + if not auth: + assert_equal(cmd.username, None) + assert_equal(cmd.password, None) + rv.append(cmd) + + # Test: outgoing onion connection through node + node.addnode("bitcoinostk4e4re.onion:8333", "onetry") + cmd = proxies[2].queue.get() + assert(isinstance(cmd, Socks5Command)) + assert_equal(cmd.atyp, AddressType.DOMAINNAME) + assert_equal(cmd.addr, "bitcoinostk4e4re.onion") + assert_equal(cmd.port, 8333) + if not auth: + assert_equal(cmd.username, None) + assert_equal(cmd.password, None) + rv.append(cmd) + + # Test: outgoing DNS name connection through node + node.addnode("node.noumenon:8333", "onetry") + cmd = proxies[3].queue.get() + assert(isinstance(cmd, Socks5Command)) + assert_equal(cmd.atyp, AddressType.DOMAINNAME) + assert_equal(cmd.addr, "node.noumenon") + assert_equal(cmd.port, 8333) + if not auth: + assert_equal(cmd.username, None) + assert_equal(cmd.password, None) + rv.append(cmd) + + return rv + + def run_test(self): + # basic -proxy + self.node_test(self.nodes[0], [self.serv1, self.serv1, self.serv1, self.serv1], False) + + # -proxy plus -onion + self.node_test(self.nodes[1], [self.serv1, self.serv1, self.serv2, self.serv1], False) + + # -proxy plus -onion, -proxyrandomize + rv = self.node_test(self.nodes[2], [self.serv2, self.serv2, self.serv2, self.serv2], True) + # Check that credentials as used for -proxyrandomize connections are unique + credentials = set((x.username,x.password) for x in rv) + assert_equal(len(credentials), 4) + + # proxy on IPv6 localhost + self.node_test(self.nodes[3], [self.serv3, self.serv3, self.serv3, self.serv3], False) + +if __name__ == '__main__': + ProxyTest().main() + diff --git a/qa/rpc-tests/pruning.py b/qa/rpc-tests/pruning.py new file mode 100755 index 0000000000..f26cbee1e2 --- /dev/null +++ b/qa/rpc-tests/pruning.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python2 +# Copyright (c) 2014 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test pruning code +# ******** +# WARNING: +# This test uses 4GB of disk space and takes in excess of 30 mins to run +# ******** + +from test_framework import BitcoinTestFramework +from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException +from util import * +import os.path + +def calc_usage(blockdir): + return sum(os.path.getsize(blockdir+f) for f in os.listdir(blockdir) if os.path.isfile(blockdir+f))/(1024*1024) + +class PruneTest(BitcoinTestFramework): + + def __init__(self): + self.utxo = [] + self.address = ["",""] + + # Some pre-processing to create a bunch of OP_RETURN txouts to insert into transactions we create + # So we have big transactions and full blocks to fill up our block files + + # create one script_pubkey + script_pubkey = "6a4d0200" #OP_RETURN OP_PUSH2 512 bytes + for i in xrange (512): + script_pubkey = script_pubkey + "01" + # concatenate 128 txouts of above script_pubkey which we'll insert before the txout for change + self.txouts = "81" + for k in xrange(128): + # add txout value + self.txouts = self.txouts + "0000000000000000" + # add length of script_pubkey + self.txouts = self.txouts + "fd0402" + # add script_pubkey + self.txouts = self.txouts + script_pubkey + + + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 3) + + def setup_network(self): + self.nodes = [] + self.is_network_split = False + + # Create nodes 0 and 1 to mine + self.nodes.append(start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=300)) + self.nodes.append(start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=300)) + + # Create node 2 to test pruning + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-prune=550"], timewait=300)) + self.prunedir = self.options.tmpdir+"/node2/regtest/blocks/" + + self.address[0] = self.nodes[0].getnewaddress() + self.address[1] = self.nodes[1].getnewaddress() + + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[1], 2) + connect_nodes(self.nodes[2], 0) + sync_blocks(self.nodes[0:3]) + + def create_big_chain(self): + # Start by creating some coinbases we can spend later + self.nodes[1].generate(200) + sync_blocks(self.nodes[0:2]) + self.nodes[0].generate(150) + # Then mine enough full blocks to create more than 550MB of data + for i in xrange(645): + self.mine_full_block(self.nodes[0], self.address[0]) + + sync_blocks(self.nodes[0:3]) + + def test_height_min(self): + if not os.path.isfile(self.prunedir+"blk00000.dat"): + raise AssertionError("blk00000.dat is missing, pruning too early") + print "Success" + print "Though we're already using more than 550MB, current usage:", calc_usage(self.prunedir) + print "Mining 25 more blocks should cause the first block file to be pruned" + # Pruning doesn't run until we're allocating another chunk, 20 full blocks past the height cutoff will ensure this + for i in xrange(25): + self.mine_full_block(self.nodes[0],self.address[0]) + + waitstart = time.time() + while os.path.isfile(self.prunedir+"blk00000.dat"): + time.sleep(0.1) + if time.time() - waitstart > 10: + raise AssertionError("blk00000.dat not pruned when it should be") + + print "Success" + usage = calc_usage(self.prunedir) + print "Usage should be below target:", usage + if (usage > 550): + raise AssertionError("Pruning target not being met") + + def create_chain_with_staleblocks(self): + # Create stale blocks in manageable sized chunks + print "Mine 24 (stale) blocks on Node 1, followed by 25 (main chain) block reorg from Node 0, for 12 rounds" + + for j in xrange(12): + # Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain + # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects + # Stopping node 0 also clears its mempool, so it doesn't have node1's transactions to accidentally mine + stop_node(self.nodes[0],0) + self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=300) + # Mine 24 blocks in node 1 + self.utxo = self.nodes[1].listunspent() + for i in xrange(24): + if j == 0: + self.mine_full_block(self.nodes[1],self.address[1]) + else: + self.nodes[1].generate(1) #tx's already in mempool from previous disconnects + + # Reorg back with 25 block chain from node 0 + self.utxo = self.nodes[0].listunspent() + for i in xrange(25): + self.mine_full_block(self.nodes[0],self.address[0]) + + # Create connections in the order so both nodes can see the reorg at the same time + connect_nodes(self.nodes[1], 0) + connect_nodes(self.nodes[2], 0) + sync_blocks(self.nodes[0:3]) + + print "Usage can be over target because of high stale rate:", calc_usage(self.prunedir) + + def reorg_test(self): + # Node 1 will mine a 300 block chain starting 287 blocks back from Node 0 and Node 2's tip + # This will cause Node 2 to do a reorg requiring 288 blocks of undo data to the reorg_test chain + # Reboot node 1 to clear its mempool (hopefully make the invalidate faster) + # Lower the block max size so we don't keep mining all our big mempool transactions (from disconnected blocks) + stop_node(self.nodes[1],1) + self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=300) + + height = self.nodes[1].getblockcount() + print "Current block height:", height + + invalidheight = height-287 + badhash = self.nodes[1].getblockhash(invalidheight) + print "Invalidating block at height:",invalidheight,badhash + self.nodes[1].invalidateblock(badhash) + + # We've now switched to our previously mined-24 block fork on node 1, but thats not what we want + # So invalidate that fork as well, until we're on the same chain as node 0/2 (but at an ancestor 288 blocks ago) + mainchainhash = self.nodes[0].getblockhash(invalidheight - 1) + curhash = self.nodes[1].getblockhash(invalidheight - 1) + while curhash != mainchainhash: + self.nodes[1].invalidateblock(curhash) + curhash = self.nodes[1].getblockhash(invalidheight - 1) + + assert(self.nodes[1].getblockcount() == invalidheight - 1) + print "New best height", self.nodes[1].getblockcount() + + # Reboot node1 to clear those giant tx's from mempool + stop_node(self.nodes[1],1) + self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=300) + + print "Generating new longer chain of 300 more blocks" + self.nodes[1].generate(300) + + print "Reconnect nodes" + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[2], 1) + sync_blocks(self.nodes[0:3]) + + print "Verify height on node 2:",self.nodes[2].getblockcount() + print "Usage possibly still high bc of stale blocks in block files:", calc_usage(self.prunedir) + + print "Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)" + self.nodes[0].generate(220) #node 0 has many large tx's in its mempool from the disconnects + sync_blocks(self.nodes[0:3]) + + usage = calc_usage(self.prunedir) + print "Usage should be below target:", usage + if (usage > 550): + raise AssertionError("Pruning target not being met") + + return invalidheight,badhash + + def reorg_back(self): + # Verify that a block on the old main chain fork has been pruned away + try: + self.nodes[2].getblock(self.forkhash) + raise AssertionError("Old block wasn't pruned so can't test redownload") + except JSONRPCException as e: + print "Will need to redownload block",self.forkheight + + # Verify that we have enough history to reorg back to the fork point + # Although this is more than 288 blocks, because this chain was written more recently + # and only its other 299 small and 220 large block are in the block files after it, + # its expected to still be retained + self.nodes[2].getblock(self.nodes[2].getblockhash(self.forkheight)) + + first_reorg_height = self.nodes[2].getblockcount() + curchainhash = self.nodes[2].getblockhash(self.mainchainheight) + self.nodes[2].invalidateblock(curchainhash) + goalbestheight = self.mainchainheight + goalbesthash = self.mainchainhash2 + + # As of 0.10 the current block download logic is not able to reorg to the original chain created in + # create_chain_with_stale_blocks because it doesn't know of any peer thats on that chain from which to + # redownload its missing blocks. + # Invalidate the reorg_test chain in node 0 as well, it can successfully switch to the original chain + # because it has all the block data. + # However it must mine enough blocks to have a more work chain than the reorg_test chain in order + # to trigger node 2's block download logic. + # At this point node 2 is within 288 blocks of the fork point so it will preserve its ability to reorg + if self.nodes[2].getblockcount() < self.mainchainheight: + blocks_to_mine = first_reorg_height + 1 - self.mainchainheight + print "Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed:", blocks_to_mine + self.nodes[0].invalidateblock(curchainhash) + assert(self.nodes[0].getblockcount() == self.mainchainheight) + assert(self.nodes[0].getbestblockhash() == self.mainchainhash2) + goalbesthash = self.nodes[0].generate(blocks_to_mine)[-1] + goalbestheight = first_reorg_height + 1 + + print "Verify node 2 reorged back to the main chain, some blocks of which it had to redownload" + waitstart = time.time() + while self.nodes[2].getblockcount() < goalbestheight: + time.sleep(0.1) + if time.time() - waitstart > 300: + raise AssertionError("Node 2 didn't reorg to proper height") + assert(self.nodes[2].getbestblockhash() == goalbesthash) + # Verify we can now have the data for a block previously pruned + assert(self.nodes[2].getblock(self.forkhash)["height"] == self.forkheight) + + def mine_full_block(self, node, address): + # Want to create a full block + # We'll generate a 66k transaction below, and 14 of them is close to the 1MB block limit + for j in xrange(14): + if len(self.utxo) < 14: + self.utxo = node.listunspent() + inputs=[] + outputs = {} + t = self.utxo.pop() + inputs.append({ "txid" : t["txid"], "vout" : t["vout"]}) + remchange = t["amount"] - Decimal("0.001000") + outputs[address]=remchange + # Create a basic transaction that will send change back to ourself after account for a fee + # And then insert the 128 generated transaction outs in the middle rawtx[92] is where the # + # of txouts is stored and is the only thing we overwrite from the original transaction + rawtx = node.createrawtransaction(inputs, outputs) + newtx = rawtx[0:92] + newtx = newtx + self.txouts + newtx = newtx + rawtx[94:] + # Appears to be ever so slightly faster to sign with SIGHASH_NONE + signresult = node.signrawtransaction(newtx,None,None,"NONE") + txid = node.sendrawtransaction(signresult["hex"], True) + # Mine a full sized block which will be these transactions we just created + node.generate(1) + + + def run_test(self): + print "Warning! This test requires 4GB of disk space and takes over 30 mins" + print "Mining a big blockchain of 995 blocks" + self.create_big_chain() + # Chain diagram key: + # * blocks on main chain + # +,&,$,@ blocks on other forks + # X invalidated block + # N1 Node 1 + # + # Start by mining a simple chain that all nodes have + # N0=N1=N2 **...*(995) + + print "Check that we haven't started pruning yet because we're below PruneAfterHeight" + self.test_height_min() + # Extend this chain past the PruneAfterHeight + # N0=N1=N2 **...*(1020) + + print "Check that we'll exceed disk space target if we have a very high stale block rate" + self.create_chain_with_staleblocks() + # Disconnect N0 + # And mine a 24 block chain on N1 and a separate 25 block chain on N0 + # N1=N2 **...*+...+(1044) + # N0 **...**...**(1045) + # + # reconnect nodes causing reorg on N1 and N2 + # N1=N2 **...*(1020) *...**(1045) + # \ + # +...+(1044) + # + # repeat this process until you have 12 stale forks hanging off the + # main chain on N1 and N2 + # N0 *************************...***************************(1320) + # + # N1=N2 **...*(1020) *...**(1045) *.. ..**(1295) *...**(1320) + # \ \ \ + # +...+(1044) &.. $...$(1319) + + # Save some current chain state for later use + self.mainchainheight = self.nodes[2].getblockcount() #1320 + self.mainchainhash2 = self.nodes[2].getblockhash(self.mainchainheight) + + print "Check that we can survive a 288 block reorg still" + (self.forkheight,self.forkhash) = self.reorg_test() #(1033, ) + # Now create a 288 block reorg by mining a longer chain on N1 + # First disconnect N1 + # Then invalidate 1033 on main chain and 1032 on fork so height is 1032 on main chain + # N1 **...*(1020) **...**(1032)X.. + # \ + # ++...+(1031)X.. + # + # Now mine 300 more blocks on N1 + # N1 **...*(1020) **...**(1032) @@...@(1332) + # \ \ + # \ X... + # \ \ + # ++...+(1031)X.. .. + # + # Reconnect nodes and mine 220 more blocks on N1 + # N1 **...*(1020) **...**(1032) @@...@@@(1552) + # \ \ + # \ X... + # \ \ + # ++...+(1031)X.. .. + # + # N2 **...*(1020) **...**(1032) @@...@@@(1552) + # \ \ + # \ *...**(1320) + # \ \ + # ++...++(1044) .. + # + # N0 ********************(1032) @@...@@@(1552) + # \ + # *...**(1320) + + print "Test that we can rerequest a block we previously pruned if needed for a reorg" + self.reorg_back() + # Verify that N2 still has block 1033 on current chain (@), but not on main chain (*) + # Invalidate 1033 on current chain (@) on N2 and we should be able to reorg to + # original main chain (*), but will require redownload of some blocks + # In order to have a peer we think we can download from, must also perform this invalidation + # on N0 and mine a new longest chain to trigger. + # Final result: + # N0 ********************(1032) **...****(1553) + # \ + # X@...@@@(1552) + # + # N2 **...*(1020) **...**(1032) **...****(1553) + # \ \ + # \ X@...@@@(1552) + # \ + # +.. + # + # N1 doesn't change because 1033 on main chain (*) is invalid + + print "Done" + +if __name__ == '__main__': + PruneTest().main() diff --git a/qa/rpc-tests/reindex.py b/qa/rpc-tests/reindex.py new file mode 100755 index 0000000000..fe767586bb --- /dev/null +++ b/qa/rpc-tests/reindex.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python2 +# Copyright (c) 2014 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test -reindex with CheckBlockIndex +# +from test_framework import BitcoinTestFramework +from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException +from util import * +import os.path + +class ReindexTest(BitcoinTestFramework): + + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 1) + + def setup_network(self): + self.nodes = [] + self.is_network_split = False + self.nodes.append(start_node(0, self.options.tmpdir)) + + def run_test(self): + self.nodes[0].generate(3) + stop_node(self.nodes[0], 0) + wait_bitcoinds() + self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug", "-reindex", "-checkblockindex=1"]) + assert_equal(self.nodes[0].getblockcount(), 3) + print "Success" + +if __name__ == '__main__': + ReindexTest().main() diff --git a/qa/rpc-tests/socks5.py b/qa/rpc-tests/socks5.py new file mode 100644 index 0000000000..1dbfb98d5d --- /dev/null +++ b/qa/rpc-tests/socks5.py @@ -0,0 +1,160 @@ +# Copyright (c) 2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +Dummy Socks5 server for testing. +''' +from __future__ import print_function, division, unicode_literals +import socket, threading, Queue +import traceback, sys + +### Protocol constants +class Command: + CONNECT = 0x01 + +class AddressType: + IPV4 = 0x01 + DOMAINNAME = 0x03 + IPV6 = 0x04 + +### Utility functions +def recvall(s, n): + '''Receive n bytes from a socket, or fail''' + rv = bytearray() + while n > 0: + d = s.recv(n) + if not d: + raise IOError('Unexpected end of stream') + rv.extend(d) + n -= len(d) + return rv + +### Implementation classes +class Socks5Configuration(object): + '''Proxy configuration''' + def __init__(self): + self.addr = None # Bind address (must be set) + self.af = socket.AF_INET # Bind address family + self.unauth = False # Support unauthenticated + self.auth = False # Support authentication + +class Socks5Command(object): + '''Information about an incoming socks5 command''' + def __init__(self, cmd, atyp, addr, port, username, password): + self.cmd = cmd # Command (one of Command.*) + self.atyp = atyp # Address type (one of AddressType.*) + self.addr = addr # Address + self.port = port # Port to connect to + self.username = username + self.password = password + def __repr__(self): + return 'Socks5Command(%s,%s,%s,%s,%s,%s)' % (self.cmd, self.atyp, self.addr, self.port, self.username, self.password) + +class Socks5Connection(object): + def __init__(self, serv, conn, peer): + self.serv = serv + self.conn = conn + self.peer = peer + + def handle(self): + ''' + Handle socks5 request according to RFC1928 + ''' + try: + # Verify socks version + ver = recvall(self.conn, 1)[0] + if ver != 0x05: + raise IOError('Invalid socks version %i' % ver) + # Choose authentication method + nmethods = recvall(self.conn, 1)[0] + methods = bytearray(recvall(self.conn, nmethods)) + method = None + if 0x02 in methods and self.serv.conf.auth: + method = 0x02 # username/password + elif 0x00 in methods and self.serv.conf.unauth: + method = 0x00 # unauthenticated + if method is None: + raise IOError('No supported authentication method was offered') + # Send response + self.conn.sendall(bytearray([0x05, method])) + # Read authentication (optional) + username = None + password = None + if method == 0x02: + ver = recvall(self.conn, 1)[0] + if ver != 0x01: + raise IOError('Invalid auth packet version %i' % ver) + ulen = recvall(self.conn, 1)[0] + username = str(recvall(self.conn, ulen)) + plen = recvall(self.conn, 1)[0] + password = str(recvall(self.conn, plen)) + # Send authentication response + self.conn.sendall(bytearray([0x01, 0x00])) + + # Read connect request + (ver,cmd,rsv,atyp) = recvall(self.conn, 4) + if ver != 0x05: + raise IOError('Invalid socks version %i in connect request' % ver) + if cmd != Command.CONNECT: + raise IOError('Unhandled command %i in connect request' % cmd) + + if atyp == AddressType.IPV4: + addr = recvall(self.conn, 4) + elif atyp == AddressType.DOMAINNAME: + n = recvall(self.conn, 1)[0] + addr = str(recvall(self.conn, n)) + elif atyp == AddressType.IPV6: + addr = recvall(self.conn, 16) + else: + raise IOError('Unknown address type %i' % atyp) + port_hi,port_lo = recvall(self.conn, 2) + port = (port_hi << 8) | port_lo + + # Send dummy response + self.conn.sendall(bytearray([0x05, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00])) + + cmdin = Socks5Command(cmd, atyp, addr, port, username, password) + self.serv.queue.put(cmdin) + print('Proxy: ', cmdin) + # Fall through to disconnect + except Exception,e: + traceback.print_exc(file=sys.stderr) + self.serv.queue.put(e) + finally: + self.conn.close() + +class Socks5Server(object): + def __init__(self, conf): + self.conf = conf + self.s = socket.socket(conf.af) + self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.s.bind(conf.addr) + self.s.listen(5) + self.running = False + self.thread = None + self.queue = Queue.Queue() # report connections and exceptions to client + + def run(self): + while self.running: + (sockconn, peer) = self.s.accept() + if self.running: + conn = Socks5Connection(self, sockconn, peer) + thread = threading.Thread(None, conn.handle) + thread.daemon = True + thread.start() + + def start(self): + assert(not self.running) + self.running = True + self.thread = threading.Thread(None, self.run) + self.thread.daemon = True + self.thread.start() + + def stop(self): + self.running = False + # connect to self to end run loop + s = socket.socket(self.conf.af) + s.connect(self.conf.addr) + s.close() + self.thread.join() + diff --git a/qa/rpc-tests/util.py b/qa/rpc-tests/util.py index 9ecee31959..cf789f48e2 100644 --- a/qa/rpc-tests/util.py +++ b/qa/rpc-tests/util.py @@ -158,7 +158,7 @@ def _rpchost_to_args(rpchost): rv += ['-rpcport=' + rpcport] return rv -def start_node(i, dirname, extra_args=None, rpchost=None): +def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None): """ Start a bitcoind and return RPC connection to it """ @@ -172,7 +172,10 @@ def start_node(i, dirname, extra_args=None, rpchost=None): ["-rpcwait", "getblockcount"], stdout=devnull) devnull.close() url = "http://rt:rt@%s:%d" % (rpchost or '127.0.0.1', rpc_port(i)) - proxy = AuthServiceProxy(url) + if timewait is not None: + proxy = AuthServiceProxy(url, timeout=timewait) + else: + proxy = AuthServiceProxy(url) proxy.url = url # store URL on proxy for info return proxy diff --git a/qa/rpc-tests/wallet.py b/qa/rpc-tests/wallet.py index 5f3178c606..08032fc538 100755 --- a/qa/rpc-tests/wallet.py +++ b/qa/rpc-tests/wallet.py @@ -151,6 +151,33 @@ class WalletTest (BitcoinTestFramework): assert(txid1 in self.nodes[3].getrawmempool()) + #check if we can list zero value tx as available coins + #1. create rawtx + #2. hex-changed one output to 0.0 + #3. sign and send + #4. check if recipient (node0) can list the zero value tx + usp = self.nodes[1].listunspent() + inputs = [{"txid":usp[0]['txid'], "vout":usp[0]['vout']}] + outputs = {self.nodes[1].getnewaddress(): 49.998, self.nodes[0].getnewaddress(): 11.11} + + rawTx = self.nodes[1].createrawtransaction(inputs, outputs).replace("c0833842", "00000000") #replace 11.11 with 0.0 (int32) + decRawTx = self.nodes[1].decoderawtransaction(rawTx) + signedRawTx = self.nodes[1].signrawtransaction(rawTx) + decRawTx = self.nodes[1].decoderawtransaction(signedRawTx['hex']) + zeroValueTxid= decRawTx['txid'] + sendResp = self.nodes[1].sendrawtransaction(signedRawTx['hex']) + + self.sync_all() + self.nodes[1].generate(1) #mine a block + self.sync_all() + + unspentTxs = self.nodes[0].listunspent() #zero value tx must be in listunspents output + found = False + for uTx in unspentTxs: + if uTx['txid'] == zeroValueTxid: + found = True + assert_equal(uTx['amount'], Decimal('0.00000000')); + assert(found) #do some -walletbroadcast tests stop_nodes(self.nodes) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 52ff3f224f..9aec13ccef 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -39,6 +39,7 @@ BITCOIN_TESTS =\ test/base32_tests.cpp \ test/base58_tests.cpp \ test/base64_tests.cpp \ + test/bip32_tests.cpp \ test/bloom_tests.cpp \ test/checkblock_tests.cpp \ test/Checkpoints_tests.cpp \ diff --git a/src/addrman.cpp b/src/addrman.cpp index 5d9527f0e1..c41ee3f9fc 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -67,9 +67,8 @@ double CAddrInfo::GetChance(int64_t nNow) const if (nSinceLastTry < 60 * 10) fChance *= 0.01; - // deprioritize 50% after each failed attempt - for (int n = 0; n < nAttempts; n++) - fChance /= 1.5; + // deprioritize 66% after each failed attempt, but at most 1/28th to avoid the search taking forever or overly penalizing outages. + fChance *= pow(0.66, min(nAttempts, 8)); return fChance; } @@ -332,10 +331,10 @@ void CAddrMan::Attempt_(const CService& addr, int64_t nTime) info.nAttempts++; } -CAddress CAddrMan::Select_() +CAddrInfo CAddrMan::Select_() { if (size() == 0) - return CAddress(); + return CAddrInfo(); // Use a 50% chance for choosing between tried and new table entries. if (nTried > 0 && (nNew == 0 || GetRandInt(2) == 0)) { diff --git a/src/addrman.h b/src/addrman.h index 8116d0b763..5badd4ef95 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -22,6 +22,10 @@ */ class CAddrInfo : public CAddress { +public: + //! last try whatsoever by us (memory only) + int64_t nLastTry; + private: //! where knowledge about this address first came from CNetAddr source; @@ -29,9 +33,6 @@ private: //! last successful connection by us int64_t nLastSuccess; - //! last try whatsoever by us: - // int64_t CAddress::nLastTry - //! connection attempts since last successful attempt int nAttempts; @@ -231,7 +232,7 @@ protected: //! Select an address to connect to. //! nUnkBias determines how much to favor new addresses over tried ones (min=0, max=100) - CAddress Select_(); + CAddrInfo Select_(); #ifdef DEBUG_ADDRMAN //! Perform consistency check. Returns an error code or zero. @@ -533,9 +534,9 @@ public: * Choose an address to connect to. * nUnkBias determines how much "new" entries are favored over "tried" ones (0-100). */ - CAddress Select() + CAddrInfo Select() { - CAddress addrRet; + CAddrInfo addrRet; { LOCK(cs); Check(); diff --git a/src/arith_uint256.h b/src/arith_uint256.h index b6ba3a1087..103c78bb8e 100644 --- a/src/arith_uint256.h +++ b/src/arith_uint256.h @@ -287,4 +287,4 @@ public: uint256 ArithToUint256(const arith_uint256 &); arith_uint256 UintToArith256(const uint256 &); -#endif // BITCOIN_UINT256_H +#endif // BITCOIN_ARITH_UINT256_H diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index a0a96c2dfa..2172f4a21b 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -21,7 +21,7 @@ * * \section intro_sec Introduction * - * This is the developer documentation of the reference client for an experimental new digital currency called Bitcoin (http://www.bitcoin.org/), + * This is the developer documentation of the reference client for an experimental new digital currency called Bitcoin (https://www.bitcoin.org/), * which enables instant payments to anyone, anywhere in the world. Bitcoin uses peer-to-peer technology to operate * with no central authority: managing transactions and issuing money are carried out collectively by the network. * diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 97312b366d..589c7b5472 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -121,6 +121,7 @@ public: vAlertPubKey = ParseHex("04fc9702847840aaf195de8442ebecedf5b095cdbb9bc716bda9110971b28a49e0ead8564ff0db22209e0374782c093bb899692d524e9d6a6956e7c5ecbcd68284"); nDefaultPort = 8333; nMinerThreads = 0; + nPruneAfterHeight = 100000; /** * Build the genesis block. Note that the output of the genesis coinbase cannot @@ -151,11 +152,11 @@ public: assert(consensus.hashGenesisBlock == uint256S("0x000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f")); assert(genesis.hashMerkleRoot == uint256S("0x4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b")); - vSeeds.push_back(CDNSSeedData("bitcoin.sipa.be", "seed.bitcoin.sipa.be")); - vSeeds.push_back(CDNSSeedData("bluematt.me", "dnsseed.bluematt.me")); - vSeeds.push_back(CDNSSeedData("dashjr.org", "dnsseed.bitcoin.dashjr.org")); - vSeeds.push_back(CDNSSeedData("bitcoinstats.com", "seed.bitcoinstats.com")); - vSeeds.push_back(CDNSSeedData("xf2.org", "bitseed.xf2.org")); + vSeeds.push_back(CDNSSeedData("bitcoin.sipa.be", "seed.bitcoin.sipa.be")); // Pieter Wuille + vSeeds.push_back(CDNSSeedData("bluematt.me", "dnsseed.bluematt.me")); // Matt Corallo + vSeeds.push_back(CDNSSeedData("dashjr.org", "dnsseed.bitcoin.dashjr.org")); // Luke Dashjr + vSeeds.push_back(CDNSSeedData("bitcoinstats.com", "seed.bitcoinstats.com")); // Addy Yeow + vSeeds.push_back(CDNSSeedData("xf2.org", "bitseed.xf2.org")); // Jeff Garzik base58Prefixes[PUBKEY_ADDRESS] = std::vector<unsigned char>(1,0); base58Prefixes[SCRIPT_ADDRESS] = std::vector<unsigned char>(1,5); @@ -198,6 +199,7 @@ public: vAlertPubKey = ParseHex("04302390343f91cc401d56d68b123028bf52e5fca1939df127f63c6467cdf9c8e2c14b61104cf817d0b780da337893ecc4aaff1309e536162dabbdb45200ca2b0a"); nDefaultPort = 18333; nMinerThreads = 0; + nPruneAfterHeight = 1000; //! Modify the testnet genesis block so the timestamp is valid for a later start. genesis.nTime = 1296688602; @@ -257,6 +259,7 @@ public: consensus.hashGenesisBlock = genesis.GetHash(); nDefaultPort = 18444; assert(consensus.hashGenesisBlock == uint256S("0x0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206")); + nPruneAfterHeight = 1000; vFixedSeeds.clear(); //! Regtest mode doesn't have any fixed seeds. vSeeds.clear(); //! Regtest mode doesn't have any DNS seeds. diff --git a/src/chainparams.h b/src/chainparams.h index 0692094478..5e974123dc 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -58,6 +58,7 @@ public: bool DefaultConsistencyChecks() const { return fDefaultConsistencyChecks; } /** Make standard checks */ bool RequireStandard() const { return fRequireStandard; } + int64_t PruneAfterHeight() const { return nPruneAfterHeight; } /** Make miner stop after a block is found. In RPC, don't return until nGenProcLimit blocks are generated */ bool MineBlocksOnDemand() const { return fMineBlocksOnDemand; } /** In the future use NetworkIDString() for RPC fields */ @@ -77,6 +78,7 @@ protected: std::vector<unsigned char> vAlertPubKey; int nDefaultPort; int nMinerThreads; + uint64_t nPruneAfterHeight; std::vector<CDNSSeedData> vSeeds; std::vector<unsigned char> base58Prefixes[MAX_BASE58_TYPES]; std::string strNetworkID; diff --git a/src/consensus/params.h b/src/consensus/params.h index 35d447b712..c480a1cce1 100644 --- a/src/consensus/params.h +++ b/src/consensus/params.h @@ -3,8 +3,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_CONSENSUS_CONSENSUS_PARAMS_H -#define BITCOIN_CONSENSUS_CONSENSUS_PARAMS_H +#ifndef BITCOIN_CONSENSUS_PARAMS_H +#define BITCOIN_CONSENSUS_PARAMS_H #include "uint256.h" @@ -28,4 +28,4 @@ struct Params { }; } // namespace Consensus -#endif // BITCOIN_CONSENSUS_CONSENSUS_PARAMS_H +#endif // BITCOIN_CONSENSUS_PARAMS_H diff --git a/src/init.cpp b/src/init.cpp index 4d9c233c8c..b648a237bf 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -123,7 +123,7 @@ public: LogPrintf("Error reading from database: %s\n", e.what()); // Starting the shutdown sequence and returning false to the caller would be // interpreted as 'entry not found' (as opposed to unable to read data), and - // could lead to invalid interpration. Just exit immediately, as we can't + // could lead to invalid interpretation. Just exit immediately, as we can't // continue anyway, and all writes should be atomic. abort(); } @@ -275,7 +275,12 @@ std::string HelpMessage(HelpMessageMode mode) #ifndef WIN32 strUsage += HelpMessageOpt("-pid=<file>", strprintf(_("Specify pid file (default: %s)"), "bitcoind.pid")); #endif - strUsage += HelpMessageOpt("-reindex", _("Rebuild block chain index from current blk000??.dat files") + " " + _("on startup")); + strUsage += HelpMessageOpt("-prune=<n>", _("Reduce storage requirements by pruning (deleting) old blocks. This mode disables wallet support and is incompatible with -txindex.") + " " + + _("Warning: Reverting this setting requires re-downloading the entire blockchain.") + " " + + _("(default: 0 = disable pruning blocks,") + " " + + strprintf(_(">%u = target size in MiB to use for block files)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024)); +strUsage += HelpMessageOpt("-reindex", _("Rebuild block chain index from current blk000??.dat files") + " " + _("on startup")); + #if !defined(WIN32) strUsage += HelpMessageOpt("-sysperms", _("Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)")); #endif @@ -301,6 +306,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-permitbaremultisig", strprintf(_("Relay non-P2SH multisig (default: %u)"), 1)); strUsage += HelpMessageOpt("-port=<port>", strprintf(_("Listen for connections on <port> (default: %u or testnet: %u)"), 8333, 18333)); strUsage += HelpMessageOpt("-proxy=<ip:port>", _("Connect through SOCKS5 proxy")); + strUsage += HelpMessageOpt("-proxyrandomize", strprintf(_("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)"), 1)); strUsage += HelpMessageOpt("-seednode=<ip>", _("Connect to a node to retrieve peer addresses, and disconnect")); strUsage += HelpMessageOpt("-timeout=<n>", strprintf(_("Specify connection timeout in milliseconds (minimum: 1, default: %d)"), DEFAULT_CONNECT_TIMEOUT)); #ifdef USE_UPNP @@ -351,7 +357,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-flushwallet", strprintf(_("Run a thread to flush wallet periodically (default: %u)"), 1)); strUsage += HelpMessageOpt("-stopafterblockimport", strprintf(_("Stop running after importing blocks from disk (default: %u)"), 0)); } - string debugCategories = "addrman, alert, bench, coindb, db, lock, rand, rpc, selectcoins, mempool, net"; // Don't translate these and qt below + string debugCategories = "addrman, alert, bench, coindb, db, lock, rand, rpc, selectcoins, mempool, net, proxy, prune"; // Don't translate these and qt below if (mode == HMM_BITCOIN_QT) debugCategories += ", qt"; strUsage += HelpMessageOpt("-debug=<category>", strprintf(_("Output debugging information (default: %u, supplying <category> is optional)"), 0) + ". " + @@ -365,8 +371,8 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-logtimestamps", strprintf(_("Prepend debug output with timestamp (default: %u)"), 1)); if (GetBoolArg("-help-debug", false)) { - strUsage += HelpMessageOpt("-limitfreerelay=<n>", strprintf(_("Continuously rate-limit free transactions to <n>*1000 bytes per minute (default:%u)"), 15)); - strUsage += HelpMessageOpt("-relaypriority", strprintf(_("Require high priority for relaying free or low-fee transactions (default:%u)"), 1)); + strUsage += HelpMessageOpt("-limitfreerelay=<n>", strprintf(_("Continuously rate-limit free transactions to <n>*1000 bytes per minute (default: %u)"), 15)); + strUsage += HelpMessageOpt("-relaypriority", strprintf(_("Require high priority for relaying free or low-fee transactions (default: %u)"), 1)); strUsage += HelpMessageOpt("-maxsigcachesize=<n>", strprintf(_("Limit size of signature cache to <n> entries (default: %u)"), 50000)); } strUsage += HelpMessageOpt("-minrelaytxfee=<amt>", strprintf(_("Fees (in BTC/Kb) smaller than this are considered zero fee for relaying (default: %s)"), FormatMoney(::minRelayTxFee.GetFeePerK()))); @@ -457,10 +463,33 @@ struct CImportingNow } }; + +// If we're using -prune with -reindex, then delete block files that will be ignored by the +// reindex. Since reindexing works by starting at block file 0 and looping until a blockfile +// is missing, and since pruning works by deleting the oldest block file first, just check +// for block file 0, and if it doesn't exist, delete all the block files in the +// directory (since they won't be read by the reindex but will take up disk space). +void DeleteAllBlockFiles() +{ + if (boost::filesystem::exists(GetBlockPosFilename(CDiskBlockPos(0, 0), "blk"))) + return; + + LogPrintf("Removing all blk?????.dat and rev?????.dat files for -reindex with -prune\n"); + boost::filesystem::path blocksdir = GetDataDir() / "blocks"; + for (boost::filesystem::directory_iterator it(blocksdir); it != boost::filesystem::directory_iterator(); it++) { + if (is_regular_file(*it)) { + if ((it->path().filename().string().length() == 12) && + (it->path().filename().string().substr(8,4) == ".dat") && + ((it->path().filename().string().substr(0,3) == "blk") || + (it->path().filename().string().substr(0,3) == "rev"))) + boost::filesystem::remove(it->path()); + } + } +} + void ThreadImport(std::vector<boost::filesystem::path> vImportFiles) { RenameThread("bitcoin-loadblk"); - // -reindex if (fReindex) { CImportingNow imp; @@ -673,6 +702,21 @@ bool AppInit2(boost::thread_group& threadGroup) if (nFD - MIN_CORE_FILEDESCRIPTORS < nMaxConnections) nMaxConnections = nFD - MIN_CORE_FILEDESCRIPTORS; + // if using block pruning, then disable txindex + // also disable the wallet (for now, until SPV support is implemented in wallet) + if (GetArg("-prune", 0)) { + if (GetBoolArg("-txindex", false)) + return InitError(_("Prune mode is incompatible with -txindex.")); +#ifdef ENABLE_WALLET + if (!GetBoolArg("-disablewallet", false)) { + if (SoftSetBoolArg("-disablewallet", true)) + LogPrintf("%s : parameter interaction: -prune -> setting -disablewallet=1\n", __func__); + else + return InitError(_("Can't run with a wallet in prune mode.")); + } +#endif + } + // ********************************************************* Step 3: parameter-to-internal-flags fDebug = !mapMultiArgs["-debug"].empty(); @@ -709,6 +753,21 @@ bool AppInit2(boost::thread_group& threadGroup) nScriptCheckThreads = MAX_SCRIPTCHECK_THREADS; fServer = GetBoolArg("-server", false); + + // block pruning; get the amount of disk space (in MB) to allot for block & undo files + int64_t nSignedPruneTarget = GetArg("-prune", 0) * 1024 * 1024; + if (nSignedPruneTarget < 0) { + return InitError(_("Prune cannot be configured with a negative value.")); + } + nPruneTarget = (uint64_t) nSignedPruneTarget; + if (nPruneTarget) { + if (nPruneTarget < MIN_DISK_SPACE_FOR_BLOCK_FILES) { + return InitError(strprintf(_("Prune configured below the minimum of %d MB. Please use a higher number."), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024)); + } + LogPrintf("Prune configured to target %uMiB on disk for block and undo files.\n", nPruneTarget / 1024 / 1024); + fPruneMode = true; + } + #ifdef ENABLE_WALLET bool fDisableWallet = GetBoolArg("-disablewallet", false); #endif @@ -891,10 +950,10 @@ bool AppInit2(boost::thread_group& threadGroup) } } - CService addrProxy; + proxyType addrProxy; bool fProxy = false; if (mapArgs.count("-proxy")) { - addrProxy = CService(mapArgs["-proxy"], 9050); + addrProxy = proxyType(CService(mapArgs["-proxy"], 9050), GetArg("-proxyrandomize", true)); if (!addrProxy.IsValid()) return InitError(strprintf(_("Invalid -proxy address: '%s'"), mapArgs["-proxy"])); @@ -904,14 +963,14 @@ bool AppInit2(boost::thread_group& threadGroup) fProxy = true; } - // -onion can override normal proxy, -noonion disables tor entirely + // -onion can override normal proxy, -noonion disables connecting to .onion entirely if (!(mapArgs.count("-onion") && mapArgs["-onion"] == "0") && (fProxy || mapArgs.count("-onion"))) { - CService addrOnion; + proxyType addrOnion; if (!mapArgs.count("-onion")) addrOnion = addrProxy; else - addrOnion = CService(mapArgs["-onion"], 9050); + addrOnion = proxyType(CService(mapArgs["-onion"], 9050), GetArg("-proxyrandomize", true)); if (!addrOnion.IsValid()) return InitError(strprintf(_("Invalid -onion address: '%s'"), mapArgs["-onion"])); SetProxy(NET_TOR, addrOnion); @@ -1029,8 +1088,12 @@ bool AppInit2(boost::thread_group& threadGroup) pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview); pcoinsTip = new CCoinsViewCache(pcoinscatcher); - if (fReindex) + if (fReindex) { pblocktree->WriteReindexing(true); + //If we're reindexing in prune mode, wipe away all our block and undo data files + if (fPruneMode) + DeleteAllBlockFiles(); + } if (!LoadBlockIndex()) { strLoadError = _("Error loading block database"); @@ -1054,7 +1117,18 @@ bool AppInit2(boost::thread_group& threadGroup) break; } + // Check for changed -prune state. What we are concerned about is a user who has pruned blocks + // in the past, but is now trying to run unpruned. + if (fHavePruned && !fPruneMode) { + strLoadError = _("You need to rebuild the database using -reindex to go back to unpruned mode. This will redownload the entire blockchain"); + break; + } + uiInterface.InitMessage(_("Verifying blocks...")); + if (fHavePruned && GetArg("-checkblocks", 288) > MIN_BLOCKS_TO_KEEP) { + LogPrintf("Prune: pruned datadir may not have more than %d blocks; -checkblocks=%d may fail\n", + MIN_BLOCKS_TO_KEEP, GetArg("-checkblocks", 288)); + } if (!CVerifyDB().VerifyDB(pcoinsdbview, GetArg("-checklevel", 3), GetArg("-checkblocks", 288))) { strLoadError = _("Corrupted block database detected"); @@ -1105,6 +1179,15 @@ bool AppInit2(boost::thread_group& threadGroup) mempool.ReadFeeEstimates(est_filein); fFeeEstimatesInitialized = true; + // if prune mode, unset NODE_NETWORK and prune block files + if (fPruneMode) { + LogPrintf("Unsetting NODE_NETWORK on prune mode\n"); + nLocalServices &= ~NODE_NETWORK; + if (!fReindex) { + PruneAndFlush(); + } + } + // ********************************************************* Step 8: load wallet #ifdef ENABLE_WALLET if (fDisableWallet) { diff --git a/src/main.cpp b/src/main.cpp index f0aa3022cd..bf32ac91e7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -52,9 +52,12 @@ int nScriptCheckThreads = 0; bool fImporting = false; bool fReindex = false; bool fTxIndex = false; +bool fHavePruned = false; +bool fPruneMode = false; bool fIsBareMultisigStd = true; bool fCheckBlockIndex = false; unsigned int nCoinCacheSize = 5000; +uint64_t nPruneTarget = 0; /** Fees smaller than this (in satoshi) are considered zero fee (for relaying and mining) */ CFeeRate minRelayTxFee = CFeeRate(1000); @@ -110,17 +113,25 @@ namespace { /** * The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself and all ancestors) and - * as good as our current tip or better. Entries may be failed, though. + * as good as our current tip or better. Entries may be failed, though, and pruning nodes may be + * missing the data for the block. */ set<CBlockIndex*, CBlockIndexWorkComparator> setBlockIndexCandidates; /** Number of nodes with fSyncStarted. */ int nSyncStarted = 0; - /** All pairs A->B, where A (or one if its ancestors) misses transactions, but B has transactions. */ + /** All pairs A->B, where A (or one if its ancestors) misses transactions, but B has transactions. + * Pruned nodes may have entries where B is missing data. + */ multimap<CBlockIndex*, CBlockIndex*> mapBlocksUnlinked; CCriticalSection cs_LastBlockFile; std::vector<CBlockFileInfo> vinfoBlockFile; int nLastBlockFile = 0; + /** Global flag to indicate we should check to see if there are + * block/undo files that should be deleted. Set on startup + * or if we allocate more file space when we're in prune mode + */ + bool fCheckForPruning = false; /** * Every received block is assigned a unique and increasing identifier, so we @@ -1233,14 +1244,14 @@ void CheckForkWarningConditions() } if (pindexBestForkTip && pindexBestForkBase) { - LogPrintf("CheckForkWarningConditions: Warning: Large valid fork found\n forking the chain at height %d (%s)\n lasting to height %d (%s).\nChain state database corruption likely.\n", + LogPrintf("%s: Warning: Large valid fork found\n forking the chain at height %d (%s)\n lasting to height %d (%s).\nChain state database corruption likely.\n", __func__, pindexBestForkBase->nHeight, pindexBestForkBase->phashBlock->ToString(), pindexBestForkTip->nHeight, pindexBestForkTip->phashBlock->ToString()); fLargeWorkForkFound = true; } else { - LogPrintf("CheckForkWarningConditions: Warning: Found invalid chain at least ~6 blocks longer than our best chain.\nChain state database corruption likely.\n"); + LogPrintf("%s: Warning: Found invalid chain at least ~6 blocks longer than our best chain.\nChain state database corruption likely.\n", __func__); fLargeWorkInvalidChainFound = true; } } @@ -1298,10 +1309,10 @@ void Misbehaving(NodeId pnode, int howmuch) int banscore = GetArg("-banscore", 100); if (state->nMisbehavior >= banscore && state->nMisbehavior - howmuch < banscore) { - LogPrintf("Misbehaving: %s (%d -> %d) BAN THRESHOLD EXCEEDED\n", state->name, state->nMisbehavior-howmuch, state->nMisbehavior); + LogPrintf("%s: %s (%d -> %d) BAN THRESHOLD EXCEEDED\n", __func__, state->name, state->nMisbehavior-howmuch, state->nMisbehavior); state->fShouldBan = true; } else - LogPrintf("Misbehaving: %s (%d -> %d)\n", state->name, state->nMisbehavior-howmuch, state->nMisbehavior); + LogPrintf("%s: %s (%d -> %d)\n", __func__, state->name, state->nMisbehavior-howmuch, state->nMisbehavior); } void static InvalidChainFound(CBlockIndex* pindexNew) @@ -1309,11 +1320,11 @@ void static InvalidChainFound(CBlockIndex* pindexNew) if (!pindexBestInvalid || pindexNew->nChainWork > pindexBestInvalid->nChainWork) pindexBestInvalid = pindexNew; - LogPrintf("InvalidChainFound: invalid block=%s height=%d log2_work=%.8g date=%s\n", + LogPrintf("%s: invalid block=%s height=%d log2_work=%.8g date=%s\n", __func__, pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, log(pindexNew->nChainWork.getdouble())/log(2.0), DateTimeStrFormat("%Y-%m-%d %H:%M:%S", pindexNew->GetBlockTime())); - LogPrintf("InvalidChainFound: current best=%s height=%d log2_work=%.8g date=%s\n", + LogPrintf("%s: current best=%s height=%d log2_work=%.8g date=%s\n", __func__, chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime())); CheckForkWarningConditions(); @@ -1849,6 +1860,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin } enum FlushStateMode { + FLUSH_STATE_NONE, FLUSH_STATE_IF_NEEDED, FLUSH_STATE_PERIODIC, FLUSH_STATE_ALWAYS @@ -1856,16 +1868,30 @@ enum FlushStateMode { /** * Update the on-disk chain state. - * The caches and indexes are flushed if either they're too large, forceWrite is set, or - * fast is not set and it's been a while since the last write. + * The caches and indexes are flushed depending on the mode we're called with + * if they're too large, if it's been a while since the last write, + * or always and in all cases if we're in prune mode and are deleting files. */ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { - LOCK(cs_main); + LOCK2(cs_main, cs_LastBlockFile); static int64_t nLastWrite = 0; + std::set<int> setFilesToPrune; + bool fFlushForPrune = false; try { + if (fPruneMode && fCheckForPruning) { + FindFilesToPrune(setFilesToPrune); + if (!setFilesToPrune.empty()) { + fFlushForPrune = true; + if (!fHavePruned) { + pblocktree->WriteFlag("prunedblockfiles", true); + fHavePruned = true; + } + } + } if ((mode == FLUSH_STATE_ALWAYS) || ((mode == FLUSH_STATE_PERIODIC || mode == FLUSH_STATE_IF_NEEDED) && pcoinsTip->GetCacheSize() > nCoinCacheSize) || - (mode == FLUSH_STATE_PERIODIC && GetTimeMicros() > nLastWrite + DATABASE_WRITE_INTERVAL * 1000000)) { + (mode == FLUSH_STATE_PERIODIC && GetTimeMicros() > nLastWrite + DATABASE_WRITE_INTERVAL * 1000000) || + fFlushForPrune) { // Typical CCoins structures on disk are around 100 bytes in size. // Pushing a new one to the database can cause it to be written // twice (once in the log, and once in the tables). This is already @@ -1893,9 +1919,16 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { return state.Abort("Files to write to block index database"); } } - // Finally flush the chainstate (which may refer to block index entries). + // Flush the chainstate (which may refer to block index entries). if (!pcoinsTip->Flush()) return state.Abort("Failed to write to coin database"); + + // Finally remove any pruned files + if (fFlushForPrune) { + UnlinkPrunedFiles(setFilesToPrune); + fCheckForPruning = false; + } + // Update best block in wallet (so we can detect restored wallets). if (mode != FLUSH_STATE_IF_NEEDED) { GetMainSignals().SetBestChain(chainActive.GetLocator()); @@ -1913,6 +1946,12 @@ void FlushStateToDisk() { FlushStateToDisk(state, FLUSH_STATE_ALWAYS); } +void PruneAndFlush() { + CValidationState state; + fCheckForPruning = true; + FlushStateToDisk(state, FLUSH_STATE_NONE); +} + /** Update chainActive and related internal data structures. */ void static UpdateTip(CBlockIndex *pindexNew) { chainActive.SetTip(pindexNew); @@ -1921,7 +1960,7 @@ void static UpdateTip(CBlockIndex *pindexNew) { nTimeBestReceived = GetTime(); mempool.AddTransactionsUpdated(1); - LogPrintf("UpdateTip: new best=%s height=%d log2_work=%.8g tx=%lu date=%s progress=%f cache=%u\n", + LogPrintf("%s: new best=%s height=%d log2_work=%.8g tx=%lu date=%s progress=%f cache=%u\n", __func__, chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), Checkpoints::GuessVerificationProgress(chainActive.Tip()), (unsigned int)pcoinsTip->GetCacheSize()); @@ -1941,7 +1980,7 @@ void static UpdateTip(CBlockIndex *pindexNew) { pindex = pindex->pprev; } if (nUpgraded > 0) - LogPrintf("SetBestChain: %d of last 100 blocks above version %d\n", nUpgraded, (int)CBlock::CURRENT_VERSION); + LogPrintf("%s: %d of last 100 blocks above version %d\n", __func__, nUpgraded, (int)CBlock::CURRENT_VERSION); if (nUpgraded > 100/2) { // strMiscWarning is read by GetWarnings(), called by Qt and the JSON-RPC code to warn the user: @@ -2083,15 +2122,29 @@ static CBlockIndex* FindMostWorkChain() { CBlockIndex *pindexTest = pindexNew; bool fInvalidAncestor = false; while (pindexTest && !chainActive.Contains(pindexTest)) { - assert(pindexTest->nStatus & BLOCK_HAVE_DATA); assert(pindexTest->nChainTx || pindexTest->nHeight == 0); - if (pindexTest->nStatus & BLOCK_FAILED_MASK) { - // Candidate has an invalid ancestor, remove entire chain from the set. - if (pindexBestInvalid == NULL || pindexNew->nChainWork > pindexBestInvalid->nChainWork) + + // Pruned nodes may have entries in setBlockIndexCandidates for + // which block files have been deleted. Remove those as candidates + // for the most work chain if we come across them; we can't switch + // to a chain unless we have all the non-active-chain parent blocks. + bool fFailedChain = pindexTest->nStatus & BLOCK_FAILED_MASK; + bool fMissingData = !(pindexTest->nStatus & BLOCK_HAVE_DATA); + if (fFailedChain || fMissingData) { + // Candidate chain is not usable (either invalid or missing data) + if (fFailedChain && (pindexBestInvalid == NULL || pindexNew->nChainWork > pindexBestInvalid->nChainWork)) pindexBestInvalid = pindexNew; CBlockIndex *pindexFailed = pindexNew; + // Remove the entire chain from the set. while (pindexTest != pindexFailed) { - pindexFailed->nStatus |= BLOCK_FAILED_CHILD; + if (fFailedChain) { + pindexFailed->nStatus |= BLOCK_FAILED_CHILD; + } else if (fMissingData) { + // If we're missing data, then add back to mapBlocksUnlinked, + // so that if the block arrives in the future we can try adding + // to setBlockIndexCandidates again. + mapBlocksUnlinked.insert(std::make_pair(pindexFailed->pprev, pindexFailed)); + } setBlockIndexCandidates.erase(pindexFailed); pindexFailed = pindexFailed->pprev; } @@ -2219,7 +2272,9 @@ bool ActivateBestChain(CValidationState &state, CBlock *pblock) { uint256 hashNewTip = pindexNewTip->GetBlockHash(); // Relay inventory, but don't relay old inventory during initial block download. int nBlockEstimate = Checkpoints::GetTotalBlocksEstimate(); - { + // Don't relay blocks if pruning -- could cause a peer to try to download, resulting + // in a stalled download if the block file is pruned before the request. + if (nLocalServices & NODE_NETWORK) { LOCK(cs_vNodes); BOOST_FOREACH(CNode* pnode, vNodes) if (chainActive.Height() > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : nBlockEstimate)) @@ -2350,10 +2405,6 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl pindexNew->nUndoPos = 0; pindexNew->nStatus |= BLOCK_HAVE_DATA; pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); - { - LOCK(cs_nBlockSequenceId); - pindexNew->nSequenceId = nBlockSequenceId++; - } setDirtyBlockIndex.insert(pindexNew); if (pindexNew->pprev == NULL || pindexNew->pprev->nChainTx) { @@ -2366,6 +2417,10 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl CBlockIndex *pindex = queue.front(); queue.pop_front(); pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; + { + LOCK(cs_nBlockSequenceId); + pindex->nSequenceId = nBlockSequenceId++; + } if (chainActive.Tip() == NULL || !setBlockIndexCandidates.value_comp()(pindex, chainActive.Tip())) { setBlockIndexCandidates.insert(pindex); } @@ -2419,6 +2474,8 @@ bool FindBlockPos(CValidationState &state, CDiskBlockPos &pos, unsigned int nAdd unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; unsigned int nNewChunks = (vinfoBlockFile[nFile].nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; if (nNewChunks > nOldChunks) { + if (fPruneMode) + fCheckForPruning = true; if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos)) { FILE *file = OpenBlockFile(pos); if (file) { @@ -2450,6 +2507,8 @@ bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigne unsigned int nOldChunks = (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; unsigned int nNewChunks = (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; if (nNewChunks > nOldChunks) { + if (fPruneMode) + fCheckForPruning = true; if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos)) { FILE *file = OpenUndoFile(pos); if (file) { @@ -2665,7 +2724,10 @@ bool AcceptBlock(CBlock& block, CValidationState& state, CBlockIndex** ppindex, if (!AcceptBlockHeader(block, state, &pindex)) return false; - if (pindex->nStatus & BLOCK_HAVE_DATA) { + // If we're pruning, ensure that we don't allow a peer to dump a copy + // of old blocks. But we might need blocks that are not on the main chain + // to handle a reorg, even if we've processed once. + if (pindex->nStatus & BLOCK_HAVE_DATA || chainActive.Contains(pindex)) { // TODO: deal better with duplicate blocks. // return state.DoS(20, error("AcceptBlock(): already have block %d %s", pindex->nHeight, pindex->GetBlockHash().ToString()), REJECT_DUPLICATE, "duplicate"); return true; @@ -2698,6 +2760,9 @@ bool AcceptBlock(CBlock& block, CValidationState& state, CBlockIndex** ppindex, return state.Abort(std::string("System error: ") + e.what()); } + if (fCheckForPruning) + FlushStateToDisk(state, FLUSH_STATE_NONE); // we just allocated more disk space for block files + return true; } @@ -2785,6 +2850,112 @@ bool AbortNode(const std::string &strMessage, const std::string &userMessage) { return false; } + +/** + * BLOCK PRUNING CODE + */ + +/* Calculate the amount of disk space the block & undo files currently use */ +uint64_t CalculateCurrentUsage() +{ + uint64_t retval = 0; + BOOST_FOREACH(const CBlockFileInfo &file, vinfoBlockFile) { + retval += file.nSize + file.nUndoSize; + } + return retval; +} + +/* Prune a block file (modify associated database entries)*/ +void PruneOneBlockFile(const int fileNumber) +{ + for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); ++it) { + CBlockIndex* pindex = it->second; + if (pindex->nFile == fileNumber) { + pindex->nStatus &= ~BLOCK_HAVE_DATA; + pindex->nStatus &= ~BLOCK_HAVE_UNDO; + pindex->nFile = 0; + pindex->nDataPos = 0; + pindex->nUndoPos = 0; + setDirtyBlockIndex.insert(pindex); + + // Prune from mapBlocksUnlinked -- any block we prune would have + // to be downloaded again in order to consider its chain, at which + // point it would be considered as a candidate for + // mapBlocksUnlinked or setBlockIndexCandidates. + std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> range = mapBlocksUnlinked.equal_range(pindex->pprev); + while (range.first != range.second) { + std::multimap<CBlockIndex *, CBlockIndex *>::iterator it = range.first; + range.first++; + if (it->second == pindex) { + mapBlocksUnlinked.erase(it); + } + } + } + } + + vinfoBlockFile[fileNumber].SetNull(); + setDirtyFileInfo.insert(fileNumber); +} + + +void UnlinkPrunedFiles(std::set<int>& setFilesToPrune) +{ + for (set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { + CDiskBlockPos pos(*it, 0); + boost::filesystem::remove(GetBlockPosFilename(pos, "blk")); + boost::filesystem::remove(GetBlockPosFilename(pos, "rev")); + LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); + } +} + +/* Calculate the block/rev files that should be deleted to remain under target*/ +void FindFilesToPrune(std::set<int>& setFilesToPrune) +{ + LOCK2(cs_main, cs_LastBlockFile); + if (chainActive.Tip() == NULL || nPruneTarget == 0) { + return; + } + if (chainActive.Tip()->nHeight <= Params().PruneAfterHeight()) { + return; + } + + unsigned int nLastBlockWeMustKeep = chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP; + uint64_t nCurrentUsage = CalculateCurrentUsage(); + // We don't check to prune until after we've allocated new space for files + // So we should leave a buffer under our target to account for another allocation + // before the next pruning. + uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE; + uint64_t nBytesToPrune; + int count=0; + + if (nCurrentUsage + nBuffer >= nPruneTarget) { + for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) { + nBytesToPrune = vinfoBlockFile[fileNumber].nSize + vinfoBlockFile[fileNumber].nUndoSize; + + if (vinfoBlockFile[fileNumber].nSize == 0) + continue; + + if (nCurrentUsage + nBuffer < nPruneTarget) // are we below our target? + break; + + // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip + if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeMustKeep) + break; + + PruneOneBlockFile(fileNumber); + // Queue up the files for removal + setFilesToPrune.insert(fileNumber); + nCurrentUsage -= nBytesToPrune; + count++; + } + } + + LogPrint("prune", "Prune: target=%dMiB actual=%dMiB diff=%dMiB min_must_keep=%d removed %d blk/rev pairs\n", + nPruneTarget/1024/1024, nCurrentUsage/1024/1024, + ((int64_t)nPruneTarget - (int64_t)nCurrentUsage)/1024/1024, + nLastBlockWeMustKeep, count); +} + bool CheckDiskSpace(uint64_t nAdditionalBytes) { uint64_t nFreeBytesAvailable = boost::filesystem::space(GetDataDir()).available; @@ -2872,7 +3043,9 @@ bool static LoadBlockIndexDB() { CBlockIndex* pindex = item.second; pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex); - if (pindex->nStatus & BLOCK_HAVE_DATA) { + // We can link the chain of blocks for which we've received transactions at some point. + // Pruned nodes may have deleted the block. + if (pindex->nTx > 0) { if (pindex->pprev) { if (pindex->pprev->nChainTx) { pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; @@ -2929,6 +3102,11 @@ bool static LoadBlockIndexDB() } } + // Check whether we have ever pruned block & undo files + pblocktree->ReadFlag("prunedblockfiles", fHavePruned); + if (fHavePruned) + LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n"); + // Check whether we need to continue reindexing bool fReindexing = false; pblocktree->ReadReindexing(fReindexing); @@ -2936,7 +3114,7 @@ bool static LoadBlockIndexDB() // Check whether we have a transaction index pblocktree->ReadFlag("txindex", fTxIndex); - LogPrintf("LoadBlockIndexDB(): transaction index %s\n", fTxIndex ? "enabled" : "disabled"); + LogPrintf("%s: transaction index %s\n", __func__, fTxIndex ? "enabled" : "disabled"); // Load pointer to end of best chain BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); @@ -2946,7 +3124,7 @@ bool static LoadBlockIndexDB() PruneBlockIndexCandidates(); - LogPrintf("LoadBlockIndexDB(): hashBestChain=%s height=%d date=%s progress=%f\n", + LogPrintf("%s: hashBestChain=%s height=%d date=%s progress=%f\n", __func__, chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), Checkpoints::GuessVerificationProgress(chainActive.Tip())); @@ -3069,6 +3247,7 @@ void UnloadBlockIndex() delete entry.second; } mapBlockIndex.clear(); + fHavePruned = false; } bool LoadBlockIndex() @@ -3232,6 +3411,14 @@ void static CheckBlockIndex() LOCK(cs_main); + // During a reindex, we read the genesis block and call CheckBlockIndex before ActivateBestChain, + // so we have the genesis block in mapBlockIndex but no active chain. (A few of the tests when + // iterating the block tree require that chainActive has been initialized.) + if (chainActive.Height() < 0) { + assert(mapBlockIndex.size() <= 1); + return; + } + // Build forward-pointing map of the entire block tree. std::multimap<CBlockIndex*,CBlockIndex*> forward; for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); it++) { @@ -3252,6 +3439,7 @@ void static CheckBlockIndex() int nHeight = 0; CBlockIndex* pindexFirstInvalid = NULL; // Oldest ancestor of pindex which is invalid. CBlockIndex* pindexFirstMissing = NULL; // Oldest ancestor of pindex which does not have BLOCK_HAVE_DATA. + CBlockIndex* pindexFirstNeverProcessed = NULL; // Oldest ancestor of pindex for which nTx == 0. CBlockIndex* pindexFirstNotTreeValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_TREE (regardless of being valid or not). CBlockIndex* pindexFirstNotTransactionsValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_TRANSACTIONS (regardless of being valid or not). CBlockIndex* pindexFirstNotChainValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_CHAIN (regardless of being valid or not). @@ -3260,6 +3448,7 @@ void static CheckBlockIndex() nNodes++; if (pindexFirstInvalid == NULL && pindex->nStatus & BLOCK_FAILED_VALID) pindexFirstInvalid = pindex; if (pindexFirstMissing == NULL && !(pindex->nStatus & BLOCK_HAVE_DATA)) pindexFirstMissing = pindex; + if (pindexFirstNeverProcessed == NULL && pindex->nTx == 0) pindexFirstNeverProcessed = pindex; if (pindex->pprev != NULL && pindexFirstNotTreeValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TREE) pindexFirstNotTreeValid = pindex; if (pindex->pprev != NULL && pindexFirstNotTransactionsValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) pindexFirstNotTransactionsValid = pindex; if (pindex->pprev != NULL && pindexFirstNotChainValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_CHAIN) pindexFirstNotChainValid = pindex; @@ -3271,11 +3460,21 @@ void static CheckBlockIndex() assert(pindex->GetBlockHash() == consensusParams.hashGenesisBlock); // Genesis block's hash must match. assert(pindex == chainActive.Genesis()); // The current active chain's genesis block must be this block. } - // HAVE_DATA is equivalent to VALID_TRANSACTIONS and equivalent to nTx > 0 (we stored the number of transactions in the block) - assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0)); - assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); - // All parents having data is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to nChainTx being set. - assert((pindexFirstMissing != NULL) == (pindex->nChainTx == 0)); // nChainTx == 0 is used to signal that all parent block's transaction data is available. + if (pindex->nChainTx == 0) assert(pindex->nSequenceId == 0); // nSequenceId can't be set for blocks that aren't linked + // VALID_TRANSACTIONS is equivalent to nTx > 0 for all nodes (whether or not pruning has occurred). + // HAVE_DATA is only equivalent to nTx > 0 (or VALID_TRANSACTIONS) if no pruning has occurred. + if (!fHavePruned) { + // If we've never pruned, then HAVE_DATA should be equivalent to nTx > 0 + assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0)); + assert(pindexFirstMissing == pindexFirstNeverProcessed); + } else { + // If we have pruned, then we can only say that HAVE_DATA implies nTx > 0 + if (pindex->nStatus & BLOCK_HAVE_DATA) assert(pindex->nTx > 0); + } + if (pindex->nStatus & BLOCK_HAVE_UNDO) assert(pindex->nStatus & BLOCK_HAVE_DATA); + assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // This is pruning-independent. + // All parents having had data (at some point) is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to nChainTx being set. + assert((pindexFirstNeverProcessed != NULL) == (pindex->nChainTx == 0)); // nChainTx != 0 is used to signal that all parent blocks have been processed (but may have been pruned). assert((pindexFirstNotTransactionsValid != NULL) == (pindex->nChainTx == 0)); assert(pindex->nHeight == nHeight); // nHeight must be consistent. assert(pindex->pprev == NULL || pindex->nChainWork >= pindex->pprev->nChainWork); // For every block except the genesis block, the chainwork must be larger than the parent's. @@ -3288,11 +3487,20 @@ void static CheckBlockIndex() // Checks for not-invalid blocks. assert((pindex->nStatus & BLOCK_FAILED_MASK) == 0); // The failed mask cannot be set for blocks without invalid parents. } - if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && pindexFirstMissing == NULL) { - if (pindexFirstInvalid == NULL) { // If this block sorts at least as good as the current tip and is valid, it must be in setBlockIndexCandidates. - assert(setBlockIndexCandidates.count(pindex)); + if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && pindexFirstNeverProcessed == NULL) { + if (pindexFirstInvalid == NULL) { + // If this block sorts at least as good as the current tip and + // is valid and we have all data for its parents, it must be in + // setBlockIndexCandidates. chainActive.Tip() must also be there + // even if some data has been pruned. + if (pindexFirstMissing == NULL || pindex == chainActive.Tip()) { + assert(setBlockIndexCandidates.count(pindex)); + } + // If some parent is missing, then it could be that this block was in + // setBlockIndexCandidates but had to be removed because of the missing data. + // In this case it must be in mapBlocksUnlinked -- see test below. } - } else { // If this block sorts worse than the current tip, it cannot be in setBlockIndexCandidates. + } else { // If this block sorts worse than the current tip or some ancestor's block has never been seen, it cannot be in setBlockIndexCandidates. assert(setBlockIndexCandidates.count(pindex) == 0); } // Check whether this block is in mapBlocksUnlinked. @@ -3306,12 +3514,28 @@ void static CheckBlockIndex() } rangeUnlinked.first++; } - if (pindex->pprev && pindex->nStatus & BLOCK_HAVE_DATA && pindexFirstMissing != NULL) { - if (pindexFirstInvalid == NULL) { // If this block has block data available, some parent doesn't, and has no invalid parents, it must be in mapBlocksUnlinked. - assert(foundInUnlinked); + if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed != NULL && pindexFirstInvalid == NULL) { + // If this block has block data available, some parent was never received, and has no invalid parents, it must be in mapBlocksUnlinked. + assert(foundInUnlinked); + } + if (!(pindex->nStatus & BLOCK_HAVE_DATA)) assert(!foundInUnlinked); // Can't be in mapBlocksUnlinked if we don't HAVE_DATA + if (pindexFirstMissing == NULL) assert(!foundInUnlinked); // We aren't missing data for any parent -- cannot be in mapBlocksUnlinked. + if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed == NULL && pindexFirstMissing != NULL) { + // We HAVE_DATA for this block, have received data for all parents at some point, but we're currently missing data for some parent. + assert(fHavePruned); // We must have pruned. + // This block may have entered mapBlocksUnlinked if: + // - it has a descendant that at some point had more work than the + // tip, and + // - we tried switching to that descendant but were missing + // data for some intermediate block between chainActive and the + // tip. + // So if this block is itself better than chainActive.Tip() and it wasn't in + // setBlockIndexCandidates, then it must be in mapBlocksUnlinked. + if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && setBlockIndexCandidates.count(pindex) == 0) { + if (pindexFirstInvalid == NULL) { + assert(foundInUnlinked); + } } - } else { // If this block does not have block data available, or all parents do, it cannot be in mapBlocksUnlinked. - assert(!foundInUnlinked); } // assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow // End: actual consistency checks. @@ -3331,6 +3555,7 @@ void static CheckBlockIndex() // If pindex was the first with a certain property, unset the corresponding variable. if (pindex == pindexFirstInvalid) pindexFirstInvalid = NULL; if (pindex == pindexFirstMissing) pindexFirstMissing = NULL; + if (pindex == pindexFirstNeverProcessed) pindexFirstNeverProcessed = NULL; if (pindex == pindexFirstNotTreeValid) pindexFirstNotTreeValid = NULL; if (pindex == pindexFirstNotTransactionsValid) pindexFirstNotTransactionsValid = NULL; if (pindex == pindexFirstNotChainValid) pindexFirstNotChainValid = NULL; @@ -3484,11 +3709,13 @@ void static ProcessGetData(CNode* pfrom) send = mi->second->IsValid(BLOCK_VALID_SCRIPTS) && (pindexBestHeader != NULL) && (mi->second->GetBlockTime() > pindexBestHeader->GetBlockTime() - 30 * 24 * 60 * 60); if (!send) { - LogPrintf("ProcessGetData(): ignoring request from peer=%i for old block that isn't in the main chain\n", pfrom->GetId()); + LogPrintf("%s: ignoring request from peer=%i for old block that isn't in the main chain\n", __func__, pfrom->GetId()); } } } - if (send) + // Pruned nodes may have deleted the block, so check whether + // it's available before trying to send. + if (send && (mi->second->nStatus & BLOCK_HAVE_DATA)) { // Send block from disk CBlock block; @@ -4386,7 +4613,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, bool ProcessMessages(CNode* pfrom) { //if (fDebug) - // LogPrintf("ProcessMessages(%u messages)\n", pfrom->vRecvMsg.size()); + // LogPrintf("%s(%u messages)\n", __func__, pfrom->vRecvMsg.size()); // // Message format @@ -4414,7 +4641,7 @@ bool ProcessMessages(CNode* pfrom) CNetMessage& msg = *it; //if (fDebug) - // LogPrintf("ProcessMessages(message %u msgsz, %u bytes, complete:%s)\n", + // LogPrintf("%s(message %u msgsz, %u bytes, complete:%s)\n", __func__, // msg.hdr.nMessageSize, msg.vRecv.size(), // msg.complete() ? "Y" : "N"); @@ -4450,7 +4677,7 @@ bool ProcessMessages(CNode* pfrom) unsigned int nChecksum = ReadLE32((unsigned char*)&hash); if (nChecksum != hdr.nChecksum) { - LogPrintf("ProcessMessages(%s, %u bytes): CHECKSUM ERROR nChecksum=%08x hdr.nChecksum=%08x\n", + LogPrintf("%s(%s, %u bytes): CHECKSUM ERROR nChecksum=%08x hdr.nChecksum=%08x\n", __func__, SanitizeString(strCommand), nMessageSize, nChecksum, hdr.nChecksum); continue; } @@ -4468,12 +4695,12 @@ bool ProcessMessages(CNode* pfrom) if (strstr(e.what(), "end of data")) { // Allow exceptions from under-length message on vRecv - LogPrintf("ProcessMessages(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", SanitizeString(strCommand), nMessageSize, e.what()); + LogPrintf("%s(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } else if (strstr(e.what(), "size too large")) { // Allow exceptions from over-long size - LogPrintf("ProcessMessages(%s, %u bytes): Exception '%s' caught\n", SanitizeString(strCommand), nMessageSize, e.what()); + LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } else { @@ -4490,7 +4717,7 @@ bool ProcessMessages(CNode* pfrom) } if (!fRet) - LogPrintf("ProcessMessage(%s, %u bytes) FAILED peer=%d\n", SanitizeString(strCommand), nMessageSize, pfrom->id); + LogPrintf("%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->id); break; } diff --git a/src/main.h b/src/main.h index bb6fd6f204..d34f6c30d1 100644 --- a/src/main.h +++ b/src/main.h @@ -132,6 +132,26 @@ extern CBlockIndex *pindexBestHeader; /** Minimum disk space required - used in CheckDiskSpace() */ static const uint64_t nMinDiskSpace = 52428800; +/** Pruning-related variables and constants */ +/** True if any block files have ever been pruned. */ +extern bool fHavePruned; +/** True if we're running in -prune mode. */ +extern bool fPruneMode; +/** Number of MiB of block files that we're trying to stay below. */ +extern uint64_t nPruneTarget; +/** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of chainActive.Tip() will not be pruned. */ +static const signed int MIN_BLOCKS_TO_KEEP = 288; + +// Require that user allocate at least 550MB for block & undo files (blk???.dat and rev???.dat) +// At 1MB per block, 288 blocks = 288MB. +// Add 15% for Undo data = 331MB +// Add 20% for Orphan block rate = 397MB +// We want the low water mark after pruning to be at least 397 MB and since we prune in +// full block file chunks, we need the high water mark which triggers the prune to be +// one 128MB block file + added 15% undo data = 147MB greater for a total of 545MB +// Setting the target to > than 550MB will make it likely we can respect the target. +static const signed int MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024; + /** Register with a network node to receive its signals */ void RegisterNodeSignals(CNodeSignals& nodeSignals); /** Unregister a network node */ @@ -186,6 +206,28 @@ bool GetTransaction(const uint256 &hash, CTransaction &tx, uint256 &hashBlock, b bool ActivateBestChain(CValidationState &state, CBlock *pblock = NULL); CAmount GetBlockValue(int nHeight, const CAmount& nFees); +/** + * Prune block and undo files (blk???.dat and undo???.dat) so that the disk space used is less than a user-defined target. + * The user sets the target (in MB) on the command line or in config file. This will be run on startup and whenever new + * space is allocated in a block or undo file, staying below the target. Changing back to unpruned requires a reindex + * (which in this case means the blockchain must be re-downloaded.) + * + * Pruning functions are called from FlushStateToDisk when the global fCheckForPruning flag has been set. + * Block and undo files are deleted in lock-step (when blk00003.dat is deleted, so is rev00003.dat.) + * Pruning cannot take place until the longest chain is at least a certain length (100000 on mainnet, 1000 on testnet, 10 on regtest). + * Pruning will never delete a block within a defined distance (currently 288) from the active chain's tip. + * The block index is updated by unsetting HAVE_DATA and HAVE_UNDO for any blocks that were stored in the deleted files. + * A db flag records the fact that at least some block files have been pruned. + * + * @param[out] setFilesToPrune The set of file indices that can be unlinked will be returned + */ +void FindFilesToPrune(std::set<int>& setFilesToPrune); + +/** + * Actually unlink the specified files + */ +void UnlinkPrunedFiles(std::set<int>& setFilesToPrune); + /** Create a new block index entry for a given block hash */ CBlockIndex * InsertBlockIndex(uint256 hash); /** Abort with a message */ @@ -196,7 +238,8 @@ bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats); void Misbehaving(NodeId nodeid, int howmuch); /** Flush all state, indexes and buffers to disk. */ void FlushStateToDisk(); - +/** Prune block files and flush state to disk. */ +void PruneAndFlush(); /** (try to) add transaction to memory pool **/ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, diff --git a/src/miner.h b/src/miner.h index 0be3152d97..96a6b70ecd 100644 --- a/src/miner.h +++ b/src/miner.h @@ -14,7 +14,7 @@ class CBlockIndex; class CReserveKey; class CScript; class CWallet; -namespace Consensus { class Params; }; +namespace Consensus { struct Params; }; struct CBlockTemplate { diff --git a/src/net.cpp b/src/net.cpp index 48e6367f20..e5f67262ce 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1221,7 +1221,7 @@ void ThreadOpenConnections() int nTries = 0; while (true) { - CAddress addr = addrman.Select(); + CAddrInfo addr = addrman.Select(); // if we selected an invalid address, restart if (!addr.IsValid() || setConnected.count(addr.GetGroup()) || IsLocal(addr)) diff --git a/src/netbase.cpp b/src/netbase.cpp index a2ac6575bc..1837cfa9c3 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -12,6 +12,7 @@ #include "hash.h" #include "sync.h" #include "uint256.h" +#include "random.h" #include "util.h" #include "utilstrencodings.h" @@ -38,7 +39,7 @@ using namespace std; // Settings static proxyType proxyInfo[NET_MAX]; -static CService nameProxy; +static proxyType nameProxy; static CCriticalSection cs_proxyInfos; int nConnectTimeout = DEFAULT_CONNECT_TIMEOUT; bool fNameLookup = false; @@ -285,59 +286,100 @@ bool static InterruptibleRecv(char* data, size_t len, int timeout, SOCKET& hSock return len == 0; } -bool static Socks5(string strDest, int port, SOCKET& hSocket) +struct ProxyCredentials +{ + std::string username; + std::string password; +}; + +/** Connect using SOCKS5 (as described in RFC1928) */ +bool static Socks5(string strDest, int port, const ProxyCredentials *auth, SOCKET& hSocket) { LogPrintf("SOCKS5 connecting %s\n", strDest); - if (strDest.size() > 255) - { + if (strDest.size() > 255) { CloseSocket(hSocket); return error("Hostname too long"); } - char pszSocks5Init[] = "\5\1\0"; - ssize_t nSize = sizeof(pszSocks5Init) - 1; - - ssize_t ret = send(hSocket, pszSocks5Init, nSize, MSG_NOSIGNAL); - if (ret != nSize) - { + // Accepted authentication methods + std::vector<uint8_t> vSocks5Init; + vSocks5Init.push_back(0x05); + if (auth) { + vSocks5Init.push_back(0x02); // # METHODS + vSocks5Init.push_back(0x00); // X'00' NO AUTHENTICATION REQUIRED + vSocks5Init.push_back(0x02); // X'02' USERNAME/PASSWORD (RFC1929) + } else { + vSocks5Init.push_back(0x01); // # METHODS + vSocks5Init.push_back(0x00); // X'00' NO AUTHENTICATION REQUIRED + } + ssize_t ret = send(hSocket, (const char*)begin_ptr(vSocks5Init), vSocks5Init.size(), MSG_NOSIGNAL); + if (ret != (ssize_t)vSocks5Init.size()) { CloseSocket(hSocket); return error("Error sending to proxy"); } char pchRet1[2]; - if (!InterruptibleRecv(pchRet1, 2, SOCKS5_RECV_TIMEOUT, hSocket)) - { + if (!InterruptibleRecv(pchRet1, 2, SOCKS5_RECV_TIMEOUT, hSocket)) { CloseSocket(hSocket); return error("Error reading proxy response"); } - if (pchRet1[0] != 0x05 || pchRet1[1] != 0x00) - { + if (pchRet1[0] != 0x05) { CloseSocket(hSocket); return error("Proxy failed to initialize"); } - string strSocks5("\5\1"); - strSocks5 += '\000'; strSocks5 += '\003'; - strSocks5 += static_cast<char>(std::min((int)strDest.size(), 255)); - strSocks5 += strDest; - strSocks5 += static_cast<char>((port >> 8) & 0xFF); - strSocks5 += static_cast<char>((port >> 0) & 0xFF); - ret = send(hSocket, strSocks5.data(), strSocks5.size(), MSG_NOSIGNAL); - if (ret != (ssize_t)strSocks5.size()) - { + if (pchRet1[1] == 0x02 && auth) { + // Perform username/password authentication (as described in RFC1929) + std::vector<uint8_t> vAuth; + vAuth.push_back(0x01); + if (auth->username.size() > 255 || auth->password.size() > 255) + return error("Proxy username or password too long"); + vAuth.push_back(auth->username.size()); + vAuth.insert(vAuth.end(), auth->username.begin(), auth->username.end()); + vAuth.push_back(auth->password.size()); + vAuth.insert(vAuth.end(), auth->password.begin(), auth->password.end()); + ret = send(hSocket, (const char*)begin_ptr(vAuth), vAuth.size(), MSG_NOSIGNAL); + if (ret != (ssize_t)vAuth.size()) { + CloseSocket(hSocket); + return error("Error sending authentication to proxy"); + } + LogPrint("proxy", "SOCKS5 sending proxy authentication %s:%s\n", auth->username, auth->password); + char pchRetA[2]; + if (!InterruptibleRecv(pchRetA, 2, SOCKS5_RECV_TIMEOUT, hSocket)) { + CloseSocket(hSocket); + return error("Error reading proxy authentication response"); + } + if (pchRetA[0] != 0x01 || pchRetA[1] != 0x00) { + CloseSocket(hSocket); + return error("Proxy authentication unsuccesful"); + } + } else if (pchRet1[1] == 0x00) { + // Perform no authentication + } else { + CloseSocket(hSocket); + return error("Proxy requested wrong authentication method %02x", pchRet1[1]); + } + std::vector<uint8_t> vSocks5; + vSocks5.push_back(0x05); // VER protocol version + vSocks5.push_back(0x01); // CMD CONNECT + vSocks5.push_back(0x00); // RSV Reserved + vSocks5.push_back(0x03); // ATYP DOMAINNAME + vSocks5.push_back(strDest.size()); // Length<=255 is checked at beginning of function + vSocks5.insert(vSocks5.end(), strDest.begin(), strDest.end()); + vSocks5.push_back((port >> 8) & 0xFF); + vSocks5.push_back((port >> 0) & 0xFF); + ret = send(hSocket, (const char*)begin_ptr(vSocks5), vSocks5.size(), MSG_NOSIGNAL); + if (ret != (ssize_t)vSocks5.size()) { CloseSocket(hSocket); return error("Error sending to proxy"); } char pchRet2[4]; - if (!InterruptibleRecv(pchRet2, 4, SOCKS5_RECV_TIMEOUT, hSocket)) - { + if (!InterruptibleRecv(pchRet2, 4, SOCKS5_RECV_TIMEOUT, hSocket)) { CloseSocket(hSocket); return error("Error reading proxy response"); } - if (pchRet2[0] != 0x05) - { + if (pchRet2[0] != 0x05) { CloseSocket(hSocket); return error("Proxy failed to accept request"); } - if (pchRet2[1] != 0x00) - { + if (pchRet2[1] != 0x00) { CloseSocket(hSocket); switch (pchRet2[1]) { @@ -352,8 +394,7 @@ bool static Socks5(string strDest, int port, SOCKET& hSocket) default: return error("Proxy error: unknown"); } } - if (pchRet2[2] != 0x00) - { + if (pchRet2[2] != 0x00) { CloseSocket(hSocket); return error("Error: malformed proxy response"); } @@ -375,13 +416,11 @@ bool static Socks5(string strDest, int port, SOCKET& hSocket) } default: CloseSocket(hSocket); return error("Error: malformed proxy response"); } - if (!ret) - { + if (!ret) { CloseSocket(hSocket); return error("Error reading from proxy"); } - if (!InterruptibleRecv(pchRet3, 2, SOCKS5_RECV_TIMEOUT, hSocket)) - { + if (!InterruptibleRecv(pchRet3, 2, SOCKS5_RECV_TIMEOUT, hSocket)) { CloseSocket(hSocket); return error("Error reading from proxy"); } @@ -471,7 +510,7 @@ bool static ConnectSocketDirectly(const CService &addrConnect, SOCKET& hSocketRe return true; } -bool SetProxy(enum Network net, CService addrProxy) { +bool SetProxy(enum Network net, const proxyType &addrProxy) { assert(net >= 0 && net < NET_MAX); if (!addrProxy.IsValid()) return false; @@ -489,7 +528,7 @@ bool GetProxy(enum Network net, proxyType &proxyInfoOut) { return true; } -bool SetNameProxy(CService addrProxy) { +bool SetNameProxy(const proxyType &addrProxy) { if (!addrProxy.IsValid()) return false; LOCK(cs_proxyInfos); @@ -497,7 +536,7 @@ bool SetNameProxy(CService addrProxy) { return true; } -bool GetNameProxy(CService &nameProxyOut) { +bool GetNameProxy(proxyType &nameProxyOut) { LOCK(cs_proxyInfos); if(!nameProxy.IsValid()) return false; @@ -513,37 +552,49 @@ bool HaveNameProxy() { bool IsProxy(const CNetAddr &addr) { LOCK(cs_proxyInfos); for (int i = 0; i < NET_MAX; i++) { - if (addr == (CNetAddr)proxyInfo[i]) + if (addr == (CNetAddr)proxyInfo[i].proxy) return true; } return false; } -bool ConnectSocket(const CService &addrDest, SOCKET& hSocketRet, int nTimeout, bool *outProxyConnectionFailed) +static bool ConnectThroughProxy(const proxyType &proxy, const std::string strDest, int port, SOCKET& hSocketRet, int nTimeout, bool *outProxyConnectionFailed) { - proxyType proxy; - if (outProxyConnectionFailed) - *outProxyConnectionFailed = false; - // no proxy needed (none set for target network) - if (!GetProxy(addrDest.GetNetwork(), proxy)) - return ConnectSocketDirectly(addrDest, hSocketRet, nTimeout); - SOCKET hSocket = INVALID_SOCKET; - // first connect to proxy server - if (!ConnectSocketDirectly(proxy, hSocket, nTimeout)) { + if (!ConnectSocketDirectly(proxy.proxy, hSocket, nTimeout)) { if (outProxyConnectionFailed) *outProxyConnectionFailed = true; return false; } // do socks negotiation - if (!Socks5(addrDest.ToStringIP(), addrDest.GetPort(), hSocket)) - return false; + if (proxy.randomize_credentials) { + ProxyCredentials random_auth; + random_auth.username = strprintf("%i", insecure_rand()); + random_auth.password = strprintf("%i", insecure_rand()); + if (!Socks5(strDest, (unsigned short)port, &random_auth, hSocket)) + return false; + } else { + if (!Socks5(strDest, (unsigned short)port, 0, hSocket)) + return false; + } hSocketRet = hSocket; return true; } +bool ConnectSocket(const CService &addrDest, SOCKET& hSocketRet, int nTimeout, bool *outProxyConnectionFailed) +{ + proxyType proxy; + if (outProxyConnectionFailed) + *outProxyConnectionFailed = false; + + if (GetProxy(addrDest.GetNetwork(), proxy)) + return ConnectThroughProxy(proxy, addrDest.ToStringIP(), addrDest.GetPort(), hSocketRet, nTimeout, outProxyConnectionFailed); + else // no proxy needed (none set for target network) + return ConnectSocketDirectly(addrDest, hSocketRet, nTimeout); +} + bool ConnectSocketByName(CService &addr, SOCKET& hSocketRet, const char *pszDest, int portDefault, int nTimeout, bool *outProxyConnectionFailed) { string strDest; @@ -554,9 +605,7 @@ bool ConnectSocketByName(CService &addr, SOCKET& hSocketRet, const char *pszDest SplitHostPort(string(pszDest), port, strDest); - SOCKET hSocket = INVALID_SOCKET; - - CService nameProxy; + proxyType nameProxy; GetNameProxy(nameProxy); CService addrResolved(CNetAddr(strDest, fNameLookup && !HaveNameProxy()), port); @@ -569,18 +618,7 @@ bool ConnectSocketByName(CService &addr, SOCKET& hSocketRet, const char *pszDest if (!HaveNameProxy()) return false; - // first connect to name proxy server - if (!ConnectSocketDirectly(nameProxy, hSocket, nTimeout)) { - if (outProxyConnectionFailed) - *outProxyConnectionFailed = true; - return false; - } - // do socks negotiation - if (!Socks5(strDest, (unsigned short)port, hSocket)) - return false; - - hSocketRet = hSocket; - return true; + return ConnectThroughProxy(nameProxy, strDest, port, hSocketRet, nTimeout, outProxyConnectionFailed); } void CNetAddr::Init() diff --git a/src/netbase.h b/src/netbase.h index b42c2dffa4..6d2ca4afb2 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -168,15 +168,25 @@ class CService : public CNetAddr } }; -typedef CService proxyType; +class proxyType +{ +public: + proxyType(): randomize_credentials(false) {} + proxyType(const CService &proxy, bool randomize_credentials=false): proxy(proxy), randomize_credentials(randomize_credentials) {} + + bool IsValid() const { return proxy.IsValid(); } + + CService proxy; + bool randomize_credentials; +}; enum Network ParseNetwork(std::string net); std::string GetNetworkName(enum Network net); void SplitHostPort(std::string in, int &portOut, std::string &hostOut); -bool SetProxy(enum Network net, CService addrProxy); +bool SetProxy(enum Network net, const proxyType &addrProxy); bool GetProxy(enum Network net, proxyType &proxyInfoOut); bool IsProxy(const CNetAddr &addr); -bool SetNameProxy(CService addrProxy); +bool SetNameProxy(const proxyType &addrProxy); bool HaveNameProxy(); bool LookupHost(const char *pszName, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions = 0, bool fAllowLookup = true); bool Lookup(const char *pszName, CService& addr, int portDefault = 0, bool fAllowLookup = true); diff --git a/src/protocol.cpp b/src/protocol.cpp index 568580a595..dd855aa33a 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -89,7 +89,6 @@ void CAddress::Init() { nServices = NODE_NETWORK; nTime = 100000000; - nLastTry = 0; } CInv::CInv() diff --git a/src/protocol.h b/src/protocol.h index fd23eae1fc..b5e65032a2 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -116,9 +116,6 @@ public: // disk and network only unsigned int nTime; - - // memory only - int64_t nLastTry; }; /** inv message data */ diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 3ae780abfd..069601ab67 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -107,7 +107,6 @@ static QString GetLangTerritory() /** Set up translations */ static void initTranslations(QTranslator &qtTranslatorBase, QTranslator &qtTranslator, QTranslator &translatorBase, QTranslator &translator) { - // Remove old translators QApplication::removeTranslator(&qtTranslatorBase); QApplication::removeTranslator(&qtTranslator); diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 198dd1fdf9..670d54c7e3 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -164,7 +164,7 @@ BitcoinGUI::BitcoinGUI(const NetworkStyle *networkStyle, QWidget *parent) : // Create status bar statusBar(); - + // Disable size grip because it looks ugly and nobody needs it statusBar()->setSizeGripEnabled(false); diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index e4e9015c85..0a60632bfa 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -8,12 +8,12 @@ #include "addresstablemodel.h" #include "bitcoinunits.h" #include "guiutil.h" -#include "init.h" #include "optionsmodel.h" #include "scicon.h" #include "walletmodel.h" #include "coincontrol.h" +#include "init.h" #include "main.h" #include "wallet/wallet.h" diff --git a/src/qt/forms/sendcoinsentry.ui b/src/qt/forms/sendcoinsentry.ui index b362928438..48d0dd093c 100644 --- a/src/qt/forms/sendcoinsentry.ui +++ b/src/qt/forms/sendcoinsentry.ui @@ -195,7 +195,7 @@ </item> </layout> </widget> - <widget class="QFrame" name="SendCoins_InsecurePaymentRequest"> + <widget class="QFrame" name="SendCoins_UnauthenticatedPaymentRequest"> <property name="palette"> <palette> <active> @@ -612,7 +612,7 @@ </palette> </property> <property name="toolTip"> - <string>This is an unverified payment request.</string> + <string>This is an unauthenticated payment request.</string> </property> <property name="autoFillBackground"> <bool>true</bool> @@ -700,7 +700,7 @@ </item> </layout> </widget> - <widget class="QFrame" name="SendCoins_SecurePaymentRequest"> + <widget class="QFrame" name="SendCoins_AuthenticatedPaymentRequest"> <property name="palette"> <palette> <active> @@ -1144,7 +1144,7 @@ </palette> </property> <property name="toolTip"> - <string>This is a verified payment request.</string> + <string>This is an authenticated payment request.</string> </property> <property name="autoFillBackground"> <bool>true</bool> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index a5ee81db6c..4a1f728e18 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -741,14 +741,14 @@ LSSharedFileListItemRef findStartupItemInList(LSSharedFileListRef list, CFURLRef CFURLRef currentItemURL = NULL; #if defined(MAC_OS_X_VERSION_MAX_ALLOWED) && MAC_OS_X_VERSION_MAX_ALLOWED >= 10100 - if(&LSSharedFileListItemCopyResolvedURL) - currentItemURL = LSSharedFileListItemCopyResolvedURL(item, resolutionFlags, NULL); + if(&LSSharedFileListItemCopyResolvedURL) + currentItemURL = LSSharedFileListItemCopyResolvedURL(item, resolutionFlags, NULL); #if defined(MAC_OS_X_VERSION_MIN_REQUIRED) && MAC_OS_X_VERSION_MIN_REQUIRED < 10100 - else - LSSharedFileListItemResolve(item, resolutionFlags, ¤tItemURL, NULL); + else + LSSharedFileListItemResolve(item, resolutionFlags, ¤tItemURL, NULL); #endif #else - LSSharedFileListItemResolve(item, resolutionFlags, ¤tItemURL, NULL); + LSSharedFileListItemResolve(item, resolutionFlags, ¤tItemURL, NULL); #endif if(currentItemURL && CFEqual(currentItemURL, findUrl)) { diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index a169ed6b55..41d6acf358 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -335,8 +335,8 @@ bool OptionsModel::getProxySettings(QNetworkProxy& proxy) const proxyType curProxy; if (GetProxy(NET_IPV4, curProxy)) { proxy.setType(QNetworkProxy::Socks5Proxy); - proxy.setHostName(QString::fromStdString(curProxy.ToStringIP())); - proxy.setPort(curProxy.GetPort()); + proxy.setHostName(QString::fromStdString(curProxy.proxy.ToStringIP())); + proxy.setPort(curProxy.proxy.GetPort()); return true; } diff --git a/src/qt/recentrequeststablemodel.cpp b/src/qt/recentrequeststablemodel.cpp index 5221ec3e24..543b977d8f 100644 --- a/src/qt/recentrequeststablemodel.cpp +++ b/src/qt/recentrequeststablemodel.cpp @@ -5,9 +5,10 @@ #include "recentrequeststablemodel.h" #include "bitcoinunits.h" -#include "clientversion.h" #include "guiutil.h" #include "optionsmodel.h" + +#include "clientversion.h" #include "streams.h" #include <boost/foreach.hpp> diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 59939fa871..0360f160d8 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -274,11 +274,11 @@ void SendCoinsDialog::on_sendButton_clicked() recipientElement = tr("%1 to %2").arg(amount, address); } } - else if(!rcp.authenticatedMerchant.isEmpty()) // secure payment request + else if(!rcp.authenticatedMerchant.isEmpty()) // authenticated payment request { recipientElement = tr("%1 to %2").arg(amount, GUIUtil::HtmlEscape(rcp.authenticatedMerchant)); } - else // insecure payment request + else // unauthenticated payment request { recipientElement = tr("%1 to %2").arg(amount, address); } diff --git a/src/qt/sendcoinsentry.cpp b/src/qt/sendcoinsentry.cpp index 6ac650e74f..ea35ed1d53 100644 --- a/src/qt/sendcoinsentry.cpp +++ b/src/qt/sendcoinsentry.cpp @@ -99,11 +99,11 @@ void SendCoinsEntry::clear() ui->messageTextLabel->clear(); ui->messageTextLabel->hide(); ui->messageLabel->hide(); - // clear UI elements for insecure payment request + // clear UI elements for unauthenticated payment request ui->payTo_is->clear(); ui->memoTextLabel_is->clear(); ui->payAmount_is->clear(); - // clear UI elements for secure payment request + // clear UI elements for authenticated payment request ui->payTo_s->clear(); ui->memoTextLabel_s->clear(); ui->payAmount_s->clear(); @@ -190,21 +190,21 @@ void SendCoinsEntry::setValue(const SendCoinsRecipient &value) if (recipient.paymentRequest.IsInitialized()) // payment request { - if (recipient.authenticatedMerchant.isEmpty()) // insecure + if (recipient.authenticatedMerchant.isEmpty()) // unauthenticated { ui->payTo_is->setText(recipient.address); ui->memoTextLabel_is->setText(recipient.message); ui->payAmount_is->setValue(recipient.amount); ui->payAmount_is->setReadOnly(true); - setCurrentWidget(ui->SendCoins_InsecurePaymentRequest); + setCurrentWidget(ui->SendCoins_UnauthenticatedPaymentRequest); } - else // secure + else // authenticated { ui->payTo_s->setText(recipient.authenticatedMerchant); ui->memoTextLabel_s->setText(recipient.message); ui->payAmount_s->setValue(recipient.amount); ui->payAmount_s->setReadOnly(true); - setCurrentWidget(ui->SendCoins_SecurePaymentRequest); + setCurrentWidget(ui->SendCoins_AuthenticatedPaymentRequest); } } else // normal payment diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index 414fe02ff9..8430e017c1 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -4,11 +4,12 @@ #include "splashscreen.h" +#include "networkstyle.h" + #include "clientversion.h" #include "init.h" -#include "networkstyle.h" -#include "ui_interface.h" #include "util.h" +#include "ui_interface.h" #include "version.h" #ifdef ENABLE_WALLET diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index 9b235f9130..4a2c51477e 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -10,7 +10,6 @@ #include "transactionrecord.h" #include "base58.h" -#include "wallet/db.h" #include "main.h" #include "script/script.h" #include "timedata.h" diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 09ed8ce9fd..9b8be76beb 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -12,7 +12,6 @@ #include "transactiontablemodel.h" #include "base58.h" -#include "wallet/db.h" #include "keystore.h" #include "main.h" #include "sync.h" diff --git a/src/rpcmining.cpp b/src/rpcmining.cpp index 851d113f3b..aba250ba49 100644 --- a/src/rpcmining.cpp +++ b/src/rpcmining.cpp @@ -196,6 +196,8 @@ Value setgenerate(const Array& params, bool fHelp) if (pwalletMain == NULL) throw JSONRPCError(RPC_METHOD_NOT_FOUND, "Method not found (disabled)"); + if (Params().MineBlocksOnDemand()) + throw JSONRPCError(RPC_METHOD_NOT_FOUND, "Use the generate method instead of setgenerate on this network"); bool fGenerate = true; if (params.size() > 0) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 938d79513f..f5bef2a077 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -90,7 +90,7 @@ Value getinfo(const Array& params, bool fHelp) obj.push_back(Pair("blocks", (int)chainActive.Height())); obj.push_back(Pair("timeoffset", GetTimeOffset())); obj.push_back(Pair("connections", (int)vNodes.size())); - obj.push_back(Pair("proxy", (proxy.IsValid() ? proxy.ToStringIPPort() : string()))); + obj.push_back(Pair("proxy", (proxy.IsValid() ? proxy.proxy.ToStringIPPort() : string()))); obj.push_back(Pair("difficulty", (double)GetDifficulty())); obj.push_back(Pair("testnet", Params().TestnetToBeDeprecatedFieldRPC())); #ifdef ENABLE_WALLET diff --git a/src/rpcnet.cpp b/src/rpcnet.cpp index 6306fd4406..bdee5b9f2e 100644 --- a/src/rpcnet.cpp +++ b/src/rpcnet.cpp @@ -371,7 +371,8 @@ static Array GetNetworksInfo() obj.push_back(Pair("name", GetNetworkName(network))); obj.push_back(Pair("limited", IsLimited(network))); obj.push_back(Pair("reachable", IsReachable(network))); - obj.push_back(Pair("proxy", proxy.IsValid() ? proxy.ToStringIPPort() : string())); + obj.push_back(Pair("proxy", proxy.IsValid() ? proxy.proxy.ToStringIPPort() : string())); + obj.push_back(Pair("proxy_randomize_credentials", proxy.randomize_credentials)); networks.push_back(obj); } return networks; diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index c979217a13..8393a8502e 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -419,7 +419,7 @@ Value signrawtransaction(const Array& params, bool fHelp) "\nResult:\n" "{\n" " \"hex\": \"value\", (string) The raw transaction with signature(s) (hex-encoded string)\n" - " \"complete\": n (numeric) if transaction has a complete set of signature (0 if not)\n" + " \"complete\": true|false (boolean) if transaction has a complete set of signature\n" "}\n" "\nExamples:\n" diff --git a/src/support/allocators/secure.h b/src/support/allocators/secure.h index 7a74d87bb4..5e7bb66ea2 100644 --- a/src/support/allocators/secure.h +++ b/src/support/allocators/secure.h @@ -3,8 +3,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_ALLOCATORS_SECURE_H -#define BITCOIN_ALLOCATORS_SECURE_H +#ifndef BITCOIN_SUPPORT_ALLOCATORS_SECURE_H +#define BITCOIN_SUPPORT_ALLOCATORS_SECURE_H #include "support/pagelocker.h" @@ -59,4 +59,4 @@ struct secure_allocator : public std::allocator<T> { // This is exactly like std::string, but with a custom allocator. typedef std::basic_string<char, std::char_traits<char>, secure_allocator<char> > SecureString; -#endif // BITCOIN_ALLOCATORS_SECURE_H +#endif // BITCOIN_SUPPORT_ALLOCATORS_SECURE_H diff --git a/src/support/allocators/zeroafterfree.h b/src/support/allocators/zeroafterfree.h index b01fcd088b..41e23392e8 100644 --- a/src/support/allocators/zeroafterfree.h +++ b/src/support/allocators/zeroafterfree.h @@ -3,8 +3,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_ALLOCATORS_ZEROAFTERFREE_H -#define BITCOIN_ALLOCATORS_ZEROAFTERFREE_H +#ifndef BITCOIN_SUPPORT_ALLOCATORS_ZEROAFTERFREE_H +#define BITCOIN_SUPPORT_ALLOCATORS_ZEROAFTERFREE_H #include "support/cleanse.h" @@ -45,4 +45,4 @@ struct zero_after_free_allocator : public std::allocator<T> { // Byte-vector that clears its contents before deletion. typedef std::vector<char, zero_after_free_allocator<char> > CSerializeData; -#endif // BITCOIN_ALLOCATORS_ZEROAFTERFREE_H +#endif // BITCOIN_SUPPORT_ALLOCATORS_ZEROAFTERFREE_H diff --git a/src/support/pagelocker.h b/src/support/pagelocker.h index 964be1aec4..3fd793072f 100644 --- a/src/support/pagelocker.h +++ b/src/support/pagelocker.h @@ -3,8 +3,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_ALLOCATORS_PAGELOCKER_H -#define BITCOIN_ALLOCATORS_PAGELOCKER_H +#ifndef BITCOIN_SUPPORT_PAGELOCKER_H +#define BITCOIN_SUPPORT_PAGELOCKER_H #include "support/cleanse.h" @@ -175,4 +175,4 @@ void UnlockObject(const T& t) LockedPageManager::Instance().UnlockRange((void*)(&t), sizeof(T)); } -#endif // BITCOIN_ALLOCATORS_PAGELOCKER_H +#endif // BITCOIN_SUPPORT_PAGELOCKER_H diff --git a/src/sync.h b/src/sync.h index 27e80e813b..78b9043477 100644 --- a/src/sync.h +++ b/src/sync.h @@ -21,9 +21,6 @@ //////////////////////////////////////////////// /* - - - CCriticalSection mutex; boost::recursive_mutex mutex; @@ -42,20 +39,18 @@ ENTER_CRITICAL_SECTION(mutex); // no RAII LEAVE_CRITICAL_SECTION(mutex); // no RAII mutex.unlock(); - - - */ - /////////////////////////////// // // // THE ACTUAL IMPLEMENTATION // // // /////////////////////////////// -// Template mixin that adds -Wthread-safety locking annotations to a -// subset of the mutex API. +/** + * Template mixin that adds -Wthread-safety locking + * annotations to a subset of the mutex API. + */ template <typename PARENT> class LOCKABLE AnnotatedMixin : public PARENT { @@ -76,8 +71,10 @@ public: } }; -/** Wrapped boost mutex: supports recursive locking, but no waiting */ -// TODO: We should move away from using the recursive lock by default. +/** + * Wrapped boost mutex: supports recursive locking, but no waiting + * TODO: We should move away from using the recursive lock by default. + */ typedef AnnotatedMixin<boost::recursive_mutex> CCriticalSection; /** Wrapped boost mutex: supports waiting but not recursive locking */ @@ -92,9 +89,7 @@ void LeaveCritical(); std::string LocksHeld(); void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs); #else -void static inline EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs, bool fTry = false) -{ -} +void static inline EnterCritical(const char* pszName, const char* pszFile, int nLine, void* cs, bool fTry = false) {} void static inline LeaveCritical() {} void static inline AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, void* cs) {} #endif diff --git a/src/test/bip32_tests.cpp b/src/test/bip32_tests.cpp index d738851c1f..0d815c27fd 100644 --- a/src/test/bip32_tests.cpp +++ b/src/test/bip32_tests.cpp @@ -8,6 +8,7 @@ #include "key.h" #include "uint256.h" #include "util.h" +#include "utilstrencodings.h" #include "test/test_bitcoin.h" #include <string> diff --git a/src/wallet/crypter.h b/src/wallet/crypter.h index 32746b00df..70aeb76723 100644 --- a/src/wallet/crypter.h +++ b/src/wallet/crypter.h @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_CRYPTER_H -#define BITCOIN_CRYPTER_H +#ifndef BITCOIN_WALLET_CRYPTER_H +#define BITCOIN_WALLET_CRYPTER_H #include "keystore.h" #include "serialize.h" @@ -193,4 +193,4 @@ public: boost::signals2::signal<void (CCryptoKeyStore* wallet)> NotifyStatusChanged; }; -#endif // BITCOIN_CRYPTER_H +#endif // BITCOIN_WALLET_CRYPTER_H diff --git a/src/wallet/db.h b/src/wallet/db.h index 790ae50413..9596723b21 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -3,8 +3,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_DB_H -#define BITCOIN_DB_H +#ifndef BITCOIN_WALLET_DB_H +#define BITCOIN_WALLET_DB_H #include "clientversion.h" #include "serialize.h" @@ -307,4 +307,4 @@ public: bool static Rewrite(const std::string& strFile, const char* pszSkip = NULL); }; -#endif // BITCOIN_DB_H +#endif // BITCOIN_WALLET_DB_H diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index e03cd5b84e..c31c09d922 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -2301,7 +2301,7 @@ Value listunspent(const Array& params, bool fHelp) vector<COutput> vecOutputs; assert(pwalletMain != NULL); LOCK2(cs_main, pwalletMain->cs_wallet); - pwalletMain->AvailableCoins(vecOutputs, false); + pwalletMain->AvailableCoins(vecOutputs, false, NULL, true); BOOST_FOREACH(const COutput& out, vecOutputs) { if (out.nDepth < nMinDepth || out.nDepth > nMaxDepth) continue; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 2566b2712b..f0ee834913 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1481,7 +1481,7 @@ CAmount CWallet::GetImmatureWatchOnlyBalance() const /** * populate vCoins with vector of available COutputs. */ -void CWallet::AvailableCoins(vector<COutput>& vCoins, bool fOnlyConfirmed, const CCoinControl *coinControl) const +void CWallet::AvailableCoins(vector<COutput>& vCoins, bool fOnlyConfirmed, const CCoinControl *coinControl, bool fIncludeZeroValue) const { vCoins.clear(); @@ -1508,7 +1508,7 @@ void CWallet::AvailableCoins(vector<COutput>& vCoins, bool fOnlyConfirmed, const for (unsigned int i = 0; i < pcoin->vout.size(); i++) { isminetype mine = IsMine(pcoin->vout[i]); if (!(IsSpent(wtxid, i)) && mine != ISMINE_NO && - !IsLockedCoin((*it).first, i) && pcoin->vout[i].nValue > 0 && + !IsLockedCoin((*it).first, i) && (pcoin->vout[i].nValue > 0 || fIncludeZeroValue) && (!coinControl || !coinControl->HasSelected() || coinControl->IsSelected((*it).first, i))) vCoins.push_back(COutput(pcoin, i, nDepth, (mine & ISMINE_SPENDABLE) != ISMINE_NO)); } @@ -2038,7 +2038,7 @@ DBErrors CWallet::LoadWallet(bool& fFirstRunRet) setKeyPool.clear(); // Note: can't top-up keypool here, because wallet is locked. // User will be prompted to unlock wallet the next operation - // the requires a new key. + // that requires a new key. } } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 4dbb0e2e57..1c900b0315 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -3,8 +3,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_WALLET_H -#define BITCOIN_WALLET_H +#ifndef BITCOIN_WALLET_WALLET_H +#define BITCOIN_WALLET_WALLET_H #include "amount.h" #include "key.h" @@ -540,7 +540,7 @@ public: //! check whether we are allowed to upgrade (or already support) to the named feature bool CanSupportFeature(enum WalletFeature wf) { AssertLockHeld(cs_wallet); return nWalletMaxVersion >= wf; } - void AvailableCoins(std::vector<COutput>& vCoins, bool fOnlyConfirmed=true, const CCoinControl *coinControl = NULL) const; + void AvailableCoins(std::vector<COutput>& vCoins, bool fOnlyConfirmed=true, const CCoinControl *coinControl = NULL, bool fIncludeZeroValue=false) const; bool SelectCoinsMinConf(const CAmount& nTargetValue, int nConfMine, int nConfTheirs, std::vector<COutput> vCoins, std::set<std::pair<const CWalletTx*,unsigned int> >& setCoinsRet, CAmount& nValueRet) const; bool IsSpent(const uint256& hash, unsigned int n) const; @@ -869,4 +869,4 @@ private: std::vector<char> _ssExtra; }; -#endif // BITCOIN_WALLET_H +#endif // BITCOIN_WALLET_WALLET_H diff --git a/src/wallet/wallet_ismine.h b/src/wallet/wallet_ismine.h index 6293df8b10..5b9b0e0841 100644 --- a/src/wallet/wallet_ismine.h +++ b/src/wallet/wallet_ismine.h @@ -3,8 +3,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_WALLET_ISMINE_H -#define BITCOIN_WALLET_ISMINE_H +#ifndef BITCOIN_WALLET_WALLET_ISMINE_H +#define BITCOIN_WALLET_WALLET_ISMINE_H #include "key.h" #include "script/standard.h" @@ -26,4 +26,4 @@ typedef uint8_t isminefilter; isminetype IsMine(const CKeyStore& keystore, const CScript& scriptPubKey); isminetype IsMine(const CKeyStore& keystore, const CTxDestination& dest); -#endif // BITCOIN_WALLET_ISMINE_H +#endif // BITCOIN_WALLET_WALLET_ISMINE_H diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index e5f64ffaae..bc1a104b5b 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -3,8 +3,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_WALLETDB_H -#define BITCOIN_WALLETDB_H +#ifndef BITCOIN_WALLET_WALLETDB_H +#define BITCOIN_WALLET_WALLETDB_H #include "amount.h" #include "wallet/db.h" @@ -140,4 +140,4 @@ private: bool BackupWallet(const CWallet& wallet, const std::string& strDest); void ThreadFlushWalletDB(const std::string& strFile); -#endif // BITCOIN_WALLETDB_H +#endif // BITCOIN_WALLET_WALLETDB_H |
