aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rwxr-xr-xqa/rpc-tests/pruning.py356
-rw-r--r--qa/rpc-tests/util.py7
-rw-r--r--src/chainparams.cpp3
-rw-r--r--src/chainparams.h2
-rw-r--r--src/init.cpp90
-rw-r--r--src/main.cpp278
-rw-r--r--src/main.h45
7 files changed, 744 insertions, 37 deletions
diff --git a/qa/rpc-tests/pruning.py b/qa/rpc-tests/pruning.py
new file mode 100755
index 0000000000..f26cbee1e2
--- /dev/null
+++ b/qa/rpc-tests/pruning.py
@@ -0,0 +1,356 @@
+#!/usr/bin/env python2
+# Copyright (c) 2014 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#
+# Test pruning code
+# ********
+# WARNING:
+# This test uses 4GB of disk space and takes in excess of 30 mins to run
+# ********
+
+from test_framework import BitcoinTestFramework
+from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
+from util import *
+import os.path
+
+def calc_usage(blockdir):
+ return sum(os.path.getsize(blockdir+f) for f in os.listdir(blockdir) if os.path.isfile(blockdir+f))/(1024*1024)
+
+class PruneTest(BitcoinTestFramework):
+
+ def __init__(self):
+ self.utxo = []
+ self.address = ["",""]
+
+ # Some pre-processing to create a bunch of OP_RETURN txouts to insert into transactions we create
+ # So we have big transactions and full blocks to fill up our block files
+
+ # create one script_pubkey
+ script_pubkey = "6a4d0200" #OP_RETURN OP_PUSH2 512 bytes
+ for i in xrange (512):
+ script_pubkey = script_pubkey + "01"
+ # concatenate 128 txouts of above script_pubkey which we'll insert before the txout for change
+ self.txouts = "81"
+ for k in xrange(128):
+ # add txout value
+ self.txouts = self.txouts + "0000000000000000"
+ # add length of script_pubkey
+ self.txouts = self.txouts + "fd0402"
+ # add script_pubkey
+ self.txouts = self.txouts + script_pubkey
+
+
+ def setup_chain(self):
+ print("Initializing test directory "+self.options.tmpdir)
+ initialize_chain_clean(self.options.tmpdir, 3)
+
+ def setup_network(self):
+ self.nodes = []
+ self.is_network_split = False
+
+ # Create nodes 0 and 1 to mine
+ self.nodes.append(start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=300))
+ self.nodes.append(start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=300))
+
+ # Create node 2 to test pruning
+ self.nodes.append(start_node(2, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-prune=550"], timewait=300))
+ self.prunedir = self.options.tmpdir+"/node2/regtest/blocks/"
+
+ self.address[0] = self.nodes[0].getnewaddress()
+ self.address[1] = self.nodes[1].getnewaddress()
+
+ connect_nodes(self.nodes[0], 1)
+ connect_nodes(self.nodes[1], 2)
+ connect_nodes(self.nodes[2], 0)
+ sync_blocks(self.nodes[0:3])
+
+ def create_big_chain(self):
+ # Start by creating some coinbases we can spend later
+ self.nodes[1].generate(200)
+ sync_blocks(self.nodes[0:2])
+ self.nodes[0].generate(150)
+ # Then mine enough full blocks to create more than 550MB of data
+ for i in xrange(645):
+ self.mine_full_block(self.nodes[0], self.address[0])
+
+ sync_blocks(self.nodes[0:3])
+
+ def test_height_min(self):
+ if not os.path.isfile(self.prunedir+"blk00000.dat"):
+ raise AssertionError("blk00000.dat is missing, pruning too early")
+ print "Success"
+ print "Though we're already using more than 550MB, current usage:", calc_usage(self.prunedir)
+ print "Mining 25 more blocks should cause the first block file to be pruned"
+ # Pruning doesn't run until we're allocating another chunk, 20 full blocks past the height cutoff will ensure this
+ for i in xrange(25):
+ self.mine_full_block(self.nodes[0],self.address[0])
+
+ waitstart = time.time()
+ while os.path.isfile(self.prunedir+"blk00000.dat"):
+ time.sleep(0.1)
+ if time.time() - waitstart > 10:
+ raise AssertionError("blk00000.dat not pruned when it should be")
+
+ print "Success"
+ usage = calc_usage(self.prunedir)
+ print "Usage should be below target:", usage
+ if (usage > 550):
+ raise AssertionError("Pruning target not being met")
+
+ def create_chain_with_staleblocks(self):
+ # Create stale blocks in manageable sized chunks
+ print "Mine 24 (stale) blocks on Node 1, followed by 25 (main chain) block reorg from Node 0, for 12 rounds"
+
+ for j in xrange(12):
+ # Disconnect node 0 so it can mine a longer reorg chain without knowing about node 1's soon-to-be-stale chain
+ # Node 2 stays connected, so it hears about the stale blocks and then reorg's when node0 reconnects
+ # Stopping node 0 also clears its mempool, so it doesn't have node1's transactions to accidentally mine
+ stop_node(self.nodes[0],0)
+ self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=999000", "-checkblocks=5"], timewait=300)
+ # Mine 24 blocks in node 1
+ self.utxo = self.nodes[1].listunspent()
+ for i in xrange(24):
+ if j == 0:
+ self.mine_full_block(self.nodes[1],self.address[1])
+ else:
+ self.nodes[1].generate(1) #tx's already in mempool from previous disconnects
+
+ # Reorg back with 25 block chain from node 0
+ self.utxo = self.nodes[0].listunspent()
+ for i in xrange(25):
+ self.mine_full_block(self.nodes[0],self.address[0])
+
+ # Create connections in the order so both nodes can see the reorg at the same time
+ connect_nodes(self.nodes[1], 0)
+ connect_nodes(self.nodes[2], 0)
+ sync_blocks(self.nodes[0:3])
+
+ print "Usage can be over target because of high stale rate:", calc_usage(self.prunedir)
+
+ def reorg_test(self):
+ # Node 1 will mine a 300 block chain starting 287 blocks back from Node 0 and Node 2's tip
+ # This will cause Node 2 to do a reorg requiring 288 blocks of undo data to the reorg_test chain
+ # Reboot node 1 to clear its mempool (hopefully make the invalidate faster)
+ # Lower the block max size so we don't keep mining all our big mempool transactions (from disconnected blocks)
+ stop_node(self.nodes[1],1)
+ self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=300)
+
+ height = self.nodes[1].getblockcount()
+ print "Current block height:", height
+
+ invalidheight = height-287
+ badhash = self.nodes[1].getblockhash(invalidheight)
+ print "Invalidating block at height:",invalidheight,badhash
+ self.nodes[1].invalidateblock(badhash)
+
+ # We've now switched to our previously mined-24 block fork on node 1, but thats not what we want
+ # So invalidate that fork as well, until we're on the same chain as node 0/2 (but at an ancestor 288 blocks ago)
+ mainchainhash = self.nodes[0].getblockhash(invalidheight - 1)
+ curhash = self.nodes[1].getblockhash(invalidheight - 1)
+ while curhash != mainchainhash:
+ self.nodes[1].invalidateblock(curhash)
+ curhash = self.nodes[1].getblockhash(invalidheight - 1)
+
+ assert(self.nodes[1].getblockcount() == invalidheight - 1)
+ print "New best height", self.nodes[1].getblockcount()
+
+ # Reboot node1 to clear those giant tx's from mempool
+ stop_node(self.nodes[1],1)
+ self.nodes[1]=start_node(1, self.options.tmpdir, ["-debug","-maxreceivebuffer=20000","-blockmaxsize=5000", "-checkblocks=5", "-disablesafemode"], timewait=300)
+
+ print "Generating new longer chain of 300 more blocks"
+ self.nodes[1].generate(300)
+
+ print "Reconnect nodes"
+ connect_nodes(self.nodes[0], 1)
+ connect_nodes(self.nodes[2], 1)
+ sync_blocks(self.nodes[0:3])
+
+ print "Verify height on node 2:",self.nodes[2].getblockcount()
+ print "Usage possibly still high bc of stale blocks in block files:", calc_usage(self.prunedir)
+
+ print "Mine 220 more blocks so we have requisite history (some blocks will be big and cause pruning of previous chain)"
+ self.nodes[0].generate(220) #node 0 has many large tx's in its mempool from the disconnects
+ sync_blocks(self.nodes[0:3])
+
+ usage = calc_usage(self.prunedir)
+ print "Usage should be below target:", usage
+ if (usage > 550):
+ raise AssertionError("Pruning target not being met")
+
+ return invalidheight,badhash
+
+ def reorg_back(self):
+ # Verify that a block on the old main chain fork has been pruned away
+ try:
+ self.nodes[2].getblock(self.forkhash)
+ raise AssertionError("Old block wasn't pruned so can't test redownload")
+ except JSONRPCException as e:
+ print "Will need to redownload block",self.forkheight
+
+ # Verify that we have enough history to reorg back to the fork point
+ # Although this is more than 288 blocks, because this chain was written more recently
+ # and only its other 299 small and 220 large block are in the block files after it,
+ # its expected to still be retained
+ self.nodes[2].getblock(self.nodes[2].getblockhash(self.forkheight))
+
+ first_reorg_height = self.nodes[2].getblockcount()
+ curchainhash = self.nodes[2].getblockhash(self.mainchainheight)
+ self.nodes[2].invalidateblock(curchainhash)
+ goalbestheight = self.mainchainheight
+ goalbesthash = self.mainchainhash2
+
+ # As of 0.10 the current block download logic is not able to reorg to the original chain created in
+ # create_chain_with_stale_blocks because it doesn't know of any peer thats on that chain from which to
+ # redownload its missing blocks.
+ # Invalidate the reorg_test chain in node 0 as well, it can successfully switch to the original chain
+ # because it has all the block data.
+ # However it must mine enough blocks to have a more work chain than the reorg_test chain in order
+ # to trigger node 2's block download logic.
+ # At this point node 2 is within 288 blocks of the fork point so it will preserve its ability to reorg
+ if self.nodes[2].getblockcount() < self.mainchainheight:
+ blocks_to_mine = first_reorg_height + 1 - self.mainchainheight
+ print "Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed:", blocks_to_mine
+ self.nodes[0].invalidateblock(curchainhash)
+ assert(self.nodes[0].getblockcount() == self.mainchainheight)
+ assert(self.nodes[0].getbestblockhash() == self.mainchainhash2)
+ goalbesthash = self.nodes[0].generate(blocks_to_mine)[-1]
+ goalbestheight = first_reorg_height + 1
+
+ print "Verify node 2 reorged back to the main chain, some blocks of which it had to redownload"
+ waitstart = time.time()
+ while self.nodes[2].getblockcount() < goalbestheight:
+ time.sleep(0.1)
+ if time.time() - waitstart > 300:
+ raise AssertionError("Node 2 didn't reorg to proper height")
+ assert(self.nodes[2].getbestblockhash() == goalbesthash)
+ # Verify we can now have the data for a block previously pruned
+ assert(self.nodes[2].getblock(self.forkhash)["height"] == self.forkheight)
+
+ def mine_full_block(self, node, address):
+ # Want to create a full block
+ # We'll generate a 66k transaction below, and 14 of them is close to the 1MB block limit
+ for j in xrange(14):
+ if len(self.utxo) < 14:
+ self.utxo = node.listunspent()
+ inputs=[]
+ outputs = {}
+ t = self.utxo.pop()
+ inputs.append({ "txid" : t["txid"], "vout" : t["vout"]})
+ remchange = t["amount"] - Decimal("0.001000")
+ outputs[address]=remchange
+ # Create a basic transaction that will send change back to ourself after account for a fee
+ # And then insert the 128 generated transaction outs in the middle rawtx[92] is where the #
+ # of txouts is stored and is the only thing we overwrite from the original transaction
+ rawtx = node.createrawtransaction(inputs, outputs)
+ newtx = rawtx[0:92]
+ newtx = newtx + self.txouts
+ newtx = newtx + rawtx[94:]
+ # Appears to be ever so slightly faster to sign with SIGHASH_NONE
+ signresult = node.signrawtransaction(newtx,None,None,"NONE")
+ txid = node.sendrawtransaction(signresult["hex"], True)
+ # Mine a full sized block which will be these transactions we just created
+ node.generate(1)
+
+
+ def run_test(self):
+ print "Warning! This test requires 4GB of disk space and takes over 30 mins"
+ print "Mining a big blockchain of 995 blocks"
+ self.create_big_chain()
+ # Chain diagram key:
+ # * blocks on main chain
+ # +,&,$,@ blocks on other forks
+ # X invalidated block
+ # N1 Node 1
+ #
+ # Start by mining a simple chain that all nodes have
+ # N0=N1=N2 **...*(995)
+
+ print "Check that we haven't started pruning yet because we're below PruneAfterHeight"
+ self.test_height_min()
+ # Extend this chain past the PruneAfterHeight
+ # N0=N1=N2 **...*(1020)
+
+ print "Check that we'll exceed disk space target if we have a very high stale block rate"
+ self.create_chain_with_staleblocks()
+ # Disconnect N0
+ # And mine a 24 block chain on N1 and a separate 25 block chain on N0
+ # N1=N2 **...*+...+(1044)
+ # N0 **...**...**(1045)
+ #
+ # reconnect nodes causing reorg on N1 and N2
+ # N1=N2 **...*(1020) *...**(1045)
+ # \
+ # +...+(1044)
+ #
+ # repeat this process until you have 12 stale forks hanging off the
+ # main chain on N1 and N2
+ # N0 *************************...***************************(1320)
+ #
+ # N1=N2 **...*(1020) *...**(1045) *.. ..**(1295) *...**(1320)
+ # \ \ \
+ # +...+(1044) &.. $...$(1319)
+
+ # Save some current chain state for later use
+ self.mainchainheight = self.nodes[2].getblockcount() #1320
+ self.mainchainhash2 = self.nodes[2].getblockhash(self.mainchainheight)
+
+ print "Check that we can survive a 288 block reorg still"
+ (self.forkheight,self.forkhash) = self.reorg_test() #(1033, )
+ # Now create a 288 block reorg by mining a longer chain on N1
+ # First disconnect N1
+ # Then invalidate 1033 on main chain and 1032 on fork so height is 1032 on main chain
+ # N1 **...*(1020) **...**(1032)X..
+ # \
+ # ++...+(1031)X..
+ #
+ # Now mine 300 more blocks on N1
+ # N1 **...*(1020) **...**(1032) @@...@(1332)
+ # \ \
+ # \ X...
+ # \ \
+ # ++...+(1031)X.. ..
+ #
+ # Reconnect nodes and mine 220 more blocks on N1
+ # N1 **...*(1020) **...**(1032) @@...@@@(1552)
+ # \ \
+ # \ X...
+ # \ \
+ # ++...+(1031)X.. ..
+ #
+ # N2 **...*(1020) **...**(1032) @@...@@@(1552)
+ # \ \
+ # \ *...**(1320)
+ # \ \
+ # ++...++(1044) ..
+ #
+ # N0 ********************(1032) @@...@@@(1552)
+ # \
+ # *...**(1320)
+
+ print "Test that we can rerequest a block we previously pruned if needed for a reorg"
+ self.reorg_back()
+ # Verify that N2 still has block 1033 on current chain (@), but not on main chain (*)
+ # Invalidate 1033 on current chain (@) on N2 and we should be able to reorg to
+ # original main chain (*), but will require redownload of some blocks
+ # In order to have a peer we think we can download from, must also perform this invalidation
+ # on N0 and mine a new longest chain to trigger.
+ # Final result:
+ # N0 ********************(1032) **...****(1553)
+ # \
+ # X@...@@@(1552)
+ #
+ # N2 **...*(1020) **...**(1032) **...****(1553)
+ # \ \
+ # \ X@...@@@(1552)
+ # \
+ # +..
+ #
+ # N1 doesn't change because 1033 on main chain (*) is invalid
+
+ print "Done"
+
+if __name__ == '__main__':
+ PruneTest().main()
diff --git a/qa/rpc-tests/util.py b/qa/rpc-tests/util.py
index 9ecee31959..cf789f48e2 100644
--- a/qa/rpc-tests/util.py
+++ b/qa/rpc-tests/util.py
@@ -158,7 +158,7 @@ def _rpchost_to_args(rpchost):
rv += ['-rpcport=' + rpcport]
return rv
-def start_node(i, dirname, extra_args=None, rpchost=None):
+def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None):
"""
Start a bitcoind and return RPC connection to it
"""
@@ -172,7 +172,10 @@ def start_node(i, dirname, extra_args=None, rpchost=None):
["-rpcwait", "getblockcount"], stdout=devnull)
devnull.close()
url = "http://rt:rt@%s:%d" % (rpchost or '127.0.0.1', rpc_port(i))
- proxy = AuthServiceProxy(url)
+ if timewait is not None:
+ proxy = AuthServiceProxy(url, timeout=timewait)
+ else:
+ proxy = AuthServiceProxy(url)
proxy.url = url # store URL on proxy for info
return proxy
diff --git a/src/chainparams.cpp b/src/chainparams.cpp
index 2d4fde2476..589c7b5472 100644
--- a/src/chainparams.cpp
+++ b/src/chainparams.cpp
@@ -121,6 +121,7 @@ public:
vAlertPubKey = ParseHex("04fc9702847840aaf195de8442ebecedf5b095cdbb9bc716bda9110971b28a49e0ead8564ff0db22209e0374782c093bb899692d524e9d6a6956e7c5ecbcd68284");
nDefaultPort = 8333;
nMinerThreads = 0;
+ nPruneAfterHeight = 100000;
/**
* Build the genesis block. Note that the output of the genesis coinbase cannot
@@ -198,6 +199,7 @@ public:
vAlertPubKey = ParseHex("04302390343f91cc401d56d68b123028bf52e5fca1939df127f63c6467cdf9c8e2c14b61104cf817d0b780da337893ecc4aaff1309e536162dabbdb45200ca2b0a");
nDefaultPort = 18333;
nMinerThreads = 0;
+ nPruneAfterHeight = 1000;
//! Modify the testnet genesis block so the timestamp is valid for a later start.
genesis.nTime = 1296688602;
@@ -257,6 +259,7 @@ public:
consensus.hashGenesisBlock = genesis.GetHash();
nDefaultPort = 18444;
assert(consensus.hashGenesisBlock == uint256S("0x0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206"));
+ nPruneAfterHeight = 1000;
vFixedSeeds.clear(); //! Regtest mode doesn't have any fixed seeds.
vSeeds.clear(); //! Regtest mode doesn't have any DNS seeds.
diff --git a/src/chainparams.h b/src/chainparams.h
index 0692094478..5e974123dc 100644
--- a/src/chainparams.h
+++ b/src/chainparams.h
@@ -58,6 +58,7 @@ public:
bool DefaultConsistencyChecks() const { return fDefaultConsistencyChecks; }
/** Make standard checks */
bool RequireStandard() const { return fRequireStandard; }
+ int64_t PruneAfterHeight() const { return nPruneAfterHeight; }
/** Make miner stop after a block is found. In RPC, don't return until nGenProcLimit blocks are generated */
bool MineBlocksOnDemand() const { return fMineBlocksOnDemand; }
/** In the future use NetworkIDString() for RPC fields */
@@ -77,6 +78,7 @@ protected:
std::vector<unsigned char> vAlertPubKey;
int nDefaultPort;
int nMinerThreads;
+ uint64_t nPruneAfterHeight;
std::vector<CDNSSeedData> vSeeds;
std::vector<unsigned char> base58Prefixes[MAX_BASE58_TYPES];
std::string strNetworkID;
diff --git a/src/init.cpp b/src/init.cpp
index 665aebe1c5..b648a237bf 100644
--- a/src/init.cpp
+++ b/src/init.cpp
@@ -275,7 +275,12 @@ std::string HelpMessage(HelpMessageMode mode)
#ifndef WIN32
strUsage += HelpMessageOpt("-pid=<file>", strprintf(_("Specify pid file (default: %s)"), "bitcoind.pid"));
#endif
- strUsage += HelpMessageOpt("-reindex", _("Rebuild block chain index from current blk000??.dat files") + " " + _("on startup"));
+ strUsage += HelpMessageOpt("-prune=<n>", _("Reduce storage requirements by pruning (deleting) old blocks. This mode disables wallet support and is incompatible with -txindex.") + " " +
+ _("Warning: Reverting this setting requires re-downloading the entire blockchain.") + " " +
+ _("(default: 0 = disable pruning blocks,") + " " +
+ strprintf(_(">%u = target size in MiB to use for block files)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
+strUsage += HelpMessageOpt("-reindex", _("Rebuild block chain index from current blk000??.dat files") + " " + _("on startup"));
+
#if !defined(WIN32)
strUsage += HelpMessageOpt("-sysperms", _("Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)"));
#endif
@@ -352,7 +357,7 @@ std::string HelpMessage(HelpMessageMode mode)
strUsage += HelpMessageOpt("-flushwallet", strprintf(_("Run a thread to flush wallet periodically (default: %u)"), 1));
strUsage += HelpMessageOpt("-stopafterblockimport", strprintf(_("Stop running after importing blocks from disk (default: %u)"), 0));
}
- string debugCategories = "addrman, alert, bench, coindb, db, lock, rand, rpc, selectcoins, mempool, net, proxy"; // Don't translate these and qt below
+ string debugCategories = "addrman, alert, bench, coindb, db, lock, rand, rpc, selectcoins, mempool, net, proxy, prune"; // Don't translate these and qt below
if (mode == HMM_BITCOIN_QT)
debugCategories += ", qt";
strUsage += HelpMessageOpt("-debug=<category>", strprintf(_("Output debugging information (default: %u, supplying <category> is optional)"), 0) + ". " +
@@ -458,10 +463,33 @@ struct CImportingNow
}
};
+
+// If we're using -prune with -reindex, then delete block files that will be ignored by the
+// reindex. Since reindexing works by starting at block file 0 and looping until a blockfile
+// is missing, and since pruning works by deleting the oldest block file first, just check
+// for block file 0, and if it doesn't exist, delete all the block files in the
+// directory (since they won't be read by the reindex but will take up disk space).
+void DeleteAllBlockFiles()
+{
+ if (boost::filesystem::exists(GetBlockPosFilename(CDiskBlockPos(0, 0), "blk")))
+ return;
+
+ LogPrintf("Removing all blk?????.dat and rev?????.dat files for -reindex with -prune\n");
+ boost::filesystem::path blocksdir = GetDataDir() / "blocks";
+ for (boost::filesystem::directory_iterator it(blocksdir); it != boost::filesystem::directory_iterator(); it++) {
+ if (is_regular_file(*it)) {
+ if ((it->path().filename().string().length() == 12) &&
+ (it->path().filename().string().substr(8,4) == ".dat") &&
+ ((it->path().filename().string().substr(0,3) == "blk") ||
+ (it->path().filename().string().substr(0,3) == "rev")))
+ boost::filesystem::remove(it->path());
+ }
+ }
+}
+
void ThreadImport(std::vector<boost::filesystem::path> vImportFiles)
{
RenameThread("bitcoin-loadblk");
-
// -reindex
if (fReindex) {
CImportingNow imp;
@@ -674,6 +702,21 @@ bool AppInit2(boost::thread_group& threadGroup)
if (nFD - MIN_CORE_FILEDESCRIPTORS < nMaxConnections)
nMaxConnections = nFD - MIN_CORE_FILEDESCRIPTORS;
+ // if using block pruning, then disable txindex
+ // also disable the wallet (for now, until SPV support is implemented in wallet)
+ if (GetArg("-prune", 0)) {
+ if (GetBoolArg("-txindex", false))
+ return InitError(_("Prune mode is incompatible with -txindex."));
+#ifdef ENABLE_WALLET
+ if (!GetBoolArg("-disablewallet", false)) {
+ if (SoftSetBoolArg("-disablewallet", true))
+ LogPrintf("%s : parameter interaction: -prune -> setting -disablewallet=1\n", __func__);
+ else
+ return InitError(_("Can't run with a wallet in prune mode."));
+ }
+#endif
+ }
+
// ********************************************************* Step 3: parameter-to-internal-flags
fDebug = !mapMultiArgs["-debug"].empty();
@@ -710,6 +753,21 @@ bool AppInit2(boost::thread_group& threadGroup)
nScriptCheckThreads = MAX_SCRIPTCHECK_THREADS;
fServer = GetBoolArg("-server", false);
+
+ // block pruning; get the amount of disk space (in MB) to allot for block & undo files
+ int64_t nSignedPruneTarget = GetArg("-prune", 0) * 1024 * 1024;
+ if (nSignedPruneTarget < 0) {
+ return InitError(_("Prune cannot be configured with a negative value."));
+ }
+ nPruneTarget = (uint64_t) nSignedPruneTarget;
+ if (nPruneTarget) {
+ if (nPruneTarget < MIN_DISK_SPACE_FOR_BLOCK_FILES) {
+ return InitError(strprintf(_("Prune configured below the minimum of %d MB. Please use a higher number."), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
+ }
+ LogPrintf("Prune configured to target %uMiB on disk for block and undo files.\n", nPruneTarget / 1024 / 1024);
+ fPruneMode = true;
+ }
+
#ifdef ENABLE_WALLET
bool fDisableWallet = GetBoolArg("-disablewallet", false);
#endif
@@ -1030,8 +1088,12 @@ bool AppInit2(boost::thread_group& threadGroup)
pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview);
pcoinsTip = new CCoinsViewCache(pcoinscatcher);
- if (fReindex)
+ if (fReindex) {
pblocktree->WriteReindexing(true);
+ //If we're reindexing in prune mode, wipe away all our block and undo data files
+ if (fPruneMode)
+ DeleteAllBlockFiles();
+ }
if (!LoadBlockIndex()) {
strLoadError = _("Error loading block database");
@@ -1055,7 +1117,18 @@ bool AppInit2(boost::thread_group& threadGroup)
break;
}
+ // Check for changed -prune state. What we are concerned about is a user who has pruned blocks
+ // in the past, but is now trying to run unpruned.
+ if (fHavePruned && !fPruneMode) {
+ strLoadError = _("You need to rebuild the database using -reindex to go back to unpruned mode. This will redownload the entire blockchain");
+ break;
+ }
+
uiInterface.InitMessage(_("Verifying blocks..."));
+ if (fHavePruned && GetArg("-checkblocks", 288) > MIN_BLOCKS_TO_KEEP) {
+ LogPrintf("Prune: pruned datadir may not have more than %d blocks; -checkblocks=%d may fail\n",
+ MIN_BLOCKS_TO_KEEP, GetArg("-checkblocks", 288));
+ }
if (!CVerifyDB().VerifyDB(pcoinsdbview, GetArg("-checklevel", 3),
GetArg("-checkblocks", 288))) {
strLoadError = _("Corrupted block database detected");
@@ -1106,6 +1179,15 @@ bool AppInit2(boost::thread_group& threadGroup)
mempool.ReadFeeEstimates(est_filein);
fFeeEstimatesInitialized = true;
+ // if prune mode, unset NODE_NETWORK and prune block files
+ if (fPruneMode) {
+ LogPrintf("Unsetting NODE_NETWORK on prune mode\n");
+ nLocalServices &= ~NODE_NETWORK;
+ if (!fReindex) {
+ PruneAndFlush();
+ }
+ }
+
// ********************************************************* Step 8: load wallet
#ifdef ENABLE_WALLET
if (fDisableWallet) {
diff --git a/src/main.cpp b/src/main.cpp
index 4352c719a1..bf32ac91e7 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -52,9 +52,12 @@ int nScriptCheckThreads = 0;
bool fImporting = false;
bool fReindex = false;
bool fTxIndex = false;
+bool fHavePruned = false;
+bool fPruneMode = false;
bool fIsBareMultisigStd = true;
bool fCheckBlockIndex = false;
unsigned int nCoinCacheSize = 5000;
+uint64_t nPruneTarget = 0;
/** Fees smaller than this (in satoshi) are considered zero fee (for relaying and mining) */
CFeeRate minRelayTxFee = CFeeRate(1000);
@@ -110,17 +113,25 @@ namespace {
/**
* The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself and all ancestors) and
- * as good as our current tip or better. Entries may be failed, though.
+ * as good as our current tip or better. Entries may be failed, though, and pruning nodes may be
+ * missing the data for the block.
*/
set<CBlockIndex*, CBlockIndexWorkComparator> setBlockIndexCandidates;
/** Number of nodes with fSyncStarted. */
int nSyncStarted = 0;
- /** All pairs A->B, where A (or one if its ancestors) misses transactions, but B has transactions. */
+ /** All pairs A->B, where A (or one if its ancestors) misses transactions, but B has transactions.
+ * Pruned nodes may have entries where B is missing data.
+ */
multimap<CBlockIndex*, CBlockIndex*> mapBlocksUnlinked;
CCriticalSection cs_LastBlockFile;
std::vector<CBlockFileInfo> vinfoBlockFile;
int nLastBlockFile = 0;
+ /** Global flag to indicate we should check to see if there are
+ * block/undo files that should be deleted. Set on startup
+ * or if we allocate more file space when we're in prune mode
+ */
+ bool fCheckForPruning = false;
/**
* Every received block is assigned a unique and increasing identifier, so we
@@ -1849,6 +1860,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
}
enum FlushStateMode {
+ FLUSH_STATE_NONE,
FLUSH_STATE_IF_NEEDED,
FLUSH_STATE_PERIODIC,
FLUSH_STATE_ALWAYS
@@ -1856,16 +1868,30 @@ enum FlushStateMode {
/**
* Update the on-disk chain state.
- * The caches and indexes are flushed if either they're too large, forceWrite is set, or
- * fast is not set and it's been a while since the last write.
+ * The caches and indexes are flushed depending on the mode we're called with
+ * if they're too large, if it's been a while since the last write,
+ * or always and in all cases if we're in prune mode and are deleting files.
*/
bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) {
- LOCK(cs_main);
+ LOCK2(cs_main, cs_LastBlockFile);
static int64_t nLastWrite = 0;
+ std::set<int> setFilesToPrune;
+ bool fFlushForPrune = false;
try {
+ if (fPruneMode && fCheckForPruning) {
+ FindFilesToPrune(setFilesToPrune);
+ if (!setFilesToPrune.empty()) {
+ fFlushForPrune = true;
+ if (!fHavePruned) {
+ pblocktree->WriteFlag("prunedblockfiles", true);
+ fHavePruned = true;
+ }
+ }
+ }
if ((mode == FLUSH_STATE_ALWAYS) ||
((mode == FLUSH_STATE_PERIODIC || mode == FLUSH_STATE_IF_NEEDED) && pcoinsTip->GetCacheSize() > nCoinCacheSize) ||
- (mode == FLUSH_STATE_PERIODIC && GetTimeMicros() > nLastWrite + DATABASE_WRITE_INTERVAL * 1000000)) {
+ (mode == FLUSH_STATE_PERIODIC && GetTimeMicros() > nLastWrite + DATABASE_WRITE_INTERVAL * 1000000) ||
+ fFlushForPrune) {
// Typical CCoins structures on disk are around 100 bytes in size.
// Pushing a new one to the database can cause it to be written
// twice (once in the log, and once in the tables). This is already
@@ -1893,9 +1919,16 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) {
return state.Abort("Files to write to block index database");
}
}
- // Finally flush the chainstate (which may refer to block index entries).
+ // Flush the chainstate (which may refer to block index entries).
if (!pcoinsTip->Flush())
return state.Abort("Failed to write to coin database");
+
+ // Finally remove any pruned files
+ if (fFlushForPrune) {
+ UnlinkPrunedFiles(setFilesToPrune);
+ fCheckForPruning = false;
+ }
+
// Update best block in wallet (so we can detect restored wallets).
if (mode != FLUSH_STATE_IF_NEEDED) {
GetMainSignals().SetBestChain(chainActive.GetLocator());
@@ -1913,6 +1946,12 @@ void FlushStateToDisk() {
FlushStateToDisk(state, FLUSH_STATE_ALWAYS);
}
+void PruneAndFlush() {
+ CValidationState state;
+ fCheckForPruning = true;
+ FlushStateToDisk(state, FLUSH_STATE_NONE);
+}
+
/** Update chainActive and related internal data structures. */
void static UpdateTip(CBlockIndex *pindexNew) {
chainActive.SetTip(pindexNew);
@@ -2083,15 +2122,29 @@ static CBlockIndex* FindMostWorkChain() {
CBlockIndex *pindexTest = pindexNew;
bool fInvalidAncestor = false;
while (pindexTest && !chainActive.Contains(pindexTest)) {
- assert(pindexTest->nStatus & BLOCK_HAVE_DATA);
assert(pindexTest->nChainTx || pindexTest->nHeight == 0);
- if (pindexTest->nStatus & BLOCK_FAILED_MASK) {
- // Candidate has an invalid ancestor, remove entire chain from the set.
- if (pindexBestInvalid == NULL || pindexNew->nChainWork > pindexBestInvalid->nChainWork)
+
+ // Pruned nodes may have entries in setBlockIndexCandidates for
+ // which block files have been deleted. Remove those as candidates
+ // for the most work chain if we come across them; we can't switch
+ // to a chain unless we have all the non-active-chain parent blocks.
+ bool fFailedChain = pindexTest->nStatus & BLOCK_FAILED_MASK;
+ bool fMissingData = !(pindexTest->nStatus & BLOCK_HAVE_DATA);
+ if (fFailedChain || fMissingData) {
+ // Candidate chain is not usable (either invalid or missing data)
+ if (fFailedChain && (pindexBestInvalid == NULL || pindexNew->nChainWork > pindexBestInvalid->nChainWork))
pindexBestInvalid = pindexNew;
CBlockIndex *pindexFailed = pindexNew;
+ // Remove the entire chain from the set.
while (pindexTest != pindexFailed) {
- pindexFailed->nStatus |= BLOCK_FAILED_CHILD;
+ if (fFailedChain) {
+ pindexFailed->nStatus |= BLOCK_FAILED_CHILD;
+ } else if (fMissingData) {
+ // If we're missing data, then add back to mapBlocksUnlinked,
+ // so that if the block arrives in the future we can try adding
+ // to setBlockIndexCandidates again.
+ mapBlocksUnlinked.insert(std::make_pair(pindexFailed->pprev, pindexFailed));
+ }
setBlockIndexCandidates.erase(pindexFailed);
pindexFailed = pindexFailed->pprev;
}
@@ -2219,7 +2272,9 @@ bool ActivateBestChain(CValidationState &state, CBlock *pblock) {
uint256 hashNewTip = pindexNewTip->GetBlockHash();
// Relay inventory, but don't relay old inventory during initial block download.
int nBlockEstimate = Checkpoints::GetTotalBlocksEstimate();
- {
+ // Don't relay blocks if pruning -- could cause a peer to try to download, resulting
+ // in a stalled download if the block file is pruned before the request.
+ if (nLocalServices & NODE_NETWORK) {
LOCK(cs_vNodes);
BOOST_FOREACH(CNode* pnode, vNodes)
if (chainActive.Height() > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : nBlockEstimate))
@@ -2419,6 +2474,8 @@ bool FindBlockPos(CValidationState &state, CDiskBlockPos &pos, unsigned int nAdd
unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE;
unsigned int nNewChunks = (vinfoBlockFile[nFile].nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE;
if (nNewChunks > nOldChunks) {
+ if (fPruneMode)
+ fCheckForPruning = true;
if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos)) {
FILE *file = OpenBlockFile(pos);
if (file) {
@@ -2450,6 +2507,8 @@ bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigne
unsigned int nOldChunks = (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE;
unsigned int nNewChunks = (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE;
if (nNewChunks > nOldChunks) {
+ if (fPruneMode)
+ fCheckForPruning = true;
if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos)) {
FILE *file = OpenUndoFile(pos);
if (file) {
@@ -2665,7 +2724,10 @@ bool AcceptBlock(CBlock& block, CValidationState& state, CBlockIndex** ppindex,
if (!AcceptBlockHeader(block, state, &pindex))
return false;
- if (pindex->nStatus & BLOCK_HAVE_DATA) {
+ // If we're pruning, ensure that we don't allow a peer to dump a copy
+ // of old blocks. But we might need blocks that are not on the main chain
+ // to handle a reorg, even if we've processed once.
+ if (pindex->nStatus & BLOCK_HAVE_DATA || chainActive.Contains(pindex)) {
// TODO: deal better with duplicate blocks.
// return state.DoS(20, error("AcceptBlock(): already have block %d %s", pindex->nHeight, pindex->GetBlockHash().ToString()), REJECT_DUPLICATE, "duplicate");
return true;
@@ -2698,6 +2760,9 @@ bool AcceptBlock(CBlock& block, CValidationState& state, CBlockIndex** ppindex,
return state.Abort(std::string("System error: ") + e.what());
}
+ if (fCheckForPruning)
+ FlushStateToDisk(state, FLUSH_STATE_NONE); // we just allocated more disk space for block files
+
return true;
}
@@ -2785,6 +2850,112 @@ bool AbortNode(const std::string &strMessage, const std::string &userMessage) {
return false;
}
+
+/**
+ * BLOCK PRUNING CODE
+ */
+
+/* Calculate the amount of disk space the block & undo files currently use */
+uint64_t CalculateCurrentUsage()
+{
+ uint64_t retval = 0;
+ BOOST_FOREACH(const CBlockFileInfo &file, vinfoBlockFile) {
+ retval += file.nSize + file.nUndoSize;
+ }
+ return retval;
+}
+
+/* Prune a block file (modify associated database entries)*/
+void PruneOneBlockFile(const int fileNumber)
+{
+ for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); ++it) {
+ CBlockIndex* pindex = it->second;
+ if (pindex->nFile == fileNumber) {
+ pindex->nStatus &= ~BLOCK_HAVE_DATA;
+ pindex->nStatus &= ~BLOCK_HAVE_UNDO;
+ pindex->nFile = 0;
+ pindex->nDataPos = 0;
+ pindex->nUndoPos = 0;
+ setDirtyBlockIndex.insert(pindex);
+
+ // Prune from mapBlocksUnlinked -- any block we prune would have
+ // to be downloaded again in order to consider its chain, at which
+ // point it would be considered as a candidate for
+ // mapBlocksUnlinked or setBlockIndexCandidates.
+ std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> range = mapBlocksUnlinked.equal_range(pindex->pprev);
+ while (range.first != range.second) {
+ std::multimap<CBlockIndex *, CBlockIndex *>::iterator it = range.first;
+ range.first++;
+ if (it->second == pindex) {
+ mapBlocksUnlinked.erase(it);
+ }
+ }
+ }
+ }
+
+ vinfoBlockFile[fileNumber].SetNull();
+ setDirtyFileInfo.insert(fileNumber);
+}
+
+
+void UnlinkPrunedFiles(std::set<int>& setFilesToPrune)
+{
+ for (set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) {
+ CDiskBlockPos pos(*it, 0);
+ boost::filesystem::remove(GetBlockPosFilename(pos, "blk"));
+ boost::filesystem::remove(GetBlockPosFilename(pos, "rev"));
+ LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it);
+ }
+}
+
+/* Calculate the block/rev files that should be deleted to remain under target*/
+void FindFilesToPrune(std::set<int>& setFilesToPrune)
+{
+ LOCK2(cs_main, cs_LastBlockFile);
+ if (chainActive.Tip() == NULL || nPruneTarget == 0) {
+ return;
+ }
+ if (chainActive.Tip()->nHeight <= Params().PruneAfterHeight()) {
+ return;
+ }
+
+ unsigned int nLastBlockWeMustKeep = chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP;
+ uint64_t nCurrentUsage = CalculateCurrentUsage();
+ // We don't check to prune until after we've allocated new space for files
+ // So we should leave a buffer under our target to account for another allocation
+ // before the next pruning.
+ uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE;
+ uint64_t nBytesToPrune;
+ int count=0;
+
+ if (nCurrentUsage + nBuffer >= nPruneTarget) {
+ for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) {
+ nBytesToPrune = vinfoBlockFile[fileNumber].nSize + vinfoBlockFile[fileNumber].nUndoSize;
+
+ if (vinfoBlockFile[fileNumber].nSize == 0)
+ continue;
+
+ if (nCurrentUsage + nBuffer < nPruneTarget) // are we below our target?
+ break;
+
+ // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip
+ if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeMustKeep)
+ break;
+
+ PruneOneBlockFile(fileNumber);
+ // Queue up the files for removal
+ setFilesToPrune.insert(fileNumber);
+ nCurrentUsage -= nBytesToPrune;
+ count++;
+ }
+ }
+
+ LogPrint("prune", "Prune: target=%dMiB actual=%dMiB diff=%dMiB min_must_keep=%d removed %d blk/rev pairs\n",
+ nPruneTarget/1024/1024, nCurrentUsage/1024/1024,
+ ((int64_t)nPruneTarget - (int64_t)nCurrentUsage)/1024/1024,
+ nLastBlockWeMustKeep, count);
+}
+
bool CheckDiskSpace(uint64_t nAdditionalBytes)
{
uint64_t nFreeBytesAvailable = boost::filesystem::space(GetDataDir()).available;
@@ -2872,7 +3043,9 @@ bool static LoadBlockIndexDB()
{
CBlockIndex* pindex = item.second;
pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex);
- if (pindex->nStatus & BLOCK_HAVE_DATA) {
+ // We can link the chain of blocks for which we've received transactions at some point.
+ // Pruned nodes may have deleted the block.
+ if (pindex->nTx > 0) {
if (pindex->pprev) {
if (pindex->pprev->nChainTx) {
pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx;
@@ -2929,6 +3102,11 @@ bool static LoadBlockIndexDB()
}
}
+ // Check whether we have ever pruned block & undo files
+ pblocktree->ReadFlag("prunedblockfiles", fHavePruned);
+ if (fHavePruned)
+ LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n");
+
// Check whether we need to continue reindexing
bool fReindexing = false;
pblocktree->ReadReindexing(fReindexing);
@@ -3069,6 +3247,7 @@ void UnloadBlockIndex()
delete entry.second;
}
mapBlockIndex.clear();
+ fHavePruned = false;
}
bool LoadBlockIndex()
@@ -3260,6 +3439,7 @@ void static CheckBlockIndex()
int nHeight = 0;
CBlockIndex* pindexFirstInvalid = NULL; // Oldest ancestor of pindex which is invalid.
CBlockIndex* pindexFirstMissing = NULL; // Oldest ancestor of pindex which does not have BLOCK_HAVE_DATA.
+ CBlockIndex* pindexFirstNeverProcessed = NULL; // Oldest ancestor of pindex for which nTx == 0.
CBlockIndex* pindexFirstNotTreeValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_TREE (regardless of being valid or not).
CBlockIndex* pindexFirstNotTransactionsValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_TRANSACTIONS (regardless of being valid or not).
CBlockIndex* pindexFirstNotChainValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_CHAIN (regardless of being valid or not).
@@ -3268,6 +3448,7 @@ void static CheckBlockIndex()
nNodes++;
if (pindexFirstInvalid == NULL && pindex->nStatus & BLOCK_FAILED_VALID) pindexFirstInvalid = pindex;
if (pindexFirstMissing == NULL && !(pindex->nStatus & BLOCK_HAVE_DATA)) pindexFirstMissing = pindex;
+ if (pindexFirstNeverProcessed == NULL && pindex->nTx == 0) pindexFirstNeverProcessed = pindex;
if (pindex->pprev != NULL && pindexFirstNotTreeValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TREE) pindexFirstNotTreeValid = pindex;
if (pindex->pprev != NULL && pindexFirstNotTransactionsValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) pindexFirstNotTransactionsValid = pindex;
if (pindex->pprev != NULL && pindexFirstNotChainValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_CHAIN) pindexFirstNotChainValid = pindex;
@@ -3279,12 +3460,21 @@ void static CheckBlockIndex()
assert(pindex->GetBlockHash() == consensusParams.hashGenesisBlock); // Genesis block's hash must match.
assert(pindex == chainActive.Genesis()); // The current active chain's genesis block must be this block.
}
- // HAVE_DATA is equivalent to VALID_TRANSACTIONS and equivalent to nTx > 0 (we stored the number of transactions in the block)
- assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0));
- assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0));
if (pindex->nChainTx == 0) assert(pindex->nSequenceId == 0); // nSequenceId can't be set for blocks that aren't linked
- // All parents having data is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to nChainTx being set.
- assert((pindexFirstMissing != NULL) == (pindex->nChainTx == 0)); // nChainTx == 0 is used to signal that all parent block's transaction data is available.
+ // VALID_TRANSACTIONS is equivalent to nTx > 0 for all nodes (whether or not pruning has occurred).
+ // HAVE_DATA is only equivalent to nTx > 0 (or VALID_TRANSACTIONS) if no pruning has occurred.
+ if (!fHavePruned) {
+ // If we've never pruned, then HAVE_DATA should be equivalent to nTx > 0
+ assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0));
+ assert(pindexFirstMissing == pindexFirstNeverProcessed);
+ } else {
+ // If we have pruned, then we can only say that HAVE_DATA implies nTx > 0
+ if (pindex->nStatus & BLOCK_HAVE_DATA) assert(pindex->nTx > 0);
+ }
+ if (pindex->nStatus & BLOCK_HAVE_UNDO) assert(pindex->nStatus & BLOCK_HAVE_DATA);
+ assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // This is pruning-independent.
+ // All parents having had data (at some point) is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to nChainTx being set.
+ assert((pindexFirstNeverProcessed != NULL) == (pindex->nChainTx == 0)); // nChainTx != 0 is used to signal that all parent blocks have been processed (but may have been pruned).
assert((pindexFirstNotTransactionsValid != NULL) == (pindex->nChainTx == 0));
assert(pindex->nHeight == nHeight); // nHeight must be consistent.
assert(pindex->pprev == NULL || pindex->nChainWork >= pindex->pprev->nChainWork); // For every block except the genesis block, the chainwork must be larger than the parent's.
@@ -3297,11 +3487,20 @@ void static CheckBlockIndex()
// Checks for not-invalid blocks.
assert((pindex->nStatus & BLOCK_FAILED_MASK) == 0); // The failed mask cannot be set for blocks without invalid parents.
}
- if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && pindexFirstMissing == NULL) {
- if (pindexFirstInvalid == NULL) { // If this block sorts at least as good as the current tip and is valid, it must be in setBlockIndexCandidates.
- assert(setBlockIndexCandidates.count(pindex));
+ if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && pindexFirstNeverProcessed == NULL) {
+ if (pindexFirstInvalid == NULL) {
+ // If this block sorts at least as good as the current tip and
+ // is valid and we have all data for its parents, it must be in
+ // setBlockIndexCandidates. chainActive.Tip() must also be there
+ // even if some data has been pruned.
+ if (pindexFirstMissing == NULL || pindex == chainActive.Tip()) {
+ assert(setBlockIndexCandidates.count(pindex));
+ }
+ // If some parent is missing, then it could be that this block was in
+ // setBlockIndexCandidates but had to be removed because of the missing data.
+ // In this case it must be in mapBlocksUnlinked -- see test below.
}
- } else { // If this block sorts worse than the current tip, it cannot be in setBlockIndexCandidates.
+ } else { // If this block sorts worse than the current tip or some ancestor's block has never been seen, it cannot be in setBlockIndexCandidates.
assert(setBlockIndexCandidates.count(pindex) == 0);
}
// Check whether this block is in mapBlocksUnlinked.
@@ -3315,12 +3514,28 @@ void static CheckBlockIndex()
}
rangeUnlinked.first++;
}
- if (pindex->pprev && pindex->nStatus & BLOCK_HAVE_DATA && pindexFirstMissing != NULL) {
- if (pindexFirstInvalid == NULL) { // If this block has block data available, some parent doesn't, and has no invalid parents, it must be in mapBlocksUnlinked.
- assert(foundInUnlinked);
+ if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed != NULL && pindexFirstInvalid == NULL) {
+ // If this block has block data available, some parent was never received, and has no invalid parents, it must be in mapBlocksUnlinked.
+ assert(foundInUnlinked);
+ }
+ if (!(pindex->nStatus & BLOCK_HAVE_DATA)) assert(!foundInUnlinked); // Can't be in mapBlocksUnlinked if we don't HAVE_DATA
+ if (pindexFirstMissing == NULL) assert(!foundInUnlinked); // We aren't missing data for any parent -- cannot be in mapBlocksUnlinked.
+ if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed == NULL && pindexFirstMissing != NULL) {
+ // We HAVE_DATA for this block, have received data for all parents at some point, but we're currently missing data for some parent.
+ assert(fHavePruned); // We must have pruned.
+ // This block may have entered mapBlocksUnlinked if:
+ // - it has a descendant that at some point had more work than the
+ // tip, and
+ // - we tried switching to that descendant but were missing
+ // data for some intermediate block between chainActive and the
+ // tip.
+ // So if this block is itself better than chainActive.Tip() and it wasn't in
+ // setBlockIndexCandidates, then it must be in mapBlocksUnlinked.
+ if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && setBlockIndexCandidates.count(pindex) == 0) {
+ if (pindexFirstInvalid == NULL) {
+ assert(foundInUnlinked);
+ }
}
- } else { // If this block does not have block data available, or all parents do, it cannot be in mapBlocksUnlinked.
- assert(!foundInUnlinked);
}
// assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow
// End: actual consistency checks.
@@ -3340,6 +3555,7 @@ void static CheckBlockIndex()
// If pindex was the first with a certain property, unset the corresponding variable.
if (pindex == pindexFirstInvalid) pindexFirstInvalid = NULL;
if (pindex == pindexFirstMissing) pindexFirstMissing = NULL;
+ if (pindex == pindexFirstNeverProcessed) pindexFirstNeverProcessed = NULL;
if (pindex == pindexFirstNotTreeValid) pindexFirstNotTreeValid = NULL;
if (pindex == pindexFirstNotTransactionsValid) pindexFirstNotTransactionsValid = NULL;
if (pindex == pindexFirstNotChainValid) pindexFirstNotChainValid = NULL;
@@ -3497,7 +3713,9 @@ void static ProcessGetData(CNode* pfrom)
}
}
}
- if (send)
+ // Pruned nodes may have deleted the block, so check whether
+ // it's available before trying to send.
+ if (send && (mi->second->nStatus & BLOCK_HAVE_DATA))
{
// Send block from disk
CBlock block;
diff --git a/src/main.h b/src/main.h
index bb6fd6f204..d34f6c30d1 100644
--- a/src/main.h
+++ b/src/main.h
@@ -132,6 +132,26 @@ extern CBlockIndex *pindexBestHeader;
/** Minimum disk space required - used in CheckDiskSpace() */
static const uint64_t nMinDiskSpace = 52428800;
+/** Pruning-related variables and constants */
+/** True if any block files have ever been pruned. */
+extern bool fHavePruned;
+/** True if we're running in -prune mode. */
+extern bool fPruneMode;
+/** Number of MiB of block files that we're trying to stay below. */
+extern uint64_t nPruneTarget;
+/** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of chainActive.Tip() will not be pruned. */
+static const signed int MIN_BLOCKS_TO_KEEP = 288;
+
+// Require that user allocate at least 550MB for block & undo files (blk???.dat and rev???.dat)
+// At 1MB per block, 288 blocks = 288MB.
+// Add 15% for Undo data = 331MB
+// Add 20% for Orphan block rate = 397MB
+// We want the low water mark after pruning to be at least 397 MB and since we prune in
+// full block file chunks, we need the high water mark which triggers the prune to be
+// one 128MB block file + added 15% undo data = 147MB greater for a total of 545MB
+// Setting the target to > than 550MB will make it likely we can respect the target.
+static const signed int MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024;
+
/** Register with a network node to receive its signals */
void RegisterNodeSignals(CNodeSignals& nodeSignals);
/** Unregister a network node */
@@ -186,6 +206,28 @@ bool GetTransaction(const uint256 &hash, CTransaction &tx, uint256 &hashBlock, b
bool ActivateBestChain(CValidationState &state, CBlock *pblock = NULL);
CAmount GetBlockValue(int nHeight, const CAmount& nFees);
+/**
+ * Prune block and undo files (blk???.dat and undo???.dat) so that the disk space used is less than a user-defined target.
+ * The user sets the target (in MB) on the command line or in config file. This will be run on startup and whenever new
+ * space is allocated in a block or undo file, staying below the target. Changing back to unpruned requires a reindex
+ * (which in this case means the blockchain must be re-downloaded.)
+ *
+ * Pruning functions are called from FlushStateToDisk when the global fCheckForPruning flag has been set.
+ * Block and undo files are deleted in lock-step (when blk00003.dat is deleted, so is rev00003.dat.)
+ * Pruning cannot take place until the longest chain is at least a certain length (100000 on mainnet, 1000 on testnet, 10 on regtest).
+ * Pruning will never delete a block within a defined distance (currently 288) from the active chain's tip.
+ * The block index is updated by unsetting HAVE_DATA and HAVE_UNDO for any blocks that were stored in the deleted files.
+ * A db flag records the fact that at least some block files have been pruned.
+ *
+ * @param[out] setFilesToPrune The set of file indices that can be unlinked will be returned
+ */
+void FindFilesToPrune(std::set<int>& setFilesToPrune);
+
+/**
+ * Actually unlink the specified files
+ */
+void UnlinkPrunedFiles(std::set<int>& setFilesToPrune);
+
/** Create a new block index entry for a given block hash */
CBlockIndex * InsertBlockIndex(uint256 hash);
/** Abort with a message */
@@ -196,7 +238,8 @@ bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats);
void Misbehaving(NodeId nodeid, int howmuch);
/** Flush all state, indexes and buffers to disk. */
void FlushStateToDisk();
-
+/** Prune block files and flush state to disk. */
+void PruneAndFlush();
/** (try to) add transaction to memory pool **/
bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree,
generated by cgit v1.2.3 (git 2.26.2) at 2024-07-08 16:22:25 +0000