Merge #15946: Allow maintaining the blockfilterindex when using prune

84716b134e9afca2fba7731de4af1f22fa1b1518 Add "index/blockfilterindex -> validation -> index/blockfilterindex" to expected circular dependencies (Jonas Schnelli)
ab3a0a2fb915d8b8384c30a8b38b4b5cc35236fd Add functional test for blockfilterindex in prune-mode (Jonas Schnelli)
c286a22f7b63a8bd336d5d7606c339053ee0054b Add debug startup parameter -fastprune for more effective pruning tests (Jonas Schnelli)
5e112269c311a559bfded814d3c3c438349a1986 Avoid pruning below the blockfilterindex sync height (Jonas Schnelli)
00d57ff76854938ead800767fb673a8af46eac8e Avoid accessing nullpointer in BaseIndex::GetSummary() (Jonas Schnelli)
6abe9f5b11cd4a5ecb6caca8443fe2950a417842 Allow blockfilter in conjunction with prune (Jonas Schnelli)

Pull request description:

  Maintaining the blockfilterindexes in prune mode is possible and may lead to efficient p2p based rescans of wallets (restore backups, import/sweep keys) beyond the prune height (rescans not part of that PR).

  This PR allows running the blockfilterindex(es) in conjunction with pruning.
  * Bitcoind/Qt will shutdown during startup when missing block data has been detected ([re]enable `-blockfilterindex` when we already have pruned)
  * manual block pruning is disabled during blockfilterindex sync
  * auto-pruning is delayed during blockfilterindex sync

  ToDos:
  * [x] Functional tests

ACKs for top commit:
  fjahr:
    Code review ACK 84716b1
  ryanofsky:
    Code review ACK 84716b134e9afca2fba7731de4af1f22fa1b1518. Only changes since last review were suggested new FindFilesToPrune argument and test.
  benthecarman:
    tACK 84716b134e9afca2fba7731de4af1f22fa1b1518

Tree-SHA512: 91d832c6c562c463f7ec7655c08956385413a99a896640b9737bda0183607fac530435d03d87c3c0e70c61ccdfe73fe8f3639bc7d26d33ca7e60925ebb97d77a

3 files changed, 51 insertions, 0 deletions

diff --git a/test/functional/feature_blockfilterindex_prune.py b/test/functional/feature_blockfilterindex_prune.py
new file mode 100755
index 0000000000..a380aae098
--- /dev/null
+++ b/test/functional/feature_blockfilterindex_prune.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test blockfilterindex in conjunction with prune."""
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+    assert_raises_rpc_error,
+    assert_greater_than,
+)
+
+class FeatureBlockfilterindexPruneTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 2
+        self.extra_args = [["-fastprune", "-prune=1"], ["-fastprune", "-prune=1", "-blockfilterindex=1"]]
+
+    def run_test(self):
+        # test basic pruning compatibility & filter access of pruned blocks
+        self.log.info("check if we can access a blockfilter when pruning is enabled but no blocks are actually pruned")
+        assert(len(self.nodes[1].getblockfilter(self.nodes[1].getbestblockhash())['filter']) > 0)
+        self.nodes[1].generate(500)
+        self.sync_all()
+        self.log.info("prune some blocks")
+        pruneheight = self.nodes[1].pruneblockchain(400)
+        assert(pruneheight != 0)
+        self.log.info("check if we can access the tips blockfilter when we have pruned some blocks")
+        assert(len(self.nodes[1].getblockfilter(self.nodes[1].getbestblockhash())['filter']) > 0)
+        self.log.info("check if we can access the blockfilter of a pruned block")
+        assert(len(self.nodes[1].getblockfilter(self.nodes[1].getblockhash(2))['filter']) > 0)
+        self.log.info("start node without blockfilterindex")
+        self.stop_node(1)
+        self.start_node(1, extra_args=self.extra_args[0])
+        self.log.info("make sure accessing the blockfilters throws an error")
+        assert_raises_rpc_error(-1,"Index is not enabled for filtertype basic", self.nodes[1].getblockfilter, self.nodes[1].getblockhash(2))
+        self.nodes[1].generate(1000)
+        self.log.info("prune below the blockfilterindexes best block while blockfilters are disabled")
+        pruneheight_new = self.nodes[1].pruneblockchain(1000)
+        assert_greater_than(pruneheight_new, pruneheight)
+        self.stop_node(1)
+        self.log.info("make sure we get an init error when starting the node again with block filters")
+        self.nodes[1].assert_start_raises_init_error(extra_args=self.extra_args[1])
+        self.nodes[1].assert_debug_log(["basic block filter index best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)"])
+        self.log.info("make sure the node starts again with the -reindex arg")
+        reindex_args = self.extra_args[1]
+        reindex_args.append("-reindex")
+        self.start_node(1, extra_args=reindex_args)
+
+if __name__ == '__main__':
+    FeatureBlockfilterindexPruneTest().main()
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index 354cacf95a..d742ef4eee 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -289,6 +289,7 @@ BASE_SCRIPTS = [
     'feature_help.py',
     'feature_shutdown.py',
     'p2p_ibd_txrelay.py',
+    'feature_blockfilterindex_prune.py'
     # Don't append tests at the end to avoid merge conflicts
     # Put them in a random line within the section that fits their approximate run-time
 ]
diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh
index 5312dbbfdb..0b15f99448 100755
--- a/test/lint/lint-circular-dependencies.sh
+++ b/test/lint/lint-circular-dependencies.sh
@@ -11,6 +11,7 @@ export LC_ALL=C
 EXPECTED_CIRCULAR_DEPENDENCIES=(
     "chainparamsbase -> util/system -> chainparamsbase"
     "index/txindex -> validation -> index/txindex"
+    "index/blockfilterindex -> validation -> index/blockfilterindex"
     "policy/fees -> txmempool -> policy/fees"
     "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel"
     "qt/bitcoingui -> qt/walletframe -> qt/bitcoingui"