Merge #19044: net processing: Add support for getcfilters

9e36067d8cd02830c7e5a88a391dff6ac3adbe0c [test] Add test for cfilters. (Jim Posen)
11106a4722558765a44ae45c7892724a73ce514c [net processing] Message handling for getcfilters. (Jim Posen)
e535670726952e43483763dfca6fc6ec2f4b0691 [indexes] Fix default [de]serialization of BlockFilter. (Jim Posen)
bb911ae7f5cbe4974ec61266d2334b95067fa49d [refactor] Pass CNode and CConnman by reference (John Newbery)

Pull request description:

  Support `getcfilters` requests when `-peerblockfilters` is set.

  Does not advertise compact filter support in version messages.

ACKs for top commit:
  Empact:
    re-Code Review ACK https://github.com/bitcoin/bitcoin/pull/19044/commits/9e36067d8cd02830c7e5a88a391dff6ac3adbe0c
  MarcoFalke:
    re-ACK 9e36067d8c , only change is adding commit "[refactor] Pass CNode and CConnman by reference" 🥑
  jkczyz:
    ACK 9e36067d8cd02830c7e5a88a391dff6ac3adbe0c
  fjahr:
    Code review ACK 9e36067d8cd02830c7e5a88a391dff6ac3adbe0c

Tree-SHA512: b45b42a25905ef0bd9e195029185300c86856c87f78cbe17921f4a25e159ae0f6f003e61714fa43779017eb97cd89d3568419be88e47d19dc8095562939e7887

3 files changed, 127 insertions, 7 deletions

diff --git a/test/functional/p2p_blockfilters.py b/test/functional/p2p_blockfilters.py
index 9ff76b4b3d..6d947ac660 100755
--- a/test/functional/p2p_blockfilters.py
+++ b/test/functional/p2p_blockfilters.py
@@ -13,6 +13,7 @@ from test_framework.messages import (
     hash256,
     msg_getcfcheckpt,
     msg_getcfheaders,
+    msg_getcfilters,
     ser_uint256,
     uint256_from_str,
 )
@@ -25,6 +26,21 @@ from test_framework.util import (
     wait_until,
 )
 
+class CFiltersClient(P2PInterface):
+    def __init__(self):
+        super().__init__()
+        # Store the cfilters received.
+        self.cfilters = []
+
+    def pop_cfilters(self):
+        cfilters = self.cfilters
+        self.cfilters = []
+        return cfilters
+
+    def on_cfilter(self, message):
+        """Store cfilters received in a list."""
+        self.cfilters.append(message)
+
 class CompactFiltersTest(BitcoinTestFramework):
     def set_test_params(self):
         self.setup_clean_chain = True
@@ -37,8 +53,8 @@ class CompactFiltersTest(BitcoinTestFramework):
 
     def run_test(self):
         # Node 0 supports COMPACT_FILTERS, node 1 does not.
-        node0 = self.nodes[0].add_p2p_connection(P2PInterface())
-        node1 = self.nodes[1].add_p2p_connection(P2PInterface())
+        node0 = self.nodes[0].add_p2p_connection(CFiltersClient())
+        node1 = self.nodes[1].add_p2p_connection(CFiltersClient())
 
         # Nodes 0 & 1 share the same first 999 blocks in the chain.
         self.nodes[0].generate(999)
@@ -112,7 +128,8 @@ class CompactFiltersTest(BitcoinTestFramework):
         )
         node0.send_and_ping(request)
         response = node0.last_message['cfheaders']
-        assert_equal(len(response.hashes), 1000)
+        main_cfhashes = response.hashes
+        assert_equal(len(main_cfhashes), 1000)
         assert_equal(
             compute_last_header(response.prev_header, response.hashes),
             int(main_cfcheckpt, 16)
@@ -126,12 +143,50 @@ class CompactFiltersTest(BitcoinTestFramework):
         )
         node0.send_and_ping(request)
         response = node0.last_message['cfheaders']
-        assert_equal(len(response.hashes), 1000)
+        stale_cfhashes = response.hashes
+        assert_equal(len(stale_cfhashes), 1000)
         assert_equal(
             compute_last_header(response.prev_header, response.hashes),
             int(stale_cfcheckpt, 16)
         )
 
+        self.log.info("Check that peers can fetch cfilters.")
+        stop_hash = self.nodes[0].getblockhash(10)
+        request = msg_getcfilters(
+            filter_type=FILTER_TYPE_BASIC,
+            start_height=1,
+            stop_hash=int(stop_hash, 16)
+        )
+        node0.send_message(request)
+        node0.sync_with_ping()
+        response = node0.pop_cfilters()
+        assert_equal(len(response), 10)
+
+        self.log.info("Check that cfilter responses are correct.")
+        for cfilter, cfhash, height in zip(response, main_cfhashes, range(1, 11)):
+            block_hash = self.nodes[0].getblockhash(height)
+            assert_equal(cfilter.filter_type, FILTER_TYPE_BASIC)
+            assert_equal(cfilter.block_hash, int(block_hash, 16))
+            computed_cfhash = uint256_from_str(hash256(cfilter.filter_data))
+            assert_equal(computed_cfhash, cfhash)
+
+        self.log.info("Check that peers can fetch cfilters for stale blocks.")
+        request = msg_getcfilters(
+            filter_type=FILTER_TYPE_BASIC,
+            start_height=1000,
+            stop_hash=int(stale_block_hash, 16)
+        )
+        node0.send_message(request)
+        node0.sync_with_ping()
+        response = node0.pop_cfilters()
+        assert_equal(len(response), 1)
+
+        cfilter = response[0]
+        assert_equal(cfilter.filter_type, FILTER_TYPE_BASIC)
+        assert_equal(cfilter.block_hash, int(stale_block_hash, 16))
+        computed_cfhash = uint256_from_str(hash256(cfilter.filter_data))
+        assert_equal(computed_cfhash, stale_cfhashes[999])
+
         self.log.info("Requests to node 1 without NODE_COMPACT_FILTERS results in disconnection.")
         requests = [
             msg_getcfcheckpt(
@@ -143,6 +198,11 @@ class CompactFiltersTest(BitcoinTestFramework):
                 start_height=1000,
                 stop_hash=int(main_block_hash, 16)
             ),
+            msg_getcfilters(
+                filter_type=FILTER_TYPE_BASIC,
+                start_height=1000,
+                stop_hash=int(main_block_hash, 16)
+            ),
         ]
         for request in requests:
             node1 = self.nodes[1].add_p2p_connection(P2PInterface())
@@ -151,6 +211,12 @@ class CompactFiltersTest(BitcoinTestFramework):
 
         self.log.info("Check that invalid requests result in disconnection.")
         requests = [
+            # Requesting too many filters results in disconnection.
+            msg_getcfilters(
+                filter_type=FILTER_TYPE_BASIC,
+                start_height=0,
+                stop_hash=int(main_block_hash, 16)
+            ),
             # Requesting too many filter headers results in disconnection.
             msg_getcfheaders(
                 filter_type=FILTER_TYPE_BASIC,
diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py
index d178e79541..4d1dd4422e 100755
--- a/test/functional/test_framework/messages.py
+++ b/test/functional/test_framework/messages.py
@@ -1516,6 +1516,57 @@ class msg_no_witness_blocktxn(msg_blocktxn):
     def serialize(self):
         return self.block_transactions.serialize(with_witness=False)
 
+
+class msg_getcfilters:
+    __slots__ = ("filter_type", "start_height", "stop_hash")
+    msgtype =  b"getcfilters"
+
+    def __init__(self, filter_type, start_height, stop_hash):
+        self.filter_type = filter_type
+        self.start_height = start_height
+        self.stop_hash = stop_hash
+
+    def deserialize(self, f):
+        self.filter_type = struct.unpack("<B", f.read(1))[0]
+        self.start_height = struct.unpack("<I", f.read(4))[0]
+        self.stop_hash = deser_uint256(f)
+
+    def serialize(self):
+        r = b""
+        r += struct.pack("<B", self.filter_type)
+        r += struct.pack("<I", self.start_height)
+        r += ser_uint256(self.stop_hash)
+        return r
+
+    def __repr__(self):
+        return "msg_getcfilters(filter_type={:#x}, start_height={}, stop_hash={:x})".format(
+            self.filter_type, self.start_height, self.stop_hash)
+
+class msg_cfilter:
+    __slots__ = ("filter_type", "block_hash", "filter_data")
+    msgtype =  b"cfilter"
+
+    def __init__(self, filter_type=None, block_hash=None, filter_data=None):
+        self.filter_type = filter_type
+        self.block_hash = block_hash
+        self.filter_data = filter_data
+
+    def deserialize(self, f):
+        self.filter_type = struct.unpack("<B", f.read(1))[0]
+        self.block_hash = deser_uint256(f)
+        self.filter_data = deser_string(f)
+
+    def serialize(self):
+        r = b""
+        r += struct.pack("<B", self.filter_type)
+        r += ser_uint256(self.block_hash)
+        r += ser_string(self.filter_data)
+        return r
+
+    def __repr__(self):
+        return "msg_cfilter(filter_type={:#x}, block_hash={:x})".format(
+            self.filter_type, self.block_hash)
+
 class msg_getcfheaders:
     __slots__ = ("filter_type", "start_height", "stop_hash")
     msgtype =  b"getcfheaders"
diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py
index 337939909e..45063aaff2 100755
--- a/test/functional/test_framework/mininode.py
+++ b/test/functional/test_framework/mininode.py
@@ -31,8 +31,9 @@ from test_framework.messages import (
     msg_block,
     MSG_BLOCK,
     msg_blocktxn,
-    msg_cfheaders,
     msg_cfcheckpt,
+    msg_cfheaders,
+    msg_cfilter,
     msg_cmpctblock,
     msg_feefilter,
     msg_filteradd,
@@ -69,8 +70,9 @@ MESSAGEMAP = {
     b"addr": msg_addr,
     b"block": msg_block,
     b"blocktxn": msg_blocktxn,
-    b"cfheaders": msg_cfheaders,
     b"cfcheckpt": msg_cfcheckpt,
+    b"cfheaders": msg_cfheaders,
+    b"cfilter": msg_cfilter,
     b"cmpctblock": msg_cmpctblock,
     b"feefilter": msg_feefilter,
     b"filteradd": msg_filteradd,
@@ -332,8 +334,9 @@ class P2PInterface(P2PConnection):
     def on_addr(self, message): pass
     def on_block(self, message): pass
     def on_blocktxn(self, message): pass
-    def on_cfheaders(self, message): pass
     def on_cfcheckpt(self, message): pass
+    def on_cfheaders(self, message): pass
+    def on_cfilter(self, message): pass
     def on_cmpctblock(self, message): pass
     def on_feefilter(self, message): pass
     def on_filteradd(self, message): pass