net: advertise support for ADDRv2 via new message

Introduce a new message `sendaddrv2` to signal support for ADDRv2.
Send the new message immediately after sending the `VERACK` message.

Add support for receiving and parsing ADDRv2 messages.

Send ADDRv2 messages (instead of ADDR) to a peer if he has
advertised support for it.

Co-authored-by: Carl Dong <contact@carldong.me>

5 files changed, 278 insertions, 12 deletions

diff --git a/test/functional/p2p_addrv2_relay.py b/test/functional/p2p_addrv2_relay.py
new file mode 100755
index 0000000000..23ce3e5d04
--- /dev/null
+++ b/test/functional/p2p_addrv2_relay.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""
+Test addrv2 relay
+"""
+
+import time
+
+from test_framework.messages import (
+    CAddress,
+    msg_addrv2,
+    NODE_NETWORK,
+    NODE_WITNESS,
+)
+from test_framework.p2p import P2PInterface
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal
+
+ADDRS = []
+for i in range(10):
+    addr = CAddress()
+    addr.time = int(time.time()) + i
+    addr.nServices = NODE_NETWORK | NODE_WITNESS
+    addr.ip = "123.123.123.{}".format(i % 256)
+    addr.port = 8333 + i
+    ADDRS.append(addr)
+
+
+class AddrReceiver(P2PInterface):
+    addrv2_received_and_checked = False
+
+    def __init__(self):
+        super().__init__(support_addrv2 = True)
+
+    def on_addrv2(self, message):
+        for addr in message.addrs:
+            assert_equal(addr.nServices, 9)
+            assert addr.ip.startswith('123.123.123.')
+            assert (8333 <= addr.port < 8343)
+        self.addrv2_received_and_checked = True
+
+    def wait_for_addrv2(self):
+        self.wait_until(lambda: "addrv2" in self.last_message)
+
+
+class AddrTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.setup_clean_chain = True
+        self.num_nodes = 1
+
+    def run_test(self):
+        self.log.info('Create connection that sends addrv2 messages')
+        addr_source = self.nodes[0].add_p2p_connection(P2PInterface())
+        msg = msg_addrv2()
+
+        self.log.info('Send too-large addrv2 message')
+        msg.addrs = ADDRS * 101
+        with self.nodes[0].assert_debug_log(['addrv2 message size = 1010']):
+            addr_source.send_and_ping(msg)
+
+        self.log.info('Check that addrv2 message content is relayed and added to addrman')
+        addr_receiver = self.nodes[0].add_p2p_connection(AddrReceiver())
+        msg.addrs = ADDRS
+        with self.nodes[0].assert_debug_log([
+                'Added 10 addresses from 127.0.0.1: 0 tried',
+                'received: addrv2 (131 bytes) peer=0',
+                'sending addrv2 (131 bytes) peer=1',
+        ]):
+            addr_source.send_and_ping(msg)
+            self.nodes[0].setmocktime(int(time.time()) + 30 * 60)
+            addr_receiver.wait_for_addrv2()
+
+        assert addr_receiver.addrv2_received_and_checked
+
+
+if __name__ == '__main__':
+    AddrTest().main()
diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py
index fbe58c5e2f..db72a361d9 100755
--- a/test/functional/p2p_invalid_messages.py
+++ b/test/functional/p2p_invalid_messages.py
@@ -4,6 +4,9 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 """Test node responses to invalid network messages."""
 
+import struct
+import time
+
 from test_framework.messages import (
     CBlockHeader,
     CInv,
@@ -22,7 +25,10 @@ from test_framework.p2p import (
     P2PInterface,
 )
 from test_framework.test_framework import BitcoinTestFramework
-from test_framework.util import assert_equal
+from test_framework.util import (
+    assert_equal,
+    hex_str_to_bytes,
+)
 
 VALID_DATA_LIMIT = MAX_PROTOCOL_MESSAGE_LENGTH - 5  # Account for the 5-byte length prefix
 
@@ -42,6 +48,11 @@ class msg_unrecognized:
         return "{}(data={})".format(self.msgtype, self.str_data)
 
 
+class SenderOfAddrV2(P2PInterface):
+    def wait_for_sendaddrv2(self):
+        self.wait_until(lambda: 'sendaddrv2' in self.last_message)
+
+
 class InvalidMessagesTest(BitcoinTestFramework):
     def set_test_params(self):
         self.num_nodes = 1
@@ -53,6 +64,10 @@ class InvalidMessagesTest(BitcoinTestFramework):
         self.test_checksum()
         self.test_size()
         self.test_msgtype()
+        self.test_addrv2_empty()
+        self.test_addrv2_no_addresses()
+        self.test_addrv2_too_long_address()
+        self.test_addrv2_unrecognized_network()
         self.test_oversized_inv_msg()
         self.test_oversized_getdata_msg()
         self.test_oversized_headers_msg()
@@ -127,6 +142,84 @@ class InvalidMessagesTest(BitcoinTestFramework):
         assert_equal(self.nodes[0].getpeerinfo()[0]['bytesrecv_per_msg']['*other*'], 26)
         self.nodes[0].disconnect_p2ps()
 
+    def test_addrv2(self, label, required_log_messages, raw_addrv2):
+        node = self.nodes[0]
+        conn = node.add_p2p_connection(SenderOfAddrV2())
+
+        # Make sure bitcoind signals support for ADDRv2, otherwise this test
+        # will bombard an old node with messages it does not recognize which
+        # will produce unexpected results.
+        conn.wait_for_sendaddrv2()
+
+        self.log.info('Test addrv2: ' + label)
+
+        msg = msg_unrecognized(str_data=b'')
+        msg.msgtype = b'addrv2'
+        with node.assert_debug_log(required_log_messages):
+            # override serialize() which would include the length of the data
+            msg.serialize = lambda: raw_addrv2
+            conn.send_raw_message(conn.build_message(msg))
+            conn.sync_with_ping()
+
+        node.disconnect_p2ps()
+
+    def test_addrv2_empty(self):
+        self.test_addrv2('empty',
+            [
+                'received: addrv2 (0 bytes)',
+                'ProcessMessages(addrv2, 0 bytes): Exception',
+                'end of data',
+            ],
+            b'')
+
+    def test_addrv2_no_addresses(self):
+        self.test_addrv2('no addresses',
+            [
+                'received: addrv2 (1 bytes)',
+            ],
+            hex_str_to_bytes('00'))
+
+    def test_addrv2_too_long_address(self):
+        self.test_addrv2('too long address',
+            [
+                'received: addrv2 (525 bytes)',
+                'ProcessMessages(addrv2, 525 bytes): Exception',
+                'Address too long: 513 > 512',
+            ],
+            hex_str_to_bytes(
+                '01' +       # number of entries
+                '61bc6649' + # time, Fri Jan  9 02:54:25 UTC 2009
+                '00' +       # service flags, COMPACTSIZE(NODE_NONE)
+                '01' +       # network type (IPv4)
+                'fd0102' +   # address length (COMPACTSIZE(513))
+                'ab' * 513 + # address
+                '208d'))     # port
+
+    def test_addrv2_unrecognized_network(self):
+        now_hex = struct.pack('<I', int(time.time())).hex()
+        self.test_addrv2('unrecognized network',
+            [
+                'received: addrv2 (25 bytes)',
+                'IP 9.9.9.9 mapped',
+                'Added 1 addresses',
+            ],
+            hex_str_to_bytes(
+                '02' +     # number of entries
+                # this should be ignored without impeding acceptance of subsequent ones
+                now_hex +  # time
+                '01' +     # service flags, COMPACTSIZE(NODE_NETWORK)
+                '99' +     # network type (unrecognized)
+                '02' +     # address length (COMPACTSIZE(2))
+                'ab' * 2 + # address
+                '208d' +   # port
+                # this should be added:
+                now_hex +  # time
+                '01' +     # service flags, COMPACTSIZE(NODE_NETWORK)
+                '01' +     # network type (IPv4)
+                '04' +     # address length (COMPACTSIZE(4))
+                '09' * 4 + # address
+                '208d'))   # port
+
     def test_oversized_msg(self, msg, size):
         msg_type = msg.msgtype.decode('ascii')
         self.log.info("Test {} message of size {} is logged as misbehaving".format(msg_type, size))
diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py
index 00cf1ef66d..ff7f73bdf4 100755
--- a/test/functional/test_framework/messages.py
+++ b/test/functional/test_framework/messages.py
@@ -136,12 +136,17 @@ def uint256_from_compact(c):
     return v
 
 
-def deser_vector(f, c):
+# deser_function_name: Allow for an alternate deserialization function on the
+# entries in the vector.
+def deser_vector(f, c, deser_function_name=None):
     nit = deser_compact_size(f)
     r = []
     for _ in range(nit):
         t = c()
-        t.deserialize(f)
+        if deser_function_name:
+            getattr(t, deser_function_name)(f)
+        else:
+            t.deserialize(f)
         r.append(t)
     return r
 
@@ -204,38 +209,82 @@ def ToHex(obj):
 
 
 class CAddress:
-    __slots__ = ("ip", "nServices", "pchReserved", "port", "time")
+    __slots__ = ("net", "ip", "nServices", "port", "time")
+
+    # see https://github.com/bitcoin/bips/blob/master/bip-0155.mediawiki
+    NET_IPV4 = 1
+
+    ADDRV2_NET_NAME = {
+        NET_IPV4: "IPv4"
+    }
+
+    ADDRV2_ADDRESS_LENGTH = {
+        NET_IPV4: 4
+    }
 
     def __init__(self):
         self.time = 0
         self.nServices = 1
-        self.pchReserved = b"\x00" * 10 + b"\xff" * 2
+        self.net = self.NET_IPV4
         self.ip = "0.0.0.0"
         self.port = 0
 
     def deserialize(self, f, *, with_time=True):
+        """Deserialize from addrv1 format (pre-BIP155)"""
         if with_time:
             # VERSION messages serialize CAddress objects without time
-            self.time = struct.unpack("<i", f.read(4))[0]
+            self.time = struct.unpack("<I", f.read(4))[0]
         self.nServices = struct.unpack("<Q", f.read(8))[0]
-        self.pchReserved = f.read(12)
+        # We only support IPv4 which means skip 12 bytes and read the next 4 as IPv4 address.
+        f.read(12)
+        self.net = self.NET_IPV4
         self.ip = socket.inet_ntoa(f.read(4))
         self.port = struct.unpack(">H", f.read(2))[0]
 
     def serialize(self, *, with_time=True):
+        """Serialize in addrv1 format (pre-BIP155)"""
+        assert self.net == self.NET_IPV4
         r = b""
         if with_time:
             # VERSION messages serialize CAddress objects without time
-            r += struct.pack("<i", self.time)
+            r += struct.pack("<I", self.time)
         r += struct.pack("<Q", self.nServices)
-        r += self.pchReserved
+        r += b"\x00" * 10 + b"\xff" * 2
+        r += socket.inet_aton(self.ip)
+        r += struct.pack(">H", self.port)
+        return r
+
+    def deserialize_v2(self, f):
+        """Deserialize from addrv2 format (BIP155)"""
+        self.time = struct.unpack("<I", f.read(4))[0]
+
+        self.nServices = deser_compact_size(f)
+
+        self.net = struct.unpack("B", f.read(1))[0]
+        assert self.net == self.NET_IPV4
+
+        address_length = deser_compact_size(f)
+        assert address_length == self.ADDRV2_ADDRESS_LENGTH[self.net]
+
+        self.ip = socket.inet_ntoa(f.read(4))
+
+        self.port = struct.unpack(">H", f.read(2))[0]
+
+    def serialize_v2(self):
+        """Serialize in addrv2 format (BIP155)"""
+        assert self.net == self.NET_IPV4
+        r = b""
+        r += struct.pack("<I", self.time)
+        r += ser_compact_size(self.nServices)
+        r += struct.pack("B", self.net)
+        r += ser_compact_size(self.ADDRV2_ADDRESS_LENGTH[self.net])
         r += socket.inet_aton(self.ip)
         r += struct.pack(">H", self.port)
         return r
 
     def __repr__(self):
-        return "CAddress(nServices=%i ip=%s port=%i)" % (self.nServices,
-                                                         self.ip, self.port)
+        return ("CAddress(nServices=%i net=%s addr=%s port=%i)"
+                % (self.nServices, self.ADDRV2_NET_NAME[self.net], self.ip, self.port))
 
 
 class CInv:
@@ -1064,6 +1113,40 @@ class msg_addr:
         return "msg_addr(addrs=%s)" % (repr(self.addrs))
 
 
+class msg_addrv2:
+    __slots__ = ("addrs",)
+    msgtype = b"addrv2"
+
+    def __init__(self):
+        self.addrs = []
+
+    def deserialize(self, f):
+        self.addrs = deser_vector(f, CAddress, "deserialize_v2")
+
+    def serialize(self):
+        return ser_vector(self.addrs, "serialize_v2")
+
+    def __repr__(self):
+        return "msg_addrv2(addrs=%s)" % (repr(self.addrs))
+
+
+class msg_sendaddrv2:
+    __slots__ = ()
+    msgtype = b"sendaddrv2"
+
+    def __init__(self):
+        pass
+
+    def deserialize(self, f):
+        pass
+
+    def serialize(self):
+        return b""
+
+    def __repr__(self):
+        return "msg_sendaddrv2()"
+
+
 class msg_inv:
     __slots__ = ("inv",)
     msgtype = b"inv"
diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py
index 5f9b316b18..6846d31221 100755
--- a/test/functional/test_framework/p2p.py
+++ b/test/functional/test_framework/p2p.py
@@ -33,6 +33,7 @@ from test_framework.messages import (
     MAX_HEADERS_RESULTS,
     MIN_VERSION_SUPPORTED,
     msg_addr,
+    msg_addrv2,
     msg_block,
     MSG_BLOCK,
     msg_blocktxn,
@@ -56,6 +57,7 @@ from test_framework.messages import (
     msg_notfound,
     msg_ping,
     msg_pong,
+    msg_sendaddrv2,
     msg_sendcmpct,
     msg_sendheaders,
     msg_tx,
@@ -75,6 +77,7 @@ logger = logging.getLogger("TestFramework.p2p")
 
 MESSAGEMAP = {
     b"addr": msg_addr,
+    b"addrv2": msg_addrv2,
     b"block": msg_block,
     b"blocktxn": msg_blocktxn,
     b"cfcheckpt": msg_cfcheckpt,
@@ -97,6 +100,7 @@ MESSAGEMAP = {
     b"notfound": msg_notfound,
     b"ping": msg_ping,
     b"pong": msg_pong,
+    b"sendaddrv2": msg_sendaddrv2,
     b"sendcmpct": msg_sendcmpct,
     b"sendheaders": msg_sendheaders,
     b"tx": msg_tx,
@@ -285,7 +289,7 @@ class P2PInterface(P2PConnection):
 
     Individual testcases should subclass this and override the on_* methods
     if they want to alter message handling behaviour."""
-    def __init__(self):
+    def __init__(self, support_addrv2=False):
         super().__init__()
 
         # Track number of messages of each type received.
@@ -303,6 +307,8 @@ class P2PInterface(P2PConnection):
         # The network services received from the peer
         self.nServices = 0
 
+        self.support_addrv2 = support_addrv2
+
     def peer_connect(self, *args, services=NODE_NETWORK|NODE_WITNESS, send_version=True, **kwargs):
         create_conn = super().peer_connect(*args, **kwargs)
 
@@ -345,6 +351,7 @@ class P2PInterface(P2PConnection):
         pass
 
     def on_addr(self, message): pass
+    def on_addrv2(self, message): pass
     def on_block(self, message): pass
     def on_blocktxn(self, message): pass
     def on_cfcheckpt(self, message): pass
@@ -365,6 +372,7 @@ class P2PInterface(P2PConnection):
     def on_merkleblock(self, message): pass
     def on_notfound(self, message): pass
     def on_pong(self, message): pass
+    def on_sendaddrv2(self, message): pass
     def on_sendcmpct(self, message): pass
     def on_sendheaders(self, message): pass
     def on_tx(self, message): pass
@@ -389,6 +397,8 @@ class P2PInterface(P2PConnection):
         if message.nVersion >= 70016:
             self.send_message(msg_wtxidrelay())
         self.send_message(msg_verack())
+        if self.support_addrv2:
+            self.send_message(msg_sendaddrv2())
         self.nServices = message.nServices
 
     # Connection helper methods
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index c8cf173d5f..ba32f64065 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -154,6 +154,7 @@ BASE_SCRIPTS = [
     'feature_proxy.py',
     'rpc_signrawtransaction.py',
     'wallet_groups.py',
+    'p2p_addrv2_relay.py',
     'p2p_disconnect_ban.py',
     'rpc_decodescript.py',
     'rpc_blockchain.py',