Merge bitcoin/bitcoin#24358: test: USDT tracepoint interface tests

76c60d7b31ccc50b226cdbc5e38be0bd67603408 test: validation:block_connected tracepoint test (0xb10c)
260e28ece87ba2e732ff8d8a379c4b27e77e3c0d test: utxocache:* tracepoint tests (0xb10c)
34b27bac684f2f373c5e1d90697d6bc8a014f45a test: net:in/out_message tracepoint tests (0xb10c)
c934087b627f7d368458781944f990b0eb479634 test: checks for tracepoint tests (0xb10c)

Pull request description:

  This adds functional tests for the USDT tracepoints added in https://github.com/bitcoin/bitcoin/pull/22006 and https://github.com/bitcoin/bitcoin/pull/22902. This partially fixes #23296. The tests **are probably skipped** on most systems as these tests require:
  - a Linux system with a kernel that supports BPF (and available kernel headers)
  - that Bitcoin Core is compiled with tracepoints for USDT support (default when compiled with depends)
  - [bcc](https://github.com/iovisor/bcc) installed
  - the tests are run with a privileged user that is able to e.g. do BPF syscalls and load BPF maps

  The tests are not yet run in our CI as the CirrusCI containers lack the required permissions (see https://github.com/bitcoin/bitcoin/issues/23296#issuecomment-1024920845). Running the tests in a VM in the CI could work, but I haven't experimented with this yet. The priority was to get the actual tests done first to ensure the tracepoints work as intended for the v23.0 release. Running the tracepoint tests in the CI is planned as the next step to finish #23296.

  The tests can, however, be run against e.g. release candidates by hand. Additionally, they provide a starting point for tests for future tracepoints. PRs adding new tracepoint should include tests. This makes reviewing these PRs easier.

  The tests require privileges to execute BPF sycalls (`CAP_SYS_ADMIN` before Linux kernel 5.8 and `CAP_BPF` and `CAP_PERFMON` on 5.8+) and permissions to `/sys/kernel/debug/tracing/`. It's currently recommended to run the tests in a virtual machine (or on a VPS) where it's sensible to use the `root` user to gain these privileges. Never run python scripts you haven't carefully reviewed with `root` permissions! It's unclear if a non-root user can even gain the required privileges. This needs more experimenting.

  The goal here is to test the tracepoint interface to make sure the [documented interface](https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#tracepoint-documentation) does not break by accident. The tracepoints expose implementation details. This means we also need to rely on implementation details of Bitcoin Core in these functional tests to trigger the tracepoints. An example is the test of the `utxocache:flush` tracepoint: On Bitcoin Core shutdown, the UTXO cache is flushed twice. The corresponding tracepoint test expects two flushes, too - if not, the test fails. Changing implementation details could cause these tests to fail and the tracepoint API to break. However, we purposefully treat the tracepoints only as [**semi-stable**](https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#semi-stable-api). The tracepoints should not block refactors or changes to other internals.

ACKs for top commit:
  jb55:
    tACK 76c60d7b31ccc50b226cdbc5e38be0bd67603408
  laanwj:
    Tested ACK 76c60d7b31ccc50b226cdbc5e38be0bd67603408

Tree-SHA512: 9a63d945c68102e59d751bd8d2805ddd7b37185408fa831d28a9cb6641b701961389b55f216c475df7d4771154e735625067ee957fc74f454ad7a7921255364c

5 files changed, 745 insertions, 0 deletions

diff --git a/test/functional/interface_usdt_net.py b/test/functional/interface_usdt_net.py
new file mode 100755
index 0000000000..9522cd8c59
--- /dev/null
+++ b/test/functional/interface_usdt_net.py
@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+"""  Tests the net:* tracepoint API interface.
+     See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#context-net
+"""
+
+import ctypes
+from io import BytesIO
+# Test will be skipped if we don't have bcc installed
+try:
+    from bcc import BPF, USDT # type: ignore[import]
+except ImportError:
+    pass
+from test_framework.messages import msg_version
+from test_framework.p2p import P2PInterface
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal
+
+# Tor v3 addresses are 62 chars + 6 chars for the port (':12345').
+MAX_PEER_ADDR_LENGTH = 68
+MAX_PEER_CONN_TYPE_LENGTH = 20
+MAX_MSG_TYPE_LENGTH = 20
+# We won't process messages larger than 150 byte in this test. For reading
+# larger messanges see contrib/tracing/log_raw_p2p_msgs.py
+MAX_MSG_DATA_LENGTH = 150
+
+net_tracepoints_program = """
+#include <uapi/linux/ptrace.h>
+
+#define MAX_PEER_ADDR_LENGTH {}
+#define MAX_PEER_CONN_TYPE_LENGTH {}
+#define MAX_MSG_TYPE_LENGTH {}
+#define MAX_MSG_DATA_LENGTH {}
+""".format(
+    MAX_PEER_ADDR_LENGTH,
+    MAX_PEER_CONN_TYPE_LENGTH,
+    MAX_MSG_TYPE_LENGTH,
+    MAX_MSG_DATA_LENGTH
+) + """
+#define MIN(a,b) ({ __typeof__ (a) _a = (a); __typeof__ (b) _b = (b); _a < _b ? _a : _b; })
+
+struct p2p_message
+{
+    u64     peer_id;
+    char    peer_addr[MAX_PEER_ADDR_LENGTH];
+    char    peer_conn_type[MAX_PEER_CONN_TYPE_LENGTH];
+    char    msg_type[MAX_MSG_TYPE_LENGTH];
+    u64     msg_size;
+    u8      msg[MAX_MSG_DATA_LENGTH];
+};
+
+BPF_PERF_OUTPUT(inbound_messages);
+int trace_inbound_message(struct pt_regs *ctx) {
+    struct p2p_message msg = {};
+    bpf_usdt_readarg(1, ctx, &msg.peer_id);
+    bpf_usdt_readarg_p(2, ctx, &msg.peer_addr, MAX_PEER_ADDR_LENGTH);
+    bpf_usdt_readarg_p(3, ctx, &msg.peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH);
+    bpf_usdt_readarg_p(4, ctx, &msg.msg_type, MAX_MSG_TYPE_LENGTH);
+    bpf_usdt_readarg(5, ctx, &msg.msg_size);
+    bpf_usdt_readarg_p(6, ctx, &msg.msg, MIN(msg.msg_size, MAX_MSG_DATA_LENGTH));
+    inbound_messages.perf_submit(ctx, &msg, sizeof(msg));
+    return 0;
+}
+
+BPF_PERF_OUTPUT(outbound_messages);
+int trace_outbound_message(struct pt_regs *ctx) {
+    struct p2p_message msg = {};
+    bpf_usdt_readarg(1, ctx, &msg.peer_id);
+    bpf_usdt_readarg_p(2, ctx, &msg.peer_addr, MAX_PEER_ADDR_LENGTH);
+    bpf_usdt_readarg_p(3, ctx, &msg.peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH);
+    bpf_usdt_readarg_p(4, ctx, &msg.msg_type, MAX_MSG_TYPE_LENGTH);
+    bpf_usdt_readarg(5, ctx, &msg.msg_size);
+    bpf_usdt_readarg_p(6, ctx, &msg.msg, MIN(msg.msg_size, MAX_MSG_DATA_LENGTH));
+    outbound_messages.perf_submit(ctx, &msg, sizeof(msg));
+    return 0;
+};
+"""
+
+
+class NetTracepointTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 1
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_platform_not_linux()
+        self.skip_if_no_bitcoind_tracepoints()
+        self.skip_if_no_python_bcc()
+        self.skip_if_no_bpf_permissions()
+
+    def run_test(self):
+        # Tests the net:inbound_message and net:outbound_message tracepoints
+        # See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#context-net
+
+        class P2PMessage(ctypes.Structure):
+            _fields_ = [
+                ("peer_id", ctypes.c_uint64),
+                ("peer_addr", ctypes.c_char * MAX_PEER_ADDR_LENGTH),
+                ("peer_conn_type", ctypes.c_char * MAX_PEER_CONN_TYPE_LENGTH),
+                ("msg_type", ctypes.c_char * MAX_MSG_TYPE_LENGTH),
+                ("msg_size", ctypes.c_uint64),
+                ("msg", ctypes.c_ubyte * MAX_MSG_DATA_LENGTH),
+            ]
+
+            def __repr__(self):
+                return f"P2PMessage(peer={self.peer_id}, addr={self.peer_addr.decode('utf-8')}, conn_type={self.peer_conn_type.decode('utf-8')}, msg_type={self.msg_type.decode('utf-8')}, msg_size={self.msg_size})"
+
+        self.log.info(
+            "hook into the net:inbound_message and net:outbound_message tracepoints")
+        ctx = USDT(path=str(self.options.bitcoind))
+        ctx.enable_probe(probe="net:inbound_message",
+                         fn_name="trace_inbound_message")
+        ctx.enable_probe(probe="net:outbound_message",
+                         fn_name="trace_outbound_message")
+        bpf = BPF(text=net_tracepoints_program, usdt_contexts=[ctx], debug=0)
+
+        # The handle_* function is a ctypes callback function called from C. When
+        # we assert in the handle_* function, the AssertError doesn't propagate
+        # back to Python. The exception is ignored. We manually count and assert
+        # that the handle_* functions succeeded.
+        EXPECTED_INOUTBOUND_VERSION_MSG = 1
+        checked_inbound_version_msg = 0
+        checked_outbound_version_msg = 0
+
+        def check_p2p_message(event, inbound):
+            nonlocal checked_inbound_version_msg, checked_outbound_version_msg
+            if event.msg_type.decode("utf-8") == "version":
+                self.log.info(
+                    f"check_p2p_message(): {'inbound' if inbound else 'outbound'} {event}")
+                peer = self.nodes[0].getpeerinfo()[0]
+                msg = msg_version()
+                msg.deserialize(BytesIO(bytes(event.msg[:event.msg_size])))
+                assert_equal(peer["id"], event.peer_id, peer["id"])
+                assert_equal(peer["addr"], event.peer_addr.decode("utf-8"))
+                assert_equal(peer["connection_type"],
+                             event.peer_conn_type.decode("utf-8"))
+                if inbound:
+                    checked_inbound_version_msg += 1
+                else:
+                    checked_outbound_version_msg += 1
+
+        def handle_inbound(_, data, __):
+            event = ctypes.cast(data, ctypes.POINTER(P2PMessage)).contents
+            check_p2p_message(event, True)
+
+        def handle_outbound(_, data, __):
+            event = ctypes.cast(data, ctypes.POINTER(P2PMessage)).contents
+            check_p2p_message(event, False)
+
+        bpf["inbound_messages"].open_perf_buffer(handle_inbound)
+        bpf["outbound_messages"].open_perf_buffer(handle_outbound)
+
+        self.log.info("connect a P2P test node to our bitcoind node")
+        test_node = P2PInterface()
+        self.nodes[0].add_p2p_connection(test_node)
+        bpf.perf_buffer_poll(timeout=200)
+
+        self.log.info(
+            "check that we got both an inbound and outbound version message")
+        assert_equal(EXPECTED_INOUTBOUND_VERSION_MSG,
+                     checked_inbound_version_msg)
+        assert_equal(EXPECTED_INOUTBOUND_VERSION_MSG,
+                     checked_outbound_version_msg)
+
+        bpf.cleanup()
+
+
+if __name__ == '__main__':
+    NetTracepointTest().main()
diff --git a/test/functional/interface_usdt_utxocache.py b/test/functional/interface_usdt_utxocache.py
new file mode 100755
index 0000000000..0c7f351e66
--- /dev/null
+++ b/test/functional/interface_usdt_utxocache.py
@@ -0,0 +1,407 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+""" Tests the utxocache:* tracepoint API interface.
+    See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#context-utxocache
+"""
+
+import ctypes
+# Test will be skipped if we don't have bcc installed
+try:
+    from bcc import BPF, USDT # type: ignore[import]
+except ImportError:
+    pass
+from test_framework.messages import COIN
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal
+from test_framework.wallet import MiniWallet
+
+utxocache_changes_program = """
+#include <uapi/linux/ptrace.h>
+
+typedef signed long long i64;
+
+struct utxocache_change
+{
+    char        txid[32];
+    u32         index;
+    u32         height;
+    i64         value;
+    bool        is_coinbase;
+};
+
+BPF_PERF_OUTPUT(utxocache_add);
+int trace_utxocache_add(struct pt_regs *ctx) {
+    struct utxocache_change add = {};
+    bpf_usdt_readarg_p(1, ctx, &add.txid, 32);
+    bpf_usdt_readarg(2, ctx, &add.index);
+    bpf_usdt_readarg(3, ctx, &add.height);
+    bpf_usdt_readarg(4, ctx, &add.value);
+    bpf_usdt_readarg(5, ctx, &add.is_coinbase);
+    utxocache_add.perf_submit(ctx, &add, sizeof(add));
+    return 0;
+}
+
+BPF_PERF_OUTPUT(utxocache_spent);
+int trace_utxocache_spent(struct pt_regs *ctx) {
+    struct utxocache_change spent = {};
+    bpf_usdt_readarg_p(1, ctx, &spent.txid, 32);
+    bpf_usdt_readarg(2, ctx, &spent.index);
+    bpf_usdt_readarg(3, ctx, &spent.height);
+    bpf_usdt_readarg(4, ctx, &spent.value);
+    bpf_usdt_readarg(5, ctx, &spent.is_coinbase);
+    utxocache_spent.perf_submit(ctx, &spent, sizeof(spent));
+    return 0;
+}
+
+BPF_PERF_OUTPUT(utxocache_uncache);
+int trace_utxocache_uncache(struct pt_regs *ctx) {
+    struct utxocache_change uncache = {};
+    bpf_usdt_readarg_p(1, ctx, &uncache.txid, 32);
+    bpf_usdt_readarg(2, ctx, &uncache.index);
+    bpf_usdt_readarg(3, ctx, &uncache.height);
+    bpf_usdt_readarg(4, ctx, &uncache.value);
+    bpf_usdt_readarg(5, ctx, &uncache.is_coinbase);
+    utxocache_uncache.perf_submit(ctx, &uncache, sizeof(uncache));
+    return 0;
+}
+"""
+
+utxocache_flushes_program = """
+#include <uapi/linux/ptrace.h>
+
+typedef signed long long i64;
+
+struct utxocache_flush
+{
+    i64         duration;
+    u32         mode;
+    u64         size;
+    u64         memory;
+    bool        for_prune;
+};
+
+BPF_PERF_OUTPUT(utxocache_flush);
+int trace_utxocache_flush(struct pt_regs *ctx) {
+    struct utxocache_flush flush = {};
+    bpf_usdt_readarg(1, ctx, &flush.duration);
+    bpf_usdt_readarg(2, ctx, &flush.mode);
+    bpf_usdt_readarg(3, ctx, &flush.size);
+    bpf_usdt_readarg(4, ctx, &flush.memory);
+    bpf_usdt_readarg(5, ctx, &flush.for_prune);
+    utxocache_flush.perf_submit(ctx, &flush, sizeof(flush));
+    return 0;
+}
+"""
+
+FLUSHMODE_NAME = {
+    0: "NONE",
+    1: "IF_NEEDED",
+    2: "PERIODIC",
+    3: "ALWAYS",
+}
+
+
+class UTXOCacheChange(ctypes.Structure):
+    _fields_ = [
+        ("txid", ctypes.c_ubyte * 32),
+        ("index", ctypes.c_uint32),
+        ("height", ctypes.c_uint32),
+        ("value", ctypes.c_uint64),
+        ("is_coinbase", ctypes.c_bool),
+    ]
+
+    def __repr__(self):
+        return f"UTXOCacheChange(outpoint={bytes(self.txid[::-1]).hex()}:{self.index}, height={self.height}, value={self.value}sat, is_coinbase={self.is_coinbase})"
+
+
+class UTXOCacheFlush(ctypes.Structure):
+    _fields_ = [
+        ("duration", ctypes.c_int64),
+        ("mode", ctypes.c_uint32),
+        ("size", ctypes.c_uint64),
+        ("memory", ctypes.c_uint64),
+        ("for_prune", ctypes.c_bool),
+    ]
+
+    def __repr__(self):
+        return f"UTXOCacheFlush(duration={self.duration}, mode={FLUSHMODE_NAME[self.mode]}, size={self.size}, memory={self.memory}, for_prune={self.for_prune})"
+
+
+class UTXOCacheTracepointTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.setup_clean_chain = False
+        self.num_nodes = 1
+        self.extra_args = [["-txindex"]]
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_platform_not_linux()
+        self.skip_if_no_bitcoind_tracepoints()
+        self.skip_if_no_python_bcc()
+        self.skip_if_no_bpf_permissions()
+
+    def run_test(self):
+        self.wallet = MiniWallet(self.nodes[0])
+        self.generate(self.wallet, 101)
+
+        self.test_uncache()
+        self.test_add_spent()
+        self.test_flush()
+
+    def test_uncache(self):
+        """ Tests the utxocache:uncache tracepoint API.
+        https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#tracepoint-utxocacheuncache
+        """
+        # To trigger an UTXO uncache from the cache, we create an invalid transaction
+        # spending a not-cached, but existing UTXO. During transaction validation, this
+        # the UTXO is added to the utxo cache, but as the transaction is invalid, it's
+        # uncached again.
+        self.log.info("testing the utxocache:uncache tracepoint API")
+
+        # Retrieve the txid for the UTXO created in the first block. This UTXO is not
+        # in our UTXO cache.
+        EARLY_BLOCK_HEIGHT = 1
+        block_1_hash = self.nodes[0].getblockhash(EARLY_BLOCK_HEIGHT)
+        block_1 = self.nodes[0].getblock(block_1_hash)
+        block_1_coinbase_txid = block_1["tx"][0]
+
+        # Create a transaction and invalidate it by changing the txid of the previous
+        # output to the coinbase txid of the block at height 1.
+        invalid_tx = self.wallet.create_self_transfer(
+            from_node=self.nodes[0])["tx"]
+        invalid_tx.vin[0].prevout.hash = int(block_1_coinbase_txid, 16)
+
+        self.log.info("hooking into the utxocache:uncache tracepoint")
+        ctx = USDT(path=str(self.options.bitcoind))
+        ctx.enable_probe(probe="utxocache:uncache",
+                         fn_name="trace_utxocache_uncache")
+        bpf = BPF(text=utxocache_changes_program, usdt_contexts=[ctx], debug=0)
+
+        # The handle_* function is a ctypes callback function called from C. When
+        # we assert in the handle_* function, the AssertError doesn't propagate
+        # back to Python. The exception is ignored. We manually count and assert
+        # that the handle_* functions succeeded.
+        EXPECTED_HANDLE_UNCACHE_SUCCESS = 1
+        handle_uncache_succeeds = 0
+
+        def handle_utxocache_uncache(_, data, __):
+            nonlocal handle_uncache_succeeds
+            event = ctypes.cast(data, ctypes.POINTER(UTXOCacheChange)).contents
+            self.log.info(f"handle_utxocache_uncache(): {event}")
+            assert_equal(block_1_coinbase_txid, bytes(event.txid[::-1]).hex())
+            assert_equal(0, event.index)  # prevout index
+            assert_equal(EARLY_BLOCK_HEIGHT, event.height)
+            assert_equal(50 * COIN, event.value)
+            assert_equal(True, event.is_coinbase)
+
+            handle_uncache_succeeds += 1
+
+        bpf["utxocache_uncache"].open_perf_buffer(handle_utxocache_uncache)
+
+        self.log.info(
+            "testmempoolaccept the invalid transaction to trigger an UTXO-cache uncache")
+        result = self.nodes[0].testmempoolaccept(
+            [invalid_tx.serialize().hex()])[0]
+        assert_equal(result["allowed"], False)
+
+        bpf.perf_buffer_poll(timeout=100)
+        bpf.cleanup()
+
+        self.log.info(
+            f"check that we successfully traced {EXPECTED_HANDLE_UNCACHE_SUCCESS} uncaches")
+        assert_equal(EXPECTED_HANDLE_UNCACHE_SUCCESS, handle_uncache_succeeds)
+
+    def test_add_spent(self):
+        """ Tests the utxocache:add utxocache:spent tracepoint API
+            See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#tracepoint-utxocacheadd
+            and https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#tracepoint-utxocachespent
+        """
+
+        self.log.info(
+            "test the utxocache:add and utxocache:spent tracepoint API")
+
+        self.log.info("create an unconfirmed transaction")
+        self.wallet.send_self_transfer(from_node=self.nodes[0])
+
+        # We mine a block to trace changes (add/spent) to the active in-memory cache
+        # of the UTXO set (see CoinsTip() of CCoinsViewCache). However, in some cases
+        # temporary clones of the active cache are made. For example, during mining with
+        # the generate RPC call, the block is first tested in TestBlockValidity(). There,
+        # a clone of the active cache is modified during a test ConnectBlock() call.
+        # These are implementation details we don't want to test here. Thus, after
+        # mining, we invalidate the block, start the tracing, and then trace the cache
+        # changes to the active utxo cache.
+        self.log.info("mine and invalidate a block that is later reconsidered")
+        block_hash = self.generate(self.wallet, 1)[0]
+        self.nodes[0].invalidateblock(block_hash)
+
+        self.log.info(
+            "hook into the utxocache:add and utxocache:spent tracepoints")
+        ctx = USDT(path=str(self.options.bitcoind))
+        ctx.enable_probe(probe="utxocache:add", fn_name="trace_utxocache_add")
+        ctx.enable_probe(probe="utxocache:spent",
+                         fn_name="trace_utxocache_spent")
+        bpf = BPF(text=utxocache_changes_program, usdt_contexts=[ctx], debug=0)
+
+        # The handle_* function is a ctypes callback function called from C. When
+        # we assert in the handle_* function, the AssertError doesn't propagate
+        # back to Python. The exception is ignored. We manually count and assert
+        # that the handle_* functions succeeded.
+        EXPECTED_HANDLE_ADD_SUCCESS = 2
+        EXPECTED_HANDLE_SPENT_SUCCESS = 1
+        handle_add_succeeds = 0
+        handle_spent_succeeds = 0
+
+        expected_utxocache_spents = []
+        expected_utxocache_adds = []
+
+        def handle_utxocache_add(_, data, __):
+            nonlocal handle_add_succeeds
+            event = ctypes.cast(data, ctypes.POINTER(UTXOCacheChange)).contents
+            self.log.info(f"handle_utxocache_add(): {event}")
+            add = expected_utxocache_adds.pop(0)
+            assert_equal(add["txid"], bytes(event.txid[::-1]).hex())
+            assert_equal(add["index"], event.index)
+            assert_equal(add["height"], event.height)
+            assert_equal(add["value"], event.value)
+            assert_equal(add["is_coinbase"], event.is_coinbase)
+            handle_add_succeeds += 1
+
+        def handle_utxocache_spent(_, data, __):
+            nonlocal handle_spent_succeeds
+            event = ctypes.cast(data, ctypes.POINTER(UTXOCacheChange)).contents
+            self.log.info(f"handle_utxocache_spent(): {event}")
+            spent = expected_utxocache_spents.pop(0)
+            assert_equal(spent["txid"], bytes(event.txid[::-1]).hex())
+            assert_equal(spent["index"], event.index)
+            assert_equal(spent["height"], event.height)
+            assert_equal(spent["value"], event.value)
+            assert_equal(spent["is_coinbase"], event.is_coinbase)
+            handle_spent_succeeds += 1
+
+        bpf["utxocache_add"].open_perf_buffer(handle_utxocache_add)
+        bpf["utxocache_spent"].open_perf_buffer(handle_utxocache_spent)
+
+        # We trigger a block re-connection. This causes changes (add/spent)
+        # to the UTXO-cache which in turn triggers the tracepoints.
+        self.log.info("reconsider the previously invalidated block")
+        self.nodes[0].reconsiderblock(block_hash)
+
+        block = self.nodes[0].getblock(block_hash, 2)
+        for (block_index, tx) in enumerate(block["tx"]):
+            for vin in tx["vin"]:
+                if "coinbase" not in vin:
+                    prevout_tx = self.nodes[0].getrawtransaction(
+                        vin["txid"], True)
+                    prevout_tx_block = self.nodes[0].getblockheader(
+                        prevout_tx["blockhash"])
+                    spends_coinbase = "coinbase" in prevout_tx["vin"][0]
+                    expected_utxocache_spents.append({
+                        "txid": vin["txid"],
+                        "index": vin["vout"],
+                        "height": prevout_tx_block["height"],
+                        "value": int(prevout_tx["vout"][vin["vout"]]["value"] * COIN),
+                        "is_coinbase": spends_coinbase,
+                    })
+            for (i, vout) in enumerate(tx["vout"]):
+                if vout["scriptPubKey"]["type"] != "nulldata":
+                    expected_utxocache_adds.append({
+                        "txid": tx["txid"],
+                        "index": i,
+                        "height": block["height"],
+                        "value": int(vout["value"] * COIN),
+                        "is_coinbase": block_index == 0,
+                    })
+
+        assert_equal(EXPECTED_HANDLE_ADD_SUCCESS, len(expected_utxocache_adds))
+        assert_equal(EXPECTED_HANDLE_SPENT_SUCCESS,
+                     len(expected_utxocache_spents))
+
+        bpf.perf_buffer_poll(timeout=200)
+        bpf.cleanup()
+
+        self.log.info(
+            f"check that we successfully traced {EXPECTED_HANDLE_ADD_SUCCESS} adds and {EXPECTED_HANDLE_SPENT_SUCCESS} spent")
+        assert_equal(0, len(expected_utxocache_adds))
+        assert_equal(0, len(expected_utxocache_spents))
+        assert_equal(EXPECTED_HANDLE_ADD_SUCCESS, handle_add_succeeds)
+        assert_equal(EXPECTED_HANDLE_SPENT_SUCCESS, handle_spent_succeeds)
+
+    def test_flush(self):
+        """ Tests the utxocache:flush tracepoint API.
+            See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#tracepoint-utxocacheflush"""
+
+        self.log.info("test the utxocache:flush tracepoint API")
+        self.log.info("hook into the utxocache:flush tracepoint")
+        ctx = USDT(path=str(self.options.bitcoind))
+        ctx.enable_probe(probe="utxocache:flush",
+                         fn_name="trace_utxocache_flush")
+        bpf = BPF(text=utxocache_flushes_program, usdt_contexts=[ctx], debug=0)
+
+        # The handle_* function is a ctypes callback function called from C. When
+        # we assert in the handle_* function, the AssertError doesn't propagate
+        # back to Python. The exception is ignored. We manually count and assert
+        # that the handle_* functions succeeded.
+        EXPECTED_HANDLE_FLUSH_SUCCESS = 3
+        handle_flush_succeeds = 0
+        possible_cache_sizes = set()
+        expected_flushes = []
+
+        def handle_utxocache_flush(_, data, __):
+            nonlocal handle_flush_succeeds
+            event = ctypes.cast(data, ctypes.POINTER(UTXOCacheFlush)).contents
+            self.log.info(f"handle_utxocache_flush(): {event}")
+            expected = expected_flushes.pop(0)
+            assert_equal(expected["mode"], FLUSHMODE_NAME[event.mode])
+            possible_cache_sizes.remove(event.size)  # fails if size not in set
+            # sanity checks only
+            assert(event.memory > 0)
+            assert(event.duration > 0)
+            handle_flush_succeeds += 1
+
+        bpf["utxocache_flush"].open_perf_buffer(handle_utxocache_flush)
+
+        self.log.info("stop the node to flush the UTXO cache")
+        UTXOS_IN_CACHE = 104  # might need to be changed if the eariler tests are modified
+        # A node shutdown causes two flushes. One that flushes UTXOS_IN_CACHE
+        # UTXOs and one that flushes 0 UTXOs. Normally the 0-UTXO-flush is the
+        # second flush, however it can happen that the order changes.
+        possible_cache_sizes = {UTXOS_IN_CACHE, 0}
+        flush_for_shutdown = {"mode": "ALWAYS", "for_prune": False}
+        expected_flushes.extend([flush_for_shutdown, flush_for_shutdown])
+        self.stop_node(0)
+
+        bpf.perf_buffer_poll(timeout=200)
+
+        self.log.info("check that we don't expect additional flushes")
+        assert_equal(0, len(expected_flushes))
+        assert_equal(0, len(possible_cache_sizes))
+
+        self.log.info("restart the node with -prune")
+        self.start_node(0, ["-fastprune=1", "-prune=1"])
+
+        BLOCKS_TO_MINE = 350
+        self.log.info(f"mine {BLOCKS_TO_MINE} blocks to be able to prune")
+        self.generate(self.wallet, BLOCKS_TO_MINE)
+        # we added BLOCKS_TO_MINE coinbase UTXOs to the cache
+        possible_cache_sizes = {BLOCKS_TO_MINE}
+        expected_flushes.append(
+            {"mode": "NONE", "for_prune": True, "size_fn": lambda x: x == BLOCKS_TO_MINE})
+
+        self.log.info(f"prune blockchain to trigger a flush for pruning")
+        self.nodes[0].pruneblockchain(315)
+
+        bpf.perf_buffer_poll(timeout=500)
+        bpf.cleanup()
+
+        self.log.info(
+            f"check that we don't expect additional flushes and that the handle_* function succeeded")
+        assert_equal(0, len(expected_flushes))
+        assert_equal(0, len(possible_cache_sizes))
+        assert_equal(EXPECTED_HANDLE_FLUSH_SUCCESS, handle_flush_succeeds)
+
+
+if __name__ == '__main__':
+    UTXOCacheTracepointTest().main()
diff --git a/test/functional/interface_usdt_validation.py b/test/functional/interface_usdt_validation.py
new file mode 100755
index 0000000000..d11809273b
--- /dev/null
+++ b/test/functional/interface_usdt_validation.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+""" Tests the validation:* tracepoint API interface.
+    See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#context-validation
+"""
+
+import ctypes
+
+# Test will be skipped if we don't have bcc installed
+try:
+    from bcc import BPF, USDT # type: ignore[import]
+except ImportError:
+    pass
+
+from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal
+
+
+validation_blockconnected_program = """
+#include <uapi/linux/ptrace.h>
+
+typedef signed long long i64;
+
+struct connected_block
+{
+    char        hash[32];
+    int         height;
+    i64         transactions;
+    int         inputs;
+    i64         sigops;
+    u64         duration;
+};
+
+BPF_PERF_OUTPUT(block_connected);
+int trace_block_connected(struct pt_regs *ctx) {
+    struct connected_block block = {};
+    bpf_usdt_readarg_p(1, ctx, &block.hash, 32);
+    bpf_usdt_readarg(2, ctx, &block.height);
+    bpf_usdt_readarg(3, ctx, &block.transactions);
+    bpf_usdt_readarg(4, ctx, &block.inputs);
+    bpf_usdt_readarg(5, ctx, &block.sigops);
+    bpf_usdt_readarg(6, ctx, &block.duration);
+    block_connected.perf_submit(ctx, &block, sizeof(block));
+    return 0;
+}
+"""
+
+
+class ValidationTracepointTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 1
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_platform_not_linux()
+        self.skip_if_no_bitcoind_tracepoints()
+        self.skip_if_no_python_bcc()
+        self.skip_if_no_bpf_permissions()
+
+    def run_test(self):
+        # Tests the validation:block_connected tracepoint by generating blocks
+        # and comparing the values passed in the tracepoint arguments with the
+        # blocks.
+        # See https://github.com/bitcoin/bitcoin/blob/master/doc/tracing.md#tracepoint-validationblock_connected
+
+        class Block(ctypes.Structure):
+            _fields_ = [
+                ("hash", ctypes.c_ubyte * 32),
+                ("height", ctypes.c_int),
+                ("transactions", ctypes.c_int64),
+                ("inputs", ctypes.c_int),
+                ("sigops", ctypes.c_int64),
+                ("duration", ctypes.c_uint64),
+            ]
+
+            def __repr__(self):
+                return "ConnectedBlock(hash=%s height=%d, transactions=%d, inputs=%d, sigops=%d, duration=%d)" % (
+                    bytes(self.hash[::-1]).hex(),
+                    self.height,
+                    self.transactions,
+                    self.inputs,
+                    self.sigops,
+                    self.duration)
+
+        # The handle_* function is a ctypes callback function called from C. When
+        # we assert in the handle_* function, the AssertError doesn't propagate
+        # back to Python. The exception is ignored. We manually count and assert
+        # that the handle_* functions succeeded.
+        BLOCKS_EXPECTED = 2
+        blocks_checked = 0
+        expected_blocks = list()
+
+        self.log.info("hook into the validation:block_connected tracepoint")
+        ctx = USDT(path=str(self.options.bitcoind))
+        ctx.enable_probe(probe="validation:block_connected",
+                         fn_name="trace_block_connected")
+        bpf = BPF(text=validation_blockconnected_program,
+                  usdt_contexts=[ctx], debug=0)
+
+        def handle_blockconnected(_, data, __):
+            nonlocal expected_blocks, blocks_checked
+            event = ctypes.cast(data, ctypes.POINTER(Block)).contents
+            self.log.info(f"handle_blockconnected(): {event}")
+            block = expected_blocks.pop(0)
+            assert_equal(block["hash"], bytes(event.hash[::-1]).hex())
+            assert_equal(block["height"], event.height)
+            assert_equal(len(block["tx"]), event.transactions)
+            assert_equal(len([tx["vin"] for tx in block["tx"]]), event.inputs)
+            assert_equal(0, event.sigops)  # no sigops in coinbase tx
+            # only plausibility checks
+            assert(event.duration > 0)
+
+            blocks_checked += 1
+
+        bpf["block_connected"].open_perf_buffer(
+            handle_blockconnected)
+
+        self.log.info(f"mine {BLOCKS_EXPECTED} blocks")
+        block_hashes = self.generatetoaddress(
+            self.nodes[0], BLOCKS_EXPECTED, ADDRESS_BCRT1_UNSPENDABLE)
+        for block_hash in block_hashes:
+            expected_blocks.append(self.nodes[0].getblock(block_hash, 2))
+
+        bpf.perf_buffer_poll(timeout=200)
+        bpf.cleanup()
+
+        self.log.info(f"check that we traced {BLOCKS_EXPECTED} blocks")
+        assert_equal(BLOCKS_EXPECTED, blocks_checked)
+        assert_equal(0, len(expected_blocks))
+
+
+if __name__ == '__main__':
+    ValidationTracepointTest().main()
diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py
index ecdc3bf424..2fb9ec0942 100755
--- a/test/functional/test_framework/test_framework.py
+++ b/test/functional/test_framework/test_framework.py
@@ -9,6 +9,7 @@ from enum import Enum
 import argparse
 import logging
 import os
+import platform
 import pdb
 import random
 import re
@@ -826,6 +827,29 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
         except ImportError:
             raise SkipTest("python3-zmq module not available.")
 
+    def skip_if_no_python_bcc(self):
+        """Attempt to import the bcc package and skip the tests if the import fails."""
+        try:
+            import bcc  # type: ignore[import] # noqa: F401
+        except ImportError:
+            raise SkipTest("bcc python module not available")
+
+    def skip_if_no_bitcoind_tracepoints(self):
+        """Skip the running test if bitcoind has not been compiled with USDT tracepoint support."""
+        if not self.is_usdt_compiled():
+            raise SkipTest("bitcoind has not been built with USDT tracepoints enabled.")
+
+    def skip_if_no_bpf_permissions(self):
+        """Skip the running test if we don't have permissions to do BPF syscalls and load BPF maps."""
+        # check for 'root' permissions
+        if os.geteuid() != 0:
+            raise SkipTest("no permissions to use BPF (please review the tests carefully before running them with higher privileges)")
+
+    def skip_if_platform_not_linux(self):
+        """Skip the running test if we are not on a Linux platform"""
+        if platform.system() != "Linux":
+            raise SkipTest("not on a Linux system")
+
     def skip_if_no_bitcoind_zmq(self):
         """Skip the running test if bitcoind has not been compiled with zmq support."""
         if not self.is_zmq_compiled():
@@ -907,6 +931,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
         """Checks whether the zmq module was compiled."""
         return self.config["components"].getboolean("ENABLE_ZMQ")
 
+    def is_usdt_compiled(self):
+        """Checks whether the USDT tracepoints were compiled."""
+        return self.config["components"].getboolean("ENABLE_USDT_TRACEPOINTS")
+
     def is_sqlite_compiled(self):
         """Checks whether the wallet module was compiled with Sqlite support."""
         return self.config["components"].getboolean("USE_SQLITE")
diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py
index 381d2b9426..1f0f806d91 100755
--- a/test/functional/test_runner.py
+++ b/test/functional/test_runner.py
@@ -168,6 +168,9 @@ BASE_SCRIPTS = [
     'wallet_reorgsrestore.py',
     'interface_http.py',
     'interface_rpc.py',
+    'interface_usdt_net.py',
+    'interface_usdt_utxocache.py',
+    'interface_usdt_validation.py',
     'rpc_psbt.py --legacy-wallet',
     'rpc_psbt.py --descriptors',
     'rpc_users.py',