Merge bitcoin/bitcoin#25528: ci: run USDT interface tests in the CI

cc7335edc87c6ef34429b4df94f53973db520aac ci: run USDT interface test in a VM (0xb10c)
dba6f8234217565957e37516a0ea655f1180d99c test: adopt USDT utxocache interface tests (0xb10c)
220a5a2841172a07d6d7849596316f0e0933e272 test: hook into PID in tracing tests (0xb10c)

Pull request description:

  Changes a CI task that runs test the previously not run `test/functional/interface_usdt_*.py` functional tests (added in https://github.com/bitcoin/bitcoin/pull/24358).

  This task is run as CirussCI `compute_engine_instance` VM as hooking into the tracepoints is not possible in CirrusCI docker containers (https://github.com/bitcoin/bitcoin/issues/23296#issuecomment-1024920845). We use an unoffical PPA and untrusted  `bpfcc-tools` package in the CI as the Ubuntu jammy and Debian bullseye packages are outdated. We hope use an official package when new Ubuntu/Debian releases are available for the use with Google Compute Engine.

  We make sure to hook into `bitcoind` binaries in USDT interface tests via their PID, instead of their path. This makes sure multiple functional tests running in parallel don't interfere with each other.

  The utxocache USDT interface tests is adopted to a change of the functional test framework that wasn't detected as the tests weren't run in the CI. As the tracepoints expose internals, it can happen that we need to adopt the interface test when internals change. This is a bit awkward, and if it happens to frequently, we should consider generalizing the tests a bit more. For now it's fine, I think.

  See the individual commit messages for more details on the changes.

  Fixes https://github.com/bitcoin/bitcoin/issues/24782
  Fixes https://github.com/bitcoin/bitcoin/issues/23296

  I'd like to hear from reviewers:
  - Are we OK with using the [`hadret/bpfcc`](https://launchpad.net/~hadret/+archive/ubuntu/bpfcc) PPA for now? There is a clear plan when to drop it and as is currently, it could only impact the newly added VM task.
  - ~~Adding a new task increases CI runtime and costs. Should an existing `container` CI task be ported to a VM and reused instead?~~ Yes, see https://github.com/bitcoin/bitcoin/pull/25528#issuecomment-1179509525

ACKs for top commit:
  MarcoFalke:
    cr ACK cc7335edc87c6ef34429b4df94f53973db520aac

Tree-SHA512: b7fddccc0a77d82371229d048abe0bf2c4ccaa45906497ef3040cf99e7f05561890aef4c253c40e4afc96bb838c9787fae81c8454c6fd9db583276e005a4ccb3

3 files changed, 24 insertions, 20 deletions

diff --git a/test/functional/interface_usdt_net.py b/test/functional/interface_usdt_net.py
index 9522cd8c59..2235da702b 100755
--- a/test/functional/interface_usdt_net.py
+++ b/test/functional/interface_usdt_net.py
@@ -109,7 +109,7 @@ class NetTracepointTest(BitcoinTestFramework):
 
         self.log.info(
             "hook into the net:inbound_message and net:outbound_message tracepoints")
-        ctx = USDT(path=str(self.options.bitcoind))
+        ctx = USDT(pid=self.nodes[0].process.pid)
         ctx.enable_probe(probe="net:inbound_message",
                          fn_name="trace_inbound_message")
         ctx.enable_probe(probe="net:outbound_message",
diff --git a/test/functional/interface_usdt_utxocache.py b/test/functional/interface_usdt_utxocache.py
index f48ff9699d..2280de1479 100755
--- a/test/functional/interface_usdt_utxocache.py
+++ b/test/functional/interface_usdt_utxocache.py
@@ -173,7 +173,7 @@ class UTXOCacheTracepointTest(BitcoinTestFramework):
         invalid_tx.vin[0].prevout.hash = int(block_1_coinbase_txid, 16)
 
         self.log.info("hooking into the utxocache:uncache tracepoint")
-        ctx = USDT(path=str(self.options.bitcoind))
+        ctx = USDT(pid=self.nodes[0].process.pid)
         ctx.enable_probe(probe="utxocache:uncache",
                          fn_name="trace_utxocache_uncache")
         bpf = BPF(text=utxocache_changes_program, usdt_contexts=[ctx], debug=0)
@@ -238,7 +238,7 @@ class UTXOCacheTracepointTest(BitcoinTestFramework):
 
         self.log.info(
             "hook into the utxocache:add and utxocache:spent tracepoints")
-        ctx = USDT(path=str(self.options.bitcoind))
+        ctx = USDT(pid=self.nodes[0].process.pid)
         ctx.enable_probe(probe="utxocache:add", fn_name="trace_utxocache_add")
         ctx.enable_probe(probe="utxocache:spent",
                          fn_name="trace_utxocache_spent")
@@ -334,7 +334,7 @@ class UTXOCacheTracepointTest(BitcoinTestFramework):
 
         self.log.info("test the utxocache:flush tracepoint API")
         self.log.info("hook into the utxocache:flush tracepoint")
-        ctx = USDT(path=str(self.options.bitcoind))
+        ctx = USDT(pid=self.nodes[0].process.pid)
         ctx.enable_probe(probe="utxocache:flush",
                          fn_name="trace_utxocache_flush")
         bpf = BPF(text=utxocache_flushes_program, usdt_contexts=[ctx], debug=0)
@@ -345,16 +345,17 @@ class UTXOCacheTracepointTest(BitcoinTestFramework):
         # that the handle_* functions succeeded.
         EXPECTED_HANDLE_FLUSH_SUCCESS = 3
         handle_flush_succeeds = 0
-        possible_cache_sizes = set()
-        expected_flushes = []
+        expected_flushes = list()
 
         def handle_utxocache_flush(_, data, __):
             nonlocal handle_flush_succeeds
             event = ctypes.cast(data, ctypes.POINTER(UTXOCacheFlush)).contents
             self.log.info(f"handle_utxocache_flush(): {event}")
-            expected = expected_flushes.pop(0)
-            assert_equal(expected["mode"], FLUSHMODE_NAME[event.mode])
-            possible_cache_sizes.remove(event.size)  # fails if size not in set
+            expected_flushes.remove({
+                "mode": FLUSHMODE_NAME[event.mode],
+                "for_prune": event.for_prune,
+                "size": event.size
+            })
             # sanity checks only
             assert(event.memory > 0)
             assert(event.duration > 0)
@@ -363,20 +364,19 @@ class UTXOCacheTracepointTest(BitcoinTestFramework):
         bpf["utxocache_flush"].open_perf_buffer(handle_utxocache_flush)
 
         self.log.info("stop the node to flush the UTXO cache")
-        UTXOS_IN_CACHE = 104  # might need to be changed if the eariler tests are modified
+        UTXOS_IN_CACHE = 2 # might need to be changed if the eariler tests are modified
         # A node shutdown causes two flushes. One that flushes UTXOS_IN_CACHE
         # UTXOs and one that flushes 0 UTXOs. Normally the 0-UTXO-flush is the
         # second flush, however it can happen that the order changes.
-        possible_cache_sizes = {UTXOS_IN_CACHE, 0}
-        flush_for_shutdown = {"mode": "ALWAYS", "for_prune": False}
-        expected_flushes.extend([flush_for_shutdown, flush_for_shutdown])
+        expected_flushes.append({"mode": "ALWAYS", "for_prune": False, "size": UTXOS_IN_CACHE})
+        expected_flushes.append({"mode": "ALWAYS", "for_prune": False, "size": 0})
         self.stop_node(0)
 
         bpf.perf_buffer_poll(timeout=200)
+        bpf.cleanup()
 
         self.log.info("check that we don't expect additional flushes")
         assert_equal(0, len(expected_flushes))
-        assert_equal(0, len(possible_cache_sizes))
 
         self.log.info("restart the node with -prune")
         self.start_node(0, ["-fastprune=1", "-prune=1"])
@@ -384,12 +384,17 @@ class UTXOCacheTracepointTest(BitcoinTestFramework):
         BLOCKS_TO_MINE = 350
         self.log.info(f"mine {BLOCKS_TO_MINE} blocks to be able to prune")
         self.generate(self.wallet, BLOCKS_TO_MINE)
-        # we added BLOCKS_TO_MINE coinbase UTXOs to the cache
-        possible_cache_sizes = {BLOCKS_TO_MINE}
-        expected_flushes.append(
-            {"mode": "NONE", "for_prune": True, "size_fn": lambda x: x == BLOCKS_TO_MINE})
+
+        self.log.info("test the utxocache:flush tracepoint API with pruning")
+        self.log.info("hook into the utxocache:flush tracepoint")
+        ctx = USDT(pid=self.nodes[0].process.pid)
+        ctx.enable_probe(probe="utxocache:flush",
+                         fn_name="trace_utxocache_flush")
+        bpf = BPF(text=utxocache_flushes_program, usdt_contexts=[ctx], debug=0)
+        bpf["utxocache_flush"].open_perf_buffer(handle_utxocache_flush)
 
         self.log.info(f"prune blockchain to trigger a flush for pruning")
+        expected_flushes.append({"mode": "NONE", "for_prune": True, "size": 0})
         self.nodes[0].pruneblockchain(315)
 
         bpf.perf_buffer_poll(timeout=500)
@@ -398,7 +403,6 @@ class UTXOCacheTracepointTest(BitcoinTestFramework):
         self.log.info(
             f"check that we don't expect additional flushes and that the handle_* function succeeded")
         assert_equal(0, len(expected_flushes))
-        assert_equal(0, len(possible_cache_sizes))
         assert_equal(EXPECTED_HANDLE_FLUSH_SUCCESS, handle_flush_succeeds)
 
 
diff --git a/test/functional/interface_usdt_validation.py b/test/functional/interface_usdt_validation.py
index d11809273b..fb1a91b914 100755
--- a/test/functional/interface_usdt_validation.py
+++ b/test/functional/interface_usdt_validation.py
@@ -94,7 +94,7 @@ class ValidationTracepointTest(BitcoinTestFramework):
         expected_blocks = list()
 
         self.log.info("hook into the validation:block_connected tracepoint")
-        ctx = USDT(path=str(self.options.bitcoind))
+        ctx = USDT(pid=self.nodes[0].process.pid)
         ctx.enable_probe(probe="validation:block_connected",
                          fn_name="trace_block_connected")
         bpf = BPF(text=validation_blockconnected_program,