aboutsummaryrefslogtreecommitdiff
path: root/contrib
diff options
context:
space:
mode:
Diffstat (limited to 'contrib')
-rwxr-xr-xcontrib/devtools/security-check.py18
-rwxr-xr-xcontrib/devtools/symbol-check.py36
-rwxr-xr-xcontrib/devtools/test-security-check.py12
-rwxr-xr-xcontrib/devtools/test-symbol-check.py2
-rwxr-xr-xcontrib/guix/libexec/build.sh5
-rwxr-xr-xcontrib/install_db4.sh8
-rwxr-xr-xcontrib/macdeploy/gen-sdk2
-rw-r--r--contrib/tracing/README.md59
-rwxr-xr-xcontrib/tracing/log_utxocache_flush.py107
-rwxr-xr-xcontrib/tracing/log_utxos.bt86
10 files changed, 302 insertions, 33 deletions
diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py
index ef421aebb1..677557b8fa 100755
--- a/contrib/devtools/security-check.py
+++ b/contrib/devtools/security-check.py
@@ -121,6 +121,21 @@ def check_PE_RELOC_SECTION(binary) -> bool:
'''Check for a reloc section. This is required for functional ASLR.'''
return binary.has_relocations
+def check_PE_control_flow(binary) -> bool:
+ '''
+ Check for control flow instrumentation
+ '''
+ main = binary.get_symbol('main').value
+
+ section_addr = binary.section_from_rva(main).virtual_address
+ virtual_address = binary.optional_header.imagebase + section_addr + main
+
+ content = binary.get_content_from_virtual_address(virtual_address, 4, lief.Binary.VA_TYPES.VA)
+
+ if content == [243, 15, 30, 250]: # endbr64
+ return True
+ return False
+
def check_MACHO_NOUNDEFS(binary) -> bool:
'''
Check for no undefined references.
@@ -177,7 +192,8 @@ CHECKS = {
('DYNAMIC_BASE', check_PE_DYNAMIC_BASE),
('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA),
('NX', check_NX),
- ('RELOC_SECTION', check_PE_RELOC_SECTION)
+ ('RELOC_SECTION', check_PE_RELOC_SECTION),
+ ('CONTROL_FLOW', check_PE_control_flow),
],
'MACHO': [
('PIE', check_PIE),
diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py
index 136a9b70c1..15d4e729ac 100755
--- a/contrib/devtools/symbol-check.py
+++ b/contrib/devtools/symbol-check.py
@@ -19,35 +19,31 @@ import lief #type:ignore
# https://github.com/lief-project/LIEF/pull/562
LIEF_ELF_ARCH_RISCV = lief.ELF.ARCH(243)
-# Debian 8 (Jessie) EOL: 2020. https://wiki.debian.org/DebianReleases#Production_Releases
+# Debian 9 (Stretch) EOL: 2022. https://wiki.debian.org/DebianReleases#Production_Releases
#
-# - g++ version 4.9.2 (https://packages.debian.org/search?suite=jessie&arch=any&searchon=names&keywords=g%2B%2B)
-# - libc version 2.19 (https://packages.debian.org/search?suite=jessie&arch=any&searchon=names&keywords=libc6)
+# - g++ version 6.3.0 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=g%2B%2B)
+# - libc version 2.24 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=libc6)
#
-# Ubuntu 16.04 (Xenial) EOL: 2024. https://wiki.ubuntu.com/Releases
+# Ubuntu 16.04 (Xenial) EOL: 2026. https://wiki.ubuntu.com/Releases
#
-# - g++ version 5.3.1 (https://packages.ubuntu.com/search?keywords=g%2B%2B&searchon=names&suite=xenial&section=all)
-# - libc version 2.23.0 (https://packages.ubuntu.com/search?keywords=libc6&searchon=names&suite=xenial&section=all)
+# - g++ version 5.3.1
+# - libc version 2.23
#
-# CentOS 7 EOL: 2024. https://wiki.centos.org/FAQ/General
+# CentOS Stream 8 EOL: 2024. https://wiki.centos.org/About/Product
#
-# - g++ version 4.8.5 (http://mirror.centos.org/centos/7/os/x86_64/Packages/)
-# - libc version 2.17 (http://mirror.centos.org/centos/7/os/x86_64/Packages/)
-#
-# Taking the minimum of these as our target.
-#
-# According to GNU ABI document (https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html) this corresponds to:
-# GCC 4.8.5: GCC_4.8.0
-# (glibc) GLIBC_2_17
+# - g++ version 8.5.0 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/)
+# - libc version 2.28 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/)
#
+# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info.
+
MAX_VERSIONS = {
'GCC': (4,8,0),
'GLIBC': {
- lief.ELF.ARCH.i386: (2,17),
- lief.ELF.ARCH.x86_64: (2,17),
- lief.ELF.ARCH.ARM: (2,17),
- lief.ELF.ARCH.AARCH64:(2,17),
- lief.ELF.ARCH.PPC64: (2,17),
+ lief.ELF.ARCH.i386: (2,18),
+ lief.ELF.ARCH.x86_64: (2,18),
+ lief.ELF.ARCH.ARM: (2,18),
+ lief.ELF.ARCH.AARCH64:(2,18),
+ lief.ELF.ARCH.PPC64: (2,18),
LIEF_ELF_ARCH_RISCV: (2,27),
},
'LIBATOMIC': (1,0),
diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py
index 0af7cdf5e6..01df863ac0 100755
--- a/contrib/devtools/test-security-check.py
+++ b/contrib/devtools/test-security-check.py
@@ -70,16 +70,18 @@ class TestSecurityChecks(unittest.TestCase):
write_testcode(source)
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']),
- (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION'))
+ (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION CONTROL_FLOW'))
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']),
- (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION'))
+ (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION CONTROL_FLOW'))
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']),
- (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA'))
+ (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW'))
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']),
- (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA')) # -pie -fPIE does nothing unless --dynamicbase is also supplied
+ (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) # -pie -fPIE does nothing unless --dynamicbase is also supplied
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']),
- (1, executable+': failed HIGH_ENTROPY_VA'))
+ (1, executable+': failed HIGH_ENTROPY_VA CONTROL_FLOW'))
self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']),
+ (1, executable+': failed CONTROL_FLOW'))
+ self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE', '-fcf-protection=full']),
(0, ''))
clean_files(source, executable)
diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py
index 5246375fe3..d699e85026 100755
--- a/contrib/devtools/test-symbol-check.py
+++ b/contrib/devtools/test-symbol-check.py
@@ -44,7 +44,7 @@ class TestSymbolChecks(unittest.TestCase):
self.skipTest("test not available for RISC-V")
# nextup was introduced in GLIBC 2.24, so is newer than our supported
- # glibc (2.17), and available in our release build environment (2.24).
+ # glibc (2.18), and available in our release build environment (2.24).
with open(source, 'w', encoding="utf8") as f:
f.write('''
#define _GNU_SOURCE
diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh
index e009f97c60..596d0ca1fb 100755
--- a/contrib/guix/libexec/build.sh
+++ b/contrib/guix/libexec/build.sh
@@ -238,9 +238,6 @@ mkdir -p "$OUTDIR"
# CONFIGFLAGS
CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary"
-case "$HOST" in
- *linux*) CONFIGFLAGS+=" --disable-threadlocal" ;;
-esac
# CFLAGS
HOST_CFLAGS="-O2 -g"
@@ -263,7 +260,7 @@ case "$HOST" in
*mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;;
esac
-# Using --no-tls-get-addr-optimize retains compatibility with glibc 2.17, by
+# Using --no-tls-get-addr-optimize retains compatibility with glibc 2.18, by
# avoiding a PowerPC64 optimisation available in glibc 2.22 and later.
# https://sourceware.org/binutils/docs-2.35/ld/PowerPC64-ELF64.html
case "$HOST" in
diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh
index 36a4ea08f6..81c88a2ae7 100755
--- a/contrib/install_db4.sh
+++ b/contrib/install_db4.sh
@@ -1,5 +1,5 @@
#!/bin/sh
-# Copyright (c) 2017-2019 The Bitcoin Core developers
+# Copyright (c) 2017-2021 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
@@ -62,6 +62,12 @@ http_get() {
sha256_check "${3}" "${2}"
}
+# Ensure the commands we use exist on the system
+if ! check_exists patch; then
+ echo "Command-line tool 'patch' not found. Install patch and try again."
+ exit 1
+fi
+
mkdir -p "${BDB_PREFIX}"
http_get "${BDB_URL}" "${BDB_VERSION}.tar.gz" "${BDB_HASH}"
tar -xzvf ${BDB_VERSION}.tar.gz -C "$BDB_PREFIX"
diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk
index 457d8f5e64..ebef1d2db0 100755
--- a/contrib/macdeploy/gen-sdk
+++ b/contrib/macdeploy/gen-sdk
@@ -81,7 +81,7 @@ def run():
print("Creating output .tar.gz file...")
with out_sdktgz_path.open("wb") as fp:
- with gzip.GzipFile(fileobj=fp, compresslevel=9, mtime=0) as gzf:
+ with gzip.GzipFile(fileobj=fp, mode='wb', compresslevel=9, mtime=0) as gzf:
with tarfile.open(mode="w", fileobj=gzf) as tarfp:
print("Adding MacOSX SDK {} files...".format(sdk_version))
tarfp_add_with_base_change(tarfp, sdk_dir, out_name)
diff --git a/contrib/tracing/README.md b/contrib/tracing/README.md
index 1f93474fa0..b71ce2f34b 100644
--- a/contrib/tracing/README.md
+++ b/contrib/tracing/README.md
@@ -234,3 +234,62 @@ Histogram of block connection times in milliseconds (ms).
[16, 32) 9 | |
[32, 64) 4 | |
```
+
+### log_utxocache_flush.py
+
+A BCC Python script to log the cache and index flushes. Based on the
+`utxocache:flush` tracepoint.
+
+```bash
+$ python3 contrib/tracing/log_utxocache_flush.py ./src/bitcoind
+```
+
+```
+Logging utxocache flushes. Ctrl-C to end...
+Duration (µs) Mode Coins Count Memory Usage Prune Full Flush
+0 PERIODIC 28484 3929.87 kB False False
+1 PERIODIC 28485 3930.00 kB False False
+0 PERIODIC 28489 3930.51 kB False False
+1 PERIODIC 28490 3930.64 kB False False
+0 PERIODIC 28491 3930.77 kB False False
+0 PERIODIC 28491 3930.77 kB False False
+0 PERIODIC 28496 3931.41 kB False False
+1 PERIODIC 28496 3931.41 kB False False
+0 PERIODIC 28497 3931.54 kB False False
+1 PERIODIC 28497 3931.54 kB False False
+1 PERIODIC 28499 3931.79 kB False False
+.
+.
+.
+53788 ALWAYS 30076 4136.27 kB False False
+7463 ALWAYS 0 245.84 kB False False
+```
+
+### log_utxos.bt
+
+A `bpftrace` script to log information about the coins that are added, spent, or
+uncached from the UTXO set. Based on the `utxocache:add`, `utxocache:spend` and
+`utxocache:uncache` tracepoints.
+
+```bash
+$ bpftrace contrib/tracing/log_utxos.bt
+```
+
+It should produce an output similar to the following.
+
+```bash
+Attaching 4 probes...
+OP Outpoint Value Height Coinbase
+Added 6ba9ad857e1ef2eb2a2c94f06813c414c7ab273e3d6bd7ad64e000315a887e7c:1 10000 2094512 No
+Spent fa7dc4db56637a151f6649d8f26732956d1c5424c82aae400a83d02b2cc2c87b:0 182264897 2094512 No
+Added eeb2f099b1af6a2a12e6ddd2eeb16fc5968582241d7f08ba202d28b60ac264c7:0 10000 2094512 No
+Added eeb2f099b1af6a2a12e6ddd2eeb16fc5968582241d7f08ba202d28b60ac264c7:1 182254756 2094512 No
+Added a0c7f4ec9cccef2d89672a624a4e6c8237a17572efdd4679eea9e9ee70d2db04:0 10072679 2094513 Yes
+Spent 25e0df5cc1aeb1b78e6056bf403e5e8b7e41f138060ca0a50a50134df0549a5e:2 540 2094508 No
+Spent 42f383c04e09c26a2378272ec33aa0c1bf4883ca5ab739e8b7e06be5a5787d61:1 3848399 2007724 No
+Added f85e3b4b89270863a389395cc9a4123e417ab19384cef96533c6649abd6b0561:0 3788399 2094513 No
+Added f85e3b4b89270863a389395cc9a4123e417ab19384cef96533c6649abd6b0561:2 540 2094513 No
+Spent a05880b8c77971ed0b9f73062c7c4cdb0ff3856ab14cbf8bc481ed571cd34b83:1 5591281046 2094511 No
+Added eb689865f7d957938978d6207918748f74e6aa074f47874724327089445b0960:0 5589696005 2094513 No
+Added eb689865f7d957938978d6207918748f74e6aa074f47874724327089445b0960:1 1565556 2094513 No
+```
diff --git a/contrib/tracing/log_utxocache_flush.py b/contrib/tracing/log_utxocache_flush.py
new file mode 100755
index 0000000000..df27dc193a
--- /dev/null
+++ b/contrib/tracing/log_utxocache_flush.py
@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+
+import sys
+import ctypes
+from bcc import BPF, USDT
+
+"""Example logging Bitcoin Core utxo set cache flushes utilizing
+ the utxocache:flush tracepoint."""
+
+# USAGE: ./contrib/tracing/log_utxocache_flush.py path/to/bitcoind
+
+# BCC: The C program to be compiled to an eBPF program (by BCC) and loaded into
+# a sandboxed Linux kernel VM.
+program = """
+# include <uapi/linux/ptrace.h>
+struct data_t
+{
+ u64 duration;
+ u32 mode;
+ u64 coins_count;
+ u64 coins_mem_usage;
+ bool is_flush_prune;
+ bool is_full_flush;
+};
+
+// BPF perf buffer to push the data to user space.
+BPF_PERF_OUTPUT(flush);
+
+int trace_flush(struct pt_regs *ctx) {
+ struct data_t data = {};
+ bpf_usdt_readarg(1, ctx, &data.duration);
+ bpf_usdt_readarg(2, ctx, &data.mode);
+ bpf_usdt_readarg(3, ctx, &data.coins_count);
+ bpf_usdt_readarg(4, ctx, &data.coins_mem_usage);
+ bpf_usdt_readarg(5, ctx, &data.is_flush_prune);
+ bpf_usdt_readarg(5, ctx, &data.is_full_flush);
+ flush.perf_submit(ctx, &data, sizeof(data));
+ return 0;
+}
+"""
+
+FLUSH_MODES = [
+ 'NONE',
+ 'IF_NEEDED',
+ 'PERIODIC',
+ 'ALWAYS'
+]
+
+
+class Data(ctypes.Structure):
+ # define output data structure corresponding to struct data_t
+ _fields_ = [
+ ("duration", ctypes.c_uint64),
+ ("mode", ctypes.c_uint32),
+ ("coins_count", ctypes.c_uint64),
+ ("coins_mem_usage", ctypes.c_uint64),
+ ("is_flush_prune", ctypes.c_bool),
+ ("is_full_flush", ctypes.c_bool)
+ ]
+
+
+def print_event(event):
+ print("%-15d %-10s %-15d %-15s %-8s %-8s" % (
+ event.duration,
+ FLUSH_MODES[event.mode],
+ event.coins_count,
+ "%.2f kB" % (event.coins_mem_usage/1000),
+ event.is_flush_prune,
+ event.is_full_flush
+ ))
+
+
+def main(bitcoind_path):
+ bitcoind_with_usdts = USDT(path=str(bitcoind_path))
+
+ # attaching the trace functions defined in the BPF program
+ # to the tracepoints
+ bitcoind_with_usdts.enable_probe(
+ probe="flush", fn_name="trace_flush")
+ b = BPF(text=program, usdt_contexts=[bitcoind_with_usdts])
+
+ def handle_flush(_, data, size):
+ """ Coins Flush handler.
+ Called each time coin caches and indexes are flushed."""
+ event = ctypes.cast(data, ctypes.POINTER(Data)).contents
+ print_event(event)
+
+ b["flush"].open_perf_buffer(handle_flush)
+ print("Logging utxocache flushes. Ctrl-C to end...")
+ print("%-15s %-10s %-15s %-15s %-8s %-8s" % ("Duration (µs)", "Mode",
+ "Coins Count", "Memory Usage",
+ "Prune", "Full Flush"))
+
+ while True:
+ try:
+ b.perf_buffer_poll()
+ except KeyboardInterrupt:
+ exit(0)
+
+
+if __name__ == "__main__":
+ if len(sys.argv) < 2:
+ print("USAGE: ", sys.argv[0], "path/to/bitcoind")
+ exit(1)
+
+ path = sys.argv[1]
+ main(path)
diff --git a/contrib/tracing/log_utxos.bt b/contrib/tracing/log_utxos.bt
new file mode 100755
index 0000000000..0d47f3d62b
--- /dev/null
+++ b/contrib/tracing/log_utxos.bt
@@ -0,0 +1,86 @@
+#!/usr/bin/env bpftrace
+
+/*
+
+ USAGE:
+
+ bpftrace contrib/tracing/log_utxos.bt
+
+ This script requires a 'bitcoind' binary compiled with eBPF support and the
+ 'utxochache' tracepoints. By default, it's assumed that 'bitcoind' is
+ located in './src/bitcoind'. This can be modified in the script below.
+
+ NOTE: requires bpftrace v0.12.0 or above.
+*/
+
+BEGIN
+{
+ printf("%-7s %-71s %16s %7s %8s\n",
+ "OP", "Outpoint", "Value", "Height", "Coinbase");
+}
+
+/*
+ Attaches to the 'utxocache:add' tracepoint and prints additions to the UTXO set cache.
+*/
+usdt:./src/bitcoind:utxocache:add
+{
+ $txid = arg0;
+ $index = (uint32)arg1;
+ $height = (uint32)arg2;
+ $value = (int64)arg3;
+ $isCoinbase = arg4;
+
+ printf("Added ");
+ $p = $txid + 31;
+ unroll(32) {
+ $b = *(uint8*)$p;
+ printf("%02x", $b);
+ $p-=1;
+ }
+
+ printf(":%-6d %16ld %7d %s\n", $index, $value, $height, ($isCoinbase ? "Yes" : "No" ));
+}
+
+/*
+ Attaches to the 'utxocache:spent' tracepoint and prints spents from the UTXO set cache.
+*/
+usdt:./src/bitcoind:utxocache:spent
+{
+ $txid = arg0;
+ $index = (uint32)arg1;
+ $height = (uint32)arg2;
+ $value = (int64)arg3;
+ $isCoinbase = arg4;
+
+ printf("Spent ");
+ $p = $txid + 31;
+ unroll(32) {
+ $b = *(uint8*)$p;
+ printf("%02x", $b);
+ $p-=1;
+ }
+
+ printf(":%-6d %16ld %7d %s\n", $index, $value, $height, ($isCoinbase ? "Yes" : "No" ));
+}
+
+/*
+ Attaches to the 'utxocache:uncache' tracepoint and uncache UTXOs from the UTXO set cache.
+*/
+usdt:./src/bitcoind:utxocache:uncache
+{
+ $txid = arg0;
+ $index = (uint32)arg1;
+ $height = (uint32)arg2;
+ $value = (int64)arg3;
+ $isCoinbase = arg4;
+
+ printf("Uncache ");
+ $p = $txid + 31;
+ unroll(32) {
+ $b = *(uint8*)$p;
+ printf("%02x", $b);
+ $p-=1;
+ }
+
+ printf(":%-6d %16ld %7d %s\n", $index, $value, $height, ($isCoinbase ? "Yes" : "No" ));
+}
generated by cgit v1.2.3 (git 2.26.2) at 2024-07-20 04:36:57 +0000