diff options
Diffstat (limited to 'contrib')
| -rw-r--r-- | contrib/devtools/README.md | 4 | ||||
| -rwxr-xr-x | contrib/devtools/gen-manpages.py | 71 | ||||
| -rwxr-xr-x | contrib/devtools/gen-manpages.sh | 53 | ||||
| -rwxr-xr-x | contrib/guix/guix-build | 2 | ||||
| -rw-r--r-- | contrib/tracing/README.md | 31 | ||||
| -rwxr-xr-x | contrib/tracing/log_utxocache_flush.py | 21 | ||||
| -rwxr-xr-x | contrib/tracing/log_utxos.bt | 2 |
7 files changed, 96 insertions, 88 deletions
diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index afbad096c4..79b0134adc 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -76,7 +76,7 @@ year rather than two hyphenated years. If the file already has a copyright for `The Bitcoin Core developers`, the script will exit. -gen-manpages.sh +gen-manpages.py =============== A small script to automatically create manpages in ../../doc/man by running the release binaries with the -help option. @@ -87,7 +87,7 @@ repostitory. To use this tool with out-of-tree builds set `BUILDDIR`. For example: ```bash -BUILDDIR=$PWD/build contrib/devtools/gen-manpages.sh +BUILDDIR=$PWD/build contrib/devtools/gen-manpages.py ``` security-check.py and test-security-check.py diff --git a/contrib/devtools/gen-manpages.py b/contrib/devtools/gen-manpages.py new file mode 100755 index 0000000000..26612cc444 --- /dev/null +++ b/contrib/devtools/gen-manpages.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +import os +import subprocess +import sys +import tempfile + +BINARIES = [ +'src/bitcoind', +'src/bitcoin-cli', +'src/bitcoin-tx', +'src/bitcoin-wallet', +'src/bitcoin-util', +'src/qt/bitcoin-qt', +] + +# Paths to external utilities. +git = os.getenv('GIT', 'git') +help2man = os.getenv('HELP2MAN', 'help2man') + +# If not otherwise specified, get top directory from git. +topdir = os.getenv('TOPDIR') +if not topdir: + r = subprocess.run([git, 'rev-parse', '--show-toplevel'], stdout=subprocess.PIPE, check=True, universal_newlines=True) + topdir = r.stdout.rstrip() + +# Get input and output directories. +builddir = os.getenv('BUILDDIR', topdir) +mandir = os.getenv('MANDIR', os.path.join(topdir, 'doc/man')) + +# Verify that all the required binaries are usable, and extract copyright +# message in a first pass. +versions = [] +for relpath in BINARIES: + abspath = os.path.join(builddir, relpath) + try: + r = subprocess.run([abspath, '--version'], stdout=subprocess.PIPE, universal_newlines=True) + except IOError: + print(f'{abspath} not found or not an executable', file=sys.stderr) + sys.exit(1) + # take first line (which must contain version) + verstr = r.stdout.splitlines()[0] + # last word of line is the actual version e.g. v22.99.0-5c6b3d5b3508 + verstr = verstr.split()[-1] + assert verstr.startswith('v') + # remaining lines are copyright + copyright = r.stdout.split('\n')[1:] + assert copyright[0].startswith('Copyright (C)') + + versions.append((abspath, verstr, copyright)) + +if any(verstr.endswith('-dirty') for (_, verstr, _) in versions): + print("WARNING: Binaries were built from a dirty tree.") + print('man pages generated from dirty binaries should NOT be committed.') + print('To properly generate man pages, please commit your changes (or discard them), rebuild, then run this script again.') + print() + +with tempfile.NamedTemporaryFile('w', suffix='.h2m') as footer: + # Create copyright footer, and write it to a temporary include file. + # Copyright is the same for all binaries, so just use the first. + footer.write('[COPYRIGHT]\n') + footer.write('\n'.join(versions[0][2]).strip()) + footer.flush() + + # Call the binaries through help2man to produce a manual page for each of them. + for (abspath, verstr, _) in versions: + outname = os.path.join(mandir, os.path.basename(abspath) + '.1') + print(f'Generating {outname}…') + subprocess.run([help2man, '-N', '--version-string=' + verstr, '--include=' + footer.name, '-o', outname, abspath], check=True) diff --git a/contrib/devtools/gen-manpages.sh b/contrib/devtools/gen-manpages.sh deleted file mode 100755 index 8da6ff1204..0000000000 --- a/contrib/devtools/gen-manpages.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2016-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C -TOPDIR=${TOPDIR:-$(git rev-parse --show-toplevel)} -BUILDDIR=${BUILDDIR:-$TOPDIR} - -BINDIR=${BINDIR:-$BUILDDIR/src} -MANDIR=${MANDIR:-$TOPDIR/doc/man} - -BITCOIND=${BITCOIND:-$BINDIR/bitcoind} -BITCOINCLI=${BITCOINCLI:-$BINDIR/bitcoin-cli} -BITCOINTX=${BITCOINTX:-$BINDIR/bitcoin-tx} -WALLET_TOOL=${WALLET_TOOL:-$BINDIR/bitcoin-wallet} -BITCOINUTIL=${BITCOINQT:-$BINDIR/bitcoin-util} -BITCOINQT=${BITCOINQT:-$BINDIR/qt/bitcoin-qt} - -[ ! -x "$BITCOIND" ] && echo "$BITCOIND not found or not executable." && exit 1 - -# Don't allow man pages to be generated for binaries built from a dirty tree -DIRTY="" -for cmd in $BITCOIND $BITCOINCLI $BITCOINTX $WALLET_TOOL $BITCOINUTIL $BITCOINQT; do - VERSION_OUTPUT=$($cmd --version) - if [[ $VERSION_OUTPUT == *"dirty"* ]]; then - DIRTY="${DIRTY}${cmd}\n" - fi -done -if [ -n "$DIRTY" ] -then - echo -e "WARNING: the following binaries were built from a dirty tree:\n" - echo -e "$DIRTY" - echo "man pages generated from dirty binaries should NOT be committed." - echo "To properly generate man pages, please commit your changes to the above binaries, rebuild them, then run this script again." -fi - -# The autodetected version git tag can screw up manpage output a little bit -read -r -a BTCVER <<< "$($BITCOINCLI --version | head -n1 | awk -F'[ -]' '{ print $6, $7 }')" - -# Create a footer file with copyright content. -# This gets autodetected fine for bitcoind if --version-string is not set, -# but has different outcomes for bitcoin-qt and bitcoin-cli. -echo "[COPYRIGHT]" > footer.h2m -$BITCOIND --version | sed -n '1!p' >> footer.h2m - -for cmd in $BITCOIND $BITCOINCLI $BITCOINTX $WALLET_TOOL $BITCOINUTIL $BITCOINQT; do - cmdname="${cmd##*/}" - help2man -N --version-string="${BTCVER[0]}" --include=footer.h2m -o "${MANDIR}/${cmdname}.1" "${cmd}" - sed -i "s/\\\-${BTCVER[1]}//g" "${MANDIR}/${cmdname}.1" -done - -rm -f footer.h2m diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index 2d44ad0365..b38f91f3c7 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -192,7 +192,7 @@ fi if ! getent services http https ftp > /dev/null 2>&1; then cat << EOF -ERR: Your system's C library can not find service database entries for at least +ERR: Your system's C library cannot find service database entries for at least one of the following services: http, https, ftp. Hint: Most likely, /etc/services does not exist yet (common for docker images diff --git a/contrib/tracing/README.md b/contrib/tracing/README.md index b71ce2f34b..a409a23ef8 100644 --- a/contrib/tracing/README.md +++ b/contrib/tracing/README.md @@ -237,7 +237,7 @@ Histogram of block connection times in milliseconds (ms). ### log_utxocache_flush.py -A BCC Python script to log the cache and index flushes. Based on the +A BCC Python script to log the UTXO cache flushes. Based on the `utxocache:flush` tracepoint. ```bash @@ -246,23 +246,10 @@ $ python3 contrib/tracing/log_utxocache_flush.py ./src/bitcoind ``` Logging utxocache flushes. Ctrl-C to end... -Duration (µs) Mode Coins Count Memory Usage Prune Full Flush -0 PERIODIC 28484 3929.87 kB False False -1 PERIODIC 28485 3930.00 kB False False -0 PERIODIC 28489 3930.51 kB False False -1 PERIODIC 28490 3930.64 kB False False -0 PERIODIC 28491 3930.77 kB False False -0 PERIODIC 28491 3930.77 kB False False -0 PERIODIC 28496 3931.41 kB False False -1 PERIODIC 28496 3931.41 kB False False -0 PERIODIC 28497 3931.54 kB False False -1 PERIODIC 28497 3931.54 kB False False -1 PERIODIC 28499 3931.79 kB False False -. -. -. -53788 ALWAYS 30076 4136.27 kB False False -7463 ALWAYS 0 245.84 kB False False +Duration (µs) Mode Coins Count Memory Usage Prune +730451 IF_NEEDED 22990 3323.54 kB True +637657 ALWAYS 122320 17124.80 kB False +81349 ALWAYS 0 1383.49 kB False ``` ### log_utxos.bt @@ -275,7 +262,13 @@ uncached from the UTXO set. Based on the `utxocache:add`, `utxocache:spend` and $ bpftrace contrib/tracing/log_utxos.bt ``` -It should produce an output similar to the following. +This should produce an output similar to the following. If you see bpftrace +warnings like `Lost 24 events`, the eBPF perf ring-buffer is filled faster +than it is being read. You can increase the ring-buffer size by setting the +ENV variable `BPFTRACE_PERF_RB_PAGES` (default 64) at a cost of higher +memory usage. See the [bpftrace reference guide] for more information. + +[bpftrace reference guide]: https://github.com/iovisor/bpftrace/blob/master/docs/reference_guide.md#98-bpftrace_perf_rb_pages ```bash Attaching 4 probes... diff --git a/contrib/tracing/log_utxocache_flush.py b/contrib/tracing/log_utxocache_flush.py index 7f26504e7b..8c073bea0d 100755 --- a/contrib/tracing/log_utxocache_flush.py +++ b/contrib/tracing/log_utxocache_flush.py @@ -16,14 +16,14 @@ from bcc import BPF, USDT # a sandboxed Linux kernel VM. program = """ # include <uapi/linux/ptrace.h> + struct data_t { u64 duration; u32 mode; u64 coins_count; u64 coins_mem_usage; - bool is_flush_prune; - bool is_full_flush; + bool is_flush_for_prune; }; // BPF perf buffer to push the data to user space. @@ -35,8 +35,7 @@ int trace_flush(struct pt_regs *ctx) { bpf_usdt_readarg(2, ctx, &data.mode); bpf_usdt_readarg(3, ctx, &data.coins_count); bpf_usdt_readarg(4, ctx, &data.coins_mem_usage); - bpf_usdt_readarg(5, ctx, &data.is_flush_prune); - bpf_usdt_readarg(5, ctx, &data.is_full_flush); + bpf_usdt_readarg(5, ctx, &data.is_flush_for_prune); flush.perf_submit(ctx, &data, sizeof(data)); return 0; } @@ -57,19 +56,17 @@ class Data(ctypes.Structure): ("mode", ctypes.c_uint32), ("coins_count", ctypes.c_uint64), ("coins_mem_usage", ctypes.c_uint64), - ("is_flush_prune", ctypes.c_bool), - ("is_full_flush", ctypes.c_bool) + ("is_flush_for_prune", ctypes.c_bool) ] def print_event(event): - print("%-15d %-10s %-15d %-15s %-8s %-8s" % ( + print("%-15d %-10s %-15d %-15s %-8s" % ( event.duration, FLUSH_MODES[event.mode], event.coins_count, "%.2f kB" % (event.coins_mem_usage/1000), - event.is_flush_prune, - event.is_full_flush + event.is_flush_for_prune )) @@ -90,9 +87,9 @@ def main(bitcoind_path): b["flush"].open_perf_buffer(handle_flush) print("Logging utxocache flushes. Ctrl-C to end...") - print("%-15s %-10s %-15s %-15s %-8s %-8s" % ("Duration (µs)", "Mode", - "Coins Count", "Memory Usage", - "Prune", "Full Flush")) + print("%-15s %-10s %-15s %-15s %-8s" % ("Duration (µs)", "Mode", + "Coins Count", "Memory Usage", + "Flush for Prune")) while True: try: diff --git a/contrib/tracing/log_utxos.bt b/contrib/tracing/log_utxos.bt index 0d47f3d62b..54d5010f82 100755 --- a/contrib/tracing/log_utxos.bt +++ b/contrib/tracing/log_utxos.bt @@ -7,7 +7,7 @@ bpftrace contrib/tracing/log_utxos.bt This script requires a 'bitcoind' binary compiled with eBPF support and the - 'utxochache' tracepoints. By default, it's assumed that 'bitcoind' is + 'utxocache' tracepoints. By default, it's assumed that 'bitcoind' is located in './src/bitcoind'. This can be modified in the script below. NOTE: requires bpftrace v0.12.0 or above. |