diff options
Diffstat (limited to 'contrib')
| -rwxr-xr-x | contrib/devtools/check-deps.sh | 203 | ||||
| -rwxr-xr-x | contrib/devtools/symbol-check.py | 6 | ||||
| -rwxr-xr-x | contrib/guix/guix-build | 4 | ||||
| -rwxr-xr-x | contrib/guix/libexec/build.sh | 3 | ||||
| -rw-r--r-- | contrib/guix/libexec/prelude.bash | 2 | ||||
| -rw-r--r-- | contrib/guix/manifest.scm | 6 | ||||
| -rw-r--r-- | contrib/macdeploy/README.md | 34 | ||||
| -rwxr-xr-x | contrib/macdeploy/gen-sdk | 15 | ||||
| -rwxr-xr-x | contrib/seeds/generate-seeds.py | 13 | ||||
| -rwxr-xr-x | contrib/signet/miner | 8 |
10 files changed, 238 insertions, 56 deletions
diff --git a/contrib/devtools/check-deps.sh b/contrib/devtools/check-deps.sh new file mode 100755 index 0000000000..9d2eebe14d --- /dev/null +++ b/contrib/devtools/check-deps.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash + +export LC_ALL=C +set -Eeuo pipefail + +# Declare paths to libraries +declare -A LIBS +LIBS[cli]="libbitcoin_cli.a" +LIBS[common]="libbitcoin_common.a" +LIBS[consensus]="libbitcoin_consensus.a" +LIBS[crypto]="crypto/.libs/libbitcoin_crypto_base.a crypto/.libs/libbitcoin_crypto_x86_shani.a crypto/.libs/libbitcoin_crypto_sse41.a crypto/.libs/libbitcoin_crypto_avx2.a" +LIBS[node]="libbitcoin_node.a" +LIBS[util]="libbitcoin_util.a" +LIBS[wallet]="libbitcoin_wallet.a" +LIBS[wallet_tool]="libbitcoin_wallet_tool.a" + +# Declare allowed dependencies "X Y" where X is allowed to depend on Y. This +# list is taken from doc/design/libraries.md. +ALLOWED_DEPENDENCIES=( + "cli common" + "cli util" + "common consensus" + "common crypto" + "common util" + "consensus crypto" + "node common" + "node consensus" + "node crypto" + "node kernel" + "node util" + "util crypto" + "wallet common" + "wallet crypto" + "wallet util" + "wallet_tool util" + "wallet_tool wallet" +) + +# Add minor dependencies omitted from doc/design/libraries.md to keep the +# dependency diagram simple. +ALLOWED_DEPENDENCIES+=( + "wallet consensus" + "wallet_tool common" + "wallet_tool crypto" +) + +# Declare list of known errors that should be suppressed. +declare -A SUPPRESS +# init.cpp file currently calls Berkeley DB sanity check function on startup, so +# there is an undocumented dependency of the node library on the wallet library. +SUPPRESS["libbitcoin_node_a-init.o libbitcoin_wallet_a-bdb.o _ZN6wallet27BerkeleyDatabaseSanityCheckEv"]=1 +# init/common.cpp file calls InitError and InitWarning from interface_ui which +# is currently part of the node library. interface_ui should just be part of the +# common library instead, and is moved in +# https://github.com/bitcoin/bitcoin/issues/10102 +SUPPRESS["libbitcoin_common_a-common.o libbitcoin_node_a-interface_ui.o _Z11InitWarningRK13bilingual_str"]=1 +SUPPRESS["libbitcoin_common_a-common.o libbitcoin_node_a-interface_ui.o _Z9InitErrorRK13bilingual_str"]=1 +# rpc/external_signer.cpp adds defines node RPC methods but is built as part of the +# common library. It should be moved to the node library instead. +SUPPRESS["libbitcoin_common_a-external_signer.o libbitcoin_node_a-server.o _ZN9CRPCTable13appendCommandERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPK11CRPCCommand"]=1 + +usage() { + echo "Usage: $(basename "${BASH_SOURCE[0]}") [BUILD_DIR]" +} + +# Output makefile targets, converting library .a paths to libtool .la targets +lib_targets() { + for lib in "${!LIBS[@]}"; do + for lib_path in ${LIBS[$lib]}; do + # shellcheck disable=SC2001 + sed 's:/.libs/\(.*\)\.a$:/\1.la:g' <<<"$lib_path" + done + done +} + +# Extract symbol names and object names and write to text files +extract_symbols() { + local temp_dir="$1" + for lib in "${!LIBS[@]}"; do + for lib_path in ${LIBS[$lib]}; do + nm -o "$lib_path" | grep ' T ' | awk '{print $3, $1}' >> "${temp_dir}/${lib}_exports.txt" + nm -o "$lib_path" | grep ' U ' | awk '{print $3, $1}' >> "${temp_dir}/${lib}_imports.txt" + awk '{print $1}' "${temp_dir}/${lib}_exports.txt" | sort -u > "${temp_dir}/${lib}_exported_symbols.txt" + awk '{print $1}' "${temp_dir}/${lib}_imports.txt" | sort -u > "${temp_dir}/${lib}_imported_symbols.txt" + done + done +} + +# Lookup object name(s) corresponding to symbol name in text file +obj_names() { + local symbol="$1" + local txt_file="$2" + sed -n "s/^$symbol [^:]\\+:\\([^:]\\+\\):[^:]*\$/\\1/p" "$txt_file" | sort -u +} + +# Iterate through libraries and find disallowed dependencies +check_libraries() { + local temp_dir="$1" + local result=0 + for src in "${!LIBS[@]}"; do + for dst in "${!LIBS[@]}"; do + if [ "$src" != "$dst" ] && ! is_allowed "$src" "$dst"; then + if ! check_disallowed "$src" "$dst" "$temp_dir"; then + result=1 + fi + fi + done + done + check_not_suppressed + return $result +} + +# Return whether src library is allowed to depend on dst. +is_allowed() { + local src="$1" + local dst="$2" + for allowed in "${ALLOWED_DEPENDENCIES[@]}"; do + if [ "$src $dst" = "$allowed" ]; then + return 0 + fi + done + return 1 +} + +# Return whether src library imports any symbols from dst, assuming src is not +# allowed to depend on dst. +check_disallowed() { + local src="$1" + local dst="$2" + local temp_dir="$3" + local result=0 + + # Loop over symbol names exported by dst and imported by src + while read symbol; do + local dst_obj + dst_obj=$(obj_names "$symbol" "${temp_dir}/${dst}_exports.txt") + while read src_obj; do + if ! check_suppress "$src_obj" "$dst_obj" "$symbol"; then + echo "Error: $src_obj depends on $dst_obj symbol '$(c++filt "$symbol")', can suppess with:" + echo " SUPPRESS[\"$src_obj $dst_obj $symbol\"]=1" + result=1 + fi + done < <(obj_names "$symbol" "${temp_dir}/${src}_imports.txt") + done < <(comm -12 "${temp_dir}/${dst}_exported_symbols.txt" "${temp_dir}/${src}_imported_symbols.txt") + return $result +} + +# Declare array to track errors which were suppressed. +declare -A SUPPRESSED + +# Return whether error should be suppressed and record suppresssion in +# SUPPRESSED array. +check_suppress() { + local src_obj="$1" + local dst_obj="$2" + local symbol="$3" + for suppress in "${!SUPPRESS[@]}"; do + read suppress_src suppress_dst suppress_pattern <<<"$suppress" + if [[ "$src_obj" == "$suppress_src" && "$dst_obj" == "$suppress_dst" && "$symbol" =~ $suppress_pattern ]]; then + SUPPRESSED["$suppress"]=1 + return 0 + fi + done + return 1 +} + +# Warn about error which were supposed to be suppress, but were not encountered. +check_not_suppressed() { + for suppress in "${!SUPPRESS[@]}"; do + if [[ ! -v SUPPRESSED[$suppress] ]]; then + echo >&2 "Warning: suppression '$suppress' was ignored, consider deleting." + fi + done +} + +# Check arguments. +if [ "$#" = 0 ]; then + BUILD_DIR="$(dirname "${BASH_SOURCE[0]}")/../../src" +elif [ "$#" = 1 ]; then + BUILD_DIR="$1" +else + echo >&2 "Error: wrong number of arguments." + usage >&2 + exit 1 +fi +if [ ! -f "$BUILD_DIR/Makefile" ]; then + echo >&2 "Error: directory '$BUILD_DIR' does not contain a makefile, please specify path to build directory for library targets." + usage >&2 + exit 1 +fi + +# Build libraries and run checks. +cd "$BUILD_DIR" +# shellcheck disable=SC2046 +make -j"$(nproc)" $(lib_targets) +TEMP_DIR="$(mktemp -d)" +extract_symbols "$TEMP_DIR" +if check_libraries "$TEMP_DIR"; then + echo "Success! No unexpected dependencies were detected." +else + echo >&2 "Error: Unexpected dependencies were detected. Check previous output." +fi +rm -r "$TEMP_DIR" diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index e4a62c2072..6613874ce3 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -239,8 +239,8 @@ def check_MACHO_sdk(binary) -> bool: return True return False -def check_MACHO_ld64(binary) -> bool: - if binary.build_version.tools[0].version == [17, 0, 6]: +def check_MACHO_lld(binary) -> bool: + if binary.build_version.tools[0].version == [18, 1, 6]: return True return False @@ -282,7 +282,7 @@ lief.EXE_FORMATS.MACHO: [ ('DYNAMIC_LIBRARIES', check_MACHO_libraries), ('MIN_OS', check_MACHO_min_os), ('SDK', check_MACHO_sdk), - ('LD64', check_MACHO_ld64), + ('LLD', check_MACHO_lld), ], lief.EXE_FORMATS.PE: [ ('DYNAMIC_LIBRARIES', check_PE_libraries), diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index 870938cb52..2ea574fe4b 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -361,6 +361,10 @@ INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}: ...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' ...outputting in: '$(outdir_for_host "$HOST")' ...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")' + ADDITIONAL FLAGS (if set) + ADDITIONAL_GUIX_COMMON_FLAGS: ${ADDITIONAL_GUIX_COMMON_FLAGS} + ADDITIONAL_GUIX_ENVIRONMENT_FLAGS: ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} + ADDITIONAL_GUIX_TIMEMACHINE_FLAGS: ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} EOF # Run the build script 'contrib/guix/libexec/build.sh' in the build diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index f589ac7a55..9bc8c0e75d 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -178,8 +178,7 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \ x86_64_linux_AR=x86_64-linux-gnu-gcc-ar \ x86_64_linux_RANLIB=x86_64-linux-gnu-gcc-ranlib \ x86_64_linux_NM=x86_64-linux-gnu-gcc-nm \ - x86_64_linux_STRIP=x86_64-linux-gnu-strip \ - FORCE_USE_SYSTEM_CLANG=1 + x86_64_linux_STRIP=x86_64-linux-gnu-strip ########################### diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index ce6a9562b4..80bfb2875f 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -51,7 +51,7 @@ fi time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=dc4842797bfdc5f9f3f5f725bf189c2b68bd6b5a \ + --commit=f0bb724211872cd6158fce6162e0b8c73efed126 \ --cores="$JOBS" \ --keep-failed \ --fallback \ diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 40500ccb88..53569d7f7d 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -532,9 +532,9 @@ inspecting signatures in Mach-O binaries.") ((string-contains target "darwin") (list ;; Native GCC 11 toolchain gcc-toolchain-11 - clang-toolchain-17 - lld-17 - (make-lld-wrapper lld-17 #:lld-as-ld? #t) + clang-toolchain-18 + lld-18 + (make-lld-wrapper lld-18 #:lld-as-ld? #t) python-signapple zip)) (else '()))))) diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 78f61685e1..d47ee6774e 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -56,30 +56,22 @@ The `sha256sum` should be `c0c2e7bb92c1fee0c4e9f3a485e4530786732d6c6dd9e9f418c28 ## Deterministic macOS App Notes -macOS Applications are created in Linux using a recent LLVM. +macOS Applications are created on Linux using a recent LLVM. -Apple uses `clang` extensively for development and has upstreamed the necessary -functionality so that a vanilla clang can take advantage. It supports the use of `-F`, -`-target`, `-mmacosx-version-min`, and `-isysroot`, which are all necessary when -building for macOS. +All builds must target an Apple SDK. These SDKs are free to download, but not redistributable. +See the SDK Extraction notes above for how to obtain it. -To complicate things further, all builds must target an Apple SDK. These SDKs are free to -download, but not redistributable. See the SDK Extraction notes above for how to obtain it. +The Guix build process has been designed to avoid including the SDK's files in Guix's outputs. +All interim tarballs are fully deterministic and may be freely redistributed. -The Guix process builds 2 sets of files: Linux tools, then Apple binaries which are -created using these tools. The build process has been designed to avoid including the -SDK's files in Guix's outputs. All interim tarballs are fully deterministic and may be freely -redistributed. - -As of OS X 10.9 Mavericks, using an Apple-blessed key to sign binaries is a requirement in -order to satisfy the new Gatekeeper requirements. Because this private key cannot be -shared, we'll have to be a bit creative in order for the build process to remain somewhat -deterministic. Here's how it works: +Using an Apple-blessed key to sign binaries is a requirement to produce (distributable) macOS +binaries. Because this private key cannot be shared, we'll have to be a bit creative in order +for the build process to remain somewhat deterministic. Here's how it works: - Builders use Guix to create an unsigned release. This outputs an unsigned ZIP which - users may choose to bless and run. It also outputs an unsigned app structure in the form - of a tarball. + users may choose to bless, self-codesign, and run. It also outputs an unsigned app structure + in the form of a tarball. - The Apple keyholder uses this unsigned app to create a detached signature, using the - script that is also included there. Detached signatures are available from this [repository](https://github.com/bitcoin-core/bitcoin-detached-sigs). -- Builders feed the unsigned app + detached signature back into Guix. It uses the - pre-built tools to recombine the pieces into a deterministic ZIP. + included script. Detached signatures are available from this [repository](https://github.com/bitcoin-core/bitcoin-detached-sigs). +- Builders feed the unsigned app + detached signature back into Guix, which combines the + pieces into a deterministic ZIP. diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk index b73f5cba14..86a6262b5c 100755 --- a/contrib/macdeploy/gen-sdk +++ b/contrib/macdeploy/gen-sdk @@ -8,21 +8,6 @@ import gzip import os import contextlib -# monkey-patch Python 3.8 and older to fix wrong TAR header handling -# see https://github.com/bitcoin/bitcoin/pull/24534 -# and https://github.com/python/cpython/pull/18080 for more info -if sys.version_info < (3, 9): - _old_create_header = tarfile.TarInfo._create_header - def _create_header(info, format, encoding, errors): - buf = _old_create_header(info, format, encoding, errors) - # replace devmajor/devminor with binary zeroes - buf = buf[:329] + bytes(16) + buf[345:] - # recompute checksum - chksum = tarfile.calc_chksums(buf)[0] - buf = buf[:-364] + bytes("%06o\0" % chksum, "ascii") + buf[-357:] - return buf - tarfile.TarInfo._create_header = staticmethod(_create_header) - @contextlib.contextmanager def cd(path): """Context manager that restores PWD even if an exception was raised.""" diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py index e921757802..f67e7b0f4c 100755 --- a/contrib/seeds/generate-seeds.py +++ b/contrib/seeds/generate-seeds.py @@ -29,7 +29,6 @@ These should be pasted into `src/chainparamsseeds.h`. from base64 import b32decode from enum import Enum -import struct import sys import os import re @@ -115,13 +114,13 @@ def parse_spec(s): def ser_compact_size(l): r = b"" if l < 253: - r = struct.pack("B", l) + r = l.to_bytes(1, "little") elif l < 0x10000: - r = struct.pack("<BH", 253, l) + r = (253).to_bytes(1, "little") + l.to_bytes(2, "little") elif l < 0x100000000: - r = struct.pack("<BI", 254, l) + r = (254).to_bytes(1, "little") + l.to_bytes(4, "little") else: - r = struct.pack("<BQ", 255, l) + r = (255).to_bytes(1, "little") + l.to_bytes(8, "little") return r def bip155_serialize(spec): @@ -129,10 +128,10 @@ def bip155_serialize(spec): Serialize (networkID, addr, port) tuple to BIP155 binary format. ''' r = b"" - r += struct.pack('B', spec[0].value) + r += spec[0].value.to_bytes(1, "little") r += ser_compact_size(len(spec[1])) r += spec[1] - r += struct.pack('>H', spec[2]) + r += spec[2].to_bytes(2, "big") return r def process_nodes(g, f, structname): diff --git a/contrib/signet/miner b/contrib/signet/miner index e5daf9f993..4216ada5fa 100755 --- a/contrib/signet/miner +++ b/contrib/signet/miner @@ -52,20 +52,20 @@ def signet_txs(block, challenge): mroot = block.get_merkle_root(hashes) sd = b"" - sd += struct.pack("<i", block.nVersion) + sd += block.nVersion.to_bytes(4, "little", signed=True) sd += ser_uint256(block.hashPrevBlock) sd += ser_uint256(mroot) - sd += struct.pack("<I", block.nTime) + sd += block.nTime.to_bytes(4, "little") to_spend = CTransaction() - to_spend.nVersion = 0 + to_spend.version = 0 to_spend.nLockTime = 0 to_spend.vin = [CTxIn(COutPoint(0, 0xFFFFFFFF), b"\x00" + CScriptOp.encode_op_pushdata(sd), 0)] to_spend.vout = [CTxOut(0, challenge)] to_spend.rehash() spend = CTransaction() - spend.nVersion = 0 + spend.version = 0 spend.nLockTime = 0 spend.vin = [CTxIn(COutPoint(to_spend.sha256, 0), b"", 0)] spend.vout = [CTxOut(0, b"\x6a")] |