diff options
Diffstat (limited to 'contrib')
25 files changed, 497 insertions, 364 deletions
diff --git a/contrib/builder-keys/README.md b/contrib/builder-keys/README.md index 56bd87d0af..a6179d6012 100644 --- a/contrib/builder-keys/README.md +++ b/contrib/builder-keys/README.md @@ -19,9 +19,15 @@ gpg --refresh-keys To fetch keys of builders and active developers, feed the list of fingerprints of the primary keys into gpg: +On \*NIX: ```sh while read fingerprint keyholder_name; do gpg --keyserver hkps://keys.openpgp.org --recv-keys ${fingerprint}; done < ./keys.txt ``` +On Windows (requires Gpg4win >= 4.0.0): +``` +FOR /F "tokens=1" %i IN (keys.txt) DO gpg --keyserver hkps://keys.openpgp.org --recv-keys %i +``` + Add your key to the list if you provided Guix attestations for two major or minor releases of Bitcoin Core. diff --git a/contrib/builder-keys/keys.txt b/contrib/builder-keys/keys.txt index e8032f66ee..c70069b440 100644 --- a/contrib/builder-keys/keys.txt +++ b/contrib/builder-keys/keys.txt @@ -13,6 +13,7 @@ F20F56EF6A067F70E8A5C99FFF95FAA971697405 centaur (centaur) C060A6635913D98A3587D7DB1C2491FFEB0EF770 Cory Fields (cfields) BF6273FAEF7CC0BA1F562E50989F6B3048A116B5 Dev Random (devrandom) 6D3170C1DC2C6FD0AEEBCA6743811D1A26623924 Douglas Roark (droark) +948444FCE03B05BA5AB0591EC37B1C1D44C786EE Duncan Dean (dunxen) 1C6621605EC50319C463D56C7F81D87985D61612 Emanuele Cisbani (cisba) 9A1689B60D1B3CCE9262307A2F40A9BF167FBA47 Erik Mossberg (erkmos) D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece (fivepiece) @@ -50,5 +51,6 @@ ED9BDF7AD6A55E232E84524257FF9BDBCC301009 Sjors Provoost (sjors) 9EDAFF80E080659604F4A76B2EBB056FD847F8A7 Stephan Oeste (Emzy) 6DEEF79B050C4072509B743F8C275BC595448867 Tomas Kanocz (KanoczTomas) AEC1884398647C47413C1C3FB1179EB7347DC10D Warren Togami (wtogami) +74E2DEF5D77260B98BC19438099BAD163C70FBFA Will Clark (will8clark) 79D00BAC68B56D422F945A8F8E3A8F3247DBCBBF Willy Ko (willyko) 71A3B16735405025D447E8F274810B012346C9A6 Wladimir J. van der Laan (laanwj) diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index e6a29b73b9..05c0af029e 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -12,10 +12,6 @@ from typing import List import lief #type:ignore -# temporary constant, to be replaced with lief.ELF.ARCH.RISCV -# https://github.com/lief-project/LIEF/pull/562 -LIEF_ELF_ARCH_RISCV = lief.ELF.ARCH(243) - def check_ELF_RELRO(binary) -> bool: ''' Check for read-only relocations. @@ -101,7 +97,6 @@ def check_ELF_separate_code(binary): for segment in binary.segments: if segment.type == lief.ELF.SEGMENT_TYPES.LOAD: for section in segment.sections: - assert(section.name not in flags_per_section) flags_per_section[section.name] = segment.flags # Spot-check ELF LOAD program header flags per section # If these sections exist, check them against the expected R/W/E flags @@ -222,7 +217,7 @@ CHECKS = { lief.ARCHITECTURES.ARM: BASE_ELF, lief.ARCHITECTURES.ARM64: BASE_ELF, lief.ARCHITECTURES.PPC: BASE_ELF, - LIEF_ELF_ARCH_RISCV: BASE_ELF, + lief.ARCHITECTURES.RISCV: BASE_ELF, }, lief.EXE_FORMATS.PE: { lief.ARCHITECTURES.X86: BASE_PE, @@ -250,12 +245,9 @@ if __name__ == '__main__': continue if arch == lief.ARCHITECTURES.NONE: - if binary.header.machine_type == LIEF_ELF_ARCH_RISCV: - arch = LIEF_ELF_ARCH_RISCV - else: - print(f'{filename}: unknown architecture') - retval = 1 - continue + print(f'{filename}: unknown architecture') + retval = 1 + continue failed: List[str] = [] for (name, func) in CHECKS[etype][arch]: diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 461132ae63..a419e392ee 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -15,10 +15,6 @@ from typing import List, Dict import lief #type:ignore -# temporary constant, to be replaced with lief.ELF.ARCH.RISCV -# https://github.com/lief-project/LIEF/pull/562 -LIEF_ELF_ARCH_RISCV = lief.ELF.ARCH(243) - # Debian 9 (Stretch) EOL: 2022. https://wiki.debian.org/DebianReleases#Production_Releases # # - g++ version 6.3.0 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=g%2B%2B) @@ -44,7 +40,7 @@ MAX_VERSIONS = { lief.ELF.ARCH.ARM: (2,18), lief.ELF.ARCH.AARCH64:(2,18), lief.ELF.ARCH.PPC64: (2,18), - LIEF_ELF_ARCH_RISCV: (2,27), + lief.ELF.ARCH.RISCV: (2,27), }, 'LIBATOMIC': (1,0), 'V': (0,5,0), # xkb (bitcoin-qt only) @@ -78,7 +74,7 @@ ELF_INTERPRETER_NAMES: Dict[lief.ELF.ARCH, Dict[lief.ENDIANNESS, str]] = { lief.ENDIANNESS.BIG: "/lib64/ld64.so.1", lief.ENDIANNESS.LITTLE: "/lib64/ld64.so.2", }, - LIEF_ELF_ARCH_RISCV: { + lief.ELF.ARCH.RISCV: { lief.ENDIANNESS.LITTLE: "/lib/ld-linux-riscv64-lp64d.so.1", }, } @@ -200,7 +196,7 @@ def check_exported_symbols(binary) -> bool: if not symbol.exported: continue name = symbol.name - if binary.header.machine_type == LIEF_ELF_ARCH_RISCV or name in IGNORE_EXPORTS: + if binary.header.machine_type == lief.ELF.ARCH.RISCV or name in IGNORE_EXPORTS: continue print(f'{binary.name}: export of symbol {name} not allowed!') ok = False diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py index e1a2ebc491..b4c112b266 100755 --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -187,7 +187,7 @@ class TestSymbolChecks(unittest.TestCase): executable = 'test3.exe' with open(source, 'w', encoding="utf8") as f: f.write(''' - #include <windows.h> + #include <combaseapi.h> int main() { diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 90289f9d40..af5607c710 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -75,7 +75,7 @@ crucial differences: 1. Since only Windows and macOS build outputs require codesigning, the `HOSTS` environment variable will have a sane default value of `x86_64-w64-mingw32 - x86_64-apple-darwin` instead of all the platforms. + x86_64-apple-darwin arm64-apple-darwin` instead of all the platforms. 2. The `guix-codesign` command ***requires*** a `DETACHED_SIGS_REPO` flag. * _**DETACHED_SIGS_REPO**_ diff --git a/contrib/guix/guix-attest b/contrib/guix/guix-attest index 6e12cbead7..b0ef28dc3f 100755 --- a/contrib/guix/guix-attest +++ b/contrib/guix/guix-attest @@ -19,8 +19,16 @@ source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" ################ check_tools cat env basename mkdir diff sort + if [ -z "$NO_SIGN" ]; then - check_tools gpg + # make it possible to override the gpg binary + GPG=${GPG:-gpg} + + # $GPG can contain extra arguments passed to the binary + # so let's check only the existence of arg[0] + # shellcheck disable=SC2206 + GPG_ARRAY=($GPG) + check_tools "${GPG_ARRAY[0]}" fi ################ @@ -90,7 +98,7 @@ if [ -z "${signer_name}" ]; then signer_name="$gpg_key_name" fi -if [ -z "$NO_SIGN" ] && ! gpg --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then +if [ -z "$NO_SIGN" ] && ! ${GPG} --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then echo "ERR: GPG can't seem to find any key named '${gpg_key_name}'" exit 1 fi @@ -239,11 +247,11 @@ mkdir -p "$outsigdir" echo "Signing SHA256SUMS to produce SHA256SUMS.asc" for i in *.SHA256SUMS; do if [ ! -e "$i".asc ]; then - gpg --detach-sign \ - --digest-algo sha256 \ - --local-user "$gpg_key_name" \ - --armor \ - --output "$i".asc "$i" + ${GPG} --detach-sign \ + --digest-algo sha256 \ + --local-user "$gpg_key_name" \ + --armor \ + --output "$i".asc "$i" else echo "Signature already there" fi diff --git a/contrib/guix/guix-codesign b/contrib/guix/guix-codesign index 94895d3fee..3279d431aa 100755 --- a/contrib/guix/guix-codesign +++ b/contrib/guix/guix-codesign @@ -152,10 +152,10 @@ outdir_for_host() { unsigned_tarball_for_host() { case "$1" in *mingw*) - echo "$(outdir_for_host "$1")/${DISTNAME}-win-unsigned.tar.gz" + echo "$(outdir_for_host "$1")/${DISTNAME}-win64-unsigned.tar.gz" ;; *darwin*) - echo "$(outdir_for_host "$1")/${DISTNAME}-osx-unsigned.tar.gz" + echo "$(outdir_for_host "$1")/${DISTNAME}-${1}-unsigned.tar.gz" ;; *) exit 1 diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index e06a469338..4eeb360603 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -167,7 +167,6 @@ case "$HOST" in *linux*) glibc_dynamic_linker=$( case "$HOST" in - i686-linux-gnu) echo /lib/ld-linux.so.2 ;; x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; @@ -204,19 +203,12 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \ ${SOURCES_PATH+SOURCES_PATH="$SOURCES_PATH"} \ ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} \ ${SDK_PATH+SDK_PATH="$SDK_PATH"} \ - i686_linux_CC=i686-linux-gnu-gcc \ - i686_linux_CXX=i686-linux-gnu-g++ \ - i686_linux_AR=i686-linux-gnu-ar \ - i686_linux_RANLIB=i686-linux-gnu-ranlib \ - i686_linux_NM=i686-linux-gnu-nm \ - i686_linux_STRIP=i686-linux-gnu-strip \ x86_64_linux_CC=x86_64-linux-gnu-gcc \ x86_64_linux_CXX=x86_64-linux-gnu-g++ \ x86_64_linux_AR=x86_64-linux-gnu-ar \ x86_64_linux_RANLIB=x86_64-linux-gnu-ranlib \ x86_64_linux_NM=x86_64-linux-gnu-nm \ x86_64_linux_STRIP=x86_64-linux-gnu-strip \ - qt_config_opts_i686_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ qt_config_opts_x86_64_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ FORCE_USE_SYSTEM_CLANG=1 @@ -340,7 +332,7 @@ mkdir -p "$DISTSRC" mkdir -p "unsigned-app-${HOST}" cp --target-directory="unsigned-app-${HOST}" \ osx_volname \ - contrib/macdeploy/detached-sig-{apply,create}.sh \ + contrib/macdeploy/detached-sig-create.sh \ "${BASEPREFIX}/${HOST}"/native/bin/dmg mv --target-directory="unsigned-app-${HOST}" dist ( @@ -348,10 +340,10 @@ mkdir -p "$DISTSRC" find . -print0 \ | sort --zero-terminated \ | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz" && exit 1 ) + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" && exit 1 ) ) - make deploy ${V:+V=1} OSX_DMG="${OUTDIR}/${DISTNAME}-osx-unsigned.dmg" + make deploy ${V:+V=1} OSX_DMG="${OUTDIR}/${DISTNAME}-${HOST}-unsigned.dmg" ;; esac ( @@ -423,8 +415,8 @@ mkdir -p "$DISTSRC" find "${DISTNAME}" -print0 \ | sort --zero-terminated \ | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin/osx64}.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin/osx64}.tar.gz" && exit 1 ) + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}.tar.gz" && exit 1 ) ;; esac ) # $DISTSRC/installed @@ -439,8 +431,8 @@ mkdir -p "$DISTSRC" find . -print0 \ | sort --zero-terminated \ | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz" && exit 1 ) + | gzip -9n > "${OUTDIR}/${DISTNAME}-win64-unsigned.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-win64-unsigned.tar.gz" && exit 1 ) ) ;; esac diff --git a/contrib/guix/libexec/codesign.sh b/contrib/guix/libexec/codesign.sh index a8867ca351..6ede95f42b 100755 --- a/contrib/guix/libexec/codesign.sh +++ b/contrib/guix/libexec/codesign.sh @@ -91,7 +91,7 @@ mkdir -p "$DISTSRC" -- -volume_date all_file_dates ="$SOURCE_DATE_EPOCH" # Compress uncompressed.dmg and output to OUTDIR - ./dmg dmg uncompressed.dmg "${OUTDIR}/${DISTNAME}-osx-signed.dmg" + ./dmg dmg uncompressed.dmg "${OUTDIR}/${DISTNAME}-${HOST}.dmg" ;; *) exit 1 diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index 086df48fbe..f24c120863 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -51,7 +51,7 @@ fi time-machine() { # shellcheck disable=SC2086 guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=ae03f401381e956c4c41b4cf495cbde964fa43d0 \ + --commit=34e9eae68c9583acce5abc4100add3d88932a5ae \ --cores="$JOBS" \ --keep-failed \ --fallback \ diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index a0afb7b4f6..708d2e698d 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -162,9 +162,10 @@ desirable for building Bitcoin Core release binaries." (define (make-gcc-with-pthreads gcc) (package-with-extra-configure-variable gcc "--enable-threads" "posix")) -;; Required to support std::filesystem for mingw-w64 target. -(define (make-gcc-without-newlib gcc) - (package-with-extra-configure-variable gcc "--with-newlib" "no")) +(define (make-mingw-w64-cross-gcc cross-gcc) + (package-with-extra-patches cross-gcc + (search-our-patches "vmov-alignment.patch" + "gcc-broken-longjmp.patch"))) (define (make-mingw-pthreads-cross-toolchain target) "Create a cross-compilation toolchain package for TARGET" @@ -172,7 +173,7 @@ desirable for building Bitcoin Core release binaries." (pthreads-xlibc mingw-w64-x86_64-winpthreads) (pthreads-xgcc (make-gcc-with-pthreads (cross-gcc target - #:xgcc (make-gcc-without-newlib (make-ssp-fixed-gcc base-gcc)) + #:xgcc (make-ssp-fixed-gcc (make-mingw-w64-cross-gcc base-gcc)) #:xbinutils xbinutils #:libc pthreads-xlibc)))) ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and @@ -201,7 +202,7 @@ chain for " target " development.")) (define-public lief (package (name "python-lief") - (version "0.11.5") + (version "0.12.0") (source (origin (method git-fetch) @@ -211,7 +212,7 @@ chain for " target " development.")) (file-name (git-file-name name version)) (sha256 (base32 - "0qahjfg1n0x76ps2mbyljvws1l3qhkqvmxqbahps4qgywl2hbdkj")))) + "026jchj56q25v6gc0754dj9cj5hz5zaza8ij93y5ga94w20kzm9q")))) (build-system python-build-system) (native-inputs `(("cmake" ,cmake))) @@ -485,7 +486,7 @@ and endian independent.") (license license:expat))) (define-public python-signapple - (let ((commit "0777ce58e61b0e6be753a5f524149d6d47905186")) + (let ((commit "8a945a2e7583be2665cf3a6a89d665b70ecd1ab6")) (package (name "python-signapple") (version (git-version "0.1" "1" commit)) @@ -498,7 +499,7 @@ and endian independent.") (file-name (git-file-name name commit)) (sha256 (base32 - "19axspyyfqbrfw2r53c17mi9bvm8zsb39mz8v9h7c173qkm3x5ym")))) + "0fr1hangvfyiwflca6jg5g8zvg3jc9qr7vd2c12ff89pznf38dlg")))) (build-system python-build-system) (propagated-inputs `(("python-asn1crypto" ,python-asn1crypto) @@ -506,8 +507,7 @@ and endian independent.") ("python-certvalidator" ,python-certvalidator) ("python-elfesteem" ,python-elfesteem) ("python-requests" ,python-requests) - ("python-macholib" ,python-macholib) - ("libcrypto" ,openssl))) + ("python-macholib" ,python-macholib))) ;; There are no tests, but attempting to run python setup.py test leads to ;; problems, just disable the test (arguments '(#:tests? #f)) diff --git a/contrib/guix/patches/gcc-broken-longjmp.patch b/contrib/guix/patches/gcc-broken-longjmp.patch new file mode 100644 index 0000000000..1cfc0918b0 --- /dev/null +++ b/contrib/guix/patches/gcc-broken-longjmp.patch @@ -0,0 +1,68 @@ +commit eb5698897c52702498938592d7f76e67d126451f +Author: Eric Botcazou <ebotcazou@adacore.com> +Date: Wed May 5 22:48:51 2021 +0200 + + Fix PR target/100402 + + This is a regression for 64-bit Windows present from mainline down to the 9 + branch and introduced by the fix for PR target/99234. Again SEH, but with + a twist related to the way MinGW implements setjmp/longjmp, which turns out + to be piggybacked on SEH with recent versions of MinGW, i.e. the longjmp + performs a bona-fide unwinding of the stack, because it calls RtlUnwindEx + with the second argument initially passed to setjmp, which is the result of + __builtin_frame_address (0) in the MinGW header file: + + define setjmp(BUF) _setjmp((BUF), __builtin_frame_address (0)) + + This means that we directly expose the frame pointer to the SEH machinery + here (unlike with regular exception handling where we use an intermediate + CFA) and thus that we cannot do whatever we want with it. The old code + would leave it unaligned, i.e. not multiple of 16, whereas the new code + aligns it, but this breaks for some reason; at least it appears that a + .seh_setframe directive with 0 as second argument always works, so the + fix aligns it this way. + + gcc/ + PR target/100402 + * config/i386/i386.c (ix86_compute_frame_layout): For a SEH target, + always return the establisher frame for __builtin_frame_address (0). + gcc/testsuite/ + * gcc.c-torture/execute/20210505-1.c: New test. + +diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c +index 2f838840e96..06ad1b2274e 100644 +--- a/gcc/config/i386/i386.c ++++ b/gcc/config/i386/i386.c +@@ -6356,12 +6356,29 @@ ix86_compute_frame_layout (void) + area, see the SEH code in config/i386/winnt.c for the rationale. */ + frame->hard_frame_pointer_offset = frame->sse_reg_save_offset; + +- /* If we can leave the frame pointer where it is, do so. Also, return ++ /* If we can leave the frame pointer where it is, do so; however return + the establisher frame for __builtin_frame_address (0) or else if the +- frame overflows the SEH maximum frame size. */ ++ frame overflows the SEH maximum frame size. ++ ++ Note that the value returned by __builtin_frame_address (0) is quite ++ constrained, because setjmp is piggybacked on the SEH machinery with ++ recent versions of MinGW: ++ ++ # elif defined(__SEH__) ++ # if defined(__aarch64__) || defined(_ARM64_) ++ # define setjmp(BUF) _setjmp((BUF), __builtin_sponentry()) ++ # elif (__MINGW_GCC_VERSION < 40702) ++ # define setjmp(BUF) _setjmp((BUF), mingw_getsp()) ++ # else ++ # define setjmp(BUF) _setjmp((BUF), __builtin_frame_address (0)) ++ # endif ++ ++ and the second argument passed to _setjmp, if not null, is forwarded ++ to the TargetFrame parameter of RtlUnwindEx by longjmp (after it has ++ built an ExceptionRecord on the fly describing the setjmp buffer). */ + const HOST_WIDE_INT diff + = frame->stack_pointer_offset - frame->hard_frame_pointer_offset; +- if (diff <= 255) ++ if (diff <= 255 && !crtl->accesses_prior_frames) + { + /* The resulting diff will be a multiple of 16 lower than 255, + i.e. at most 240 as required by the unwind data structure. */ diff --git a/contrib/guix/patches/vmov-alignment.patch b/contrib/guix/patches/vmov-alignment.patch new file mode 100644 index 0000000000..072f76eafd --- /dev/null +++ b/contrib/guix/patches/vmov-alignment.patch @@ -0,0 +1,267 @@ +Description: Use unaligned VMOV instructions +Author: Stephen Kitt <skitt@debian.org> +Bug-Debian: https://bugs.debian.org/939559 + +Based on a patch originally by Claude Heiland-Allen <claude@mathr.co.uk> + +--- a/gcc/config/i386/sse.md ++++ b/gcc/config/i386/sse.md +@@ -1058,17 +1058,11 @@ + { + if (FLOAT_MODE_P (GET_MODE_INNER (<MODE>mode))) + { +- if (misaligned_operand (operands[1], <MODE>mode)) +- return "vmovu<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; +- else +- return "vmova<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; ++ return "vmovu<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; + } + else + { +- if (misaligned_operand (operands[1], <MODE>mode)) +- return "vmovdqu<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; +- else +- return "vmovdqa<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; ++ return "vmovdqu<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"; + } + } + [(set_attr "type" "ssemov") +@@ -1184,17 +1178,11 @@ + { + if (FLOAT_MODE_P (GET_MODE_INNER (<MODE>mode))) + { +- if (misaligned_operand (operands[0], <MODE>mode)) +- return "vmovu<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; +- else +- return "vmova<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; ++ return "vmovu<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; + } + else + { +- if (misaligned_operand (operands[0], <MODE>mode)) +- return "vmovdqu<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; +- else +- return "vmovdqa<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; ++ return "vmovdqu<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"; + } + } + [(set_attr "type" "ssemov") +@@ -7806,7 +7794,7 @@ + "TARGET_SSE && !(MEM_P (operands[0]) && MEM_P (operands[1]))" + "@ + %vmovlps\t{%1, %0|%q0, %1} +- %vmovaps\t{%1, %0|%0, %1} ++ %vmovups\t{%1, %0|%0, %1} + %vmovlps\t{%1, %d0|%d0, %q1}" + [(set_attr "type" "ssemov") + (set_attr "prefix" "maybe_vex") +@@ -13997,29 +13985,15 @@ + switch (<MODE>mode) + { + case E_V8DFmode: +- if (misaligned_operand (operands[2], <ssequartermode>mode)) +- return "vmovupd\t{%2, %x0|%x0, %2}"; +- else +- return "vmovapd\t{%2, %x0|%x0, %2}"; ++ return "vmovupd\t{%2, %x0|%x0, %2}"; + case E_V16SFmode: +- if (misaligned_operand (operands[2], <ssequartermode>mode)) +- return "vmovups\t{%2, %x0|%x0, %2}"; +- else +- return "vmovaps\t{%2, %x0|%x0, %2}"; ++ return "vmovups\t{%2, %x0|%x0, %2}"; + case E_V8DImode: +- if (misaligned_operand (operands[2], <ssequartermode>mode)) +- return which_alternative == 2 ? "vmovdqu64\t{%2, %x0|%x0, %2}" ++ return which_alternative == 2 ? "vmovdqu64\t{%2, %x0|%x0, %2}" + : "vmovdqu\t{%2, %x0|%x0, %2}"; +- else +- return which_alternative == 2 ? "vmovdqa64\t{%2, %x0|%x0, %2}" +- : "vmovdqa\t{%2, %x0|%x0, %2}"; + case E_V16SImode: +- if (misaligned_operand (operands[2], <ssequartermode>mode)) +- return which_alternative == 2 ? "vmovdqu32\t{%2, %x0|%x0, %2}" ++ return which_alternative == 2 ? "vmovdqu32\t{%2, %x0|%x0, %2}" + : "vmovdqu\t{%2, %x0|%x0, %2}"; +- else +- return which_alternative == 2 ? "vmovdqa32\t{%2, %x0|%x0, %2}" +- : "vmovdqa\t{%2, %x0|%x0, %2}"; + default: + gcc_unreachable (); + } +@@ -21225,63 +21199,27 @@ + switch (get_attr_mode (insn)) + { + case MODE_V16SF: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- return "vmovups\t{%1, %t0|%t0, %1}"; +- else +- return "vmovaps\t{%1, %t0|%t0, %1}"; ++ return "vmovups\t{%1, %t0|%t0, %1}"; + case MODE_V8DF: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- return "vmovupd\t{%1, %t0|%t0, %1}"; +- else +- return "vmovapd\t{%1, %t0|%t0, %1}"; ++ return "vmovupd\t{%1, %t0|%t0, %1}"; + case MODE_V8SF: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- return "vmovups\t{%1, %x0|%x0, %1}"; +- else +- return "vmovaps\t{%1, %x0|%x0, %1}"; ++ return "vmovups\t{%1, %x0|%x0, %1}"; + case MODE_V4DF: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- return "vmovupd\t{%1, %x0|%x0, %1}"; +- else +- return "vmovapd\t{%1, %x0|%x0, %1}"; ++ return "vmovupd\t{%1, %x0|%x0, %1}"; + case MODE_XI: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- { +- if (which_alternative == 2) +- return "vmovdqu\t{%1, %t0|%t0, %1}"; +- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) +- return "vmovdqu64\t{%1, %t0|%t0, %1}"; +- else +- return "vmovdqu32\t{%1, %t0|%t0, %1}"; +- } ++ if (which_alternative == 2) ++ return "vmovdqu\t{%1, %t0|%t0, %1}"; ++ else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) ++ return "vmovdqu64\t{%1, %t0|%t0, %1}"; + else +- { +- if (which_alternative == 2) +- return "vmovdqa\t{%1, %t0|%t0, %1}"; +- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) +- return "vmovdqa64\t{%1, %t0|%t0, %1}"; +- else +- return "vmovdqa32\t{%1, %t0|%t0, %1}"; +- } ++ return "vmovdqu32\t{%1, %t0|%t0, %1}"; + case MODE_OI: +- if (misaligned_operand (operands[1], <ssehalfvecmode>mode)) +- { +- if (which_alternative == 2) +- return "vmovdqu\t{%1, %x0|%x0, %1}"; +- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) +- return "vmovdqu64\t{%1, %x0|%x0, %1}"; +- else +- return "vmovdqu32\t{%1, %x0|%x0, %1}"; +- } ++ if (which_alternative == 2) ++ return "vmovdqu\t{%1, %x0|%x0, %1}"; ++ else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) ++ return "vmovdqu64\t{%1, %x0|%x0, %1}"; + else +- { +- if (which_alternative == 2) +- return "vmovdqa\t{%1, %x0|%x0, %1}"; +- else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8) +- return "vmovdqa64\t{%1, %x0|%x0, %1}"; +- else +- return "vmovdqa32\t{%1, %x0|%x0, %1}"; +- } ++ return "vmovdqu32\t{%1, %x0|%x0, %1}"; + default: + gcc_unreachable (); + } +--- a/gcc/config/i386/i386.c ++++ b/gcc/config/i386/i386.c +@@ -4981,13 +4981,13 @@ + switch (type) + { + case opcode_int: +- opcode = misaligned_p ? "vmovdqu32" : "vmovdqa32"; ++ opcode = "vmovdqu32"; + break; + case opcode_float: +- opcode = misaligned_p ? "vmovups" : "vmovaps"; ++ opcode = "vmovups"; + break; + case opcode_double: +- opcode = misaligned_p ? "vmovupd" : "vmovapd"; ++ opcode = "vmovupd"; + break; + } + } +@@ -4996,16 +4996,16 @@ + switch (scalar_mode) + { + case E_SFmode: +- opcode = misaligned_p ? "%vmovups" : "%vmovaps"; ++ opcode = "%vmovups"; + break; + case E_DFmode: +- opcode = misaligned_p ? "%vmovupd" : "%vmovapd"; ++ opcode = "%vmovupd"; + break; + case E_TFmode: + if (evex_reg_p) +- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; ++ opcode = "vmovdqu64"; + else +- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; ++ opcode = "%vmovdqu"; + break; + default: + gcc_unreachable (); +@@ -5017,48 +5017,32 @@ + { + case E_QImode: + if (evex_reg_p) +- opcode = (misaligned_p +- ? (TARGET_AVX512BW +- ? "vmovdqu8" +- : "vmovdqu64") +- : "vmovdqa64"); ++ opcode = TARGET_AVX512BW ? "vmovdqu8" : "vmovdqu64"; + else +- opcode = (misaligned_p +- ? (TARGET_AVX512BW +- ? "vmovdqu8" +- : "%vmovdqu") +- : "%vmovdqa"); ++ opcode = TARGET_AVX512BW ? "vmovdqu8" : "%vmovdqu"; + break; + case E_HImode: + if (evex_reg_p) +- opcode = (misaligned_p +- ? (TARGET_AVX512BW +- ? "vmovdqu16" +- : "vmovdqu64") +- : "vmovdqa64"); ++ opcode = TARGET_AVX512BW ? "vmovdqu16" : "vmovdqu64"; + else +- opcode = (misaligned_p +- ? (TARGET_AVX512BW +- ? "vmovdqu16" +- : "%vmovdqu") +- : "%vmovdqa"); ++ opcode = TARGET_AVX512BW ? "vmovdqu16" : "%vmovdqu"; + break; + case E_SImode: + if (evex_reg_p) +- opcode = misaligned_p ? "vmovdqu32" : "vmovdqa32"; ++ opcode = "vmovdqu32"; + else +- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; ++ opcode = "%vmovdqu"; + break; + case E_DImode: + case E_TImode: + case E_OImode: + if (evex_reg_p) +- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; ++ opcode = "vmovdqu64"; + else +- opcode = misaligned_p ? "%vmovdqu" : "%vmovdqa"; ++ opcode = "%vmovdqu"; + break; + case E_XImode: +- opcode = misaligned_p ? "vmovdqu64" : "vmovdqa64"; ++ opcode = "vmovdqu64"; + break; + default: + gcc_unreachable (); diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index 441b5da764..7510204bb1 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -20,49 +20,9 @@ from collections import namedtuple settings = {} -def hex_switchEndian(s): - """ Switches the endianness of a hex string (in pairs of hex chars) """ - pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] - return b''.join(pairList[::-1]).decode() - -def uint32(x): - return x & 0xffffffff - -def bytereverse(x): - return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) | - (((x) >> 8) & 0x0000ff00) | ((x) >> 24) )) - -def bufreverse(in_buf): - out_words = [] - for i in range(0, len(in_buf), 4): - word = struct.unpack('@I', in_buf[i:i+4])[0] - out_words.append(struct.pack('@I', bytereverse(word))) - return b''.join(out_words) - -def wordreverse(in_buf): - out_words = [] - for i in range(0, len(in_buf), 4): - out_words.append(in_buf[i:i+4]) - out_words.reverse() - return b''.join(out_words) - -def calc_hdr_hash(blk_hdr): - hash1 = hashlib.sha256() - hash1.update(blk_hdr) - hash1_o = hash1.digest() - - hash2 = hashlib.sha256() - hash2.update(hash1_o) - hash2_o = hash2.digest() - - return hash2_o - def calc_hash_str(blk_hdr): - hash = calc_hdr_hash(blk_hdr) - hash = bufreverse(hash) - hash = wordreverse(hash) - hash_str = hash.hex() - return hash_str + blk_hdr_hash = hashlib.sha256(hashlib.sha256(blk_hdr).digest()).digest() + return blk_hdr_hash[::-1].hex() def get_blk_dt(blk_hdr): members = struct.unpack("<I", blk_hdr[68:68+4]) @@ -78,7 +38,7 @@ def get_block_hashes(settings): for line in f: line = line.rstrip() if settings['rev_hash_bytes'] == 'true': - line = hex_switchEndian(line) + line = bytes.fromhex(line)[::-1].hex() blkindex.append(line) print("Read " + str(len(blkindex)) + " hashes") diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py index fed6e665b8..0a316eb818 100755 --- a/contrib/linearize/linearize-hashes.py +++ b/contrib/linearize/linearize-hashes.py @@ -17,11 +17,6 @@ import os.path settings = {} -def hex_switchEndian(s): - """ Switches the endianness of a hex string (in pairs of hex chars) """ - pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] - return b''.join(pairList[::-1]).decode() - class BitcoinRPC: def __init__(self, host, port, username, password): authpair = "%s:%s" % (username, password) @@ -85,7 +80,7 @@ def get_block_hashes(settings, max_blocks_per_call=10000): sys.exit(1) assert(resp_obj['id'] == x) # assume replies are in-sequence if settings['rev_hash_bytes'] == 'true': - resp_obj['result'] = hex_switchEndian(resp_obj['result']) + resp_obj['result'] = bytes.fromhex(resp_obj['result'])[::-1].hex() print(resp_obj['result']) height += num_blocks diff --git a/contrib/macdeploy/detached-sig-apply.sh b/contrib/macdeploy/detached-sig-apply.sh deleted file mode 100755 index c7296387eb..0000000000 --- a/contrib/macdeploy/detached-sig-apply.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -# Copyright (c) 2014-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C -set -e - -UNSIGNED="$1" -SIGNATURE="$2" -ROOTDIR=dist -OUTDIR=signed-app -SIGNAPPLE=signapple - -if [ -z "$UNSIGNED" ]; then - echo "usage: $0 <unsigned app> <signature>" - exit 1 -fi - -if [ -z "$SIGNATURE" ]; then - echo "usage: $0 <unsigned app> <signature>" - exit 1 -fi - -${SIGNAPPLE} apply "${UNSIGNED}" "${SIGNATURE}" -mv ${ROOTDIR} ${OUTDIR} -echo "Signed: ${OUTDIR}" diff --git a/contrib/macdeploy/macdeployqtplus b/contrib/macdeploy/macdeployqtplus index 0455a137f1..cc24e0317b 100755 --- a/contrib/macdeploy/macdeployqtplus +++ b/contrib/macdeploy/macdeployqtplus @@ -544,6 +544,22 @@ ds.close() if platform.system() == "Darwin": subprocess.check_call(f"codesign --deep --force --sign - {target}", shell=True) +print("+ Installing background.tiff +") + +bg_path = os.path.join('dist', '.background', 'background.tiff') +os.mkdir(os.path.dirname(bg_path)) + +tiff_path = os.path.join('contrib', 'macdeploy', 'background.tiff') +shutil.copy2(tiff_path, bg_path) + +# ------------------------------------------------ + +print("+ Generating symlink for /Applications +") + +os.symlink("/Applications", os.path.join('dist', "Applications")) + +# ------------------------------------------------ + if config.dmg is not None: print("+ Preparing .dmg disk image +") @@ -567,19 +583,6 @@ if config.dmg is not None: print("Attaching temp image...") output = run(["hdiutil", "attach", tempname, "-readwrite"], check=True, universal_newlines=True, stdout=PIPE).stdout - m = re.search(r"/Volumes/(.+$)", output) - disk_root = m.group(0) - - print("+ Applying fancy settings +") - - bg_path = os.path.join(disk_root, ".background", os.path.basename('background.tiff')) - os.mkdir(os.path.dirname(bg_path)) - if verbose: - print('background.tiff', "->", bg_path) - shutil.copy2('contrib/macdeploy/background.tiff', bg_path) - - os.symlink("/Applications", os.path.join(disk_root, "Applications")) - print("+ Finalizing .dmg disk image +") run(["hdiutil", "detach", f"/Volumes/{appname}"], universal_newlines=True) diff --git a/contrib/seeds/makeseeds.py b/contrib/seeds/makeseeds.py index 2b377f6c01..78eb04a836 100755 --- a/contrib/seeds/makeseeds.py +++ b/contrib/seeds/makeseeds.py @@ -10,18 +10,16 @@ import re import sys import dns.resolver import collections +from typing import List, Dict, Union NSEEDS=512 -MAX_SEEDS_PER_ASN=2 - -MIN_BLOCKS = 337600 - -# These are hosts that have been observed to be behaving strangely (e.g. -# aggressively connecting to every node). -with open("suspicious_hosts.txt", mode="r", encoding="utf-8") as f: - SUSPICIOUS_HOSTS = {s.strip() for s in f if s.strip()} +MAX_SEEDS_PER_ASN = { + 'ipv4': 2, + 'ipv6': 10, +} +MIN_BLOCKS = 730000 PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$") PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$") @@ -40,9 +38,13 @@ PATTERN_AGENT = re.compile( r"23.99" r")") -def parseline(line): +def parseline(line: str) -> Union[dict, None]: + """ Parses a line from `seeds_main.txt` into a dictionary of details for that line. + or `None`, if the line could not be parsed. + """ sline = line.split() if len(sline) < 11: + # line too short to be valid, skip it. return None m = PATTERN_IPV4.match(sline[0]) sortkey = None @@ -107,25 +109,26 @@ def parseline(line): 'sortkey': sortkey, } -def dedup(ips): - '''deduplicate by address,port''' +def dedup(ips: List[Dict]) -> List[Dict]: + """ Remove duplicates from `ips` where multiple ips share address and port. """ d = {} for ip in ips: d[ip['ip'],ip['port']] = ip return list(d.values()) -def filtermultiport(ips): - '''Filter out hosts with more nodes per IP''' +def filtermultiport(ips: List[Dict]) -> List[Dict]: + """ Filter out hosts with more nodes per IP""" hist = collections.defaultdict(list) for ip in ips: hist[ip['sortkey']].append(ip) return [value[0] for (key,value) in list(hist.items()) if len(value)==1] -def lookup_asn(net, ip): - ''' - Look up the asn for an IP (4 or 6) address by querying cymru.com, or None - if it could not be found. - ''' +def lookup_asn(net: str, ip: str) -> Union[int, None]: + """ Look up the asn for an `ip` address by querying cymru.com + on network `net` (e.g. ipv4 or ipv6). + + Returns in integer ASN or None if it could not be found. + """ try: if net == 'ipv4': ipaddr = ip @@ -147,20 +150,33 @@ def lookup_asn(net, ip): return None # Based on Greg Maxwell's seed_filter.py -def filterbyasn(ips, max_per_asn, max_per_net): +def filterbyasn(ips: List[Dict], max_per_asn: Dict, max_per_net: int) -> List[Dict]: + """ Prunes `ips` by + (a) trimming ips to have at most `max_per_net` ips from each net (e.g. ipv4, ipv6); and + (b) trimming ips to have at most `max_per_asn` ips from each asn in each net. + """ # Sift out ips by type ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']] ips_onion = [ip for ip in ips if ip['net'] == 'onion'] # Filter IPv46 by ASN, and limit to max_per_net per network result = [] - net_count = collections.defaultdict(int) - asn_count = collections.defaultdict(int) - for ip in ips_ipv46: + net_count: Dict[str, int] = collections.defaultdict(int) + asn_count: Dict[int, int] = collections.defaultdict(int) + + for i, ip in enumerate(ips_ipv46): + if i % 10 == 0: + # give progress update + print(f"{i:6d}/{len(ips_ipv46)} [{100*i/len(ips_ipv46):04.1f}%]\r", file=sys.stderr, end='', flush=True) + if net_count[ip['net']] == max_per_net: + # do not add this ip as we already too many + # ips from this network continue asn = lookup_asn(ip['net'], ip['ip']) - if asn is None or asn_count[asn] == max_per_asn: + if asn is None or asn_count[asn] == max_per_asn[ip['net']]: + # do not add this ip as we already have too many + # ips from this ASN on this network continue asn_count[asn] += 1 net_count[ip['net']] += 1 @@ -170,35 +186,33 @@ def filterbyasn(ips, max_per_asn, max_per_net): result.extend(ips_onion[0:max_per_net]) return result -def ip_stats(ips): - hist = collections.defaultdict(int) +def ip_stats(ips: List[Dict]) -> str: + """ Format and return pretty string from `ips`. """ + hist: Dict[str, int] = collections.defaultdict(int) for ip in ips: if ip is not None: hist[ip['net']] += 1 - return '%6d %6d %6d' % (hist['ipv4'], hist['ipv6'], hist['onion']) + return f"{hist['ipv4']:6d} {hist['ipv6']:6d} {hist['onion']:6d}" def main(): lines = sys.stdin.readlines() ips = [parseline(line) for line in lines] print('\x1b[7m IPv4 IPv6 Onion Pass \x1b[0m', file=sys.stderr) - print('%s Initial' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Initial', file=sys.stderr) # Skip entries with invalid address. ips = [ip for ip in ips if ip is not None] - print('%s Skip entries with invalid address' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Skip entries with invalid address', file=sys.stderr) # Skip duplicates (in case multiple seeds files were concatenated) ips = dedup(ips) - print('%s After removing duplicates' % (ip_stats(ips)), file=sys.stderr) - # Skip entries from suspicious hosts. - ips = [ip for ip in ips if ip['ip'] not in SUSPICIOUS_HOSTS] - print('%s Skip entries from suspicious hosts' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} After removing duplicates', file=sys.stderr) # Enforce minimal number of blocks. ips = [ip for ip in ips if ip['blocks'] >= MIN_BLOCKS] - print('%s Enforce minimal number of blocks' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Enforce minimal number of blocks', file=sys.stderr) # Require service bit 1. ips = [ip for ip in ips if (ip['service'] & 1) == 1] - print('%s Require service bit 1' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Require service bit 1', file=sys.stderr) # Require at least 50% 30-day uptime for clearnet, 10% for onion. req_uptime = { 'ipv4': 50, @@ -206,18 +220,18 @@ def main(): 'onion': 10, } ips = [ip for ip in ips if ip['uptime'] > req_uptime[ip['net']]] - print('%s Require minimum uptime' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Require minimum uptime', file=sys.stderr) # Require a known and recent user agent. ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])] - print('%s Require a known and recent user agent' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Require a known and recent user agent', file=sys.stderr) # Sort by availability (and use last success as tie breaker) ips.sort(key=lambda x: (x['uptime'], x['lastsuccess'], x['ip']), reverse=True) # Filter out hosts with multiple bitcoin ports, these are likely abusive ips = filtermultiport(ips) - print('%s Filter out hosts with multiple bitcoin ports' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Filter out hosts with multiple bitcoin ports', file=sys.stderr) # Look up ASNs and limit results, both per ASN and globally. ips = filterbyasn(ips, MAX_SEEDS_PER_ASN, NSEEDS) - print('%s Look up ASNs and limit results per ASN and per net' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Look up ASNs and limit results per ASN and per net', file=sys.stderr) # Sort the results by IP address (for deterministic output). ips.sort(key=lambda x: (x['net'], x['sortkey'])) for ip in ips: diff --git a/contrib/seeds/suspicious_hosts.txt b/contrib/seeds/suspicious_hosts.txt deleted file mode 100644 index 13385cc816..0000000000 --- a/contrib/seeds/suspicious_hosts.txt +++ /dev/null @@ -1,16 +0,0 @@ -130.211.129.106 -148.251.238.178 -176.9.46.6 -178.63.107.226 -54.173.72.127 -54.174.10.182 -54.183.64.54 -54.194.231.211 -54.66.214.167 -54.66.220.137 -54.67.33.14 -54.77.251.214 -54.94.195.96 -54.94.200.247 -83.81.130.26 -88.198.17.7
\ No newline at end of file diff --git a/contrib/signet/miner b/contrib/signet/miner index 012bd6cc31..b366b98e2d 100755 --- a/contrib/signet/miner +++ b/contrib/signet/miner @@ -8,7 +8,7 @@ import base64 import json import logging import math -import os.path +import os import re import struct import sys @@ -493,10 +493,11 @@ def do_generate(args): logging.debug("Mining block delta=%s start=%s mine=%s", seconds_to_hms(mine_time-bestheader["time"]), mine_time, is_mine) mined_blocks += 1 psbt = generate_psbt(tmpl, reward_spk, blocktime=mine_time) - psbt_signed = json.loads(args.bcli("-stdin", "walletprocesspsbt", input=psbt.encode('utf8'))) + input_stream = os.linesep.join([psbt, "true", "ALL"]).encode('utf8') + psbt_signed = json.loads(args.bcli("-stdin", "walletprocesspsbt", input=input_stream)) if not psbt_signed.get("complete",False): logging.debug("Generated PSBT: %s" % (psbt,)) - sys.stderr.write("PSBT signing failed") + sys.stderr.write("PSBT signing failed\n") return 1 block, signet_solution = do_decode_psbt(psbt_signed["psbt"]) block = finish_block(block, signet_solution, args.grind_cmd) diff --git a/contrib/testgen/README.md b/contrib/testgen/README.md index fcc5a378e2..2f0288df16 100644 --- a/contrib/testgen/README.md +++ b/contrib/testgen/README.md @@ -2,7 +2,7 @@ Utilities to generate test vectors for the data-driven Bitcoin tests. -Usage: +To use inside a scripted-diff (or just execute directly): - PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py valid 70 > ../../src/test/data/key_io_valid.json - PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py invalid 70 > ../../src/test/data/key_io_invalid.json + ./gen_key_io_test_vectors.py valid 70 > ../../src/test/data/key_io_valid.json + ./gen_key_io_test_vectors.py invalid 70 > ../../src/test/data/key_io_invalid.json diff --git a/contrib/testgen/base58.py b/contrib/testgen/base58.py deleted file mode 100644 index 87341ccf96..0000000000 --- a/contrib/testgen/base58.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright (c) 2012-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' -Bitcoin base58 encoding and decoding. - -Based on https://bitcointalk.org/index.php?topic=1026.0 (public domain) -''' -import hashlib - -# for compatibility with following code... -class SHA256: - new = hashlib.sha256 - -if str != bytes: - # Python 3.x - def ord(c): - return c - def chr(n): - return bytes( (n,) ) - -__b58chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' -__b58base = len(__b58chars) -b58chars = __b58chars - -def b58encode(v): - """ encode v, which is a string of bytes, to base58. - """ - long_value = 0 - for (i, c) in enumerate(v[::-1]): - if isinstance(c, str): - c = ord(c) - long_value += (256**i) * c - - result = '' - while long_value >= __b58base: - div, mod = divmod(long_value, __b58base) - result = __b58chars[mod] + result - long_value = div - result = __b58chars[long_value] + result - - # Bitcoin does a little leading-zero-compression: - # leading 0-bytes in the input become leading-1s - nPad = 0 - for c in v: - if c == 0: - nPad += 1 - else: - break - - return (__b58chars[0]*nPad) + result - -def b58decode(v, length = None): - """ decode v into a string of len bytes - """ - long_value = 0 - for i, c in enumerate(v[::-1]): - pos = __b58chars.find(c) - assert pos != -1 - long_value += pos * (__b58base**i) - - result = bytes() - while long_value >= 256: - div, mod = divmod(long_value, 256) - result = chr(mod) + result - long_value = div - result = chr(long_value) + result - - nPad = 0 - for c in v: - if c == __b58chars[0]: - nPad += 1 - continue - break - - result = bytes(nPad) + result - if length is not None and len(result) != length: - return None - - return result - -def checksum(v): - """Return 32-bit checksum based on SHA256""" - return SHA256.new(SHA256.new(v).digest()).digest()[0:4] - -def b58encode_chk(v): - """b58encode a string, with 32-bit checksum""" - return b58encode(v + checksum(v)) - -def b58decode_chk(v): - """decode a base58 string, check and remove checksum""" - result = b58decode(v) - if result is None: - return None - if result[-4:] == checksum(result[:-4]): - return result[:-4] - else: - return None - -def get_bcaddress_version(strAddress): - """ Returns None if strAddress is invalid. Otherwise returns integer version of address. """ - addr = b58decode_chk(strAddress) - if addr is None or len(addr)!=21: - return None - version = addr[0] - return ord(version) - -if __name__ == '__main__': - # Test case (from http://gitorious.org/bitcoin/python-base58.git) - assert get_bcaddress_version('15VjRaDX9zpbA8LVnbrCAFzrVzN7ixHNsC') == 0 - _ohai = 'o hai'.encode('ascii') - _tmp = b58encode(_ohai) - assert _tmp == 'DYB3oMS' - assert b58decode(_tmp, 5) == _ohai - print("Tests passed") diff --git a/contrib/testgen/gen_key_io_test_vectors.py b/contrib/testgen/gen_key_io_test_vectors.py index eafaaaeceb..7bfb1d76a8 100755 --- a/contrib/testgen/gen_key_io_test_vectors.py +++ b/contrib/testgen/gen_key_io_test_vectors.py @@ -1,21 +1,21 @@ #!/usr/bin/env python3 -# Copyright (c) 2012-2021 The Bitcoin Core developers +# Copyright (c) 2012-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Generate valid and invalid base58/bech32(m) address and private key test vectors. - -Usage: - PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py valid 70 > ../../src/test/data/key_io_valid.json - PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py invalid 70 > ../../src/test/data/key_io_invalid.json ''' -# 2012 Wladimir J. van der Laan -# Released under MIT License -import os + from itertools import islice -from base58 import b58encode_chk, b58decode_chk, b58chars +import os import random -from segwit_addr import bech32_encode, decode_segwit_address, convertbits, CHARSET, Encoding +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), '../../test/functional')) + +from test_framework.address import base58_to_byte, byte_to_base58, b58chars # noqa: E402 +from test_framework.script import OP_0, OP_1, OP_2, OP_3, OP_16, OP_DUP, OP_EQUAL, OP_EQUALVERIFY, OP_HASH160, OP_CHECKSIG # noqa: E402 +from test_framework.segwit_addr import bech32_encode, decode_segwit_address, convertbits, CHARSET, Encoding # noqa: E402 # key types PUBKEY_ADDRESS = 0 @@ -29,16 +29,6 @@ PRIVKEY_TEST = 239 PRIVKEY_REGTEST = 239 # script -OP_0 = 0x00 -OP_1 = 0x51 -OP_2 = 0x52 -OP_3 = 0x53 -OP_16 = 0x60 -OP_DUP = 0x76 -OP_EQUAL = 0x87 -OP_EQUALVERIFY = 0x88 -OP_HASH160 = 0xa9 -OP_CHECKSIG = 0xac pubkey_prefix = (OP_DUP, OP_HASH160, 20) pubkey_suffix = (OP_EQUALVERIFY, OP_CHECKSIG) script_prefix = (OP_HASH160, 20) @@ -114,8 +104,10 @@ def is_valid(v): '''Check vector v for validity''' if len(set(v) - set(b58chars)) > 0: return is_valid_bech32(v) - result = b58decode_chk(v) - if result is None: + try: + payload, version = base58_to_byte(v) + result = bytes([version]) + payload + except ValueError: # thrown if checksum doesn't match return is_valid_bech32(v) for template in templates: prefix = bytearray(template[0]) @@ -135,18 +127,19 @@ def is_valid_bech32(v): def gen_valid_base58_vector(template): '''Generate valid base58 vector''' prefix = bytearray(template[0]) - payload = bytearray(os.urandom(template[1])) + payload = rand_bytes(size=template[1]) suffix = bytearray(template[2]) dst_prefix = bytearray(template[4]) dst_suffix = bytearray(template[5]) - rv = b58encode_chk(prefix + payload + suffix) + assert len(prefix) == 1 + rv = byte_to_base58(payload + suffix, prefix[0]) return rv, dst_prefix + payload + dst_suffix def gen_valid_bech32_vector(template): '''Generate valid bech32 vector''' hrp = template[0] witver = template[1] - witprog = bytearray(os.urandom(template[2])) + witprog = rand_bytes(size=template[2]) encoding = template[4] dst_prefix = bytearray(template[5]) rv = bech32_encode(encoding, hrp, [witver] + convertbits(witprog, 8, 5)) @@ -176,21 +169,22 @@ def gen_invalid_base58_vector(template): corrupt_suffix = randbool(0.2) if corrupt_prefix: - prefix = os.urandom(1) + prefix = rand_bytes(size=1) else: prefix = bytearray(template[0]) if randomize_payload_size: - payload = os.urandom(max(int(random.expovariate(0.5)), 50)) + payload = rand_bytes(size=max(int(random.expovariate(0.5)), 50)) else: - payload = os.urandom(template[1]) + payload = rand_bytes(size=template[1]) if corrupt_suffix: - suffix = os.urandom(len(template[2])) + suffix = rand_bytes(size=len(template[2])) else: suffix = bytearray(template[2]) - val = b58encode_chk(prefix + payload + suffix) + assert len(prefix) == 1 + val = byte_to_base58(payload + suffix, prefix[0]) if random.randint(0,10)<1: # line corruption if randbool(): # add random character to end val += random.choice(b58chars) @@ -206,7 +200,7 @@ def gen_invalid_bech32_vector(template): to_upper = randbool(0.1) hrp = template[0] witver = template[1] - witprog = bytearray(os.urandom(template[2])) + witprog = rand_bytes(size=template[2]) encoding = template[3] if no_data: @@ -236,6 +230,9 @@ def randbool(p = 0.5): '''Return True with P(p)''' return random.random() < p +def rand_bytes(*, size): + return bytearray(random.getrandbits(8) for _ in range(size)) + def gen_invalid_vectors(): '''Generate invalid test vectors''' # start with some manual edge-cases @@ -250,9 +247,9 @@ def gen_invalid_vectors(): yield val, if __name__ == '__main__': - import sys import json iters = {'valid':gen_valid_vectors, 'invalid':gen_invalid_vectors} + random.seed(42) try: uiter = iters[sys.argv[1]] except IndexError: diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index 99ca305fe7..6efe49254b 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -113,16 +113,6 @@ fun:GetCoin } { - Suppress boost warning - Memcheck:Leak - fun:_Znwm - ... - fun:_ZN5boost9unit_test9framework5state17execute_test_treeEmjPKNS2_23random_generator_helperE - fun:_ZN5boost9unit_test9framework3runEmb - fun:_ZN5boost9unit_test14unit_test_mainEPFbvEiPPc - fun:main -} -{ Suppress LogInstance still reachable memory warning Memcheck:Leak match-leak-kinds: reachable |