diff options
30 files changed, 218 insertions, 291 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index ead597bf22..0507d664f1 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,12 +1,5 @@ ### Global defaults -timeout_in: 120m # https://cirrus-ci.org/faq/#instance-timed-out -container: - # https://cirrus-ci.org/faq/#are-there-any-limits - # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel - cpu: 2 - memory: 8G # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers - kvm: true # Use kvm to avoid spurious CI failures in the default virtualization cluster, see https://github.com/bitcoin/bitcoin/issues/20093 env: PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y" MAKEJOBS: "-j4" @@ -15,11 +8,27 @@ env: CCACHE_SIZE: "200M" CCACHE_DIR: "/tmp/ccache_dir" -### Global task template - # https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks -global_task_template: &GLOBAL_TASK_TEMPLATE +base_template: &BASE_TEMPLATE skip: $CIRRUS_REPO_FULL_NAME == "bitcoin-core/gui" && $CIRRUS_PR == "" # No need to run on the read-only mirror, unless it is a PR. https://cirrus-ci.org/guide/writing-tasks/#conditional-task-execution + merge_base_script: + - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi + - bash -c "$PACKAGE_MANAGER_INSTALL git" + - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH + - git config --global user.email "ci@ci.ci" + - git config --global user.name "ci" + - git merge FETCH_HEAD # Merge base to detect silent merge conflicts + stateful: false # https://cirrus-ci.org/guide/writing-tasks/#stateful-tasks + +global_task_template: &GLOBAL_TASK_TEMPLATE + << : *BASE_TEMPLATE + timeout_in: 120m # https://cirrus-ci.org/faq/#instance-timed-out + container: + # https://cirrus-ci.org/faq/#are-there-any-limits + # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel + cpu: 2 + memory: 8G # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers + kvm: true # Use kvm to avoid spurious CI failures in the default virtualization cluster, see https://github.com/bitcoin/bitcoin/issues/20093 ccache_cache: folder: "/tmp/ccache_dir" depends_built_cache: @@ -28,16 +37,14 @@ global_task_template: &GLOBAL_TASK_TEMPLATE folder: "/tmp/cirrus-ci-build/depends/sdk-sources" depends_releases_cache: folder: "/tmp/cirrus-ci-build/releases" - merge_base_script: - - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi - - bash -c "$PACKAGE_MANAGER_INSTALL git" - - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH - - git config --global user.email "ci@ci.ci" - - git config --global user.name "ci" - - git merge FETCH_HEAD # Merge base to detect silent merge conflicts ci_script: - ./ci/test_run_all.sh +compute_credits_template: &CREDITS_TEMPLATE + # https://cirrus-ci.org/pricing/#compute-credits + # Only use credits for pull requests to the main repo + use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'bitcoin/bitcoin' && $CIRRUS_PR != "" + #task: # name: "Windows" # windows_container: @@ -55,6 +62,26 @@ global_task_template: &GLOBAL_TASK_TEMPLATE # - choco install python --version=3.7.7 -y task: + name: 'lint [bionic]' + << : *BASE_TEMPLATE + container: + image: ubuntu:bionic # For python 3.6, oldest supported version according to doc/dependencies.md + cpu: 1 + memory: 1G + # For faster CI feedback, immediately schedule the linters + << : *CREDITS_TEMPLATE + setup_script: + - set -o errexit; source ./ci/test/00_setup_env.sh + before_install_script: + - set -o errexit; source ./ci/test/03_before_install.sh + install_script: + - set -o errexit; source ./ci/lint/04_install.sh + before_script: + - set -o errexit; source ./ci/lint/05_before_script.sh + lint_script: + - set -o errexit; source ./ci/lint/06_script.sh + +task: name: 'ARM [unit tests, no functional tests] [buster]' << : *GLOBAL_TASK_TEMPLATE container: @@ -81,8 +108,8 @@ task: task: name: '[previous releases, uses qt5 dev package and some depends packages] [unsigned char] [bionic]' - # For faster CI feedback, immediately schedule a task that compiles most modules. https://cirrus-ci.org/pricing/#compute-credits - use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'bitcoin/bitcoin' + # For faster CI feedback, immediately schedule a task that compiles most modules + << : *CREDITS_TEMPLATE << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:bionic diff --git a/.gitignore b/.gitignore index 5726b18928..810ef5db6b 100644 --- a/.gitignore +++ b/.gitignore @@ -76,6 +76,7 @@ src/qt/bitcoin-qt.includes *.log *.trs *.dmg +*.iso *.json.h *.raw.h diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index dd76aaacaf..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,66 +0,0 @@ -# Travis caches can be manually removed if necessary. This is one of the very -# few manual operations that is possible with Travis, and it can be done by a -# Bitcoin Core GitHub member via the Travis web interface [0]. -# -# Travis CI uploads the cache after the script phase of the build [1]. -# However, the build is terminated without saving the cache if it takes over -# 50 minutes [2]. Thus, if we spent too much time in early build stages, fail -# with an error and save the cache. -# -# [0] https://travis-ci.org/bitcoin/bitcoin/caches -# [1] https://docs.travis-ci.com/user/caching/#build-phases -# [2] https://docs.travis-ci.com/user/customizing-the-build#build-timeouts - -version: ~> 1.0 - -dist: bionic -os: linux -language: minimal -arch: amd64 -cache: - directories: - - $TRAVIS_BUILD_DIR/depends/built - - $TRAVIS_BUILD_DIR/depends/sdk-sources - - $TRAVIS_BUILD_DIR/ci/scratch/.ccache - - $TRAVIS_BUILD_DIR/releases/$HOST -stages: - - lint - - test -env: - global: - - CI_RETRY_EXE="travis_retry" - - CACHE_ERR_MSG="Error! Initial build successful, but not enough time remains to run later build stages and tests. See https://docs.travis-ci.com/user/customizing-the-build#build-timeouts . Please manually re-run this job by using the travis restart button. The next run should not time out because the build cache has been saved." -before_install: - - set -o errexit; source ./ci/test/00_setup_env.sh - - set -o errexit; source ./ci/test/03_before_install.sh -install: - - set -o errexit; source ./ci/test/04_install.sh -before_script: - # Temporary workaround for https://github.com/bitcoin/bitcoin/issues/16368 - - for i in {1..4}; do echo "$(sleep 500)" ; done & - - set -o errexit; source ./ci/test/05_before_script.sh &> "/dev/null" -script: - - export CONTINUE=1 - - if [ $SECONDS -gt 1200 ]; then export CONTINUE=0; fi # Likely the depends build took very long - - if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi # continue on repos with extended build time (90 minutes) - - if [ $CONTINUE = "1" ]; then set -o errexit; source ./ci/test/06_script_a.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi - - if [[ $SECONDS -gt 50*60-$EXPECTED_TESTS_DURATION_IN_SECONDS ]]; then export CONTINUE=0; fi - - if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi # continue on repos with extended build time (90 minutes) - - if [ $CONTINUE = "1" ]; then set -o errexit; source ./ci/test/06_script_b.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi -after_script: - - echo $TRAVIS_COMMIT_RANGE -jobs: - include: - - - stage: lint - name: 'lint' - env: - cache: pip - language: python - python: '3.6' # Oldest supported version according to doc/dependencies.md - install: - - set -o errexit; source ./ci/lint/04_install.sh - before_script: - - set -o errexit; source ./ci/lint/05_before_script.sh - script: - - set -o errexit; source ./ci/lint/06_script.sh diff --git a/Makefile.am b/Makefile.am index 958d68061f..6573f91d97 100644 --- a/Makefile.am +++ b/Makefile.am @@ -35,6 +35,7 @@ space := $(empty) $(empty) OSX_APP=Bitcoin-Qt.app OSX_VOLNAME = $(subst $(space),-,$(PACKAGE_NAME)) OSX_DMG = $(OSX_VOLNAME).dmg +OSX_TEMP_ISO = $(OSX_DMG:.dmg=).temp.iso OSX_BACKGROUND_SVG=background.svg OSX_BACKGROUND_IMAGE=background.tiff OSX_BACKGROUND_IMAGE_DPIS=36 72 @@ -135,8 +136,12 @@ $(APP_DIST_DIR)/Applications: $(APP_DIST_EXTRAS): $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt -$(OSX_DMG): $(APP_DIST_EXTRAS) - $(GENISOIMAGE) -no-cache-inodes -D -l -probe -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -apple -o $@ dist +.INTERMEDIATE: $(OSX_TEMP_ISO) +$(OSX_TEMP_ISO): $(APP_DIST_EXTRAS) + $(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ dist + +$(OSX_DMG): $(OSX_TEMP_ISO) + $(DMG) dmg "$<" "$@" dpi%.$(OSX_BACKGROUND_IMAGE): contrib/macdeploy/$(OSX_BACKGROUND_SVG) sed 's/PACKAGE_NAME/$(PACKAGE_NAME)/' < "$<" | $(RSVG_CONVERT) -f png -d $* -p $* | $(IMAGEMAGICK_CONVERT) - $@ diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index fae424051d..43d33b3f3b 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -6,14 +6,14 @@ export LC_ALL=C -travis_retry sudo apt update && sudo apt install -y clang-format-9 -sudo update-alternatives --install /usr/bin/clang-format clang-format $(which clang-format-9 ) 100 -sudo update-alternatives --install /usr/bin/clang-format-diff clang-format-diff $(which clang-format-diff-9) 100 +${CI_RETRY_EXE} apt update && apt install -y clang-format-9 python3-pip curl git +update-alternatives --install /usr/bin/clang-format clang-format $(which clang-format-9 ) 100 +update-alternatives --install /usr/bin/clang-format-diff clang-format-diff $(which clang-format-diff-9) 100 -travis_retry pip3 install codespell==1.17.1 -travis_retry pip3 install flake8==3.8.3 -travis_retry pip3 install yq -travis_retry pip3 install mypy==0.781 +${CI_RETRY_EXE} pip3 install codespell==1.17.1 +${CI_RETRY_EXE} pip3 install flake8==3.8.3 +${CI_RETRY_EXE} pip3 install yq +${CI_RETRY_EXE} pip3 install mypy==0.781 SHELLCHECK_VERSION=v0.7.1 curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ diff --git a/ci/lint/05_before_script.sh b/ci/lint/05_before_script.sh index 2987812c8e..8e5a177b01 100755 --- a/ci/lint/05_before_script.sh +++ b/ci/lint/05_before_script.sh @@ -6,4 +6,4 @@ export LC_ALL=C -git fetch --unshallow +git fetch diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index 6f81ecb22e..ba582e7bf6 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -6,11 +6,11 @@ export LC_ALL=C -if [ "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then - # TRAVIS_BRANCH will be present in a Travis environment. For builds triggered +if [ -n "$CIRRUS_PR" ]; then + # CIRRUS_PR will be present in a Cirrus environment. For builds triggered # by a pull request this is the name of the branch targeted by the pull request. - # https://docs.travis-ci.com/user/environment-variables/ - COMMIT_RANGE="$TRAVIS_BRANCH..HEAD" + # https://cirrus-ci.org/guide/writing-tasks/#environment-variables + COMMIT_RANGE="$CIRRUS_BRANCH..HEAD" test/lint/commit-script-check.sh $COMMIT_RANGE fi @@ -25,8 +25,8 @@ test/lint/check-doc.py test/lint/check-rpc-mappings.py . test/lint/lint-all.sh -if [ "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" ] && [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then +if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ -n "$CIRRUS_CRON" ]; then git log --merges --before="2 days ago" -1 --format='%H' > ./contrib/verify-commits/trusted-sha512-root-commit - travis_retry gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys $(<contrib/verify-commits/trusted-keys) && + ${CI_RETRY_EXE} gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys $(<contrib/verify-commits/trusted-keys) && ./contrib/verify-commits/verify-commits.py --clean-merge=2; fi diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index 5bda37d46d..b0de2ec0bb 100644 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to macos (bionic is used in the gitian build as well) export HOST=x86_64-apple-darwin18 -export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools" +export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools xorriso" export XCODE_VERSION=11.3.1 export XCODE_BUILD_ID=11C505 export RUN_UNIT_TESTS=false diff --git a/configure.ac b/configure.ac index e772268b04..ef2f31e582 100644 --- a/configure.ac +++ b/configure.ac @@ -675,7 +675,8 @@ case $host in AC_PATH_TOOL([DSYMUTIL], [dsymutil], dsymutil) AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], install_name_tool) AC_PATH_TOOL([OTOOL], [otool], otool) - AC_PATH_PROGS([GENISOIMAGE], [genisoimage mkisofs],genisoimage) + AC_PATH_PROGS([XORRISOFS], [xorrisofs], xorrisofs) + AC_PATH_PROGS([DMG], [dmg], dmg) AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg],rsvg-convert) AC_PATH_PROGS([IMAGEMAGICK_CONVERT], [convert],convert) AC_PATH_PROGS([TIFFCP], [tiffcp],tiffcp) diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml index a4f3219c22..2330ff7736 100644 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ b/contrib/gitian-descriptors/gitian-osx-signer.yml @@ -7,6 +7,7 @@ architectures: - "amd64" packages: - "faketime" +- "xorriso" remotes: - "url": "https://github.com/bitcoin-core/bitcoin-detached-sigs.git" "dir": "signature" @@ -18,7 +19,7 @@ script: | WRAP_DIR=$HOME/wrapped mkdir -p ${WRAP_DIR} export PATH="$PWD":$PATH - FAKETIME_PROGS="dmg genisoimage" + FAKETIME_PROGS="dmg xorrisofs" # Create global faketime wrappers for prog in ${FAKETIME_PROGS}; do @@ -36,5 +37,5 @@ script: | tar -xf ${UNSIGNED} OSX_VOLNAME="$(cat osx_volname)" ./detached-sig-apply.sh ${UNSIGNED} signature/osx - ${WRAP_DIR}/genisoimage -no-cache-inodes -D -l -probe -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -apple -o uncompressed.dmg signed-app + ${WRAP_DIR}/xorrisofs -D -l -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -o uncompressed.dmg signed-app ${WRAP_DIR}/dmg dmg uncompressed.dmg ${OUTDIR}/${SIGNED} diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 4119e88003..9a7dd13c9c 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -28,6 +28,7 @@ packages: - "python3-dev" - "python3-setuptools" - "fonts-tuffy" +- "xorriso" remotes: - "url": "https://github.com/bitcoin/bitcoin.git" "dir": "bitcoin" @@ -38,9 +39,9 @@ script: | WRAP_DIR=$HOME/wrapped HOSTS="x86_64-apple-darwin18" - CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage" + CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests XORRISOFS=${WRAP_DIR}/xorrisofs DMG=${WRAP_DIR}/dmg" FAKETIME_HOST_PROGS="" - FAKETIME_PROGS="ar ranlib date dmg genisoimage" + FAKETIME_PROGS="ar ranlib date dmg xorrisofs" export QT_RCC_TEST=1 export QT_RCC_SOURCE_DATE_OVERRIDE=1 @@ -132,12 +133,11 @@ script: | make osx_volname make deploydir - OSX_VOLNAME="$(cat osx_volname)" mkdir -p unsigned-app-${i} cp osx_volname unsigned-app-${i}/ cp contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i} cp contrib/macdeploy/detached-sig-create.sh unsigned-app-${i} - cp ${BASEPREFIX}/${i}/native/bin/dmg ${BASEPREFIX}/${i}/native/bin/genisoimage unsigned-app-${i} + cp ${BASEPREFIX}/${i}/native/bin/dmg unsigned-app-${i} cp ${BASEPREFIX}/${i}/native/bin/${i}-codesign_allocate unsigned-app-${i}/codesign_allocate cp ${BASEPREFIX}/${i}/native/bin/${i}-pagestuff unsigned-app-${i}/pagestuff mv dist unsigned-app-${i} @@ -145,8 +145,7 @@ script: | find . | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz popd - make deploy - ${WRAP_DIR}/dmg dmg "${OSX_VOLNAME}.dmg" ${OUTDIR}/${DISTNAME}-osx-unsigned.dmg + make deploy OSX_DMG="${OUTDIR}/${DISTNAME}-osx-unsigned.dmg" cd installed find . -name "lib*.la" -delete diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 6c3db2620b..2d9a4a2153 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -92,19 +92,15 @@ created using these tools. The build process has been designed to avoid includin SDK's files in Gitian's outputs. All interim tarballs are fully deterministic and may be freely redistributed. -`genisoimage` is used to create the initial DMG. It is not deterministic as-is, so it has been -patched. A system `genisoimage` will work fine, but it will not be deterministic because -the file-order will change between invocations. The patch can be seen here: [cdrkit-deterministic.patch](https://github.com/bitcoin/bitcoin/blob/master/depends/patches/native_cdrkit/cdrkit-deterministic.patch). -No effort was made to fix this cleanly, so it likely leaks memory badly, however it's only used for -a single invocation, so that's no real concern. +[`xorrisofs`](https://www.gnu.org/software/xorriso/) is used to create the DMG. -`genisoimage` cannot compress DMGs, so afterwards, the DMG tool from the -`libdmg-hfsplus` project is used to compress it. There are several bugs in this tool and its -maintainer has seemingly abandoned the project. +`xorrisofs` cannot compress DMGs, so afterwards, the DMG tool from the +`libdmg-hfsplus` project is used to compress it. There are several bugs in this +tool and its maintainer has seemingly abandoned the project. The DMG tool has the ability to create DMGs from scratch as well, but this functionality is broken. Only the compression feature is currently used. Ideally, the creation could be fixed -and `genisoimage` would no longer be necessary. +and `xorrisofs` would no longer be necessary. Background images and other features can be added to DMG files by inserting a `.DS_Store` during creation. diff --git a/depends/packages/native_cdrkit.mk b/depends/packages/native_cdrkit.mk deleted file mode 100644 index 7bdf2d7dfd..0000000000 --- a/depends/packages/native_cdrkit.mk +++ /dev/null @@ -1,28 +0,0 @@ -package=native_cdrkit -$(package)_version=1.1.11 -$(package)_download_path=https://distro.ibiblio.org/fatdog/source/600/c -$(package)_file_name=cdrkit-$($(package)_version).tar.bz2 -$(package)_sha256_hash=b50d64c214a65b1a79afe3a964c691931a4233e2ba605d793eb85d0ac3652564 -$(package)_patches=cdrkit-deterministic.patch - -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/cdrkit-deterministic.patch -endef - -# Starting with 10.1, GCC defaults to -fno-common, resulting in linking errors. -# Pass -fcommon to retain the legacy behaviour. -define $(package)_config_cmds - $($(package)_cmake) -DCMAKE_C_FLAGS="$$($(1)_cflags) -fcommon" -endef - -define $(package)_build_cmds - $(MAKE) genisoimage -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) -C genisoimage install -endef - -define $(package)_postprocess_cmds - rm bin/isovfy bin/isoinfo bin/isodump bin/isodebug bin/devdump -endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 4b4cc7d9ff..d4fd23a47b 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -25,5 +25,5 @@ darwin_native_packages = native_ds_store native_mac_alias $(host_arch)_$(host_os)_native_packages += native_b2 ifneq ($(build_os),darwin) -darwin_native_packages += native_cctools native_cdrkit native_libdmg-hfsplus +darwin_native_packages += native_cctools native_libdmg-hfsplus endif diff --git a/depends/patches/native_cdrkit/cdrkit-deterministic.patch b/depends/patches/native_cdrkit/cdrkit-deterministic.patch deleted file mode 100644 index 8ab0993dc4..0000000000 --- a/depends/patches/native_cdrkit/cdrkit-deterministic.patch +++ /dev/null @@ -1,86 +0,0 @@ ---- cdrkit-1.1.11.old/genisoimage/tree.c 2008-10-21 19:57:47.000000000 -0400 -+++ cdrkit-1.1.11/genisoimage/tree.c 2013-12-06 00:23:18.489622668 -0500 -@@ -1139,8 +1139,9 @@ - scan_directory_tree(struct directory *this_dir, char *path, - struct directory_entry *de) - { -- DIR *current_dir; -+ int current_file; - char whole_path[PATH_MAX]; -+ struct dirent **d_list; - struct dirent *d_entry; - struct directory *parent; - int dflag; -@@ -1164,7 +1165,8 @@ - this_dir->dir_flags |= DIR_WAS_SCANNED; - - errno = 0; /* Paranoia */ -- current_dir = opendir(path); -+ //current_dir = opendir(path); -+ current_file = scandir(path, &d_list, NULL, alphasort); - d_entry = NULL; - - /* -@@ -1173,12 +1175,12 @@ - */ - old_path = path; - -- if (current_dir) { -+ if (current_file >= 0) { - errno = 0; -- d_entry = readdir(current_dir); -+ d_entry = d_list[0]; - } - -- if (!current_dir || !d_entry) { -+ if (current_file < 0 || !d_entry) { - int ret = 1; - - #ifdef USE_LIBSCHILY -@@ -1191,8 +1193,8 @@ - de->isorec.flags[0] &= ~ISO_DIRECTORY; - ret = 0; - } -- if (current_dir) -- closedir(current_dir); -+ if(d_list) -+ free(d_list); - return (ret); - } - #ifdef ABORT_DEEP_ISO_ONLY -@@ -1208,7 +1210,7 @@ - errmsgno(EX_BAD, "use Rock Ridge extensions via -R or -r,\n"); - errmsgno(EX_BAD, "or allow deep ISO9660 directory nesting via -D.\n"); - } -- closedir(current_dir); -+ free(d_list); - return (1); - } - #endif -@@ -1250,13 +1252,13 @@ - * The first time through, skip this, since we already asked - * for the first entry when we opened the directory. - */ -- if (dflag) -- d_entry = readdir(current_dir); -+ if (dflag && current_file >= 0) -+ d_entry = d_list[current_file]; - dflag++; - -- if (!d_entry) -+ if (current_file < 0) - break; -- -+ current_file--; - /* OK, got a valid entry */ - - /* If we do not want all files, then pitch the backups. */ -@@ -1348,7 +1350,7 @@ - insert_file_entry(this_dir, whole_path, d_entry->d_name); - #endif /* APPLE_HYB */ - } -- closedir(current_dir); -+ free(d_list); - - #ifdef APPLE_HYB - /* diff --git a/doc/JSON-RPC-interface.md b/doc/JSON-RPC-interface.md index 40d8e330e2..c66e79af71 100644 --- a/doc/JSON-RPC-interface.md +++ b/doc/JSON-RPC-interface.md @@ -127,3 +127,14 @@ However, the wallet may not be up-to-date with the current state of the mempool or the state of the mempool by an RPC that returned before this RPC. For example, a wallet transaction that was BIP-125-replaced in the mempool prior to this RPC may not yet be reflected as such in this RPC response. + +## Limitations + +There is a known issue in the JSON-RPC interface that can cause a node to crash if +too many http connections are being opened at the same time because the system runs +out of available file descriptors. To prevent this from happening you might +want to increase the number of maximum allowed file descriptors in your system +and try to prevent opening too many connections to your JSON-RPC interface at the +same time if this is under your control. It is hard to give general advice +since this depends on your system but if you make several hundred requests at +once you are definitely at risk of encountering this issue. diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 842a3964df..3b127703b7 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -12,6 +12,18 @@ REST Interface consistency guarantees The [same guarantees as for the RPC Interface](/doc/JSON-RPC-interface.md#rpc-consistency-guarantees) apply. +Limitations +----------- + +There is a known issue in the REST interface that can cause a node to crash if +too many http connections are being opened at the same time because the system runs +out of available file descriptors. To prevent this from happening you might +want to increase the number of maximum allowed file descriptors in your system +and try to prevent opening too many connections to your rest interface at the +same time if this is under your control. It is hard to give general advice +since this depends on your system but if you make several hundred requests at +once you are definitely at risk of encountering this issue. + Supported API ------------- diff --git a/doc/tor.md b/doc/tor.md index 692041ccea..86e5d9ddf3 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -5,6 +5,16 @@ It is possible to run Bitcoin Core as a Tor onion service, and connect to such s The following directions assume you have a Tor proxy running on port 9050. Many distributions default to having a SOCKS proxy listening on port 9050, but others may not. In particular, the Tor Browser Bundle defaults to listening on port 9150. See [Tor Project FAQ:TBBSocksPort](https://www.torproject.org/docs/faq.html.en#TBBSocksPort) for how to properly configure Tor. +## How to see information about your Tor configuration via Bitcoin Core + +There are several ways to see your local onion address in Bitcoin Core: +- in the debug log (grep for "tor:" or "AddLocal") +- in the output of RPC `getnetworkinfo` in the "localaddresses" section +- in the output of the CLI `-netinfo` peer connections dashboard + +You may set the `-debug=tor` config logging option to have additional +information in the debug log about your Tor configuration. + ## 1. Run Bitcoin Core behind a Tor proxy @@ -51,14 +61,19 @@ The directory can be different of course, but virtual port numbers should be equ your bitcoind's P2P listen port (8333 by default), and target addresses and ports should be equal to binding address and port for inbound Tor connections (127.0.0.1:8334 by default). - -externalip=X You can tell bitcoin about its publicly reachable address using - this option, and this can be a .onion address. Given the above - configuration, you can find your .onion address in + -externalip=X You can tell bitcoin about its publicly reachable addresses using + this option, and this can be an onion address. Given the above + configuration, you can find your onion address in /var/lib/tor/bitcoin-service/hostname. For connections coming from unroutable addresses (such as 127.0.0.1, where the - Tor proxy typically runs), .onion addresses are given + Tor proxy typically runs), onion addresses are given preference for your node to advertise itself with. + You can set multiple local addresses with -externalip. The + one that will be rumoured to a particular peer is the most + compatible one and also using heuristics, e.g. the address + with the most incoming connections, etc. + -listen You'll need to enable listening for incoming connections, as this is off by default behind a proxy. @@ -71,7 +86,7 @@ should be equal to binding address and port for inbound Tor connections (127.0.0 In a typical situation, where you're only reachable via Tor, this should suffice: - ./bitcoind -proxy=127.0.0.1:9050 -externalip=57qr3yd1nyntf5k.onion -listen + ./bitcoind -proxy=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -listen (obviously, replace the .onion address with your own). It should be noted that you still listen on all devices and another node could establish a clearnet connection, when knowing @@ -89,7 +104,7 @@ and open port 8333 on your firewall (or use -upnp). If you only want to use Tor to reach .onion addresses, but not use it as a proxy for normal IPv4/IPv6 communication, use: - ./bitcoind -onion=127.0.0.1:9050 -externalip=57qr3yd1nyntf5k.onion -discover + ./bitcoind -onion=127.0.0.1:9050 -externalip=7zvj7a2imdgkdbg4f2dryd5rgtrn7upivr5eeij4cicjh65pooxeshid.onion -discover ## 3. Automatically create a Bitcoin Core onion service diff --git a/src/bench/chacha_poly_aead.cpp b/src/bench/chacha_poly_aead.cpp index 3b1d3e697a..e994279a4d 100644 --- a/src/bench/chacha_poly_aead.cpp +++ b/src/bench/chacha_poly_aead.cpp @@ -31,12 +31,15 @@ static void CHACHA20_POLY1305_AEAD(benchmark::Bench& bench, size_t buffersize, b uint32_t len = 0; bench.batch(buffersize).unit("byte").run([&] { // encrypt or decrypt the buffer with a static key - assert(aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, out.data(), out.size(), in.data(), buffersize, true)); + const bool crypt_ok_1 = aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, out.data(), out.size(), in.data(), buffersize, true); + assert(crypt_ok_1); if (include_decryption) { // if we decrypt, include the GetLength - assert(aead.GetLength(&len, seqnr_aad, aad_pos, in.data())); - assert(aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, out.data(), out.size(), in.data(), buffersize, true)); + const bool get_length_ok = aead.GetLength(&len, seqnr_aad, aad_pos, in.data()); + assert(get_length_ok); + const bool crypt_ok_2 = aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, out.data(), out.size(), in.data(), buffersize, true); + assert(crypt_ok_2); } // increase main sequence number diff --git a/src/init.cpp b/src/init.cpp index 46dce6aa97..1220f39b14 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -450,7 +450,7 @@ void SetupServerArgs(NodeContext& node) argsman.AddArg("-peerbloomfilters", strprintf("Support filtering of blocks and transaction with bloom filters (default: %u)", DEFAULT_PEERBLOOMFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerblockfilters", strprintf("Serve compact block filters to peers per BIP 157 (default: %u)", DEFAULT_PEERBLOCKFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-permitbaremultisig", strprintf("Relay non-P2SH multisig (default: %u)", DEFAULT_PERMIT_BAREMULTISIG), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-port=<port>", strprintf("Listen for connections on <port> (default: %u, testnet: %u signet: %u, regtest: %u)", defaultChainParams->GetDefaultPort(), testnetChainParams->GetDefaultPort(), signetChainParams->GetDefaultPort(), regtestChainParams->GetDefaultPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); + argsman.AddArg("-port=<port>", strprintf("Listen for connections on <port>. Nodes not using the default ports (default: %u, testnet: %u, signet: %u, regtest: %u) are unlikely to get incoming connections.", defaultChainParams->GetDefaultPort(), testnetChainParams->GetDefaultPort(), signetChainParams->GetDefaultPort(), regtestChainParams->GetDefaultPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-proxy=<ip:port>", "Connect through SOCKS5 proxy, set -noproxy to disable (default: disabled)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-proxyrandomize", strprintf("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)", DEFAULT_PROXYRANDOMIZE), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-seednode=<ip>", "Connect to a node to retrieve peer addresses, and disconnect. This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); diff --git a/src/net.cpp b/src/net.cpp index db4b541ca7..b3c521116b 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1228,7 +1228,7 @@ void CConnman::InactivityCheck(CNode *pnode) LogPrintf("socket sending timeout: %is\n", nTime - pnode->nLastSend); pnode->fDisconnect = true; } - else if (nTime - pnode->nLastRecv > (pnode->GetCommonVersion() > BIP0031_VERSION ? TIMEOUT_INTERVAL : 90*60)) + else if (nTime - pnode->nLastRecv > TIMEOUT_INTERVAL) { LogPrintf("socket receive timeout: %is\n", nTime - pnode->nLastRecv); pnode->fDisconnect = true; @@ -2074,7 +2074,11 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) continue; } - // do not allow non-default ports, unless after 50 invalid addresses selected already + // Do not allow non-default ports, unless after 50 invalid + // addresses selected already. This is to prevent malicious peers + // from advertising themselves as a service on another host and + // port, causing a DoS attack as nodes around the network attempt + // to connect to it fruitlessly. if (addr.GetPort() != Params().GetDefaultPort() && nTries < 50) continue; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index a33c4a0bd4..17aa889ab0 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -2394,8 +2394,8 @@ void PeerManager::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDat // empty and no one will know who we are, so these mechanisms are // important to help us connect to the network. // - // We skip this for BLOCK_RELAY peers to avoid potentially leaking - // information about our BLOCK_RELAY connections via address relay. + // We skip this for block-relay-only peers to avoid potentially leaking + // information about our block-relay-only connections via address relay. if (fListen && !::ChainstateActive().IsInitialBlockDownload()) { CAddress addr = GetLocalAddress(&pfrom.addr, pfrom.GetLocalServices()); @@ -3963,10 +3963,10 @@ void PeerManager::EvictExtraOutboundPeers(int64_t time_in_seconds) }); } - // Check whether we have too many OUTBOUND_FULL_RELAY peers + // Check whether we have too many outbound-full-relay peers if (m_connman.GetExtraFullOutboundCount() > 0) { - // If we have more OUTBOUND_FULL_RELAY peers than we target, disconnect one. - // Pick the OUTBOUND_FULL_RELAY peer that least recently announced + // If we have more outbound-full-relay peers than we target, disconnect one. + // Pick the outbound-full-relay peer that least recently announced // us a new block, with ties broken by choosing the more recent // connection (higher node id) NodeId worst_peer = -1; @@ -3975,7 +3975,7 @@ void PeerManager::EvictExtraOutboundPeers(int64_t time_in_seconds) m_connman.ForEachNode([&](CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { AssertLockHeld(::cs_main); - // Only consider OUTBOUND_FULL_RELAY peers that are not already + // Only consider outbound-full-relay peers that are not already // marked for disconnection if (!pnode->IsFullOutboundConn() || pnode->fDisconnect) return; CNodeState *state = State(pnode->GetId()); diff --git a/src/qt/bitcoinunits.cpp b/src/qt/bitcoinunits.cpp index fd55c547fc..5402ed371d 100644 --- a/src/qt/bitcoinunits.cpp +++ b/src/qt/bitcoinunits.cpp @@ -8,6 +8,8 @@ #include <cassert> +static constexpr auto MAX_DIGITS_BTC = 16; + BitcoinUnits::BitcoinUnits(QObject *parent): QAbstractListModel(parent), unitlist(availableUnits()) @@ -108,7 +110,9 @@ QString BitcoinUnits::format(int unit, const CAmount& nIn, bool fPlus, Separator qint64 n_abs = (n > 0 ? n : -n); qint64 quotient = n_abs / coin; QString quotient_str = QString::number(quotient); - if (justify) quotient_str = quotient_str.rightJustified(16 - num_decimals, ' '); + if (justify) { + quotient_str = quotient_str.rightJustified(MAX_DIGITS_BTC - num_decimals, ' '); + } // Use SI-style thin space separators as these are locale independent and can't be // confused with the decimal marker. diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index 3362b8d17c..cb66d5164e 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -172,20 +172,30 @@ BOOST_AUTO_TEST_CASE(key_signature_tests) } BOOST_CHECK(found); - // When entropy is not specified, we should always see low R signatures that are less than 70 bytes in 256 tries + // When entropy is not specified, we should always see low R signatures that are less than or equal to 70 bytes in 256 tries + // The low R signatures should always have the value of their "length of R" byte less than or equal to 32 // We should see at least one signature that is less than 70 bytes. - found = true; bool found_small = false; + bool found_big = false; + bool bad_sign = false; for (int i = 0; i < 256; ++i) { sig.clear(); std::string msg = "A message to be signed" + ToString(i); msg_hash = Hash(msg); - BOOST_CHECK(key.Sign(msg_hash, sig)); - found = sig[3] == 0x20; - BOOST_CHECK(sig.size() <= 70); + if (!key.Sign(msg_hash, sig)) { + bad_sign = true; + break; + } + // sig.size() > 70 implies sig[3] > 32, because S is always low. + // But check both conditions anyway, just in case this implication is broken for some reason + if (sig[3] > 32 || sig.size() > 70) { + found_big = true; + break; + } found_small |= sig.size() < 70; } - BOOST_CHECK(found); + BOOST_CHECK(!bad_sign); + BOOST_CHECK(!found_big); BOOST_CHECK(found_small); } diff --git a/src/validation.h b/src/validation.h index 6d8c6d431a..d10b260d8a 100644 --- a/src/validation.h +++ b/src/validation.h @@ -562,7 +562,7 @@ public: //! @returns whether or not the CoinsViews object has been fully initialized and we can //! safely flush this object to disk. - bool CanFlushToDisk() EXCLUSIVE_LOCKS_REQUIRED(cs_main) { + bool CanFlushToDisk() const EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return m_coins_views && m_coins_views->m_cacheview; } diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 4127cd45f8..019161415c 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -64,7 +64,8 @@ static void add_coin(CWallet& wallet, const CAmount& nValue, int nAge = 6*24, bo if (spendable) { CTxDestination dest; std::string error; - assert(wallet.GetNewDestination(OutputType::BECH32, "", dest, error)); + const bool destination_ok = wallet.GetNewDestination(OutputType::BECH32, "", dest, error); + assert(destination_ok); tx.vout[nInput].scriptPubKey = GetScriptForDestination(dest); } if (fIsFromMe) { diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 3e37491a23..740e671595 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1178,9 +1178,8 @@ void CWallet::transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRe // Handle transactions that were removed from the mempool because they // conflict with transactions in a newly connected block. if (reason == MemPoolRemovalReason::CONFLICT) { - // Call SyncNotifications, so external -walletnotify notifications will - // be triggered for these transactions. Set Status::UNCONFIRMED instead - // of Status::CONFLICTED for a few reasons: + // Trigger external -walletnotify notifications for these transactions. + // Set Status::UNCONFIRMED instead of Status::CONFLICTED for a few reasons: // // 1. The transactionRemovedFromMempool callback does not currently // provide the conflicting block's hash and height, and for backwards diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index e99ecd8026..a9d8b12d70 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -37,7 +37,6 @@ from test_framework.messages import ( msg_tx, msg_block, msg_no_witness_tx, - msg_verack, ser_uint256, ser_vector, sha256, @@ -146,7 +145,7 @@ def test_witness_block(node, p2p, block, accepted, with_witness=True, reason=Non class TestP2PConn(P2PInterface): def __init__(self, wtxidrelay=False): - super().__init__() + super().__init__(wtxidrelay=wtxidrelay) self.getdataset = set() self.last_wtxidrelay = [] self.lastgetdata = [] @@ -157,13 +156,6 @@ class TestP2PConn(P2PInterface): def on_inv(self, message): pass - def on_version(self, message): - if self.wtxidrelay: - super().on_version(message) - else: - self.send_message(msg_verack()) - self.nServices = message.nServices - def on_getdata(self, message): self.lastgetdata = message.inv for inv in message.inv: diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index 16d9302db8..8a751c6b54 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -30,8 +30,8 @@ import time class TestP2PConn(P2PInterface): - def __init__(self): - super().__init__() + def __init__(self, wtxidrelay=True): + super().__init__(wtxidrelay=wtxidrelay) self.tx_getdata_count = 0 def on_getdata(self, message): @@ -47,6 +47,7 @@ TXID_RELAY_DELAY = 2 # seconds OVERLOADED_PEER_DELAY = 2 # seconds MAX_GETDATA_IN_FLIGHT = 100 MAX_PEER_TX_ANNOUNCEMENTS = 5000 +NONPREF_PEER_TX_DELAY = 2 # Python test constants NUM_INBOUND = 10 @@ -168,8 +169,6 @@ class TxDownloadTest(BitcoinTestFramework): assert_equal(peer_fallback.tx_getdata_count, 0) self.nodes[0].setmocktime(int(time.time()) + GETDATA_TX_INTERVAL + 1) # Wait for request to peer_expiry to expire peer_fallback.wait_until(lambda: peer_fallback.tx_getdata_count >= 1, timeout=1) - with p2p_lock: - assert_equal(peer_fallback.tx_getdata_count, 1) self.restart_node(0) # reset mocktime def test_disconnect_fallback(self): @@ -187,8 +186,6 @@ class TxDownloadTest(BitcoinTestFramework): peer_disconnect.peer_disconnect() peer_disconnect.wait_for_disconnect() peer_fallback.wait_until(lambda: peer_fallback.tx_getdata_count >= 1, timeout=1) - with p2p_lock: - assert_equal(peer_fallback.tx_getdata_count, 1) def test_notfound_fallback(self): self.log.info('Check that notfounds will select another peer for download immediately') @@ -204,17 +201,42 @@ class TxDownloadTest(BitcoinTestFramework): assert_equal(peer_fallback.tx_getdata_count, 0) peer_notfound.send_and_ping(msg_notfound(vec=[CInv(MSG_WTX, WTXID)])) # Send notfound, so that fallback peer is selected peer_fallback.wait_until(lambda: peer_fallback.tx_getdata_count >= 1, timeout=1) - with p2p_lock: - assert_equal(peer_fallback.tx_getdata_count, 1) - def test_preferred_inv(self): - self.log.info('Check that invs from preferred peers are downloaded immediately') - self.restart_node(0, extra_args=['-whitelist=noban@127.0.0.1']) + def test_preferred_inv(self, preferred=False): + if preferred: + self.log.info('Check invs from preferred peers are downloaded immediately') + self.restart_node(0, extra_args=['-whitelist=noban@127.0.0.1']) + else: + self.log.info('Check invs from non-preferred peers are downloaded after {} s'.format(NONPREF_PEER_TX_DELAY)) + mock_time = int(time.time() + 1) + self.nodes[0].setmocktime(mock_time) peer = self.nodes[0].add_p2p_connection(TestP2PConn()) peer.send_message(msg_inv([CInv(t=MSG_WTX, h=0xff00ff00)])) - peer.wait_until(lambda: peer.tx_getdata_count >= 1, timeout=1) + peer.sync_with_ping() + if preferred: + peer.wait_until(lambda: peer.tx_getdata_count >= 1, timeout=1) + else: + with p2p_lock: + assert_equal(peer.tx_getdata_count, 0) + self.nodes[0].setmocktime(mock_time + NONPREF_PEER_TX_DELAY) + peer.wait_until(lambda: peer.tx_getdata_count >= 1, timeout=1) + + def test_txid_inv_delay(self, glob_wtxid=False): + self.log.info('Check that inv from a txid-relay peers are delayed by {} s, with a wtxid peer {}'.format(TXID_RELAY_DELAY, glob_wtxid)) + self.restart_node(0, extra_args=['-whitelist=noban@127.0.0.1']) + mock_time = int(time.time() + 1) + self.nodes[0].setmocktime(mock_time) + peer = self.nodes[0].add_p2p_connection(TestP2PConn(wtxidrelay=False)) + if glob_wtxid: + # Add a second wtxid-relay connection otherwise TXID_RELAY_DELAY is waived in + # lack of wtxid-relay peers + self.nodes[0].add_p2p_connection(TestP2PConn(wtxidrelay=True)) + peer.send_message(msg_inv([CInv(t=MSG_TX, h=0xff11ff11)])) + peer.sync_with_ping() with p2p_lock: - assert_equal(peer.tx_getdata_count, 1) + assert_equal(peer.tx_getdata_count, 0 if glob_wtxid else 1) + self.nodes[0].setmocktime(mock_time + TXID_RELAY_DELAY) + peer.wait_until(lambda: peer.tx_getdata_count >= 1, timeout=1) def test_large_inv_batch(self): self.log.info('Test how large inv batches are handled with relay permission') @@ -229,8 +251,6 @@ class TxDownloadTest(BitcoinTestFramework): peer.send_message(msg_inv([CInv(t=MSG_WTX, h=wtxid) for wtxid in range(MAX_PEER_TX_ANNOUNCEMENTS + 1)])) peer.wait_until(lambda: peer.tx_getdata_count == MAX_PEER_TX_ANNOUNCEMENTS) peer.sync_with_ping() - with p2p_lock: - assert_equal(peer.tx_getdata_count, MAX_PEER_TX_ANNOUNCEMENTS) def test_spurious_notfound(self): self.log.info('Check that spurious notfound is ignored') @@ -242,6 +262,9 @@ class TxDownloadTest(BitcoinTestFramework): self.test_disconnect_fallback() self.test_notfound_fallback() self.test_preferred_inv() + self.test_preferred_inv(True) + self.test_txid_inv_delay() + self.test_txid_inv_delay(True) self.test_large_inv_batch() self.test_spurious_notfound() diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index 8b79a4dc2f..ea769ddfa2 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -289,7 +289,7 @@ class P2PInterface(P2PConnection): Individual testcases should subclass this and override the on_* methods if they want to alter message handling behaviour.""" - def __init__(self, support_addrv2=False): + def __init__(self, support_addrv2=False, wtxidrelay=True): super().__init__() # Track number of messages of each type received. @@ -309,6 +309,9 @@ class P2PInterface(P2PConnection): self.support_addrv2 = support_addrv2 + # If the peer supports wtxid-relay + self.wtxidrelay = wtxidrelay + def peer_connect(self, *args, services=NODE_NETWORK|NODE_WITNESS, send_version=True, **kwargs): create_conn = super().peer_connect(*args, **kwargs) @@ -394,7 +397,7 @@ class P2PInterface(P2PConnection): def on_version(self, message): assert message.nVersion >= MIN_VERSION_SUPPORTED, "Version {} received. Test framework only supports versions greater than {}".format(message.nVersion, MIN_VERSION_SUPPORTED) - if message.nVersion >= 70016: + if message.nVersion >= 70016 and self.wtxidrelay: self.send_message(msg_wtxidrelay()) if self.support_addrv2: self.send_message(msg_sendaddrv2()) |