diff options
712 files changed, 25099 insertions, 13917 deletions
diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 3ca7818eca..0000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,63 +0,0 @@ -version: '{branch}.{build}' -skip_tags: true -image: Visual Studio 2019 -configuration: Release -platform: x64 -clone_depth: 5 -environment: - PATH: 'C:\Python37-x64;C:\Python37-x64\Scripts;%PATH%' - PYTHONUTF8: 1 - QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/qt51211x64_static_vs2019_16101/Qt5.12.11_x64_static_vs2019_16101.zip' - QT_DOWNLOAD_HASH: 'cf1b58107fadbf0d9a957d14dab16cde6b6eb6936a1908472da1f967dda34a3a' - QT_LOCAL_PATH: 'C:\Qt5.12.11_x64_static_vs2019_16101' - VCPKG_TAG: '75522bb1f2e7d863078bcd06322348f053a9e33f' -install: -# Disable zmq test for now since python zmq library on Windows would cause Access violation sometimes. -# - cmd: pip install zmq -# The powershell block below is to set up vcpkg to install the c++ dependencies. The pseudo code is: -# a. Checkout the vcpkg source (including port files) for the specific checkout and build the vcpkg binary, -# b. Append a setting to the vcpkg cmake config file to only do release builds of dependencies (skipping deubg builds saves ~5 mins). -# Note originally this block also installed the dependencies using 'vcpkg install'. Dependencies are now installed -# as part of the msbuild command using vcpkg mainfests. -- ps: | - cd c:\tools\vcpkg - $env:GIT_REDIRECT_STDERR = '2>&1' # git is writing non-errors to STDERR when doing git pull. Send to STDOUT instead. - git -c advice.detachedHead=false checkout $env:VCPKG_TAG - .\bootstrap-vcpkg.bat > $null - Add-Content "C:\tools\vcpkg\triplets\$env:PLATFORM-windows-static.cmake" "set(VCPKG_BUILD_TYPE release)" - cd "$env:APPVEYOR_BUILD_FOLDER" -before_build: -# Powershell block below is to download and extract the Qt static libraries. The pseudo code is: -# a. Download the zip file with the prebuilt Qt static libraries. -# b. Check that the downloaded file matches the expected hash. -# c. Extract the zip file to the specific destination path expected by the msbuild projects. -- ps: | - Write-Host "Downloading Qt binaries."; - Invoke-WebRequest -Uri $env:QT_DOWNLOAD_URL -Out qtdownload.zip; - Write-Host "Qt binaries successfully downloaded, checking hash against $env:QT_DOWNLOAD_HASH..."; - if((Get-FileHash qtdownload.zip).Hash -eq $env:QT_DOWNLOAD_HASH) { - Expand-Archive qtdownload.zip -DestinationPath $env:QT_LOCAL_PATH; - Write-Host "Qt binary download matched the expected hash."; - } - else { - Write-Host "ERROR: Qt binary download did not match the expected hash."; - Exit-AppveyorBuild; - } -- cmd: python build_msvc\msvc-autogen.py -build_script: -- cmd: msbuild /p:TrackFileAccess=false build_msvc\bitcoin.sln /m /v:q /nologo -after_build: -#- 7z a bitcoin-%APPVEYOR_BUILD_VERSION%.zip %APPVEYOR_BUILD_FOLDER%\build_msvc\%platform%\%configuration%\*.exe -test_script: -- cmd: src\test_bitcoin.exe -l test_suite -- cmd: src\bench_bitcoin.exe > NUL -- ps: python test\util\bitcoin-util-test.py -- cmd: python test\util\rpcauth-test.py -# Fee estimation test failing on appveyor with: WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted. -# functional tests disabled for now. See -# https://github.com/bitcoin/bitcoin/pull/18626#issuecomment-613396202 -# https://github.com/bitcoin/bitcoin/issues/18623 -# - cmd: python test\functional\test_runner.py --ci --quiet --combinedlogslen=4000 --failfast --exclude feature_fee_estimation -artifacts: -#- path: bitcoin-%APPVEYOR_BUILD_VERSION%.zip -deploy: off diff --git a/.cirrus.yml b/.cirrus.yml index 26bd27754f..5491f2c04b 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,6 +1,4 @@ -### Global defaults - -env: +env: # Global defaults PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y" MAKEJOBS: "-j4" TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache @@ -18,19 +16,23 @@ persistent_worker_template: &PERSISTENT_WORKER_TEMPLATE persistent_worker: {} # https://cirrus-ci.org/guide/persistent-workers/ # https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks -base_template: &BASE_TEMPLATE +filter_template: &FILTER_TEMPLATE skip: $CIRRUS_REPO_FULL_NAME == "bitcoin-core/gui" && $CIRRUS_PR == "" # No need to run on the read-only mirror, unless it is a PR. https://cirrus-ci.org/guide/writing-tasks/#conditional-task-execution + stateful: false # https://cirrus-ci.org/guide/writing-tasks/#stateful-tasks + +base_template: &BASE_TEMPLATE + << : *FILTER_TEMPLATE merge_base_script: - - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi + # Unconditionally install git (used in fingerprint_script) and set the + # default git author name (used in verify-commits.py) - bash -c "$PACKAGE_MANAGER_INSTALL git" - - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH - git config --global user.email "ci@ci.ci" - git config --global user.name "ci" + - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi + - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH - git merge FETCH_HEAD # Merge base to detect silent merge conflicts - stateful: false # https://cirrus-ci.org/guide/writing-tasks/#stateful-tasks -global_task_template: &GLOBAL_TASK_TEMPLATE - << : *BASE_TEMPLATE +main_template: &MAIN_TEMPLATE timeout_in: 120m # https://cirrus-ci.org/faq/#instance-timed-out container: # https://cirrus-ci.org/faq/#are-there-any-limits @@ -41,9 +43,14 @@ global_task_template: &GLOBAL_TASK_TEMPLATE folder: "/tmp/ccache_dir" depends_built_cache: folder: "depends/built" + fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-list -1 HEAD ./depends) ci_script: - ./ci/test_run_all.sh +global_task_template: &GLOBAL_TASK_TEMPLATE + << : *BASE_TEMPLATE + << : *MAIN_TEMPLATE + depends_sdk_cache_template: &DEPENDS_SDK_CACHE_TEMPLATE depends_sdk_cache: folder: "depends/sdk-sources" @@ -53,22 +60,6 @@ compute_credits_template: &CREDITS_TEMPLATE # Only use credits for pull requests to the main repo use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'bitcoin/bitcoin' && $CIRRUS_PR != "" -#task: -# name: "Windows" -# windows_container: -# image: cirrusci/windowsservercore:2019 -# env: -# CIRRUS_SHELL: powershell -# PATH: 'C:\Python37;C:\Python37\Scripts;%PATH%' -# PYTHONUTF8: 1 -# QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.6/Qt5.9.8_x64_static_vs2019.zip' -# QT_DOWNLOAD_HASH: '9a8c6eb20967873785057fdcd329a657c7f922b0af08c5fde105cc597dd37e21' -# QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019' -# VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed' -# VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74' -# install_script: -# - choco install python --version=3.7.7 -y - task: name: 'lint [bionic]' << : *BASE_TEMPLATE @@ -84,13 +75,95 @@ task: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV task: - name: 'ARM [unit tests, no functional tests] [buster]' + name: "Win64 native [msvc]" + << : *FILTER_TEMPLATE + windows_container: + cpu: 4 + memory: 16G + image: cirrusci/windowsservercore:visualstudio2019 + timeout_in: 120m + env: + PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Current\Bin;%PATH%' + PYTHONUTF8: 1 + VCPKG_TAG: '75522bb1f2e7d863078bcd06322348f053a9e33f' + VCPKG_FEATURE_FLAGS: 'manifests' + QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.12/5.12.11/single/qt-everywhere-src-5.12.11.zip' + QT_LOCAL_PATH: 'C:\qt-everywhere-src-5.12.11.zip' + QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.12.11' + QTBASEDIR: 'C:\Qt_static' + x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"' + IgnoreWarnIntDirInTempDetected: 'true' + merge_script: + - git config --global user.email "ci@ci.ci" + - git config --global user.name "ci" + # Windows filesystem loses the executable bit, and all of the executable + # files are considered "modified" now. It will break the following `git merge` + # command. The next two commands make git ignore this issue. + - git config core.filemode false + - git reset --hard + - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL $env:CIRRUS_BASE_BRANCH; git merge FETCH_HEAD; } + msvc_qt_built_cache: + folder: "%QTBASEDIR%" + reupload_on_changes: false + fingerprint_script: + - echo %QT_DOWNLOAD_URL% + - msbuild -version + populate_script: + - curl -L -o C:\jom.zip http://download.qt.io/official_releases/jom/jom.zip + - mkdir C:\jom + - tar -xf C:\jom.zip -C C:\jom + - curl -L -o %QT_LOCAL_PATH% %QT_DOWNLOAD_URL% + - tar -xf %QT_LOCAL_PATH% -C C:\ + - '%x64_NATIVE_TOOLS%' + - cd %QT_SOURCE_DIR% + - mkdir build + - cd build + - ..\configure -release -silent -opensource -confirm-license -opengl desktop -no-shared -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -no-libjpeg -nomake examples -nomake tests -nomake tools -no-dbus -no-libudev -no-icu -no-gtk -no-opengles3 -no-angle -no-sql-sqlite -no-sql-odbc -no-sqlite -no-libudev -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcanvas3d -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquickcontrols -skip qtquickcontrols2 -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-sql -no-feature-sqlmodel -prefix %QTBASEDIR% + - jom + - jom install + vcpkg_cache: + folder: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' + install_python_script: + - choco install --yes --no-progress python3 --version=3.9.6 + - pip install zmq + - python -VV + install_vcpkg_script: + - cd .. + - git clone --quiet https://github.com/microsoft/vcpkg.git + - cd vcpkg + - git -c advice.detachedHead=false checkout %VCPKG_TAG% + - .\bootstrap-vcpkg -disableMetrics + - echo set(VCPKG_BUILD_TYPE release) >> triplets\x64-windows-static.cmake + - .\vcpkg integrate install + - .\vcpkg version + build_script: + - cd %CIRRUS_WORKING_DIR% + - python build_msvc\msvc-autogen.py + - msbuild build_msvc\bitcoin.sln -property:Configuration=Release -maxCpuCount -verbosity:minimal -noLogo + unit_tests_script: + - src\test_bitcoin.exe -l test_suite + - src\bench_bitcoin.exe > NUL + - python test\util\test_runner.py + - python test\util\rpcauth-test.py + functional_tests_script: + # Increase the dynamic port range to the maximum allowed value to mitigate "OSError: [WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted". + # See: https://docs.microsoft.com/en-us/biztalk/technical-guides/settings-that-can-be-modified-to-improve-network-performance + - netsh int ipv4 set dynamicport tcp start=1025 num=64511 + - netsh int ipv6 set dynamicport tcp start=1025 num=64511 + # Exclude feature_dbcrash for now due to timeout + - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=4000 --jobs=4 --timeout-factor=8 --failfast --extended --exclude feature_dbcrash + +task: + name: 'ARM [unit tests, no functional tests] [bullseye]' << : *GLOBAL_TASK_TEMPLATE - container: - image: debian:buster + arm_container: + image: debian:bullseye + cpu: 2 + memory: 8G env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_arm.sh" + QEMU_USER_CMD: "" # Disable qemu and run the test natively task: name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [focal]' @@ -153,6 +226,7 @@ task: task: name: '[no depends, sanitizers: fuzzer,address,undefined,integer] [focal]' + only_if: $CIRRUS_BRANCH == $CIRRUS_DEFAULT_BRANCH || $CIRRUS_BASE_BRANCH == $CIRRUS_DEFAULT_BRANCH << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal @@ -164,7 +238,7 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh" task: - name: '[multiprocess, DEBUG] [focal]' + name: '[multiprocess, i686, DEBUG] [focal]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal @@ -173,7 +247,7 @@ task: env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV MAKEJOBS: "-j8" - FILE_ENV: "./ci/test/00_setup_env_native_multiprocess.sh" + FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh" task: name: '[no wallet] [bionic]' @@ -185,7 +259,7 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_nowallet.sh" task: - name: 'macOS 10.14 [gui, no tests] [focal]' + name: 'macOS 10.15 [gui, no tests] [focal]' << : *DEPENDS_SDK_CACHE_TEMPLATE << : *GLOBAL_TASK_TEMPLATE container: @@ -211,9 +285,11 @@ task: task: name: 'ARM64 Android APK [focal]' << : *DEPENDS_SDK_CACHE_TEMPLATE + << : *BASE_TEMPLATE depends_sources_cache: folder: "depends/sources" - << : *GLOBAL_TASK_TEMPLATE + fingerprint_script: git rev-list -1 HEAD ./depends + << : *MAIN_TEMPLATE container: image: ubuntu:focal env: diff --git a/.editorconfig b/.editorconfig index 4967e675f6..ae7e92d1c8 100644 --- a/.editorconfig +++ b/.editorconfig @@ -13,7 +13,7 @@ trim_trailing_whitespace = true [*.{h,cpp,py,sh}] indent_size = 4 -# .cirrus.yml, .appveyor.yml, .fuzzbuzz.yml, etc. +# .cirrus.yml, .fuzzbuzz.yml, etc. [*.yml] indent_size = 2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5cd4715ef0..acf5cc08d1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -60,8 +60,8 @@ Most communication about Bitcoin Core development happens on IRC, in the `#bitcoin-core-dev` channel on Libera Chat. The easiest way to participate on IRC is with the web client, [web.libera.chat](https://web.libera.chat/#bitcoin-core-dev). Chat history logs can be found -on [http://www.erisian.com.au/bitcoin-core-dev/](http://www.erisian.com.au/bitcoin-core-dev/) -and [http://gnusha.org/bitcoin-core-dev/](http://gnusha.org/bitcoin-core-dev/). +on [https://www.erisian.com.au/bitcoin-core-dev/](https://www.erisian.com.au/bitcoin-core-dev/) +and [https://gnusha.org/bitcoin-core-dev/](https://gnusha.org/bitcoin-core-dev/). Discussion about codebase improvements happens in GitHub issues and pull requests. @@ -81,7 +81,7 @@ facilitates social contribution, easy testing and peer review. To contribute a patch, the workflow is as follows: - 1. Fork repository ([only for the first time](https://help.github.com/en/articles/fork-a-repo)) + 1. Fork repository ([only for the first time](https://docs.github.com/en/get-started/quickstart/fork-a-repo)) 1. Create topic branch 1. Commit patches @@ -182,7 +182,7 @@ for more information on helping with translations. ### Work in Progress Changes and Requests for Comments If a pull request is not to be considered for merging (yet), please -prefix the title with [WIP] or use [Tasks Lists](https://help.github.com/articles/basic-writing-and-formatting-syntax/#task-lists) +prefix the title with [WIP] or use [Tasks Lists](https://docs.github.com/en/github/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#task-lists) in the body of the pull request to indicate tasks are pending. ### Address Feedback @@ -386,7 +386,7 @@ about: - It may be because your code is too complex for all but a few people, and those people may not have realized your pull request even exists. A great way to find people who are qualified and care about the code you are touching is the - [Git Blame feature](https://help.github.com/articles/tracing-changes-in-a-file/). Simply + [Git Blame feature](https://docs.github.com/en/github/managing-files-in-a-repository/managing-files-on-github/tracking-changes-in-a-file). Simply look up who last modified the code you are changing and see if you can find them and give them a nudge. Don't be incessant about the nudging, though. - Finally, if all else fails, ask on IRC or elsewhere for someone to give your pull request diff --git a/INSTALL.md b/INSTALL.md index 520a47d960..4cead03036 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,5 +1 @@ -Building Bitcoin -================ - -See doc/build-*.md for instructions on building the various -elements of the Bitcoin Core reference implementation of Bitcoin. +See [doc/build-\*.md](/doc)
\ No newline at end of file diff --git a/Makefile.am b/Makefile.am index d84dab1842..ce66331910 100644 --- a/Makefile.am +++ b/Makefile.am @@ -3,7 +3,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. # Pattern rule to print variables, e.g. make print-top_srcdir -print-%: +print-%: FORCE @echo '$*'='$($*)' ACLOCAL_AMFLAGS = -I build-aux/m4 @@ -58,6 +58,7 @@ DIST_SHARE = \ BIN_CHECKS=$(top_srcdir)/contrib/devtools/symbol-check.py \ $(top_srcdir)/contrib/devtools/security-check.py \ + $(top_srcdir)/contrib/devtools/utils.py \ $(top_srcdir)/contrib/devtools/pixie.py WINDOWS_PACKAGING = $(top_srcdir)/share/pixmaps/bitcoin.ico \ @@ -285,7 +286,7 @@ EXTRA_DIST += \ test/fuzz EXTRA_DIST += \ - test/util/bitcoin-util-test.py \ + test/util/test_runner.py \ test/util/data/bitcoin-util-test.json \ test/util/data/blanktxv1.hex \ test/util/data/blanktxv1.json \ @@ -366,14 +367,14 @@ clean-local: clean-docs test-security-check: if TARGET_DARWIN - $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_MACHO - $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_MACHO + $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_MACHO + $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_MACHO endif if TARGET_WINDOWS - $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_PE - $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_PE + $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_PE + $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_PE endif if TARGET_LINUX - $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_ELF - $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_ELF + $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_ELF + $(AM_V_at) CC='$(CC)' CPPFILT='$(CPPFILT)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_ELF endif @@ -41,7 +41,6 @@ /doc/dependencies.md @fanquake /doc/developer-notes.md @laanwj /doc/files.md @hebasto -/doc/gitian-building.md @laanwj /doc/reduce-memory.md @fanquake /doc/reduce-traffic.md @jonasschnelli /doc/release-process.md @laanwj @@ -77,8 +76,7 @@ /contrib/devtools/test-security-check.py @fanquake /contrib/devtools/symbol-check.py @fanquake -# Gitian/Guix -/contrib/gitian-build.py @hebasto +# Guix /contrib/guix/ @dongcarl # Compatibility diff --git a/build-aux/m4/bitcoin_find_bdb48.m4 b/build-aux/m4/bitcoin_find_bdb48.m4 index 5fc5b493d3..3d6c8210ed 100644 --- a/build-aux/m4/bitcoin_find_bdb48.m4 +++ b/build-aux/m4/bitcoin_find_bdb48.m4 @@ -48,15 +48,22 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ if test "x$bdbpath" = "xX"; then use_bdb=no AC_MSG_RESULT([no]) - AC_MSG_ERROR([libdb_cxx headers missing, ]AC_PACKAGE_NAME[ requires this library for BDB wallet support (--without-bdb to disable BDB wallet support)]) + AC_MSG_WARN([libdb_cxx headers missing]) + AC_MSG_WARN(AC_PACKAGE_NAME[ requires this library for BDB (legacy) wallet support]) + AC_MSG_WARN([Passing --without-bdb will suppress this warning]) elif test "x$bdb48path" = "xX"; then BITCOIN_SUBDIR_TO_INCLUDE(BDB_CPPFLAGS,[${bdbpath}],db_cxx) AC_ARG_WITH([incompatible-bdb],[AS_HELP_STRING([--with-incompatible-bdb], [allow using a bdb version other than 4.8])],[ - AC_MSG_WARN([Found Berkeley DB other than 4.8; BDB wallets opened by this build will not be portable!]) + AC_MSG_WARN([Found Berkeley DB other than 4.8]) + AC_MSG_WARN([BDB (legacy) wallets opened by this build will not be portable!]) + use_bdb=yes ],[ - AC_MSG_ERROR([Found Berkeley DB other than 4.8, required for portable BDB wallets (--with-incompatible-bdb to ignore or --without-bdb to disable BDB wallet support)]) + AC_MSG_WARN([Found Berkeley DB other than 4.8]) + AC_MSG_WARN([BDB (legacy) wallets opened by this build would not be portable!]) + AC_MSG_WARN([If this is intended, pass --with-incompatible-bdb]) + AC_MSG_WARN([Passing --without-bdb will suppress this warning]) + use_bdb=no ]) - use_bdb=yes else BITCOIN_SUBDIR_TO_INCLUDE(BDB_CPPFLAGS,[${bdb48path}],db_cxx) bdbpath="${bdb48path}" @@ -78,7 +85,9 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ ]) done if test "x$BDB_LIBS" = "x"; then - AC_MSG_ERROR([libdb_cxx missing, ]AC_PACKAGE_NAME[ requires this library for BDB wallet support (--without-bdb to disable BDB wallet support)]) + AC_MSG_WARN([libdb_cxx headers missing]) + AC_MSG_WARN(AC_PACKAGE_NAME[ requires this library for BDB (legacy) wallet support]) + AC_MSG_WARN([Passing --without-bdb will suppress this warning]) fi fi if test "x$use_bdb" != "xno"; then diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index 5b5a8ed16e..1e979edf0f 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -350,7 +350,7 @@ AC_DEFUN([_BITCOIN_QT_CHECK_STATIC_LIBS], [ PKG_CHECK_MODULES([QT_FONTDATABASE], [${qt_lib_prefix}FontDatabaseSupport${qt_lib_suffix}], [QT_LIBS="$QT_FONTDATABASE_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_THEME], [${qt_lib_prefix}ThemeSupport${qt_lib_suffix}], [QT_LIBS="$QT_THEME_LIBS $QT_LIBS"]) if test "x$TARGET_OS" = xlinux; then - PKG_CHECK_MODULES([QT_INPUT], [${qt_lib_prefix}XcbQpa], [QT_LIBS="$QT_INPUT_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_INPUT], [${qt_lib_prefix}InputSupport], [QT_LIBS="$QT_INPUT_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_SERVICE], [${qt_lib_prefix}ServiceSupport], [QT_LIBS="$QT_SERVICE_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_XCBQPA], [${qt_lib_prefix}XcbQpa], [QT_LIBS="$QT_XCBQPA_LIBS $QT_LIBS"]) elif test "x$TARGET_OS" = xdarwin; then diff --git a/build-aux/m4/bitcoin_runtime_lib.m4 b/build-aux/m4/bitcoin_runtime_lib.m4 new file mode 100644 index 0000000000..1a6922deca --- /dev/null +++ b/build-aux/m4/bitcoin_runtime_lib.m4 @@ -0,0 +1,42 @@ +# On some platforms clang builtin implementations +# require compiler-rt as a runtime library to use. +# +# See: +# - https://bugs.llvm.org/show_bug.cgi?id=28629 + +m4_define([_CHECK_RUNTIME_testbody], [[ + bool f(long long x, long long y, long long* p) + { + return __builtin_mul_overflow(x, y, p); + } + int main() { return 0; } +]]) + +AC_DEFUN([CHECK_RUNTIME_LIB], [ + + AC_LANG_PUSH([C++]) + + AC_MSG_CHECKING([for __builtin_mul_overflow]) + AC_LINK_IFELSE( + [AC_LANG_SOURCE([_CHECK_RUNTIME_testbody])], + [ + AC_MSG_RESULT([yes]) + AC_DEFINE([HAVE_BUILTIN_MUL_OVERFLOW], [1], [Define if you have a working __builtin_mul_overflow]) + ], + [ + ax_check_save_flags="$LDFLAGS" + LDFLAGS="$LDFLAGS --rtlib=compiler-rt -lgcc_s" + AC_LINK_IFELSE( + [AC_LANG_SOURCE([_CHECK_RUNTIME_testbody])], + [ + AC_MSG_RESULT([yes, with additional linker flags]) + RUNTIME_LDFLAGS="--rtlib=compiler-rt -lgcc_s" + AC_DEFINE([HAVE_BUILTIN_MUL_OVERFLOW], [1], [Define if you have a working __builtin_mul_overflow]) + ], + [AC_MSG_RESULT([no])]) + LDFLAGS="$ax_check_save_flags" + ]) + + AC_LANG_POP + AC_SUBST([RUNTIME_LDFLAGS]) +]) diff --git a/build_msvc/README.md b/build_msvc/README.md index 88a05644a7..36fb942c8e 100644 --- a/build_msvc/README.md +++ b/build_msvc/README.md @@ -3,78 +3,65 @@ Building Bitcoin Core with Visual Studio Introduction --------------------- -Solution and project files to build the Bitcoin Core applications `msbuild` or Visual Studio can be found in the `build_msvc` directory. The build has been tested with Visual Studio 2019 (building with earlier versions of Visual Studio should not be expected to work). +Solution and project files to build Bitcoin Core with `msbuild` or Visual Studio can be found in the `build_msvc` directory. The build has been tested with Visual Studio 2019 (building with earlier versions of Visual Studio should not be expected to work). -Building with Visual Studio is an alternative to the Linux based [cross-compiler build](https://github.com/bitcoin/bitcoin/blob/master/doc/build-windows.md). +To build Bitcoin Core from the command-line, it is sufficient to only install the Visual Studio Build Tools component. -Quick Start ---------------------- -The minimal steps required to build Bitcoin Core with the msbuild toolchain are below. More detailed instructions are contained in the following sections. +Building with Visual Studio is an alternative to the Linux based [cross-compiler build](../doc/build-windows.md). -``` -cd build_msvc -py -3 msvc-autogen.py -msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build -``` -Dependencies +Prerequisites --------------------- -A number of [open source libraries](https://github.com/bitcoin/bitcoin/blob/master/doc/dependencies.md) are required in order to be able to build Bitcoin Core. - -Options for installing the dependencies in a Visual Studio compatible manner are: +To build [dependencies](../doc/dependencies.md) (except for [Qt](#qt)), +the default approach is to use the [vcpkg](https://docs.microsoft.com/en-us/cpp/vcpkg) package manager from Microsoft: -- Use Microsoft's [vcpkg](https://docs.microsoft.com/en-us/cpp/vcpkg) to download the source packages and build locally. This is the recommended approach. -- Download the source code, build each dependency, add the required include paths, link libraries and binary tools to the Visual Studio project files. -- Use [nuget](https://www.nuget.org/) packages with the understanding that any binary files have been compiled by an untrusted third party. +1. [Install](https://vcpkg.io/en/getting-started.html) vcpkg. -The [external dependencies](https://github.com/bitcoin/bitcoin/blob/master/doc/dependencies.md) required for building are listed in the `build_msvc/vcpkg.json` file. To ensure `msbuild` project files automatically install the `vcpkg` dependencies use: +2. By default, vcpkg makes both `release` and `debug` builds for each package. +To save build time and disk space, one could skip `debug` builds (example uses PowerShell): +```powershell -``` -vcpkg integrate install +Add-Content -Path "vcpkg\triplets\x64-windows-static.cmake" -Value "set(VCPKG_BUILD_TYPE release)" ``` Qt --------------------- -In order to build Bitcoin Core a static build of Qt is required. The runtime library version (e.g. v142) and platform type (x86 or x64) must also match. +To build Bitcoin Core with the GUI, a static build of Qt is required. + +1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-src-5.12.11.zip`](https://download.qt.io/official_releases/qt/5.12/5.12.11/single/qt-everywhere-src-5.12.11.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`. + +2. Open "x64 Native Tools Command Prompt for VS 2019", and input the following commands: +```cmd +cd C:\dev\qt-source +mkdir build +cd build +..\configure -release -silent -opensource -confirm-license -opengl desktop -no-shared -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -no-libjpeg -nomake examples -nomake tests -nomake tools -no-dbus -no-libudev -no-icu -no-gtk -no-opengles3 -no-angle -no-sql-sqlite -no-sql-odbc -no-sqlite -no-libudev -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcanvas3d -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquickcontrols -skip qtquickcontrols2 -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-sql -no-feature-sqlmodel -prefix C:\Qt_static +nmake +nmake install +``` -Some prebuilt x64 versions of Qt can be downloaded from [here](https://github.com/sipsorcery/qt_win_binary/releases). Please be aware these downloads are NOT officially sanctioned by Bitcoin Core and are provided for developer convenience only. They should NOT be used for builds that will be used in a production environment or with real funds. +One could speed up building with [`jom`](https://wiki.qt.io/Jom), a replacement for `nmake` which makes use of all CPU cores. -To determine which Qt prebuilt version to download open the `.appveyor.yml` file and note the `QT_DOWNLOAD_URL`. When extracting the zip file the destination path must be set to `C:\`. This is due to the way that Qt includes, libraries and tools use internal paths. +To build Bitcoin Core without Qt, unload or disable the `bitcoin-qt`, `libbitcoin_qt` and `test_bitcoin-qt` projects. -To build Bitcoin Core without Qt unload or disable the `bitcoin-qt`, `libbitcoin_qt` and `test_bitcoin-qt` projects. Building --------------------- -The instructions below use `vcpkg` to install the dependencies. - -- Install [`vcpkg`](https://github.com/Microsoft/vcpkg). - -- Use Python to generate `*.vcxproj` from Makefile +1. Use Python to generate `*.vcxproj` from Makefile: ``` PS >py -3 msvc-autogen.py ``` -- An optional step is to adjust the settings in the `build_msvc` directory and the `common.init.vcxproj` file. This project file contains settings that are common to all projects such as the runtime library version and target Windows SDK version. The Qt directories can also be set. +2. An optional step is to adjust the settings in the `build_msvc` directory and the `common.init.vcxproj` file. This project file contains settings that are common to all projects such as the runtime library version and target Windows SDK version. The Qt directories can also be set. To specify a non-default path to a static Qt package directory, use the `QTBASEDIR` environment variable. -- To build from the command line with the Visual Studio 2019 toolchain use: +3. To build from the command-line with the Visual Studio 2019 toolchain use: +```cmd +msbuild -property:Configuration=Release -maxCpuCount -verbosity:minimal bitcoin.sln ``` -msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build -``` - -- Alternatively, open the `build_msvc/bitcoin.sln` file in Visual Studio 2019. - -AppVeyor ---------------------- -The .appveyor.yml in the root directory is suitable to perform builds on [AppVeyor](https://www.appveyor.com/) Continuous Integration servers. The simplest way to perform an AppVeyor build is to fork Bitcoin Core and then configure a new AppVeyor Project pointing to the forked repository. -For safety reasons the Bitcoin Core .appveyor.yml file has the artifact options disabled. The build will be performed but no executable files will be available. To enable artifacts on a forked repository uncomment the lines shown below: - -``` - #- 7z a bitcoin-%APPVEYOR_BUILD_VERSION%.zip %APPVEYOR_BUILD_FOLDER%\build_msvc\%platform%\%configuration%\*.exe - #- path: bitcoin-%APPVEYOR_BUILD_VERSION%.zip -``` +Alternatively, open the `build_msvc/bitcoin.sln` file in Visual Studio 2019. Security --------------------- diff --git a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj index a697c1dfb6..724dae1969 100644 --- a/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj +++ b/build_msvc/bitcoin-qt/bitcoin-qt.vcxproj @@ -9,6 +9,7 @@ </PropertyGroup> <ItemGroup> <ClCompile Include="..\..\src\qt\main.cpp" /> + <ClCompile Include="..\..\src\init\bitcoind.cpp" /> <ResourceCompile Include="..\..\src\qt\res\bitcoin-qt-res.rc" /> </ItemGroup> <ItemGroup> diff --git a/build_msvc/bitcoin-util/bitcoin-util.vcxproj b/build_msvc/bitcoin-util/bitcoin-util.vcxproj index 3a6aa4a837..8a0964824b 100644 --- a/build_msvc/bitcoin-util/bitcoin-util.vcxproj +++ b/build_msvc/bitcoin-util/bitcoin-util.vcxproj @@ -2,7 +2,7 @@ <Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Import Project="..\common.init.vcxproj" /> <PropertyGroup Label="Globals"> - <ProjectGuid>{D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}</ProjectGuid> + <ProjectGuid>{57A04EC9-542A-4E40-83D0-AC3BE1F36805}</ProjectGuid> </PropertyGroup> <PropertyGroup Label="Configuration"> <ConfigurationType>Application</ConfigurationType> diff --git a/build_msvc/bitcoin.sln b/build_msvc/bitcoin.sln index 7d8591c10b..1b1f27a8a9 100644 --- a/build_msvc/bitcoin.sln +++ b/build_msvc/bitcoin.sln @@ -32,7 +32,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bench_bitcoin", "bench_bitc EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bitcoin-tx", "bitcoin-tx\bitcoin-tx.vcxproj", "{D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bitcoin-util", "bitcoin-util\bitcoin-util.vcxproj", "{D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bitcoin-util", "bitcoin-util\bitcoin-util.vcxproj", "{57A04EC9-542A-4E40-83D0-AC3BE1F36805}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bitcoin-wallet", "bitcoin-wallet\bitcoin-wallet.vcxproj", "{84DE8790-EDE3-4483-81AC-C32F15E861F4}" EndProject @@ -53,195 +53,105 @@ EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 Release|x64 = Release|x64 - Release|x86 = Release|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Debug|x64.ActiveCfg = Debug|x64 {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Debug|x64.Build.0 = Debug|x64 - {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Debug|x86.ActiveCfg = Debug|Win32 - {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Debug|x86.Build.0 = Debug|Win32 {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Release|x64.ActiveCfg = Release|x64 {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Release|x64.Build.0 = Release|x64 - {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Release|x86.ActiveCfg = Release|Win32 - {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Release|x86.Build.0 = Release|Win32 {E78473E9-B850-456C-9120-276301E04C06}.Debug|x64.ActiveCfg = Debug|x64 {E78473E9-B850-456C-9120-276301E04C06}.Debug|x64.Build.0 = Debug|x64 - {E78473E9-B850-456C-9120-276301E04C06}.Debug|x86.ActiveCfg = Debug|Win32 - {E78473E9-B850-456C-9120-276301E04C06}.Debug|x86.Build.0 = Debug|Win32 {E78473E9-B850-456C-9120-276301E04C06}.Release|x64.ActiveCfg = Release|x64 {E78473E9-B850-456C-9120-276301E04C06}.Release|x64.Build.0 = Release|x64 - {E78473E9-B850-456C-9120-276301E04C06}.Release|x86.ActiveCfg = Release|Win32 - {E78473E9-B850-456C-9120-276301E04C06}.Release|x86.Build.0 = Release|Win32 {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Debug|x64.ActiveCfg = Debug|x64 {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Debug|x64.Build.0 = Debug|x64 - {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Debug|x86.ActiveCfg = Debug|Win32 - {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Debug|x86.Build.0 = Debug|Win32 {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Release|x64.ActiveCfg = Release|x64 {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Release|x64.Build.0 = Release|x64 - {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Release|x86.ActiveCfg = Release|Win32 - {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Release|x86.Build.0 = Release|Win32 {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Debug|x64.ActiveCfg = Debug|x64 {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Debug|x64.Build.0 = Debug|x64 - {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Debug|x86.ActiveCfg = Debug|Win32 - {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Debug|x86.Build.0 = Debug|Win32 {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Release|x64.ActiveCfg = Release|x64 {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Release|x64.Build.0 = Release|x64 - {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Release|x86.ActiveCfg = Release|Win32 - {B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}.Release|x86.Build.0 = Release|Win32 {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Debug|x64.ActiveCfg = Debug|x64 {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Debug|x64.Build.0 = Debug|x64 - {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Debug|x86.ActiveCfg = Debug|Win32 - {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Debug|x86.Build.0 = Debug|Win32 {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Release|x64.ActiveCfg = Release|x64 {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Release|x64.Build.0 = Release|x64 - {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Release|x86.ActiveCfg = Release|Win32 - {7C87E378-DF58-482E-AA2F-1BC129BC19CE}.Release|x86.Build.0 = Release|Win32 {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Debug|x64.ActiveCfg = Debug|x64 {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Debug|x64.Build.0 = Debug|x64 - {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Debug|x86.ActiveCfg = Debug|Win32 - {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Debug|x86.Build.0 = Debug|Win32 {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Release|x64.ActiveCfg = Release|x64 {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Release|x64.Build.0 = Release|x64 - {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Release|x86.ActiveCfg = Release|Win32 - {6190199C-6CF4-4DAD-BFBD-93FA72A760C1}.Release|x86.Build.0 = Release|Win32 {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Debug|x64.ActiveCfg = Debug|x64 {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Debug|x64.Build.0 = Debug|x64 - {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Debug|x86.ActiveCfg = Debug|Win32 - {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Debug|x86.Build.0 = Debug|Win32 {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Release|x64.ActiveCfg = Release|x64 {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Release|x64.Build.0 = Release|x64 - {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Release|x86.ActiveCfg = Release|Win32 - {460FEE33-1FE1-483F-B3BF-931FF8E969A5}.Release|x86.Build.0 = Release|Win32 {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Debug|x64.ActiveCfg = Debug|x64 {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Debug|x64.Build.0 = Debug|x64 - {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Debug|x86.ActiveCfg = Debug|Win32 - {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Debug|x86.Build.0 = Debug|Win32 {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Release|x64.ActiveCfg = Release|x64 {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Release|x64.Build.0 = Release|x64 - {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Release|x86.ActiveCfg = Release|Win32 - {5724BA7D-A09A-4BA8-800B-C4C1561B3D69}.Release|x86.Build.0 = Release|Win32 {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Debug|x64.ActiveCfg = Debug|x64 {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Debug|x64.Build.0 = Debug|x64 - {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Debug|x86.ActiveCfg = Debug|Win32 - {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Debug|x86.Build.0 = Debug|Win32 {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Release|x64.ActiveCfg = Release|x64 {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Release|x64.Build.0 = Release|x64 - {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Release|x86.ActiveCfg = Release|Win32 - {93B86837-B543-48A5-A89B-7C87ABB77DF2}.Release|x86.Build.0 = Release|Win32 {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Debug|x64.ActiveCfg = Debug|x64 {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Debug|x64.Build.0 = Debug|x64 - {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Debug|x86.ActiveCfg = Debug|Win32 - {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Debug|x86.Build.0 = Debug|Win32 {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Release|x64.ActiveCfg = Release|x64 {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Release|x64.Build.0 = Release|x64 - {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Release|x86.ActiveCfg = Release|Win32 - {792D487F-F14C-49FC-A9DE-3FC150F31C3F}.Release|x86.Build.0 = Release|Win32 {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Debug|x64.ActiveCfg = Debug|x64 {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Debug|x64.Build.0 = Debug|x64 - {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Debug|x86.ActiveCfg = Debug|Win32 - {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Debug|x86.Build.0 = Debug|Win32 {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Release|x64.ActiveCfg = Release|x64 {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Release|x64.Build.0 = Release|x64 - {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Release|x86.ActiveCfg = Release|Win32 - {A56B73DB-D46D-4882-8374-1FE3FFA08F07}.Release|x86.Build.0 = Release|Win32 {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Debug|x64.ActiveCfg = Debug|x64 {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Debug|x64.Build.0 = Debug|x64 - {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Debug|x86.ActiveCfg = Debug|Win32 - {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Debug|x86.Build.0 = Debug|Win32 {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Release|x64.ActiveCfg = Release|x64 {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Release|x64.Build.0 = Release|x64 - {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Release|x86.ActiveCfg = Release|Win32 - {0667528C-D734-4009-ADF9-C0D6C4A5A5A6}.Release|x86.Build.0 = Release|Win32 {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Debug|x64.ActiveCfg = Debug|x64 {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Debug|x64.Build.0 = Debug|x64 - {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Debug|x86.ActiveCfg = Debug|Win32 - {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Debug|x86.Build.0 = Debug|Win32 {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Release|x64.ActiveCfg = Release|x64 {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Release|x64.Build.0 = Release|x64 - {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Release|x86.ActiveCfg = Release|Win32 - {0B2D7431-F876-4A58-87BF-F748338CD3BF}.Release|x86.Build.0 = Release|Win32 {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Debug|x64.ActiveCfg = Debug|x64 {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Debug|x64.Build.0 = Debug|x64 - {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Debug|x86.ActiveCfg = Debug|Win32 - {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Debug|x86.Build.0 = Debug|Win32 {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Release|x64.ActiveCfg = Release|x64 {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Release|x64.Build.0 = Release|x64 - {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Release|x86.ActiveCfg = Release|Win32 - {1125654E-E1B2-4431-8B5C-62EA9A2FEECB}.Release|x86.Build.0 = Release|Win32 {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Debug|x64.ActiveCfg = Debug|x64 {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Debug|x64.Build.0 = Debug|x64 - {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Debug|x86.ActiveCfg = Debug|Win32 - {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Debug|x86.Build.0 = Debug|Win32 {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Release|x64.ActiveCfg = Release|x64 {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Release|x64.Build.0 = Release|x64 - {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Release|x86.ActiveCfg = Release|Win32 - {D3022AF6-AD33-4CE3-B358-87CB6A1B29CF}.Release|x86.Build.0 = Release|Win32 + {57A04EC9-542A-4E40-83D0-AC3BE1F36805}.Debug|x64.ActiveCfg = Debug|x64 + {57A04EC9-542A-4E40-83D0-AC3BE1F36805}.Debug|x64.Build.0 = Debug|x64 + {57A04EC9-542A-4E40-83D0-AC3BE1F36805}.Release|x64.ActiveCfg = Release|x64 + {57A04EC9-542A-4E40-83D0-AC3BE1F36805}.Release|x64.Build.0 = Release|x64 {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Debug|x64.ActiveCfg = Debug|x64 {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Debug|x64.Build.0 = Debug|x64 - {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Debug|x86.ActiveCfg = Debug|Win32 - {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Debug|x86.Build.0 = Debug|Win32 {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Release|x64.ActiveCfg = Release|x64 {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Release|x64.Build.0 = Release|x64 - {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Release|x86.ActiveCfg = Release|Win32 - {84DE8790-EDE3-4483-81AC-C32F15E861F4}.Release|x86.Build.0 = Release|Win32 {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Debug|x64.ActiveCfg = Debug|x64 {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Debug|x64.Build.0 = Debug|x64 - {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Debug|x86.ActiveCfg = Debug|Win32 - {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Debug|x86.Build.0 = Debug|Win32 {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Release|x64.ActiveCfg = Release|x64 {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Release|x64.Build.0 = Release|x64 - {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Release|x86.ActiveCfg = Release|Win32 - {F91AC55E-6F5E-4C58-9AC5-B40DB7DEEF93}.Release|x86.Build.0 = Release|Win32 {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Debug|x64.ActiveCfg = Debug|x64 {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Debug|x64.Build.0 = Debug|x64 - {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Debug|x86.ActiveCfg = Debug|Win32 - {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Debug|x86.Build.0 = Debug|Win32 {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Release|x64.ActiveCfg = Release|x64 {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Release|x64.Build.0 = Release|x64 - {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Release|x86.ActiveCfg = Release|Win32 - {BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}.Release|x86.Build.0 = Release|Win32 {18430FEF-6B61-4C53-B396-718E02850F1B}.Debug|x64.ActiveCfg = Debug|x64 {18430FEF-6B61-4C53-B396-718E02850F1B}.Debug|x64.Build.0 = Debug|x64 - {18430FEF-6B61-4C53-B396-718E02850F1B}.Debug|x86.ActiveCfg = Debug|Win32 - {18430FEF-6B61-4C53-B396-718E02850F1B}.Debug|x86.Build.0 = Debug|Win32 {18430FEF-6B61-4C53-B396-718E02850F1B}.Release|x64.ActiveCfg = Release|x64 {18430FEF-6B61-4C53-B396-718E02850F1B}.Release|x64.Build.0 = Release|x64 - {18430FEF-6B61-4C53-B396-718E02850F1B}.Release|x86.ActiveCfg = Release|Win32 - {18430FEF-6B61-4C53-B396-718E02850F1B}.Release|x86.Build.0 = Release|Win32 {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Debug|x64.ActiveCfg = Debug|x64 {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Debug|x64.Build.0 = Debug|x64 - {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Debug|x86.ActiveCfg = Debug|Win32 - {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Debug|x86.Build.0 = Debug|Win32 {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Release|x64.ActiveCfg = Release|x64 {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Release|x64.Build.0 = Release|x64 - {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Release|x86.ActiveCfg = Release|Win32 - {2B4ABFF8-D1FD-4845-88C9-1F3C0A6512BF}.Release|x86.Build.0 = Release|Win32 {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Debug|x64.ActiveCfg = Debug|x64 {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Debug|x64.Build.0 = Debug|x64 - {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Debug|x86.ActiveCfg = Debug|Win32 - {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Debug|x86.Build.0 = Debug|Win32 {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Release|x64.ActiveCfg = Release|x64 {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Release|x64.Build.0 = Release|x64 - {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Release|x86.ActiveCfg = Release|Win32 - {7E99172D-7FF2-4CB6-B736-AC9B76ED412A}.Release|x86.Build.0 = Release|Win32 {868474FD-35F6-4400-8EED-30A33E7521D4}.Debug|x64.ActiveCfg = Debug|x64 {868474FD-35F6-4400-8EED-30A33E7521D4}.Debug|x64.Build.0 = Debug|x64 - {868474FD-35F6-4400-8EED-30A33E7521D4}.Debug|x86.ActiveCfg = Debug|Win32 - {868474FD-35F6-4400-8EED-30A33E7521D4}.Debug|x86.Build.0 = Debug|Win32 {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x64.ActiveCfg = Release|x64 {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x64.Build.0 = Release|x64 - {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x86.ActiveCfg = Release|Win32 - {868474FD-35F6-4400-8EED-30A33E7521D4}.Release|x86.Build.0 = Release|Win32 {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x64.ActiveCfg = Debug|x64 {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x64.Build.0 = Debug|x64 - {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x86.ActiveCfg = Debug|Win32 - {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x86.Build.0 = Debug|Win32 {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x64.ActiveCfg = Release|x64 {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x64.Build.0 = Release|x64 - {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x86.ActiveCfg = Release|Win32 - {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x86.Build.0 = Release|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h index aba5607eb6..e2930f3ea9 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h @@ -5,9 +5,6 @@ #ifndef BITCOIN_BITCOIN_CONFIG_H #define BITCOIN_BITCOIN_CONFIG_H -/* Define if building universal (internal helper macro) */ -/* #undef AC_APPLE_UNIVERSAL_BUILD */ - /* Version Build */ #define CLIENT_VERSION_BUILD 0 @@ -15,7 +12,7 @@ #define CLIENT_VERSION_IS_RELEASE false /* Major version */ -#define CLIENT_VERSION_MAJOR 21 +#define CLIENT_VERSION_MAJOR 22 /* Minor version */ #define CLIENT_VERSION_MINOR 99 @@ -30,7 +27,7 @@ #define COPYRIGHT_HOLDERS_SUBSTITUTION "Bitcoin Core" /* Copyright year */ -#define COPYRIGHT_YEAR 2019 +#define COPYRIGHT_YEAR 2021 /* Define to 1 to enable wallet functions */ #define ENABLE_WALLET 1 @@ -59,14 +56,11 @@ /* define if the Boost::Unit_Test_Framework library is available */ #define HAVE_BOOST_UNIT_TEST_FRAMEWORK /**/ -/* Define to 1 if you have the <byteswap.h> header file. */ -/* #undef HAVE_BYTESWAP_H */ - /* Define this symbol if the consensus lib has been built */ #define HAVE_CONSENSUS_LIB 1 -/* define if the compiler supports basic C++11 syntax */ -#define HAVE_CXX11 1 +/* define if the compiler supports basic C++17 syntax */ +#define HAVE_CXX17 1 /* Define to 1 if you have the declaration of `be16toh', and to 0 if you don't. */ @@ -144,37 +138,12 @@ don't. */ #define HAVE_DECL_STRNLEN 1 -/* Define to 1 if you have the <dlfcn.h> header file. */ -/* #undef HAVE_DLFCN_H */ - -/* Define to 1 if you have the <endian.h> header file. */ -/* #undef HAVE_ENDIAN_H */ - -/* Define to 1 if the system has the `dllexport' function attribute */ -#define HAVE_FUNC_ATTRIBUTE_DLLEXPORT 1 - -/* Define to 1 if the system has the `dllimport' function attribute */ -#define HAVE_FUNC_ATTRIBUTE_DLLIMPORT 1 - -/* Define to 1 if the system has the `visibility' function attribute */ -#define HAVE_FUNC_ATTRIBUTE_VISIBILITY 1 - -/* Define this symbol if the BSD getentropy system call is available */ -/* #undef HAVE_GETENTROPY */ - -/* Define this symbol if the BSD getentropy system call is available with - sys/random.h */ -/* #undef HAVE_GETENTROPY_RAND */ +/* Define if the dllexport attribute is supported. */ +#define HAVE_DLLEXPORT_ATTRIBUTE 1 /* Define to 1 if you have the <inttypes.h> header file. */ #define HAVE_INTTYPES_H 1 -/* Define this symbol if you have malloc_info */ -/* #undef HAVE_MALLOC_INFO */ - -/* Define this symbol if you have mallopt with M_ARENA_MAX */ -/* #undef HAVE_MALLOPT_ARENA_MAX */ - /* Define to 1 if you have the <memory.h> header file. */ #define HAVE_MEMORY_H 1 @@ -187,18 +156,6 @@ /* Define to 1 if you have the <miniupnpc/upnperrors.h> header file. */ #define HAVE_MINIUPNPC_UPNPERRORS_H 1 -/* Define this symbol if you have MSG_DONTWAIT */ -/* #undef HAVE_MSG_DONTWAIT */ - -/* Define this symbol if you have MSG_NOSIGNAL */ -/* #undef HAVE_MSG_NOSIGNAL */ - -/* Define if you have POSIX threads libraries and header files. */ -//#define HAVE_PTHREAD 1 - -/* Have PTHREAD_PRIO_INHERIT. */ -//#define HAVE_PTHREAD_PRIO_INHERIT 1 - /* Define to 1 if you have the <stdint.h> header file. */ #define HAVE_STDINT_H 1 @@ -208,45 +165,18 @@ /* Define to 1 if you have the <stdlib.h> header file. */ #define HAVE_STDLIB_H 1 -/* Define to 1 if you have the `strerror_r' function. */ -/* #undef HAVE_STRERROR_R */ - /* Define to 1 if you have the <strings.h> header file. */ #define HAVE_STRINGS_H 1 /* Define to 1 if you have the <string.h> header file. */ #define HAVE_STRING_H 1 -/* Define this symbol if the BSD sysctl(KERN_ARND) is available */ -/* #undef HAVE_SYSCTL_ARND */ - -/* Define to 1 if you have the <sys/endian.h> header file. */ -/* #undef HAVE_SYS_ENDIAN_H */ - -/* Define this symbol if the Linux getrandom system call is available */ -/* #undef HAVE_SYS_GETRANDOM */ - -/* Define to 1 if you have the <sys/prctl.h> header file. */ -/* #undef HAVE_SYS_PRCTL_H */ - -/* Define to 1 if you have the <sys/select.h> header file. */ -/* #undef HAVE_SYS_SELECT_H */ - /* Define to 1 if you have the <sys/stat.h> header file. */ #define HAVE_SYS_STAT_H 1 /* Define to 1 if you have the <sys/types.h> header file. */ #define HAVE_SYS_TYPES_H 1 -/* Define to 1 if you have the <unistd.h> header file. */ -//#define HAVE_UNISTD_H 1 - -/* Define if the visibility attribute is supported. */ -#define HAVE_VISIBILITY_ATTRIBUTE 1 - -/* Define to the sub-directory where libtool stores uninstalled libraries. */ -#define LT_OBJDIR ".libs/" - /* Define to the address where bug reports for this package should be sent. */ #define PACKAGE_BUGREPORT "https://github.com/bitcoin/bitcoin/issues" @@ -254,23 +184,13 @@ #define PACKAGE_NAME "Bitcoin Core" /* Define to the full name and version of this package. */ -#define PACKAGE_STRING "Bitcoin Core 21.99.0" - -/* Define to the one symbol short name of this package. */ -#define PACKAGE_TARNAME "bitcoin" +#define PACKAGE_STRING "Bitcoin Core 22.99.0" /* Define to the home page for this package. */ #define PACKAGE_URL "https://bitcoincore.org/" /* Define to the version of this package. */ -#define PACKAGE_VERSION "21.99.0" - -/* Define to necessary symbol if this constant uses a non-standard name on - your system. */ -/* #undef PTHREAD_CREATE_JOINABLE */ - -/* Define this symbol if the qt platform is cocoa */ -/* #undef QT_QPA_PLATFORM_COCOA */ +#define PACKAGE_VERSION "22.99.0" /* Define this symbol if the minimal qt platform exists */ #define QT_QPA_PLATFORM_MINIMAL 1 @@ -278,54 +198,9 @@ /* Define this symbol if the qt platform is windows */ #define QT_QPA_PLATFORM_WINDOWS 1 -/* Define this symbol if the qt platform is xcb */ -/* #undef QT_QPA_PLATFORM_XCB */ - /* Define this symbol if qt plugins are static */ #define QT_STATICPLUGIN 1 -/* Define to 1 if you have the ANSI C header files. */ -#define STDC_HEADERS 1 - -/* Define to 1 if strerror_r returns char *. */ -/* #undef STRERROR_R_CHAR_P */ - -/* Define this symbol to build in assembly routines */ -//#define USE_ASM 1 - -/* Define if dbus support should be compiled in */ -/* #undef USE_DBUS */ - -/* Define if QR support should be compiled in */ -//#define USE_QRCODE 1 - -/* UPnP support not compiled if undefined, otherwise value (0 or 1) determines - default state */ -//#define USE_UPNP 0 - -/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most - significant byte first (like Motorola and SPARC, unlike Intel). */ -#if defined AC_APPLE_UNIVERSAL_BUILD -# if defined __BIG_ENDIAN__ -# define WORDS_BIGENDIAN 1 -# endif -#else -# ifndef WORDS_BIGENDIAN -/* # undef WORDS_BIGENDIAN */ -# endif -#endif - -/* Enable large inode numbers on Mac OS X 10.5. */ -#ifndef _DARWIN_USE_64_BIT_INODE -# define _DARWIN_USE_64_BIT_INODE 1 -#endif - -/* Number of bits in a file offset, on hosts where this is settable. */ -#define _FILE_OFFSET_BITS 64 - -/* Define for large files, on AIX-style hosts. */ -/* #undef _LARGE_FILES */ - /* Windows Universal Platform constraints */ #if !defined(WINAPI_FAMILY) || (WINAPI_FAMILY == WINAPI_FAMILY_DESKTOP_APP) /* Either a desktop application without API restrictions, or and older system diff --git a/build_msvc/bitcoind/bitcoind.vcxproj b/build_msvc/bitcoind/bitcoind.vcxproj index c2c32af838..d56c359fe0 100644 --- a/build_msvc/bitcoind/bitcoind.vcxproj +++ b/build_msvc/bitcoind/bitcoind.vcxproj @@ -58,6 +58,8 @@ <ReplaceInFile FilePath="$(ConfigIniOut)" Replace="@PACKAGE_NAME@" By="Bitcoin Core"></ReplaceInFile> <ReplaceInFile FilePath="$(ConfigIniOut)" + Replace="@PACKAGE_BUGREPORT@" By="https://github.com/bitcoin/bitcoin/issues"></ReplaceInFile> + <ReplaceInFile FilePath="$(ConfigIniOut)" Replace="@abs_top_srcdir@" By="..\.." ToFullPath="true"></ReplaceInFile> <ReplaceInFile FilePath="$(ConfigIniOut)" Replace="@abs_top_builddir@" By="..\.." ToFullPath="true"></ReplaceInFile> @@ -72,11 +74,15 @@ <ReplaceInFile FilePath="$(ConfigIniOut)" Replace="@BUILD_BITCOIN_CLI_TRUE@" By=""></ReplaceInFile> <ReplaceInFile FilePath="$(ConfigIniOut)" + Replace="@BUILD_BITCOIN_WALLET_TRUE@" By=""></ReplaceInFile> + <ReplaceInFile FilePath="$(ConfigIniOut)" Replace="@BUILD_BITCOIND_TRUE@" By=""></ReplaceInFile> <ReplaceInFile FilePath="$(ConfigIniOut)" Replace="@ENABLE_FUZZ_TRUE@" By=""></ReplaceInFile> <ReplaceInFile FilePath="$(ConfigIniOut)" Replace="@ENABLE_ZMQ_TRUE@" By=""></ReplaceInFile> + <ReplaceInFile FilePath="$(ConfigIniOut)" + Replace="@ENABLE_EXTERNAL_SIGNER_TRUE@" By=""></ReplaceInFile> </Target> <Import Project="..\common.vcxproj" /> </Project> diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj index 6ea018d846..fb4d70cc78 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj @@ -14,7 +14,6 @@ <VcpkgUseStatic>true</VcpkgUseStatic> <VcpkgAutoLink>true</VcpkgAutoLink> <VcpkgConfiguration>$(Configuration)</VcpkgConfiguration> - <VcpkgTriplet Condition="'$(Platform)'=='Win32'">x86-windows-static</VcpkgTriplet> <VcpkgTriplet Condition="'$(Platform)'=='x64'">x64-windows-static</VcpkgTriplet> </PropertyGroup> @@ -35,14 +34,6 @@ <Configuration>Debug</Configuration> <Platform>x64</Platform> </ProjectConfiguration> - <ProjectConfiguration Include="Release|Win32"> - <Configuration>Release</Configuration> - <Platform>Win32</Platform> - </ProjectConfiguration> - <ProjectConfiguration Include="Debug|Win32"> - <Configuration>Debug</Configuration> - <Platform>Win32</Platform> - </ProjectConfiguration> </ItemGroup> <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration"> diff --git a/build_msvc/common.qt.init.vcxproj b/build_msvc/common.qt.init.vcxproj index ce66a7ab34..cc8063e545 100644 --- a/build_msvc/common.qt.init.vcxproj +++ b/build_msvc/common.qt.init.vcxproj @@ -2,7 +2,7 @@ <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <PropertyGroup Label="QtGlobals"> - <QtBaseDir>C:\Qt5.12.11_x64_static_vs2019_16101</QtBaseDir> + <QtBaseDir Condition="'$(QTBASEDIR)' == ''">C:\Qt_static</QtBaseDir> <QtPluginsLibraryDir>$(QtBaseDir)\plugins</QtPluginsLibraryDir> <QtLibraryDir>$(QtBaseDir)\lib</QtLibraryDir> <QtIncludeDir>$(QtBaseDir)\include</QtIncludeDir> diff --git a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj index 96bb584375..a64ae881f2 100644 --- a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj +++ b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj @@ -24,6 +24,7 @@ <ClCompile Include="..\..\src\qt\csvmodelwriter.cpp" /> <ClCompile Include="..\..\src\qt\editaddressdialog.cpp" /> <ClCompile Include="..\..\src\qt\guiutil.cpp" /> + <ClCompile Include="..\..\src\qt\initexecutor.cpp" /> <ClCompile Include="..\..\src\qt\intro.cpp" /> <ClCompile Include="..\..\src\qt\modaloverlay.cpp" /> <ClCompile Include="..\..\src\qt\networkstyle.cpp" /> @@ -78,6 +79,7 @@ <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_csvmodelwriter.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_editaddressdialog.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_guiutil.cpp" /> + <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_initexecutor.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_intro.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_modaloverlay.cpp" /> <ClCompile Include="$(GeneratedFilesOutDir)\moc\moc_networkstyle.cpp" /> @@ -138,20 +140,6 @@ </ClCompile> </ItemDefinitionGroup> - <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> - <ClCompile> - <PreprocessorDefinitions>_X86_;%(PreprocessorDefinitions)</PreprocessorDefinitions> - <AdditionalIncludeDirectories>$(QtIncludes);$(GeneratedFilesOutDir)\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - </ClCompile> - </ItemDefinitionGroup> - - <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> - <ClCompile> - <PreprocessorDefinitions>_X86_;%(PreprocessorDefinitions)</PreprocessorDefinitions> - <AdditionalIncludeDirectories>$(QtIncludes);$(GeneratedFilesOutDir)\..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - </ClCompile> - </ItemDefinitionGroup> - <ItemGroup> <QT_MOC Include="..\..\src\qt\bitcoinamountfield.cpp" /> <QT_MOC Include="..\..\src\qt\intro.cpp" /> diff --git a/build_msvc/msvc-autogen.py b/build_msvc/msvc-autogen.py index d99b17d381..a1ed935996 100755 --- a/build_msvc/msvc-autogen.py +++ b/build_msvc/msvc-autogen.py @@ -9,7 +9,7 @@ import argparse from shutil import copyfile SOURCE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'src')) -DEFAULT_PLATFORM_TOOLSET = R'v141' +DEFAULT_PLATFORM_TOOLSET = R'v142' libs = [ 'libbitcoin_cli', diff --git a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj index 1d2c86b7ac..08b12bdd85 100644 --- a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj +++ b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj @@ -8,6 +8,7 @@ <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir> </PropertyGroup> <ItemGroup> + <ClCompile Include="..\..\src\init\bitcoind.cpp" /> <ClCompile Include="..\..\src\test\util\setup_common.cpp" /> <ClCompile Include="..\..\src\qt\test\addressbooktests.cpp" /> <ClCompile Include="..\..\src\qt\test\apptests.cpp" /> diff --git a/build_msvc/test_bitcoin/test_bitcoin.vcxproj b/build_msvc/test_bitcoin/test_bitcoin.vcxproj index 5c4b777d51..bb1a780bfa 100644 --- a/build_msvc/test_bitcoin/test_bitcoin.vcxproj +++ b/build_msvc/test_bitcoin/test_bitcoin.vcxproj @@ -16,6 +16,7 @@ <ClCompile Include="..\..\src\test\util\*.cpp" /> <ClCompile Include="..\..\src\wallet\test\*_fixture.cpp" /> <ClCompile Include="..\..\src\wallet\test\*_tests.cpp" /> + <ClCompile Include="..\..\src\wallet\test\util.cpp" /> </ItemGroup> <ItemGroup> <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj"> diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 2c63a9efac..5587618f2d 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -13,7 +13,6 @@ update-alternatives --install /usr/bin/clang-format-diff clang-format-diff $(whi ${CI_RETRY_EXE} pip3 install codespell==2.0.0 ${CI_RETRY_EXE} pip3 install flake8==3.8.3 -${CI_RETRY_EXE} pip3 install yq ${CI_RETRY_EXE} pip3 install mypy==0.781 ${CI_RETRY_EXE} pip3 install vulture==2.3 diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index e38cfe8eef..f7dacd8512 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -23,10 +23,15 @@ test/lint/git-subtree-check.sh src/crc32c test/lint/check-doc.py test/lint/lint-all.sh -if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ -n "$CIRRUS_CRON" ]; then - git log --merges --before="2 days ago" -1 --format='%H' > ./contrib/verify-commits/trusted-sha512-root-commit - ${CI_RETRY_EXE} gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys $(<contrib/verify-commits/trusted-keys) && - ./contrib/verify-commits/verify-commits.py --clean-merge=2; +if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ "$CIRRUS_PR" = "" ] ; then + # Sanity check only the last few commits to get notified of missing sigs, + # missing keys, or expired keys. Usually there is only one new merge commit + # per push on the master branch and a few commits on release branches, so + # sanity checking only a few (10) commits seems sufficient and cheap. + git log HEAD~10 -1 --format='%H' > ./contrib/verify-commits/trusted-sha512-root-commit + git log HEAD~10 -1 --format='%H' > ./contrib/verify-commits/trusted-git-root + ${CI_RETRY_EXE} gpg --keyserver hkps://keys.openpgp.org --recv-keys $(<contrib/verify-commits/trusted-keys) && + ./contrib/verify-commits/verify-commits.py; fi echo diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index fa4d0410fa..8a9d808f5d 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -47,7 +47,7 @@ export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} export EXPECTED_TESTS_DURATION_IN_SECONDS=${EXPECTED_TESTS_DURATION_IN_SECONDS:-1000} export CONTAINER_NAME=${CONTAINER_NAME:-ci_unnamed} -export DOCKER_NAME_TAG=${DOCKER_NAME_TAG:-ubuntu:18.04} +export DOCKER_NAME_TAG=${DOCKER_NAME_TAG:-ubuntu:20.04} # Randomize test order. # See https://www.boost.org/doc/libs/1_71_0/libs/test/doc/html/boost_test/utf_reference/rt_param_reference/random.html export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1} diff --git a/ci/test/00_setup_env_android.sh b/ci/test/00_setup_env_android.sh index f78a84eeac..4ef3ae1ceb 100755 --- a/ci/test/00_setup_env_android.sh +++ b/ci/test/00_setup_env_android.sh @@ -16,7 +16,7 @@ export RUN_FUNCTIONAL_TESTS=false export ANDROID_API_LEVEL=28 export ANDROID_BUILD_TOOLS_VERSION=28.0.3 -export ANDROID_NDK_VERSION=21.1.6352462 +export ANDROID_NDK_VERSION=22.1.7171670 export ANDROID_TOOLS_URL=https://dl.google.com/android/repository/commandlinetools-linux-6609375_latest.zip export ANDROID_HOME="${DEPENDS_DIR}/SDKs/android" export ANDROID_NDK_HOME="${ANDROID_HOME}/ndk/${ANDROID_NDK_VERSION}" diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index 8d2b70e549..f18052fe37 100755 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -18,11 +18,11 @@ if [ -n "$QEMU_USER_CMD" ]; then fi export CONTAINER_NAME=ci_arm_linux # Use debian to avoid 404 apt errors when cross compiling -export DOCKER_NAME_TAG="debian:buster" +export DOCKER_NAME_TAG="debian:bullseye" export USE_BUSY_BOX=true export RUN_UNIT_TESTS=true export RUN_FUNCTIONAL_TESTS=false export GOAL="install" # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1" # This could be removed once the ABI change warning does not show up by default -export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CXXFLAGS=-Wno-psabi" +export BITCOIN_CONFIG="--enable-reduce-exports CXXFLAGS=-Wno-psabi" diff --git a/ci/test/00_setup_env_native_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh index 1418dfbc51..a25c98a004 100755 --- a/ci/test/00_setup_env_native_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -6,12 +6,13 @@ export LC_ALL=C.UTF-8 -export CONTAINER_NAME=ci_native_multiprocess +export HOST=i686-pc-linux-gnu +export CONTAINER_NAME=ci_i686_multiprocess export DOCKER_NAME_TAG=ubuntu:20.04 -export PACKAGES="cmake python3 python3-pip llvm clang" +export PACKAGES="cmake python3 python3-pip llvm clang g++-multilib" export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" -export BITCOIN_CONFIG="--enable-debug CC=clang CXX=clang++" # Use clang to avoid OOM +export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' LDFLAGS='--rtlib=compiler-rt -lgcc_s'" export TEST_RUNNER_ENV="BITCOIND=bitcoin-node" -export RUN_SECURITY_TESTS="true" +export TEST_RUNNER_EXTRA="--nosandbox" export PIP_PACKAGES="lief" diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index 73ac09c1de..8bccf4fc67 100755 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -7,8 +7,8 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross -export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos (Focal is used in the gitian build as well) -export HOST=x86_64-apple-darwin18 +export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos +export HOST=x86_64-apple-darwin19 export PACKAGES="cmake imagemagick librsvg2-bin libz-dev libtiff-tools libtinfo5 python3-setuptools xorriso" export XCODE_VERSION=12.1 export XCODE_BUILD_ID=12A7403 diff --git a/ci/test/00_setup_env_mac_host.sh b/ci/test/00_setup_env_mac_host.sh index c0d951a041..02889ec936 100755 --- a/ci/test/00_setup_env_mac_host.sh +++ b/ci/test/00_setup_env_mac_host.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-apple-darwin18 +export HOST=x86_64-apple-darwin19 export PIP_PACKAGES="zmq lief" export GOAL="install" export BITCOIN_CONFIG="--with-gui --enable-reduce-exports" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index 58388fa928..b8ac691346 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -14,5 +14,5 @@ export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC=clang CXX=clang++" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC='clang -ftrivial-auto-var-init=pattern' CXX='clang++ -ftrivial-auto-var-init=pattern'" export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh index d167c9198a..e9a20fca7d 100755 --- a/ci/test/00_setup_env_native_nowallet.sh +++ b/ci/test/00_setup_env_native_nowallet.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet export DOCKER_NAME_TAG=ubuntu:18.04 # Use bionic to have one config run the tests in python3.6, see doc/dependencies.md -export PACKAGES="python3-zmq clang-5.0 llvm-5.0" # Use clang-5 to test C++17 compatibility, see doc/dependencies.md -export DEP_OPTS="NO_WALLET=1" +export PACKAGES="python3-zmq clang-7 llvm-7 libc++abi-7-dev libc++-7-dev" # Use clang-7 to test C++17 compatibility, see doc/dependencies.md +export DEP_OPTS="NO_WALLET=1 CC=clang-7 CXX='clang++-7 -stdlib=libc++'" export GOAL="install" -export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CC=clang-5.0 CXX=clang++-5.0" +export BITCOIN_CONFIG="--enable-reduce-exports CC=clang-7 CXX='clang++-7 -stdlib=libc++'" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 9c57eba62a..8176179f0b 100755 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -7,13 +7,13 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_qt5 -export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic gcc-7 can compile our c++17 and run our functional tests in python3, see doc/dependencies.md -export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" +export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic gcc-8 can compile our C++17 and run our functional tests in python3, see doc/dependencies.md +export PACKAGES="gcc-8 g++-8 python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" export DEP_OPTS="NO_QT=1 NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" -export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1" -export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports ---enable-debug --disable-fuzz-binary CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\"" +export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1 v0.20.1" +export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-reduce-exports +--enable-debug --disable-fuzz-binary CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" CC=gcc-8 CXX=g++-8" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index e079a7057c..78af869e70 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -6,10 +6,11 @@ export LC_ALL=C.UTF-8 +export DOCKER_NAME_TAG="ubuntu:20.04" export CONTAINER_NAME=ci_native_valgrind export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 -export TEST_RUNNER_EXTRA="--exclude rpc_bind" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +export TEST_RUNNER_EXTRA="--exclude rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index 51a0fd9117..fd253123e6 100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -18,9 +18,9 @@ if [ -n "$QEMU_USER_CMD" ]; then fi # Use debian to avoid 404 apt errors export CONTAINER_NAME=ci_s390x -export DOCKER_NAME_TAG="debian:buster" -export RUN_UNIT_TESTS=true +export DOCKER_NAME_TAG="debian:bookworm" export TEST_RUNNER_ENV="LC_ALL=C" +export TEST_RUNNER_EXTRA="--exclude rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export RUN_FUNCTIONAL_TESTS=true export GOAL="install" export BITCOIN_CONFIG="--enable-reduce-exports --with-incompatible-bdb" diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index 4d5bde13fd..4dff335e4e 100755 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -7,14 +7,10 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win64 -export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to win64 (Focal is used in the gitian build as well) +export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to win64 export HOST=x86_64-w64-mingw32 export DPKG_ADD_ARCH="i386" export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64 wine32 file" export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests --disable-external-signer" - -# Compiler for MinGW-w64 causes false -Wreturn-type warning. -# See https://sourceforge.net/p/mingw-w64/bugs/306/ -export NO_WERROR=1 diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 01dbfe221b..2079d2ed2b 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -11,6 +11,7 @@ if [[ $QEMU_USER_CMD == qemu-s390* ]]; then fi if [ "$CI_OS_NAME" == "macos" ]; then + sudo -H pip3 install --upgrade pip IN_GETOPT_BIN="/usr/local/opt/gnu-getopt/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES fi @@ -83,11 +84,14 @@ fi DOCKER_EXEC echo "Free disk space:" DOCKER_EXEC df -h -if [ ! -d ${DIR_QA_ASSETS} ]; then - DOCKER_EXEC git clone --depth=1 https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} +if [ "$RUN_FUZZ_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then + if [ ! -d ${DIR_QA_ASSETS} ]; then + DOCKER_EXEC git clone --depth=1 https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} + fi + + export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ + export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/ fi -export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ -export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/ DOCKER_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" diff --git a/configure.ac b/configure.ac index 20ba0f72d7..5a6f21fe42 100644 --- a/configure.ac +++ b/configure.ac @@ -1,5 +1,5 @@ AC_PREREQ([2.69]) -define(_CLIENT_VERSION_MAJOR, 21) +define(_CLIENT_VERSION_MAJOR, 22) define(_CLIENT_VERSION_MINOR, 99) define(_CLIENT_VERSION_BUILD, 0) define(_CLIENT_VERSION_RC, 0) @@ -71,6 +71,12 @@ case $host in ;; esac +AC_ARG_WITH([seccomp], + [AS_HELP_STRING([--with-seccomp], + [enable experimental syscall sandbox feature (-sandbox), default is yes if seccomp-bpf is detected under Linux x86_64])], + [seccomp_found=$withval], + [seccomp_found=auto]) + dnl Require C++17 compiler (no GNU extensions) AX_CXX_COMPILE_STDCXX([17], [noext], [mandatory]) @@ -101,7 +107,7 @@ AC_PATH_TOOL(GCOV, gcov) AC_PATH_TOOL(LLVM_COV, llvm-cov) AC_PATH_PROG(LCOV, lcov) dnl Python 3.6 is specified in .python-version and should be used if available, see doc/dependencies.md -AC_PATH_PROGS([PYTHON], [python3.6 python3.7 python3.8 python3.9 python3 python]) +AC_PATH_PROGS([PYTHON], [python3.6 python3.7 python3.8 python3.9, python3.10, python3 python]) AC_PATH_PROG(GENHTML, genhtml) AC_PATH_PROG([GIT], [git]) AC_PATH_PROG(CCACHE,ccache) @@ -240,15 +246,9 @@ AC_ARG_ENABLE([lcov-branch-coverage], [use_lcov_branch=yes], [use_lcov_branch=no]) -AC_ARG_ENABLE([glibc-back-compat], - [AS_HELP_STRING([--enable-glibc-back-compat], - [enable backwards compatibility with glibc])], - [use_glibc_compat=$enableval], - [use_glibc_compat=no]) - AC_ARG_ENABLE([threadlocal], [AS_HELP_STRING([--enable-threadlocal], - [enable features that depend on the c++ thread_local keyword (currently just thread names in debug logs). (default is to enabled if there is platform support and glibc-back-compat is not enabled)])], + [enable features that depend on the c++ thread_local keyword (currently just thread names in debug logs). (default is to enable if there is platform support)])], [use_thread_local=$enableval], [use_thread_local=auto]) @@ -318,13 +318,6 @@ AC_ARG_ENABLE([gprof], [enable_gprof=$enableval], [enable_gprof=no]) -dnl Pass compiler & linker flags that make builds deterministic -AC_ARG_ENABLE([determinism], - [AS_HELP_STRING([--enable-determinism], - [Enable compilation flags that make builds deterministic (default is no)])], - [enable_determinism=$enableval], - [enable_determinism=no]) - dnl Turn warnings into errors AC_ARG_ENABLE([werror], [AS_HELP_STRING([--enable-werror], @@ -424,7 +417,13 @@ if test "x$enable_werror" = "xyes"; then AX_CHECK_COMPILE_FLAG([-Werror=range-loop-analysis],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=range-loop-analysis"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=unused-variable],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unused-variable"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=date-time],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=date-time"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=return-type],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=return-type"],,[[$CXXFLAG_WERROR]]) + + dnl -Wreturn-type is broken in GCC for MinGW-w64. + dnl https://sourceforge.net/p/mingw-w64/bugs/306/ + AX_CHECK_COMPILE_FLAG([-Werror=return-type], [ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=return-type"], [], [$CXXFLAG_WERROR], + [AC_LANG_SOURCE([[#include <cassert> + int f(){ assert(false); }]])]) + AX_CHECK_COMPILE_FLAG([-Werror=conditional-uninitialized],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=conditional-uninitialized"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=sign-compare],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=sign-compare"],,[[$CXXFLAG_WERROR]]) dnl -Wsuggest-override is broken with GCC before 9.2 @@ -433,6 +432,7 @@ if test "x$enable_werror" = "xyes"; then [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) AX_CHECK_COMPILE_FLAG([-Werror=unreachable-code-loop-increment],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unreachable-code-loop-increment"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Werror=mismatched-tags], [ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=mismatched-tags"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Werror=implicit-fallthrough], [ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=implicit-fallthrough"], [], [$CXXFLAG_WERROR]) if test x$suppress_external_warnings != xno ; then AX_CHECK_COMPILE_FLAG([-Werror=documentation],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=documentation"],,[[$CXXFLAG_WERROR]]) @@ -463,6 +463,7 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wsuggest-override],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsuggest-override"],,[[$CXXFLAG_WERROR]], [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) AX_CHECK_COMPILE_FLAG([-Wunreachable-code-loop-increment],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunreachable-code-loop-increment"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wimplicit-fallthrough"], [], [$CXXFLAG_WERROR]) if test x$suppress_external_warnings != xno ; then AX_CHECK_COMPILE_FLAG([-Wdocumentation],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdocumentation"],,[[$CXXFLAG_WERROR]]) @@ -474,7 +475,6 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wunused-parameter],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-parameter"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wself-assign],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-self-assign"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wunused-local-typedef],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-local-typedef"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-implicit-fallthrough"],,[[$CXXFLAG_WERROR]]) if test x$suppress_external_warnings != xyes ; then AX_CHECK_COMPILE_FLAG([-Wdeprecated-copy],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-deprecated-copy"],,[[$CXXFLAG_WERROR]]) fi @@ -574,13 +574,17 @@ AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto],[[ARM_CRC_CXXFLAGS="-march=arm TEMP_CXXFLAGS="$CXXFLAGS" CXXFLAGS="$CXXFLAGS $ARM_CRC_CXXFLAGS" -AC_MSG_CHECKING(for ARM CRC32 intrinsics) +AC_MSG_CHECKING(for AArch64 CRC32 intrinsics) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include <arm_acle.h> #include <arm_neon.h> ]],[[ +#ifdef __aarch64__ __crc32cb(0, 0); __crc32ch(0, 0); __crc32cw(0, 0); __crc32cd(0, 0); vmull_p64(0, 0); +#else +#error "crc32c library does not support hardware acceleration on 32-bit ARM" +#endif ]])], [ AC_MSG_RESULT(yes); enable_arm_crc=yes; ], [ AC_MSG_RESULT(no)] @@ -705,6 +709,33 @@ case $host in if $BREW list --versions qt5 >/dev/null; then export PKG_CONFIG_PATH="$($BREW --prefix qt5 2>/dev/null)/lib/pkgconfig:$PKG_CONFIG_PATH" fi + + case $host in + *aarch64*) + dnl The preferred Homebrew prefix for Apple Silicon is /opt/homebrew. + dnl Therefore, as we do not use pkg-config to detect miniupnpc and libnatpmp + dnl packages, we should set the CPPFLAGS and LDFLAGS variables for them + dnl explicitly. + if test "x$use_upnp" != xno && $BREW list --versions miniupnpc >/dev/null; then + miniupnpc_prefix=$($BREW --prefix miniupnpc 2>/dev/null) + if test "x$suppress_external_warnings" != xno; then + CPPFLAGS="$CPPFLAGS -isystem $miniupnpc_prefix/include" + else + CPPFLAGS="$CPPFLAGS -I$miniupnpc_prefix/include" + fi + LDFLAGS="$LDFLAGS -L$miniupnpc_prefix/lib" + fi + if test "x$use_natpmp" != xno && $BREW list --versions libnatpmp >/dev/null; then + libnatpmp_prefix=$($BREW --prefix libnatpmp 2>/dev/null) + if test "x$suppress_external_warnings" != xno; then + CPPFLAGS="$CPPFLAGS -isystem $libnatpmp_prefix/include" + else + CPPFLAGS="$CPPFLAGS -I$libnatpmp_prefix/include" + fi + LDFLAGS="$LDFLAGS -L$libnatpmp_prefix/lib" + fi + ;; + esac fi else case $build_os in @@ -836,17 +867,7 @@ if test x$ac_cv_sys_large_files != x && CPPFLAGS="$CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files" fi -if test x$use_glibc_compat != xno; then - AX_CHECK_LINK_FLAG([[-Wl,--wrap=__divmoddi4]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=__divmoddi4"]) - AX_CHECK_LINK_FLAG([[-Wl,--wrap=log2f]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--wrap=log2f"]) - case $host in - powerpc64* | ppc64*) - AX_CHECK_LINK_FLAG([[-Wl,--no-tls-get-addr-optimize]], [COMPAT_LDFLAGS="$COMPAT_LDFLAGS -Wl,--no-tls-get-addr-optimize"]) - ;; - esac -else - AC_SEARCH_LIBS([clock_gettime],[rt]) -fi +AC_SEARCH_LIBS([clock_gettime],[rt]) if test "x$enable_gprof" = xyes; then dnl -pg is incompatible with -pie. Since hardening and profiling together doesn't make sense, @@ -907,6 +928,7 @@ if test x$use_hardening != xno; then ]) fi + AX_CHECK_LINK_FLAG([[-Wl,--enable-reloc-section]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--enable-reloc-section"],, [[$LDFLAG_WERROR]]) AX_CHECK_LINK_FLAG([[-Wl,--dynamicbase]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--dynamicbase"],, [[$LDFLAG_WERROR]]) AX_CHECK_LINK_FLAG([[-Wl,--nxcompat]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--nxcompat"],, [[$LDFLAG_WERROR]]) AX_CHECK_LINK_FLAG([[-Wl,--high-entropy-va]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--high-entropy-va"],, [[$LDFLAG_WERROR]]) @@ -931,12 +953,6 @@ if test x$TARGET_OS = xdarwin; then AX_CHECK_LINK_FLAG([[-Wl,-bind_at_load]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-bind_at_load"],, [[$LDFLAG_WERROR]]) fi -if test x$enable_determinism = xyes; then - if test x$TARGET_OS = xwindows; then - AX_CHECK_LINK_FLAG([[-Wl,--no-insert-timestamp]], [LDFLAGS="$LDFLAGS -Wl,--no-insert-timestamp"],, [[$LDFLAG_WERROR]]) - fi -fi - AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) AC_CHECK_DECLS([getifaddrs, freeifaddrs],[CHECK_SOCKET],, @@ -1043,7 +1059,7 @@ AC_COMPILE_IFELSE([AC_LANG_SOURCE([ dnl thread_local is currently disabled when building with glibc back compat. dnl Our minimum supported glibc is 2.17, however support for thread_local dnl did not arrive in glibc until 2.18. -if test "x$use_thread_local" = xyes || { test "x$use_thread_local" = xauto && test "x$use_glibc_compat" = xno; }; then +if test "x$use_thread_local" = xyes || test "x$use_thread_local" = xauto; then TEMP_LDFLAGS="$LDFLAGS" LDFLAGS="$TEMP_LDFLAGS $PTHREAD_CFLAGS" AC_MSG_CHECKING([for thread_local support]) @@ -1200,8 +1216,6 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <xmmintrin.h>]], [[ AC_MSG_CHECKING(for strong getauxval support in the system headers) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #include <arm_acle.h> - #include <arm_neon.h> #include <sys/auxv.h> ]], [[ getauxval(AT_HWCAP); @@ -1210,26 +1224,14 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ [ AC_MSG_RESULT(no); HAVE_STRONG_GETAUXVAL=0 ] ) -AC_MSG_CHECKING(for weak getauxval support in the compiler) -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ - #ifdef __linux__ - unsigned long getauxval(unsigned long type) __attribute__((weak)); - #define AT_HWCAP 16 - #endif - ]], [[ - getauxval(AT_HWCAP); - ]])], - [ AC_MSG_RESULT(yes); HAVE_WEAK_GETAUXVAL=1; AC_DEFINE(HAVE_WEAK_GETAUXVAL, 1, [Define this symbol to build code that uses getauxval (weak linking)]) ], - [ AC_MSG_RESULT(no); HAVE_WEAK_GETAUXVAL=0 ] -) - +have_any_system=no AC_MSG_CHECKING([for std::system]) AC_LINK_IFELSE( [ AC_LANG_PROGRAM( [[ #include <cstdlib> ]], [[ int nErr = std::system(""); ]] )], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_STD__SYSTEM, 1, Define to 1 if std::system is available.)], + [ AC_MSG_RESULT(yes); have_any_system=yes], [ AC_MSG_RESULT(no) ] ) @@ -1239,11 +1241,13 @@ AC_LINK_IFELSE( [[ ]], [[ int nErr = ::_wsystem(""); ]] )], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_WSYSTEM, 1, Define to 1 if ::wsystem is available.)], + [ AC_MSG_RESULT(yes); have_any_system=yes], [ AC_MSG_RESULT(no) ] ) -AC_DEFINE([HAVE_SYSTEM], [HAVE_STD__SYSTEM || HAVE_WSYSTEM], [std::system or ::wsystem]) +if test "x$have_any_system" != "xno"; then + AC_DEFINE(HAVE_SYSTEM, 1, Define to 1 if std::system or ::wsystem is available.) +fi LEVELDB_CPPFLAGS= LIBLEVELDB= @@ -1434,6 +1438,36 @@ if test "x$use_external_signer" != xno; then fi AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER], [test "x$use_external_signer" = "xyes"]) +dnl Do not compile with syscall sandbox support when compiling under the sanitizers. +dnl The sanitizers introduce use of syscalls that are not typically used in bitcoind +dnl (such as execve when the sanitizers execute llvm-symbolizer). +if test x$use_sanitizers != x; then + AC_MSG_WARN(Specifying --with-sanitizers forces --without-seccomp since the sanitizers introduce use of syscalls not allowed by the bitcoind syscall sandbox (-sandbox=<mode>).) + seccomp_found=no +fi +if test "x$seccomp_found" != "xno"; then + AC_MSG_CHECKING([for seccomp-bpf (Linux x86-64)]) + AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ + @%:@include <linux/seccomp.h> + ]], [[ + #if !defined(__x86_64__) + # error Syscall sandbox is an experimental feature currently available only under Linux x86-64. + #endif + ]])],[ + AC_MSG_RESULT(yes) + seccomp_found="yes" + AC_DEFINE(USE_SYSCALL_SANDBOX, 1, [Define this symbol to build with syscall sandbox support.]) + ],[ + AC_MSG_RESULT(no) + seccomp_found="no" + ]) +fi +dnl Currently only enable -sandbox=<mode> feature if seccomp is found. +dnl In the future, sandboxing could be also be supported with other +dnl sandboxing mechanisms besides seccomp. +use_syscall_sandbox=$seccomp_found +AM_CONDITIONAL([ENABLE_SYSCALL_SANDBOX], [test "x$use_syscall_sandbox" != "xno"]) + dnl Check for reduced exports if test x$use_reduce_exports = xyes; then AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[CXXFLAGS="$CXXFLAGS -fvisibility=hidden"], @@ -1763,6 +1797,10 @@ if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_ AC_MSG_ERROR([No targets! Please specify at least one of: --with-utils --with-libs --with-daemon --with-gui --enable-bench or --enable-tests]) fi +if test x$enable_fuzz_binary = xyes; then + CHECK_RUNTIME_LIB +fi + AM_CONDITIONAL([TARGET_DARWIN], [test x$TARGET_OS = xdarwin]) AM_CONDITIONAL([BUILD_DARWIN], [test x$BUILD_OS = xdarwin]) AM_CONDITIONAL([TARGET_LINUX], [test x$TARGET_OS = xlinux]) @@ -1780,7 +1818,6 @@ AM_CONDITIONAL([ENABLE_BENCH],[test x$use_bench = xyes]) AM_CONDITIONAL([USE_QRCODE], [test x$use_qr = xyes]) AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes]) AM_CONDITIONAL([USE_LIBEVENT],[test x$use_libevent = xyes]) -AM_CONDITIONAL([GLIBC_BACK_COMPAT],[test x$use_glibc_compat = xyes]) AM_CONDITIONAL([HARDEN],[test x$use_hardening = xyes]) AM_CONDITIONAL([ENABLE_SSE42],[test x$enable_sse42 = xyes]) AM_CONDITIONAL([ENABLE_SSE41],[test x$enable_sse41 = xyes]) @@ -1823,7 +1860,6 @@ AC_SUBST(DEBUG_CPPFLAGS) AC_SUBST(WARN_CXXFLAGS) AC_SUBST(NOWARN_CXXFLAGS) AC_SUBST(DEBUG_CXXFLAGS) -AC_SUBST(COMPAT_LDFLAGS) AC_SUBST(ERROR_CXXFLAGS) AC_SUBST(GPROF_CXXFLAGS) AC_SUBST(GPROF_LDFLAGS) @@ -1863,7 +1899,6 @@ AC_SUBST(HAVE_O_CLOEXEC) AC_SUBST(HAVE_BUILTIN_PREFETCH) AC_SUBST(HAVE_MM_PREFETCH) AC_SUBST(HAVE_STRONG_GETAUXVAL) -AC_SUBST(HAVE_WEAK_GETAUXVAL) AC_SUBST(ANDROID_ARCH) AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/config.ini]) AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh]) @@ -1875,7 +1910,7 @@ AC_CONFIG_LINKS([contrib/devtools/test-symbol-check.py:contrib/devtools/test-sym AC_CONFIG_LINKS([contrib/filter-lcov.py:contrib/filter-lcov.py]) AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py]) AC_CONFIG_LINKS([test/fuzz/test_runner.py:test/fuzz/test_runner.py]) -AC_CONFIG_LINKS([test/util/bitcoin-util-test.py:test/util/bitcoin-util-test.py]) +AC_CONFIG_LINKS([test/util/test_runner.py:test/util/test_runner.py]) AC_CONFIG_LINKS([test/util/rpcauth-test.py:test/util/rpcauth-test.py]) dnl boost's m4 checks do something really nasty: they export these vars. As a @@ -1922,6 +1957,7 @@ echo echo "Options used to compile and link:" echo " external signer = $use_external_signer" echo " multiprocess = $build_multiprocess" +echo " with experimental syscall sandbox support = $use_syscall_sandbox" echo " with libs = $build_bitcoin_libs" echo " with wallet = $enable_wallet" if test "x$enable_wallet" != "xno"; then diff --git a/contrib/README.md b/contrib/README.md index 361975baa4..ae1372e95d 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -26,18 +26,12 @@ The [Debian](/contrib/debian) subfolder contains the copyright file. All other packaging related files can be found in the [bitcoin-core/packaging](https://github.com/bitcoin-core/packaging) repository. -### [Gitian-descriptors](/contrib/gitian-descriptors) ### -Files used during the gitian build process. For more information about gitian, see the [the Bitcoin Core documentation repository](https://github.com/bitcoin-core/docs). - -### [Gitian-keys](/contrib/gitian-keys) -PGP keys used for signing Bitcoin Core [Gitian release](/doc/release-process.md) results. +### [Builder keys](/contrib/builder-keys) +PGP keys used for signing Bitcoin Core [release](/doc/release-process.md) results. ### [MacDeploy](/contrib/macdeploy) ### Scripts and notes for Mac builds. -### [Gitian-build](/contrib/gitian-build.py) ### -Script for running full Gitian builds. - Test and Verify Tools --------------------- diff --git a/contrib/gitian-keys/README.md b/contrib/builder-keys/README.md index ffe4fb144b..56bd87d0af 100644 --- a/contrib/gitian-keys/README.md +++ b/contrib/builder-keys/README.md @@ -1,10 +1,10 @@ -## PGP keys of Gitian builders and Developers +## PGP keys of builders and Developers -The file `keys.txt` contains fingerprints of the public keys of Gitian builders -and active developers. +The file `keys.txt` contains fingerprints of the public keys of builders and +active developers. The associated keys are mainly used to sign git commits or the build results -of Gitian builds. +of Guix builds. The most recent version of each pgp key can be found on most pgp key servers. @@ -16,12 +16,12 @@ To fetch the latest version of all pgp keys in your gpg homedir, gpg --refresh-keys ``` -To fetch keys of Gitian builders and active developers, feed the list of -fingerprints of the primary keys into gpg: +To fetch keys of builders and active developers, feed the list of fingerprints +of the primary keys into gpg: ```sh -while read fingerprint keyholder_name; do gpg --keyserver hkp://subset.pool.sks-keyservers.net --recv-keys ${fingerprint}; done < ./keys.txt +while read fingerprint keyholder_name; do gpg --keyserver hkps://keys.openpgp.org --recv-keys ${fingerprint}; done < ./keys.txt ``` -Add your key to the list if you provided Gitian signatures for two major or +Add your key to the list if you provided Guix attestations for two major or minor releases of Bitcoin Core. diff --git a/contrib/gitian-keys/keys.txt b/contrib/builder-keys/keys.txt index db28cd07a0..e8032f66ee 100644 --- a/contrib/gitian-keys/keys.txt +++ b/contrib/builder-keys/keys.txt @@ -5,6 +5,7 @@ E944AE667CF960B1004BC32FCA662BE18B877A60 Andreas Schildbach (aschildbach) 590B7292695AFFA5B672CBB2E13FC145CD3F4304 Antoine Poinsot (darosior) 0AD83877C1F0CD1EE9BD660AD7CC770B81FD22A8 Ben Carman (benthecarman) 912FD3228387123DC97E0E57D5566241A0295FA9 BtcDrak (btcdrak) +04017A2A6D9A0CCDC81D8EC296AB007F1A7ED999 Carl Dong (dongcarl) C519EBCF3B926298946783EFF6430754120EC2F4 Christian Decker (cdecker) 18AE2F798E0D239755DA4FD24B79F986CBDF8736 Chun Kuan Le (ken2812221) 101598DC823C1B5F9A6624ABA5E0907A0380E6C3 CoinForensics (CoinForensics) @@ -19,6 +20,7 @@ D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece (fivepiece) 01CDF4627A3B88AAE4A571C87588242FBE38D3A8 Gavin Andresen (gavinandresen) D1DBF2C4B96F2DEBF4C16654410108112E7EA81F Hennadii Stepanov (hebasto) A2FD494D0021AA9B4FA58F759102B7AE654A4A5A Ilyas Ridhuan (IlyasRidhuan) +2688F5A9A4BE0F295E921E8A25F27A38A47AD566 James O'Beirne (jamesob) D3F22A3A4C366C2DCB66D3722DA9C5A7FA81EA35 Jarol Rodriguez (jarolrod) 7480909378D544EA6B6DCEB7535B12980BB8A4D3 Jeffri H Frontz (jhfrontz) D3CC177286005BB8FF673294C5242A1AB3936517 jl2012 (jl2012) @@ -26,6 +28,7 @@ D3CC177286005BB8FF673294C5242A1AB3936517 jl2012 (jl2012) 32EE5C4C3FA15CCADB46ABE529D4BCB6416F53EC Jonas Schnelli (jonasschnelli) 4B4E840451149DD7FB0D633477DFAB5C3108B9A8 Jorge Timon (jtimon) C42AFF7C61B3E44A1454CD3557AF762DB3353322 Karl-Johan Alm (kallewoof) +70A1D47DD44F59DF8B22244333E472FE870C7E5D Kristaps Kaupe (kristapsk) 30DE693AE0DE9E37B3E7EB6BBFF0F67810C1EED1 Lisa Neigut (niftynei) E463A93F5F3117EEDE6C7316BD02942421F4889F Luke Dashjr (luke-jr) B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B Marco Falke (marco) diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 1fa850af1a..afbad096c4 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -98,7 +98,7 @@ Perform basic security checks on a series of executables. symbol-check.py =============== -A script to check that the executables produced by gitian only contain +A script to check that release executables only contain certain symbols and are only linked against allowed libraries. For Linux this means checking for allowed gcc, glibc and libstdc++ version symbols. @@ -106,9 +106,9 @@ This makes sure they are still compatible with the minimum supported distributio For macOS and Windows we check that the executables are only linked against libraries we allow. -Example usage after a gitian build: +Example usage: - find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py + find ../path/to/executables -type f -executable | xargs python3 contrib/devtools/symbol-check.py If no errors occur the return value will be 0 and the output will be empty. diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 56e4313d78..98cab1b7fc 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -3,21 +3,22 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' -A script to check that the executables produced by gitian only contain -certain symbols and are only linked against allowed libraries. +A script to check that release executables only contain certain symbols +and are only linked against allowed libraries. Example usage: - find ../gitian-builder/build -type f -executable | xargs python3 contrib/devtools/symbol-check.py + find ../path/to/binaries -type f -executable | xargs python3 contrib/devtools/symbol-check.py ''' import subprocess import sys -import os from typing import List, Optional import lief import pixie +from utils import determine_wellknown_cmd + # Debian 8 (Jessie) EOL: 2020. https://wiki.debian.org/DebianReleases#Production_Releases # # - g++ version 4.9.2 (https://packages.debian.org/search?suite=jessie&arch=any&searchon=names&keywords=g%2B%2B) @@ -60,7 +61,6 @@ IGNORE_EXPORTS = { '_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', '__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr', 'environ', '_environ', '__environ', } -CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') # Allowed NEEDED libraries ELF_ALLOWED_LIBRARIES = { @@ -140,7 +140,7 @@ class CPPFilt(object): Use a pipe to the 'c++filt' command. ''' def __init__(self): - self.proc = subprocess.Popen(CPPFILT_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) + self.proc = subprocess.Popen(determine_wellknown_cmd('CPPFILT', 'c++filt'), stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) def __call__(self, mangled): self.proc.stdin.write(mangled + '\n') @@ -217,7 +217,7 @@ def check_MACHO_libraries(filename) -> bool: def check_MACHO_min_os(filename) -> bool: binary = lief.parse(filename) - if binary.build_version.minos == [10,14,0]: + if binary.build_version.minos == [10,15,0]: return True return False diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index c079fe5b4d..14058e2cc8 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -9,6 +9,8 @@ import os import subprocess import unittest +from utils import determine_wellknown_cmd + def write_testcode(filename): with open(filename, 'w', encoding="utf8") as f: f.write(''' @@ -25,7 +27,7 @@ def clean_files(source, executable): os.remove(executable) def call_security_check(cc, source, executable, options): - subprocess.run([cc,source,'-o',executable] + options, check=True) + subprocess.run([*cc,source,'-o',executable] + options, check=True) p = subprocess.run(['./contrib/devtools/security-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) return (p.returncode, p.stdout.rstrip()) @@ -33,7 +35,7 @@ class TestSecurityChecks(unittest.TestCase): def test_ELF(self): source = 'test1.c' executable = 'test1' - cc = 'gcc' + cc = determine_wellknown_cmd('CC', 'gcc') write_testcode(source) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), @@ -54,18 +56,20 @@ class TestSecurityChecks(unittest.TestCase): def test_PE(self): source = 'test1.c' executable = 'test1.exe' - cc = 'x86_64-w64-mingw32-gcc' + cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed HIGH_ENTROPY_VA RELOC_SECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed RELOC_SECTION')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA')) # -pie -fPIE does nothing unless --dynamicbase is also supplied + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']), + (1, executable+': failed HIGH_ENTROPY_VA')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']), (0, '')) clean_files(source, executable) @@ -73,7 +77,7 @@ class TestSecurityChecks(unittest.TestCase): def test_MACHO(self): source = 'test1.c' executable = 'test1' - cc = 'clang' + cc = determine_wellknown_cmd('CC', 'clang') write_testcode(source) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']), @@ -95,4 +99,3 @@ class TestSecurityChecks(unittest.TestCase): if __name__ == '__main__': unittest.main() - diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py index 6ce2fa3560..651589c11b 100755 --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -7,41 +7,51 @@ Test script for symbol-check.py ''' import os import subprocess +from typing import List import unittest -def call_symbol_check(cc, source, executable, options): - subprocess.run([cc,source,'-o',executable] + options, check=True) +from utils import determine_wellknown_cmd + +def call_symbol_check(cc: List[str], source, executable, options): + subprocess.run([*cc,source,'-o',executable] + options, check=True) p = subprocess.run(['./contrib/devtools/symbol-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) os.remove(source) os.remove(executable) return (p.returncode, p.stdout.rstrip()) +def get_machine(cc: List[str]): + p = subprocess.run([*cc,'-dumpmachine'], stdout=subprocess.PIPE, universal_newlines=True) + return p.stdout.rstrip() + class TestSymbolChecks(unittest.TestCase): def test_ELF(self): source = 'test1.c' executable = 'test1' - cc = 'gcc' + cc = determine_wellknown_cmd('CC', 'gcc') - # renameat2 was introduced in GLIBC 2.28, so is newer than the upper limit - # of glibc for all platforms + # there's no way to do this test for RISC-V at the moment; we build for + # RISC-V in a glibc 2.27 envinonment and we allow all symbols from 2.27. + if 'riscv' in get_machine(cc): + self.skipTest("test not available for RISC-V") + + # nextup was introduced in GLIBC 2.24, so is newer than our supported + # glibc (2.17), and available in our release build environment (2.24). with open(source, 'w', encoding="utf8") as f: f.write(''' #define _GNU_SOURCE - #include <stdio.h> - #include <linux/fs.h> + #include <math.h> - int renameat2(int olddirfd, const char *oldpath, - int newdirfd, const char *newpath, unsigned int flags); + double nextup(double x); int main() { - renameat2(0, "test", 0, "test_", RENAME_EXCHANGE); + nextup(3.14); return 0; } ''') - self.assertEqual(call_symbol_check(cc, source, executable, []), - (1, executable + ': symbol renameat2 from unsupported version GLIBC_2.28\n' + + self.assertEqual(call_symbol_check(cc, source, executable, ['-lm']), + (1, executable + ': symbol nextup from unsupported version GLIBC_2.24\n' + executable + ': failed IMPORTED_SYMBOLS')) # -lutil is part of the libc6 package so a safe bet that it's installed @@ -63,26 +73,27 @@ class TestSymbolChecks(unittest.TestCase): (1, executable + ': NEEDED library libutil.so.1 is not allowed\n' + executable + ': failed LIBRARY_DEPENDENCIES')) - # finally, check a conforming file that simply uses a math function + # finally, check a simple conforming binary source = 'test3.c' executable = 'test3' with open(source, 'w', encoding="utf8") as f: f.write(''' - #include <math.h> + #include <stdio.h> int main() { - return (int)pow(2.0, 4.0); + printf("42"); + return 0; } ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-lm']), + self.assertEqual(call_symbol_check(cc, source, executable, []), (0, '')) def test_MACHO(self): source = 'test1.c' executable = 'test1' - cc = 'clang' + cc = determine_wellknown_cmd('CC', 'clang') with open(source, 'w', encoding="utf8") as f: f.write(''' @@ -96,7 +107,7 @@ class TestSymbolChecks(unittest.TestCase): ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-lexpat']), + self.assertEqual(call_symbol_check(cc, source, executable, ['-lexpat', '-Wl,-platform_version','-Wl,macos', '-Wl,11.4', '-Wl,11.4']), (1, 'libexpat.1.dylib is not in ALLOWED_LIBRARIES!\n' + f'{executable}: failed DYNAMIC_LIBRARIES MIN_OS SDK')) @@ -113,7 +124,7 @@ class TestSymbolChecks(unittest.TestCase): } ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-framework', 'CoreGraphics']), + self.assertEqual(call_symbol_check(cc, source, executable, ['-framework', 'CoreGraphics', '-Wl,-platform_version','-Wl,macos', '-Wl,11.4', '-Wl,11.4']), (1, f'{executable}: failed MIN_OS SDK')) source = 'test3.c' @@ -126,13 +137,13 @@ class TestSymbolChecks(unittest.TestCase): } ''') - self.assertEqual(call_symbol_check(cc, source, executable, ['-mmacosx-version-min=10.14']), + self.assertEqual(call_symbol_check(cc, source, executable, ['-Wl,-platform_version','-Wl,macos', '-Wl,10.15', '-Wl,11.4']), (1, f'{executable}: failed SDK')) def test_PE(self): source = 'test1.c' executable = 'test1.exe' - cc = 'x86_64-w64-mingw32-gcc' + cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') with open(source, 'w', encoding="utf8") as f: f.write(''' @@ -182,4 +193,3 @@ class TestSymbolChecks(unittest.TestCase): if __name__ == '__main__': unittest.main() - diff --git a/contrib/devtools/utils.py b/contrib/devtools/utils.py new file mode 100755 index 0000000000..68ad1c3aba --- /dev/null +++ b/contrib/devtools/utils.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +Common utility functions +''' +import shutil +import sys +import os +from typing import List + + +def determine_wellknown_cmd(envvar, progname) -> List[str]: + maybe_env = os.getenv(envvar) + maybe_which = shutil.which(progname) + if maybe_env: + return maybe_env.split(' ') # Well-known vars are often meant to be word-split + elif maybe_which: + return [ maybe_which ] + else: + sys.exit(f"{progname} not found") diff --git a/contrib/gitian-build.py b/contrib/gitian-build.py deleted file mode 100755 index 5df87d9e70..0000000000 --- a/contrib/gitian-build.py +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -import argparse -import os -import subprocess -import sys - -def setup(): - global args, workdir - programs = ['ruby', 'git', 'make', 'wget', 'curl'] - if args.kvm: - programs += ['apt-cacher-ng', 'python-vm-builder', 'qemu-kvm', 'qemu-utils'] - elif args.docker: - if not os.path.isfile('/lib/systemd/system/docker.service'): - dockers = ['docker.io', 'docker-ce'] - for i in dockers: - return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i]) - if return_code == 0: - break - if return_code != 0: - print('Cannot find any way to install Docker.', file=sys.stderr) - sys.exit(1) - else: - programs += ['apt-cacher-ng', 'lxc', 'debootstrap'] - subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) - if not os.path.isdir('gitian.sigs'): - subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin-core/gitian.sigs.git']) - if not os.path.isdir('bitcoin-detached-sigs'): - subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin-core/bitcoin-detached-sigs.git']) - if not os.path.isdir('gitian-builder'): - subprocess.check_call(['git', 'clone', 'https://github.com/devrandom/gitian-builder.git']) - if not os.path.isdir('bitcoin'): - subprocess.check_call(['git', 'clone', 'https://github.com/bitcoin/bitcoin.git']) - os.chdir('gitian-builder') - make_image_prog = ['bin/make-base-vm', '--suite', 'focal', '--arch', 'amd64'] - if args.docker: - make_image_prog += ['--docker'] - elif not args.kvm: - make_image_prog += ['--lxc', '--disksize', '13000'] - subprocess.check_call(make_image_prog) - os.chdir(workdir) - if args.is_focal and not args.kvm and not args.docker: - subprocess.check_call(['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net']) - print('Reboot is required') - sys.exit(0) - -def build(): - global args, workdir - - os.makedirs('bitcoin-binaries/' + args.version, exist_ok=True) - print('\nBuilding Dependencies\n') - os.chdir('gitian-builder') - os.makedirs('inputs', exist_ok=True) - - subprocess.check_call(['wget', '-O', 'inputs/osslsigncode-2.0.tar.gz', 'https://github.com/mtrojnar/osslsigncode/archive/2.0.tar.gz']) - subprocess.check_call(["echo '5a60e0a4b3e0b4d655317b2f12a810211c50242138322b16e7e01c6fbb89d92f inputs/osslsigncode-2.0.tar.gz' | sha256sum -c"], shell=True) - subprocess.check_call(['make', '-C', '../bitcoin/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common']) - - if args.linux: - print('\nCompiling ' + args.version + ' Linux') - subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin='+args.commit, '--url', 'bitcoin='+args.url, '../bitcoin/contrib/gitian-descriptors/gitian-linux.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-linux', '--destination', '../gitian.sigs/', '../bitcoin/contrib/gitian-descriptors/gitian-linux.yml']) - subprocess.check_call('mv build/out/bitcoin-*.tar.gz build/out/src/bitcoin-*.tar.gz ../bitcoin-binaries/'+args.version, shell=True) - - if args.windows: - print('\nCompiling ' + args.version + ' Windows') - subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin='+args.commit, '--url', 'bitcoin='+args.url, '../bitcoin/contrib/gitian-descriptors/gitian-win.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-unsigned', '--destination', '../gitian.sigs/', '../bitcoin/contrib/gitian-descriptors/gitian-win.yml']) - subprocess.check_call('mv build/out/bitcoin-*-win-unsigned.tar.gz inputs/', shell=True) - subprocess.check_call('mv build/out/bitcoin-*.zip build/out/bitcoin-*.exe build/out/src/bitcoin-*.tar.gz ../bitcoin-binaries/'+args.version, shell=True) - - if args.macos: - print('\nCompiling ' + args.version + ' MacOS') - subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'bitcoin='+args.commit, '--url', 'bitcoin='+args.url, '../bitcoin/contrib/gitian-descriptors/gitian-osx.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../bitcoin/contrib/gitian-descriptors/gitian-osx.yml']) - subprocess.check_call('mv build/out/bitcoin-*-osx-unsigned.tar.gz inputs/', shell=True) - subprocess.check_call('mv build/out/bitcoin-*.tar.gz build/out/bitcoin-*.dmg build/out/src/bitcoin-*.tar.gz ../bitcoin-binaries/'+args.version, shell=True) - - os.chdir(workdir) - - if args.commit_files: - print('\nCommitting '+args.version+' Unsigned Sigs\n') - os.chdir('gitian.sigs') - subprocess.check_call(['git', 'add', args.version+'-linux/'+args.signer]) - subprocess.check_call(['git', 'add', args.version+'-win-unsigned/'+args.signer]) - subprocess.check_call(['git', 'add', args.version+'-osx-unsigned/'+args.signer]) - subprocess.check_call(['git', 'commit', '-m', 'Add '+args.version+' unsigned sigs for '+args.signer]) - os.chdir(workdir) - -def sign(): - global args, workdir - os.chdir('gitian-builder') - - if args.windows: - print('\nSigning ' + args.version + ' Windows') - subprocess.check_call('cp inputs/bitcoin-' + args.version + '-win-unsigned.tar.gz inputs/bitcoin-win-unsigned.tar.gz', shell=True) - subprocess.check_call(['bin/gbuild', '--skip-image', '--upgrade', '--commit', 'signature='+args.commit, '../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win-signed', '--destination', '../gitian.sigs/', '../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml']) - subprocess.check_call('mv build/out/bitcoin-*win64-setup.exe ../bitcoin-binaries/'+args.version, shell=True) - - if args.macos: - print('\nSigning ' + args.version + ' MacOS') - subprocess.check_call('cp inputs/bitcoin-' + args.version + '-osx-unsigned.tar.gz inputs/bitcoin-osx-unsigned.tar.gz', shell=True) - subprocess.check_call(['bin/gbuild', '--skip-image', '--upgrade', '--commit', 'signature='+args.commit, '../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml']) - subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-signed', '--destination', '../gitian.sigs/', '../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml']) - subprocess.check_call('mv build/out/bitcoin-osx-signed.dmg ../bitcoin-binaries/'+args.version+'/bitcoin-'+args.version+'-osx.dmg', shell=True) - - os.chdir(workdir) - - if args.commit_files: - print('\nCommitting '+args.version+' Signed Sigs\n') - os.chdir('gitian.sigs') - subprocess.check_call(['git', 'add', args.version+'-win-signed/'+args.signer]) - subprocess.check_call(['git', 'add', args.version+'-osx-signed/'+args.signer]) - subprocess.check_call(['git', 'commit', '-a', '-m', 'Add '+args.version+' signed binary sigs for '+args.signer]) - os.chdir(workdir) - -def verify(): - global args, workdir - rc = 0 - os.chdir('gitian-builder') - - print('\nVerifying v'+args.version+' Linux\n') - if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../bitcoin/contrib/gitian-descriptors/gitian-linux.yml']): - print('Verifying v'+args.version+' Linux FAILED\n') - rc = 1 - - print('\nVerifying v'+args.version+' Windows\n') - if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-unsigned', '../bitcoin/contrib/gitian-descriptors/gitian-win.yml']): - print('Verifying v'+args.version+' Windows FAILED\n') - rc = 1 - - print('\nVerifying v'+args.version+' MacOS\n') - if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-unsigned', '../bitcoin/contrib/gitian-descriptors/gitian-osx.yml']): - print('Verifying v'+args.version+' MacOS FAILED\n') - rc = 1 - - print('\nVerifying v'+args.version+' Signed Windows\n') - if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win-signed', '../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml']): - print('Verifying v'+args.version+' Signed Windows FAILED\n') - rc = 1 - - print('\nVerifying v'+args.version+' Signed MacOS\n') - if subprocess.call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx-signed', '../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml']): - print('Verifying v'+args.version+' Signed MacOS FAILED\n') - rc = 1 - - os.chdir(workdir) - return rc - -def main(): - global args, workdir - - parser = argparse.ArgumentParser(description='Script for running full Gitian builds.') - parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch') - parser.add_argument('-p', '--pull', action='store_true', dest='pull', help='Indicate that the version argument is the number of a github repository pull request') - parser.add_argument('-u', '--url', dest='url', default='https://github.com/bitcoin/bitcoin', help='Specify the URL of the repository. Default is %(default)s') - parser.add_argument('-v', '--verify', action='store_true', dest='verify', help='Verify the Gitian build') - parser.add_argument('-b', '--build', action='store_true', dest='build', help='Do a Gitian build') - parser.add_argument('-s', '--sign', action='store_true', dest='sign', help='Make signed binaries for Windows and MacOS') - parser.add_argument('-B', '--buildsign', action='store_true', dest='buildsign', help='Build both signed and unsigned binaries') - parser.add_argument('-o', '--os', dest='os', default='lwm', help='Specify which Operating Systems the build is for. Default is %(default)s. l for Linux, w for Windows, m for MacOS') - parser.add_argument('-j', '--jobs', dest='jobs', default='2', help='Number of processes to use. Default %(default)s') - parser.add_argument('-m', '--memory', dest='memory', default='2000', help='Memory to allocate in MiB. Default %(default)s') - parser.add_argument('-k', '--kvm', action='store_true', dest='kvm', help='Use KVM instead of LXC') - parser.add_argument('-d', '--docker', action='store_true', dest='docker', help='Use Docker instead of LXC') - parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Only works on Debian-based systems (Ubuntu, Debian)') - parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign', help='Create the assert file for detached signing. Will not commit anything.') - parser.add_argument('-n', '--no-commit', action='store_false', dest='commit_files', help='Do not commit anything to git') - parser.add_argument('signer', nargs='?', help='GPG signer to sign each build assert file') - parser.add_argument('version', nargs='?', help='Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified') - - args = parser.parse_args() - workdir = os.getcwd() - - args.is_focal = b'focal' in subprocess.check_output(['lsb_release', '-cs']) - - if args.kvm and args.docker: - raise Exception('Error: cannot have both kvm and docker') - - # Ensure no more than one environment variable for gitian-builder (USE_LXC, USE_VBOX, USE_DOCKER) is set as they - # can interfere (e.g., USE_LXC being set shadows USE_DOCKER; for details see gitian-builder/libexec/make-clean-vm). - os.environ['USE_LXC'] = '' - os.environ['USE_VBOX'] = '' - os.environ['USE_DOCKER'] = '' - if args.docker: - os.environ['USE_DOCKER'] = '1' - elif not args.kvm: - os.environ['USE_LXC'] = '1' - if 'GITIAN_HOST_IP' not in os.environ.keys(): - os.environ['GITIAN_HOST_IP'] = '10.0.3.1' - if 'LXC_GUEST_IP' not in os.environ.keys(): - os.environ['LXC_GUEST_IP'] = '10.0.3.5' - - if args.setup: - setup() - - if args.buildsign: - args.build = True - args.sign = True - - if not args.build and not args.sign and not args.verify: - sys.exit(0) - - args.linux = 'l' in args.os - args.windows = 'w' in args.os - args.macos = 'm' in args.os - - # Disable for MacOS if no SDK found - if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz'): - print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') - args.macos = False - - args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign' - - script_name = os.path.basename(sys.argv[0]) - if not args.signer: - print(script_name+': Missing signer') - print('Try '+script_name+' --help for more information') - sys.exit(1) - if not args.version: - print(script_name+': Missing version') - print('Try '+script_name+' --help for more information') - sys.exit(1) - - # Add leading 'v' for tags - if args.commit and args.pull: - raise Exception('Cannot have both commit and pull') - args.commit = ('' if args.commit else 'v') + args.version - - os.chdir('bitcoin') - if args.pull: - subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) - os.chdir('../gitian-builder/inputs/bitcoin') - subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) - args.commit = subprocess.check_output(['git', 'show', '-s', '--format=%H', 'FETCH_HEAD'], universal_newlines=True, encoding='utf8').strip() - args.version = 'pull-' + args.version - print(args.commit) - subprocess.check_call(['git', 'fetch']) - subprocess.check_call(['git', 'checkout', args.commit]) - os.chdir(workdir) - - os.chdir('gitian-builder') - subprocess.check_call(['git', 'pull']) - os.chdir(workdir) - - if args.build: - build() - - if args.sign: - sign() - - if args.verify: - os.chdir('gitian.sigs') - subprocess.check_call(['git', 'pull']) - os.chdir(workdir) - sys.exit(verify()) - -if __name__ == '__main__': - main() diff --git a/contrib/gitian-descriptors/assign_DISTNAME b/contrib/gitian-descriptors/assign_DISTNAME deleted file mode 100644 index 330fbc041b..0000000000 --- a/contrib/gitian-descriptors/assign_DISTNAME +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# A helper script to be sourced into the gitian descriptors - -if RECENT_TAG="$(git describe --exact-match HEAD 2> /dev/null)"; then - VERSION="${RECENT_TAG#v}" -else - VERSION="$(git rev-parse --short=12 HEAD)" -fi -DISTNAME="bitcoin-${VERSION}" diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml deleted file mode 100644 index e6dce7a8c6..0000000000 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ /dev/null @@ -1,167 +0,0 @@ ---- -name: "bitcoin-core-linux-22" -enable_cache: true -distro: "ubuntu" -suites: -- "focal" -architectures: -- "amd64" -packages: -# Common dependencies. -- "autoconf" -- "automake" -- "binutils" -- "bison" -- "bsdmainutils" -- "ca-certificates" -- "curl" -- "faketime" -- "g++-8" -- "gcc-8" -- "git" -- "libtool" -- "patch" -- "pkg-config" -- "python3" -- "python3-pip" -# Cross compilation HOSTS: -# - arm-linux-gnueabihf -- "binutils-arm-linux-gnueabihf" -- "g++-8-arm-linux-gnueabihf" -# - aarch64-linux-gnu -- "binutils-aarch64-linux-gnu" -- "g++-8-aarch64-linux-gnu" -# - powerpc64-linux-gnu -- "binutils-powerpc64-linux-gnu" -- "g++-8-powerpc64-linux-gnu" -# - powerpc64le-linux-gnu -- "binutils-powerpc64le-linux-gnu" -- "g++-8-powerpc64le-linux-gnu" -# - riscv64-linux-gnu -- "binutils-riscv64-linux-gnu" -- "g++-8-riscv64-linux-gnu" -remotes: -- "url": "https://github.com/bitcoin/bitcoin.git" - "dir": "bitcoin" -files: [] -script: | - set -e -o pipefail - - WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu riscv64-linux-gnu" - CONFIGFLAGS="--enable-glibc-back-compat --enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" - FAKETIME_HOST_PROGS="gcc g++" - FAKETIME_PROGS="date ar ranlib nm" - HOST_CFLAGS="-O2 -g" - HOST_CXXFLAGS="-O2 -g" - HOST_LDFLAGS_BASE="-static-libstdc++ -Wl,-O2" - - export TZ="UTC" - export BUILD_DIR="$PWD" - mkdir -p ${WRAP_DIR} - if test -n "$GBUILD_CACHE_ENABLED"; then - export SOURCES_PATH=${GBUILD_COMMON_CACHE} - export BASE_CACHE=${GBUILD_PACKAGE_CACHE} - mkdir -p ${BASE_CACHE} ${SOURCES_PATH} - fi - - # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) - function create_global_faketime_wrappers { - for prog in ${FAKETIME_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - } - - function create_per-host_faketime_wrappers { - for i in $HOSTS; do - for prog in ${FAKETIME_HOST_PROGS}; do - if which ${i}-${prog}-8 - then - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog}-8 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - if [ "${i:0:11}" = "powerpc64le" ]; then - echo "exec \"\$REAL\" -mcpu=power8 -mtune=power9 \"\$@\"" >> $WRAP_DIR/${i}-${prog} - elif [ "${i:0:9}" = "powerpc64" ]; then - echo "exec \"\$REAL\" -mcpu=970 -mtune=power9 \"\$@\"" >> $WRAP_DIR/${i}-${prog} - else - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} - fi - chmod +x ${WRAP_DIR}/${i}-${prog} - fi - done - done - } - - pip3 install lief==0.11.5 - - # Faketime for depends so intermediate results are comparable - export PATH_orig=${PATH} - create_global_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_faketime_wrappers "2000-01-01 12:00:00" - export PATH=${WRAP_DIR}:${PATH} - - cd bitcoin - BASEPREFIX="${PWD}/depends" - # Build dependencies for each host - for i in $HOSTS; do - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" CC=${i}-gcc-8 CXX=${i}-g++-8 - done - - # Faketime for binaries - export PATH=${PATH_orig} - create_global_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" - export PATH=${WRAP_DIR}:${PATH} - - # Define DISTNAME variable. - # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME - source contrib/gitian-descriptors/assign_DISTNAME - - GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" - - # Create the source tarball - mkdir -p "$(dirname "$GIT_ARCHIVE")" - git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD - - ORIGPATH="$PATH" - # Extract the git archive into a dir for each host and build - for i in ${HOSTS}; do - export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} - if [ "${i}" = "powerpc64-linux-gnu" ]; then - # Workaround for https://bugs.launchpad.net/ubuntu/+source/gcc-8-cross-ports/+bug/1853740 - # TODO: remove this when no longer needed - HOST_LDFLAGS="${HOST_LDFLAGS_BASE} -Wl,-z,noexecstack" - else - HOST_LDFLAGS="${HOST_LDFLAGS_BASE}" - fi - mkdir -p distsrc-${i} - cd distsrc-${i} - INSTALLPATH="${PWD}/installed/${DISTNAME}" - mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf "${GIT_ARCHIVE}" - - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" CC=${i}-gcc-8 CXX=${i}-g++-8 - make ${MAKEOPTS} - make ${MAKEOPTS} -C src check-security - make ${MAKEOPTS} -C src check-symbols - make install DESTDIR=${INSTALLPATH} - cd installed - find . -name "lib*.la" -delete - find . -name "lib*.a" -delete - rm -rf ${DISTNAME}/lib/pkgconfig - find ${DISTNAME}/bin -type f -executable -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg - find ${DISTNAME}/lib -type f -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg - cp ../README.md ${DISTNAME}/ - find ${DISTNAME} -not -name "*.dbg" | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz - find ${DISTNAME} -name "*.dbg" | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz - cd ../../ - rm -rf distsrc-${i} - done diff --git a/contrib/gitian-descriptors/gitian-osx-signer.yml b/contrib/gitian-descriptors/gitian-osx-signer.yml deleted file mode 100644 index addad0a5d2..0000000000 --- a/contrib/gitian-descriptors/gitian-osx-signer.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -name: "bitcoin-dmg-signer" -distro: "ubuntu" -suites: -- "focal" -architectures: -- "amd64" -packages: -- "faketime" -- "xorriso" -- "python3-pip" -remotes: -- "url": "https://github.com/bitcoin-core/bitcoin-detached-sigs.git" - "dir": "signature" -- "url": "https://github.com/achow101/signapple.git" - "dir": "signapple" - "commit": "b084cbbf44d5330448ffce0c7d118f75781b64bd" -files: -- "bitcoin-osx-unsigned.tar.gz" -script: | - set -e -o pipefail - - WRAP_DIR=$HOME/wrapped - mkdir -p ${WRAP_DIR} - export PATH="$PWD":$PATH - FAKETIME_PROGS="dmg xorrisofs" - - # Create global faketime wrappers - for prog in ${FAKETIME_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"${REFERENCE_DATETIME}\"" >> ${WRAP_DIR}/${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - - # Install signapple - cd signapple - python3 -m pip install -U pip setuptools - python3 -m pip install . - export PATH="$HOME/.local/bin":$PATH - cd .. - - UNSIGNED_TARBALL=bitcoin-osx-unsigned.tar.gz - UNSIGNED_APP=dist/Bitcoin-Qt.app - SIGNED=bitcoin-osx-signed.dmg - - tar -xf ${UNSIGNED_TARBALL} - OSX_VOLNAME="$(cat osx_volname)" - ./detached-sig-apply.sh ${UNSIGNED_APP} signature/osx/dist - ${WRAP_DIR}/xorrisofs -D -l -V "${OSX_VOLNAME}" -no-pad -r -dir-mode 0755 -o uncompressed.dmg signed-app - ${WRAP_DIR}/dmg dmg uncompressed.dmg ${OUTDIR}/${SIGNED} diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml deleted file mode 100644 index a39618adb7..0000000000 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ /dev/null @@ -1,155 +0,0 @@ ---- -name: "bitcoin-core-osx-22" -enable_cache: true -distro: "ubuntu" -suites: -- "focal" -architectures: -- "amd64" -packages: -- "ca-certificates" -- "curl" -- "g++" -- "git" -- "pkg-config" -- "autoconf" -- "librsvg2-bin" -- "libtiff-tools" -- "libtool" -- "automake" -- "faketime" -- "bsdmainutils" -- "cmake" -- "imagemagick" -- "libz-dev" -- "python3" -- "python3-pip" -- "python3-setuptools" -- "fonts-tuffy" -- "xorriso" -- "libtinfo5" -remotes: -- "url": "https://github.com/bitcoin/bitcoin.git" - "dir": "bitcoin" -files: -- "Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz" -script: | - set -e -o pipefail - - WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-apple-darwin18" - CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary XORRISOFS=${WRAP_DIR}/xorrisofs DMG=${WRAP_DIR}/dmg" - FAKETIME_HOST_PROGS="" - FAKETIME_PROGS="ar ranlib date dmg xorrisofs" - - export TZ="UTC" - export BUILD_DIR="$PWD" - mkdir -p ${WRAP_DIR} - if test -n "$GBUILD_CACHE_ENABLED"; then - export SOURCES_PATH=${GBUILD_COMMON_CACHE} - export BASE_CACHE=${GBUILD_PACKAGE_CACHE} - mkdir -p ${BASE_CACHE} ${SOURCES_PATH} - fi - - export ZERO_AR_DATE=1 - - # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) - function create_global_faketime_wrappers { - for prog in ${FAKETIME_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - } - - function create_per-host_faketime_wrappers { - for i in $HOSTS; do - for prog in ${FAKETIME_HOST_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} - chmod +x ${WRAP_DIR}/${i}-${prog} - done - done - } - - pip3 install lief==0.11.5 - - # Faketime for depends so intermediate results are comparable - export PATH_orig=${PATH} - create_global_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_faketime_wrappers "2000-01-01 12:00:00" - export PATH=${WRAP_DIR}:${PATH} - - cd bitcoin - BASEPREFIX="${PWD}/depends" - - mkdir -p ${BASEPREFIX}/SDKs - tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz - - # Build dependencies for each host - for i in $HOSTS; do - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" - done - - # Faketime for binaries - export PATH=${PATH_orig} - create_global_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" - export PATH=${WRAP_DIR}:${PATH} - - # Define DISTNAME variable. - # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME - source contrib/gitian-descriptors/assign_DISTNAME - - GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" - - # Create the source tarball - mkdir -p "$(dirname "$GIT_ARCHIVE")" - git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD - - ORIGPATH="$PATH" - # Extract the git archive into a dir for each host and build - for i in ${HOSTS}; do - export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} - mkdir -p distsrc-${i} - cd distsrc-${i} - INSTALLPATH="${PWD}/installed/${DISTNAME}" - mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf "${GIT_ARCHIVE}" - - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} - make ${MAKEOPTS} - make ${MAKEOPTS} -C src check-security - make ${MAKEOPTS} -C src check-symbols - make install-strip DESTDIR=${INSTALLPATH} - - make osx_volname - make deploydir - mkdir -p unsigned-app-${i} - cp osx_volname unsigned-app-${i}/ - cp contrib/macdeploy/detached-sig-apply.sh unsigned-app-${i} - cp contrib/macdeploy/detached-sig-create.sh unsigned-app-${i} - cp ${BASEPREFIX}/${i}/native/bin/dmg unsigned-app-${i} - mv dist unsigned-app-${i} - pushd unsigned-app-${i} - find . | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-osx-unsigned.tar.gz - popd - - make deploy OSX_DMG="${OUTDIR}/${DISTNAME}-osx-unsigned.dmg" - - cd installed - find . -name "lib*.la" -delete - find . -name "lib*.a" -delete - rm -rf ${DISTNAME}/lib/pkgconfig - find ${DISTNAME} | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz - cd ../../ - done - - mv ${OUTDIR}/${DISTNAME}-x86_64-*.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz diff --git a/contrib/gitian-descriptors/gitian-win-signer.yml b/contrib/gitian-descriptors/gitian-win-signer.yml deleted file mode 100644 index c13c24c3cc..0000000000 --- a/contrib/gitian-descriptors/gitian-win-signer.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -name: "bitcoin-win-signer" -distro: "ubuntu" -suites: -- "focal" -architectures: -- "amd64" -packages: -- "libssl-dev" -- "autoconf" -- "automake" -- "libtool" -- "pkg-config" -remotes: -- "url": "https://github.com/bitcoin-core/bitcoin-detached-sigs.git" - "dir": "signature" -files: -- "osslsigncode-2.0.tar.gz" -- "bitcoin-win-unsigned.tar.gz" -script: | - set -e -o pipefail - - BUILD_DIR="$PWD" - SIGDIR=${BUILD_DIR}/signature/win - UNSIGNED_DIR=${BUILD_DIR}/unsigned - - echo "5a60e0a4b3e0b4d655317b2f12a810211c50242138322b16e7e01c6fbb89d92f osslsigncode-2.0.tar.gz" | sha256sum -c - - mkdir -p ${UNSIGNED_DIR} - tar -C ${UNSIGNED_DIR} -xf bitcoin-win-unsigned.tar.gz - - tar xf osslsigncode-2.0.tar.gz - cd osslsigncode-2.0 - - ./autogen.sh - ./configure --without-gsf --without-curl --disable-dependency-tracking - make - find ${UNSIGNED_DIR} -name "*-unsigned.exe" | while read i; do - INFILE="$(basename "${i}")" - OUTFILE="${INFILE/-unsigned}" - ./osslsigncode attach-signature -in "${i}" -out "${OUTDIR}/${OUTFILE}" -sigin "${SIGDIR}/${INFILE}.pem" - done diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml deleted file mode 100644 index ffe228a032..0000000000 --- a/contrib/gitian-descriptors/gitian-win.yml +++ /dev/null @@ -1,157 +0,0 @@ ---- -name: "bitcoin-core-win-22" -enable_cache: true -distro: "ubuntu" -suites: -- "focal" -architectures: -- "amd64" -packages: -- "curl" -- "g++" -- "git" -- "pkg-config" -- "autoconf" -- "libtool" -- "automake" -- "faketime" -- "bsdmainutils" -- "mingw-w64" -- "g++-mingw-w64" -- "nsis" -- "zip" -- "ca-certificates" -- "python3" -- "python3-pip" -remotes: -- "url": "https://github.com/bitcoin/bitcoin.git" - "dir": "bitcoin" -files: [] -script: | - set -e -o pipefail - - WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-w64-mingw32" - CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" - FAKETIME_HOST_PROGS="ar ranlib nm windres strip objcopy" - FAKETIME_PROGS="date makensis zip" - HOST_CFLAGS="-O2 -g -fno-ident" - HOST_CXXFLAGS="-O2 -g -fno-ident" - - export TZ="UTC" - export BUILD_DIR="$PWD" - mkdir -p ${WRAP_DIR} - if test -n "$GBUILD_CACHE_ENABLED"; then - export SOURCES_PATH=${GBUILD_COMMON_CACHE} - export BASE_CACHE=${GBUILD_PACKAGE_CACHE} - mkdir -p ${BASE_CACHE} ${SOURCES_PATH} - fi - - # Use $LIB in LD_PRELOAD to avoid hardcoding the dir (See `man ld.so`) - function create_global_faketime_wrappers { - for prog in ${FAKETIME_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} - echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${prog} - chmod +x ${WRAP_DIR}/${prog} - done - } - - function create_per-host_faketime_wrappers { - for i in $HOSTS; do - for prog in ${FAKETIME_HOST_PROGS}; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} - chmod +x ${WRAP_DIR}/${i}-${prog} - done - done - } - - function create_per-host_compiler_wrapper { - # -posix variant is required for c++11 threading. - for i in $HOSTS; do - for prog in gcc g++; do - echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} - echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} - echo "export LD_PRELOAD='/usr/\$LIB/faketime/libfaketime.so.1'" >> ${WRAP_DIR}/${i}-${prog} - echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} - echo "exec \"\$REAL\" \"\$@\"" >> $WRAP_DIR/${i}-${prog} - chmod +x ${WRAP_DIR}/${i}-${prog} - done - done - } - - pip3 install lief==0.11.5 - - # Faketime for depends so intermediate results are comparable - export PATH_orig=${PATH} - create_global_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_faketime_wrappers "2000-01-01 12:00:00" - create_per-host_compiler_wrapper "2000-01-01 12:00:00" - export PATH=${WRAP_DIR}:${PATH} - - cd bitcoin - BASEPREFIX="${PWD}/depends" - # Build dependencies for each host - for i in $HOSTS; do - make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" - done - - # Faketime for binaries - export PATH=${PATH_orig} - create_global_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" - create_per-host_compiler_wrapper "${REFERENCE_DATETIME}" - export PATH=${WRAP_DIR}:${PATH} - - # Define DISTNAME variable. - # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME - source contrib/gitian-descriptors/assign_DISTNAME - - GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" - - # Create the source tarball - mkdir -p "$(dirname "$GIT_ARCHIVE")" - git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD - - ORIGPATH="$PATH" - # Extract the git archive into a dir for each host and build - for i in ${HOSTS}; do - export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} - mkdir -p distsrc-${i} - cd distsrc-${i} - INSTALLPATH="${PWD}/installed/${DISTNAME}" - mkdir -p ${INSTALLPATH} - tar --strip-components=1 -xf "${GIT_ARCHIVE}" - - ./autogen.sh - CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" - make ${MAKEOPTS} - make ${MAKEOPTS} -C src check-security - make ${MAKEOPTS} -C src check-symbols - make deploy BITCOIN_WIN_INSTALLER="${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" - make install DESTDIR=${INSTALLPATH} - cd installed - mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/ - find . -name "lib*.la" -delete - find . -name "lib*.a" -delete - rm -rf ${DISTNAME}/lib/pkgconfig - find ${DISTNAME}/bin -type f -executable -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg - find ${DISTNAME}/lib -type f -print0 | xargs -0 -n1 -I{} ../contrib/devtools/split-debug.sh {} {} {}.dbg - cp ../doc/README_windows.txt ${DISTNAME}/readme.txt - find ${DISTNAME} -not -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i//x86_64-w64-mingw32/win64}.zip - find ${DISTNAME} -name "*.dbg" -type f | sort | zip -X@ ${OUTDIR}/${DISTNAME}-${i//x86_64-w64-mingw32/win64}-debug.zip - cd ../../ - rm -rf distsrc-${i} - done - - cp -rf contrib/windeploy $BUILD_DIR - cd $BUILD_DIR/windeploy - mkdir unsigned - cp ${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe unsigned/ - find . | sort | tar --mtime="$REFERENCE_DATETIME" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-win-unsigned.tar.gz diff --git a/contrib/guix/INSTALL.md b/contrib/guix/INSTALL.md new file mode 100644 index 0000000000..68aae18731 --- /dev/null +++ b/contrib/guix/INSTALL.md @@ -0,0 +1,801 @@ +# Guix Installation and Setup + +This only needs to be done once per machine. If you have already completed the +installation and setup, please proceed to [perform a build](./README.md). + +Otherwise, you may choose from one of the following options to install Guix: + +1. Using the official **shell installer script** [⤓ skip to section][install-script] + - Maintained by Guix developers + - Easiest (automatically performs *most* setup) + - Works on nearly all Linux distributions + - Only installs latest release + - Binary installation only, requires high level of trust + - Note: The script needs to be run as root, so it should be inspected before it's run +2. Using the official **binary tarball** [⤓ skip to section][install-bin-tarball] + - Maintained by Guix developers + - Normal difficulty (full manual setup required) + - Works on nearly all Linux distributions + - Installs any release + - Binary installation only, requires high level of trust +3. Using fanquake's **Docker image** [↗︎ external instructions][install-fanquake-docker] + - Maintained by fanquake + - Easy (automatically performs *some* setup) + - Works wherever Docker images work + - Installs any release + - Binary installation only, requires high level of trust +4. Using a **distribution-maintained package** [⤓ skip to section][install-distro-pkg] + - Maintained by distribution's Guix package maintainer + - Normal difficulty (manual setup required) + - Works only on distributions with Guix packaged, see: https://repology.org/project/guix/versions + - Installs a release decided on by package maintainer + - Source or binary installation depending on the distribution +5. Building **from source** [⤓ skip to section][install-source] + - Maintained by you + - Hard, but rewarding + - Can be made to work on most Linux distributions + - Installs any commit (more granular) + - Source installation, requires lower level of trust + +## Options 1 and 2: Using the official shell installer script or binary tarball + +The installation instructions for both the official shell installer script and +the binary tarballs can be found in the GNU Guix Manual's [Binary Installation +section](https://guix.gnu.org/manual/en/html_node/Binary-Installation.html). + +Note that running through the binary tarball installation steps is largely +equivalent to manually performing what the shell installer script does. + +Note that at the time of writing (July 5th, 2021), the shell installer script +automatically creates an `/etc/profile.d` entry which the binary tarball +installation instructions do not ask you to create. However, you will likely +need this entry for better desktop integration. Please see [this +section](#add-an-etcprofiled-entry) for instructions on how to add a +`/etc/profile.d/guix.sh` entry. + +Regardless of which installation option you chose, the changes to +`/etc/profile.d` will not take effect until the next shell or desktop session, +so you should log out and log back in. + +## Option 3: Using fanquake's Docker image + +Please refer to fanquake's instructions +[here](https://github.com/fanquake/core-review/tree/master/guix). + +Note that the `Dockerfile` is largely equivalent to running through the binary +tarball installation steps. + +## Option 4: Using a distribution-maintained package + +Note that this section is based on the distro packaging situation at the time of +writing (July 2021). Guix is expected to be more widely packaged over time. For +an up-to-date view on Guix's package status/version across distros, please see: +https://repology.org/project/guix/versions + +### Debian 11 (Bullseye)/Ubuntu 21.04 (Hirsute Hippo) + +Guix v1.2.0 is available as a distribution package starting in [Debian +11](https://packages.debian.org/bullseye/guix) and [Ubuntu +21.04](https://packages.ubuntu.com/hirsute/guix). + +Note that if you intend on using Guix without using any substitutes (more +details [here][security-model]), v1.2.0 has a known problem when building GnuTLS +from source. Solutions and workarounds are documented +[here](#gnutls-test-suite-fail-status-request-revoked). + + +To install: +```sh +sudo apt install guix +``` + +For up-to-date information on Debian and Ubuntu's release history: +- [Debian release history](https://www.debian.org/releases/) +- [Ubuntu release history](https://ubuntu.com/about/release-cycle) + +### Arch Linux + +Guix is available in the AUR as +[`guix`](https://aur.archlinux.org/packages/guix/), please follow the +installation instructions in the Arch Linux Wiki ([live +link](https://wiki.archlinux.org/index.php/Guix#AUR_Package_Installation), +[2021/03/30 +permalink](https://wiki.archlinux.org/index.php?title=Guix&oldid=637559#AUR_Package_Installation)) +to install Guix. + +At the time of writing (2021/03/30), the `check` phase will fail if the path to +guix's build directory is longer than 36 characters due to an anachronistic +character limit on the shebang line. Since the `check` phase happens after the +`build` phase, which may take quite a long time, it is recommended that users +either: + +1. Skip the `check` phase + - For `makepkg`: `makepkg --nocheck ...` + - For `yay`: `yay --mflags="--nocheck" ...` + - For `paru`: `paru --nocheck ...` +2. Or, check their build directory's length beforehand + - For those building with `makepkg`: `pwd | wc -c` + +## Option 5: Building from source + +Building Guix from source is a rather involved process but a rewarding one for +those looking to minimize trust and maximize customizability (e.g. building a +particular commit of Guix). Previous experience with using autotools-style build +systems to build packages from source will be helpful. *hic sunt dracones.* + +I strongly urge you to at least skim through the entire section once before you +start issuing commands, as it will save you a lot of unnecessary pain and +anguish. + +### Installing common build tools + +There are a few basic build tools that are required for most things we'll build, +so let's install them now: + +Text transformation/i18n: +- `autopoint` (sometimes packaged in `gettext`) +- `help2man` +- `po4a` +- `texinfo` + +Build system tools: +- `g++` w/ C++11 support +- `libtool` +- `autoconf` +- `automake` +- `pkg-config` (sometimes packaged as `pkgconf`) +- `make` +- `cmake` + +Miscellaneous: +- `git` +- `gnupg` +- `python3` + +### Building and Installing Guix's dependencies + +In order to build Guix itself from source, we need to first make sure that the +necessary dependencies are installed and discoverable. The most up-to-date list +of Guix's dependencies is kept in the ["Requirements" +section](https://guix.gnu.org/manual/en/html_node/Requirements.html) of the Guix +Reference Manual. + +Depending on your distribution, most or all of these dependencies may already be +packaged and installable without manually building and installing. + +For reference, the graphic below outlines Guix v1.3.0's dependency graph: + + + +#### Guile + +##### Choosing a Guile version and sticking to it + +One of the first things you need to decide is which Guile version you want to +use: Guile v2.2 or Guile v3.0. Unlike the python2 to python3 transition, Guile +v2.2 and Guile v3.0 are largely compatible, as evidenced by the fact that most +Guile packages and even [Guix +itself](https://guix.gnu.org/en/blog/2020/guile-3-and-guix/) support running on +both. + +What is important here is that you **choose one**, and you **remain consistent** +with your choice throughout **all Guile-related packages**, no matter if they +are installed via the distribution's package manager or installed from source. +This is because the files for Guile packages are installed to directories which +are separated based on the Guile version. + +###### Example: Checking that Ubuntu's `guile-git` is compatible with your chosen Guile version + +On Ubuntu Focal: + +```sh +$ apt show guile-git +Package: guile-git +... +Depends: guile-2.2, guile-bytestructures, libgit2-dev +... +``` + +As you can see, the package `guile-git` depends on `guile-2.2`, meaning that it +was likely built for Guile v2.2. This means that if you decided to use Guile +v3.0 on Ubuntu Focal, you would need to build guile-git from source instead of +using the distribution package. + +On Ubuntu Hirsute: + +```sh +$ apt show guile-git +Package: guile-git +... +Depends: guile-3.0 | guile-2.2, guile-bytestructures (>= 1.0.7-3~), libgit2-dev (>= 1.0) +... +``` + +In this case, `guile-git` depends on either `guile-3.0` or `guile-2.2`, meaning +that it would work no matter what Guile version you decided to use. + +###### Corner case: Multiple versions of Guile on one system + +It is recommended to only install one version of Guile, so that build systems do +not get confused about which Guile to use. + +However, if you insist on having both Guile v2.2 and Guile v3.0 installed on +your system, then you need to **consistently** specify one of +`GUILE_EFFECTIVE_VERSION=3.0` or `GUILE_EFFECTIVE_VERSION=2.2` to all +`./configure` invocations for Guix and its dependencies. + +##### Installing Guile + +Guile is most likely already packaged for your distribution, so after you have +[chosen a Guile version](#choosing-a-guile-version-and-sticking-to-it), install +it via your distribution's package manager. + +If your distribution splits packages into `-dev`-suffixed and +non-`-dev`-suffixed sub-packages (as is the case for Debian-derived +distributions), please make sure to install both. For example, to install Guile +v2.2 on Debian/Ubuntu: + +```sh +apt install guile-2.2 guile-2.2-dev +``` + +#### Mixing distribution packages and source-built packages + +At the time of writing, most distributions have _some_ of Guix's dependencies +packaged, but not all. This means that you may want to install the distribution +package for some dependencies, and manually build-from-source for others. + +Distribution packages usually install to `/usr`, which is different from the +default `./configure` prefix of source-built packages: `/usr/local`. + +This means that if you mix-and-match distribution packages and source-built +packages and do not specify exactly `--prefix=/usr` to `./configure` for +source-built packages, you will need to augment the `GUILE_LOAD_PATH` and +`GUILE_LOAD_COMPILED_PATH` environment variables so that Guile will look +under the right prefix and find your source-built packages. + +For example, if you are using Guile v2.2, and have Guile packages in the +`/usr/local` prefix, either add the following lines to your `.profile` or +`.bash_profile` so that the environment variable is properly set for all future +shell logins, or paste the lines into a POSIX-style shell to temporarily modify +the environment variables of your current shell session. + +```sh +# Help Guile v2.2.x find packages in /usr/local +export GUILE_LOAD_PATH="/usr/local/share/guile/site/2.2${GUILE_LOAD_PATH:+:}$GUILE_LOAD_PATH" +export GUILE_LOAD_COMPILED_PATH="/usr/local/lib/guile/2.2/site-ccache${GUILE_LOAD_COMPILED_PATH:+:}$GUILE_COMPILED_LOAD_PATH" +``` + +Note that these environment variables are used to check for packages during +`./configure`, so they should be set as soon as possible should you want to use +a prefix other than `/usr`. + +#### Building and installing source-built packages + +***IMPORTANT**: A few dependencies have non-obvious quirks/errata which are +documented in the sub-sections immediately below. Please read these sections +before proceeding to build and install these packages.* + +Although you should always refer to the README or INSTALL files for the most +accurate information, most of these dependencies use autoconf-style build +systems (check if there's a `configure.ac` file), and will likely do the right +thing with the following: + +Clone the repository and check out the latest release: +```sh +git clone <git-repo-of-dependency>/<dependency>.git +cd <dependency> +git tag -l # check for the latest release +git checkout <latest-release> +``` + +For autoconf-based build systems (if `./autogen.sh` or `configure.ac` exists at +the root of the repository): + +```sh +./autogen.sh || autoreconf -vfi +./configure --prefix=<prefix> +make +sudo make install +``` + +For CMake-based build systems (if `CMakeLists.txt` exists at the root of the +repository): + +```sh +mkdir build && cd build +cmake .. -DCMAKE_INSTALL_PREFIX=<prefix> +sudo cmake --build . --target install +``` + +If you choose not to specify exactly `--prefix=/usr` to `./configure`, please +make sure you've carefully read the [previous section] on mixing distribution +packages and source-built packages. + +##### Binding packages require `-dev`-suffixed packages + +Relevant for: +- Everyone + +When building bindings, the `-dev`-suffixed version of the original package +needs to be installed. For example, building `Guile-zlib` on Debian-derived +distributions requires that `zlib1g-dev` is installed. + +When using bindings, the `-dev`-suffixed version of the original package still +needs to be installed. This is particularly problematic when distribution +packages are mispackaged like `guile-sqlite3` is in Ubuntu Focal such that +installing `guile-sqlite3` does not automatically install `libsqlite3-dev` as a +dependency. + +Below is a list of relevant Guile bindings and their corresponding `-dev` +packages in Debian at the time of writing. + +| Guile binding package | -dev Debian package | +|-----------------------|---------------------| +| guile-gcrypt | libgcrypt-dev | +| guile-git | libgit2-dev | +| guile-lzlib | liblz-dev | +| guile-ssh | libssh-dev | +| guile-sqlite3 | libsqlite3-dev | +| guile-zlib | zlib1g-dev | + +##### `guile-git` actually depends on `libgit2 >= 1.1` + +Relevant for: +- Those building `guile-git` from source against `libgit2 < 1.1` +- Those installing `guile-git` from their distribution where `guile-git` is + built against `libgit2 < 1.1` + +As of v0.4.0, `guile-git` claims to only require `libgit2 >= 0.28.0`, however, +it actually requires `libgit2 >= 1.1`, otherwise, it will be confused by a +reference of `origin/keyring`: instead of interpreting the reference as "the +'keyring' branch of the 'origin' remote", the reference is interpreted as "the +branch literally named 'origin/keyring'" + +This is especially notable because Ubuntu Focal packages `libgit2 v0.28.4`, and +`guile-git` is built against it. + +Should you be in this situation, you need to build both `libgit2 v1.1.x` and +`guile-git` from source. + +Source: https://logs.guix.gnu.org/guix/2020-11-12.log#232527 + +##### `{scheme,guile}-bytestructures` v1.0.8 and v1.0.9 are broken for Guile v2.2 + +Relevant for: +- Those building `{scheme,guile}-bytestructures` from source against Guile v2.2 + +Commit +[707eea3](https://github.com/TaylanUB/scheme-bytestructures/commit/707eea3a85e1e375e86702229ebf73d496377669) +introduced a regression for Guile v2.2 and was first included in v1.0.8, this +was later corrected in commit +[ec9a721](https://github.com/TaylanUB/scheme-bytestructures/commit/ec9a721957c17bcda13148f8faa5f06934431ff7) +and included in v1.1.0. + +TL;DR If you decided to use Guile v2.2, do not use `{scheme,guile}-bytestructures` v1.0.8 or v1.0.9. + +### Building and Installing Guix itself + +Start by cloning Guix: + +``` +git clone https://git.savannah.gnu.org/git/guix.git +cd guix +``` + +You will likely want to build the latest release, however, if the latest release +when you're reading this is still 1.2.0 then you may want to use 95aca29 instead +to avoid a problem in the GnuTLS test suite. + +``` +git branch -a -l 'origin/version-*' # check for the latest release +git checkout <latest-release> +``` + +Bootstrap the build system: +``` +./bootstrap +``` + +Configure with the recommended `--localstatedir` flag: +``` +./configure --localstatedir=/var +``` + +Note: If you intend to hack on Guix in the future, you will need to supply the +same `--localstatedir=` flag for all future Guix `./configure` invocations. See +the last paragraph of this +[section](https://guix.gnu.org/manual/en/html_node/Requirements.html) for more +details. + +Build Guix (this will take a while): +``` +make -j$(nproc) +``` + +Install Guix: + +``` +sudo make install +``` + +### Post-"build from source" Setup + +#### Creating and starting a `guix-daemon-original` service with a fixed `argv[0]` + +At this point, guix will be installed to `${bindir}`, which is likely +`/usr/local/bin` if you did not override directory variables at +`./configure`-time. More information on standard Automake directory variables +can be found +[here](https://www.gnu.org/software/automake/manual/html_node/Standard-Directory-Variables.html). + +However, the Guix init scripts and service configurations for Upstart, systemd, +SysV, and OpenRC are installed (in `${libdir}`) to launch +`${localstatedir}/guix/profiles/per-user/root/current-guix/bin/guix-daemon`, +which does not yet exist, and will only exist after [`root` performs their first +`guix pull`](#guix-pull-as-root). + +We need to create a `-original` version of these init scripts that's pointed to +the binaries we just built and `make install`'ed in `${bindir}` (normally, +`/usr/local/bin`). + +Example for `systemd`, run as `root`: + +```sh +# Create guix-daemon-original.service by modifying guix-daemon.service +libdir=# set according to your PREFIX (default is /usr/local/lib) +bindir="$(dirname $(command -v guix-daemon))" +sed -E -e "s|/\S*/guix/profiles/per-user/root/current-guix/bin/guix-daemon|${bindir}/guix-daemon|" "${libdir}"/systemd/system/guix-daemon.service > /etc/systemd/system/guix-daemon-original.service +chmod 664 /etc/systemd/system/guix-daemon-original.service + +# Make systemd recognize the new service +systemctl daemon-reload + +# Make sure that the non-working guix-daemon.service is stopped and disabled +systemctl stop guix-daemon +systemctl disable guix-daemon + +# Make sure that the working guix-daemon-original.service is started and enabled +systemctl enable guix-daemon-original +systemctl start guix-daemon-original +``` + +#### Creating `guix-daemon` users / groups + +Please see the [relevant +section](https://guix.gnu.org/manual/en/html_node/Build-Environment-Setup.html) +in the Guix Reference Manual for more details. + +## Optional setup + +At this point, you are set up to [use Guix to build Bitcoin +Core](./README.md#usage). However, if you want to polish your setup a bit and +make it "what Guix intended", then read the next few subsections. + +### Add an `/etc/profile.d` entry + +This section definitely does not apply to you if you installed Guix using: +1. The shell installer script +2. fanquake's Docker image +3. Debian's `guix` package + +#### Background + +Although Guix knows how to update itself and its packages, it does so in a +non-invasive way (it does not modify `/usr/local/bin/guix`). + +Instead, it does the following: + +- After a `guix pull`, it updates + `/var/guix/profiles/per-user/$USER/current-guix`, and creates a symlink + targeting this directory at `$HOME/.config/guix/current` + +- After a `guix install`, it updates + `/var/guix/profiles/per-user/$USER/guix-profile`, and creates a symlink + targeting this directory at `$HOME/.guix-profile` + +Therefore, in order for these operations to affect your shell/desktop sessions +(and for the principle of least astonishment to hold), their corresponding +directories have to be added to well-known environment variables like `$PATH`, +`$INFOPATH`, `$XDG_DATA_DIRS`, etc. + +In other words, if `$HOME/.config/guix/current/bin` does not exist in your +`$PATH`, a `guix pull` will have no effect on what `guix` you are using. Same +goes for `$HOME/.guix-profile/bin`, `guix install`, and installed packages. + +Helpfully, after a `guix pull` or `guix install`, a message will be printed like +so: + +``` +hint: Consider setting the necessary environment variables by running: + + GUIX_PROFILE="$HOME/.guix-profile" + . "$GUIX_PROFILE/etc/profile" + +Alternately, see `guix package --search-paths -p "$HOME/.guix-profile"'. +``` + +However, this is somewhat tedious to do for both `guix pull` and `guix install` +for each user on the system that wants to properly use `guix`. I recommend that +you instead add an entry to `/etc/profile.d` instead. This is done by default +when installing the Debian package later than 1.2.0-4 and when using the shell +script installer. + +#### Instructions + +Create `/etc/profile.d/guix.sh` with the following content: +```sh +# _GUIX_PROFILE: `guix pull` profile +_GUIX_PROFILE="$HOME/.config/guix/current" +if [ -L $_GUIX_PROFILE ]; then + export PATH="$_GUIX_PROFILE/bin${PATH:+:}$PATH" + # Export INFOPATH so that the updated info pages can be found + # and read by both /usr/bin/info and/or $GUIX_PROFILE/bin/info + # When INFOPATH is unset, add a trailing colon so that Emacs + # searches 'Info-default-directory-list'. + export INFOPATH="$_GUIX_PROFILE/share/info:$INFOPATH" +fi + +# GUIX_PROFILE: User's default profile +GUIX_PROFILE="$HOME/.guix-profile" +[ -L $GUIX_PROFILE ] || return +GUIX_LOCPATH="$GUIX_PROFILE/lib/locale" +export GUIX_PROFILE GUIX_LOCPATH + +[ -f "$GUIX_PROFILE/etc/profile" ] && . "$GUIX_PROFILE/etc/profile" + +# set XDG_DATA_DIRS to include Guix installations +export XDG_DATA_DIRS="$GUIX_PROFILE/share:${XDG_DATA_DIRS:-/usr/local/share/:/usr/share/}" +``` + +Please note that this will not take effect until the next shell or desktop +session (log out and log back in). + +### `guix pull` as root + +Before you do this, you need to read the section on [choosing your security +model][security-model] and adjust `guix` and `guix-daemon` flags according to +your choice, as invoking `guix pull` may pull substitutes from substitute +servers (which you may not want). + +As mentioned in a previous section, Guix expects +`${localstatedir}/guix/profiles/per-user/root/current-guix` to be populated with +`root`'s Guix profile, `guix pull`-ed and built by some former version of Guix. +However, this is not the case when we build from source. Therefore, we need to +perform a `guix pull` as `root`: + +```sh +sudo --login guix pull --branch=version-<latest-release-version> +# or +sudo --login guix pull --commit=<particular-commit> +``` + +`guix pull` is quite a long process (especially if you're using +`--no-substitute`). If you encounter build problems, please refer to the +[troubleshooting section](#troubleshooting). + +Note that running a bare `guix pull` with no commit or branch specified will +pull the latest commit on Guix's master branch, which is likely fine, but not +recommended. + +If you installed Guix from source, you may get an error like the following: +```sh +error: while creating symlink '/root/.config/guix/current' No such file or directory +``` +To resolve this, simply: +``` +sudo mkdir -p /root/.config/guix +``` +Then try the `guix pull` command again. + +After the `guix pull` finishes successfully, +`${localstatedir}/guix/profiles/per-user/root/current-guix` should be populated. + +#### Using the newly-pulled `guix` by restarting the daemon + +Depending on how you installed Guix, you should now make sure that your init +scripts and service configurations point to the newly-pulled `guix-daemon`. + +##### If you built Guix from source + +If you followed the instructions for [fixing argv\[0\]][fix-argv0], you can now +do the following: + +```sh +systemctl stop guix-daemon-original +systemctl disable guix-daemon-original + +systemctl enable guix-daemon +systemctl start guix-daemon +``` + +##### If you installed Guix via the Debian/Ubuntu distribution packages + +You will need to create a `guix-daemon-latest` service which points to the new +`guix` rather than a pinned one. + +```sh +# Create guix-daemon-latest.service by modifying guix-daemon.service +sed -E -e "s|/usr/bin/guix-daemon|/var/guix/profiles/per-user/root/current-guix/bin/guix-daemon|" /etc/systemd/system/guix-daemon.service > /lib/systemd/system/guix-daemon-latest.service +chmod 664 /lib/systemd/system/guix-daemon-latest.service + +# Make systemd recognize the new service +systemctl daemon-reload + +# Make sure that the old guix-daemon.service is stopped and disabled +systemctl stop guix-daemon +systemctl disable guix-daemon + +# Make sure that the new guix-daemon-latest.service is started and enabled +systemctl enable guix-daemon-latest +systemctl start guix-daemon-latest +``` + +##### If you installed Guix via lantw44's Arch Linux AUR package + +At the time of writing (July 5th, 2021) the systemd unit for "updated Guix" is +`guix-daemon-latest.service`, therefore, you should do the following: + +```sh +systemctl stop guix-daemon +systemctl disable guix-daemon + +systemctl enable guix-daemon-latest +systemctl start guix-daemon-latest +``` + +##### Otherwise... + +Simply do: + +```sh +systemctl restart guix-daemon +``` + +### Checking everything + +If you followed all the steps above to make your Guix setup "prim and proper," +you can check that you did everything properly by running through this +checklist. + +1. `/etc/profile.d/guix.sh` should exist and be sourced at each shell login + +2. `guix describe` should not print `guix describe: error: failed to determine + origin`, but rather something like: + + ``` + Generation 38 Feb 22 2021 16:39:31 (current) + guix f350df4 + repository URL: https://git.savannah.gnu.org/git/guix.git + branch: version-1.2.0 + commit: f350df405fbcd5b9e27e6b6aa500da7f101f41e7 + ``` + +3. `guix-daemon` should be running from `${localstatedir}/guix/profiles/per-user/root/current-guix` + +# Troubleshooting + +## Derivation failed to build + +When you see a build failure like below: + +``` +building /gnu/store/...-foo-3.6.12.drv... +/ 'check' phasenote: keeping build directory `/tmp/guix-build-foo-3.6.12.drv-0' +builder for `/gnu/store/...-foo-3.6.12.drv' failed with exit code 1 +build of /gnu/store/...-foo-3.6.12.drv failed +View build log at '/var/log/guix/drvs/../...-foo-3.6.12.drv.bz2'. +cannot build derivation `/gnu/store/...-qux-7.69.1.drv': 1 dependencies couldn't be built +cannot build derivation `/gnu/store/...-bar-3.16.5.drv': 1 dependencies couldn't be built +cannot build derivation `/gnu/store/...-baz-2.0.5.drv': 1 dependencies couldn't be built +guix time-machine: error: build of `/gnu/store/...-baz-2.0.5.drv' failed +``` + +It means that `guix` failed to build a package named `foo`, which was a +dependency of `qux`, `bar`, and `baz`. Importantly, note that the last "failed" +line is not necessarily the root cause, the first "failed" line is. + +Most of the time, the build failure is due to a spurious test failure or the +package's build system/test suite breaking when running multi-threaded. To +rebuild _just_ this derivation in a single-threaded fashion (please don't forget +to add other `guix` flags like `--no-substitutes` as appropriate): + +```sh +$ guix build --cores=1 /gnu/store/...-foo-3.6.12.drv +``` + +If the single-threaded rebuild did not succeed, you may need to dig deeper. +You may view `foo`'s build logs in `less` like so (please replace paths with the +path you see in the build failure output): + +```sh +$ bzcat /var/log/guix/drvs/../...-foo-3.6.12.drv.bz2 | less +``` + +`foo`'s build directory is also preserved and available at +`/tmp/guix-build-foo-3.6.12.drv-0`. However, if you fail to build `foo` multiple +times, it may be `/tmp/...drv-1` or `/tmp/...drv-2`. Always consult the build +failure output for the most accurate, up-to-date information. + +### python(-minimal): [Errno 84] Invalid or incomplete multibyte or wide character + +This error occurs when your `$TMPDIR` (default: /tmp) exists on a filesystem +which rejects characters not present in the UTF-8 character code set. An example +is ZFS with the utf8only=on option set. + +More information: https://bugs.python.org/issue37584 + +### GnuTLS: test-suite FAIL: status-request-revoked + +*The derivation is likely identified by: `/gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv`* + +This unfortunate error is most common for non-substitute builders who installed +Guix v1.2.0. The problem stems from the fact that one of GnuTLS's tests uses a +hardcoded certificate which expired on 2020-10-24. + +What's more unfortunate is that this GnuTLS derivation is somewhat special in +Guix's dependency graph and is not affected by the package transformation flags +like `--without-tests=`. + +The easiest solution for those encountering this problem is to install a newer +version of Guix. However, there are ways to work around this issue: + +#### Workaround 1: Using substitutes for this single derivation + +If you've authorized the official Guix build farm's key (more info +[here](./README.md#step-1-authorize-the-signing-keys)), then you can use +substitutes just for this single derivation by invoking the following: + +```sh +guix build --substitute-urls="https://ci.guix.gnu.org" /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv +``` + +See [this section](./README.md#removing-authorized-keys) for instructions on how +to remove authorized keys if you don't want to keep the build farm's key +authorized. + +#### Workaround 2: Temporarily setting the system clock back + +This workaround was described [here](https://issues.guix.gnu.org/44559#5). + +Basically: +1. Turn off networking +2. Turn off NTP +3. Set system time to 2020-10-01 +4. guix build --no-substitutes /gnu/store/vhphki5sg9xkdhh2pbc8gi6vhpfzryf0-gnutls-3.6.12.drv +5. Set system time back to accurate current time +6. Turn NTP back on +7. Turn networking back on + +### coreutils: FAIL: tests/tail-2/inotify-dir-recreate + +The inotify-dir-create test fails on "remote" filesystems such as overlayfs +(Docker's default filesystem) due to the filesystem being mistakenly recognized +as non-remote. + +A relatively easy workaround to this is to make sure that a somewhat traditional +filesystem is mounted at `/tmp` (where `guix-daemon` performs its builds). For +Docker users, this might mean [using a volume][docker/volumes], [binding +mounting][docker/bind-mnt] from host, or (for those with enough RAM and swap) +[mounting a tmpfs][docker/tmpfs] using the `--tmpfs` flag. + +Please see the following links for more details: + +- An upstream coreutils bug has been filed: [debbugs#47940](https://debbugs.gnu.org/cgi/bugreport.cgi?bug=47940) +- A Guix bug detailing the underlying problem has been filed: [guix-issues#47935](https://issues.guix.gnu.org/47935) +- A commit to skip this test in Guix has been merged into the core-updates branch: +[savannah/guix@6ba1058](https://git.savannah.gnu.org/cgit/guix.git/commit/?id=6ba1058df0c4ce5611c2367531ae5c3cdc729ab4) + + +[install-script]: #options-1-and-2-using-the-official-shell-installer-script-or-binary-tarball +[install-bin-tarball]: #options-1-and-2-using-the-official-shell-installer-script-or-binary-tarball +[install-fanquake-docker]: #option-3-using-fanquakes-docker-image +[install-distro-pkg]: #option-4-using-a-distribution-maintained-package +[install-source]: #option-5-building-from-source + +[fix-argv0]: #creating-and-starting-a-guix-daemon-original-service-with-a-fixed-argv0 +[security-model]: ./README.md#choosing-your-security-model + +[docker/volumes]: https://docs.docker.com/storage/volumes/ +[docker/bind-mnt]: https://docs.docker.com/storage/bind-mounts/ +[docker/tmpfs]: https://docs.docker.com/storage/tmpfs/ diff --git a/contrib/guix/README.md b/contrib/guix/README.md index e604b370e3..51a034c26e 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -9,83 +9,171 @@ downloads. We achieve bootstrappability by using Guix as a functional package manager. -## Requirements +# Requirements -Conservatively, a x86_64 machine with: +Conservatively, you will need an x86_64 machine with: - 16GB of free disk space on the partition that /gnu/store will reside in -- 8GB of free disk space per platform triple you're planning on building (see - the `HOSTS` environment variable description) +- 8GB of free disk space **per platform triple** you're planning on building + (see the `HOSTS` [environment variable description][env-vars-list]) -## Setup +# Installation and Setup -### Installing Guix +If you don't have Guix installed and set up, please follow the instructions in +[INSTALL.md](./INSTALL.md) -If you're just testing this out, you can use the -[Dockerfile][fanquake/guix-docker] for convenience. It automatically speeds up -your builds by [using substitutes](#speeding-up-builds-with-substitute-servers). -If you don't want this behaviour, refer to the [next -section](#choosing-your-security-model). +# Usage -Otherwise, follow the [Guix installation guide][guix/bin-install]. +If you haven't considered your security model yet, please read [the relevant +section](#choosing-your-security-model) before proceeding to perform a build. -> Note: For those who like to keep their filesystems clean, Guix is designed to -> be very standalone and _will not_ conflict with your system's package -> manager/existing setup. It _only_ touches `/var/guix`, `/gnu`, and -> `~/.config/guix`. +## Making the Xcode SDK available for macOS cross-compilation -### Choosing your security model +In order to perform a build for macOS (which is included in the default set of +platform triples to build), you'll need to extract the macOS SDK tarball using +tools found in the [`macdeploy` directory](../macdeploy/README.md). -Guix allows us to achieve better binary security by using our CPU time to build -everything from scratch. However, it doesn't sacrifice user choice in pursuit of -this: users can decide whether or not to bootstrap and to use substitutes -(pre-built packages). +You can then either point to the SDK using the `SDK_PATH` environment variable: -After installation, you may want to consider [adding substitute -servers](#speeding-up-builds-with-substitute-servers) from which to download -pre-built packages to speed up your build if that fits your security model (say, -if you're just testing that this works). Substitute servers are set up by -default if you're using the [Dockerfile][fanquake/guix-docker]. +```sh +# Extract the SDK tarball to /path/to/parent/dir/of/extracted/SDK/Xcode-<foo>-<bar>-extracted-SDK-with-libcxx-headers +tar -C /path/to/parent/dir/of/extracted/SDK -xaf /path/to/Xcode-<foo>-<bar>-extracted-SDK-with-libcxx-headers.tar.gz -If you prefer not to use any substitutes, make sure to supply `--no-substitutes` -like in the following snippet. The first build will take a while, but the -resulting packages will be cached for future builds. +# Indicate where to locate the SDK tarball +export SDK_PATH=/path/to/parent/dir/of/extracted/SDK +``` + +or extract it into `depends/SDKs`: ```sh -export ADDITIONAL_GUIX_COMMON_FLAGS='--no-substitutes' +mkdir -p depends/SDKs +tar -C depends/SDKs -xaf /path/to/SDK/tarball ``` -Likewise, to perform a bootstrapped build (takes even longer): +## Building + +*The author highly recommends at least reading over the [common usage patterns +and examples](#common-guix-build-invocation-patterns-and-examples) section below +before starting a build. For a full list of customization options, see the +[recognized environment variables][env-vars-list] section.* + +To build Bitcoin Core reproducibly with all default options, invoke the +following from the top of a clean repository: ```sh -export ADDITIONAL_GUIX_COMMON_FLAGS='--no-substitutes' ADDITIONAL_GUIX_ENVIRONMENT_FLAGS='--bootstrap' +./contrib/guix/guix-build ``` -### Using a version of Guix with `guix time-machine` capabilities +## Codesigning build outputs -> Note: This entire section can be skipped if you are already using a version of -> Guix that has [the `guix time-machine` command][guix/time-machine]. +The `guix-codesign` command attaches codesignatures (produced by codesigners) to +existing non-codesigned outputs. Please see the [release process +documentation](/doc/release-process.md) for more context. -Once Guix is installed, if it doesn't have the `guix time-machine` command, pull -the latest `guix`. +It respects many of the same environment variable flags as `guix-build`, with 2 +crucial differences: + +1. Since only Windows and macOS build outputs require codesigning, the `HOSTS` + environment variable will have a sane default value of `x86_64-w64-mingw32 + x86_64-apple-darwin19` instead of all the platforms. +2. The `guix-codesign` command ***requires*** a `DETACHED_SIGS_REPO` flag. + * _**DETACHED_SIGS_REPO**_ + + Set the directory where detached codesignatures can be found for the current + Bitcoin Core version being built. + + _REQUIRED environment variable_ + +An invocation with all default options would look like: + +``` +env DETACHED_SIGS_REPO=<path/to/bitcoin-detached-sigs> ./contrib/guix/guix-codesign +``` + +## Cleaning intermediate work directories + +By default, `guix-build` leaves all intermediate files or "work directories" +(e.g. `depends/work`, `guix-build-*/distsrc-*`) intact at the end of a build so +that they are available to the user (to aid in debugging, etc.). However, these +directories usually take up a large amount of disk space. Therefore, a +`guix-clean` convenience script is provided which cleans the current `git` +worktree to save disk space: + +``` +./contrib/guix/guix-clean +``` + + +## Attesting to build outputs + +Much like how Gitian build outputs are attested to in a `gitian.sigs` +repository, Guix build outputs are attested to in the [`guix.sigs` +repository](https://github.com/bitcoin-core/guix.sigs). + +After you've cloned the `guix.sigs` repository, to attest to the current +worktree's commit/tag: + +``` +env GUIX_SIGS_REPO=<path/to/guix.sigs> SIGNER=<gpg-key-name> ./contrib/guix/guix-attest +``` + +See `./contrib/guix/guix-attest --help` for more information on the various ways +`guix-attest` can be invoked. + +## Verifying build output attestations + +After at least one other signer has uploaded their signatures to the `guix.sigs` +repository: + +``` +git -C <path/to/guix.sigs> pull +env GUIX_SIGS_REPO=<path/to/guix.sigs> ./contrib/guix/guix-verify +``` + + +## Common `guix-build` invocation patterns and examples + +### Keeping caches and SDKs outside of the worktree + +If you perform a lot of builds and have a bunch of worktrees, you may find it +more efficient to keep the depends tree's download cache, build cache, and SDKs +outside of the worktrees to avoid duplicate downloads and unnecessary builds. To +help with this situation, the `guix-build` script honours the `SOURCES_PATH`, +`BASE_CACHE`, and `SDK_PATH` environment variables and will pass them on to the +depends tree so that you can do something like: ```sh -guix pull --max-jobs=4 # change number of jobs accordingly +env SOURCES_PATH="$HOME/depends-SOURCES_PATH" BASE_CACHE="$HOME/depends-BASE_CACHE" SDK_PATH="$HOME/macOS-SDKs" ./contrib/guix/guix-build ``` -Make sure that you are using your current profile. (You are prompted to do this -at the end of the `guix pull`) +Note that the paths that these environment variables point to **must be +directories**, and **NOT symlinks to directories**. + +See the [recognized environment variables][env-vars-list] section for more +details. + +### Building a subset of platform triples -```bash -export PATH="${HOME}/.config/guix/current/bin${PATH:+:}$PATH" +Sometimes you only want to build a subset of the supported platform triples, in +which case you can override the default list by setting the space-separated +`HOSTS` environment variable: + +```sh +env HOSTS='x86_64-w64-mingw32 x86_64-apple-darwin19' ./contrib/guix/guix-build ``` +See the [recognized environment variables][env-vars-list] section for more +details. + ### Controlling the number of threads used by `guix` build commands +Depending on your system's RAM capacity, you may want to decrease the number of +threads used to decrease RAM usage or vice versa. + By default, the scripts under `./contrib/guix` will invoke all `guix` build commands with `--cores="$JOBS"`. Note that `$JOBS` defaults to `$(nproc)` if not -specified. However, astute manual readers will also notice that there is a -`--max-jobs=` flag (which defaults to 1 if unspecified). +specified. However, astute manual readers will also notice that `guix` build +commands also accept a `--max-jobs=` flag (which defaults to 1 if unspecified). Here is the difference between `--cores=` and `--max-jobs=`: @@ -124,31 +212,19 @@ packages when the dependency graph allows for it, you may want to try: export JOBS=1 ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=8' ``` -## Usage - -### As a Tool for Deterministic Builds - -From the top of a clean Bitcoin Core repository: - -```sh -./contrib/guix/guix-build -``` - -After the build finishes successfully (check the status code please), compare -hashes: - -```sh -find output/ -type f -print0 | sort -z | xargs -r0 sha256sum -``` +See the [recognized environment variables][env-vars-list] section for more +details. -#### Recognized environment variables +## Recognized environment variables * _**HOSTS**_ Override the space-separated list of platform triples for which to perform a - bootstrappable build. _(defaults to "x86\_64-linux-gnu arm-linux-gnueabihf - aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu - x86\_64-w64-mingw32 x86\_64-apple-darwin18")_ + bootstrappable build. + + _(defaults to "x86\_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu + riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu + x86\_64-w64-mingw32 x86\_64-apple-darwin19")_ * _**SOURCES_PATH**_ @@ -156,18 +232,27 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum depends tree. Setting this to the same directory across multiple builds of the depends tree can eliminate unnecessary redownloading of package sources. + The path that this environment variable points to **must be a directory**, and + **NOT a symlink to a directory**. + * _**BASE_CACHE**_ Set the depends tree cache for built packages. This is passed through to the depends tree. Setting this to the same directory across multiple builds of the depends tree can eliminate unnecessary building of packages. + The path that this environment variable points to **must be a directory**, and + **NOT a symlink to a directory**. + * _**SDK_PATH**_ Set the path where _extracted_ SDKs can be found. This is passed through to the depends tree. Note that this is should be set to the _parent_ directory of - the actual SDK (e.g. SDK_PATH=$HOME/Downloads/macOS-SDKs instead of - $HOME/Downloads/macOS-SDKs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers). + the actual SDK (e.g. `SDK_PATH=$HOME/Downloads/macOS-SDKs` instead of + `$HOME/Downloads/macOS-SDKs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers`). + + The path that this environment variable points to **must be a directory**, and + **NOT a symlink to a directory**. * _**JOBS**_ @@ -178,13 +263,17 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum - `make` as in `make --jobs="$JOBS"` - `xargs` as in `xargs -P"$JOBS"` + See [here](#controlling-the-number-of-threads-used-by-guix-build-commands) for + more details. + _(defaults to the value of `nproc` outside the container)_ * _**SOURCE_DATE_EPOCH**_ Override the reference UNIX timestamp used for bit-for-bit reproducibility, - the variable name conforms to [standard][r12e/source-date-epoch]. _(defaults - to the output of `$(git log --format=%at -1)`)_ + the variable name conforms to [standard][r12e/source-date-epoch]. + + _(defaults to the output of `$(git log --format=%at -1)`)_ * _**V**_ @@ -200,8 +289,7 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum A whitespace-delimited list of URLs from which to download pre-built packages. A URL is only used if its signing key is authorized (refer to the [substitute - servers section](#speeding-up-builds-with-substitute-servers) for more - details). + servers section](#option-1-building-with-substitutes) for more details). * _**ADDITIONAL_GUIX_COMMON_FLAGS**_ @@ -216,28 +304,65 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum Additional flags to be passed to the invocation of `guix environment` inside `guix time-machine`. -## Tips and Tricks - -### Speeding up builds with substitute servers +# Choosing your security model -_This whole section is automatically done in the convenience -[Dockerfiles][fanquake/guix-docker]_ +No matter how you installed Guix, you need to decide on your security model for +building packages with Guix. -For those who are used to life in the fast _(and trustful)_ lane, you can -specify [substitute servers][guix/substitutes] from which to download pre-built -packages. +Guix allows us to achieve better binary security by using our CPU time to build +everything from scratch. However, it doesn't sacrifice user choice in pursuit of +this: users can decide whether or not to use **substitutes** (pre-built +packages). + +## Option 1: Building with substitutes + +### Step 1: Authorize the signing keys + +Depending on the installation procedure you followed, you may have already +authorized the Guix build farm key. In particular, the official shell installer +script asks you if you want the key installed, and the debian distribution +package authorized the key during installation. + +You can check the current list of authorized keys at `/etc/guix/acl`. + +At the time of writing, a `/etc/guix/acl` with just the Guix build farm key +authorized looks something like: + +```lisp +(acl + (entry + (public-key + (ecc + (curve Ed25519) + (q #8D156F295D24B0D9A86FA5741A840FF2D24F60F7B6C4134814AD55625971B394#) + ) + ) + (tag + (guix import) + ) + ) + ) +``` -> For those who only want to use substitutes from the official Guix build farm -> and have authorized the build farm's signing key during Guix's installation, -> you don't need to do anything. +If you've determined that the official Guix build farm key hasn't been +authorized, and you would like to authorize it, run the following as root: -#### Step 1: Authorize the signing keys +``` +guix archive --authorize < /var/guix/profiles/per-user/root/current-guix/share/guix/ci.guix.gnu.org.pub +``` -For the official Guix build farm at https://ci.guix.gnu.org, run as root: +If +`/var/guix/profiles/per-user/root/current-guix/share/guix/ci.guix.gnu.org.pub` +doesn't exist, try: +```sh +guix archive --authorize < <PREFIX>/share/guix/ci.guix.gnu.org.pub ``` -guix archive --authorize < ~root/.config/guix/current/share/guix/ci.guix.gnu.org.pub -``` + +Where `<PREFIX>` is likely: +- `/usr` if you installed from a distribution package +- `/usr/local` if you installed Guix from source and didn't supply any + prefix-modifying flags to Guix's `./configure` For dongcarl's substitute server at https://guix.carldong.io, run as root: @@ -245,92 +370,104 @@ For dongcarl's substitute server at https://guix.carldong.io, run as root: wget -qO- 'https://guix.carldong.io/signing-key.pub' | guix archive --authorize ``` -#### Step 2: Specify the substitute servers +#### Removing authorized keys -The official Guix build farm at https://ci.guix.gnu.org is automatically used -unless the `--no-substitutes` flag is supplied. +To remove previously authorized keys, simply edit `/etc/guix/acl` and remove the +`(entry (public-key ...))` entry. -This can be overridden for all `guix` invocations by passing the -`--substitute-urls` option to your invocation of `guix-daemon`. This can also be -overridden on a call-by-call basis by passing the same `--substitute-urls` -option to client tools such at `guix environment`. +### Step 2: Specify the substitute servers -To use dongcarl's substitute server for Bitcoin Core builds after having -[authorized his signing key](#authorize-the-signing-keys): +Once its key is authorized, the official Guix build farm at +https://ci.guix.gnu.org is automatically used unless the `--no-substitutes` flag +is supplied. This default list of substitute servers is overridable both on a +`guix-daemon` level and when you invoke `guix` commands. See examples below for +the various ways of adding dongcarl's substitute server after having [authorized +his signing key](#authorize-the-signing-keys). -``` -export SUBSTITUTE_URLS='https://guix.carldong.io https://ci.guix.gnu.org' +Change the **default list** of substitute servers by starting `guix-daemon` with +the `--substitute-urls` option (you will likely need to edit your init script): + +```sh +guix-daemon <cmd> --substitute-urls='https://guix.carldong.io https://ci.guix.gnu.org' ``` -## Troubleshooting +Override the default list of substitute servers by passing the +`--substitute-urls` option for invocations of `guix` commands: -### Derivation failed to build +```sh +guix <cmd> --substitute-urls='https://guix.carldong.io https://ci.guix.gnu.org' +``` -When you see a build failure like below: +For scripts under `./contrib/guix`, set the `SUBSTITUTE_URLS` environment +variable: -``` -building /gnu/store/...-foo-3.6.12.drv... -/ 'check' phasenote: keeping build directory `/tmp/guix-build-foo-3.6.12.drv-0' -builder for `/gnu/store/...-foo-3.6.12.drv' failed with exit code 1 -build of /gnu/store/...-foo-3.6.12.drv failed -View build log at '/var/log/guix/drvs/../...-foo-3.6.12.drv.bz2'. -cannot build derivation `/gnu/store/...-qux-7.69.1.drv': 1 dependencies couldn't be built -cannot build derivation `/gnu/store/...-bar-3.16.5.drv': 1 dependencies couldn't be built -cannot build derivation `/gnu/store/...-baz-2.0.5.drv': 1 dependencies couldn't be built -guix time-machine: error: build of `/gnu/store/...-baz-2.0.5.drv' failed +```sh +export SUBSTITUTE_URLS='https://guix.carldong.io https://ci.guix.gnu.org' ``` -It means that `guix` failed to build a package named `foo`, which was a -dependency of `qux`, `bar`, and `baz`. Importantly, note that the last "failed" -line is not necessarily the root cause, the first "failed" line is. +## Option 2: Disabling substitutes on an ad-hoc basis -Most of the time, the build failure is due to a spurious test failure or the -package's build system/test suite breaking when running multi-threaded. To -rebuild _just_ this derivation in a single-threaded fashion: +If you prefer not to use any substitutes, make sure to supply `--no-substitutes` +like in the following snippet. The first build will take a while, but the +resulting packages will be cached for future builds. +For direct invocations of `guix`: ```sh -$ guix build --cores=1 /gnu/store/...-foo-3.6.12.drv +guix <cmd> --no-substitutes ``` -If the single-threaded rebuild did not succeed, you may need to dig deeper. -You may view `foo`'s build logs in `less` like so (please replace paths with the -path you see in the build failure output): - +For the scripts under `./contrib/guix/`: ```sh -$ bzcat /var/log/guix/drvs/../...-foo-3.6.12.drv.bz2 | less +export ADDITIONAL_GUIX_COMMON_FLAGS='--no-substitutes' ``` -`foo`'s build directory is also preserved and available at -`/tmp/guix-build-foo-3.6.12.drv-0`. However, if you fail to build `foo` multiple -times, it may be `/tmp/...drv-1` or `/tmp/...drv-2`. Always consult the build -failure output for the most accurate, up-to-date information. +## Option 3: Disabling substitutes by default + +`guix-daemon` accepts a `--no-substitutes` flag, which will make sure that, +unless otherwise overridden by a command line invocation, no substitutes will be +used. -#### python(-minimal): [Errno 84] Invalid or incomplete multibyte or wide character +If you start `guix-daemon` using an init script, you can edit said script to +supply this flag. -This error occurs when your `$TMPDIR` (default: /tmp) exists on a filesystem -which rejects characters not present in the UTF-8 character code set. An example -is ZFS with the utf8only=on option set. -More information: https://bugs.python.org/issue37584 +# Purging/Uninstalling Guix -## FAQ +In the extraordinarily rare case where you messed up your Guix installation in +an irreversible way, you may want to completely purge Guix from your system and +start over. -### How can I trust the binary installation? +1. Uninstall Guix itself according to the way you installed it (e.g. `sudo apt + purge guix` for Ubuntu packaging, `sudo make uninstall` for a build from source). +2. Remove all build users and groups -As mentioned at the bottom of [this manual page][guix/bin-install]: + You may check for relevant users and groups using: -> The binary installation tarballs can be (re)produced and verified simply by -> running the following command in the Guix source tree: -> -> make guix-binary.x86_64-linux.tar.xz + ``` + getent passwd | grep guix + getent group | grep guix + ``` -### Is Guix packaged in my operating system? + Then, you may remove users and groups using: -Guix is shipped starting with [Debian Bullseye][debian/guix-bullseye] and -[Ubuntu 21.04 "Hirsute Hippo"][ubuntu/guix-hirsute]. Other operating systems -are working on packaging Guix as well. + ``` + sudo userdel <user> + sudo groupdel <group> + ``` -[b17e]: http://bootstrappable.org/ +3. Remove all possible Guix-related directories + - `/var/guix/` + - `/var/log/guix/` + - `/gnu/` + - `/etc/guix/` + - `/home/*/.config/guix/` + - `/home/*/.cache/guix/` + - `/home/*/.guix-profile/` + - `/root/.config/guix/` + - `/root/.cache/guix/` + - `/root/.guix-profile/` + +[b17e]: https://bootstrappable.org/ [r12e/source-date-epoch]: https://reproducible-builds.org/docs/source-date-epoch/ [guix/install.sh]: https://git.savannah.gnu.org/cgit/guix.git/plain/etc/guix-install.sh @@ -343,3 +480,5 @@ are working on packaging Guix as well. [debian/guix-bullseye]: https://packages.debian.org/bullseye/guix [ubuntu/guix-hirsute]: https://packages.ubuntu.com/hirsute/guix [fanquake/guix-docker]: https://github.com/fanquake/core-review/tree/master/guix + +[env-vars-list]: #recognized-environment-variables diff --git a/contrib/guix/guix-attest b/contrib/guix/guix-attest index c8cf73d400..6e12cbead7 100755 --- a/contrib/guix/guix-attest +++ b/contrib/guix/guix-attest @@ -18,7 +18,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" # Required non-builtin commands should be invokable ################ -check_tools cat env basename mkdir xargs find +check_tools cat env basename mkdir diff sort if [ -z "$NO_SIGN" ]; then check_tools gpg fi @@ -162,6 +162,18 @@ EOF echo "Attesting to build outputs for version: '${VERSION}'" echo "" +# Given a SHA256SUMS file as stdin that has lines like: +# 0ba536819b221a91d3d42e978be016aac918f40984754d74058aa0c921cd3ea6 a/b/d/c/d/s/bitcoin-22.0rc2-riscv64-linux-gnu.tar.gz +# ... +# +# Replace each line's file name with its basename: +# 0ba536819b221a91d3d42e978be016aac918f40984754d74058aa0c921cd3ea6 bitcoin-22.0rc2-riscv64-linux-gnu.tar.gz +# ... +# +basenameify_SHA256SUMS() { + sed -E 's@(^[[:xdigit:]]{64}[[:space:]]+).+/([^/]+$)@\1\2@' +} + outsigdir="$GUIX_SIGS_REPO/$VERSION/$signer_name" mkdir -p "$outsigdir" ( @@ -174,7 +186,8 @@ mkdir -p "$outsigdir" cat "${noncodesigned_fragments[@]}" \ | sort -u \ | sort -k2 \ - > "$temp_noncodesigned" + | basenameify_SHA256SUMS \ + > "$temp_noncodesigned" if [ -e noncodesigned.SHA256SUMS ]; then # The SHA256SUMS already exists, make sure it's exactly what we # expect, error out if not @@ -192,8 +205,8 @@ mkdir -p "$outsigdir" exit 1 fi - temp_codesigned="$(mktemp)" - trap 'rm -rf -- "$temp_codesigned"' EXIT + temp_all="$(mktemp)" + trap 'rm -rf -- "$temp_all"' EXIT if (( ${#codesigned_fragments[@]} )); then # Note: all.SHA256SUMS attests to all of $sha256sum_fragments, but is @@ -201,18 +214,19 @@ mkdir -p "$outsigdir" cat "${sha256sum_fragments[@]}" \ | sort -u \ | sort -k2 \ - > "$temp_codesigned" - if [ -e codesigned.SHA256SUMS ]; then + | basenameify_SHA256SUMS \ + > "$temp_all" + if [ -e all.SHA256SUMS ]; then # The SHA256SUMS already exists, make sure it's exactly what we # expect, error out if not - if diff -u all.SHA256SUMS "$temp_codesigned"; then + if diff -u all.SHA256SUMS "$temp_all"; then echo "An all.SHA256SUMS file already exists for '${VERSION}' and is up-to-date." else shasum_already_exists all.SHA256SUMS exit 1 fi else - mv "$temp_codesigned" codesigned.SHA256SUMS + mv "$temp_all" all.SHA256SUMS fi else # It is fine to have the codesigned outputs be missing (perhaps the @@ -226,6 +240,7 @@ mkdir -p "$outsigdir" for i in *.SHA256SUMS; do if [ ! -e "$i".asc ]; then gpg --detach-sign \ + --digest-algo sha256 \ --local-user "$gpg_key_name" \ --armor \ --output "$i".asc "$i" diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index 29d6701b25..9317fa7fde 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -18,7 +18,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" # Required non-builtin commands should be invocable ################ -check_tools cat mkdir make git guix +check_tools cat mkdir make getent curl git guix ################ # GUIX_BUILD_OPTIONS should be empty @@ -76,7 +76,7 @@ mkdir -p "$VERSION_BASE" # Default to building for all supported HOSTs (overridable by environment) export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu x86_64-w64-mingw32 - x86_64-apple-darwin18}" + x86_64-apple-darwin19}" # Usage: distsrc_for_host HOST # @@ -186,6 +186,29 @@ fi # # However, the internal API is likely to change more than the CLI invocation +################ +# Services database must have basic entries +################ + +if ! getent services http https ftp > /dev/null 2>&1; then +cat << EOF +ERR: Your system's C library can not find service database entries for at least + one of the following services: http, https, ftp. + +Hint: Most likely, /etc/services does not exist yet (common for docker images + and minimal distros), or you don't have permissions to access it. + + If /etc/services does not exist yet, you may want to install the + appropriate package for your distro which provides it. + + On Debian/Ubuntu: netbase + On Arch Linux: iana-etc + + For more information, see: getent(1), services(5) + +EOF + +fi ######### # SETUP # @@ -209,14 +232,14 @@ host_to_commonname() { } # Determine the reference time used for determinism (overridable by environment) -SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" +SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" # Execute "$@" in a pinned, possibly older version of Guix, for reproducibility # across time. time-machine() { # shellcheck disable=SC2086 - guix time-machine --url=https://github.com/dongcarl/guix.git \ - --commit=490e39ff303f4f6873a04bfb8253755bdae1b29c \ + guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ + --commit=aa34d4d28dfe25ba47d5800d05000fb7221788c0 \ --cores="$JOBS" \ --keep-failed \ --fallback \ diff --git a/contrib/guix/guix-codesign b/contrib/guix/guix-codesign index 62853dbf4d..aff897037d 100755 --- a/contrib/guix/guix-codesign +++ b/contrib/guix/guix-codesign @@ -91,7 +91,7 @@ fi ################ # Default to building for all supported HOSTs (overridable by environment) -export HOSTS="${HOSTS:-x86_64-w64-mingw32 x86_64-apple-darwin18}" +export HOSTS="${HOSTS:-x86_64-w64-mingw32 x86_64-apple-darwin19}" # Usage: distsrc_for_host HOST # @@ -220,20 +220,20 @@ fi JOBS="${JOBS:-$(nproc)}" # Determine the reference time used for determinism (overridable by environment) -SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log --format=%at -1)}" +SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" # Execute "$@" in a pinned, possibly older version of Guix, for reproducibility # across time. time-machine() { # shellcheck disable=SC2086 - guix time-machine --url=https://github.com/dongcarl/guix.git \ - --commit=490e39ff303f4f6873a04bfb8253755bdae1b29c \ - --cores="$JOBS" \ - --keep-failed \ - --fallback \ - ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ - ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ - -- "$@" + guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ + --commit=aa34d4d28dfe25ba47d5800d05000fb7221788c0 \ + --cores="$JOBS" \ + --keep-failed \ + --fallback \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ + -- "$@" } # Make sure an output directory exists for our builds diff --git a/contrib/guix/guix-verify b/contrib/guix/guix-verify index a6e2c4065e..02ae022741 100755 --- a/contrib/guix/guix-verify +++ b/contrib/guix/guix-verify @@ -28,7 +28,11 @@ cmd_usage() { cat <<EOF Synopsis: - env GUIX_SIGS_REPO=<path/to/guix.sigs> ./contrib/guix/guix-verify + env GUIX_SIGS_REPO=<path/to/guix.sigs> [ SIGNER=<signer> ] ./contrib/guix/guix-verify + +Example overriding signer's manifest to use as base + + env GUIX_SIGS_REPO=/home/dongcarl/guix.sigs SIGNER=achow101 ./contrib/guix/guix-verify EOF } @@ -73,11 +77,13 @@ verify() { echo "" echo "Hint: Either the signature is invalid or the public key is missing" echo "" + failure=1 elif ! diff --report-identical "$compare_manifest" "$current_manifest" 1>&2; then echo "ERR: The SHA256SUMS attestation in these two directories differ:" echo " '${compare_manifest}'" echo " '${current_manifest}'" echo "" + failure=1 else echo "Verified: '${current_manifest}'" echo "" @@ -92,6 +98,17 @@ echo "--------------------" echo "" if (( ${#all_noncodesigned[@]} )); then compare_noncodesigned="${all_noncodesigned[0]}" + if [[ -n "$SIGNER" ]]; then + signer_noncodesigned="$OUTSIGDIR_BASE/$SIGNER/noncodesigned.SHA256SUMS" + if [[ -f "$signer_noncodesigned" ]]; then + echo "Using $SIGNER's manifest as the base to compare against" + compare_noncodesigned="$signer_noncodesigned" + else + echo "Unable to find $SIGNER's manifest, using the first one found" + fi + else + echo "No SIGNER provided, using the first manifest found" + fi for current_manifest in "${all_noncodesigned[@]}"; do verify "$compare_noncodesigned" "$current_manifest" @@ -112,6 +129,17 @@ echo "--------------------" echo "" if (( ${#all_all[@]} )); then compare_all="${all_all[0]}" + if [[ -n "$SIGNER" ]]; then + signer_all="$OUTSIGDIR_BASE/$SIGNER/all.SHA256SUMS" + if [[ -f "$signer_all" ]]; then + echo "Using $SIGNER's manifest as the base to compare against" + compare_all="$signer_all" + else + echo "Unable to find $SIGNER's manifest, using the first one found" + fi + else + echo "No SIGNER provided, using the first manifest found" + fi for current_manifest in "${all_all[@]}"; do verify "$compare_all" "$current_manifest" @@ -140,3 +168,7 @@ if (( ${#all_noncodesigned[@]} + ${#all_all[@]} == 0 )); then echo "" exit 1 fi + +if [ -n "$failure" ]; then + exit 1 +fi diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 6741328473..93526f8c45 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -147,7 +147,7 @@ case "$HOST" in # # After the native packages in depends are built, the ld wrapper should # no longer affect our build, as clang would instead reach for - # x86_64-apple-darwin18-ld from cctools + # x86_64-apple-darwin19-ld from cctools ;; *) export GUIX_LD_WRAPPER_DISABLE_RPATH=yes ;; esac @@ -214,6 +214,7 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \ x86_64_linux_NM=x86_64-linux-gnu-nm \ x86_64_linux_STRIP=x86_64-linux-gnu-strip \ qt_config_opts_i686_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ + qt_config_opts_x86_64_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ FORCE_USE_SYSTEM_CLANG=1 @@ -226,7 +227,6 @@ GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}.tar.gz" # Create the source tarball if not already there if [ ! -e "$GIT_ARCHIVE" ]; then mkdir -p "$(dirname "$GIT_ARCHIVE")" - touch "${DIST_ARCHIVE_BASE}"/SKIPATTEST.TAG git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD fi @@ -239,7 +239,7 @@ mkdir -p "$OUTDIR" # CONFIGFLAGS CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" case "$HOST" in - *linux*) CONFIGFLAGS+=" --enable-glibc-back-compat" ;; + *linux*) CONFIGFLAGS+=" --disable-threadlocal" ;; esac # CFLAGS @@ -253,12 +253,23 @@ esac # CXXFLAGS HOST_CXXFLAGS="$HOST_CFLAGS" +case "$HOST" in + arm-linux-gnueabihf) HOST_CXXFLAGS="${HOST_CXXFLAGS} -Wno-psabi" ;; +esac + # LDFLAGS case "$HOST" in *linux*) HOST_LDFLAGS="-Wl,--as-needed -Wl,--dynamic-linker=$glibc_dynamic_linker -static-libstdc++ -Wl,-O2" ;; *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; esac +# Using --no-tls-get-addr-optimize retains compatibility with glibc 2.17, by +# avoiding a PowerPC64 optimisation available in glibc 2.22 and later. +# https://sourceware.org/binutils/docs-2.35/ld/PowerPC64-ELF64.html +case "$HOST" in + *powerpc64*) HOST_LDFLAGS="${HOST_LDFLAGS} -Wl,--no-tls-get-addr-optimize" ;; +esac + case "$HOST" in powerpc64-linux-*|riscv64-linux-*) HOST_LDFLAGS="${HOST_LDFLAGS} -Wl,-z,noexecstack" ;; esac @@ -291,10 +302,11 @@ mkdir -p "$DISTSRC" # Build Bitcoin Core make --jobs="$JOBS" ${V:+V=1} - # Perform basic ELF security checks on a series of executables. + # Check that symbol/security checks tools are sane. + make test-security-check ${V:+V=1} + # Perform basic security checks on a series of executables. make -C src --jobs=1 check-security ${V:+V=1} - # Check that executables only contain allowed gcc, glibc and libstdc++ - # version symbols for Linux distro back-compatibility. + # Check that executables only contain allowed version symbols. make -C src --jobs=1 check-symbols ${V:+V=1} mkdir -p "$OUTDIR" @@ -411,8 +423,8 @@ mkdir -p "$DISTSRC" find "${DISTNAME}" -print0 \ | sort --zero-terminated \ | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin18/osx64}.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin18/osx64}.tar.gz" && exit 1 ) + | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin19/osx64}.tar.gz" \ + || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST//x86_64-apple-darwin19/osx64}.tar.gz" && exit 1 ) ;; esac ) # $DISTSRC/installed @@ -445,5 +457,6 @@ mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ find "$ACTUAL_OUTDIR" -type f } | xargs realpath --relative-base="$PWD" \ | xargs sha256sum \ + | sort -k2 \ | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part ) diff --git a/contrib/guix/libexec/codesign.sh b/contrib/guix/libexec/codesign.sh index b1eec686ec..f484ac5774 100755 --- a/contrib/guix/libexec/codesign.sh +++ b/contrib/guix/libexec/codesign.sh @@ -108,5 +108,6 @@ mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" \ find "$ACTUAL_OUTDIR" -type f } | xargs realpath --relative-base="$PWD" \ | xargs sha256sum \ + | sort -k2 \ | sponge "$ACTUAL_OUTDIR"/SHA256SUMS.part ) diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index 9705607119..40ae4b5208 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -49,7 +49,7 @@ fi # Set common variables ################ -VERSION="${VERSION:-$(git_head_version)}" +VERSION="${FORCE_VERSION:-$(git_head_version)}" DISTNAME="${DISTNAME:-bitcoin-${VERSION}}" version_base_prefix="${PWD}/guix-build-" diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index ba168a2a4a..5805006053 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -80,6 +80,10 @@ http://www.linuxfromscratch.org/hlfs/view/development/chapter05/gcc-pass1.html" (("-rpath=") "-rpath-link=")) #t)))))))) +(define (make-binutils-with-mingw-w64-disable-flags xbinutils) + (package-with-extra-patches xbinutils + (search-our-patches "binutils-mingw-w64-disable-flags.patch"))) + (define (make-cross-toolchain target base-gcc-for-libc base-kernel-headers @@ -135,11 +139,25 @@ chain for " target " development.")) (package-with-extra-patches gcc-8 (search-our-patches "gcc-8-sort-libtool-find-output.patch"))) +;; Building glibc with stack smashing protector first landed in glibc 2.25, use +;; this function to disable for older glibcs +;; +;; From glibc 2.25 changelog: +;; +;; * Most of glibc can now be built with the stack smashing protector enabled. +;; It is recommended to build glibc with --enable-stack-protector=strong. +;; Implemented by Nick Alcock (Oracle). +(define (make-glibc-without-ssp xglibc) + (package-with-extra-configure-variable + (package-with-extra-configure-variable + xglibc "libc_cv_ssp" "no") + "libc_cv_ssp_strong" "no")) + (define* (make-bitcoin-cross-toolchain target #:key (base-gcc-for-libc gcc-7) - (base-kernel-headers linux-libre-headers-5.4) - (base-libc glibc) ; glibc 2.31 + (base-kernel-headers linux-libre-headers-4.9) + (base-libc (make-glibc-without-ssp glibc-2.24)) (base-gcc (make-gcc-rpath-link base-gcc))) "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values desirable for building Bitcoin Core release binaries." @@ -154,7 +172,7 @@ desirable for building Bitcoin Core release binaries." (define (make-mingw-pthreads-cross-toolchain target) "Create a cross-compilation toolchain package for TARGET" - (let* ((xbinutils (cross-binutils target)) + (let* ((xbinutils (make-binutils-with-mingw-w64-disable-flags (cross-binutils target))) (pthreads-xlibc mingw-w64-x86_64-winpthreads) (pthreads-xgcc (make-gcc-with-pthreads (cross-gcc target @@ -557,6 +575,28 @@ and endian independent.") inspecting signatures in Mach-O binaries.") (license license:expat)))) +(define-public glibc-2.24 + (package + (inherit glibc) + (version "2.24") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://sourceware.org/git/glibc.git") + (commit "0d7f1ed30969886c8dde62fbf7d2c79967d4bace"))) + (file-name (git-file-name "glibc" "0d7f1ed30969886c8dde62fbf7d2c79967d4bace")) + (sha256 + (base32 + "0g5hryia5v1k0qx97qffgwzrz4lr4jw3s5kj04yllhswsxyjbic3")) + (patches (search-our-patches "glibc-ldd-x86_64.patch" + "glibc-versioned-locpath.patch" + "glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch" + "glibc-2.24-no-build-time-cxx-header-run.patch")))))) + +(define glibc-2.27/bitcoin-patched + (package-with-extra-patches glibc-2.27 + (search-our-patches "glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch"))) + (packages->manifest (append (list ;; The Basics @@ -606,7 +646,12 @@ inspecting signatures in Mach-O binaries.") (make-nsis-with-sde-support nsis-x86_64) osslsigncode)) ((string-contains target "-linux-") - (list (make-bitcoin-cross-toolchain target))) + (list (cond ((string-contains target "riscv64-") + (make-bitcoin-cross-toolchain target + #:base-libc glibc-2.27/bitcoin-patched + #:base-kernel-headers linux-libre-headers-4.19)) + (else + (make-bitcoin-cross-toolchain target))))) ((string-contains target "darwin") (list clang-toolchain-10 binutils imagemagick libtiff librsvg font-tuffy cmake xorriso python-signapple)) (else '()))))) diff --git a/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch b/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch new file mode 100644 index 0000000000..8f88eb9dfd --- /dev/null +++ b/contrib/guix/patches/binutils-mingw-w64-disable-flags.patch @@ -0,0 +1,171 @@ +Description: Add disable opposites to the security-related flags +Author: Stephen Kitt <skitt@debian.org> + +This patch adds "no-" variants to disable the various security flags: +"no-dynamicbase", "no-nxcompat", "no-high-entropy-va", "disable-reloc-section". + +--- a/ld/emultempl/pe.em ++++ b/ld/emultempl/pe.em +@@ -259,9 +261,11 @@ + (OPTION_ENABLE_LONG_SECTION_NAMES + 1) + /* DLLCharacteristics flags. */ + #define OPTION_DYNAMIC_BASE (OPTION_DISABLE_LONG_SECTION_NAMES + 1) +-#define OPTION_FORCE_INTEGRITY (OPTION_DYNAMIC_BASE + 1) ++#define OPTION_NO_DYNAMIC_BASE (OPTION_DYNAMIC_BASE + 1) ++#define OPTION_FORCE_INTEGRITY (OPTION_NO_DYNAMIC_BASE + 1) + #define OPTION_NX_COMPAT (OPTION_FORCE_INTEGRITY + 1) +-#define OPTION_NO_ISOLATION (OPTION_NX_COMPAT + 1) ++#define OPTION_NO_NX_COMPAT (OPTION_NX_COMPAT + 1) ++#define OPTION_NO_ISOLATION (OPTION_NO_NX_COMPAT + 1) + #define OPTION_NO_SEH (OPTION_NO_ISOLATION + 1) + #define OPTION_NO_BIND (OPTION_NO_SEH + 1) + #define OPTION_WDM_DRIVER (OPTION_NO_BIND + 1) +@@ -271,6 +275,7 @@ + #define OPTION_NO_INSERT_TIMESTAMP (OPTION_INSERT_TIMESTAMP + 1) + #define OPTION_BUILD_ID (OPTION_NO_INSERT_TIMESTAMP + 1) + #define OPTION_ENABLE_RELOC_SECTION (OPTION_BUILD_ID + 1) ++#define OPTION_DISABLE_RELOC_SECTION (OPTION_ENABLE_RELOC_SECTION + 1) + + static void + gld${EMULATION_NAME}_add_options +@@ -342,8 +347,10 @@ + {"enable-long-section-names", no_argument, NULL, OPTION_ENABLE_LONG_SECTION_NAMES}, + {"disable-long-section-names", no_argument, NULL, OPTION_DISABLE_LONG_SECTION_NAMES}, + {"dynamicbase",no_argument, NULL, OPTION_DYNAMIC_BASE}, ++ {"no-dynamicbase", no_argument, NULL, OPTION_NO_DYNAMIC_BASE}, + {"forceinteg", no_argument, NULL, OPTION_FORCE_INTEGRITY}, + {"nxcompat", no_argument, NULL, OPTION_NX_COMPAT}, ++ {"no-nxcompat", no_argument, NULL, OPTION_NO_NX_COMPAT}, + {"no-isolation", no_argument, NULL, OPTION_NO_ISOLATION}, + {"no-seh", no_argument, NULL, OPTION_NO_SEH}, + {"no-bind", no_argument, NULL, OPTION_NO_BIND}, +@@ -351,6 +358,7 @@ + {"tsaware", no_argument, NULL, OPTION_TERMINAL_SERVER_AWARE}, + {"build-id", optional_argument, NULL, OPTION_BUILD_ID}, + {"enable-reloc-section", no_argument, NULL, OPTION_ENABLE_RELOC_SECTION}, ++ {"disable-reloc-section", no_argument, NULL, OPTION_DISABLE_RELOC_SECTION}, + {NULL, no_argument, NULL, 0} + }; + +@@ -485,9 +494,12 @@ + in object files\n")); + fprintf (file, _(" --dynamicbase Image base address may be relocated using\n\ + address space layout randomization (ASLR)\n")); ++ fprintf (file, _(" --no-dynamicbase Image base address may not be relocated\n")); + fprintf (file, _(" --enable-reloc-section Create the base relocation table\n")); ++ fprintf (file, _(" --disable-reloc-section Disable the base relocation table\n")); + fprintf (file, _(" --forceinteg Code integrity checks are enforced\n")); + fprintf (file, _(" --nxcompat Image is compatible with data execution prevention\n")); ++ fprintf (file, _(" --no-nxcompat Image is not compatible with data execution prevention\n")); + fprintf (file, _(" --no-isolation Image understands isolation but do not isolate the image\n")); + fprintf (file, _(" --no-seh Image does not use SEH. No SE handler may\n\ + be called in this image\n")); +@@ -862,12 +874,21 @@ + case OPTION_ENABLE_RELOC_SECTION: + pe_dll_enable_reloc_section = 1; + break; ++ case OPTION_DISABLE_RELOC_SECTION: ++ pe_dll_enable_reloc_section = 0; ++ /* fall through */ ++ case OPTION_NO_DYNAMIC_BASE: ++ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE; ++ break; + case OPTION_FORCE_INTEGRITY: + pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_FORCE_INTEGRITY; + break; + case OPTION_NX_COMPAT: + pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; + break; ++ case OPTION_NO_NX_COMPAT: ++ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; ++ break; + case OPTION_NO_ISOLATION: + pe_dll_characteristics |= IMAGE_DLLCHARACTERISTICS_NO_ISOLATION; + break; +--- a/ld/emultempl/pep.em ++++ b/ld/emultempl/pep.em +@@ -237,9 +240,12 @@ + OPTION_ENABLE_LONG_SECTION_NAMES, + OPTION_DISABLE_LONG_SECTION_NAMES, + OPTION_HIGH_ENTROPY_VA, ++ OPTION_NO_HIGH_ENTROPY_VA, + OPTION_DYNAMIC_BASE, ++ OPTION_NO_DYNAMIC_BASE, + OPTION_FORCE_INTEGRITY, + OPTION_NX_COMPAT, ++ OPTION_NO_NX_COMPAT, + OPTION_NO_ISOLATION, + OPTION_NO_SEH, + OPTION_NO_BIND, +@@ -248,7 +254,8 @@ + OPTION_NO_INSERT_TIMESTAMP, + OPTION_TERMINAL_SERVER_AWARE, + OPTION_BUILD_ID, +- OPTION_ENABLE_RELOC_SECTION ++ OPTION_ENABLE_RELOC_SECTION, ++ OPTION_DISABLE_RELOC_SECTION + }; + + static void +@@ -315,9 +322,12 @@ + {"enable-long-section-names", no_argument, NULL, OPTION_ENABLE_LONG_SECTION_NAMES}, + {"disable-long-section-names", no_argument, NULL, OPTION_DISABLE_LONG_SECTION_NAMES}, + {"high-entropy-va", no_argument, NULL, OPTION_HIGH_ENTROPY_VA}, ++ {"no-high-entropy-va", no_argument, NULL, OPTION_NO_HIGH_ENTROPY_VA}, + {"dynamicbase",no_argument, NULL, OPTION_DYNAMIC_BASE}, ++ {"no-dynamicbase", no_argument, NULL, OPTION_NO_DYNAMIC_BASE}, + {"forceinteg", no_argument, NULL, OPTION_FORCE_INTEGRITY}, + {"nxcompat", no_argument, NULL, OPTION_NX_COMPAT}, ++ {"no-nxcompat", no_argument, NULL, OPTION_NO_NX_COMPAT}, + {"no-isolation", no_argument, NULL, OPTION_NO_ISOLATION}, + {"no-seh", no_argument, NULL, OPTION_NO_SEH}, + {"no-bind", no_argument, NULL, OPTION_NO_BIND}, +@@ -327,6 +337,7 @@ + {"no-insert-timestamp", no_argument, NULL, OPTION_NO_INSERT_TIMESTAMP}, + {"build-id", optional_argument, NULL, OPTION_BUILD_ID}, + {"enable-reloc-section", no_argument, NULL, OPTION_ENABLE_RELOC_SECTION}, ++ {"disable-reloc-section", no_argument, NULL, OPTION_DISABLE_RELOC_SECTION}, + {NULL, no_argument, NULL, 0} + }; + +@@ -448,11 +461,15 @@ + in object files\n")); + fprintf (file, _(" --high-entropy-va Image is compatible with 64-bit address space\n\ + layout randomization (ASLR)\n")); ++ fprintf (file, _(" --no-high-entropy-va Image is not compatible with 64-bit ASLR\n")); + fprintf (file, _(" --dynamicbase Image base address may be relocated using\n\ + address space layout randomization (ASLR)\n")); ++ fprintf (file, _(" --no-dynamicbase Image base address may not be relocated\n")); + fprintf (file, _(" --enable-reloc-section Create the base relocation table\n")); ++ fprintf (file, _(" --disable-reloc-section Disable the base relocation table\n")); + fprintf (file, _(" --forceinteg Code integrity checks are enforced\n")); + fprintf (file, _(" --nxcompat Image is compatible with data execution prevention\n")); ++ fprintf (file, _(" --no-nxcompat Image is not compatible with data execution prevention\n")); + fprintf (file, _(" --no-isolation Image understands isolation but do not isolate the image\n")); + fprintf (file, _(" --no-seh Image does not use SEH; no SE handler may\n\ + be called in this image\n")); +@@ -809,12 +826,24 @@ + case OPTION_ENABLE_RELOC_SECTION: + pep_dll_enable_reloc_section = 1; + break; ++ case OPTION_DISABLE_RELOC_SECTION: ++ pep_dll_enable_reloc_section = 0; ++ /* fall through */ ++ case OPTION_NO_DYNAMIC_BASE: ++ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE; ++ /* fall through */ ++ case OPTION_NO_HIGH_ENTROPY_VA: ++ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA; ++ break; + case OPTION_FORCE_INTEGRITY: + pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_FORCE_INTEGRITY; + break; + case OPTION_NX_COMPAT: + pe_dll_characteristics |= IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; + break; ++ case OPTION_NO_NX_COMPAT: ++ pe_dll_characteristics &= ~IMAGE_DLL_CHARACTERISTICS_NX_COMPAT; ++ break; + case OPTION_NO_ISOLATION: + pe_dll_characteristics |= IMAGE_DLLCHARACTERISTICS_NO_ISOLATION; + break; diff --git a/contrib/guix/patches/gcc-8-sort-libtool-find-output.patch b/contrib/guix/patches/gcc-8-sort-libtool-find-output.patch index 1dfe3ba132..f327c464f3 100644 --- a/contrib/guix/patches/gcc-8-sort-libtool-find-output.patch +++ b/contrib/guix/patches/gcc-8-sort-libtool-find-output.patch @@ -1,3 +1,11 @@ +guix: repro: Sort find output in libtool for gcc-8 + +Otherwise the resulting .a static libraries (e.g. libstdc++.a) will not +be reproducible and end up making the Bitcoin binaries non-reproducible +as well. + +See: https://reproducible-builds.org/docs/archives/#gnu-libtool + diff --git a/gcc/configure b/gcc/configure index 97ba7d7d69c..e37a96f0c0c 100755 --- a/gcc/configure diff --git a/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch b/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch new file mode 100644 index 0000000000..5c4d0c6ebe --- /dev/null +++ b/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch @@ -0,0 +1,62 @@ +https://sourceware.org/git/?p=glibc.git;a=commit;h=a68ba2f3cd3cbe32c1f31e13c20ed13487727b32 + +commit 6b02af31e9a721bb15a11380cd22d53b621711f8 +Author: Szabolcs Nagy <szabolcs.nagy@arm.com> +Date: Wed Oct 18 17:26:23 2017 +0100 + + [AARCH64] Rewrite elf_machine_load_address using _DYNAMIC symbol + + This patch rewrites aarch64 elf_machine_load_address to use special _DYNAMIC + symbol instead of _dl_start. + + The static address of _DYNAMIC symbol is stored in the first GOT entry. + Here is the change which makes this solution work (part of binutils 2.24): + https://sourceware.org/ml/binutils/2013-06/msg00248.html + + i386, x86_64 targets use the same method to do this as well. + + The original implementation relies on a trick that R_AARCH64_ABS32 relocation + being resolved at link time and the static address fits in the 32bits. + However, in LP64, normally, the address is defined to be 64 bit. + + Here is the C version one which should be portable in all cases. + + * sysdeps/aarch64/dl-machine.h (elf_machine_load_address): Use + _DYNAMIC symbol to calculate load address. + +diff --git a/sysdeps/aarch64/dl-machine.h b/sysdeps/aarch64/dl-machine.h +index e86d8b5b63..5a5b8a5de5 100644 +--- a/sysdeps/aarch64/dl-machine.h ++++ b/sysdeps/aarch64/dl-machine.h +@@ -49,26 +49,11 @@ elf_machine_load_address (void) + /* To figure out the load address we use the definition that for any symbol: + dynamic_addr(symbol) = static_addr(symbol) + load_addr + +- The choice of symbol is arbitrary. The static address we obtain +- by constructing a non GOT reference to the symbol, the dynamic +- address of the symbol we compute using adrp/add to compute the +- symbol's address relative to the PC. +- This depends on 32bit relocations being resolved at link time +- and that the static address fits in the 32bits. */ +- +- ElfW(Addr) static_addr; +- ElfW(Addr) dynamic_addr; +- +- asm (" \n" +-" adrp %1, _dl_start; \n" +-" add %1, %1, #:lo12:_dl_start \n" +-" ldr %w0, 1f \n" +-" b 2f \n" +-"1: \n" +-" .word _dl_start \n" +-"2: \n" +- : "=r" (static_addr), "=r" (dynamic_addr)); +- return dynamic_addr - static_addr; ++ _DYNAMIC sysmbol is used here as its link-time address stored in ++ the special unrelocated first GOT entry. */ ++ ++ extern ElfW(Dyn) _DYNAMIC[] attribute_hidden; ++ return (ElfW(Addr)) &_DYNAMIC - elf_machine_dynamic (); + } + + /* Set up the loaded object described by L so its unrelocated PLT diff --git a/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch b/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch new file mode 100644 index 0000000000..11fe7fdc99 --- /dev/null +++ b/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch @@ -0,0 +1,100 @@ +https://sourceware.org/git/?p=glibc.git;a=commit;h=fc3e1337be1c6935ab58bd13520f97a535cf70cc + +commit dc23a45db566095e83ff0b7a57afc87fb5ca89a1 +Author: Florian Weimer <fweimer@redhat.com> +Date: Wed Sep 21 10:45:32 2016 +0200 + + Avoid running $(CXX) during build to obtain header file paths + + This reduces the build time somewhat and is particularly noticeable + during rebuilds with few code changes. + +diff --git a/Makerules b/Makerules +index 7e4077ee50..c338850de5 100644 +--- a/Makerules ++++ b/Makerules +@@ -121,14 +121,10 @@ ifneq (,$(CXX)) + # will be used instead of /usr/include/stdlib.h and /usr/include/math.h. + before-compile := $(common-objpfx)cstdlib $(common-objpfx)cmath \ + $(before-compile) +-cstdlib=$(shell echo "\#include <cstdlib>" | $(CXX) -M -MP -x c++ - \ +- | sed -n "/cstdlib:/{s/:$$//;p}") +-$(common-objpfx)cstdlib: $(cstdlib) ++$(common-objpfx)cstdlib: $(c++-cstdlib-header) + $(INSTALL_DATA) $< $@T + $(move-if-change) $@T $@ +-cmath=$(shell echo "\#include <cmath>" | $(CXX) -M -MP -x c++ - \ +- | sed -n "/cmath:/{s/:$$//;p}") +-$(common-objpfx)cmath: $(cmath) ++$(common-objpfx)cmath: $(c++-cmath-header) + $(INSTALL_DATA) $< $@T + $(move-if-change) $@T $@ + endif +diff --git a/config.make.in b/config.make.in +index 95c6f36876..04a8b3ed7f 100644 +--- a/config.make.in ++++ b/config.make.in +@@ -45,6 +45,8 @@ defines = @DEFINES@ + sysheaders = @sysheaders@ + sysincludes = @SYSINCLUDES@ + c++-sysincludes = @CXX_SYSINCLUDES@ ++c++-cstdlib-header = @CXX_CSTDLIB_HEADER@ ++c++-cmath-header = @CXX_CMATH_HEADER@ + all-warnings = @all_warnings@ + enable-werror = @enable_werror@ + +diff --git a/configure b/configure +index 17625e1041..6ff252744b 100755 +--- a/configure ++++ b/configure +@@ -635,6 +635,8 @@ BISON + INSTALL_INFO + PERL + BASH_SHELL ++CXX_CMATH_HEADER ++CXX_CSTDLIB_HEADER + CXX_SYSINCLUDES + SYSINCLUDES + AUTOCONF +@@ -5054,6 +5056,18 @@ fi + + + ++# Obtain some C++ header file paths. This is used to make a local ++# copy of those headers in Makerules. ++if test -n "$CXX"; then ++ find_cxx_header () { ++ echo "#include <$1>" | $CXX -M -MP -x c++ - | sed -n "/$1:/{s/:\$//;p}" ++ } ++ CXX_CSTDLIB_HEADER="$(find_cxx_header cstdlib)" ++ CXX_CMATH_HEADER="$(find_cxx_header cmath)" ++fi ++ ++ ++ + # Test if LD_LIBRARY_PATH contains the notation for the current directory + # since this would lead to problems installing/building glibc. + # LD_LIBRARY_PATH contains the current directory if one of the following +diff --git a/configure.ac b/configure.ac +index 33bcd62180..9938ab0dc2 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -1039,6 +1039,18 @@ fi + AC_SUBST(SYSINCLUDES) + AC_SUBST(CXX_SYSINCLUDES) + ++# Obtain some C++ header file paths. This is used to make a local ++# copy of those headers in Makerules. ++if test -n "$CXX"; then ++ find_cxx_header () { ++ echo "#include <$1>" | $CXX -M -MP -x c++ - | sed -n "/$1:/{s/:\$//;p}" ++ } ++ CXX_CSTDLIB_HEADER="$(find_cxx_header cstdlib)" ++ CXX_CMATH_HEADER="$(find_cxx_header cmath)" ++fi ++AC_SUBST(CXX_CSTDLIB_HEADER) ++AC_SUBST(CXX_CMATH_HEADER) ++ + # Test if LD_LIBRARY_PATH contains the notation for the current directory + # since this would lead to problems installing/building glibc. + # LD_LIBRARY_PATH contains the current directory if one of the following diff --git a/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch new file mode 100644 index 0000000000..d6217157ee --- /dev/null +++ b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch @@ -0,0 +1,72 @@ +https://sourceware.org/git/?p=glibc.git;a=commit;h=0b9c84906f653978fb8768c7ebd0ee14a47e662e + +From 562c52cc81a4e456a62e6455feb32732049e9070 Mon Sep 17 00:00:00 2001 +From: "H.J. Lu" <hjl.tools@gmail.com> +Date: Mon, 31 Dec 2018 09:26:42 -0800 +Subject: [PATCH] riscv: Use __has_include__ to include <asm/syscalls.h> [BZ + #24022] + +<asm/syscalls.h> has been removed by + +commit 27f8899d6002e11a6e2d995e29b8deab5aa9cc25 +Author: David Abdurachmanov <david.abdurachmanov@gmail.com> +Date: Thu Nov 8 20:02:39 2018 +0100 + + riscv: add asm/unistd.h UAPI header + + Marcin Juszkiewicz reported issues while generating syscall table for riscv + using 4.20-rc1. The patch refactors our unistd.h files to match some other + architectures. + + - Add asm/unistd.h UAPI header, which has __ARCH_WANT_NEW_STAT only for 64-bit + - Remove asm/syscalls.h UAPI header and merge to asm/unistd.h + - Adjust kernel asm/unistd.h + + So now asm/unistd.h UAPI header should show all syscalls for riscv. + +<asm/syscalls.h> may be restored by + +Subject: [PATCH] riscv: restore asm/syscalls.h UAPI header +Date: Tue, 11 Dec 2018 09:09:35 +0100 + +UAPI header asm/syscalls.h was merged into UAPI asm/unistd.h header, +which did resolve issue with missing syscalls macros resulting in +glibc (2.28) build failure. It also broke glibc in a different way: +asm/syscalls.h is being used by glibc. I noticed this while doing +Fedora 30/Rawhide mass rebuild. + +The patch returns asm/syscalls.h header and incl. it into asm/unistd.h. +I plan to send a patch to glibc to use asm/unistd.h instead of +asm/syscalls.h + +In the meantime, we use __has_include__, which was added to GCC 5, to +check if <asm/syscalls.h> exists before including it. Tested with +build-many-glibcs.py for riscv against kernel 4.19.12 and 4.20-rc7. + + [BZ #24022] + * sysdeps/unix/sysv/linux/riscv/flush-icache.c: Check if + <asm/syscalls.h> exists with __has_include__ before including it. +--- + sysdeps/unix/sysv/linux/riscv/flush-icache.c | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/sysdeps/unix/sysv/linux/riscv/flush-icache.c b/sysdeps/unix/sysv/linux/riscv/flush-icache.c +index d612ef4c6c..0b2042620b 100644 +--- a/sysdeps/unix/sysv/linux/riscv/flush-icache.c ++++ b/sysdeps/unix/sysv/linux/riscv/flush-icache.c +@@ -21,7 +21,11 @@ + #include <stdlib.h> + #include <atomic.h> + #include <sys/cachectl.h> +-#include <asm/syscalls.h> ++#if __has_include__ (<asm/syscalls.h>) ++# include <asm/syscalls.h> ++#else ++# include <asm/unistd.h> ++#endif + + typedef int (*func_type) (void *, void *, unsigned long int); + +-- +2.31.1 + diff --git a/contrib/guix/patches/glibc-ldd-x86_64.patch b/contrib/guix/patches/glibc-ldd-x86_64.patch new file mode 100644 index 0000000000..b1b6d5a548 --- /dev/null +++ b/contrib/guix/patches/glibc-ldd-x86_64.patch @@ -0,0 +1,10 @@ +By default, 'RTDLLIST' in 'ldd' refers to 'lib64/ld-linux-x86-64.so', whereas +it's in 'lib/' for us. This patch fixes that. + +--- glibc-2.17/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed 2012-12-25 04:02:13.000000000 +0100 ++++ glibc-2.17/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed 2013-09-15 23:08:03.000000000 +0200 +@@ -1,3 +1,3 @@ + /LD_TRACE_LOADED_OBJECTS=1/a\ + add_env="$add_env LD_LIBRARY_VERSION=\\$verify_out" +-s_^\(RTLDLIST=\)\(.*lib\)\(\|64\|x32\)\(/[^/]*\)\(-x86-64\|-x32\)\(\.so\.[0-9.]*\)[ ]*$_\1"\2\4\6 \264\4-x86-64\6 \2x32\4-x32\6"_ ++s_^\(RTLDLIST=\)\(.*lib\)\(\|64\|x32\)\(/[^/]*\)\(-x86-64\|-x32\)\(\.so\.[0-9.]*\)[ ]*$_\1"\2\4\6 \2\4-x86-64\6 \2x32\4-x32\6"_ diff --git a/contrib/guix/patches/glibc-versioned-locpath.patch b/contrib/guix/patches/glibc-versioned-locpath.patch new file mode 100644 index 0000000000..bc7652127f --- /dev/null +++ b/contrib/guix/patches/glibc-versioned-locpath.patch @@ -0,0 +1,240 @@ +The format of locale data can be incompatible between libc versions, and +loading incompatible data can lead to 'setlocale' returning EINVAL at best +or triggering an assertion failure at worst. See +https://lists.gnu.org/archive/html/guix-devel/2015-09/msg00717.html +for background information. + +To address that, this patch changes libc to honor a new 'GUIX_LOCPATH' +variable, and to look for locale data in version-specific sub-directories of +that variable. So, if GUIX_LOCPATH=/foo:/bar, locale data is searched for in +/foo/X.Y and /bar/X.Y, where X.Y is the libc version number. + +That way, a single 'GUIX_LOCPATH' setting can work even if different libc +versions coexist on the system. + +--- a/locale/newlocale.c ++++ b/locale/newlocale.c +@@ -30,6 +30,7 @@ + /* Lock for protecting global data. */ + __libc_rwlock_define (extern , __libc_setlocale_lock attribute_hidden) + ++extern error_t compute_locale_search_path (char **, size_t *); + + /* Use this when we come along an error. */ + #define ERROR_RETURN \ +@@ -48,7 +49,6 @@ __newlocale (int category_mask, const char *locale, __locale_t base) + __locale_t result_ptr; + char *locale_path; + size_t locale_path_len; +- const char *locpath_var; + int cnt; + size_t names_len; + +@@ -102,17 +102,8 @@ __newlocale (int category_mask, const char *locale, __locale_t base) + locale_path = NULL; + locale_path_len = 0; + +- locpath_var = getenv ("LOCPATH"); +- if (locpath_var != NULL && locpath_var[0] != '\0') +- { +- if (__argz_create_sep (locpath_var, ':', +- &locale_path, &locale_path_len) != 0) +- return NULL; +- +- if (__argz_add_sep (&locale_path, &locale_path_len, +- _nl_default_locale_path, ':') != 0) +- return NULL; +- } ++ if (compute_locale_search_path (&locale_path, &locale_path_len) != 0) ++ return NULL; + + /* Get the names for the locales we are interested in. We either + allow a composite name or a single name. */ +diff --git a/locale/setlocale.c b/locale/setlocale.c +index ead030d..0c0e314 100644 +--- a/locale/setlocale.c ++++ b/locale/setlocale.c +@@ -215,12 +215,65 @@ setdata (int category, struct __locale_data *data) + } + } + ++/* Return in *LOCALE_PATH and *LOCALE_PATH_LEN the locale data search path as ++ a colon-separated list. Return ENOMEN on error, zero otherwise. */ ++error_t ++compute_locale_search_path (char **locale_path, size_t *locale_path_len) ++{ ++ char* guix_locpath_var = getenv ("GUIX_LOCPATH"); ++ char *locpath_var = getenv ("LOCPATH"); ++ ++ if (guix_locpath_var != NULL && guix_locpath_var[0] != '\0') ++ { ++ /* Entries in 'GUIX_LOCPATH' take precedence over 'LOCPATH'. These ++ entries are systematically prefixed with "/X.Y" where "X.Y" is the ++ libc version. */ ++ if (__argz_create_sep (guix_locpath_var, ':', ++ locale_path, locale_path_len) != 0 ++ || __argz_suffix_entries (locale_path, locale_path_len, ++ "/" VERSION) != 0) ++ goto bail_out; ++ } ++ ++ if (locpath_var != NULL && locpath_var[0] != '\0') ++ { ++ char *reg_locale_path = NULL; ++ size_t reg_locale_path_len = 0; ++ ++ if (__argz_create_sep (locpath_var, ':', ++ ®_locale_path, ®_locale_path_len) != 0) ++ goto bail_out; ++ ++ if (__argz_append (locale_path, locale_path_len, ++ reg_locale_path, reg_locale_path_len) != 0) ++ goto bail_out; ++ ++ free (reg_locale_path); ++ } ++ ++ if (*locale_path != NULL) ++ { ++ /* Append the system default locale directory. */ ++ if (__argz_add_sep (locale_path, locale_path_len, ++ _nl_default_locale_path, ':') != 0) ++ goto bail_out; ++ } ++ ++ return 0; ++ ++ bail_out: ++ free (*locale_path); ++ *locale_path = NULL; ++ *locale_path_len = 0; ++ ++ return ENOMEM; ++} ++ + char * + setlocale (int category, const char *locale) + { + char *locale_path; + size_t locale_path_len; +- const char *locpath_var; + char *composite; + + /* Sanity check for CATEGORY argument. */ +@@ -251,17 +304,10 @@ setlocale (int category, const char *locale) + locale_path = NULL; + locale_path_len = 0; + +- locpath_var = getenv ("LOCPATH"); +- if (locpath_var != NULL && locpath_var[0] != '\0') ++ if (compute_locale_search_path (&locale_path, &locale_path_len) != 0) + { +- if (__argz_create_sep (locpath_var, ':', +- &locale_path, &locale_path_len) != 0 +- || __argz_add_sep (&locale_path, &locale_path_len, +- _nl_default_locale_path, ':') != 0) +- { +- __libc_rwlock_unlock (__libc_setlocale_lock); +- return NULL; +- } ++ __libc_rwlock_unlock (__libc_setlocale_lock); ++ return NULL; + } + + if (category == LC_ALL) +diff --git a/string/Makefile b/string/Makefile +index 8424a61..f925503 100644 +--- a/string/Makefile ++++ b/string/Makefile +@@ -38,7 +38,7 @@ routines := strcat strchr strcmp strcoll strcpy strcspn \ + swab strfry memfrob memmem rawmemchr strchrnul \ + $(addprefix argz-,append count create ctsep next \ + delete extract insert stringify \ +- addsep replace) \ ++ addsep replace suffix) \ + envz basename \ + strcoll_l strxfrm_l string-inlines memrchr \ + xpg-strerror strerror_l +diff --git a/string/argz-suffix.c b/string/argz-suffix.c +new file mode 100644 +index 0000000..505b0f2 +--- /dev/null ++++ b/string/argz-suffix.c +@@ -0,0 +1,56 @@ ++/* Copyright (C) 2015 Free Software Foundation, Inc. ++ This file is part of the GNU C Library. ++ Contributed by Ludovic Courtès <ludo@gnu.org>. ++ ++ The GNU C Library is free software; you can redistribute it and/or ++ modify it under the terms of the GNU Lesser General Public ++ License as published by the Free Software Foundation; either ++ version 2.1 of the License, or (at your option) any later version. ++ ++ The GNU C Library is distributed in the hope that it will be useful, ++ but WITHOUT ANY WARRANTY; without even the implied warranty of ++ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ++ Lesser General Public License for more details. ++ ++ You should have received a copy of the GNU Lesser General Public ++ License along with the GNU C Library; if not, see ++ <http://www.gnu.org/licenses/>. */ ++ ++#include <argz.h> ++#include <errno.h> ++#include <stdlib.h> ++#include <string.h> ++ ++ ++error_t ++__argz_suffix_entries (char **argz, size_t *argz_len, const char *suffix) ++ ++{ ++ size_t suffix_len = strlen (suffix); ++ size_t count = __argz_count (*argz, *argz_len); ++ size_t new_argz_len = *argz_len + count * suffix_len; ++ char *new_argz = malloc (new_argz_len); ++ ++ if (new_argz) ++ { ++ char *p = new_argz, *entry; ++ ++ for (entry = *argz; ++ entry != NULL; ++ entry = argz_next (*argz, *argz_len, entry)) ++ { ++ p = stpcpy (p, entry); ++ p = stpcpy (p, suffix); ++ p++; ++ } ++ ++ free (*argz); ++ *argz = new_argz; ++ *argz_len = new_argz_len; ++ ++ return 0; ++ } ++ else ++ return ENOMEM; ++} ++weak_alias (__argz_suffix_entries, argz_suffix_entries) +diff --git a/string/argz.h b/string/argz.h +index bb62a31..d276a35 100644 +--- a/string/argz.h ++++ b/string/argz.h +@@ -134,6 +134,16 @@ extern error_t argz_replace (char **__restrict __argz, + const char *__restrict __str, + const char *__restrict __with, + unsigned int *__restrict __replace_count); ++ ++/* Suffix each entry of ARGZ & ARGZ_LEN with SUFFIX. Return 0 on success, ++ and ENOMEN if memory cannot be allocated. */ ++extern error_t __argz_suffix_entries (char **__restrict __argz, ++ size_t *__restrict __argz_len, ++ const char *__restrict __suffix); ++extern error_t argz_suffix_entries (char **__restrict __argz, ++ size_t *__restrict __argz_len, ++ const char *__restrict __suffix); ++ + + /* Returns the next entry in ARGZ & ARGZ_LEN after ENTRY, or NULL if there + are no more. If entry is NULL, then the first entry is returned. This diff --git a/contrib/guix/patches/nsis-SConstruct-sde-support.patch b/contrib/guix/patches/nsis-SConstruct-sde-support.patch index 5edf1b7c8e..f58406a7a0 100644 --- a/contrib/guix/patches/nsis-SConstruct-sde-support.patch +++ b/contrib/guix/patches/nsis-SConstruct-sde-support.patch @@ -1,3 +1,6 @@ +https://github.com/kichik/nsis/pull/13 +https://sourceforge.net/p/nsis/code/7248/ + diff --git a/SConstruct b/SConstruct index e8252c9..41786f2 100755 --- a/SConstruct diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh index 4037936404..dd4d862dee 100755 --- a/contrib/install_db4.sh +++ b/contrib/install_db4.sh @@ -221,10 +221,10 @@ EOF # The packaged config.guess and config.sub are ancient (2009) and can cause build issues. # Replace them with modern versions. # See https://github.com/bitcoin/bitcoin/issues/16064 -CONFIG_GUESS_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=55eaf3e779455c4e5cc9f82efb5278be8f8f900b' -CONFIG_GUESS_HASH='2d1ff7bca773d2ec3c6217118129220fa72d8adda67c7d2bf79994b3129232c1' -CONFIG_SUB_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=55eaf3e779455c4e5cc9f82efb5278be8f8f900b' -CONFIG_SUB_HASH='3a4befde9bcdf0fdb2763fc1bfa74e8696df94e1ad7aac8042d133c8ff1d2e32' +CONFIG_GUESS_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=4550d2f15b3a7ce2451c1f29500b9339430c877f' +CONFIG_GUESS_HASH='c8f530e01840719871748a8071113435bdfdf75b74c57e78e47898edea8754ae' +CONFIG_SUB_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=4550d2f15b3a7ce2451c1f29500b9339430c877f' +CONFIG_SUB_HASH='3969f7d5f6967ccc6f792401b8ef3916a1d1b1d0f0de5a4e354c95addb8b800e' rm -f "dist/config.guess" rm -f "dist/config.sub" diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index 73f54cd488..9a8bcc57a5 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -17,7 +17,6 @@ import datetime import time import glob from collections import namedtuple -from binascii import unhexlify settings = {} @@ -332,7 +331,7 @@ if __name__ == '__main__': settings['max_out_sz'] = int(settings['max_out_sz']) settings['split_timestamp'] = int(settings['split_timestamp']) settings['file_timestamp'] = int(settings['file_timestamp']) - settings['netmagic'] = unhexlify(settings['netmagic'].encode('utf-8')) + settings['netmagic'] = bytes.fromhex(settings['netmagic']) settings['out_of_order_cache_sz'] = int(settings['out_of_order_cache_sz']) settings['debug_output'] = settings['debug_output'].lower() diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 21f6ba2eb3..a685aac1c0 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -16,7 +16,10 @@ Our current macOS SDK (`Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz`) can be extracted from [Xcode_12.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.1/Xcode_12.1.xip). -An Apple ID is needed to download this. +Alternatively, after logging in to your account go to 'Downloads', then 'More' +and look for [`Xcode_12.1`](https://download.developer.apple.com/Developer_Tools/Xcode_12.1/Xcode_12.1.xip). +An Apple ID and cookies enabled for the hostname are needed to download this. +The `sha256sum` of the archive should be `612443b1894b39368a596ea1607f30cbb0481ad44d5e29c75edb71a6d2cf050f`. After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip` archive. This makes the SDK less-trivial to extract on non-macOS machines. One @@ -76,7 +79,7 @@ and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done To complicate things further, all builds must target an Apple SDK. These SDKs are free to download, but not redistributable. To obtain it, register for an Apple Developer Account, -then download [Xcode_11.3.1](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip). +then download [Xcode_12.1](https://download.developer.apple.com/Developer_Tools/Xcode_12.1/Xcode_12.1.xip). This file is many gigabytes in size, but most (but not all) of what we need is contained only in a single directory: @@ -87,9 +90,9 @@ Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk See the SDK Extraction notes above for how to obtain it. -The Gitian descriptors build 2 sets of files: Linux tools, then Apple binaries which are +The Guix process build 2 sets of files: Linux tools, then Apple binaries which are created using these tools. The build process has been designed to avoid including the -SDK's files in Gitian's outputs. All interim tarballs are fully deterministic and may be freely +SDK's files in Guix's outputs. All interim tarballs are fully deterministic and may be freely redistributed. [`xorrisofs`](https://www.gnu.org/software/xorriso/) is used to create the DMG. @@ -110,11 +113,11 @@ order to satisfy the new Gatekeeper requirements. Because this private key canno shared, we'll have to be a bit creative in order for the build process to remain somewhat deterministic. Here's how it works: -- Builders use Gitian to create an unsigned release. This outputs an unsigned DMG which +- Builders use Guix to create an unsigned release. This outputs an unsigned DMG which users may choose to bless and run. It also outputs an unsigned app structure in the form of a tarball, which also contains all of the tools that have been previously (deterministically) built in order to create a final DMG. - The Apple keyholder uses this unsigned app to create a detached signature, using the script that is also included there. Detached signatures are available from this [repository](https://github.com/bitcoin-core/bitcoin-detached-sigs). -- Builders feed the unsigned app + detached signature back into Gitian. It uses the +- Builders feed the unsigned app + detached signature back into Guix. It uses the pre-built tools to recombine the pieces into a deterministic DMG. diff --git a/contrib/macdeploy/macdeployqtplus b/contrib/macdeploy/macdeployqtplus index 9bf3305288..055a932eee 100755 --- a/contrib/macdeploy/macdeployqtplus +++ b/contrib/macdeploy/macdeployqtplus @@ -16,7 +16,6 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. # -import plistlib import sys, re, os, shutil, stat, os.path from argparse import ArgumentParser from ds_store import DSStore @@ -53,7 +52,7 @@ class FrameworkInfo(object): return False def __str__(self): - return f""" Framework name: {frameworkName} + return f""" Framework name: {self.frameworkName} Framework directory: {self.frameworkDirectory} Framework path: {self.frameworkPath} Binary name: {self.binaryName} @@ -85,8 +84,8 @@ class FrameworkInfo(object): if line == "": return None - # Don't deploy system libraries (exception for libQtuitools and libQtlucene). - if line.startswith("/System/Library/") or line.startswith("@executable_path") or (line.startswith("/usr/lib/") and "libQt" not in line): + # Don't deploy system libraries + if line.startswith("/System/Library/") or line.startswith("@executable_path") or line.startswith("/usr/lib/"): return None m = cls.reOLine.match(line) @@ -287,14 +286,6 @@ def copyFramework(framework: FrameworkInfo, path: str, verbose: int) -> Optional if verbose: print("Copied Contents:", fromContentsDir) print(" to:", toContentsDir) - elif framework.frameworkName.startswith("libQtGui"): # Copy qt_menu.nib (applies to non-framework layout) - qtMenuNibSourcePath = os.path.join(framework.frameworkDirectory, "Resources", "qt_menu.nib") - qtMenuNibDestinationPath = os.path.join(path, "Contents", "Resources", "qt_menu.nib") - if os.path.exists(qtMenuNibSourcePath) and not os.path.exists(qtMenuNibDestinationPath): - shutil.copytree(qtMenuNibSourcePath, qtMenuNibDestinationPath, symlinks=True) - if verbose: - print("Copied for libQtGui:", qtMenuNibSourcePath) - print(" to:", qtMenuNibDestinationPath) return toPath @@ -351,115 +342,20 @@ def deployFrameworksForAppBundle(applicationBundle: ApplicationBundleInfo, strip return deployFrameworks(frameworks, applicationBundle.path, applicationBundle.binaryPath, strip, verbose) def deployPlugins(appBundleInfo: ApplicationBundleInfo, deploymentInfo: DeploymentInfo, strip: bool, verbose: int): - # Lookup available plugins, exclude unneeded plugins = [] if deploymentInfo.pluginPath is None: return for dirpath, dirnames, filenames in os.walk(deploymentInfo.pluginPath): pluginDirectory = os.path.relpath(dirpath, deploymentInfo.pluginPath) - if pluginDirectory == "designer": - # Skip designer plugins - continue - elif pluginDirectory == "printsupport": - # Skip printsupport plugins - continue - elif pluginDirectory == "imageformats": - # Skip imageformats plugins + + if pluginDirectory not in ['styles', 'platforms']: continue - elif pluginDirectory == "sqldrivers": - # Deploy the sql plugins only if QtSql is in use - if not deploymentInfo.usesFramework("QtSql"): - continue - elif pluginDirectory == "script": - # Deploy the script plugins only if QtScript is in use - if not deploymentInfo.usesFramework("QtScript"): - continue - elif pluginDirectory == "qmltooling" or pluginDirectory == "qml1tooling": - # Deploy the qml plugins only if QtDeclarative is in use - if not deploymentInfo.usesFramework("QtDeclarative"): - continue - elif pluginDirectory == "bearer": - # Deploy the bearer plugins only if QtNetwork is in use - if not deploymentInfo.usesFramework("QtNetwork"): - continue - elif pluginDirectory == "position": - # Deploy the position plugins only if QtPositioning is in use - if not deploymentInfo.usesFramework("QtPositioning"): - continue - elif pluginDirectory == "sensors" or pluginDirectory == "sensorgestures": - # Deploy the sensor plugins only if QtSensors is in use - if not deploymentInfo.usesFramework("QtSensors"): - continue - elif pluginDirectory == "audio" or pluginDirectory == "playlistformats": - # Deploy the audio plugins only if QtMultimedia is in use - if not deploymentInfo.usesFramework("QtMultimedia"): - continue - elif pluginDirectory == "mediaservice": - # Deploy the mediaservice plugins only if QtMultimediaWidgets is in use - if not deploymentInfo.usesFramework("QtMultimediaWidgets"): - continue - elif pluginDirectory == "canbus": - # Deploy the canbus plugins only if QtSerialBus is in use - if not deploymentInfo.usesFramework("QtSerialBus"): - continue - elif pluginDirectory == "webview": - # Deploy the webview plugins only if QtWebView is in use - if not deploymentInfo.usesFramework("QtWebView"): - continue - elif pluginDirectory == "gamepads": - # Deploy the webview plugins only if QtGamepad is in use - if not deploymentInfo.usesFramework("QtGamepad"): - continue - elif pluginDirectory == "geoservices": - # Deploy the webview plugins only if QtLocation is in use - if not deploymentInfo.usesFramework("QtLocation"): - continue - elif pluginDirectory == "texttospeech": - # Deploy the texttospeech plugins only if QtTextToSpeech is in use - if not deploymentInfo.usesFramework("QtTextToSpeech"): - continue - elif pluginDirectory == "virtualkeyboard": - # Deploy the virtualkeyboard plugins only if QtVirtualKeyboard is in use - if not deploymentInfo.usesFramework("QtVirtualKeyboard"): - continue - elif pluginDirectory == "sceneparsers": - # Deploy the virtualkeyboard plugins only if Qt3DCore is in use - if not deploymentInfo.usesFramework("Qt3DCore"): - continue - elif pluginDirectory == "renderplugins": - # Deploy the renderplugins plugins only if Qt3DCore is in use - if not deploymentInfo.usesFramework("Qt3DCore"): - continue - elif pluginDirectory == "geometryloaders": - # Deploy the geometryloaders plugins only if Qt3DCore is in use - if not deploymentInfo.usesFramework("Qt3DCore"): - continue for pluginName in filenames: pluginPath = os.path.join(pluginDirectory, pluginName) - if pluginName.endswith("_debug.dylib"): - # Skip debug plugins + + if pluginName.split('.')[0] not in ['libqminimal', 'libqcocoa', 'libqmacstyle']: continue - elif pluginPath == "imageformats/libqsvg.dylib" or pluginPath == "iconengines/libqsvgicon.dylib": - # Deploy the svg plugins only if QtSvg is in use - if not deploymentInfo.usesFramework("QtSvg"): - continue - elif pluginPath == "accessible/libqtaccessiblecompatwidgets.dylib": - # Deploy accessibility for Qt3Support only if the Qt3Support is in use - if not deploymentInfo.usesFramework("Qt3Support"): - continue - elif pluginPath == "graphicssystems/libqglgraphicssystem.dylib": - # Deploy the opengl graphicssystem plugin only if QtOpenGL is in use - if not deploymentInfo.usesFramework("QtOpenGL"): - continue - elif pluginPath == "accessible/libqtaccessiblequick.dylib": - # Deploy the accessible qtquick plugin only if QtQuick is in use - if not deploymentInfo.usesFramework("QtQuick"): - continue - elif pluginPath == "platforminputcontexts/libqtvirtualkeyboardplugin.dylib": - # Deploy the virtualkeyboardplugin plugin only if QtVirtualKeyboard is in use - if not deploymentInfo.usesFramework("QtVirtualKeyboard"): - continue plugins.append((pluginDirectory, pluginName)) @@ -527,6 +423,9 @@ if os.path.exists(appname + ".dmg"): print("+ Removing existing DMG +") os.unlink(appname + ".dmg") +if os.path.exists(appname + ".temp.dmg"): + os.unlink(appname + ".temp.dmg") + # ------------------------------------------------ target = os.path.join("dist", "Bitcoin-Qt.app") diff --git a/contrib/seeds/nodes_main.txt b/contrib/seeds/nodes_main.txt index a62150a930..b9dfdb4b0a 100644 --- a/contrib/seeds/nodes_main.txt +++ b/contrib/seeds/nodes_main.txt @@ -677,12 +677,20 @@ tddeij4qigtjr6jfnrmq6btnirmq5msgwcsdpcdjr7atftm7cxlqztid.onion:8333 vi5bnbxkleeqi6hfccjochnn65lcxlfqs4uwgmhudph554zibiusqnad.onion:8333 xqt25cobm5zqucac3634zfght72he6u3eagfyej5ellbhcdgos7t2had.onion:8333 -# manually added 2021-05 for minimal i2p bootstrap support -72l3ucjkuscrbiiepoehuwqgknyzgo7zuix5ty4puwrkyhtmnsga.b32.i2p:8333 -c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p:8333 -gehtac45oaghz54ypyopim64mql7oad2bqclla74l6tfeolzmodq.b32.i2p:8333 -h3r6bkn46qxftwja53pxiykntegfyfjqtnzbm6iv6r5mungmqgmq.b32.i2p:8333 -hnbbyjpxx54623l555sta7pocy3se4sdgmuebi5k6reesz5rjp6q.b32.i2p:8333 -pjs7or2ctvteeo5tu4bwyrtydeuhqhvdprtujn4daxr75jpebjxa.b32.i2p:8333 -wwbw7nqr3ahkqv62cuqfwgtneekvvpnuc4i4f6yo7tpoqjswvcwa.b32.i2p:8333 -zsxwyo6qcn3chqzwxnseusqgsnuw3maqnztkiypyfxtya4snkoka.b32.i2p:8333 +# manually added 2021-08 for minimal i2p bootstrap support +4hllr6w55mbtemb3ebvlzl4zj6qke4si7zcob5qdyg63mjgq624a.b32.i2p:0 +6s33jtpvwzkiej3nff5qm72slgqljxhxn62hdt6m7nvynqsxqdda.b32.i2p:0 +a5qsnv3maw77mlmmzlcglu6twje6ttctd3fhpbfwcbpmewx6fczq.b32.i2p:0 +bitcornrd36coazsbzsz4pdebyzvaplmsalq4kpoljmn6cg6x5zq.b32.i2p:0 +c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p:0 +dhtq2p76tyhi442aidb3vd2bv7yxxjuddpb2jydnnrl2ons5bhha.b32.i2p:0 +gehtac45oaghz54ypyopim64mql7oad2bqclla74l6tfeolzmodq.b32.i2p:0 +h3r6bkn46qxftwja53pxiykntegfyfjqtnzbm6iv6r5mungmqgmq.b32.i2p:0 +hnbbyjpxx54623l555sta7pocy3se4sdgmuebi5k6reesz5rjp6q.b32.i2p:0 +i3hcdakiz2tyvggkwefvdjoi7444kgvd2mbdfizjvv43q7zukezq.b32.i2p:0 +jz3s4eurm5vzjresf4mwo7oni4bk36daolwxh4iqtewakylgkxmq.b32.i2p:0 +kokkmpquqlkptu5hkmzqlttsmtwxicldr4so7wqsufk6bwf32nma.b32.i2p:0 +kvrde7mcgjhz3xzeltwy4gs2rxdfbnbs2wc67mh2pt43wjmjnmbq.b32.i2p:0 +shh2ewyegnuwnmdse5kl5toybdvzkvk2yj4zcowz6iwhhh3ykdfa.b32.i2p:0 +wwbw7nqr3ahkqv62cuqfwgtneekvvpnuc4i4f6yo7tpoqjswvcwa.b32.i2p:0 +zsxwyo6qcn3chqzwxnseusqgsnuw3maqnztkiypyfxtya4snkoka.b32.i2p:0 diff --git a/contrib/signet/getcoins.py b/contrib/signet/getcoins.py index 691f0bb1b6..3d0aa5d132 100755 --- a/contrib/signet/getcoins.py +++ b/contrib/signet/getcoins.py @@ -4,33 +4,153 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. import argparse -import subprocess +import io import requests +import subprocess import sys +DEFAULT_GLOBAL_FAUCET = 'https://signetfaucet.com/claim' +DEFAULT_GLOBAL_CAPTCHA = 'https://signetfaucet.com/captcha' +GLOBAL_FIRST_BLOCK_HASH = '00000086d6b2636cb2a392d45edc4ec544a10024d30141c9adf4bfd9de533b53' + +# braille unicode block +BASE = 0x2800 +BIT_PER_PIXEL = [ + [0x01, 0x08], + [0x02, 0x10], + [0x04, 0x20], + [0x40, 0x80], +] +BW = 2 +BH = 4 + +# imagemagick or compatible fork (used for converting SVG) +CONVERT = 'convert' + +class PPMImage: + ''' + Load a PPM image (Pillow-ish API). + ''' + def __init__(self, f): + if f.readline() != b'P6\n': + raise ValueError('Invalid ppm format: header') + line = f.readline() + (width, height) = (int(x) for x in line.rstrip().split(b' ')) + if f.readline() != b'255\n': + raise ValueError('Invalid ppm format: color depth') + data = f.read(width * height * 3) + stride = width * 3 + self.size = (width, height) + self._grid = [[tuple(data[stride * y + 3 * x:stride * y + 3 * (x + 1)]) for x in range(width)] for y in range(height)] + + def getpixel(self, pos): + return self._grid[pos[1]][pos[0]] + +def print_image(img, threshold=128): + '''Print black-and-white image to terminal in braille unicode characters.''' + x_blocks = (img.size[0] + BW - 1) // BW + y_blocks = (img.size[1] + BH - 1) // BH + + for yb in range(y_blocks): + line = [] + for xb in range(x_blocks): + ch = BASE + for y in range(BH): + for x in range(BW): + try: + val = img.getpixel((xb * BW + x, yb * BH + y)) + except IndexError: + pass + else: + if val[0] < threshold: + ch |= BIT_PER_PIXEL[y][x] + line.append(chr(ch)) + print(''.join(line)) + parser = argparse.ArgumentParser(description='Script to get coins from a faucet.', epilog='You may need to start with double-dash (--) when providing bitcoin-cli arguments.') parser.add_argument('-c', '--cmd', dest='cmd', default='bitcoin-cli', help='bitcoin-cli command to use') -parser.add_argument('-f', '--faucet', dest='faucet', default='https://signetfaucet.com/claim', help='URL of the faucet') +parser.add_argument('-f', '--faucet', dest='faucet', default=DEFAULT_GLOBAL_FAUCET, help='URL of the faucet') +parser.add_argument('-g', '--captcha', dest='captcha', default=DEFAULT_GLOBAL_CAPTCHA, help='URL of the faucet captcha, or empty if no captcha is needed') parser.add_argument('-a', '--addr', dest='addr', default='', help='Bitcoin address to which the faucet should send') parser.add_argument('-p', '--password', dest='password', default='', help='Faucet password, if any') +parser.add_argument('-n', '--amount', dest='amount', default='0.001', help='Amount to request (0.001-0.1, default is 0.001)') +parser.add_argument('-i', '--imagemagick', dest='imagemagick', default=CONVERT, help='Path to imagemagick convert utility') parser.add_argument('bitcoin_cli_args', nargs='*', help='Arguments to pass on to bitcoin-cli (default: -signet)') args = parser.parse_args() +if args.bitcoin_cli_args == []: + args.bitcoin_cli_args = ['-signet'] + + +def bitcoin_cli(rpc_command_and_params): + argv = [args.cmd] + args.bitcoin_cli_args + rpc_command_and_params + try: + return subprocess.check_output(argv).strip().decode() + except FileNotFoundError: + print('The binary', args.cmd, 'could not be found.') + exit(1) + except subprocess.CalledProcessError: + cmdline = ' '.join(argv) + print(f'-----\nError while calling "{cmdline}" (see output above).') + exit(1) + + +if args.faucet.lower() == DEFAULT_GLOBAL_FAUCET: + # Get the hash of the block at height 1 of the currently active signet chain + curr_signet_hash = bitcoin_cli(['getblockhash', '1']) + if curr_signet_hash != GLOBAL_FIRST_BLOCK_HASH: + print('The global faucet cannot be used with a custom Signet network. Please use the global signet or setup your custom faucet to use this functionality.\n') + exit(1) +else: + # For custom faucets, don't request captcha by default. + if args.captcha == DEFAULT_GLOBAL_CAPTCHA: + args.captcha = '' + if args.addr == '': - if args.bitcoin_cli_args == []: - args.bitcoin_cli_args = ['-signet'] # get address for receiving coins + args.addr = bitcoin_cli(['getnewaddress', 'faucet', 'bech32']) + +data = {'address': args.addr, 'password': args.password, 'amount': args.amount} + +# Store cookies +# for debugging: print(session.cookies.get_dict()) +session = requests.Session() + +if args.captcha != '': # Retrieve a captcha + try: + res = session.get(args.captcha) + except: + print('Unexpected error when contacting faucet:', sys.exc_info()[0]) + exit(1) + + # Convert SVG image to PPM, and load it try: - args.addr = subprocess.check_output([args.cmd] + args.bitcoin_cli_args + ['getnewaddress', 'faucet', 'bech32']).strip() + rv = subprocess.run([args.imagemagick, '-', '-depth', '8', 'ppm:-'], input=res.content, check=True, capture_output=True) except FileNotFoundError: - print('The binary', args.cmd, 'could not be found.') - exit() + print('The binary', args.imagemagick, 'could not be found. Please make sure ImageMagick (or a compatible fork) is installed and that the correct path is specified.') + exit(1) + img = PPMImage(io.BytesIO(rv.stdout)) + + # Terminal interaction + print_image(img) + print('Enter captcha: ', end='') + data['captcha'] = input() -data = {'address': args.addr, 'password': args.password} try: - res = requests.post(args.faucet, data=data) + res = session.post(args.faucet, data=data) except: print('Unexpected error when contacting faucet:', sys.exc_info()[0]) - exit() -print(res.text) + exit(1) + +# Display the output as per the returned status code +if res: + # When the return code is in between 200 and 400 i.e. successful + print(res.text) +elif res.status_code == 404: + print('The specified faucet URL does not exist. Please check for any server issues/typo.') +elif res.status_code == 429: + print('The script does not allow for repeated transactions as the global faucet is rate-limitied to 1 request/IP/day. You can access the faucet website to get more coins manually') +else: + print(f'Returned Error Code {res.status_code}\n{res.text}\n') + print('Please check the provided arguments for their validity and/or any possible typo.') diff --git a/contrib/signet/miner b/contrib/signet/miner index 78e1fa5ecd..012bd6cc31 100755 --- a/contrib/signet/miner +++ b/contrib/signet/miner @@ -15,7 +15,6 @@ import sys import time import subprocess -from binascii import unhexlify from io import BytesIO PATH_BASE_CONTRIB_SIGNET = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) @@ -202,7 +201,7 @@ def finish_block(block, signet_solution, grind_cmd): def generate_psbt(tmpl, reward_spk, *, blocktime=None): signet_spk = tmpl["signet_challenge"] - signet_spk_bin = unhexlify(signet_spk) + signet_spk_bin = bytes.fromhex(signet_spk) cbtx = create_coinbase(height=tmpl["height"], value=tmpl["coinbasevalue"], spk=reward_spk) cbtx.vin[0].nSequence = 2**32-2 @@ -258,7 +257,7 @@ def get_reward_addr_spk(args, height): return args.address, args.reward_spk reward_addr = get_reward_address(args, height) - reward_spk = unhexlify(json.loads(args.bcli("getaddressinfo", reward_addr))["scriptPubKey"]) + reward_spk = bytes.fromhex(json.loads(args.bcli("getaddressinfo", reward_addr))["scriptPubKey"]) if args.address is not None: # will always be the same, so cache args.reward_spk = reward_spk diff --git a/contrib/tracing/README.md b/contrib/tracing/README.md new file mode 100644 index 0000000000..047354cda1 --- /dev/null +++ b/contrib/tracing/README.md @@ -0,0 +1,241 @@ +Example scripts for User-space, Statically Defined Tracing (USDT) +================================================================= + +This directory contains scripts showcasing User-space, Statically Defined +Tracing (USDT) support for Bitcoin Core on Linux using. For more information on +USDT support in Bitcoin Core see the [USDT documentation]. + +[USDT documentation]: ../../doc/tracing.md + + +Examples for the two main eBPF front-ends, [bpftrace] and +[BPF Compiler Collection (BCC)], with support for USDT, are listed. BCC is used +for complex tools and daemons and `bpftrace` is preferred for one-liners and +shorter scripts. + +[bpftrace]: https://github.com/iovisor/bpftrace +[BPF Compiler Collection (BCC)]: https://github.com/iovisor/bcc + + +To develop and run bpftrace and BCC scripts you need to install the +corresponding packages. See [installing bpftrace] and [installing BCC] for more +information. For development there exist a [bpftrace Reference Guide], a +[BCC Reference Guide], and a [bcc Python Developer Tutorial]. + +[installing bpftrace]: https://github.com/iovisor/bpftrace/blob/master/INSTALL.md +[installing BCC]: https://github.com/iovisor/bcc/blob/master/INSTALL.md +[bpftrace Reference Guide]: https://github.com/iovisor/bpftrace/blob/master/docs/reference_guide.md +[BCC Reference Guide]: https://github.com/iovisor/bcc/blob/master/docs/reference_guide.md +[bcc Python Developer Tutorial]: https://github.com/iovisor/bcc/blob/master/docs/tutorial_bcc_python_developer.md + +## Examples + +The bpftrace examples contain a relative path to the `bitcoind` binary. By +default, the scripts should be run from the repository-root and assume a +self-compiled `bitcoind` binary. The paths in the examples can be changed, for +example, to point to release builds if needed. See the +[Bitcoin Core USDT documentation] on how to list available tracepoints in your +`bitcoind` binary. + +[Bitcoin Core USDT documentation]: ../../doc/tracing.md#listing-available-tracepoints + +**WARNING: eBPF programs require root privileges to be loaded into a Linux +kernel VM. This means the bpftrace and BCC examples must be executed with root +privileges. Make sure to carefully review any scripts that you run with root +privileges first!** + +### log_p2p_traffic.bt + +A bpftrace script logging information about inbound and outbound P2P network +messages. Based on the `net:inbound_message` and `net:outbound_message` +tracepoints. + +By default, `bpftrace` limits strings to 64 bytes due to the limited stack size +in the eBPF VM. For example, Tor v3 addresses exceed the string size limit which +results in the port being cut off during logging. The string size limit can be +increased with the `BPFTRACE_STRLEN` environment variable (`BPFTRACE_STRLEN=70` +works fine). + +``` +$ bpftrace contrib/tracing/log_p2p_traffic.bt +``` + +Output +``` +outbound 'ping' msg to peer 11 (outbound-full-relay, [2a02:b10c:f747:1:ef:fake:ipv6:addr]:8333) with 8 bytes +inbound 'pong' msg from peer 11 (outbound-full-relay, [2a02:b10c:f747:1:ef:fake:ipv6:addr]:8333) with 8 bytes +inbound 'inv' msg from peer 16 (outbound-full-relay, XX.XX.XXX.121:8333) with 37 bytes +outbound 'getdata' msg to peer 16 (outbound-full-relay, XX.XX.XXX.121:8333) with 37 bytes +inbound 'tx' msg from peer 16 (outbound-full-relay, XX.XX.XXX.121:8333) with 222 bytes +outbound 'inv' msg to peer 9 (outbound-full-relay, faketorv3addressa2ufa6odvoi3s77j4uegey0xb10csyfyve2t33curbyd.onion:8333) with 37 bytes +outbound 'inv' msg to peer 7 (outbound-full-relay, XX.XX.XXX.242:8333) with 37 bytes +… +``` + +### p2p_monitor.py + +A BCC Python script using curses for an interactive P2P message monitor. Based +on the `net:inbound_message` and `net:outbound_message` tracepoints. + +Inbound and outbound traffic is listed for each peer together with information +about the connection. Peers can be selected individually to view recent P2P +messages. + +``` +$ python3 contrib/tracing/p2p_monitor.py ./src/bitcoind +``` + +Lists selectable peers and traffic and connection information. +``` + P2P Message Monitor + Navigate with UP/DOWN or J/K and select a peer with ENTER or SPACE to see individual P2P messages + + PEER OUTBOUND INBOUND TYPE ADDR + 0 46 398 byte 61 1407590 byte block-relay-only XX.XX.XXX.196:8333 + 11 1156 253570 byte 3431 2394924 byte outbound-full-relay XXX.X.XX.179:8333 + 13 3425 1809620 byte 1236 305458 byte inbound XXX.X.X.X:60380 + 16 1046 241633 byte 1589 1199220 byte outbound-full-relay 4faketorv2pbfu7x.onion:8333 + 19 577 181679 byte 390 148951 byte outbound-full-relay kfake4vctorjv2o2.onion:8333 + 20 11 1248 byte 13 1283 byte block-relay-only [2600:fake:64d9:b10c:4436:aaaa:fe:bb]:8333 + 21 11 1248 byte 13 1299 byte block-relay-only XX.XXX.X.155:8333 + 22 5 103 byte 1 102 byte feeler XX.XX.XXX.173:8333 + 23 11 1248 byte 12 1255 byte block-relay-only XX.XXX.XXX.220:8333 + 24 3 103 byte 1 102 byte feeler XXX.XXX.XXX.64:8333 +… +``` + +Showing recent P2P messages between our node and a selected peer. + +``` + ---------------------------------------------------------------------- + | PEER 16 (4faketorv2pbfu7x.onion:8333) | + | OUR NODE outbound-full-relay PEER | + | <--- sendcmpct (9 bytes) | + | inv (37 byte) ---> | + | <--- ping (8 bytes) | + | pong (8 byte) ---> | + | inv (37 byte) ---> | + | <--- addr (31 bytes) | + | inv (37 byte) ---> | + | <--- getheaders (1029 bytes) | + | headers (1 byte) ---> | + | <--- feefilter (8 bytes) | + | <--- pong (8 bytes) | + | <--- headers (82 bytes) | + | <--- addr (30003 bytes) | + | inv (1261 byte) ---> | + | … | + +``` + +### log_raw_p2p_msgs.py + +A BCC Python script showcasing eBPF and USDT limitations when passing data +larger than about 32kb. Based on the `net:inbound_message` and +`net:outbound_message` tracepoints. + +Bitcoin P2P messages can be larger than 32kb (e.g. `tx`, `block`, ...). The +eBPF VM's stack is limited to 512 bytes, and we can't allocate more than about +32kb for a P2P message in the eBPF VM. The **message data is cut off** when the +message is larger than MAX_MSG_DATA_LENGTH (see script). This can be detected +in user-space by comparing the data length to the message length variable. The +message is cut off when the data length is smaller than the message length. +A warning is included with the printed message data. + +Data is submitted to user-space (i.e. to this script) via a ring buffer. The +throughput of the ring buffer is limited. Each p2p_message is about 32kb in +size. In- or outbound messages submitted to the ring buffer in rapid +succession fill the ring buffer faster than it can be read. Some messages are +lost. BCC prints: `Possibly lost 2 samples` on lost messages. + + +``` +$ python3 contrib/tracing/log_raw_p2p_msgs.py ./src/bitcoind +``` + +``` +Logging raw P2P messages. +Messages larger that about 32kb will be cut off! +Some messages might be lost! + outbound msg 'inv' from peer 4 (outbound-full-relay, XX.XXX.XX.4:8333) with 253 bytes: 0705000000be2245c8f844c9f763748e1a7… +… +Warning: incomplete message (only 32568 out of 53552 bytes)! inbound msg 'tx' from peer 32 (outbound-full-relay, XX.XXX.XXX.43:8333) with 53552 bytes: 020000000001fd3c01939c85ad6756ed9fc… +… +Possibly lost 2 samples +``` + +### connectblock_benchmark.bt + +A `bpftrace` script to benchmark the `ConnectBlock()` function during, for +example, a blockchain re-index. Based on the `validation:block_connected` USDT +tracepoint. + +The script takes three positional arguments. The first two arguments, the start, +and end height indicate between which blocks the benchmark should be run. The +third acts as a duration threshold in milliseconds. When the `ConnectBlock()` +function takes longer than the threshold, information about the block, is +printed. For more details, see the header comment in the script. + +By default, `bpftrace` limits strings to 64 bytes due to the limited stack size +in the kernel VM. Block hashes as zero-terminated hex strings are 65 bytes which +exceed the string limit. The string size limit can be set to 65 bytes with the +environment variable `BPFTRACE_STRLEN`. + +The following command can be used to benchmark, for example, `ConnectBlock()` +between height 20000 and 38000 on SigNet while logging all blocks that take +longer than 25ms to connect. + +``` +$ BPFTRACE_STRLEN=65 bpftrace contrib/tracing/connectblock_benchmark.bt 20000 38000 25 +``` + +In a different terminal, starting Bitcoin Core in SigNet mode and with +re-indexing enabled. + +``` +$ ./src/bitcoind -signet -reindex +``` + +This produces the following output. +``` +Attaching 5 probes... +ConnectBlock Benchmark between height 20000 and 38000 inclusive +Logging blocks taking longer than 25 ms to connect. +Starting Connect Block Benchmark between height 20000 and 38000. +BENCH 39 blk/s 59 tx/s 59 inputs/s 20 sigops/s (height 20038) +Block 20492 (000000f555653bb05e2f3c6e79925e01a20dd57033f4dc7c354b46e34735d32b) 20 tx 2319 ins 2318 sigops took 38 ms +BENCH 1840 blk/s 2117 tx/s 4478 inputs/s 2471 sigops/s (height 21879) +BENCH 1816 blk/s 4972 tx/s 4982 inputs/s 125 sigops/s (height 23695) +BENCH 2095 blk/s 2890 tx/s 2910 inputs/s 152 sigops/s (height 25790) +BENCH 1684 blk/s 3979 tx/s 4053 inputs/s 288 sigops/s (height 27474) +BENCH 1155 blk/s 3216 tx/s 3252 inputs/s 115 sigops/s (height 28629) +BENCH 1797 blk/s 2488 tx/s 2503 inputs/s 111 sigops/s (height 30426) +BENCH 1849 blk/s 6318 tx/s 6569 inputs/s 12189 sigops/s (height 32275) +BENCH 946 blk/s 20209 tx/s 20775 inputs/s 83809 sigops/s (height 33221) +Block 33406 (0000002adfe4a15cfcd53bd890a89bbae836e5bb7f38bac566f61ad4548c87f6) 25 tx 2045 ins 2090 sigops took 29 ms +Block 33687 (00000073231307a9828e5607ceb8156b402efe56747271a4442e75eb5b77cd36) 52 tx 1797 ins 1826 sigops took 26 ms +BENCH 582 blk/s 21581 tx/s 27673 inputs/s 60345 sigops/s (height 33803) +BENCH 1035 blk/s 19735 tx/s 19776 inputs/s 51355 sigops/s (height 34838) +Block 35625 (0000006b00b347390c4768ea9df2655e9ff4b120f29d78594a2a702f8a02c997) 20 tx 3374 ins 3371 sigops took 49 ms +BENCH 887 blk/s 17857 tx/s 22191 inputs/s 24404 sigops/s (height 35725) +Block 35937 (000000d816d13d6e39b471cd4368db60463a764ba1f29168606b04a22b81ea57) 75 tx 3943 ins 3940 sigops took 61 ms +BENCH 823 blk/s 16298 tx/s 21031 inputs/s 18440 sigops/s (height 36548) +Block 36583 (000000c3e260556dbf42968aae3f904dba8b8c1ff96a6f6e3aa5365d2e3ad317) 24 tx 2198 ins 2194 sigops took 34 ms +Block 36700 (000000b3b173de9e65a3cfa738d976af6347aaf83fa17ab3f2a4d2ede3ddfac4) 73 tx 1615 ins 1611 sigops took 31 ms +Block 36832 (0000007859578c02c1ac37dabd1b9ec19b98f350b56935f5dd3a41e9f79f836e) 34 tx 1440 ins 1436 sigops took 26 ms +BENCH 613 blk/s 16718 tx/s 25074 inputs/s 23022 sigops/s (height 37161) +Block 37870 (000000f5c1086291ba2d943fb0c3bc82e71c5ee341ee117681d1456fbf6c6c38) 25 tx 1517 ins 1514 sigops took 29 ms +BENCH 811 blk/s 16031 tx/s 20921 inputs/s 18696 sigops/s (height 37972) + +Took 14055 ms to connect the blocks between height 20000 and 38000. + +Histogram of block connection times in milliseconds (ms). +@durations: +[0] 16838 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@| +[1] 882 |@@ | +[2, 4) 236 | | +[4, 8) 23 | | +[8, 16) 9 | | +[16, 32) 9 | | +[32, 64) 4 | | +``` diff --git a/contrib/tracing/connectblock_benchmark.bt b/contrib/tracing/connectblock_benchmark.bt new file mode 100755 index 0000000000..d268eff7f8 --- /dev/null +++ b/contrib/tracing/connectblock_benchmark.bt @@ -0,0 +1,150 @@ +#!/usr/bin/env bpftrace + +/* + + USAGE: + + BPFTRACE_STRLEN=65 bpftrace contrib/tracing/connectblock_benchmark.bt <start height> <end height> <logging threshold in ms> + + - The environment variable BPFTRACE_STRLEN needs to be set to 65 chars as + strings are limited to 64 chars by default. Hex strings with Bitcoin block + hashes are 64 hex chars + 1 null-termination char. + - <start height> sets the height at which the benchmark should start. Setting + the start height to 0 starts the benchmark immediately, even before the + first block is connected. + - <end height> sets the height after which the benchmark should end. Setting + the end height to 0 disables the benchmark. The script only logs blocks + over <logging threshold in ms>. + - Threshold <logging threshold in ms> + + This script requires a 'bitcoind' binary compiled with eBPF support and the + 'validation:block_connected' USDT. By default, it's assumed that 'bitcoind' is + located in './src/bitcoind'. This can be modified in the script below. + + EXAMPLES: + + BPFTRACE_STRLEN=65 bpftrace contrib/tracing/connectblock_benchmark.bt 300000 680000 1000 + + When run together 'bitcoind -reindex', this benchmarks the time it takes to + connect the blocks between height 300.000 and 680.000 (inclusive) and prints + details about all blocks that take longer than 1000ms to connect. Prints a + histogram with block connection times when the benchmark is finished. + + + BPFTRACE_STRLEN=65 bpftrace contrib/tracing/connectblock_benchmark.bt 0 0 500 + + When running together 'bitcoind', all newly connected blocks that + take longer than 500ms to connect are logged. A histogram with block + connection times is shown when the script is terminated. + +*/ + +BEGIN +{ + $start_height = $1; + $end_height = $2; + $logging_threshold_ms = $3; + + if ($end_height < $start_height) { + printf("Error: start height (%d) larger than end height (%d)!\n", $start_height, $end_height); + exit(); + } + + if ($end_height > 0) { + printf("ConnectBlock benchmark between height %d and %d inclusive\n", $start_height, $end_height); + } else { + printf("ConnectBlock logging starting at height %d\n", $start_height); + } + + if ($logging_threshold_ms > 0) { + printf("Logging blocks taking longer than %d ms to connect.\n", $3); + } + + if ($start_height == 0) { + @start = nsecs; + } +} + +/* + Attaches to the 'validation:block_connected' USDT and collects stats when the + connected block is between the start and end height (or the end height is + unset). +*/ +usdt:./src/bitcoind:validation:block_connected /arg1 >= $1 && (arg1 <= $2 || $2 == 0 )/ +{ + $height = arg1; + $transactions = arg2; + $inputs = arg3; + $sigops = arg4; + $duration = (uint64) arg5; + + @height = $height; + + @blocks = @blocks + 1; + @transactions = @transactions + $transactions; + @inputs = @inputs + $inputs; + @sigops = @sigops + $sigops; + + @durations = hist($duration / 1000); + + if ($height == $1 && $height != 0) { + @start = nsecs; + printf("Starting Connect Block Benchmark between height %d and %d.\n", $1, $2); + } + + if ($2 > 0 && $height >= $2) { + @end = nsecs; + $duration = @end - @start; + printf("\nTook %d ms to connect the blocks between height %d and %d.\n", $duration / 1000000, $1, $2); + exit(); + } +} + +/* + Attaches to the 'validation:block_connected' USDT and logs information about + blocks where the time it took to connect the block is above the + <logging threshold in ms>. +*/ +usdt:./src/bitcoind:validation:block_connected / (uint64) arg5 / 1000> $3 / +{ + $hash_str = str(arg0); + $height = (int32) arg1; + $transactions = (uint64) arg2; + $inputs = (int32) arg3; + $sigops = (int64) arg4; + $duration = (int64) arg5; + + printf("Block %d (%s) %4d tx %5d ins %5d sigops took %4d ms\n", $height, $hash_str, $transactions, $inputs, $sigops, (uint64) $duration / 1000); +} + + +/* + Prints stats about the blocks, transactions, inputs, and sigops processed in + the last second (if any). +*/ +interval:s:1 { + if (@blocks > 0) { + printf("BENCH %4d blk/s %6d tx/s %7d inputs/s %8d sigops/s (height %d)\n", @blocks, @transactions, @inputs, @sigops, @height); + + zero(@blocks); + zero(@transactions); + zero(@inputs); + zero(@sigops); + } +} + +END +{ + printf("\nHistogram of block connection times in milliseconds (ms).\n"); + print(@durations); + + clear(@durations); + clear(@blocks); + clear(@transactions); + clear(@inputs); + clear(@sigops); + clear(@height); + clear(@start); + clear(@end); +} + diff --git a/contrib/tracing/log_p2p_traffic.bt b/contrib/tracing/log_p2p_traffic.bt new file mode 100755 index 0000000000..f62956aa5e --- /dev/null +++ b/contrib/tracing/log_p2p_traffic.bt @@ -0,0 +1,28 @@ +#!/usr/bin/env bpftrace + +BEGIN +{ + printf("Logging P2P traffic\n") +} + +usdt:./src/bitcoind:net:inbound_message +{ + $peer_id = (int64) arg0; + $peer_addr = str(arg1); + $peer_type = str(arg2); + $msg_type = str(arg3); + $msg_len = arg4; + printf("inbound '%s' msg from peer %d (%s, %s) with %d bytes\n", $msg_type, $peer_id, $peer_type, $peer_addr, $msg_len); +} + +usdt:./src/bitcoind:net:outbound_message +{ + $peer_id = (int64) arg0; + $peer_addr = str(arg1); + $peer_type = str(arg2); + $msg_type = str(arg3); + $msg_len = arg4; + + printf("outbound '%s' msg to peer %d (%s, %s) with %d bytes\n", $msg_type, $peer_id, $peer_type, $peer_addr, $msg_len); +} + diff --git a/contrib/tracing/log_raw_p2p_msgs.py b/contrib/tracing/log_raw_p2p_msgs.py new file mode 100755 index 0000000000..b5b5755632 --- /dev/null +++ b/contrib/tracing/log_raw_p2p_msgs.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 + +""" Demonstration of eBPF limitations and the effect on USDT with the + net:inbound_message and net:outbound_message tracepoints. """ + +# This script shows a limitation of eBPF when data larger than 32kb is passed to +# user-space. It uses BCC (https://github.com/iovisor/bcc) to load a sandboxed +# eBPF program into the Linux kernel (root privileges are required). The eBPF +# program attaches to two statically defined tracepoints. The tracepoint +# 'net:inbound_message' is called when a new P2P message is received, and +# 'net:outbound_message' is called on outbound P2P messages. The eBPF program +# submits the P2P messages to this script via a BPF ring buffer. The submitted +# messages are printed. + +# eBPF Limitations: +# +# Bitcoin P2P messages can be larger than 32kb (e.g. tx, block, ...). The eBPF +# VM's stack is limited to 512 bytes, and we can't allocate more than about 32kb +# for a P2P message in the eBPF VM. The message data is cut off when the message +# is larger than MAX_MSG_DATA_LENGTH (see definition below). This can be detected +# in user-space by comparing the data length to the message length variable. The +# message is cut off when the data length is smaller than the message length. +# A warning is included with the printed message data. +# +# Data is submitted to user-space (i.e. to this script) via a ring buffer. The +# throughput of the ring buffer is limited. Each p2p_message is about 32kb in +# size. In- or outbound messages submitted to the ring buffer in rapid +# succession fill the ring buffer faster than it can be read. Some messages are +# lost. +# +# BCC prints: "Possibly lost 2 samples" on lost messages. + +import sys +from bcc import BPF, USDT + +# BCC: The C program to be compiled to an eBPF program (by BCC) and loaded into +# a sandboxed Linux kernel VM. +program = """ +#include <uapi/linux/ptrace.h> + +#define MIN(a,b) ({ __typeof__ (a) _a = (a); __typeof__ (b) _b = (b); _a < _b ? _a : _b; }) + +// Maximum possible allocation size +// from include/linux/percpu.h in the Linux kernel +#define PCPU_MIN_UNIT_SIZE (32 << 10) + +// Tor v3 addresses are 62 chars + 6 chars for the port (':12345'). +#define MAX_PEER_ADDR_LENGTH 62 + 6 +#define MAX_PEER_CONN_TYPE_LENGTH 20 +#define MAX_MSG_TYPE_LENGTH 20 +#define MAX_MSG_DATA_LENGTH PCPU_MIN_UNIT_SIZE - 200 + +struct p2p_message +{ + u64 peer_id; + char peer_addr[MAX_PEER_ADDR_LENGTH]; + char peer_conn_type[MAX_PEER_CONN_TYPE_LENGTH]; + char msg_type[MAX_MSG_TYPE_LENGTH]; + u64 msg_size; + u8 msg[MAX_MSG_DATA_LENGTH]; +}; + +// We can't store the p2p_message struct on the eBPF stack as it is limited to +// 512 bytes and P2P message can be bigger than 512 bytes. However, we can use +// an BPF-array with a length of 1 to allocate up to 32768 bytes (this is +// defined by PCPU_MIN_UNIT_SIZE in include/linux/percpu.h in the Linux kernel). +// Also see https://github.com/iovisor/bcc/issues/2306 +BPF_ARRAY(msg_arr, struct p2p_message, 1); + +// Two BPF perf buffers for pushing data (here P2P messages) to user-space. +BPF_PERF_OUTPUT(inbound_messages); +BPF_PERF_OUTPUT(outbound_messages); + +int trace_inbound_message(struct pt_regs *ctx) { + int idx = 0; + struct p2p_message *msg = msg_arr.lookup(&idx); + + // lookup() does not return a NULL pointer. However, the BPF verifier + // requires an explicit check that that the `msg` pointer isn't a NULL + // pointer. See https://github.com/iovisor/bcc/issues/2595 + if (msg == NULL) return 1; + + bpf_usdt_readarg(1, ctx, &msg->peer_id); + bpf_usdt_readarg_p(2, ctx, &msg->peer_addr, MAX_PEER_ADDR_LENGTH); + bpf_usdt_readarg_p(3, ctx, &msg->peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); + bpf_usdt_readarg_p(4, ctx, &msg->msg_type, MAX_MSG_TYPE_LENGTH); + bpf_usdt_readarg(5, ctx, &msg->msg_size); + bpf_usdt_readarg_p(6, ctx, &msg->msg, MIN(msg->msg_size, MAX_MSG_DATA_LENGTH)); + + inbound_messages.perf_submit(ctx, msg, sizeof(*msg)); + return 0; +}; + +int trace_outbound_message(struct pt_regs *ctx) { + int idx = 0; + struct p2p_message *msg = msg_arr.lookup(&idx); + + // lookup() does not return a NULL pointer. However, the BPF verifier + // requires an explicit check that that the `msg` pointer isn't a NULL + // pointer. See https://github.com/iovisor/bcc/issues/2595 + if (msg == NULL) return 1; + + bpf_usdt_readarg(1, ctx, &msg->peer_id); + bpf_usdt_readarg_p(2, ctx, &msg->peer_addr, MAX_PEER_ADDR_LENGTH); + bpf_usdt_readarg_p(3, ctx, &msg->peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); + bpf_usdt_readarg_p(4, ctx, &msg->msg_type, MAX_MSG_TYPE_LENGTH); + bpf_usdt_readarg(5, ctx, &msg->msg_size); + bpf_usdt_readarg_p(6, ctx, &msg->msg, MIN(msg->msg_size, MAX_MSG_DATA_LENGTH)); + + outbound_messages.perf_submit(ctx, msg, sizeof(*msg)); + return 0; +}; +""" + + +def print_message(event, inbound): + print(f"%s %s msg '%s' from peer %d (%s, %s) with %d bytes: %s" % + ( + f"Warning: incomplete message (only %d out of %d bytes)!" % ( + len(event.msg), event.msg_size) if len(event.msg) < event.msg_size else "", + "inbound" if inbound else "outbound", + event.msg_type.decode("utf-8"), + event.peer_id, + event.peer_conn_type.decode("utf-8"), + event.peer_addr.decode("utf-8"), + event.msg_size, + bytes(event.msg[:event.msg_size]).hex(), + ) + ) + + +def main(bitcoind_path): + bitcoind_with_usdts = USDT(path=str(bitcoind_path)) + + # attaching the trace functions defined in the BPF program to the tracepoints + bitcoind_with_usdts.enable_probe( + probe="inbound_message", fn_name="trace_inbound_message") + bitcoind_with_usdts.enable_probe( + probe="outbound_message", fn_name="trace_outbound_message") + bpf = BPF(text=program, usdt_contexts=[bitcoind_with_usdts]) + + # BCC: perf buffer handle function for inbound_messages + def handle_inbound(_, data, size): + """ Inbound message handler. + + Called each time a message is submitted to the inbound_messages BPF table.""" + + event = bpf["inbound_messages"].event(data) + print_message(event, True) + + # BCC: perf buffer handle function for outbound_messages + + def handle_outbound(_, data, size): + """ Outbound message handler. + + Called each time a message is submitted to the outbound_messages BPF table.""" + + event = bpf["outbound_messages"].event(data) + print_message(event, False) + + # BCC: add handlers to the inbound and outbound perf buffers + bpf["inbound_messages"].open_perf_buffer(handle_inbound) + bpf["outbound_messages"].open_perf_buffer(handle_outbound) + + print("Logging raw P2P messages.") + print("Messages larger that about 32kb will be cut off!") + print("Some messages might be lost!") + while True: + try: + bpf.perf_buffer_poll() + except KeyboardInterrupt: + exit() + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("USAGE:", sys.argv[0], "path/to/bitcoind") + exit() + path = sys.argv[1] + main(path) diff --git a/contrib/tracing/p2p_monitor.py b/contrib/tracing/p2p_monitor.py new file mode 100755 index 0000000000..14e3e3a801 --- /dev/null +++ b/contrib/tracing/p2p_monitor.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 + +""" Interactive bitcoind P2P network traffic monitor utilizing USDT and the + net:inbound_message and net:outbound_message tracepoints. """ + +# This script demonstrates what USDT for Bitcoin Core can enable. It uses BCC +# (https://github.com/iovisor/bcc) to load a sandboxed eBPF program into the +# Linux kernel (root privileges are required). The eBPF program attaches to two +# statically defined tracepoints. The tracepoint 'net:inbound_message' is called +# when a new P2P message is received, and 'net:outbound_message' is called on +# outbound P2P messages. The eBPF program submits the P2P messages to +# this script via a BPF ring buffer. + +import sys +import curses +from curses import wrapper, panel +from bcc import BPF, USDT + +# BCC: The C program to be compiled to an eBPF program (by BCC) and loaded into +# a sandboxed Linux kernel VM. +program = """ +#include <uapi/linux/ptrace.h> + +// Tor v3 addresses are 62 chars + 6 chars for the port (':12345'). +// I2P addresses are 60 chars + 6 chars for the port (':12345'). +#define MAX_PEER_ADDR_LENGTH 62 + 6 +#define MAX_PEER_CONN_TYPE_LENGTH 20 +#define MAX_MSG_TYPE_LENGTH 20 + +struct p2p_message +{ + u64 peer_id; + char peer_addr[MAX_PEER_ADDR_LENGTH]; + char peer_conn_type[MAX_PEER_CONN_TYPE_LENGTH]; + char msg_type[MAX_MSG_TYPE_LENGTH]; + u64 msg_size; +}; + + +// Two BPF perf buffers for pushing data (here P2P messages) to user space. +BPF_PERF_OUTPUT(inbound_messages); +BPF_PERF_OUTPUT(outbound_messages); + +int trace_inbound_message(struct pt_regs *ctx) { + struct p2p_message msg = {}; + + bpf_usdt_readarg(1, ctx, &msg.peer_id); + bpf_usdt_readarg_p(2, ctx, &msg.peer_addr, MAX_PEER_ADDR_LENGTH); + bpf_usdt_readarg_p(3, ctx, &msg.peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); + bpf_usdt_readarg_p(4, ctx, &msg.msg_type, MAX_MSG_TYPE_LENGTH); + bpf_usdt_readarg(5, ctx, &msg.msg_size); + + inbound_messages.perf_submit(ctx, &msg, sizeof(msg)); + return 0; +}; + +int trace_outbound_message(struct pt_regs *ctx) { + struct p2p_message msg = {}; + + bpf_usdt_readarg(1, ctx, &msg.peer_id); + bpf_usdt_readarg_p(2, ctx, &msg.peer_addr, MAX_PEER_ADDR_LENGTH); + bpf_usdt_readarg_p(3, ctx, &msg.peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); + bpf_usdt_readarg_p(4, ctx, &msg.msg_type, MAX_MSG_TYPE_LENGTH); + bpf_usdt_readarg(5, ctx, &msg.msg_size); + + outbound_messages.perf_submit(ctx, &msg, sizeof(msg)); + return 0; +}; +""" + + +class Message: + """ A P2P network message. """ + msg_type = "" + size = 0 + data = bytes() + inbound = False + + def __init__(self, msg_type, size, inbound): + self.msg_type = msg_type + self.size = size + self.inbound = inbound + + +class Peer: + """ A P2P network peer. """ + id = 0 + address = "" + connection_type = "" + last_messages = list() + + total_inbound_msgs = 0 + total_inbound_bytes = 0 + total_outbound_msgs = 0 + total_outbound_bytes = 0 + + def __init__(self, id, address, connection_type): + self.id = id + self.address = address + self.connection_type = connection_type + self.last_messages = list() + + def add_message(self, message): + self.last_messages.append(message) + if len(self.last_messages) > 25: + self.last_messages.pop(0) + if message.inbound: + self.total_inbound_bytes += message.size + self.total_inbound_msgs += 1 + else: + self.total_outbound_bytes += message.size + self.total_outbound_msgs += 1 + + +def main(bitcoind_path): + peers = dict() + + bitcoind_with_usdts = USDT(path=str(bitcoind_path)) + + # attaching the trace functions defined in the BPF program to the tracepoints + bitcoind_with_usdts.enable_probe( + probe="inbound_message", fn_name="trace_inbound_message") + bitcoind_with_usdts.enable_probe( + probe="outbound_message", fn_name="trace_outbound_message") + bpf = BPF(text=program, usdt_contexts=[bitcoind_with_usdts]) + + # BCC: perf buffer handle function for inbound_messages + def handle_inbound(_, data, size): + """ Inbound message handler. + + Called each time a message is submitted to the inbound_messages BPF table.""" + event = bpf["inbound_messages"].event(data) + if event.peer_id not in peers: + peer = Peer(event.peer_id, event.peer_addr.decode( + "utf-8"), event.peer_conn_type.decode("utf-8")) + peers[peer.id] = peer + peers[event.peer_id].add_message( + Message(event.msg_type.decode("utf-8"), event.msg_size, True)) + + # BCC: perf buffer handle function for outbound_messages + def handle_outbound(_, data, size): + """ Outbound message handler. + + Called each time a message is submitted to the outbound_messages BPF table.""" + event = bpf["outbound_messages"].event(data) + if event.peer_id not in peers: + peer = Peer(event.peer_id, event.peer_addr.decode( + "utf-8"), event.peer_conn_type.decode("utf-8")) + peers[peer.id] = peer + peers[event.peer_id].add_message( + Message(event.msg_type.decode("utf-8"), event.msg_size, False)) + + # BCC: add handlers to the inbound and outbound perf buffers + bpf["inbound_messages"].open_perf_buffer(handle_inbound) + bpf["outbound_messages"].open_perf_buffer(handle_outbound) + + wrapper(loop, bpf, peers) + + +def loop(screen, bpf, peers): + screen.nodelay(1) + cur_list_pos = 0 + win = curses.newwin(30, 70, 2, 7) + win.erase() + win.border(ord("|"), ord("|"), ord("-"), ord("-"), + ord("-"), ord("-"), ord("-"), ord("-")) + info_panel = panel.new_panel(win) + info_panel.hide() + + ROWS_AVALIABLE_FOR_LIST = curses.LINES - 5 + scroll = 0 + + while True: + try: + # BCC: poll the perf buffers for new events or timeout after 50ms + bpf.perf_buffer_poll(timeout=50) + + ch = screen.getch() + if (ch == curses.KEY_DOWN or ch == ord("j")) and cur_list_pos < len( + peers.keys()) -1 and info_panel.hidden(): + cur_list_pos += 1 + if cur_list_pos >= ROWS_AVALIABLE_FOR_LIST: + scroll += 1 + if (ch == curses.KEY_UP or ch == ord("k")) and cur_list_pos > 0 and info_panel.hidden(): + cur_list_pos -= 1 + if scroll > 0: + scroll -= 1 + if ch == ord('\n') or ch == ord(' '): + if info_panel.hidden(): + info_panel.show() + else: + info_panel.hide() + screen.erase() + render(screen, peers, cur_list_pos, scroll, ROWS_AVALIABLE_FOR_LIST, info_panel) + curses.panel.update_panels() + screen.refresh() + except KeyboardInterrupt: + exit() + + +def render(screen, peers, cur_list_pos, scroll, ROWS_AVALIABLE_FOR_LIST, info_panel): + """ renders the list of peers and details panel + + This code is unrelated to USDT, BCC and BPF. + """ + header_format = "%6s %-20s %-20s %-22s %-67s" + row_format = "%6s %-5d %9d byte %-5d %9d byte %-22s %-67s" + + screen.addstr(0, 1, (" P2P Message Monitor "), curses.A_REVERSE) + screen.addstr( + 1, 0, (" Navigate with UP/DOWN or J/K and select a peer with ENTER or SPACE to see individual P2P messages"), curses.A_NORMAL) + screen.addstr(3, 0, + header_format % ("PEER", "OUTBOUND", "INBOUND", "TYPE", "ADDR"), curses.A_BOLD | curses.A_UNDERLINE) + peer_list = sorted(peers.keys())[scroll:ROWS_AVALIABLE_FOR_LIST+scroll] + for i, peer_id in enumerate(peer_list): + peer = peers[peer_id] + screen.addstr(i + 4, 0, + row_format % (peer.id, peer.total_outbound_msgs, peer.total_outbound_bytes, + peer.total_inbound_msgs, peer.total_inbound_bytes, + peer.connection_type, peer.address), + curses.A_REVERSE if i + scroll == cur_list_pos else curses.A_NORMAL) + if i + scroll == cur_list_pos: + info_window = info_panel.window() + info_window.erase() + info_window.border( + ord("|"), ord("|"), ord("-"), ord("-"), + ord("-"), ord("-"), ord("-"), ord("-")) + + info_window.addstr( + 1, 1, f"PEER {peer.id} ({peer.address})".center(68), curses.A_REVERSE | curses.A_BOLD) + info_window.addstr( + 2, 1, f" OUR NODE{peer.connection_type:^54}PEER ", + curses.A_BOLD) + for i, msg in enumerate(peer.last_messages): + if msg.inbound: + info_window.addstr( + i + 3, 1, "%68s" % + (f"<--- {msg.msg_type} ({msg.size} bytes) "), curses.A_NORMAL) + else: + info_window.addstr( + i + 3, 1, " %s (%d byte) --->" % + (msg.msg_type, msg.size), curses.A_NORMAL) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("USAGE:", sys.argv[0], "path/to/bitcoind") + exit() + path = sys.argv[1] + main(path) diff --git a/contrib/verify-commits/README.md b/contrib/verify-commits/README.md index e95a57586f..b8b15280ba 100644 --- a/contrib/verify-commits/README.md +++ b/contrib/verify-commits/README.md @@ -40,7 +40,7 @@ Import trusted keys In order to check the commit signatures, you must add the trusted PGP keys to your machine. [GnuPG](https://gnupg.org/) may be used to import the trusted keys by running the following command: ```sh -gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys $(<contrib/verify-commits/trusted-keys) +gpg --keyserver hkps://keys.openpgp.org --recv-keys $(<contrib/verify-commits/trusted-keys) ``` Key expiry/revocation diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py index 6cbaf2dc0c..51c151add8 100755 --- a/contrib/verifybinaries/verify.py +++ b/contrib/verifybinaries/verify.py @@ -2,7 +2,7 @@ # Copyright (c) 2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Script for verifying Bitoin Core release binaries +"""Script for verifying Bitcoin Core release binaries This script attempts to download the signature file SHA256SUMS.asc from bitcoincore.org and bitcoin.org and compares them. diff --git a/contrib/zmq/zmq_sub.py b/contrib/zmq/zmq_sub.py index 9cb887e2dc..6269269d37 100755 --- a/contrib/zmq/zmq_sub.py +++ b/contrib/zmq/zmq_sub.py @@ -23,7 +23,6 @@ https://github.com/bitcoin/bitcoin/blob/37a7fe9e440b83e2364d5498931253937abe9294/contrib/zmq/zmq_sub.py """ -import binascii import asyncio import zmq import zmq.asyncio @@ -58,18 +57,18 @@ class ZMQHandler(): sequence = str(struct.unpack('<I', seq)[-1]) if topic == b"hashblock": print('- HASH BLOCK ('+sequence+') -') - print(binascii.hexlify(body)) + print(body.hex()) elif topic == b"hashtx": print('- HASH TX ('+sequence+') -') - print(binascii.hexlify(body)) + print(body.hex()) elif topic == b"rawblock": print('- RAW BLOCK HEADER ('+sequence+') -') - print(binascii.hexlify(body[:80])) + print(body[:80].hex()) elif topic == b"rawtx": print('- RAW TX ('+sequence+') -') - print(binascii.hexlify(body)) + print(body.hex()) elif topic == b"sequence": - hash = binascii.hexlify(body[:32]) + hash = body[:32].hex() label = chr(body[32]) mempool_sequence = None if len(body) != 32+1+8 else struct.unpack("<Q", body[32+1:])[0] print('- SEQUENCE ('+sequence+') -') diff --git a/depends/Makefile b/depends/Makefile index ac12e91e49..a3b9cd2099 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -1,7 +1,7 @@ .NOTPARALLEL : # Pattern rule to print variables, e.g. make print-top_srcdir -print-%: +print-%: FORCE @echo '$*'='$($*)' # When invoking a sub-make, keep only the command line variable definitions @@ -284,3 +284,4 @@ download: download-osx download-linux download-win $(foreach package,$(all_packages),$(eval $(call ext_add_stages,$(package)))) .PHONY: install cached clean clean-all download-one download-osx download-linux download-win download check-packages check-sources +.PHONY: FORCE diff --git a/depends/README.md b/depends/README.md index 50e1a32c70..15c82cddf2 100644 --- a/depends/README.md +++ b/depends/README.md @@ -28,7 +28,7 @@ Common `host-platform-triplet`s for cross compilation are: - `i686-pc-linux-gnu` for Linux 32 bit - `x86_64-pc-linux-gnu` for x86 Linux - `x86_64-w64-mingw32` for Win64 -- `x86_64-apple-darwin18` for macOS +- `x86_64-apple-darwin19` for macOS - `arm-linux-gnueabihf` for Linux ARM 32 bit - `aarch64-linux-gnu` for Linux ARM 64 bit - `powerpc64-linux-gnu` for Linux POWER 64-bit (big endian) @@ -87,6 +87,14 @@ For linux S390X cross compilation: sudo apt-get install g++-s390x-linux-gnu binutils-s390x-linux-gnu +### Install the required dependencies: M1-based macOS + +To be able to build the `qt` package, ensure that Rosetta 2 is installed: + +``` +softwareupdate --install-rosetta +``` + ### Dependency Options The following can be set when running make: `make FOO=bar` diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 5a7ae2df9a..ea92bb7793 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -1,4 +1,4 @@ -OSX_MIN_VERSION=10.14 +OSX_MIN_VERSION=10.15 OSX_SDK_VERSION=10.15.6 XCODE_VERSION=12.1 XCODE_BUILD_ID=12A7403 diff --git a/depends/packages/bdb.mk b/depends/packages/bdb.mk index d45ac3d03f..8a3116bb3b 100644 --- a/depends/packages/bdb.mk +++ b/depends/packages/bdb.mk @@ -12,7 +12,7 @@ $(package)_config_opts_mingw32=--enable-mingw $(package)_config_opts_linux=--with-pic $(package)_config_opts_android=--with-pic $(package)_cflags+=-Wno-error=implicit-function-declaration -$(package)_cxxflags=-std=c++17 +$(package)_cxxflags+=-std=c++17 $(package)_cppflags_mingw32=-DUNICODE -D_UNICODE endef diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index f879d176f5..21df50b040 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -23,7 +23,7 @@ else $(package)_toolset_$(host_os)=gcc endif $(package)_config_libraries=filesystem,system,test -$(package)_cxxflags=-std=c++17 -fvisibility=hidden +$(package)_cxxflags+=-std=c++17 -fvisibility=hidden $(package)_cxxflags_linux=-fPIC $(package)_cxxflags_android=-fPIC $(package)_cxxflags_x86_64_darwin=-fcf-protection=full diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index dad317193c..0af5412d94 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -16,6 +16,10 @@ define $(package)_set_vars $(package)_cppflags_mingw32=-D_WIN32_WINNT=0x0601 endef +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub build-aux +endef + define $(package)_config_cmds $($(package)_autoconf) endef diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index 885207fce9..d169eb6723 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -16,6 +16,10 @@ define $(package)_set_vars $(package)_cxx=$(clangxx_prog) endef +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub cctools +endef + define $(package)_config_cmds $($(package)_autoconf) endef diff --git a/depends/packages/native_clang.mk b/depends/packages/native_clang.mk index 36adeb196d..25ac77c1a3 100644 --- a/depends/packages/native_clang.mk +++ b/depends/packages/native_clang.mk @@ -1,9 +1,15 @@ package=native_clang $(package)_version=10.0.1 $(package)_download_path=https://github.com/llvm/llvm-project/releases/download/llvmorg-$($(package)_version) +ifneq (,$(findstring aarch64,$(BUILD))) +$(package)_download_file=clang+llvm-$($(package)_version)-aarch64-linux-gnu.tar.xz +$(package)_file_name=clang+llvm-$($(package)_version)-aarch64-linux-gnu.tar.xz +$(package)_sha256_hash=90dc69a4758ca15cd0ffa45d07fbf5bf4309d47d2c7745a9f0735ecffde9c31f +else $(package)_download_file=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-16.04.tar.xz $(package)_file_name=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-16.04.tar.xz $(package)_sha256_hash=48b83ef827ac2c213d5b64f5ad7ed082c8bcb712b46644e0dc5045c6f462c231 +endif define $(package)_preprocess_cmds rm -f $($(package)_extract_dir)/lib/libc++abi.so* diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 165c27f5e3..9004b064d6 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -6,9 +6,11 @@ $(package)_file_name=qtbase-$($(package)_suffix) $(package)_sha256_hash=1c1b4e33137ca77881074c140d54c3c9747e845a31338cfe8680f171f0bc3a39 $(package)_linux_dependencies=freetype fontconfig libxcb libxkbcommon $(package)_qt_libs=corelib network widgets gui plugins testlib -$(package)_patches=fix_qt_pkgconfig.patch mac-qmake.conf fix_no_printer.patch no-xlib.patch -$(package)_patches+= fix_android_qmake_conf.patch fix_android_jni_static.patch dont_hardcode_pwd.patch -$(package)_patches+= drop_lrelease_dependency.patch no_sdk_version_check.patch +$(package)_linguist_tools = lrelease lupdate lconvert +$(package)_patches = qt.pro qttools_src.pro +$(package)_patches += fix_qt_pkgconfig.patch mac-qmake.conf fix_no_printer.patch no-xlib.patch +$(package)_patches += support_new_android_ndks.patch fix_android_jni_static.patch dont_hardcode_pwd.patch +$(package)_patches+= no_sdk_version_check.patch $(package)_patches+= fix_lib_paths.patch fix_android_pch.patch $(package)_patches+= qtbase-moc-ignore-gcc-macro.patch fix_limits_header.patch @@ -64,6 +66,7 @@ $(package)_config_opts += -no-system-proxies $(package)_config_opts += -no-use-gold-linker $(package)_config_opts += -nomake examples $(package)_config_opts += -nomake tests +$(package)_config_opts += -nomake tools $(package)_config_opts += -opensource $(package)_config_opts += -pkg-config $(package)_config_opts += -prefix $(host_prefix) @@ -113,14 +116,13 @@ $(package)_config_opts_darwin = -no-dbus $(package)_config_opts_darwin += -no-opengl $(package)_config_opts_darwin += -pch $(package)_config_opts_darwin += -no-feature-corewlan -$(package)_config_opts_darwin += -device-option QMAKE_MACOSX_DEPLOYMENT_TARGET=$(OSX_MIN_VERSION) +$(package)_config_opts_darwin += QMAKE_MACOSX_DEPLOYMENT_TARGET=$(OSX_MIN_VERSION) ifneq ($(build_os),darwin) $(package)_config_opts_darwin += -xplatform macx-clang-linux $(package)_config_opts_darwin += -device-option MAC_SDK_PATH=$(OSX_SDK) $(package)_config_opts_darwin += -device-option MAC_SDK_VERSION=$(OSX_SDK_VERSION) $(package)_config_opts_darwin += -device-option CROSS_COMPILE="$(host)-" -$(package)_config_opts_darwin += -device-option MAC_MIN_VERSION=$(OSX_MIN_VERSION) $(package)_config_opts_darwin += -device-option MAC_TARGET=$(host) $(package)_config_opts_darwin += -device-option XCODE_VERSION=$(XCODE_VERSION) endif @@ -201,30 +203,28 @@ endef # # 1. Apply our patches to the extracted source. See each patch for more info. # -# 2. Point to lrelease in qttools/bin/lrelease; otherwise Qt will look for it in -# $(host)/native/bin/lrelease and not find it. +# 2. Create a macOS-Clang-Linux mkspec using our mac-qmake.conf. # -# 3. Create a macOS-Clang-Linux mkspec using our mac-qmake.conf. -# -# 4. After making a copy of the mkspec for the linux-arm-gnueabi host, named +# 3. After making a copy of the mkspec for the linux-arm-gnueabi host, named # bitcoin-linux-g++, replace instances of linux-arm-gnueabi with $(host). This # way we can generically support hosts like riscv64-linux-gnu, which Qt doesn't # ship a mkspec for. See it's usage in config_opts_* above. # -# 5. Put our C, CXX and LD FLAGS into gcc-base.conf. Only used for non-host builds. +# 4. Put our C, CXX and LD FLAGS into gcc-base.conf. Only used for non-host builds. # -# 6. Do similar for the win32-g++ mkspec. +# 5. Do similar for the win32-g++ mkspec. # -# 7. In clang.conf, swap out clang & clang++, for our compiler + flags. See #17466. +# 6. In clang.conf, swap out clang & clang++, for our compiler + flags. See #17466. # -# 8. Adjust a regex in toolchain.prf, to accommodate Guix's usage of +# 7. Adjust a regex in toolchain.prf, to accommodate Guix's usage of # CROSS_LIBRARY_PATH. See #15277. define $(package)_preprocess_cmds - patch -p1 -i $($(package)_patch_dir)/drop_lrelease_dependency.patch && \ + cp $($(package)_patch_dir)/qt.pro qt.pro && \ + cp $($(package)_patch_dir)/qttools_src.pro qttools/src/src.pro && \ patch -p1 -i $($(package)_patch_dir)/dont_hardcode_pwd.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_android_qmake_conf.patch && \ + patch -p1 -i $($(package)_patch_dir)/support_new_android_ndks.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_android_jni_static.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_android_pch.patch && \ patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \ @@ -232,7 +232,6 @@ define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/fix_lib_paths.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_limits_header.patch && \ - sed -i.old "s|updateqm.commands = \$$$$\$$$$LRELEASE|updateqm.commands = $($(package)_extract_dir)/qttools/bin/lrelease|" qttranslations/translations/translations.pro && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ cp -f qtbase/mkspecs/macx-clang/qplatformdefs.h qtbase/mkspecs/macx-clang-linux/ &&\ cp -f $($(package)_patch_dir)/mac-qmake.conf qtbase/mkspecs/macx-clang-linux/qmake.conf && \ @@ -249,35 +248,22 @@ endef define $(package)_config_cmds export PKG_CONFIG_SYSROOT_DIR=/ && \ export PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig && \ - export PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig && \ + export PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig && \ cd qtbase && \ - ./configure $($(package)_config_opts) && \ - cd .. && \ - $(MAKE) -C qtbase sub-src-clean && \ - qtbase/bin/qmake -o qttranslations/Makefile qttranslations/qttranslations.pro && \ - qtbase/bin/qmake -o qttranslations/translations/Makefile qttranslations/translations/translations.pro && \ - qtbase/bin/qmake -o qttools/src/linguist/lrelease/Makefile qttools/src/linguist/lrelease/lrelease.pro && \ - qtbase/bin/qmake -o qttools/src/linguist/lupdate/Makefile qttools/src/linguist/lupdate/lupdate.pro && \ - qtbase/bin/qmake -o qttools/src/linguist/lconvert/Makefile qttools/src/linguist/lconvert/lconvert.pro + ./configure -top-level $($(package)_config_opts) endef define $(package)_build_cmds - $(MAKE) -C qtbase/src $(addprefix sub-,$($(package)_qt_libs)) && \ - $(MAKE) -C qttools/src/linguist/lrelease && \ - $(MAKE) -C qttools/src/linguist/lupdate && \ - $(MAKE) -C qttools/src/linguist/lconvert && \ - $(MAKE) -C qttranslations + $(MAKE) endef define $(package)_stage_cmds $(MAKE) -C qtbase/src INSTALL_ROOT=$($(package)_staging_dir) $(addsuffix -install_subtargets,$(addprefix sub-,$($(package)_qt_libs))) && \ - $(MAKE) -C qttools/src/linguist/lrelease INSTALL_ROOT=$($(package)_staging_dir) install_target && \ - $(MAKE) -C qttools/src/linguist/lupdate INSTALL_ROOT=$($(package)_staging_dir) install_target && \ - $(MAKE) -C qttools/src/linguist/lconvert INSTALL_ROOT=$($(package)_staging_dir) install_target && \ + $(MAKE) -C qttools/src/linguist INSTALL_ROOT=$($(package)_staging_dir) $(addsuffix -install_subtargets,$(addprefix sub-,$($(package)_linguist_tools))) && \ $(MAKE) -C qttranslations INSTALL_ROOT=$($(package)_staging_dir) install_subtargets endef define $(package)_postprocess_cmds rm -rf native/mkspecs/ native/lib/ lib/cmake/ && \ - rm -f lib/lib*.la lib/*.prl plugins/*/*.prl + rm -f lib/lib*.la endef diff --git a/depends/packages/sqlite.mk b/depends/packages/sqlite.mk index 5b3a61b239..af5e0d09c9 100644 --- a/depends/packages/sqlite.mk +++ b/depends/packages/sqlite.mk @@ -9,6 +9,10 @@ $(package)_config_opts=--disable-shared --disable-readline --disable-dynamic-ext $(package)_config_opts_linux=--with-pic endef +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . +endef + define $(package)_config_cmds $($(package)_autoconf) endef diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index 3b7f3690a4..9798248c61 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -12,7 +12,7 @@ define $(package)_set_vars $(package)_config_opts += --disable-Werror --disable-drafts --enable-option-checking $(package)_config_opts_linux=--with-pic $(package)_config_opts_android=--with-pic - $(package)_cxxflags=-std=c++17 + $(package)_cxxflags+=-std=c++17 endef define $(package)_preprocess_cmds diff --git a/depends/patches/qt/drop_lrelease_dependency.patch b/depends/patches/qt/drop_lrelease_dependency.patch deleted file mode 100644 index 9b918af77c..0000000000 --- a/depends/patches/qt/drop_lrelease_dependency.patch +++ /dev/null @@ -1,20 +0,0 @@ -commit 67b3ed7406e1d0762188dbad2c44a06824ba0778 -Author: fanquake <fanquake@gmail.com> -Date: Tue Aug 18 15:24:01 2020 +0800 - - Drop dependency on lrelease - - Qts buildsystem insists on using the installed lrelease, but gets - confused about how to find it. Since we manually control the build - order, just drop the dependency. - - See #9469 - -diff --git a/qttranslations/translations/translations.pro b/qttranslations/translations/translations.pro -index 694544c..eff339d 100644 ---- a/qttranslations/translations/translations.pro -+++ b/qttranslations/translations/translations.pro -@@ -107,3 +107,2 @@ updateqm.commands = $$LRELEASE ${QMAKE_FILE_IN} -qm ${QMAKE_FILE_OUT} - silent:updateqm.commands = @echo lrelease ${QMAKE_FILE_IN} && $$updateqm.commands --updateqm.depends = $$LRELEASE_EXE - updateqm.name = LRELEASE ${QMAKE_FILE_IN} diff --git a/depends/patches/qt/fix_android_pch.patch b/depends/patches/qt/fix_android_pch.patch index bed6e4bb63..195e1c5e59 100644 --- a/depends/patches/qt/fix_android_pch.patch +++ b/depends/patches/qt/fix_android_pch.patch @@ -1,6 +1,6 @@ --- old/qtbase/mkspecs/common/android-base-head.conf +++ new/qtbase/mkspecs/common/android-base-head.conf -@@ -73,6 +73,6 @@ CROSS_COMPILE = $$NDK_TOOLCHAIN_PATH/bin/$$NDK_TOOLS_PREFIX- +@@ -72,6 +72,6 @@ CROSS_COMPILE = $$NDK_TOOLCHAIN_PATH/bin/$$NDK_TOOLS_PREFIX- QMAKE_PCH_OUTPUT_EXT = .gch QMAKE_CFLAGS_PRECOMPILE = -x c-header -c ${QMAKE_PCH_INPUT} -o ${QMAKE_PCH_OUTPUT} diff --git a/depends/patches/qt/fix_android_qmake_conf.patch b/depends/patches/qt/fix_android_qmake_conf.patch deleted file mode 100644 index 3a8753fd1d..0000000000 --- a/depends/patches/qt/fix_android_qmake_conf.patch +++ /dev/null @@ -1,10 +0,0 @@ ---- old/qtbase/mkspecs/android-clang/qmake.conf -+++ new/qtbase/mkspecs/android-clang/qmake.conf -@@ -47,7 +47,7 @@ ANDROID_STDCPP_PATH = $$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++_shared.so - ANDROID_USE_LLVM = true - - exists($$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++.so): \ -- ANDROID_CXX_STL_LIBS = -lc++ -+ ANDROID_CXX_STL_LIBS = -lc++_shared - else: \ - ANDROID_CXX_STL_LIBS = $$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++.so.$$replace(ANDROID_PLATFORM, "android-", "") diff --git a/depends/patches/qt/mac-qmake.conf b/depends/patches/qt/mac-qmake.conf index 0142667547..190ab7a160 100644 --- a/depends/patches/qt/mac-qmake.conf +++ b/depends/patches/qt/mac-qmake.conf @@ -8,7 +8,6 @@ include(../common/clang-mac.conf) QMAKE_MAC_SDK_PATH=$${MAC_SDK_PATH} QMAKE_XCODE_VERSION = $${XCODE_VERSION} QMAKE_XCODE_DEVELOPER_PATH=/Developer -QMAKE_MACOSX_DEPLOYMENT_TARGET = $${MAC_MIN_VERSION} QMAKE_MAC_SDK=macosx QMAKE_MAC_SDK.macosx.Path = $${MAC_SDK_PATH} QMAKE_MAC_SDK.macosx.platform_name = macosx diff --git a/depends/patches/qt/qt.pro b/depends/patches/qt/qt.pro new file mode 100644 index 0000000000..8f2e900a84 --- /dev/null +++ b/depends/patches/qt/qt.pro @@ -0,0 +1,16 @@ +# Create the super cache so modules will add themselves to it. +cache(, super) + +!QTDIR_build: cache(CONFIG, add, $$list(QTDIR_build)) + +prl = no_install_prl +CONFIG += $$prl +cache(CONFIG, add stash, prl) + +TEMPLATE = subdirs +SUBDIRS = qtbase qttools qttranslations + +qttools.depends = qtbase +qttranslations.depends = qttools + +load(qt_configure) diff --git a/depends/patches/qt/qttools_src.pro b/depends/patches/qt/qttools_src.pro new file mode 100644 index 0000000000..6ef71a0942 --- /dev/null +++ b/depends/patches/qt/qttools_src.pro @@ -0,0 +1,6 @@ +TEMPLATE = subdirs +SUBDIRS = linguist + +fb = force_bootstrap +CONFIG += $$fb +cache(CONFIG, add, fb) diff --git a/depends/patches/qt/support_new_android_ndks.patch b/depends/patches/qt/support_new_android_ndks.patch new file mode 100644 index 0000000000..85c8ae2132 --- /dev/null +++ b/depends/patches/qt/support_new_android_ndks.patch @@ -0,0 +1,122 @@ +Follow Google's BuildSystemMaintainers doc to support future NDK releases. + +Upstream commit: + - Qt 5.14: 9b14950ff600a4ce5a8698b67ab38907c50417f1 + +--- old/qtbase/mkspecs/android-clang/qmake.conf ++++ new/qtbase/mkspecs/android-clang/qmake.conf +@@ -14,43 +14,29 @@ + QMAKE_CC = $$NDK_LLVM_PATH/bin/clang + QMAKE_CXX = $$NDK_LLVM_PATH/bin/clang++ + ++# Follow https://android.googlesource.com/platform/ndk/+/ndk-release-r20/docs/BuildSystemMaintainers.md ++ + equals(ANDROID_TARGET_ARCH, armeabi-v7a): \ +- QMAKE_CFLAGS += -target armv7-none-linux-androideabi +-else: equals(ANDROID_TARGET_ARCH, armeabi): \ +- QMAKE_CFLAGS += -target armv5te-none-linux-androideabi ++ QMAKE_CFLAGS = -target armv7a-linux-androideabi$$replace(ANDROID_PLATFORM, "android-", "") + else: equals(ANDROID_TARGET_ARCH, arm64-v8a): \ +- QMAKE_CFLAGS += -target aarch64-none-linux-android ++ QMAKE_CFLAGS = -target aarch64-linux-android$$replace(ANDROID_PLATFORM, "android-", "") + else: equals(ANDROID_TARGET_ARCH, x86): \ +- QMAKE_CFLAGS += -target i686-none-linux-android -mstackrealign ++ QMAKE_CFLAGS = -target i686-linux-android$$replace(ANDROID_PLATFORM, "android-", "") -mstackrealign + else: equals(ANDROID_TARGET_ARCH, x86_64): \ +- QMAKE_CFLAGS += -target x86_64-none-linux-android +-else: equals(ANDROID_TARGET_ARCH, mips): \ +- QMAKE_CFLAGS += -target mipsel-none-linux-android +-else: equals(ANDROID_TARGET_ARCH, mips64): \ +- QMAKE_CFLAGS += -target mips64el-none-linux-android +- +-QMAKE_CFLAGS += -gcc-toolchain $$NDK_TOOLCHAIN_PATH -fno-limit-debug-info +- +-QMAKE_LINK = $$QMAKE_CXX $$QMAKE_CFLAGS -Wl,--exclude-libs,libgcc.a -Wl,--exclude-libs,libatomic.a -nostdlib++ +-equals(ANDROID_TARGET_ARCH, armeabi-v7a): QMAKE_LINK += -Wl,--exclude-libs,libunwind.a +- +-QMAKE_CFLAGS += -DANDROID_HAS_WSTRING --sysroot=$$NDK_ROOT/sysroot \ +- -isystem $$NDK_ROOT/sysroot/usr/include/$$NDK_TOOLS_PREFIX \ +- -isystem $$NDK_ROOT/sources/cxx-stl/llvm-libc++/include \ +- -isystem $$NDK_ROOT/sources/android/support/include \ +- -isystem $$NDK_ROOT/sources/cxx-stl/llvm-libc++abi/include ++ QMAKE_CFLAGS = -target x86_64-linux-android$$replace(ANDROID_PLATFORM, "android-", "") + +-ANDROID_SOURCES_CXX_STL_LIBDIR = $$NDK_ROOT/sources/cxx-stl/llvm-libc++/libs/$$ANDROID_TARGET_ARCH ++QMAKE_CFLAGS += -fno-limit-debug-info + +-ANDROID_STDCPP_PATH = $$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++_shared.so ++QMAKE_LINK = $$QMAKE_CXX $$QMAKE_CFLAGS + +-ANDROID_USE_LLVM = true ++ANDROID_STDCPP_PATH = $$NDK_LLVM_PATH/sysroot/usr/lib/$$NDK_TOOLS_PREFIX/libc++_shared.so + +-exists($$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++.so): \ +- ANDROID_CXX_STL_LIBS = -lc++ +-else: \ +- ANDROID_CXX_STL_LIBS = $$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++.so.$$replace(ANDROID_PLATFORM, "android-", "") ++ANDROID_USE_LLVM = true + +-QMAKE_CFLAGS_OPTIMIZE_SIZE = -Oz ++QMAKE_CFLAGS_OPTIMIZE_SIZE = -Oz ++QMAKE_LIBDIR_POST = ++QMAKE_LFLAGS = ++QMAKE_LIBS_PRIVATE = ++ANDROID_CXX_STL_LIBS = + + include(../common/android-base-tail.conf) + +--- old/qtbase/mkspecs/common/android-base-head.conf ++++ new/qtbase/mkspecs/common/android-base-head.conf +@@ -64,7 +58,6 @@ + } + + CONFIG += $$ANDROID_PLATFORM +-QMAKE_CFLAGS = -D__ANDROID_API__=$$replace(ANDROID_PLATFORM, "android-", "") + + ANDROID_PLATFORM_ROOT_PATH = $$NDK_ROOT/platforms/$$ANDROID_PLATFORM/arch-$$ANDROID_ARCHITECTURE/ + +--- old/qtbase/mkspecs/common/android-base-tail.conf ++++ new/qtbase/mkspecs/common/android-base-tail.conf +@@ -6,22 +6,17 @@ + QMAKE_CFLAGS += -fstack-protector-strong -DANDROID + + equals(ANDROID_TARGET_ARCH, armeabi-v7a): \ +- QMAKE_CFLAGS += -march=armv7-a -mfloat-abi=softfp -mfpu=vfp -fno-builtin-memmove ++ QMAKE_CFLAGS += -march=armv7-a -mfloat-abi=softfp -mfpu=vfp + else: equals(ANDROID_TARGET_ARCH, armeabi): \ +- QMAKE_CFLAGS += -march=armv5te -mtune=xscale -msoft-float -fno-builtin-memmove +-# -fno-builtin-memmove is used to workaround https://code.google.com/p/android/issues/detail?id=81692 ++ QMAKE_CFLAGS += -march=armv5te -mtune=xscale -msoft-float + + QMAKE_CFLAGS_WARN_ON = -Wall -W + QMAKE_CFLAGS_WARN_OFF = + equals(ANDROID_TARGET_ARCH, armeabi-v7a) | equals(ANDROID_TARGET_ARCH, armeabi) { + CONFIG += optimize_size + QMAKE_CFLAGS_DEBUG = -g -marm -O0 +- equals(ANDROID_TARGET_ARCH, armeabi):if(equals(NDK_TOOLCHAIN_VERSION, 4.8)|equals(NDK_TOOLCHAIN_VERSION, 4.9)) { +- DEFINES += QT_OS_ANDROID_GCC_48_WORKAROUND +- } else { +- QMAKE_CFLAGS_RELEASE += -mthumb +- QMAKE_CFLAGS_RELEASE_WITH_DEBUGINFO += -mthumb +- } ++ QMAKE_CFLAGS_RELEASE += -mthumb ++ QMAKE_CFLAGS_RELEASE_WITH_DEBUGINFO += -mthumb + } + + QMAKE_CFLAGS_SHLIB = -fPIC +@@ -61,15 +56,12 @@ + QMAKE_RANLIB = $${CROSS_COMPILE}ranlib + + QMAKE_INCDIR_POST = +-QMAKE_LIBDIR_POST = $$ANDROID_SOURCES_CXX_STL_LIBDIR + QMAKE_INCDIR_X11 = + QMAKE_LIBDIR_X11 = + QMAKE_INCDIR_OPENGL = + QMAKE_LIBDIR_OPENGL = + + QMAKE_LINK_SHLIB = $$QMAKE_LINK +-QMAKE_LFLAGS = --sysroot=$$ANDROID_PLATFORM_ROOT_PATH +-equals(ANDROID_TARGET_ARCH, x86_64) QMAKE_LFLAGS += -L$$ANDROID_PLATFORM_ROOT_PATH/usr/lib64 + QMAKE_LFLAGS_APP = -Wl,--no-undefined -Wl,-z,noexecstack -shared + QMAKE_LFLAGS_SHLIB = -Wl,--no-undefined -Wl,-z,noexecstack -shared + QMAKE_LFLAGS_PLUGIN = $$QMAKE_LFLAGS_SHLIB diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index 21bf587eaf..d8fd46d1c7 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -863,9 +863,7 @@ RECURSIVE = YES EXCLUDE = src/crc32c \ src/leveldb \ - src/json \ - src/test \ - src/qt/test + src/json # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded diff --git a/doc/README.md b/doc/README.md index 38f6b1d327..aabfe220bc 100644 --- a/doc/README.md +++ b/doc/README.md @@ -46,7 +46,6 @@ The following are developer notes on how to build Bitcoin Core on your native pl - [OpenBSD Build Notes](build-openbsd.md) - [NetBSD Build Notes](build-netbsd.md) - [Android Build Notes](build-android.md) -- [Gitian Building Guide (External Link)](https://github.com/bitcoin-core/docs/blob/master/gitian-building.md) Development --------------------- @@ -77,6 +76,7 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th - [Fuzz-testing](fuzzing.md) - [I2P Support](i2p.md) - [Init Scripts (systemd/upstart/openrc)](init.md) +- [Managing Wallets](managing-wallets.md) - [PSBT support](psbt.md) - [Reduce Memory](reduce-memory.md) - [Reduce Traffic](reduce-traffic.md) diff --git a/doc/benchmarking.md b/doc/benchmarking.md index b6cd86eafe..84d5f2c444 100644 --- a/doc/benchmarking.md +++ b/doc/benchmarking.md @@ -8,8 +8,10 @@ thread queue, wallet balance. Running --------------------- -For benchmarks purposes you only need to compile `bitcoin_bench`. Beware of configuring without `--enable-debug` as this would impact -benchmarking by unlatching log printers and lock analysis. +For benchmarking, you only need to compile `bitcoin_bench`. The bench runner +warns if you configure with `--enable-debug`, but consider if building without +it will impact the benchmark(s) you are interested in by unlatching log printers +and lock analysis. make -C src bitcoin_bench @@ -19,19 +21,28 @@ After compiling bitcoin-core, the benchmarks can be run with: The output will look similar to: ``` -| ns/byte | byte/s | error % | benchmark -|--------------------:|--------------------:|--------:|:---------------------------------------------- -| 64.13 | 15,592,356.01 | 0.1% | `Base58CheckEncode` -| 24.56 | 40,722,672.68 | 0.2% | `Base58Decode` +| ns/op | op/s | err% | total | benchmark +|--------------------:|--------------------:|--------:|----------:|:---------- +| 57,927,463.00 | 17.26 | 3.6% | 0.66 | `AddrManAdd` +| 677,816.00 | 1,475.33 | 4.9% | 0.01 | `AddrManGetAddr` + +... + +| ns/byte | byte/s | err% | total | benchmark +|--------------------:|--------------------:|--------:|----------:|:---------- +| 127.32 | 7,854,302.69 | 0.3% | 0.00 | `Base58CheckEncode` +| 31.95 | 31,303,226.99 | 0.2% | 0.00 | `Base58Decode` + ... ``` Help --------------------- - src/bench/bench_bitcoin --help + src/bench/bench_bitcoin -? -To print options like scaling factor or per-benchmark filter. +To print the various options, like listing the benchmarks without running them +or using a regex filter to only run certain benchmarks. Notes --------------------- diff --git a/doc/bitcoin-conf.md b/doc/bitcoin-conf.md index 9a312bc33c..8c9035c45b 100644 --- a/doc/bitcoin-conf.md +++ b/doc/bitcoin-conf.md @@ -4,6 +4,8 @@ The configuration file is used by `bitcoind`, `bitcoin-qt` and `bitcoin-cli`. All command-line options (except for `-?`, `-help`, `-version` and `-conf`) may be specified in a configuration file, and all configuration file options (except for `includeconf`) may also be specified on the command line. Command-line options override values set in the configuration file and configuration file options override values set in the GUI. +Changes to the configuration file while `bitcoind` or `bitcoin-qt` is running only take effect after restarting. + ## Configuration File Format The configuration file is a plain text file and consists of `option=value` entries, one per line. Leading and trailing whitespaces are removed. diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index 613aea438f..6e54f67edc 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -1,6 +1,6 @@ OpenBSD build guide ====================== -(updated for OpenBSD 6.7) +(updated for OpenBSD 6.9) This guide describes how to build bitcoind, bitcoin-qt, and command-line utilities on OpenBSD. @@ -67,9 +67,16 @@ export AUTOMAKE_VERSION=1.16 ``` Make sure `BDB_PREFIX` is set to the appropriate path from the above steps. +Note that building with external signer support currently fails on OpenBSD, +hence you have to explicitly disable it by passing the parameter +`--disable-external-signer` to the configure script. +(Background: the feature requires the header-only library boost::process, which +is available on OpenBSD 6.9 via Boost 1.72.0, but contains certain system calls +and preprocessor defines like `waitid()` and `WEXITED` that are not available.) + To configure with wallet: ```bash -./configure --with-gui=no CC=cc CXX=c++ \ +./configure --with-gui=no --disable-external-signer CC=cc CXX=c++ \ BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ BDB_CFLAGS="-I${BDB_PREFIX}/include" \ MAKE=gmake @@ -77,12 +84,12 @@ To configure with wallet: To configure without wallet: ```bash -./configure --disable-wallet --with-gui=no CC=cc CC_FOR_BUILD=cc CXX=c++ MAKE=gmake +./configure --disable-wallet --with-gui=no --disable-external-signer CC=cc CC_FOR_BUILD=cc CXX=c++ MAKE=gmake ``` To configure with GUI: ```bash -./configure --with-gui=yes CC=cc CXX=c++ \ +./configure --with-gui=yes --disable-external-signer CC=cc CXX=c++ \ BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ BDB_CFLAGS="-I${BDB_PREFIX}/include" \ MAKE=gmake diff --git a/doc/build-unix.md b/doc/build-unix.md index 73c0bf8779..02c36eea7c 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -48,6 +48,7 @@ Optional dependencies: univalue | Utility | JSON parsing and encoding (bundled version will be used unless --with-system-univalue passed to configure) libzmq3 | ZMQ notification | Optional, allows generating ZMQ notifications (requires ZMQ version >= 4.0.0) sqlite3 | SQLite DB | Optional, wallet storage (only needed when wallet enabled) + systemtap | Tracing (USDT) | Optional, statically defined tracepoints For the versions used, see [dependencies.md](dependencies.md) @@ -107,6 +108,10 @@ ZMQ dependencies (provides ZMQ API): sudo apt-get install libzmq3-dev +User-Space, Statically Defined Tracing (USDT) dependencies: + + sudo apt install systemtap-sdt-dev + GUI dependencies: If you want to build bitcoin-qt, make sure that the required packages for Qt development @@ -117,6 +122,10 @@ To build with Qt 5 you need the following: sudo apt-get install libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools +Additionally, to support Wayland protocol for modern desktop environments: + + sudo apt install qtwayland5 + libqrencode (optional) can be installed with: sudo apt-get install libqrencode-dev @@ -162,6 +171,10 @@ ZMQ dependencies (provides ZMQ API): sudo dnf install zeromq-devel +User-Space, Statically Defined Tracing (USDT) dependencies: + + sudo dnf install systemtap + GUI dependencies: If you want to build bitcoin-qt, make sure that the required packages for Qt development @@ -172,6 +185,10 @@ To build with Qt 5 you need the following: sudo dnf install qt5-qttools-devel qt5-qtbase-devel +Additionally, to support Wayland protocol for modern desktop environments: + + sudo dnf install qt5-qtwayland + libqrencode (optional) can be installed with: sudo dnf install qrencode-devel @@ -331,7 +348,7 @@ To build executables for ARM: make HOST=arm-linux-gnueabihf NO_QT=1 cd .. ./autogen.sh - CONFIG_SITE=$PWD/depends/arm-linux-gnueabihf/share/config.site ./configure --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++ + CONFIG_SITE=$PWD/depends/arm-linux-gnueabihf/share/config.site ./configure --enable-reduce-exports LDFLAGS=-static-libstdc++ make diff --git a/doc/build-windows.md b/doc/build-windows.md index f88b9739de..0b895eadfb 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -5,11 +5,9 @@ Below are some notes on how to build Bitcoin Core for Windows. The options known to work for building Bitcoin Core on Windows are: -* On Linux, using the [Mingw-w64](https://mingw-w64.org/doku.php) cross compiler tool chain. Ubuntu Bionic 18.04 is required -and is the platform used to build the Bitcoin Core Windows release binaries. -* On Windows, using [Windows -Subsystem for Linux (WSL)](https://docs.microsoft.com/windows/wsl/about) and the Mingw-w64 cross compiler tool chain. -* On Windows, using a native compiler tool chain such as [Visual Studio](https://www.visualstudio.com). See [README.md](/build_msvc/README.md). +* On Linux, using the [Mingw-w64](https://www.mingw-w64.org/) cross compiler tool chain. +* On Windows, using [Windows Subsystem for Linux (WSL)](https://docs.microsoft.com/windows/wsl/about) and Mingw-w64. +* On Windows, using [Microsoft Visual Studio](https://www.visualstudio.com). See [README.md](/build_msvc/README.md). Other options which may work, but which have not been extensively tested are (please contribute instructions): @@ -18,40 +16,12 @@ Other options which may work, but which have not been extensively tested are (pl Installing Windows Subsystem for Linux --------------------------------------- -With Windows 10, Microsoft has released a new feature named the [Windows -Subsystem for Linux (WSL)](https://docs.microsoft.com/windows/wsl/about). This -feature allows you to run a bash shell directly on Windows in an Ubuntu-based -environment. Within this environment you can cross compile for Windows without -the need for a separate Linux VM or server. Note that while WSL can be installed with -other Linux variants, such as OpenSUSE, the following instructions have only been -tested with Ubuntu. - -This feature is not supported in versions of Windows prior to Windows 10 or on -Windows Server SKUs. In addition, it is available [only for 64-bit versions of -Windows](https://docs.microsoft.com/windows/wsl/install-win10). - -Full instructions to install WSL are available on the above link. -To install WSL on Windows 10 with Fall Creators Update installed (version >= 16215.0) do the following: - -1. Enable the Windows Subsystem for Linux feature - * Open the Windows Features dialog (`OptionalFeatures.exe`) - * Enable 'Windows Subsystem for Linux' - * Click 'OK' and restart if necessary -2. Install Ubuntu - * Open Microsoft Store and search for "Ubuntu 18.04" or use [this link](https://www.microsoft.com/store/productId/9N9TNGVNDL3Q) - * Click Install -3. Complete Installation - * Open a cmd prompt and type "Ubuntu1804" - * Create a new UNIX user account (this is a separate account from your Windows account) - -After the bash shell is active, you can follow the instructions below, starting -with the "Cross-compilation" section. Compiling the 64-bit version is -recommended, but it is possible to compile the 32-bit version. +Follow the upstream installation instructions, available [here](https://docs.microsoft.com/windows/wsl/install-win10). Cross-compilation for Ubuntu and Windows Subsystem for Linux ------------------------------------------------------------ -The steps below can be performed on Ubuntu (including in a VM) or WSL. The depends system +The steps below can be performed on Ubuntu or WSL. The depends system will also work on other Linux distributions, however the commands for installing the toolchain will be different. diff --git a/doc/dependencies.md b/doc/dependencies.md index 66c5a76b3b..0c1fd6ba98 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -7,11 +7,12 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | --- | --- | --- | --- | --- | --- | | Berkeley DB | [4.8.30](https://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html) | 4.8.x | No | | | | Boost | [1.71.0](https://www.boost.org/users/download/) | [1.64.0](https://github.com/bitcoin/bitcoin/pull/22320) | No | | | -| Clang<sup>[ \* ](#note1)</sup> | | [5.0+](https://releases.llvm.org/download.html) (C++17 support) | | | | +| Clang<sup>[ \* ](#note1)</sup> | | [7.0](https://releases.llvm.org/download.html) (C++17 & std::filesystem support) | | | | | Expat | [2.2.7](https://libexpat.github.io/) | | No | Yes | | | fontconfig | [2.12.1](https://www.freedesktop.org/software/fontconfig/release/) | | No | Yes | | | FreeType | [2.7.1](https://download.savannah.gnu.org/releases/freetype) | | No | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Android only) | -| GCC | | [7+](https://gcc.gnu.org/) (C++17 support) | | | | +| GCC | | [8.1](https://gcc.gnu.org/) (C++17 & std::filesystem support) | | | | +| glibc | | [2.17](https://www.gnu.org/software/libc/) | | | | | | HarfBuzz-NG | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | libevent | [2.1.12-stable](https://github.com/libevent/libevent/releases) | [2.0.21](https://github.com/bitcoin/bitcoin/pull/18676) | No | | | | libnatpmp | git commit [4536032...](https://github.com/miniupnp/libnatpmp/tree/4536032ae32268a45c073a4d5e91bbab4534773a) | | No | | | @@ -24,6 +25,7 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | Qt | [5.12.11](https://download.qt.io/official_releases/qt/) | [5.9.5](https://github.com/bitcoin/bitcoin/issues/20104) | No | | | | SQLite | [3.32.1](https://sqlite.org/download.html) | [3.7.17](https://github.com/bitcoin/bitcoin/pull/19077) | | | | | XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | +| systemtap ([tracing](tracing.md))| | | | | | | xkbcommon | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | | ZeroMQ | [4.3.1](https://github.com/zeromq/libzmq/releases) | 4.0.0 | No | | | | zlib | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | @@ -41,6 +43,7 @@ Some dependencies are not needed in all configurations. The following are some f * SQLite is not needed with `--disable-wallet` or `--without-sqlite`. * Qt is not needed with `--without-gui`. * If the qrencode dependency is absent, QR support won't be added. To force an error when that happens, pass `--with-qrencode`. +* If the systemtap dependency is absent, USDT support won't compiled in. * ZeroMQ is needed only with the `--with-zmq` option. #### Other diff --git a/doc/descriptors.md b/doc/descriptors.md index e27ff87546..3bbb626a42 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -99,7 +99,7 @@ Descriptors consist of several types of expressions. The top level expression is `ADDR` expressions are any type of supported address: - P2PKH addresses (base58, of the form `1...` for mainnet or `[nm]...` for testnet). Note that P2PKH addresses in descriptors cannot be used for P2PK outputs (use the `pk` function instead). - P2SH addresses (base58, of the form `3...` for mainnet or `2...` for testnet, defined in [BIP 13](https://github.com/bitcoin/bips/blob/master/bip-0013.mediawiki)). -- Segwit addresses (bech32, of the form `bc1...` for mainnet or `tb1...` for testnet, defined in [BIP 173](https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki)). +- Segwit addresses (bech32 and bech32m, of the form `bc1...` for mainnet or `tb1...` for testnet, defined in [BIP 173](https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki) and [BIP 350](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki)). ## Explanation diff --git a/doc/developer-notes.md b/doc/developer-notes.md index be83257740..a05ea93a46 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -89,7 +89,11 @@ code. separate words (snake_case). - Class member variables have a `m_` prefix. - Global variables have a `g_` prefix. - - Compile-time constant names are all uppercase, and use `_` to separate words. + - Constant names are all uppercase, and use `_` to separate words. + - Enumerator constants may be `snake_case`, `PascalCase` or `ALL_CAPS`. + This is a more tolerant policy than the [C++ Core + Guidelines](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Renum-caps), + which recommend using `snake_case`. Please use what seems appropriate. - Class names, function names, and method names are UpperCamelCase (PascalCase). Do not prefix class names with `C`. - Test suite naming convention: The Boost test suite in file @@ -375,7 +379,7 @@ make cov Profiling is a good way to get a precise idea of where time is being spent in code. One tool for doing profiling on Linux platforms is called -[`perf`](http://www.brendangregg.com/perf.html), and has been integrated into +[`perf`](https://www.brendangregg.com/perf.html), and has been integrated into the functional test framework. Perf can observe a running process and sample (at some frequency) where its execution is. @@ -699,19 +703,19 @@ Foo(vec); ```cpp enum class Tabs { - INFO, - CONSOLE, - GRAPH, - PEERS + info, + console, + network_graph, + peers }; int GetInt(Tabs tab) { switch (tab) { - case Tabs::INFO: return 0; - case Tabs::CONSOLE: return 1; - case Tabs::GRAPH: return 2; - case Tabs::PEERS: return 3; + case Tabs::info: return 0; + case Tabs::console: return 1; + case Tabs::network_graph: return 2; + case Tabs::peers: return 3; } // no default case, so the compiler can warn about missing cases assert(false); } @@ -989,37 +993,41 @@ Subtrees Several parts of the repository are subtrees of software maintained elsewhere. -Some of these are maintained by active developers of Bitcoin Core, in which case changes should probably go -directly upstream without being PRed directly against the project. They will be merged back in the next -subtree merge. +Some of these are maintained by active developers of Bitcoin Core, in which case +changes should go directly upstream without being PRed directly against the project. +They will be merged back in the next subtree merge. -Others are external projects without a tight relationship with our project. Changes to these should also -be sent upstream, but bugfixes may also be prudent to PR against Bitcoin Core so that they can be integrated -quickly. Cosmetic changes should be purely taken upstream. +Others are external projects without a tight relationship with our project. Changes +to these should also be sent upstream, but bugfixes may also be prudent to PR against +a Bitcoin Core subtree, so that they can be integrated quickly. Cosmetic changes +should be taken upstream. -There is a tool in `test/lint/git-subtree-check.sh` ([instructions](../test/lint#git-subtree-checksh)) to check a subtree directory for consistency with -its upstream repository. +There is a tool in `test/lint/git-subtree-check.sh` ([instructions](../test/lint#git-subtree-checksh)) +to check a subtree directory for consistency with its upstream repository. Current subtrees include: - src/leveldb - - Upstream at https://github.com/google/leveldb ; Maintained by Google, but - open important PRs to Core to avoid delay. + - Subtree at https://github.com/bitcoin-core/leveldb-subtree ; maintained by Core contributors. + - Upstream at https://github.com/google/leveldb ; maintained by Google. Open + important PRs to the subtree to avoid delay. - **Note**: Follow the instructions in [Upgrading LevelDB](#upgrading-leveldb) when merging upstream changes to the LevelDB subtree. - src/crc32c - Used by leveldb for hardware acceleration of CRC32C checksums for data integrity. - - Upstream at https://github.com/google/crc32c ; Maintained by Google. + - Subtree at https://github.com/bitcoin-core/crc32c-subtree ; maintained by Core contributors. + - Upstream at https://github.com/google/crc32c ; maintained by Google. - src/secp256k1 - - Upstream at https://github.com/bitcoin-core/secp256k1/ ; actively maintained by Core contributors. + - Upstream at https://github.com/bitcoin-core/secp256k1/ ; maintained by Core contributors. - src/crypto/ctaes - - Upstream at https://github.com/bitcoin-core/ctaes ; actively maintained by Core contributors. + - Upstream at https://github.com/bitcoin-core/ctaes ; maintained by Core contributors. - src/univalue - - Upstream at https://github.com/bitcoin-core/univalue ; actively maintained by Core contributors, deviates from upstream https://github.com/jgarzik/univalue + - Subtree at https://github.com/bitcoin-core/univalue-subtree ; maintained by Core contributors. + - Deviates from upstream https://github.com/jgarzik/univalue. Upgrading LevelDB --------------------- diff --git a/doc/files.md b/doc/files.md index e670d77ae5..f88d3f91a1 100644 --- a/doc/files.md +++ b/doc/files.md @@ -56,7 +56,6 @@ Subdirectory | File(s) | Description `indexes/coinstats/db/` | LevelDB database | Coinstats index; *optional*, used if `-coinstatsindex=1` `wallets/` | | [Contains wallets](#multi-wallet-environment); can be specified by `-walletdir` option; if `wallets/` subdirectory does not exist, wallets reside in the [data directory](#data-directory-location) `./` | `anchors.dat` | Anchor IP address database, created on shutdown and deleted at startup. Anchors are last known outgoing block-relay-only peers that are tried to re-connect to on startup -`./` | `banlist.dat` | Stores the addresses/subnets of banned nodes (deprecated). `bitcoind` or `bitcoin-qt` no longer save the banlist to this file, but read it on startup if `banlist.json` is not present. `./` | `banlist.json` | Stores the addresses/subnets of banned nodes. `./` | `bitcoin.conf` | User-defined [configuration settings](bitcoin-conf.md) for `bitcoind` or `bitcoin-qt`. File is not written to by the software and must be created manually. Path can be specified by `-conf` option `./` | `bitcoind.pid` | Stores the process ID (PID) of `bitcoind` or `bitcoin-qt` while running; created at start and deleted on shutdown; can be specified by `-pid` option @@ -114,6 +113,7 @@ These subdirectories and files are no longer used by Bitcoin Core: Path | Description | Repository notes ---------------|-------------|----------------- +`banlist.dat` | Stores the addresses/subnets of banned nodes; superseded by `banlist.json` in 22.0 and completely ignored in 23.0 | [PR #20966](https://github.com/bitcoin/bitcoin/pull/20966), [PR #22570](https://github.com/bitcoin/bitcoin/pull/22570) `blktree/` | Blockchain index; replaced by `blocks/index/` in [0.8.0](https://github.com/bitcoin/bitcoin/blob/master/doc/release-notes/release-notes-0.8.0.md#improvements) | [PR #2231](https://github.com/bitcoin/bitcoin/pull/2231), [`8fdc94cc`](https://github.com/bitcoin/bitcoin/commit/8fdc94cc8f0341e96b1edb3a5b56811c0b20bd15) `coins/` | Unspent transaction output database; replaced by `chainstate/` in 0.8.0 | [PR #2231](https://github.com/bitcoin/bitcoin/pull/2231), [`8fdc94cc`](https://github.com/bitcoin/bitcoin/commit/8fdc94cc8f0341e96b1edb3a5b56811c0b20bd15) `blkindex.dat` | Blockchain index BDB database; replaced by {`chainstate/`, `blocks/index/`, `blocks/revNNNNN.dat`<sup>[\[2\]](#note2)</sup>} in 0.8.0 | [PR #1677](https://github.com/bitcoin/bitcoin/pull/1677) diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 6fc9077e4c..0880f9f581 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -16,6 +16,9 @@ $ FUZZ=process_message src/test/fuzz/fuzz # abort fuzzing using ctrl-c ``` +There is also a runner script to execute all fuzz targets. Refer to +`./test/fuzz/test_runner.py --help` for more details. + ## Fuzzing harnesses and output [`process_message`](https://github.com/bitcoin/bitcoin/blob/master/src/test/fuzz/process_message.cpp) is a fuzzing harness for the [`ProcessMessage(...)` function (`net_processing`)](https://github.com/bitcoin/bitcoin/blob/master/src/net_processing.cpp). The available fuzzing harnesses are found in [`src/test/fuzz/`](https://github.com/bitcoin/bitcoin/tree/master/src/test/fuzz). @@ -83,6 +86,10 @@ INFO: seed corpus: files: 991 min: 1b max: 1858b total: 288291b rss: 150Mb … ``` +## Run without sanitizers for increased throughput + +Fuzzing on a harness compiled with `--with-sanitizers=address,fuzzer,undefined` is good for finding bugs. However, the very slow execution even under libFuzzer will limit the ability to find new coverage. A good approach is to perform occasional long runs without the additional bug-detectors (configure `--with-sanitizers=fuzzer`) and then merge new inputs into a corpus as described in the qa-assets repo (https://github.com/bitcoin-core/qa-assets/blob/main/.github/PULL_REQUEST_TEMPLATE.md). Patience is useful; even with improved throughput, libFuzzer may need days and 10s of millions of executions to reach deep/hard targets. + ## Reproduce a fuzzer crash reported by the CI - `cd` into the `qa-assets` directory and update it with `git pull qa-assets` @@ -247,6 +254,73 @@ $ honggfuzz/honggfuzz --exit_upon_crash --quiet --timeout 4 -n 1 -Q \ -debug ``` +# Fuzzing Bitcoin Core using Eclipser (v1.x) + +## Quickstart guide + +To quickly get started fuzzing Bitcoin Core using [Eclipser v1.x](https://github.com/SoftSec-KAIST/Eclipser/tree/v1.x): + +```sh +$ git clone https://github.com/bitcoin/bitcoin +$ cd bitcoin/ +$ sudo vim /etc/apt/sources.list # Uncomment the lines starting with 'deb-src'. +$ sudo apt-get update +$ sudo apt-get build-dep qemu +$ sudo apt-get install libtool libtool-bin wget automake autoconf bison gdb +``` + +At this point, you must install the .NET core. The process differs, depending on your Linux distribution. +See [this link](https://docs.microsoft.com/en-us/dotnet/core/install/linux) for details. +On ubuntu 20.04, the following should work: + +```sh +$ wget -q https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb +$ sudo dpkg -i packages-microsoft-prod.deb +$ rm packages-microsoft-prod.deb +$ sudo apt-get update +$ sudo apt-get install -y dotnet-sdk-2.1 +``` + +You will also want to make sure Python is installed as `python` for the Eclipser install to succeed. + +```sh +$ git clone https://github.com/SoftSec-KAIST/Eclipser.git +$ cd Eclipser +$ git checkout v1.x +$ make +$ cd .. +$ ./autogen.sh +$ ./configure --enable-fuzz +$ make +$ mkdir -p outputs/ +$ FUZZ=bech32 dotnet Eclipser/build/Eclipser.dll fuzz -p src/test/fuzz/fuzz -t 36000 -o outputs --src stdin +``` + +This will perform 10 hours of fuzzing. + +To make further use of the inputs generated by Eclipser, you +must first decode them: + +```sh +$ dotnet Eclipser/build/Eclipser.dll decode -i outputs/testcase -o decoded_outputs +``` +This will place raw inputs in the directory `decoded_outputs/decoded_stdins`. Crashes are in the `outputs/crashes` directory, and must +be decoded in the same way. + +Fuzzing with Eclipser will likely be much more effective if using an existing corpus: + +```sh +$ git clone https://github.com/bitcoin-core/qa-assets +$ FUZZ=bech32 dotnet Eclipser/build/Eclipser.dll fuzz -p src/test/fuzz/fuzz -t 36000 -i qa-assets/fuzz_seed_corpus/bech32 outputs --src stdin +``` + +Note that fuzzing with Eclipser on certain targets (those that create 'full nodes', e.g. `process_message*`) will, +for now, slowly fill `/tmp/` with improperly cleaned-up files, which will cause spurious crashes. +See [this proposed patch](https://github.com/bitcoin/bitcoin/pull/22472) for more information. + +Read the [Eclipser documentation for v1.x](https://github.com/SoftSec-KAIST/Eclipser/tree/v1.x) for more details on using Eclipser. + + # OSS-Fuzz Bitcoin Core participates in Google's [OSS-Fuzz](https://github.com/google/oss-fuzz/tree/master/projects/bitcoin-core) diff --git a/doc/gitian-building.md b/doc/gitian-building.md deleted file mode 100644 index 3a48f4a0b3..0000000000 --- a/doc/gitian-building.md +++ /dev/null @@ -1,4 +0,0 @@ -Gitian building -================ - -This file was moved to [the Bitcoin Core documentation repository](https://github.com/bitcoin-core/docs/blob/master/gitian-building.md) at [https://github.com/bitcoin-core/docs](https://github.com/bitcoin-core/docs). diff --git a/doc/i2p.md b/doc/i2p.md index 8b4607208a..5f631c11ca 100644 --- a/doc/i2p.md +++ b/doc/i2p.md @@ -10,11 +10,22 @@ started with I2P terminology. ## Run Bitcoin Core with an I2P router (proxy) A running I2P router (proxy) with [SAM](https://geti2p.net/en/docs/api/samv3) -enabled is required (there is an [official one](https://geti2p.net) and -[a few alternatives](https://en.wikipedia.org/wiki/I2P#Routers)). Notice the IP -address and port the SAM proxy is listening to; usually, it is -`127.0.0.1:7656`. Once it is up and running with SAM enabled, use the following -Bitcoin Core options: +enabled is required. Options include: + +- [i2prouter (I2P Router)](https://geti2p.net), the official implementation in + Java +- [i2pd (I2P Daemon)](https://github.com/PurpleI2P/i2pd) + ([documentation](https://i2pd.readthedocs.io/en/latest)), a lighter + alternative in C++ (successfully tested with version 2.23 and up; version 2.36 + or later recommended) +- [i2p-zero](https://github.com/i2p-zero/i2p-zero) +- [other alternatives](https://en.wikipedia.org/wiki/I2P#Routers) + +Note the IP address and port the SAM proxy is listening to; usually, it is +`127.0.0.1:7656`. + +Once an I2P router with SAM enabled is up and running, use the following Bitcoin +Core configuration options: ``` -i2psam=<ip:port> @@ -42,18 +53,41 @@ named `i2p_private_key` in the Bitcoin Core data directory. ## Additional configuration options related to I2P -You may set the `debug=i2p` config logging option to have additional -information in the debug log about your I2P configuration and connections. Run -`bitcoin-cli help logging` for more information. +``` +-debug=i2p +``` + +Set the `debug=i2p` config logging option to see additional information in the +debug log about your I2P configuration and connections. Run `bitcoin-cli help +logging` for more information. + +``` +-onlynet=i2p +``` + +Make outgoing connections only to I2P addresses. Incoming connections are not +affected by this option. It can be specified multiple times to allow multiple +network types, e.g. onlynet=ipv4, onlynet=ipv6, onlynet=onion, onlynet=i2p. -It is possible to restrict outgoing connections in the usual way with -`onlynet=i2p`. I2P support was added to Bitcoin Core in version 22.0 (mid 2021) -and there may be fewer I2P peers than Tor or IP ones. Therefore, using -`onlynet=i2p` alone (without other `onlynet=`) may make a node more susceptible -to [Sybil attacks](https://en.bitcoin.it/wiki/Weaknesses#Sybil_attack). Use +Warning: if you use -onlynet with values other than onion, and the -onion or +-proxy option is set, then outgoing onion connections will still be made; use +-noonion or -onion=0 to disable outbound onion connections in this case. + +I2P support was added to Bitcoin Core in version 22.0 and there may be fewer I2P +peers than Tor or IP ones. Therefore, using I2P alone without other networks may +make a node more susceptible to [Sybil +attacks](https://en.bitcoin.it/wiki/Weaknesses#Sybil_attack). You can use `bitcoin-cli -addrinfo` to see the number of I2P addresses known to your node. -## I2P related information in Bitcoin Core +Another consideration with `onlynet=i2p` is that the initial blocks download +phase when syncing up a new node can be very slow. This phase can be sped up by +using other networks, for instance `onlynet=onion`, at the same time. + +In general, a node can be run with both onion and I2P hidden services (or +any/all of IPv4/IPv6/onion/I2P), which can provide a potential fallback if one +of the networks has issues. + +## I2P-related information in Bitcoin Core There are several ways to see your I2P address in Bitcoin Core: - in the debug log (grep for `AddLocal`, the I2P address ends in `.b32.i2p`) @@ -70,3 +104,18 @@ RPC. Bitcoin Core uses the [SAM v3.1](https://geti2p.net/en/docs/api/samv3) protocol to connect to the I2P network. Any I2P router that supports it can be used. + +## Ports in I2P and Bitcoin Core + +Bitcoin Core uses the [SAM v3.1](https://geti2p.net/en/docs/api/samv3) +protocol. One particularity of SAM v3.1 is that it does not support ports, +unlike newer versions of SAM (v3.2 and up) that do support them and default the +port numbers to 0. From the point of view of peers that use newer versions of +SAM or other protocols that support ports, a SAM v3.1 peer is connecting to them +on port 0, from source port 0. + +To allow future upgrades to newer versions of SAM, Bitcoin Core sets its +listening port to 0 when listening for incoming I2P connections and advertises +its own I2P address with port 0. Furthermore, it will not attempt to connect to +I2P addresses with a non-zero port number because with SAM v3.1 the destination +port (`TO_PORT`) is always set to 0 and is not in the control of Bitcoin Core. diff --git a/doc/managing-wallets.md b/doc/managing-wallets.md new file mode 100644 index 0000000000..aab6d131bd --- /dev/null +++ b/doc/managing-wallets.md @@ -0,0 +1,125 @@ +# Managing the Wallet + +## 1. Backing Up and Restoring The Wallet + +### 1.1 Creating the Wallet + +Since version 0.21, Bitcoin Core no longer has a default wallet. +Wallets can be created with the `createwallet` RPC or with the `Create wallet` GUI menu item. + +In the GUI, the `Create a new wallet` button is displayed on the main screen when there is no wallet loaded. Alternatively, there is the option `File` ->`Create wallet`. + +The following command, for example, creates a descriptor wallet. More information about this command may be found by running `bitcoin-cli help createwallet`. + +``` +$ bitcoin-cli -named createwallet wallet_name="wallet-01" descriptors=true +``` + +The `descriptors` parameter can be omitted if the intention is to create a legacy wallet. For now, the default type is the legacy wallet, but that is expected to change in a future release. + +By default, wallets are created in the `wallets` folder of the data directory, which varies by operating system, as shown below. The user can change the default by using the `-datadir` or `-walletdir` initialization parameters. + +| Operating System | Default wallet directory | +| -----------------|:------------------------------------------------------------| +| Linux | `/home/<user>/.bitcoin/wallets` | +| Windows | `C:\Users\<user>\AppData\Roaming\Bitcoin\wallets` | +| macOS | `/Users/<user>/Library/Application Support/Bitcoin/wallets` | + +### 1.2 Encrypting the Wallet + +The `wallet.dat` file is not encrypted by default and is, therefore, vulnerable if an attacker gains access to the device where the wallet or the backups are stored. + +Wallet encryption may prevent unauthorized access. However, this significantly increases the risk of losing coins due to forgotten passphrases. There is no way to recover a passphrase. This tradeoff should be well thought out by the user. + +Wallet encryption may also not protect against more sophisticated attacks. An attacker can, for example, obtain the password by installing a keylogger on the user's machine. + +After encrypting the wallet or changing the passphrase, a new backup needs to be created immediately. The reason is that the keypool is flushed and a new HD seed is generated after encryption. Any bitcoins received by the new seed cannot be recovered from the previous backups. + +The wallet's private key may be encrypted with the following command: + +``` +$ bitcoin-cli -rpcwallet="wallet-01" encryptwallet "passphrase" +``` + +Once encrypted, the passphrase can be changed with the `walletpassphrasechange` command. + +``` +$ bitcoin-cli -rpcwallet="wallet-01" walletpassphrasechange "oldpassphrase" "newpassphrase" +``` + +The argument passed to `-rpcwallet` is the name of the wallet to be encrypted. + +Only the wallet's private key is encrypted. All other wallet information, such as transactions, is still visible. + +The wallet's private key can also be encrypted in the `createwallet` command via the `passphrase` argument: + +``` +$ bitcoin-cli -named createwallet wallet_name="wallet-01" descriptors=true passphrase="passphrase" +``` + +Note that if the passphrase is lost, all the coins in the wallet will also be lost forever. + +### 1.3 Unlocking the Wallet + +If the wallet is encrypted and the user tries any operation related to private keys, such as sending bitcoins, an error message will be displayed. + +``` +$ bitcoin-cli -rpcwallet="wallet-01" sendtoaddress "tb1qw508d6qejxtdg4y5r3zarvary0c5xw7kxpjzsx" 0.01 +error code: -13 +error message: +Error: Please enter the wallet passphrase with walletpassphrase first. +``` + +To unlock the wallet and allow it to run these operations, the `walletpassphrase` RPC is required. + +This command takes the passphrase and an argument called `timeout`, which specifies the time in seconds that the wallet decryption key is stored in memory. After this period expires, the user needs to execute this RPC again. + +``` +$ bitcoin-cli -rpcwallet="wallet-01" walletpassphrase "passphrase" 120 +``` + +In the GUI, there is no specific menu item to unlock the wallet. When the user sends bitcoins, the passphrase will be prompted automatically. + +### 1.4 Backing Up the Wallet + +To backup the wallet, the `backupwallet` RPC or the `Backup Wallet` GUI menu item must be used to ensure the file is in a safe state when the copy is made. + +In the RPC, the destination parameter must include the name of the file. Otherwise, the command will return an error message like "Error: Wallet backup failed!" for descriptor wallets. If it is a legacy wallet, it will be copied and a file will be created with the default file name `wallet.dat`. + +``` +$ bitcoin-cli -rpcwallet="wallet-01" backupwallet /home/node01/Backups/backup-01.dat +``` + +In the GUI, the wallet is selected in the `Wallet` drop-down list in the upper right corner. If this list is not present, the wallet can be loaded in `File` ->`Open wallet` if necessary. Then, the backup can be done in `File` -> `Backup Wallet...`. + +This backup file can be stored on one or multiple offline devices, which must be reliable enough to work in an emergency and be malware free. Backup files can be regularly tested to avoid problems in the future. + +If the computer has malware, it can compromise the wallet when recovering the backup file. One way to minimize this is to not connect the backup to an online device. + +If both the wallet and all backups are lost for any reason, the bitcoins related to this wallet will become permanently inaccessible. + +### 1.5 Backup Frequency + +The original Bitcoin Core wallet was a collection of unrelated private keys. If a non-HD wallet had received funds to an address and then was restored from a backup made before the address was generated, then any funds sent to that address would have been lost because there was no deterministic mechanism to derive the address again. + +Bitcoin Core [version 0.13](https://github.com/bitcoin/bitcoin/blob/master/doc/release-notes/release-notes-0.13.0.md) introduced HD wallets with deterministic key derivation. With HD wallets, users no longer lose funds when restoring old backups because all addresses are derived from the HD wallet seed. + +This means that a single backup is enough to recover the coins at any time. It is still recommended to make regular backups (once a week) or after a significant number of new transactions to maintain the metadata, such as labels. Metadata cannot be retrieved from a blockchain rescan, so if the backup is too old, the metadata will be lost forever. + +Wallets created before version 0.13 are not HD and must be backed up every 100 keys used since the previous backup, or even more often to maintain the metadata. + +### 1.6 Restoring the Wallet From a Backup + +To restore a wallet, the `restorewallet` RPC must be used. + +``` +$ bitcoin-cli restorewallet "restored-wallet" /home/node01/Backups/backup-01.dat +``` + +After that, `getwalletinfo` can be used to check if the wallet has been fully restored. + +``` +$ bitcoin-cli -rpcwallet="restored-wallet" getwalletinfo +``` + +The restored wallet can also be loaded in the GUI via `File` ->`Open wallet`.
\ No newline at end of file diff --git a/doc/release-notes-22539.md b/doc/release-notes-22539.md new file mode 100644 index 0000000000..9f56071451 --- /dev/null +++ b/doc/release-notes-22539.md @@ -0,0 +1,8 @@ +Notable changes +=============== + +P2P and network changes +----------------------- + +- Fee estimation now takes the feerate of replacement (RBF) transactions into + account. diff --git a/doc/release-notes.md b/doc/release-notes.md index dc28ccb9ed..81e79dd3a9 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -1,5 +1,3 @@ -# Release notes now being edited on https://github.com/bitcoin-core/bitcoin-devwiki/wiki/22.0-Release-Notes-draft - *After branching off for a major version release of Bitcoin Core, use this template to create the initial release notes draft.* @@ -8,7 +6,7 @@ template to create the initial release notes draft.* for the process.* *Create the draft, named* "*version* Release Notes Draft" -*(e.g. "0.20.0 Release Notes Draft"), as a collaborative wiki in:* +*(e.g. "22.0 Release Notes Draft"), as a collaborative wiki in:* https://github.com/bitcoin-core/bitcoin-devwiki/wiki/ @@ -48,94 +46,44 @@ Compatibility ============== Bitcoin Core is supported and extensively tested on operating systems -using the Linux kernel, macOS 10.14+, and Windows 7 and newer. Bitcoin +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin Core should also work on most other Unix-like systems but is not as frequently tested on them. It is not recommended to use Bitcoin Core on unsupported systems. -From Bitcoin Core 22.0 onwards, macOS versions earlier than 10.14 are no longer supported. - Notable changes =============== P2P and network changes ----------------------- -- This release removes support for Tor version 2 hidden services in favor of Tor - v3 only, as the Tor network [dropped support for Tor - v2](https://blog.torproject.org/v2-deprecation-timeline) with the release of - Tor version 0.4.6. Henceforth, Bitcoin Core ignores Tor v2 addresses; it - neither rumors them over the network to other peers, nor stores them in memory - or to `peers.dat`. (#22050) +- A bitcoind node will no longer rumour addresses to inbound peers by default. + They will become eligible for address gossip after sending an ADDR, ADDRV2, + or GETADDR message. (#21528) + +Rescan startup parameter removed +-------------------------------- -- Added NAT-PMP port mapping support via - [`libnatpmp`](https://miniupnp.tuxfamily.org/libnatpmp.html). (#18077) +The `-rescan` startup parameter has been removed. Wallets which require +rescanning due to corruption will still be rescanned on startup. +Otherwise, please use the `rescanblockchain` RPC to trigger a rescan. (#23123) Updated RPCs ------------ -- Due to [BIP 350](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki) - being implemented, behavior for all RPCs that accept addresses is changed when - a native witness version 1 (or higher) is passed. These now require a Bech32m - encoding instead of a Bech32 one, and Bech32m encoding will be used for such - addresses in RPC output as well. No version 1 addresses should be created - for mainnet until consensus rules are adopted that give them meaning - (e.g. through [BIP 341](https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki)). - Once that happens, Bech32m is expected to be used for them, so this shouldn't - affect any production systems, but may be observed on other networks where such - addresses already have meaning (like signet). (#20861) - -- The `getpeerinfo` RPC returns two new boolean fields, `bip152_hb_to` and - `bip152_hb_from`, that respectively indicate whether we selected a peer to be - in compact blocks high-bandwidth mode or whether a peer selected us as a - compact blocks high-bandwidth peer. High-bandwidth peers send new block - announcements via a `cmpctblock` message rather than the usual inv/headers - announcements. See BIP 152 for more details. (#19776) - -- `getpeerinfo` no longer returns the following fields: `addnode`, `banscore`, - and `whitelisted`, which were previously deprecated in 0.21. Instead of - `addnode`, the `connection_type` field returns manual. Instead of - `whitelisted`, the `permissions` field indicates if the peer has special - privileges. The `banscore` field has simply been removed. (#20755) - -- The following RPCs: `gettxout`, `getrawtransaction`, `decoderawtransaction`, - `decodescript`, `gettransaction`, and REST endpoints: `/rest/tx`, - `/rest/getutxos`, `/rest/block` deprecated the following fields (which are no - longer returned in the responses by default): `addresses`, `reqSigs`. - The `-deprecatedrpc=addresses` flag must be passed for these fields to be - included in the RPC response. This flag/option will be available only for this major release, after which - the deprecation will be removed entirely. Note that these fields are attributes of - the `scriptPubKey` object returned in the RPC response. However, in the response - of `decodescript` these fields are top-level attributes, and included again as attributes - of the `scriptPubKey` object. (#20286) - -- When creating a hex-encoded bitcoin transaction using the `bitcoin-tx` utility - with the `-json` option set, the following fields: `addresses`, `reqSigs` are no longer - returned in the tx output of the response. (#20286) - -- The `listbanned` RPC now returns two new numeric fields: `ban_duration` and `time_remaining`. - Respectively, these new fields indicate the duration of a ban and the time remaining until a ban expires, - both in seconds. Additionally, the `ban_created` field is repositioned to come before `banned_until`. (#21602) - -- The `getnodeaddresses` RPC now returns a "network" field indicating the - network type (ipv4, ipv6, onion, or i2p) for each address. (#21594) - -- `getnodeaddresses` now also accepts a "network" argument (ipv4, ipv6, onion, - or i2p) to return only addresses of the specified network. (#21843) - -- The `testmempoolaccept` RPC now accepts multiple transactions (still experimental at the moment, - API may be unstable). This is intended for testing transaction packages with dependency - relationships; it is not recommended for batch-validating independent transactions. In addition to - mempool policy, package policies apply: the list cannot contain more than 25 transactions or have a - total size exceeding 101K virtual bytes, and cannot conflict with (spend the same inputs as) each other or - the mempool, even if it would be a valid BIP125 replace-by-fee. There are some known limitations to - the accuracy of the test accept: it's possible for `testmempoolaccept` to return "allowed"=True for a - group of transactions, but "too-long-mempool-chain" if they are actually submitted. (#20833) - -- `addmultisigaddress` and `createmultisig` now support up to 20 keys for - Segwit addresses. (#20867) - -Changes to Wallet or GUI related RPCs can be found in the GUI or Wallet section below. +- The `-deprecatedrpc=addresses` configuration option has been removed. RPCs + `gettxout`, `getrawtransaction`, `decoderawtransaction`, `decodescript`, + `gettransaction verbose=true` and REST endpoints `/rest/tx`, `/rest/getutxos`, + `/rest/block` no longer return the `addresses` and `reqSigs` fields, which + were previously deprecated in 22.0. (#22650) + +- `listunspent` now includes `ancestorcount`, `ancestorsize`, and + `ancestorfees` for each transaction output that is still in the mempool. + (#12677) + +- `lockunspent` now optionally takes a third parameter, `persistent`, which + causes the lock to be written persistently to the wallet database. This + allows UTXOs to remain locked even after node restarts or crashes. (#23065) New RPCs -------- @@ -143,76 +91,61 @@ New RPCs Build System ------------ +Files +----- + +* On startup, the list of banned hosts and networks (via `setban` RPC) in + `banlist.dat` is ignored and only `banlist.json` is considered. Bitcoin Core + version 22.x is the only version that can read `banlist.dat` and also write + it to `banlist.json`. If `banlist.json` already exists, version 22.x will not + try to translate the `banlist.dat` into json. After an upgrade, `listbanned` + can be used to double check the parsed entries. (#22570) + New settings ------------ -- The `-natpmp` option has been added to use NAT-PMP to map the listening port. - If both UPnP and NAT-PMP are enabled, a successful allocation from UPnP - prevails over one from NAT-PMP. (#18077) - Updated settings ---------------- -Changes to Wallet or GUI related settings can be found in the GUI or Wallet section below. - -- Passing an invalid `-rpcauth` argument now cause bitcoind to fail to start. (#20461) +- In previous releases, the meaning of the command line option + `-persistmempool` (without a value provided) incorrectly disabled mempool + persistence. `-persistmempool` is now treated like other boolean options to + mean `-persistmempool=1`. Passing `-persistmempool=0`, `-persistmempool=1` + and `-nopersistmempool` is unaffected. (#23061) Tools and Utilities ------------------- -- A new CLI `-addrinfo` command returns the number of addresses known to the - node per network type (including Tor v2 versus v3) and total. This can be - useful to see if the node knows enough addresses in a network to use options - like `-onlynet=<network>` or to upgrade to this release of Bitcoin Core 22.0 - that supports Tor v3 only. (#21595) +- Update `-getinfo` to return data in a user-friendly format that also reduces vertical space. (#21832) -- A new `-rpcwaittimeout` argument to `bitcoin-cli` sets the timeout - in seconds to use with `-rpcwait`. If the timeout expires, - `bitcoin-cli` will report a failure. (#21056) +- CLI `-addrinfo` now returns a single field for the number of `onion` addresses + known to the node instead of separate `torv2` and `torv3` fields, as support + for Tor V2 addresses was removed from Bitcoin Core in 22.0. (#22544) Wallet ------ -- A new `listdescriptors` RPC is available to inspect the contents of descriptor-enabled wallets. - The RPC returns public versions of all imported descriptors, including their timestamp and flags. - For ranged descriptors, it also returns the range boundaries and the next index to generate addresses from. (#20226) - -- The `bumpfee` RPC is not available with wallets that have private keys - disabled. `psbtbumpfee` can be used instead. (#20891) - -- The `fundrawtransaction`, `send` and `walletcreatefundedpsbt` RPCs now support an `include_unsafe` option - that when `true` allows using unsafe inputs to fund the transaction. - Note that the resulting transaction may become invalid if one of the unsafe inputs disappears. - If that happens, the transaction must be funded with different inputs and republished. (#21359) - -- We now support up to 20 keys in `multi()` and `sortedmulti()` descriptors - under `wsh()`. (#20867) - GUI changes ----------- +- UTXOs which are locked via the GUI are now stored persistently in the + wallet database, so are not lost on node shutdown or crash. (#23065) + Low-level changes ================= RPC --- -- The RPC server can process a limited number of simultaneous RPC requests. - Previously, if this limit was exceeded, the RPC server would respond with - [status code 500 (`HTTP_INTERNAL_SERVER_ERROR`)](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#5xx_server_errors). - Now it returns status code 503 (`HTTP_SERVICE_UNAVAILABLE`). (#18335) - -- Error codes have been updated to be more accurate for the following error cases (#18466): - - `signmessage` now returns RPC_INVALID_ADDRESS_OR_KEY (-5) if the - passed address is invalid. Previously returned RPC_TYPE_ERROR (-3). - - `verifymessage` now returns RPC_INVALID_ADDRESS_OR_KEY (-5) if the - passed address is invalid. Previously returned RPC_TYPE_ERROR (-3). - - `verifymessage` now returns RPC_TYPE_ERROR (-3) if the passed signature - is malformed. Previously returned RPC_INVALID_ADDRESS_OR_KEY (-5). +- `getblockchaininfo` now returns a new `time` field, that provides the chain tip time. (#22407) Tests ----- +- For the `regtest` network the activation heights of several softforks were + set to block height 1. They can be changed by the runtime setting + `-testactivationheight=name@height`. (#22818) + Credits ======= diff --git a/doc/release-notes/release-notes-22.0.md b/doc/release-notes/release-notes-22.0.md new file mode 100644 index 0000000000..972c91aa6f --- /dev/null +++ b/doc/release-notes/release-notes-22.0.md @@ -0,0 +1,1163 @@ +22.0 Release Notes +================== + +Bitcoin Core version 22.0 is now available from: + + <https://bitcoincore.org/bin/bitcoin-core-22.0/> + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + <https://github.com/bitcoin/bitcoin/issues> + +To receive security and update notifications, please subscribe to: + + <https://bitcoincore.org/en/list/announcements/join/> + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.14+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +From Bitcoin Core 22.0 onwards, macOS versions earlier than 10.14 are no longer supported. + +Notable changes +=============== + +P2P and network changes +----------------------- +- Added support for running Bitcoin Core as an + [I2P (Invisible Internet Project)](https://en.wikipedia.org/wiki/I2P) service + and connect to such services. See [i2p.md](https://github.com/bitcoin/bitcoin/blob/22.x/doc/i2p.md) for details. (#20685) +- This release removes support for Tor version 2 hidden services in favor of Tor + v3 only, as the Tor network [dropped support for Tor + v2](https://blog.torproject.org/v2-deprecation-timeline) with the release of + Tor version 0.4.6. Henceforth, Bitcoin Core ignores Tor v2 addresses; it + neither rumors them over the network to other peers, nor stores them in memory + or to `peers.dat`. (#22050) + +- Added NAT-PMP port mapping support via + [`libnatpmp`](https://miniupnp.tuxfamily.org/libnatpmp.html). (#18077) + +New and Updated RPCs +-------------------- + +- Due to [BIP 350](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki) + being implemented, behavior for all RPCs that accept addresses is changed when + a native witness version 1 (or higher) is passed. These now require a Bech32m + encoding instead of a Bech32 one, and Bech32m encoding will be used for such + addresses in RPC output as well. No version 1 addresses should be created + for mainnet until consensus rules are adopted that give them meaning + (as will happen through [BIP 341](https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki)). + Once that happens, Bech32m is expected to be used for them, so this shouldn't + affect any production systems, but may be observed on other networks where such + addresses already have meaning (like signet). (#20861) + +- The `getpeerinfo` RPC returns two new boolean fields, `bip152_hb_to` and + `bip152_hb_from`, that respectively indicate whether we selected a peer to be + in compact blocks high-bandwidth mode or whether a peer selected us as a + compact blocks high-bandwidth peer. High-bandwidth peers send new block + announcements via a `cmpctblock` message rather than the usual inv/headers + announcements. See BIP 152 for more details. (#19776) + +- `getpeerinfo` no longer returns the following fields: `addnode`, `banscore`, + and `whitelisted`, which were previously deprecated in 0.21. Instead of + `addnode`, the `connection_type` field returns manual. Instead of + `whitelisted`, the `permissions` field indicates if the peer has special + privileges. The `banscore` field has simply been removed. (#20755) + +- The following RPCs: `gettxout`, `getrawtransaction`, `decoderawtransaction`, + `decodescript`, `gettransaction`, and REST endpoints: `/rest/tx`, + `/rest/getutxos`, `/rest/block` deprecated the following fields (which are no + longer returned in the responses by default): `addresses`, `reqSigs`. + The `-deprecatedrpc=addresses` flag must be passed for these fields to be + included in the RPC response. This flag/option will be available only for this major release, after which + the deprecation will be removed entirely. Note that these fields are attributes of + the `scriptPubKey` object returned in the RPC response. However, in the response + of `decodescript` these fields are top-level attributes, and included again as attributes + of the `scriptPubKey` object. (#20286) + +- When creating a hex-encoded bitcoin transaction using the `bitcoin-tx` utility + with the `-json` option set, the following fields: `addresses`, `reqSigs` are no longer + returned in the tx output of the response. (#20286) + +- The `listbanned` RPC now returns two new numeric fields: `ban_duration` and `time_remaining`. + Respectively, these new fields indicate the duration of a ban and the time remaining until a ban expires, + both in seconds. Additionally, the `ban_created` field is repositioned to come before `banned_until`. (#21602) + +- The `setban` RPC can ban onion addresses again. This fixes a regression introduced in version 0.21.0. (#20852) + +- The `getnodeaddresses` RPC now returns a "network" field indicating the + network type (ipv4, ipv6, onion, or i2p) for each address. (#21594) + +- `getnodeaddresses` now also accepts a "network" argument (ipv4, ipv6, onion, + or i2p) to return only addresses of the specified network. (#21843) + +- The `testmempoolaccept` RPC now accepts multiple transactions (still experimental at the moment, + API may be unstable). This is intended for testing transaction packages with dependency + relationships; it is not recommended for batch-validating independent transactions. In addition to + mempool policy, package policies apply: the list cannot contain more than 25 transactions or have a + total size exceeding 101K virtual bytes, and cannot conflict with (spend the same inputs as) each other or + the mempool, even if it would be a valid BIP125 replace-by-fee. There are some known limitations to + the accuracy of the test accept: it's possible for `testmempoolaccept` to return "allowed"=True for a + group of transactions, but "too-long-mempool-chain" if they are actually submitted. (#20833) + +- `addmultisigaddress` and `createmultisig` now support up to 20 keys for + Segwit addresses. (#20867) + +Changes to Wallet or GUI related RPCs can be found in the GUI or Wallet section below. + +Build System +------------ + +- Release binaries are now produced using the new `guix`-based build system. + The [/doc/release-process.md](/doc/release-process.md) document has been updated accordingly. + +Files +----- + +- The list of banned hosts and networks (via `setban` RPC) is now saved on disk + in JSON format in `banlist.json` instead of `banlist.dat`. `banlist.dat` is + only read on startup if `banlist.json` is not present. Changes are only written to the new + `banlist.json`. A future version of Bitcoin Core may completely ignore + `banlist.dat`. (#20966) + +New settings +------------ + +- The `-natpmp` option has been added to use NAT-PMP to map the listening port. + If both UPnP and NAT-PMP are enabled, a successful allocation from UPnP + prevails over one from NAT-PMP. (#18077) + +Updated settings +---------------- + +Changes to Wallet or GUI related settings can be found in the GUI or Wallet section below. + +- Passing an invalid `-rpcauth` argument now cause bitcoind to fail to start. (#20461) + +Tools and Utilities +------------------- + +- A new CLI `-addrinfo` command returns the number of addresses known to the + node per network type (including Tor v2 versus v3) and total. This can be + useful to see if the node knows enough addresses in a network to use options + like `-onlynet=<network>` or to upgrade to this release of Bitcoin Core 22.0 + that supports Tor v3 only. (#21595) + +- A new `-rpcwaittimeout` argument to `bitcoin-cli` sets the timeout + in seconds to use with `-rpcwait`. If the timeout expires, + `bitcoin-cli` will report a failure. (#21056) + +Wallet +------ + +- External signers such as hardware wallets can now be used through the new RPC methods `enumeratesigners` and `displayaddress`. Support is also added to the `send` RPC call. This feature is experimental. See [external-signer.md](https://github.com/bitcoin/bitcoin/blob/22.x/doc/external-signer.md) for details. (#16546) + +- A new `listdescriptors` RPC is available to inspect the contents of descriptor-enabled wallets. + The RPC returns public versions of all imported descriptors, including their timestamp and flags. + For ranged descriptors, it also returns the range boundaries and the next index to generate addresses from. (#20226) + +- The `bumpfee` RPC is not available with wallets that have private keys + disabled. `psbtbumpfee` can be used instead. (#20891) + +- The `fundrawtransaction`, `send` and `walletcreatefundedpsbt` RPCs now support an `include_unsafe` option + that when `true` allows using unsafe inputs to fund the transaction. + Note that the resulting transaction may become invalid if one of the unsafe inputs disappears. + If that happens, the transaction must be funded with different inputs and republished. (#21359) + +- We now support up to 20 keys in `multi()` and `sortedmulti()` descriptors + under `wsh()`. (#20867) + +- Taproot descriptors can be imported into the wallet only after activation has occurred on the network (e.g. mainnet, testnet, signet) in use. See [descriptors.md](https://github.com/bitcoin/bitcoin/blob/22.x/doc/descriptors.md) for supported descriptors. + +GUI changes +----------- + +- External signers such as hardware wallets can now be used. These require an external tool such as [HWI](https://github.com/bitcoin-core/HWI) to be installed and configured under Options -> Wallet. When creating a new wallet a new option "External signer" will appear in the dialog. If the device is detected, its name is suggested as the wallet name. The watch-only keys are then automatically imported. Receive addresses can be verified on the device. The send dialog will automatically use the connected device. This feature is experimental and the UI may freeze for a few seconds when performing these actions. + +Low-level changes +================= + +RPC +--- + +- The RPC server can process a limited number of simultaneous RPC requests. + Previously, if this limit was exceeded, the RPC server would respond with + [status code 500 (`HTTP_INTERNAL_SERVER_ERROR`)](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#5xx_server_errors). + Now it returns status code 503 (`HTTP_SERVICE_UNAVAILABLE`). (#18335) + +- Error codes have been updated to be more accurate for the following error cases (#18466): + - `signmessage` now returns RPC_INVALID_ADDRESS_OR_KEY (-5) if the + passed address is invalid. Previously returned RPC_TYPE_ERROR (-3). + - `verifymessage` now returns RPC_INVALID_ADDRESS_OR_KEY (-5) if the + passed address is invalid. Previously returned RPC_TYPE_ERROR (-3). + - `verifymessage` now returns RPC_TYPE_ERROR (-3) if the passed signature + is malformed. Previously returned RPC_INVALID_ADDRESS_OR_KEY (-5). + +Tests +----- + +22.0 change log +=============== + +A detailed list of changes in this version follows. To keep the list to a manageable length, small refactors and typo fixes are not included, and similar changes are sometimes condensed into one line. + +### Consensus +- bitcoin/bitcoin#19438 Introduce deploymentstatus (ajtowns) +- bitcoin/bitcoin#20207 Follow-up extra comments on taproot code and tests (sipa) +- bitcoin/bitcoin#21330 Deal with missing data in signature hashes more consistently (sipa) + +### Policy +- bitcoin/bitcoin#18766 Disable fee estimation in blocksonly mode (by removing the fee estimates global) (darosior) +- bitcoin/bitcoin#20497 Add `MAX_STANDARD_SCRIPTSIG_SIZE` to policy (sanket1729) +- bitcoin/bitcoin#20611 Move `TX_MAX_STANDARD_VERSION` to policy (MarcoFalke) + +### Mining +- bitcoin/bitcoin#19937, bitcoin/bitcoin#20923 Signet mining utility (ajtowns) + +### Block and transaction handling +- bitcoin/bitcoin#14501 Fix possible data race when committing block files (luke-jr) +- bitcoin/bitcoin#15946 Allow maintaining the blockfilterindex when using prune (jonasschnelli) +- bitcoin/bitcoin#18710 Add local thread pool to CCheckQueue (hebasto) +- bitcoin/bitcoin#19521 Coinstats Index (fjahr) +- bitcoin/bitcoin#19806 UTXO snapshot activation (jamesob) +- bitcoin/bitcoin#19905 Remove dead CheckForkWarningConditionsOnNewFork (MarcoFalke) +- bitcoin/bitcoin#19935 Move SaltedHashers to separate file and add some new ones (achow101) +- bitcoin/bitcoin#20054 Remove confusing and useless "unexpected version" warning (MarcoFalke) +- bitcoin/bitcoin#20519 Handle rename failure in `DumpMempool(…)` by using the `RenameOver(…)` return value (practicalswift) +- bitcoin/bitcoin#20749, bitcoin/bitcoin#20750, bitcoin/bitcoin#21055, bitcoin/bitcoin#21270, bitcoin/bitcoin#21525, bitcoin/bitcoin#21391, bitcoin/bitcoin#21767, bitcoin/bitcoin#21866 Prune `g_chainman` usage (dongcarl) +- bitcoin/bitcoin#20833 rpc/validation: enable packages through testmempoolaccept (glozow) +- bitcoin/bitcoin#20834 Locks and docs in ATMP and CheckInputsFromMempoolAndCache (glozow) +- bitcoin/bitcoin#20854 Remove unnecessary try-block (amitiuttarwar) +- bitcoin/bitcoin#20868 Remove redundant check on pindex (jarolrod) +- bitcoin/bitcoin#20921 Don't try to invalidate genesis block in CChainState::InvalidateBlock (theStack) +- bitcoin/bitcoin#20972 Locks: Annotate CTxMemPool::check to require `cs_main` (dongcarl) +- bitcoin/bitcoin#21009 Remove RewindBlockIndex logic (dhruv) +- bitcoin/bitcoin#21025 Guard chainman chainstates with `cs_main` (dongcarl) +- bitcoin/bitcoin#21202 Two small clang lock annotation improvements (amitiuttarwar) +- bitcoin/bitcoin#21523 Run VerifyDB on all chainstates (jamesob) +- bitcoin/bitcoin#21573 Update libsecp256k1 subtree to latest master (sipa) +- bitcoin/bitcoin#21582, bitcoin/bitcoin#21584, bitcoin/bitcoin#21585 Fix assumeutxo crashes (MarcoFalke) +- bitcoin/bitcoin#21681 Fix ActivateSnapshot to use hardcoded nChainTx (jamesob) +- bitcoin/bitcoin#21796 index: Avoid async shutdown on init error (MarcoFalke) +- bitcoin/bitcoin#21946 Document and test lack of inherited signaling in RBF policy (ariard) +- bitcoin/bitcoin#22084 Package testmempoolaccept followups (glozow) +- bitcoin/bitcoin#22102 Remove `Warning:` from warning message printed for unknown new rules (prayank23) +- bitcoin/bitcoin#22112 Force port 0 in I2P (vasild) +- bitcoin/bitcoin#22135 CRegTestParams: Use `args` instead of `gArgs` (kiminuo) +- bitcoin/bitcoin#22146 Reject invalid coin height and output index when loading assumeutxo (MarcoFalke) +- bitcoin/bitcoin#22253 Distinguish between same tx and same-nonwitness-data tx in mempool (glozow) +- bitcoin/bitcoin#22261 Two small fixes to node broadcast logic (jnewbery) +- bitcoin/bitcoin#22415 Make `m_mempool` optional in CChainState (jamesob) +- bitcoin/bitcoin#22499 Update assumed chain params (sriramdvt) +- bitcoin/bitcoin#22589 net, doc: update I2P hardcoded seeds and docs for 22.0 (jonatack) + +### P2P protocol and network code +- bitcoin/bitcoin#18077 Add NAT-PMP port forwarding support (hebasto) +- bitcoin/bitcoin#18722 addrman: improve performance by using more suitable containers (vasild) +- bitcoin/bitcoin#18819 Replace `cs_feeFilter` with simple std::atomic (MarcoFalke) +- bitcoin/bitcoin#19203 Add regression fuzz harness for CVE-2017-18350. Add FuzzedSocket (practicalswift) +- bitcoin/bitcoin#19288 fuzz: Add fuzzing harness for TorController (practicalswift) +- bitcoin/bitcoin#19415 Make DNS lookup mockable, add fuzzing harness (practicalswift) +- bitcoin/bitcoin#19509 Per-Peer Message Capture (troygiorshev) +- bitcoin/bitcoin#19763 Don't try to relay to the address' originator (vasild) +- bitcoin/bitcoin#19771 Replace enum CConnMan::NumConnections with enum class ConnectionDirection (luke-jr) +- bitcoin/bitcoin#19776 net, rpc: expose high bandwidth mode state via getpeerinfo (theStack) +- bitcoin/bitcoin#19832 Put disconnecting logs into BCLog::NET category (hebasto) +- bitcoin/bitcoin#19858 Periodically make block-relay connections and sync headers (sdaftuar) +- bitcoin/bitcoin#19884 No delay in adding fixed seeds if -dnsseed=0 and peers.dat is empty (dhruv) +- bitcoin/bitcoin#20079 Treat handshake misbehavior like unknown message (MarcoFalke) +- bitcoin/bitcoin#20138 Assume that SetCommonVersion is called at most once per peer (MarcoFalke) +- bitcoin/bitcoin#20162 p2p: declare Announcement::m_state as uint8_t, add getter/setter (jonatack) +- bitcoin/bitcoin#20197 Protect onions in AttemptToEvictConnection(), add eviction protection test coverage (jonatack) +- bitcoin/bitcoin#20210 assert `CNode::m_inbound_onion` is inbound in ctor, add getter, unit tests (jonatack) +- bitcoin/bitcoin#20228 addrman: Make addrman a top-level component (jnewbery) +- bitcoin/bitcoin#20234 Don't bind on 0.0.0.0 if binds are restricted to Tor (vasild) +- bitcoin/bitcoin#20477 Add unit testing of node eviction logic (practicalswift) +- bitcoin/bitcoin#20516 Well-defined CAddress disk serialization, and addrv2 anchors.dat (sipa) +- bitcoin/bitcoin#20557 addrman: Fix new table bucketing during unserialization (jnewbery) +- bitcoin/bitcoin#20561 Periodically clear `m_addr_known` (sdaftuar) +- bitcoin/bitcoin#20599 net processing: Tolerate sendheaders and sendcmpct messages before verack (jnewbery) +- bitcoin/bitcoin#20616 Check CJDNS address is valid (lontivero) +- bitcoin/bitcoin#20617 Remove `m_is_manual_connection` from CNodeState (ariard) +- bitcoin/bitcoin#20624 net processing: Remove nStartingHeight check from block relay (jnewbery) +- bitcoin/bitcoin#20651 Make p2p recv buffer timeout 20 minutes for all peers (jnewbery) +- bitcoin/bitcoin#20661 Only select from addrv2-capable peers for torv3 address relay (sipa) +- bitcoin/bitcoin#20685 Add I2P support using I2P SAM (vasild) +- bitcoin/bitcoin#20690 Clean up logging of outbound connection type (sdaftuar) +- bitcoin/bitcoin#20721 Move ping data to `net_processing` (jnewbery) +- bitcoin/bitcoin#20724 Cleanup of -debug=net log messages (ajtowns) +- bitcoin/bitcoin#20747 net processing: Remove dropmessagestest (jnewbery) +- bitcoin/bitcoin#20764 cli -netinfo peer connections dashboard updates 🎄 ✨ (jonatack) +- bitcoin/bitcoin#20788 add RAII socket and use it instead of bare SOCKET (vasild) +- bitcoin/bitcoin#20791 remove unused legacyWhitelisted in AcceptConnection() (jonatack) +- bitcoin/bitcoin#20816 Move RecordBytesSent() call out of `cs_vSend` lock (jnewbery) +- bitcoin/bitcoin#20845 Log to net debug in MaybeDiscourageAndDisconnect except for noban and manual peers (MarcoFalke) +- bitcoin/bitcoin#20864 Move SocketSendData lock annotation to header (MarcoFalke) +- bitcoin/bitcoin#20965 net, rpc: return `NET_UNROUTABLE` as `not_publicly_routable`, automate helps (jonatack) +- bitcoin/bitcoin#20966 banman: save the banlist in a JSON format on disk (vasild) +- bitcoin/bitcoin#21015 Make all of `net_processing` (and some of net) use std::chrono types (dhruv) +- bitcoin/bitcoin#21029 bitcoin-cli: Correct docs (no "generatenewaddress" exists) (luke-jr) +- bitcoin/bitcoin#21148 Split orphan handling from `net_processing` into txorphanage (ajtowns) +- bitcoin/bitcoin#21162 Net Processing: Move RelayTransaction() into PeerManager (jnewbery) +- bitcoin/bitcoin#21167 make `CNode::m_inbound_onion` public, initialize explicitly (jonatack) +- bitcoin/bitcoin#21186 net/net processing: Move addr data into `net_processing` (jnewbery) +- bitcoin/bitcoin#21187 Net processing: Only call PushAddress() from `net_processing` (jnewbery) +- bitcoin/bitcoin#21198 Address outstanding review comments from PR20721 (jnewbery) +- bitcoin/bitcoin#21222 log: Clarify log message when file does not exist (MarcoFalke) +- bitcoin/bitcoin#21235 Clarify disconnect log message in ProcessGetBlockData, remove send bool (MarcoFalke) +- bitcoin/bitcoin#21236 Net processing: Extract `addr` send functionality into MaybeSendAddr() (jnewbery) +- bitcoin/bitcoin#21261 update inbound eviction protection for multiple networks, add I2P peers (jonatack) +- bitcoin/bitcoin#21328 net, refactor: pass uint16 CService::port as uint16 (jonatack) +- bitcoin/bitcoin#21387 Refactor sock to add I2P fuzz and unit tests (vasild) +- bitcoin/bitcoin#21395 Net processing: Remove unused CNodeState.address member (jnewbery) +- bitcoin/bitcoin#21407 i2p: limit the size of incoming messages (vasild) +- bitcoin/bitcoin#21506 p2p, refactor: make NetPermissionFlags an enum class (jonatack) +- bitcoin/bitcoin#21509 Don't send FEEFILTER in blocksonly mode (mzumsande) +- bitcoin/bitcoin#21560 Add Tor v3 hardcoded seeds (laanwj) +- bitcoin/bitcoin#21563 Restrict period when `cs_vNodes` mutex is locked (hebasto) +- bitcoin/bitcoin#21564 Avoid calling getnameinfo when formatting IPv4 addresses in CNetAddr::ToStringIP (practicalswift) +- bitcoin/bitcoin#21631 i2p: always check the return value of Sock::Wait() (vasild) +- bitcoin/bitcoin#21644 p2p, bugfix: use NetPermissions::HasFlag() in CConnman::Bind() (jonatack) +- bitcoin/bitcoin#21659 flag relevant Sock methods with [[nodiscard]] (vasild) +- bitcoin/bitcoin#21750 remove unnecessary check of `CNode::cs_vSend` (vasild) +- bitcoin/bitcoin#21756 Avoid calling `getnameinfo` when formatting IPv6 addresses in `CNetAddr::ToStringIP` (practicalswift) +- bitcoin/bitcoin#21775 Limit `m_block_inv_mutex` (MarcoFalke) +- bitcoin/bitcoin#21825 Add I2P hardcoded seeds (jonatack) +- bitcoin/bitcoin#21843 p2p, rpc: enable GetAddr, GetAddresses, and getnodeaddresses by network (jonatack) +- bitcoin/bitcoin#21845 net processing: Don't require locking `cs_main` before calling RelayTransactions() (jnewbery) +- bitcoin/bitcoin#21872 Sanitize message type for logging (laanwj) +- bitcoin/bitcoin#21914 Use stronger AddLocal() for our I2P address (vasild) +- bitcoin/bitcoin#21985 Return IPv6 scope id in `CNetAddr::ToStringIP()` (laanwj) +- bitcoin/bitcoin#21992 Remove -feefilter option (amadeuszpawlik) +- bitcoin/bitcoin#21996 Pass strings to NetPermissions::TryParse functions by const ref (jonatack) +- bitcoin/bitcoin#22013 ignore block-relay-only peers when skipping DNS seed (ajtowns) +- bitcoin/bitcoin#22050 Remove tor v2 support (jonatack) +- bitcoin/bitcoin#22096 AddrFetch - don't disconnect on self-announcements (mzumsande) +- bitcoin/bitcoin#22141 net processing: Remove hash and fValidatedHeaders from QueuedBlock (jnewbery) +- bitcoin/bitcoin#22144 Randomize message processing peer order (sipa) +- bitcoin/bitcoin#22147 Protect last outbound HB compact block peer (sdaftuar) +- bitcoin/bitcoin#22179 Torv2 removal followups (vasild) +- bitcoin/bitcoin#22211 Relay I2P addresses even if not reachable (by us) (vasild) +- bitcoin/bitcoin#22284 Performance improvements to ProtectEvictionCandidatesByRatio() (jonatack) +- bitcoin/bitcoin#22387 Rate limit the processing of rumoured addresses (sipa) +- bitcoin/bitcoin#22455 addrman: detect on-disk corrupted nNew and nTried during unserialization (vasild) + +### Wallet +- bitcoin/bitcoin#15710 Catch `ios_base::failure` specifically (Bushstar) +- bitcoin/bitcoin#16546 External signer support - Wallet Box edition (Sjors) +- bitcoin/bitcoin#17331 Use effective values throughout coin selection (achow101) +- bitcoin/bitcoin#18418 Increase `OUTPUT_GROUP_MAX_ENTRIES` to 100 (fjahr) +- bitcoin/bitcoin#18842 Mark replaced tx to not be in the mempool anymore (MarcoFalke) +- bitcoin/bitcoin#19136 Add `parent_desc` to `getaddressinfo` (achow101) +- bitcoin/bitcoin#19137 wallettool: Add dump and createfromdump commands (achow101) +- bitcoin/bitcoin#19651 `importdescriptor`s update existing (S3RK) +- bitcoin/bitcoin#20040 Refactor OutputGroups to handle fees and spending eligibility on grouping (achow101) +- bitcoin/bitcoin#20202 Make BDB support optional (achow101) +- bitcoin/bitcoin#20226, bitcoin/bitcoin#21277, - bitcoin/bitcoin#21063 Add `listdescriptors` command (S3RK) +- bitcoin/bitcoin#20267 Disable and fix tests for when BDB is not compiled (achow101) +- bitcoin/bitcoin#20275 List all wallets in non-SQLite and non-BDB builds (ryanofsky) +- bitcoin/bitcoin#20365 wallettool: Add parameter to create descriptors wallet (S3RK) +- bitcoin/bitcoin#20403 `upgradewallet` fixes, improvements, test coverage (jonatack) +- bitcoin/bitcoin#20448 `unloadwallet`: Allow specifying `wallet_name` param matching RPC endpoint wallet (luke-jr) +- bitcoin/bitcoin#20536 Error with "Transaction too large" if the funded tx will end up being too large after signing (achow101) +- bitcoin/bitcoin#20687 Add missing check for -descriptors wallet tool option (MarcoFalke) +- bitcoin/bitcoin#20952 Add BerkeleyDB version sanity check at init time (laanwj) +- bitcoin/bitcoin#21127 Load flags before everything else (Sjors) +- bitcoin/bitcoin#21141 Add new format string placeholders for walletnotify (maayank) +- bitcoin/bitcoin#21238 A few descriptor improvements to prepare for Taproot support (sipa) +- bitcoin/bitcoin#21302 `createwallet` examples for descriptor wallets (S3RK) +- bitcoin/bitcoin#21329 descriptor wallet: Cache last hardened xpub and use in normalized descriptors (achow101) +- bitcoin/bitcoin#21365 Basic Taproot signing support for descriptor wallets (sipa) +- bitcoin/bitcoin#21417 Misc external signer improvement and HWI 2 support (Sjors) +- bitcoin/bitcoin#21467 Move external signer out of wallet module (Sjors) +- bitcoin/bitcoin#21572 Fix wrong wallet RPC context set after #21366 (ryanofsky) +- bitcoin/bitcoin#21574 Drop JSONRPCRequest constructors after #21366 (ryanofsky) +- bitcoin/bitcoin#21666 Miscellaneous external signer changes (fanquake) +- bitcoin/bitcoin#21759 Document coin selection code (glozow) +- bitcoin/bitcoin#21786 Ensure sat/vB feerates are in range (mantissa of 3) (jonatack) +- bitcoin/bitcoin#21944 Fix issues when `walletdir` is root directory (prayank23) +- bitcoin/bitcoin#22042 Replace size/weight estimate tuple with struct for named fields (instagibbs) +- bitcoin/bitcoin#22051 Basic Taproot derivation support for descriptors (sipa) +- bitcoin/bitcoin#22154 Add OutputType::BECH32M and related wallet support for fetching bech32m addresses (achow101) +- bitcoin/bitcoin#22156 Allow tr() import only when Taproot is active (achow101) +- bitcoin/bitcoin#22166 Add support for inferring tr() descriptors (sipa) +- bitcoin/bitcoin#22173 Do not load external signers wallets when unsupported (achow101) +- bitcoin/bitcoin#22308 Add missing BlockUntilSyncedToCurrentChain (MarcoFalke) +- bitcoin/bitcoin#22334 Do not spam about non-existent spk managers (S3RK) +- bitcoin/bitcoin#22379 Erase spkmans rather than setting to nullptr (achow101) +- bitcoin/bitcoin#22421 Make IsSegWitOutput return true for taproot outputs (sipa) +- bitcoin/bitcoin#22461 Change ScriptPubKeyMan::Upgrade default to True (achow101) +- bitcoin/bitcoin#22492 Reorder locks in dumpwallet to avoid lock order assertion (achow101) +- bitcoin/bitcoin#22686 Use GetSelectionAmount in ApproximateBestSubset (achow101) + +### RPC and other APIs +- bitcoin/bitcoin#18335, bitcoin/bitcoin#21484 cli: Print useful error if bitcoind rpc work queue exceeded (LarryRuane) +- bitcoin/bitcoin#18466 Fix invalid parameter error codes for `{sign,verify}message` RPCs (theStack) +- bitcoin/bitcoin#18772 Calculate fees in `getblock` using BlockUndo data (robot-visions) +- bitcoin/bitcoin#19033 http: Release work queue after event base finish (promag) +- bitcoin/bitcoin#19055 Add MuHash3072 implementation (fjahr) +- bitcoin/bitcoin#19145 Add `hash_type` MUHASH for gettxoutsetinfo (fjahr) +- bitcoin/bitcoin#19847 Avoid duplicate set lookup in `gettxoutproof` (promag) +- bitcoin/bitcoin#20286 Deprecate `addresses` and `reqSigs` from RPC outputs (mjdietzx) +- bitcoin/bitcoin#20459 Fail to return undocumented return values (MarcoFalke) +- bitcoin/bitcoin#20461 Validate `-rpcauth` arguments (promag) +- bitcoin/bitcoin#20556 Properly document return values (`submitblock`, `gettxout`, `getblocktemplate`, `scantxoutset`) (MarcoFalke) +- bitcoin/bitcoin#20755 Remove deprecated fields from `getpeerinfo` (amitiuttarwar) +- bitcoin/bitcoin#20832 Better error messages for invalid addresses (eilx2) +- bitcoin/bitcoin#20867 Support up to 20 keys for multisig under Segwit context (darosior) +- bitcoin/bitcoin#20877 cli: `-netinfo` user help and argument parsing improvements (jonatack) +- bitcoin/bitcoin#20891 Remove deprecated bumpfee behavior (achow101) +- bitcoin/bitcoin#20916 Return wtxid from `testmempoolaccept` (MarcoFalke) +- bitcoin/bitcoin#20917 Add missing signet mentions in network name lists (theStack) +- bitcoin/bitcoin#20941 Document `RPC_TRANSACTION_ALREADY_IN_CHAIN` exception (jarolrod) +- bitcoin/bitcoin#20944 Return total fee in `getmempoolinfo` (MarcoFalke) +- bitcoin/bitcoin#20964 Add specific error code for "wallet already loaded" (laanwj) +- bitcoin/bitcoin#21053 Document {previous,next}blockhash as optional (theStack) +- bitcoin/bitcoin#21056 Add a `-rpcwaittimeout` parameter to limit time spent waiting (cdecker) +- bitcoin/bitcoin#21192 cli: Treat high detail levels as maximum in `-netinfo` (laanwj) +- bitcoin/bitcoin#21311 Document optional fields for `getchaintxstats` result (theStack) +- bitcoin/bitcoin#21359 `include_unsafe` option for fundrawtransaction (t-bast) +- bitcoin/bitcoin#21426 Remove `scantxoutset` EXPERIMENTAL warning (jonatack) +- bitcoin/bitcoin#21544 Missing doc updates for bumpfee psbt update (MarcoFalke) +- bitcoin/bitcoin#21594 Add `network` field to `getnodeaddresses` (jonatack) +- bitcoin/bitcoin#21595, bitcoin/bitcoin#21753 cli: Create `-addrinfo` (jonatack) +- bitcoin/bitcoin#21602 Add additional ban time fields to `listbanned` (jarolrod) +- bitcoin/bitcoin#21679 Keep default argument value in correct type (promag) +- bitcoin/bitcoin#21718 Improve error message for `getblock` invalid datatype (klementtan) +- bitcoin/bitcoin#21913 RPCHelpMan fixes (kallewoof) +- bitcoin/bitcoin#22021 `bumpfee`/`psbtbumpfee` fixes and updates (jonatack) +- bitcoin/bitcoin#22043 `addpeeraddress` test coverage, code simplify/constness (jonatack) +- bitcoin/bitcoin#22327 cli: Avoid truncating `-rpcwaittimeout` (MarcoFalke) + +### GUI +- bitcoin/bitcoin#18948 Call setParent() in the parent's context (hebasto) +- bitcoin/bitcoin#20482 Add depends qt fix for ARM macs (jonasschnelli) +- bitcoin/bitcoin#21836 scripted-diff: Replace three dots with ellipsis in the ui strings (hebasto) +- bitcoin/bitcoin#21935 Enable external signer support for GUI builds (Sjors) +- bitcoin/bitcoin#22133 Make QWindowsVistaStylePlugin available again (regression) (hebasto) +- bitcoin-core/gui#4 UI external signer support (e.g. hardware wallet) (Sjors) +- bitcoin-core/gui#13 Hide peer detail view if multiple are selected (promag) +- bitcoin-core/gui#18 Add peertablesortproxy module (hebasto) +- bitcoin-core/gui#21 Improve pruning tooltip (fluffypony, BitcoinErrorLog) +- bitcoin-core/gui#72 Log static plugins meta data and used style (hebasto) +- bitcoin-core/gui#79 Embed monospaced font (hebasto) +- bitcoin-core/gui#85 Remove unused "What's This" button in dialogs on Windows OS (hebasto) +- bitcoin-core/gui#115 Replace "Hide tray icon" option with positive "Show tray icon" one (hebasto) +- bitcoin-core/gui#118 Remove BDB version from the Information tab (hebasto) +- bitcoin-core/gui#121 Early subscribe core signals in transaction table model (promag) +- bitcoin-core/gui#123 Do not accept command while executing another one (hebasto) +- bitcoin-core/gui#125 Enable changing the autoprune block space size in intro dialog (luke-jr) +- bitcoin-core/gui#138 Unlock encrypted wallet "OK" button bugfix (mjdietzx) +- bitcoin-core/gui#139 doc: Improve gui/src/qt README.md (jarolrod) +- bitcoin-core/gui#154 Support macOS Dark mode (goums, Uplab) +- bitcoin-core/gui#162 Add network to peers window and peer details (jonatack) +- bitcoin-core/gui#163, bitcoin-core/gui#180 Peer details: replace Direction with Connection Type (jonatack) +- bitcoin-core/gui#164 Handle peer addition/removal in a right way (hebasto) +- bitcoin-core/gui#165 Save QSplitter state in QSettings (hebasto) +- bitcoin-core/gui#173 Follow Qt docs when implementing rowCount and columnCount (hebasto) +- bitcoin-core/gui#179 Add Type column to peers window, update peer details name/tooltip (jonatack) +- bitcoin-core/gui#186 Add information to "Confirm fee bump" window (prayank23) +- bitcoin-core/gui#189 Drop workaround for QTBUG-42503 which was fixed in Qt 5.5.0 (prusnak) +- bitcoin-core/gui#194 Save/restore RPCConsole geometry only for window (hebasto) +- bitcoin-core/gui#202 Fix right panel toggle in peers tab (RandyMcMillan) +- bitcoin-core/gui#203 Display plain "Inbound" in peer details (jonatack) +- bitcoin-core/gui#204 Drop buggy TableViewLastColumnResizingFixer class (hebasto) +- bitcoin-core/gui#205, bitcoin-core/gui#229 Save/restore TransactionView and recentRequestsView tables column sizes (hebasto) +- bitcoin-core/gui#206 Display fRelayTxes and `bip152_highbandwidth_{to, from}` in peer details (jonatack) +- bitcoin-core/gui#213 Add Copy Address Action to Payment Requests (jarolrod) +- bitcoin-core/gui#214 Disable requests context menu actions when appropriate (jarolrod) +- bitcoin-core/gui#217 Make warning label look clickable (jarolrod) +- bitcoin-core/gui#219 Prevent the main window popup menu (hebasto) +- bitcoin-core/gui#220 Do not translate file extensions (hebasto) +- bitcoin-core/gui#221 RPCConsole translatable string fixes and improvements (jonatack) +- bitcoin-core/gui#226 Add "Last Block" and "Last Tx" rows to peer details area (jonatack) +- bitcoin-core/gui#233 qt test: Don't bind to regtest port (achow101) +- bitcoin-core/gui#243 Fix issue when disabling the auto-enabled blank wallet checkbox (jarolrod) +- bitcoin-core/gui#246 Revert "qt: Use "fusion" style on macOS Big Sur with old Qt" (hebasto) +- bitcoin-core/gui#248 For values of "Bytes transferred" and "Bytes/s" with 1000-based prefix names use 1000-based divisor instead of 1024-based (wodry) +- bitcoin-core/gui#251 Improve URI/file handling message (hebasto) +- bitcoin-core/gui#256 Save/restore column sizes of the tables in the Peers tab (hebasto) +- bitcoin-core/gui#260 Handle exceptions isntead of crash (hebasto) +- bitcoin-core/gui#263 Revamp context menus (hebasto) +- bitcoin-core/gui#271 Don't clear console prompt when font resizing (jarolrod) +- bitcoin-core/gui#275 Support runtime appearance adjustment on macOS (hebasto) +- bitcoin-core/gui#276 Elide long strings in their middle in the Peers tab (hebasto) +- bitcoin-core/gui#281 Set shortcuts for console's resize buttons (jarolrod) +- bitcoin-core/gui#293 Enable wordWrap for Services (RandyMcMillan) +- bitcoin-core/gui#296 Do not use QObject::tr plural syntax for numbers with a unit symbol (hebasto) +- bitcoin-core/gui#297 Avoid unnecessary translations (hebasto) +- bitcoin-core/gui#298 Peertableview alternating row colors (RandyMcMillan) +- bitcoin-core/gui#300 Remove progress bar on modal overlay (brunoerg) +- bitcoin-core/gui#309 Add access to the Peers tab from the network icon (hebasto) +- bitcoin-core/gui#311 Peers Window rename 'Peer id' to 'Peer' (jarolrod) +- bitcoin-core/gui#313 Optimize string concatenation by default (hebasto) +- bitcoin-core/gui#325 Align numbers in the "Peer Id" column to the right (hebasto) +- bitcoin-core/gui#329 Make console buttons look clickable (jarolrod) +- bitcoin-core/gui#330 Allow prompt icon to be colorized (jarolrod) +- bitcoin-core/gui#331 Make RPC console welcome message translation-friendly (hebasto) +- bitcoin-core/gui#332 Replace disambiguation strings with translator comments (hebasto) +- bitcoin-core/gui#335 test: Use QSignalSpy instead of QEventLoop (jarolrod) +- bitcoin-core/gui#343 Improve the GUI responsiveness when progress dialogs are used (hebasto) +- bitcoin-core/gui#361 Fix GUI segfault caused by bitcoin/bitcoin#22216 (ryanofsky) +- bitcoin-core/gui#362 Add keyboard shortcuts to context menus (luke-jr) +- bitcoin-core/gui#366 Dark Mode fixes/portability (luke-jr) +- bitcoin-core/gui#375 Emit dataChanged signal to dynamically re-sort Peers table (hebasto) +- bitcoin-core/gui#393 Fix regression in "Encrypt Wallet" menu item (hebasto) +- bitcoin-core/gui#396 Ensure external signer option remains disabled without signers (achow101) +- bitcoin-core/gui#406 Handle new added plurals in `bitcoin_en.ts` (hebasto) + +### Build system +- bitcoin/bitcoin#17227 Add Android packaging support (icota) +- bitcoin/bitcoin#17920 guix: Build support for macOS (dongcarl) +- bitcoin/bitcoin#18298 Fix Qt processing of configure script for depends with DEBUG=1 (hebasto) +- bitcoin/bitcoin#19160 multiprocess: Add basic spawn and IPC support (ryanofsky) +- bitcoin/bitcoin#19504 Bump minimum python version to 3.6 (ajtowns) +- bitcoin/bitcoin#19522 fix building libconsensus with reduced exports for Darwin targets (fanquake) +- bitcoin/bitcoin#19683 Pin clang search paths for darwin host (dongcarl) +- bitcoin/bitcoin#19764 Split boost into build/host packages + bump + cleanup (dongcarl) +- bitcoin/bitcoin#19817 libtapi 1100.0.11 (fanquake) +- bitcoin/bitcoin#19846 enable unused member function diagnostic (Zero-1729) +- bitcoin/bitcoin#19867 Document and cleanup Qt hacks (fanquake) +- bitcoin/bitcoin#20046 Set `CMAKE_INSTALL_RPATH` for native packages (ryanofsky) +- bitcoin/bitcoin#20223 Drop the leading 0 from the version number (achow101) +- bitcoin/bitcoin#20333 Remove `native_biplist` dependency (fanquake) +- bitcoin/bitcoin#20353 configure: Support -fdebug-prefix-map and -fmacro-prefix-map (ajtowns) +- bitcoin/bitcoin#20359 Various config.site.in improvements and linting (dongcarl) +- bitcoin/bitcoin#20413 Require C++17 compiler (MarcoFalke) +- bitcoin/bitcoin#20419 Set minimum supported macOS to 10.14 (fanquake) +- bitcoin/bitcoin#20421 miniupnpc 2.2.2 (fanquake) +- bitcoin/bitcoin#20422 Mac deployment unification (fanquake) +- bitcoin/bitcoin#20424 Update univalue subtree (MarcoFalke) +- bitcoin/bitcoin#20449 Fix Windows installer build (achow101) +- bitcoin/bitcoin#20468 Warn when generating man pages for binaries built from a dirty branch (tylerchambers) +- bitcoin/bitcoin#20469 Avoid secp256k1.h include from system (dergoegge) +- bitcoin/bitcoin#20470 Replace genisoimage with xorriso (dongcarl) +- bitcoin/bitcoin#20471 Use C++17 in depends (fanquake) +- bitcoin/bitcoin#20496 Drop unneeded macOS framework dependencies (hebasto) +- bitcoin/bitcoin#20520 Do not force Precompiled Headers (PCH) for building Qt on Linux (hebasto) +- bitcoin/bitcoin#20549 Support make src/bitcoin-node and src/bitcoin-gui (promag) +- bitcoin/bitcoin#20565 Ensure PIC build for bdb on Android (BlockMechanic) +- bitcoin/bitcoin#20594 Fix getauxval calls in randomenv.cpp (jonasschnelli) +- bitcoin/bitcoin#20603 Update crc32c subtree (MarcoFalke) +- bitcoin/bitcoin#20609 configure: output notice that test binary is disabled by fuzzing (apoelstra) +- bitcoin/bitcoin#20619 guix: Quality of life improvements (dongcarl) +- bitcoin/bitcoin#20629 Improve id string robustness (dongcarl) +- bitcoin/bitcoin#20641 Use Qt top-level build facilities (hebasto) +- bitcoin/bitcoin#20650 Drop workaround for a fixed bug in Qt build system (hebasto) +- bitcoin/bitcoin#20673 Use more legible qmake commands in qt package (hebasto) +- bitcoin/bitcoin#20684 Define .INTERMEDIATE target once only (hebasto) +- bitcoin/bitcoin#20720 more robustly check for fcf-protection support (fanquake) +- bitcoin/bitcoin#20734 Make platform-specific targets available for proper platform builds only (hebasto) +- bitcoin/bitcoin#20936 build fuzz tests by default (danben) +- bitcoin/bitcoin#20937 guix: Make nsis reproducible by respecting SOURCE-DATE-EPOCH (dongcarl) +- bitcoin/bitcoin#20938 fix linking against -latomic when building for riscv (fanquake) +- bitcoin/bitcoin#20939 fix `RELOC_SECTION` security check for bitcoin-util (fanquake) +- bitcoin/bitcoin#20963 gitian-linux: Build binaries for 64-bit POWER (continued) (laanwj) +- bitcoin/bitcoin#21036 gitian: Bump descriptors to focal for 22.0 (fanquake) +- bitcoin/bitcoin#21045 Adds switch to enable/disable randomized base address in MSVC builds (EthanHeilman) +- bitcoin/bitcoin#21065 make macOS HOST in download-osx generic (fanquake) +- bitcoin/bitcoin#21078 guix: only download sources for hosts being built (fanquake) +- bitcoin/bitcoin#21116 Disable --disable-fuzz-binary for gitian/guix builds (hebasto) +- bitcoin/bitcoin#21182 remove mostly pointless `BOOST_PROCESS` macro (fanquake) +- bitcoin/bitcoin#21205 actually fail when Boost is missing (fanquake) +- bitcoin/bitcoin#21209 use newer source for libnatpmp (fanquake) +- bitcoin/bitcoin#21226 Fix fuzz binary compilation under windows (danben) +- bitcoin/bitcoin#21231 Add /opt/homebrew to path to look for boost libraries (fyquah) +- bitcoin/bitcoin#21239 guix: Add codesignature attachment support for osx+win (dongcarl) +- bitcoin/bitcoin#21250 Make `HAVE_O_CLOEXEC` available outside LevelDB (bugfix) (theStack) +- bitcoin/bitcoin#21272 guix: Passthrough `SDK_PATH` into container (dongcarl) +- bitcoin/bitcoin#21274 assumptions: Assume C++17 (fanquake) +- bitcoin/bitcoin#21286 Bump minimum Qt version to 5.9.5 (hebasto) +- bitcoin/bitcoin#21298 guix: Bump time-machine, glibc, and linux-headers (dongcarl) +- bitcoin/bitcoin#21304 guix: Add guix-clean script + establish gc-root for container profiles (dongcarl) +- bitcoin/bitcoin#21320 fix libnatpmp macos cross compile (fanquake) +- bitcoin/bitcoin#21321 guix: Add curl to required tool list (hebasto) +- bitcoin/bitcoin#21333 set Unicode true for NSIS installer (fanquake) +- bitcoin/bitcoin#21339 Make `AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER])` unconditional (hebasto) +- bitcoin/bitcoin#21349 Fix fuzz-cuckoocache cross-compiling with DEBUG=1 (hebasto) +- bitcoin/bitcoin#21354 build, doc: Drop no longer required packages from macOS cross-compiling dependencies (hebasto) +- bitcoin/bitcoin#21363 build, qt: Improve Qt static plugins/libs check code (hebasto) +- bitcoin/bitcoin#21375 guix: Misc feedback-based fixes + hier restructuring (dongcarl) +- bitcoin/bitcoin#21376 Qt 5.12.10 (fanquake) +- bitcoin/bitcoin#21382 Clean remnants of QTBUG-34748 fix (hebasto) +- bitcoin/bitcoin#21400 Fix regression introduced in #21363 (hebasto) +- bitcoin/bitcoin#21403 set --build when configuring packages in depends (fanquake) +- bitcoin/bitcoin#21421 don't try and use -fstack-clash-protection on Windows (fanquake) +- bitcoin/bitcoin#21423 Cleanups and follow ups after bumping Qt to 5.12.10 (hebasto) +- bitcoin/bitcoin#21427 Fix `id_string` invocations (dongcarl) +- bitcoin/bitcoin#21430 Add -Werror=implicit-fallthrough compile flag (hebasto) +- bitcoin/bitcoin#21457 Split libtapi and clang out of `native_cctools` (fanquake) +- bitcoin/bitcoin#21462 guix: Add guix-{attest,verify} scripts (dongcarl) +- bitcoin/bitcoin#21495 build, qt: Fix static builds on macOS Big Sur (hebasto) +- bitcoin/bitcoin#21497 Do not opt-in unused CoreWLAN stuff in depends for macOS (hebasto) +- bitcoin/bitcoin#21543 Enable safe warnings for msvc builds (hebasto) +- bitcoin/bitcoin#21565 Make `bitcoin_qt.m4` more generic (fanquake) +- bitcoin/bitcoin#21610 remove -Wdeprecated-register from NOWARN flags (fanquake) +- bitcoin/bitcoin#21613 enable -Wdocumentation (fanquake) +- bitcoin/bitcoin#21629 Fix configuring when building depends with `NO_BDB=1` (fanquake) +- bitcoin/bitcoin#21654 build, qt: Make Qt rcc output always deterministic (hebasto) +- bitcoin/bitcoin#21655 build, qt: No longer need to set `QT_RCC_TEST=1` for determinism (hebasto) +- bitcoin/bitcoin#21658 fix make deploy for arm64-darwin (sgulls) +- bitcoin/bitcoin#21694 Use XLIFF file to provide more context to Transifex translators (hebasto) +- bitcoin/bitcoin#21708, bitcoin/bitcoin#21593 Drop pointless sed commands (hebasto) +- bitcoin/bitcoin#21731 Update msvc build to use Qt5.12.10 binaries (sipsorcery) +- bitcoin/bitcoin#21733 Re-add command to install vcpkg (dplusplus1024) +- bitcoin/bitcoin#21793 Use `-isysroot` over `--sysroot` on macOS (fanquake) +- bitcoin/bitcoin#21869 Add missing `-D_LIBCPP_DEBUG=1` to debug flags (MarcoFalke) +- bitcoin/bitcoin#21889 macho: check for control flow instrumentation (fanquake) +- bitcoin/bitcoin#21920 Improve macro for testing -latomic requirement (MarcoFalke) +- bitcoin/bitcoin#21991 libevent 2.1.12-stable (fanquake) +- bitcoin/bitcoin#22054 Bump Qt version to 5.12.11 (hebasto) +- bitcoin/bitcoin#22063 Use Qt archive of the same version as the compiled binaries (hebasto) +- bitcoin/bitcoin#22070 Don't use cf-protection when targeting arm-apple-darwin (fanquake) +- bitcoin/bitcoin#22071 Latest config.guess and config.sub (fanquake) +- bitcoin/bitcoin#22075 guix: Misc leftover usability improvements (dongcarl) +- bitcoin/bitcoin#22123 Fix qt.mk for mac arm64 (promag) +- bitcoin/bitcoin#22174 build, qt: Fix libraries linking order for Linux hosts (hebasto) +- bitcoin/bitcoin#22182 guix: Overhaul how guix-{attest,verify} works and hierarchy (dongcarl) +- bitcoin/bitcoin#22186 build, qt: Fix compiling qt package in depends with GCC 11 (hebasto) +- bitcoin/bitcoin#22199 macdeploy: minor fixups and simplifications (fanquake) +- bitcoin/bitcoin#22230 Fix MSVC linker /SubSystem option for bitcoin-qt.exe (hebasto) +- bitcoin/bitcoin#22234 Mark print-% target as phony (dgoncharov) +- bitcoin/bitcoin#22238 improve detection of eBPF support (fanquake) +- bitcoin/bitcoin#22258 Disable deprecated-copy warning only when external warnings are enabled (MarcoFalke) +- bitcoin/bitcoin#22320 set minimum required Boost to 1.64.0 (fanquake) +- bitcoin/bitcoin#22348 Fix cross build for Windows with Boost Process (hebasto) +- bitcoin/bitcoin#22365 guix: Avoid relying on newer symbols by rebasing our cross toolchains on older glibcs (dongcarl) +- bitcoin/bitcoin#22381 guix: Test security-check sanity before performing them (with macOS) (fanquake) +- bitcoin/bitcoin#22405 Remove --enable-glibc-back-compat from Guix build (fanquake) +- bitcoin/bitcoin#22406 Remove --enable-determinism configure option (fanquake) +- bitcoin/bitcoin#22410 Avoid GCC 7.1 ABI change warning in guix build (sipa) +- bitcoin/bitcoin#22436 use aarch64 Clang if cross-compiling for darwin on aarch64 (fanquake) +- bitcoin/bitcoin#22465 guix: Pin kernel-header version, time-machine to upstream 1.3.0 commit (dongcarl) +- bitcoin/bitcoin#22511 guix: Silence `getent(1)` invocation, doc fixups (dongcarl) +- bitcoin/bitcoin#22531 guix: Fixes to guix-{attest,verify} (achow101) +- bitcoin/bitcoin#22642 release: Release with separate sha256sums and sig files (dongcarl) +- bitcoin/bitcoin#22685 clientversion: No suffix `#if CLIENT_VERSION_IS_RELEASE` (dongcarl) +- bitcoin/bitcoin#22713 Fix build with Boost 1.77.0 (sizeofvoid) + +### Tests and QA +- bitcoin/bitcoin#14604 Add test and refactor `feature_block.py` (sanket1729) +- bitcoin/bitcoin#17556 Change `feature_config_args.py` not to rely on strange regtest=0 behavior (ryanofsky) +- bitcoin/bitcoin#18795 wallet issue with orphaned rewards (domob1812) +- bitcoin/bitcoin#18847 compressor: Use a prevector in CompressScript serialization (jb55) +- bitcoin/bitcoin#19259 fuzz: Add fuzzing harness for LoadMempool(…) and DumpMempool(…) (practicalswift) +- bitcoin/bitcoin#19315 Allow outbound & block-relay-only connections in functional tests. (amitiuttarwar) +- bitcoin/bitcoin#19698 Apply strict verification flags for transaction tests and assert backwards compatibility (glozow) +- bitcoin/bitcoin#19801 Check for all possible `OP_CLTV` fail reasons in `feature_cltv.py` (BIP 65) (theStack) +- bitcoin/bitcoin#19893 Remove or explain syncwithvalidationinterfacequeue (MarcoFalke) +- bitcoin/bitcoin#19972 fuzz: Add fuzzing harness for node eviction logic (practicalswift) +- bitcoin/bitcoin#19982 Fix inconsistent lock order in `wallet_tests/CreateWallet` (hebasto) +- bitcoin/bitcoin#20000 Fix creation of "std::string"s with \0s (vasild) +- bitcoin/bitcoin#20047 Use `wait_for_{block,header}` helpers in `p2p_fingerprint.py` (theStack) +- bitcoin/bitcoin#20171 Add functional test `test_txid_inv_delay` (ariard) +- bitcoin/bitcoin#20189 Switch to BIP341's suggested scheme for outputs without script (sipa) +- bitcoin/bitcoin#20248 Fix length of R check in `key_signature_tests` (dgpv) +- bitcoin/bitcoin#20276, bitcoin/bitcoin#20385, bitcoin/bitcoin#20688, bitcoin/bitcoin#20692 Run various mempool tests even with wallet disabled (mjdietzx) +- bitcoin/bitcoin#20323 Create or use existing properly initialized NodeContexts (dongcarl) +- bitcoin/bitcoin#20354 Add `feature_taproot.py --previous_release` (MarcoFalke) +- bitcoin/bitcoin#20370 fuzz: Version handshake (MarcoFalke) +- bitcoin/bitcoin#20377 fuzz: Fill various small fuzzing gaps (practicalswift) +- bitcoin/bitcoin#20425 fuzz: Make CAddrMan fuzzing harness deterministic (practicalswift) +- bitcoin/bitcoin#20430 Sanitizers: Add suppression for unsigned-integer-overflow in libstdc++ (jonasschnelli) +- bitcoin/bitcoin#20437 fuzz: Avoid time-based "non-determinism" in fuzzing harnesses by using mocked GetTime() (practicalswift) +- bitcoin/bitcoin#20458 Add `is_bdb_compiled` helper (Sjors) +- bitcoin/bitcoin#20466 Fix intermittent `p2p_fingerprint` issue (MarcoFalke) +- bitcoin/bitcoin#20472 Add testing of ParseInt/ParseUInt edge cases with leading +/-/0:s (practicalswift) +- bitcoin/bitcoin#20507 sync: print proper lock order location when double lock is detected (vasild) +- bitcoin/bitcoin#20522 Fix sync issue in `disconnect_p2ps` (amitiuttarwar) +- bitcoin/bitcoin#20524 Move `MIN_VERSION_SUPPORTED` to p2p.py (jnewbery) +- bitcoin/bitcoin#20540 Fix `wallet_multiwallet` issue on windows (MarcoFalke) +- bitcoin/bitcoin#20560 fuzz: Link all targets once (MarcoFalke) +- bitcoin/bitcoin#20567 Add option to git-subtree-check to do full check, add help (laanwj) +- bitcoin/bitcoin#20569 Fix intermittent `wallet_multiwallet` issue with `got_loading_error` (MarcoFalke) +- bitcoin/bitcoin#20613 Use Popen.wait instead of RPC in `assert_start_raises_init_error` (MarcoFalke) +- bitcoin/bitcoin#20663 fuzz: Hide `script_assets_test_minimizer` (MarcoFalke) +- bitcoin/bitcoin#20674 fuzz: Call SendMessages after ProcessMessage to increase coverage (MarcoFalke) +- bitcoin/bitcoin#20683 Fix restart node race (MarcoFalke) +- bitcoin/bitcoin#20686 fuzz: replace CNode code with fuzz/util.h::ConsumeNode() (jonatack) +- bitcoin/bitcoin#20733 Inline non-member functions with body in fuzzing headers (pstratem) +- bitcoin/bitcoin#20737 Add missing assignment in `mempool_resurrect.py` (MarcoFalke) +- bitcoin/bitcoin#20745 Correct `epoll_ctl` data race suppression (hebasto) +- bitcoin/bitcoin#20748 Add race:SendZmqMessage tsan suppression (MarcoFalke) +- bitcoin/bitcoin#20760 Set correct nValue for multi-op-return policy check (MarcoFalke) +- bitcoin/bitcoin#20761 fuzz: Check that `NULL_DATA` is unspendable (MarcoFalke) +- bitcoin/bitcoin#20765 fuzz: Check that certain script TxoutType are nonstandard (mjdietzx) +- bitcoin/bitcoin#20772 fuzz: Bolster ExtractDestination(s) checks (mjdietzx) +- bitcoin/bitcoin#20789 fuzz: Rework strong and weak net enum fuzzing (MarcoFalke) +- bitcoin/bitcoin#20828 fuzz: Introduce CallOneOf helper to replace switch-case (MarcoFalke) +- bitcoin/bitcoin#20839 fuzz: Avoid extraneous copy of input data, using Span<> (MarcoFalke) +- bitcoin/bitcoin#20844 Add sanitizer suppressions for AMD EPYC CPUs (MarcoFalke) +- bitcoin/bitcoin#20857 Update documentation in `feature_csv_activation.py` (PiRK) +- bitcoin/bitcoin#20876 Replace getmempoolentry with testmempoolaccept in MiniWallet (MarcoFalke) +- bitcoin/bitcoin#20881 fuzz: net permission flags in net processing (MarcoFalke) +- bitcoin/bitcoin#20882 fuzz: Add missing muhash registration (MarcoFalke) +- bitcoin/bitcoin#20908 fuzz: Use mocktime in `process_message*` fuzz targets (MarcoFalke) +- bitcoin/bitcoin#20915 fuzz: Fail if message type is not fuzzed (MarcoFalke) +- bitcoin/bitcoin#20946 fuzz: Consolidate fuzzing TestingSetup initialization (dongcarl) +- bitcoin/bitcoin#20954 Declare `nodes` type `in test_framework.py` (kiminuo) +- bitcoin/bitcoin#20955 Fix `get_previous_releases.py` for aarch64 (MarcoFalke) +- bitcoin/bitcoin#20969 check that getblockfilter RPC fails without block filter index (theStack) +- bitcoin/bitcoin#20971 Work around libFuzzer deadlock (MarcoFalke) +- bitcoin/bitcoin#20993 Store subversion (user agent) as string in `msg_version` (theStack) +- bitcoin/bitcoin#20995 fuzz: Avoid initializing version to less than `MIN_PEER_PROTO_VERSION` (MarcoFalke) +- bitcoin/bitcoin#20998 Fix BlockToJsonVerbose benchmark (martinus) +- bitcoin/bitcoin#21003 Move MakeNoLogFileContext to `libtest_util`, and use it in bench (MarcoFalke) +- bitcoin/bitcoin#21008 Fix zmq test flakiness, improve speed (theStack) +- bitcoin/bitcoin#21023 fuzz: Disable shuffle when merge=1 (MarcoFalke) +- bitcoin/bitcoin#21037 fuzz: Avoid designated initialization (C++20) in fuzz tests (practicalswift) +- bitcoin/bitcoin#21042 doc, test: Improve `setup_clean_chain` documentation (fjahr) +- bitcoin/bitcoin#21080 fuzz: Configure check for main function (take 2) (MarcoFalke) +- bitcoin/bitcoin#21084 Fix timeout decrease in `feature_assumevalid` (brunoerg) +- bitcoin/bitcoin#21096 Re-add dead code detection (flack) +- bitcoin/bitcoin#21100 Remove unused function `xor_bytes` (theStack) +- bitcoin/bitcoin#21115 Fix Windows cross build (hebasto) +- bitcoin/bitcoin#21117 Remove `assert_blockchain_height` (MarcoFalke) +- bitcoin/bitcoin#21121 Small unit test improvements, including helper to make mempool transaction (amitiuttarwar) +- bitcoin/bitcoin#21124 Remove unnecessary assignment in bdb (brunoerg) +- bitcoin/bitcoin#21125 Change `BOOST_CHECK` to `BOOST_CHECK_EQUAL` for paths (kiminuo) +- bitcoin/bitcoin#21142, bitcoin/bitcoin#21512 fuzz: Add `tx_pool` fuzz target (MarcoFalke) +- bitcoin/bitcoin#21165 Use mocktime in `test_seed_peers` (dhruv) +- bitcoin/bitcoin#21169 fuzz: Add RPC interface fuzzing. Increase fuzzing coverage from 65% to 70% (practicalswift) +- bitcoin/bitcoin#21170 bench: Add benchmark to write json into a string (martinus) +- bitcoin/bitcoin#21178 Run `mempool_reorg.py` even with wallet disabled (DariusParvin) +- bitcoin/bitcoin#21185 fuzz: Remove expensive and redundant muhash from crypto fuzz target (MarcoFalke) +- bitcoin/bitcoin#21200 Speed up `rpc_blockchain.py` by removing miniwallet.generate() (MarcoFalke) +- bitcoin/bitcoin#21211 Move `P2WSH_OP_TRUE` to shared test library (MarcoFalke) +- bitcoin/bitcoin#21228 Avoid comparision of integers with different signs (jonasschnelli) +- bitcoin/bitcoin#21230 Fix `NODE_NETWORK_LIMITED_MIN_BLOCKS` disconnection (MarcoFalke) +- bitcoin/bitcoin#21252 Add missing wait for sync to `feature_blockfilterindex_prune` (MarcoFalke) +- bitcoin/bitcoin#21254 Avoid connecting to real network when running tests (MarcoFalke) +- bitcoin/bitcoin#21264 fuzz: Two scripted diff renames (MarcoFalke) +- bitcoin/bitcoin#21280 Bug fix in `transaction_tests` (glozow) +- bitcoin/bitcoin#21293 Replace accidentally placed bit-OR with logical-OR (hebasto) +- bitcoin/bitcoin#21297 `feature_blockfilterindex_prune.py` improvements (jonatack) +- bitcoin/bitcoin#21310 zmq test: fix sync-up by matching notification to generated block (theStack) +- bitcoin/bitcoin#21334 Additional BIP9 tests (Sjors) +- bitcoin/bitcoin#21338 Add functional test for anchors.dat (brunoerg) +- bitcoin/bitcoin#21345 Bring `p2p_leak.py` up to date (mzumsande) +- bitcoin/bitcoin#21357 Unconditionally check for fRelay field in test framework (jarolrod) +- bitcoin/bitcoin#21358 fuzz: Add missing include (`test/util/setup_common.h`) (MarcoFalke) +- bitcoin/bitcoin#21371 fuzz: fix gcc Woverloaded-virtual build warnings (jonatack) +- bitcoin/bitcoin#21373 Generate fewer blocks in `feature_nulldummy` to fix timeouts, speed up (jonatack) +- bitcoin/bitcoin#21390 Test improvements for UTXO set hash tests (fjahr) +- bitcoin/bitcoin#21410 increase `rpc_timeout` for fundrawtx `test_transaction_too_large` (jonatack) +- bitcoin/bitcoin#21411 add logging, reduce blocks, move `sync_all` in `wallet_` groups (jonatack) +- bitcoin/bitcoin#21438 Add ParseUInt8() test coverage (jonatack) +- bitcoin/bitcoin#21443 fuzz: Implement `fuzzed_dns_lookup_function` as a lambda (practicalswift) +- bitcoin/bitcoin#21445 cirrus: Use SSD cluster for speedup (MarcoFalke) +- bitcoin/bitcoin#21477 Add test for CNetAddr::ToString IPv6 address formatting (RFC 5952) (practicalswift) +- bitcoin/bitcoin#21487 fuzz: Use ConsumeWeakEnum in addrman for service flags (MarcoFalke) +- bitcoin/bitcoin#21488 Add ParseUInt16() unit test and fuzz coverage (jonatack) +- bitcoin/bitcoin#21491 test: remove duplicate assertions in util_tests (jonatack) +- bitcoin/bitcoin#21522 fuzz: Use PickValue where possible (MarcoFalke) +- bitcoin/bitcoin#21531 remove qt byteswap compattests (fanquake) +- bitcoin/bitcoin#21557 small cleanup in RPCNestedTests tests (fanquake) +- bitcoin/bitcoin#21586 Add missing suppression for signed-integer-overflow:txmempool.cpp (MarcoFalke) +- bitcoin/bitcoin#21592 Remove option to make TestChain100Setup non-deterministic (MarcoFalke) +- bitcoin/bitcoin#21597 Document `race:validation_chainstatemanager_tests` suppression (MarcoFalke) +- bitcoin/bitcoin#21599 Replace file level integer overflow suppression with function level suppression (practicalswift) +- bitcoin/bitcoin#21604 Document why no symbol names can be used for suppressions (MarcoFalke) +- bitcoin/bitcoin#21606 fuzz: Extend psbt fuzz target a bit (MarcoFalke) +- bitcoin/bitcoin#21617 fuzz: Fix uninitialized read in i2p test (MarcoFalke) +- bitcoin/bitcoin#21630 fuzz: split FuzzedSock interface and implementation (vasild) +- bitcoin/bitcoin#21634 Skip SQLite fsyncs while testing (achow101) +- bitcoin/bitcoin#21669 Remove spurious double lock tsan suppressions by bumping to clang-12 (MarcoFalke) +- bitcoin/bitcoin#21676 Use mocktime to avoid intermittent failure in `rpc_tests` (MarcoFalke) +- bitcoin/bitcoin#21677 fuzz: Avoid use of low file descriptor ids (which may be in use) in FuzzedSock (practicalswift) +- bitcoin/bitcoin#21678 Fix TestPotentialDeadLockDetected suppression (hebasto) +- bitcoin/bitcoin#21689 Remove intermittently failing and not very meaningful `BOOST_CHECK` in `cnetaddr_basic` (practicalswift) +- bitcoin/bitcoin#21691 Check that no versionbits are re-used (MarcoFalke) +- bitcoin/bitcoin#21707 Extend functional tests for addr relay (mzumsande) +- bitcoin/bitcoin#21712 Test default `include_mempool` value of gettxout (promag) +- bitcoin/bitcoin#21738 Use clang-12 for ASAN, Add missing suppression (MarcoFalke) +- bitcoin/bitcoin#21740 add new python linter to check file names and permissions (windsok) +- bitcoin/bitcoin#21749 Bump shellcheck version (hebasto) +- bitcoin/bitcoin#21754 Run `feature_cltv` with MiniWallet (MarcoFalke) +- bitcoin/bitcoin#21762 Speed up `mempool_spend_coinbase.py` (MarcoFalke) +- bitcoin/bitcoin#21773 fuzz: Ensure prevout is consensus-valid (MarcoFalke) +- bitcoin/bitcoin#21777 Fix `feature_notifications.py` intermittent issue (MarcoFalke) +- bitcoin/bitcoin#21785 Fix intermittent issue in `p2p_addr_relay.py` (MarcoFalke) +- bitcoin/bitcoin#21787 Fix off-by-ones in `rpc_fundrawtransaction` assertions (jonatack) +- bitcoin/bitcoin#21792 Fix intermittent issue in `p2p_segwit.py` (MarcoFalke) +- bitcoin/bitcoin#21795 fuzz: Terminate immediately if a fuzzing harness tries to perform a DNS lookup (belt and suspenders) (practicalswift) +- bitcoin/bitcoin#21798 fuzz: Create a block template in `tx_pool` targets (MarcoFalke) +- bitcoin/bitcoin#21804 Speed up `p2p_segwit.py` (jnewbery) +- bitcoin/bitcoin#21810 fuzz: Various RPC fuzzer follow-ups (practicalswift) +- bitcoin/bitcoin#21814 Fix `feature_config_args.py` intermittent issue (MarcoFalke) +- bitcoin/bitcoin#21821 Add missing test for empty P2WSH redeem (MarcoFalke) +- bitcoin/bitcoin#21822 Resolve bug in `interface_bitcoin_cli.py` (klementtan) +- bitcoin/bitcoin#21846 fuzz: Add `-fsanitize=integer` suppression needed for RPC fuzzer (`generateblock`) (practicalswift) +- bitcoin/bitcoin#21849 fuzz: Limit toxic test globals to their respective scope (MarcoFalke) +- bitcoin/bitcoin#21867 use MiniWallet for `p2p_blocksonly.py` (theStack) +- bitcoin/bitcoin#21873 minor fixes & improvements for files linter test (windsok) +- bitcoin/bitcoin#21874 fuzz: Add `WRITE_ALL_FUZZ_TARGETS_AND_ABORT` (MarcoFalke) +- bitcoin/bitcoin#21884 fuzz: Remove unused --enable-danger-fuzz-link-all option (MarcoFalke) +- bitcoin/bitcoin#21890 fuzz: Limit ParseISO8601DateTime fuzzing to 32-bit (MarcoFalke) +- bitcoin/bitcoin#21891 fuzz: Remove strprintf test cases that are known to fail (MarcoFalke) +- bitcoin/bitcoin#21892 fuzz: Avoid excessively large min fee rate in `tx_pool` (MarcoFalke) +- bitcoin/bitcoin#21895 Add TSA annotations to the WorkQueue class members (hebasto) +- bitcoin/bitcoin#21900 use MiniWallet for `feature_csv_activation.py` (theStack) +- bitcoin/bitcoin#21909 fuzz: Limit max insertions in timedata fuzz test (MarcoFalke) +- bitcoin/bitcoin#21922 fuzz: Avoid timeout in EncodeBase58 (MarcoFalke) +- bitcoin/bitcoin#21927 fuzz: Run const CScript member functions only once (MarcoFalke) +- bitcoin/bitcoin#21929 fuzz: Remove incorrect float round-trip serialization test (MarcoFalke) +- bitcoin/bitcoin#21936 fuzz: Terminate immediately if a fuzzing harness tries to create a TCP socket (belt and suspenders) (practicalswift) +- bitcoin/bitcoin#21941 fuzz: Call const member functions in addrman fuzz test only once (MarcoFalke) +- bitcoin/bitcoin#21945 add P2PK support to MiniWallet (theStack) +- bitcoin/bitcoin#21948 Fix off-by-one in mockscheduler test RPC (MarcoFalke) +- bitcoin/bitcoin#21953 fuzz: Add `utxo_snapshot` target (MarcoFalke) +- bitcoin/bitcoin#21970 fuzz: Add missing CheckTransaction before CheckTxInputs (MarcoFalke) +- bitcoin/bitcoin#21989 Use `COINBASE_MATURITY` in functional tests (kiminuo) +- bitcoin/bitcoin#22003 Add thread safety annotations (ajtowns) +- bitcoin/bitcoin#22004 fuzz: Speed up transaction fuzz target (MarcoFalke) +- bitcoin/bitcoin#22005 fuzz: Speed up banman fuzz target (MarcoFalke) +- bitcoin/bitcoin#22029 [fuzz] Improve transport deserialization fuzz test coverage (dhruv) +- bitcoin/bitcoin#22048 MiniWallet: introduce enum type for output mode (theStack) +- bitcoin/bitcoin#22057 use MiniWallet (P2PK mode) for `feature_dersig.py` (theStack) +- bitcoin/bitcoin#22065 Mark `CheckTxInputs` `[[nodiscard]]`. Avoid UUM in fuzzing harness `coins_view` (practicalswift) +- bitcoin/bitcoin#22069 fuzz: don't try and use fopencookie() when building for Android (fanquake) +- bitcoin/bitcoin#22082 update nanobench from release 4.0.0 to 4.3.4 (martinus) +- bitcoin/bitcoin#22086 remove BasicTestingSetup from unit tests that don't need it (fanquake) +- bitcoin/bitcoin#22089 MiniWallet: fix fee calculation for P2PK and check tx vsize (theStack) +- bitcoin/bitcoin#21107, bitcoin/bitcoin#22092 Convert documentation into type annotations (fanquake) +- bitcoin/bitcoin#22095 Additional BIP32 test vector for hardened derivation with leading zeros (kristapsk) +- bitcoin/bitcoin#22103 Fix IPv6 check on BSD systems (n-thumann) +- bitcoin/bitcoin#22118 check anchors.dat when node starts for the first time (brunoerg) +- bitcoin/bitcoin#22120 `p2p_invalid_block`: Check that a block rejected due to too-new tim… (willcl-ark) +- bitcoin/bitcoin#22153 Fix `p2p_leak.py` intermittent failure (mzumsande) +- bitcoin/bitcoin#22169 p2p, rpc, fuzz: various tiny follow-ups (jonatack) +- bitcoin/bitcoin#22176 Correct outstanding -Werror=sign-compare errors (Empact) +- bitcoin/bitcoin#22180 fuzz: Increase branch coverage of the float fuzz target (MarcoFalke) +- bitcoin/bitcoin#22187 Add `sync_blocks` in `wallet_orphanedreward.py` (domob1812) +- bitcoin/bitcoin#22201 Fix TestShell to allow running in Jupyter Notebook (josibake) +- bitcoin/bitcoin#22202 Add temporary coinstats suppressions (MarcoFalke) +- bitcoin/bitcoin#22203 Use ConnmanTestMsg from test lib in `denialofservice_tests` (MarcoFalke) +- bitcoin/bitcoin#22210 Use MiniWallet in `test_no_inherited_signaling` RBF test (MarcoFalke) +- bitcoin/bitcoin#22224 Update msvc and appveyor builds to use Qt5.12.11 binaries (sipsorcery) +- bitcoin/bitcoin#22249 Kill process group to avoid dangling processes when using `--failfast` (S3RK) +- bitcoin/bitcoin#22267 fuzz: Speed up crypto fuzz target (MarcoFalke) +- bitcoin/bitcoin#22270 Add bitcoin-util tests (+refactors) (MarcoFalke) +- bitcoin/bitcoin#22271 fuzz: Assert roundtrip equality for `CPubKey` (theStack) +- bitcoin/bitcoin#22279 fuzz: add missing ECCVerifyHandle to `base_encode_decode` (apoelstra) +- bitcoin/bitcoin#22292 bench, doc: benchmarking updates and fixups (jonatack) +- bitcoin/bitcoin#22306 Improvements to `p2p_addr_relay.py` (amitiuttarwar) +- bitcoin/bitcoin#22310 Add functional test for replacement relay fee check (ariard) +- bitcoin/bitcoin#22311 Add missing syncwithvalidationinterfacequeue in `p2p_blockfilters` (MarcoFalke) +- bitcoin/bitcoin#22313 Add missing `sync_all` to `feature_coinstatsindex` (MarcoFalke) +- bitcoin/bitcoin#22322 fuzz: Check banman roundtrip (MarcoFalke) +- bitcoin/bitcoin#22363 Use `script_util` helpers for creating P2{PKH,SH,WPKH,WSH} scripts (theStack) +- bitcoin/bitcoin#22399 fuzz: Rework CTxDestination fuzzing (MarcoFalke) +- bitcoin/bitcoin#22408 add tests for `bad-txns-prevout-null` reject reason (theStack) +- bitcoin/bitcoin#22445 fuzz: Move implementations of non-template fuzz helpers from util.h to util.cpp (sriramdvt) +- bitcoin/bitcoin#22446 Fix `wallet_listdescriptors.py` if bdb is not compiled (hebasto) +- bitcoin/bitcoin#22447 Whitelist `rpc_rawtransaction` peers to speed up tests (jonatack) +- bitcoin/bitcoin#22742 Use proper target in `do_fund_send` (S3RK) + +### Miscellaneous +- bitcoin/bitcoin#19337 sync: Detect double lock from the same thread (vasild) +- bitcoin/bitcoin#19809 log: Prefix log messages with function name and source code location if -logsourcelocations is set (practicalswift) +- bitcoin/bitcoin#19866 eBPF Linux tracepoints (jb55) +- bitcoin/bitcoin#20024 init: Fix incorrect warning "Reducing -maxconnections from N to N-1, because of system limitations" (practicalswift) +- bitcoin/bitcoin#20145 contrib: Add getcoins.py script to get coins from (signet) faucet (kallewoof) +- bitcoin/bitcoin#20255 util: Add assume() identity function (MarcoFalke) +- bitcoin/bitcoin#20288 script, doc: Contrib/seeds updates (jonatack) +- bitcoin/bitcoin#20358 src/randomenv.cpp: Fix build on uclibc (ffontaine) +- bitcoin/bitcoin#20406 util: Avoid invalid integer negation in formatmoney and valuefromamount (practicalswift) +- bitcoin/bitcoin#20434 contrib: Parse elf directly for symbol and security checks (laanwj) +- bitcoin/bitcoin#20451 lint: Run mypy over contrib/devtools (fanquake) +- bitcoin/bitcoin#20476 contrib: Add test for elf symbol-check (laanwj) +- bitcoin/bitcoin#20530 lint: Update cppcheck linter to c++17 and improve explicit usage (fjahr) +- bitcoin/bitcoin#20589 log: Clarify that failure to read/write `fee_estimates.dat` is non-fatal (MarcoFalke) +- bitcoin/bitcoin#20602 util: Allow use of c++14 chrono literals (MarcoFalke) +- bitcoin/bitcoin#20605 init: Signal-safe instant shutdown (laanwj) +- bitcoin/bitcoin#20608 contrib: Add symbol check test for PE binaries (fanquake) +- bitcoin/bitcoin#20689 contrib: Replace binary verification script verify.sh with python rewrite (theStack) +- bitcoin/bitcoin#20715 util: Add argsmanager::getcommand() and use it in bitcoin-wallet (MarcoFalke) +- bitcoin/bitcoin#20735 script: Remove outdated extract-osx-sdk.sh (hebasto) +- bitcoin/bitcoin#20817 lint: Update list of spelling linter false positives, bump to codespell 2.0.0 (theStack) +- bitcoin/bitcoin#20884 script: Improve robustness of bitcoind.service on startup (hebasto) +- bitcoin/bitcoin#20906 contrib: Embed c++11 patch in `install_db4.sh` (gruve-p) +- bitcoin/bitcoin#21004 contrib: Fix docker args conditional in gitian-build (setpill) +- bitcoin/bitcoin#21007 bitcoind: Add -daemonwait option to wait for initialization (laanwj) +- bitcoin/bitcoin#21041 log: Move "Pre-allocating up to position 0x[…] in […].dat" log message to debug category (practicalswift) +- bitcoin/bitcoin#21059 Drop boost/preprocessor dependencies (hebasto) +- bitcoin/bitcoin#21087 guix: Passthrough `BASE_CACHE` into container (dongcarl) +- bitcoin/bitcoin#21088 guix: Jump forwards in time-machine and adapt (dongcarl) +- bitcoin/bitcoin#21089 guix: Add support for powerpc64{,le} (dongcarl) +- bitcoin/bitcoin#21110 util: Remove boost `posix_time` usage from `gettime*` (fanquake) +- bitcoin/bitcoin#21111 Improve OpenRC initscript (parazyd) +- bitcoin/bitcoin#21123 code style: Add EditorConfig file (kiminuo) +- bitcoin/bitcoin#21173 util: Faster hexstr => 13% faster blocktojson (martinus) +- bitcoin/bitcoin#21221 tools: Allow argument/parameter bin packing in clang-format (jnewbery) +- bitcoin/bitcoin#21244 Move GetDataDir to ArgsManager (kiminuo) +- bitcoin/bitcoin#21255 contrib: Run test-symbol-check for risc-v (fanquake) +- bitcoin/bitcoin#21271 guix: Explicitly set umask in build container (dongcarl) +- bitcoin/bitcoin#21300 script: Add explanatory comment to tc.sh (dscotese) +- bitcoin/bitcoin#21317 util: Make assume() usable as unary expression (MarcoFalke) +- bitcoin/bitcoin#21336 Make .gitignore ignore src/test/fuzz/fuzz.exe (hebasto) +- bitcoin/bitcoin#21337 guix: Update darwin native packages dependencies (hebasto) +- bitcoin/bitcoin#21405 compat: remove memcpy -> memmove backwards compatibility alias (fanquake) +- bitcoin/bitcoin#21418 contrib: Make systemd invoke dependencies only when ready (laanwj) +- bitcoin/bitcoin#21447 Always add -daemonwait to known command line arguments (hebasto) +- bitcoin/bitcoin#21471 bugfix: Fix `bech32_encode` calls in `gen_key_io_test_vectors.py` (sipa) +- bitcoin/bitcoin#21615 script: Add trusted key for hebasto (hebasto) +- bitcoin/bitcoin#21664 contrib: Use lief for macos and windows symbol & security checks (fanquake) +- bitcoin/bitcoin#21695 contrib: Remove no longer used contrib/bitcoin-qt.pro (hebasto) +- bitcoin/bitcoin#21711 guix: Add full installation and usage documentation (dongcarl) +- bitcoin/bitcoin#21799 guix: Use `gcc-8` across the board (dongcarl) +- bitcoin/bitcoin#21802 Avoid UB in util/asmap (advance a dereferenceable iterator outside its valid range) (MarcoFalke) +- bitcoin/bitcoin#21823 script: Update reviewers (jonatack) +- bitcoin/bitcoin#21850 Remove `GetDataDir(net_specific)` function (kiminuo) +- bitcoin/bitcoin#21871 scripts: Add checks for minimum required os versions (fanquake) +- bitcoin/bitcoin#21966 Remove double serialization; use software encoder for fee estimation (sipa) +- bitcoin/bitcoin#22060 contrib: Add torv3 seed nodes for testnet, drop v2 ones (laanwj) +- bitcoin/bitcoin#22244 devtools: Correctly extract symbol versions in symbol-check (laanwj) +- bitcoin/bitcoin#22533 guix/build: Remove vestigial SKIPATTEST.TAG (dongcarl) +- bitcoin/bitcoin#22643 guix-verify: Non-zero exit code when anything fails (dongcarl) +- bitcoin/bitcoin#22654 guix: Don't include directory name in SHA256SUMS (achow101) + +### Documentation +- bitcoin/bitcoin#15451 clarify getdata limit after #14897 (HashUnlimited) +- bitcoin/bitcoin#15545 Explain why CheckBlock() is called before AcceptBlock (Sjors) +- bitcoin/bitcoin#17350 Add developer documentation to isminetype (HAOYUatHZ) +- bitcoin/bitcoin#17934 Use `CONFIG_SITE` variable instead of --prefix option (hebasto) +- bitcoin/bitcoin#18030 Coin::IsSpent() can also mean never existed (Sjors) +- bitcoin/bitcoin#18096 IsFinalTx comment about nSequence & `OP_CLTV` (nothingmuch) +- bitcoin/bitcoin#18568 Clarify developer notes about constant naming (ryanofsky) +- bitcoin/bitcoin#19961 doc: tor.md updates (jonatack) +- bitcoin/bitcoin#19968 Clarify CRollingBloomFilter size estimate (robot-dreams) +- bitcoin/bitcoin#20200 Rename CODEOWNERS to REVIEWERS (adamjonas) +- bitcoin/bitcoin#20329 docs/descriptors.md: Remove hardened marker in the path after xpub (dgpv) +- bitcoin/bitcoin#20380 Add instructions on how to fuzz the P2P layer using Honggfuzz NetDriver (practicalswift) +- bitcoin/bitcoin#20414 Remove generated manual pages from master branch (laanwj) +- bitcoin/bitcoin#20473 Document current boost dependency as 1.71.0 (laanwj) +- bitcoin/bitcoin#20512 Add bash as an OpenBSD dependency (emilengler) +- bitcoin/bitcoin#20568 Use FeeModes doc helper in estimatesmartfee (MarcoFalke) +- bitcoin/bitcoin#20577 libconsensus: add missing error code description, fix NBitcoin link (theStack) +- bitcoin/bitcoin#20587 Tidy up Tor doc (more stringent) (wodry) +- bitcoin/bitcoin#20592 Update wtxidrelay documentation per BIP339 (jonatack) +- bitcoin/bitcoin#20601 Update for FreeBSD 12.2, add GUI Build Instructions (jarolrod) +- bitcoin/bitcoin#20635 fix misleading comment about call to non-existing function (pox) +- bitcoin/bitcoin#20646 Refer to BIPs 339/155 in feature negotiation (jonatack) +- bitcoin/bitcoin#20653 Move addr relay comment in net to correct place (MarcoFalke) +- bitcoin/bitcoin#20677 Remove shouty enums in `net_processing` comments (sdaftuar) +- bitcoin/bitcoin#20741 Update 'Secure string handling' (prayank23) +- bitcoin/bitcoin#20757 tor.md and -onlynet help updates (jonatack) +- bitcoin/bitcoin#20829 Add -netinfo help (jonatack) +- bitcoin/bitcoin#20830 Update developer notes with signet (jonatack) +- bitcoin/bitcoin#20890 Add explicit macdeployqtplus dependencies install step (hebasto) +- bitcoin/bitcoin#20913 Add manual page generation for bitcoin-util (laanwj) +- bitcoin/bitcoin#20985 Add xorriso to macOS depends packages (fanquake) +- bitcoin/bitcoin#20986 Update developer notes to discourage very long lines (jnewbery) +- bitcoin/bitcoin#20987 Add instructions for generating RPC docs (ben-kaufman) +- bitcoin/bitcoin#21026 Document use of make-tag script to make tags (laanwj) +- bitcoin/bitcoin#21028 doc/bips: Add BIPs 43, 44, 49, and 84 (luke-jr) +- bitcoin/bitcoin#21049 Add release notes for listdescriptors RPC (S3RK) +- bitcoin/bitcoin#21060 More precise -debug and -debugexclude doc (wodry) +- bitcoin/bitcoin#21077 Clarify -timeout and -peertimeout config options (glozow) +- bitcoin/bitcoin#21105 Correctly identify script type (niftynei) +- bitcoin/bitcoin#21163 Guix is shipped in Debian and Ubuntu (MarcoFalke) +- bitcoin/bitcoin#21210 Rework internal and external links (MarcoFalke) +- bitcoin/bitcoin#21246 Correction for VerifyTaprootCommitment comments (roconnor-blockstream) +- bitcoin/bitcoin#21263 Clarify that squashing should happen before review (MarcoFalke) +- bitcoin/bitcoin#21323 guix, doc: Update default HOSTS value (hebasto) +- bitcoin/bitcoin#21324 Update build instructions for Fedora (hebasto) +- bitcoin/bitcoin#21343 Revamp macOS build doc (jarolrod) +- bitcoin/bitcoin#21346 install qt5 when building on macOS (fanquake) +- bitcoin/bitcoin#21384 doc: add signet to bitcoin.conf documentation (jonatack) +- bitcoin/bitcoin#21394 Improve comment about protected peers (amitiuttarwar) +- bitcoin/bitcoin#21398 Update fuzzing docs for afl-clang-lto (MarcoFalke) +- bitcoin/bitcoin#21444 net, doc: Doxygen updates and fixes in netbase.{h,cpp} (jonatack) +- bitcoin/bitcoin#21481 Tell howto install clang-format on Debian/Ubuntu (wodry) +- bitcoin/bitcoin#21567 Fix various misleading comments (glozow) +- bitcoin/bitcoin#21661 Fix name of script guix-build (Emzy) +- bitcoin/bitcoin#21672 Remove boostrap info from `GUIX_COMMON_FLAGS` doc (fanquake) +- bitcoin/bitcoin#21688 Note on SDK for macOS depends cross-compile (jarolrod) +- bitcoin/bitcoin#21709 Update reduce-memory.md and bitcoin.conf -maxconnections info (jonatack) +- bitcoin/bitcoin#21710 update helps for addnode rpc and -addnode/-maxconnections config options (jonatack) +- bitcoin/bitcoin#21752 Clarify that feerates are per virtual size (MarcoFalke) +- bitcoin/bitcoin#21811 Remove Visual Studio 2017 reference from readme (sipsorcery) +- bitcoin/bitcoin#21818 Fixup -coinstatsindex help, update bitcoin.conf and files.md (jonatack) +- bitcoin/bitcoin#21856 add OSS-Fuzz section to fuzzing.md doc (adamjonas) +- bitcoin/bitcoin#21912 Remove mention of priority estimation (MarcoFalke) +- bitcoin/bitcoin#21925 Update bips.md for 0.21.1 (MarcoFalke) +- bitcoin/bitcoin#21942 improve make with parallel jobs description (klementtan) +- bitcoin/bitcoin#21947 Fix OSS-Fuzz links (MarcoFalke) +- bitcoin/bitcoin#21988 note that brew installed qt is not supported (jarolrod) +- bitcoin/bitcoin#22056 describe in fuzzing.md how to reproduce a CI crash (jonatack) +- bitcoin/bitcoin#22080 add maxuploadtarget to bitcoin.conf example (jarolrod) +- bitcoin/bitcoin#22088 Improve note on choosing posix mingw32 (jarolrod) +- bitcoin/bitcoin#22109 Fix external links (IRC, …) (MarcoFalke) +- bitcoin/bitcoin#22121 Various validation doc fixups (MarcoFalke) +- bitcoin/bitcoin#22172 Update tor.md, release notes with removal of tor v2 support (jonatack) +- bitcoin/bitcoin#22204 Remove obsolete `okSafeMode` RPC guideline from developer notes (theStack) +- bitcoin/bitcoin#22208 Update `REVIEWERS` (practicalswift) +- bitcoin/bitcoin#22250 add basic I2P documentation (vasild) +- bitcoin/bitcoin#22296 Final merge of release notes snippets, mv to wiki (MarcoFalke) +- bitcoin/bitcoin#22335 recommend `--disable-external-signer` in OpenBSD build guide (theStack) +- bitcoin/bitcoin#22339 Document minimum required libc++ version (hebasto) +- bitcoin/bitcoin#22349 Repository IRC updates (jonatack) +- bitcoin/bitcoin#22360 Remove unused section from release process (MarcoFalke) +- bitcoin/bitcoin#22369 Add steps for Transifex to release process (jonatack) +- bitcoin/bitcoin#22393 Added info to bitcoin.conf doc (bliotti) +- bitcoin/bitcoin#22402 Install Rosetta on M1-macOS for qt in depends (hebasto) +- bitcoin/bitcoin#22432 Fix incorrect `testmempoolaccept` doc (glozow) +- bitcoin/bitcoin#22648 doc, test: improve i2p/tor docs and i2p reachable unit tests (jonatack) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Aaron Clauson +- Adam Jonas +- amadeuszpawlik +- Amiti Uttarwar +- Andrew Chow +- Andrew Poelstra +- Anthony Towns +- Antoine Poinsot +- Antoine Riard +- apawlik +- apitko +- Ben Carman +- Ben Woosley +- benk10 +- Bezdrighin +- Block Mechanic +- Brian Liotti +- Bruno Garcia +- Carl Dong +- Christian Decker +- coinforensics +- Cory Fields +- Dan Benjamin +- Daniel Kraft +- Darius Parvin +- Dhruv Mehta +- Dmitry Goncharov +- Dmitry Petukhov +- dplusplus1024 +- dscotese +- Duncan Dean +- Elle Mouton +- Elliott Jin +- Emil Engler +- Ethan Heilman +- eugene +- Evan Klitzke +- Fabian Jahr +- Fabrice Fontaine +- fanquake +- fdov +- flack +- Fotis Koutoupas +- Fu Yong Quah +- fyquah +- glozow +- Gregory Sanders +- Guido Vranken +- Gunar C. Gessner +- h +- HAOYUatHZ +- Hennadii Stepanov +- Igor Cota +- Ikko Ashimine +- Ivan Metlushko +- jackielove4u +- James O'Beirne +- Jarol Rodriguez +- Joel Klabo +- John Newbery +- Jon Atack +- Jonas Schnelli +- João Barbosa +- Josiah Baker +- Karl-Johan Alm +- Kiminuo +- Klement Tan +- Kristaps Kaupe +- Larry Ruane +- lisa neigut +- Lucas Ontivero +- Luke Dashjr +- Maayan Keshet +- MarcoFalke +- Martin Ankerl +- Martin Zumsande +- Michael Dietz +- Michael Polzer +- Michael Tidwell +- Niklas Gögge +- nthumann +- Oliver Gugger +- parazyd +- Patrick Strateman +- Pavol Rusnak +- Peter Bushnell +- Pierre K +- Pieter Wuille +- PiRK +- pox +- practicalswift +- Prayank +- R E Broadley +- Rafael Sadowski +- randymcmillan +- Raul Siles +- Riccardo Spagni +- Russell O'Connor +- Russell Yanofsky +- S3RK +- saibato +- Samuel Dobson +- sanket1729 +- Sawyer Billings +- Sebastian Falbesoner +- setpill +- sgulls +- sinetek +- Sjors Provoost +- Sriram +- Stephan Oeste +- Suhas Daftuar +- Sylvain Goumy +- t-bast +- Troy Giorshev +- Tushar Singla +- Tyler Chambers +- Uplab +- Vasil Dimov +- W. J. van der Laan +- willcl-ark +- William Bright +- William Casarin +- windsok +- wodry +- Yerzhan Mazhkenov +- Yuval Kogman +- Zero + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-process.md b/doc/release-process.md index 3ead1181b9..6a5202d0f9 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -37,15 +37,19 @@ Release Process - This update should be reviewed with a reindex-chainstate with assumevalid=0 to catch any defect that causes rejection of blocks in the past history. - Clear the release notes and move them to the wiki (see "Write the release notes" below). - -#### After branch-off (on master) - -- Update the version of `contrib/gitian-descriptors/*.yml`. +- Translations on Transifex + - Create [a new resource](https://www.transifex.com/bitcoin/bitcoin/content/) named after the major version with the slug `[bitcoin.qt-translation-<RRR>x]`, where `RRR` is the major branch number padded with zeros. Use `src/qt/locale/bitcoin_en.xlf` to create it. + - In the project workflow settings, ensure that [Translation Memory Fill-up](https://docs.transifex.com/translation-memory/enabling-autofill) is enabled and that [Translation Memory Context Matching](https://docs.transifex.com/translation-memory/translation-memory-with-context) is disabled. + - Update the Transifex slug in [`.tx/config`](/.tx/config) to the slug of the resource created in the first step. This identifies which resource the translations will be synchronized from. + - Make an announcement that translators can start translating for the new version. You can use one of the [previous announcements](https://www.transifex.com/bitcoin/bitcoin/announcements/) as a template. + - Change the auto-update URL for the resource to `master`, e.g. `https://raw.githubusercontent.com/bitcoin/bitcoin/master/src/qt/locale/bitcoin_en.xlf`. (Do this only after the previous steps, to prevent an auto-update from interfering.) #### After branch-off (on the major release branch) - Update the versions. - Create a pinned meta-issue for testing the release candidate (see [this issue](https://github.com/bitcoin/bitcoin/issues/17079) for an example) and provide a link to it in the release announcements where useful. +- Translations on Transifex + - Change the auto-update URL for the new major version's resource away from `master` and to the branch, e.g. `https://raw.githubusercontent.com/bitcoin/bitcoin/<branch>/src/qt/locale/bitcoin_en.xlf`. Do not forget this or it will keep tracking the translations on master instead, drifting away from the specific major release. #### Before final release @@ -64,14 +68,14 @@ This will perform a few last-minute consistency checks in the build system files ### First time / New builders -If you're using the automated script (found in [contrib/gitian-build.py](/contrib/gitian-build.py)), then at this point you should run it with the "--setup" command. Otherwise ignore this. +Install Guix using one of the installation methods detailed in +[contrib/guix/INSTALL.md](/contrib/guix/INSTALL.md). Check out the source code in the following directory hierarchy. cd /path/to/your/toplevel/build - git clone https://github.com/bitcoin-core/gitian.sigs.git + git clone https://github.com/bitcoin-core/guix.sigs.git git clone https://github.com/bitcoin-core/bitcoin-detached-sigs.git - git clone https://github.com/devrandom/gitian-builder.git git clone https://github.com/bitcoin/bitcoin.git ### Write the release notes @@ -86,110 +90,56 @@ Generate list of authors: git log --format='- %aN' v(current version, e.g. 0.20.0)..v(new version, e.g. 0.20.1) | sort -fiu -### Setup and perform Gitian builds - -If you're using the automated script (found in [contrib/gitian-build.py](/contrib/gitian-build.py)), then at this point you should run it with the "--build" command. Otherwise ignore this. - -Setup Gitian descriptors: - - pushd ./bitcoin - export SIGNER="(your Gitian key, ie bluematt, sipa, etc)" - export VERSION=(new version, e.g. 0.20.0) - git fetch - git checkout v${VERSION} - popd - -Ensure your gitian.sigs are up-to-date if you wish to gverify your builds against other Gitian signatures. - - pushd ./gitian.sigs - git pull - popd - -Ensure gitian-builder is up-to-date: - - pushd ./gitian-builder - git pull - popd - -### Fetch and create inputs: (first time, or when dependency versions change) - - pushd ./gitian-builder - mkdir -p inputs - wget -O inputs/osslsigncode-2.0.tar.gz https://github.com/mtrojnar/osslsigncode/archive/2.0.tar.gz - echo '5a60e0a4b3e0b4d655317b2f12a810211c50242138322b16e7e01c6fbb89d92f inputs/osslsigncode-2.0.tar.gz' | sha256sum -c - popd - -Create the macOS SDK tarball, see the [macdeploy instructions](/contrib/macdeploy/README.md#deterministic-macos-dmg-notes) for details, and copy it into the inputs directory. - -### Optional: Seed the Gitian sources cache and offline git repositories - -NOTE: Gitian is sometimes unable to download files. If you have errors, try the step below. - -By default, Gitian will fetch source files as needed. To cache them ahead of time, make sure you have checked out the tag you want to build in bitcoin, then: - - pushd ./gitian-builder - make -C ../bitcoin/depends download SOURCES_PATH=`pwd`/cache/common - popd - -Only missing files will be fetched, so this is safe to re-run for each build. - -NOTE: Offline builds must use the --url flag to ensure Gitian fetches only from local URLs. For example: +### Setup and perform Guix builds - pushd ./gitian-builder - ./bin/gbuild --url bitcoin=/path/to/bitcoin,signature=/path/to/sigs {rest of arguments} - popd +Checkout the Bitcoin Core version you'd like to build: -The gbuild invocations below <b>DO NOT DO THIS</b> by default. - -### Build and sign Bitcoin Core for Linux, Windows, and macOS: +```sh +pushd ./bitcoin +SIGNER='(your builder key, ie bluematt, sipa, etc)' +VERSION='(new version without v-prefix, e.g. 0.20.0)' +git fetch "v${VERSION}" +git checkout "v${VERSION}" +popd +``` - pushd ./gitian-builder - ./bin/gbuild --num-make 2 --memory 3000 --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml - ./bin/gsign --signer "$SIGNER" --release ${VERSION}-linux --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml - mv build/out/bitcoin-*.tar.gz build/out/src/bitcoin-*.tar.gz ../ +Ensure your guix.sigs are up-to-date if you wish to `guix-verify` your builds +against other `guix-attest` signatures. - ./bin/gbuild --num-make 2 --memory 3000 --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-win.yml - ./bin/gsign --signer "$SIGNER" --release ${VERSION}-win-unsigned --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-win.yml - mv build/out/bitcoin-*-win-unsigned.tar.gz inputs/bitcoin-win-unsigned.tar.gz - mv build/out/bitcoin-*.zip build/out/bitcoin-*.exe ../ +```sh +git -C ./guix.sigs pull +``` - ./bin/gbuild --num-make 2 --memory 3000 --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml - ./bin/gsign --signer "$SIGNER" --release ${VERSION}-osx-unsigned --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml - mv build/out/bitcoin-*-osx-unsigned.tar.gz inputs/bitcoin-osx-unsigned.tar.gz - mv build/out/bitcoin-*.tar.gz build/out/bitcoin-*.dmg ../ - popd +### Create the macOS SDK tarball: (first time, or when SDK version changes) -Build output expected: +Create the macOS SDK tarball, see the [macdeploy +instructions](/contrib/macdeploy/README.md#deterministic-macos-dmg-notes) for +details. - 1. source tarball (`bitcoin-${VERSION}.tar.gz`) - 2. linux 32-bit and 64-bit dist tarballs (`bitcoin-${VERSION}-linux[32|64].tar.gz`) - 3. windows 32-bit and 64-bit unsigned installers and dist zips (`bitcoin-${VERSION}-win[32|64]-setup-unsigned.exe`, `bitcoin-${VERSION}-win[32|64].zip`) - 4. macOS unsigned installer and dist tarball (`bitcoin-${VERSION}-osx-unsigned.dmg`, `bitcoin-${VERSION}-osx64.tar.gz`) - 5. Gitian signatures (in `gitian.sigs/${VERSION}-<linux|{win,osx}-unsigned>/(your Gitian key)/`) +### Build and attest to build outputs: -### Verify other gitian builders signatures to your own. (Optional) +Follow the relevant Guix README.md sections: +- [Performing a build](/contrib/guix/README.md#performing-a-build) +- [Attesting to build outputs](/contrib/guix/README.md#attesting-to-build-outputs) -Add other gitian builders keys to your gpg keyring, and/or refresh keys: See `../bitcoin/contrib/gitian-keys/README.md`. +### Verify other builders' signatures to your own. (Optional) -Verify the signatures +Add other builders keys to your gpg keyring, and/or refresh keys: See `../bitcoin/contrib/builder-keys/README.md`. - pushd ./gitian-builder - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-linux ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-win-unsigned ../bitcoin/contrib/gitian-descriptors/gitian-win.yml - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-osx-unsigned ../bitcoin/contrib/gitian-descriptors/gitian-osx.yml - popd +Follow the relevant Guix README.md sections: +- [Verifying build output attestations](/contrib/guix/README.md#verifying-build-output-attestations) ### Next steps: -Commit your signature to gitian.sigs: +Commit your signature to guix.sigs: - pushd gitian.sigs - git add ${VERSION}-linux/"${SIGNER}" - git add ${VERSION}-win-unsigned/"${SIGNER}" - git add ${VERSION}-osx-unsigned/"${SIGNER}" - git commit -m "Add ${VERSION} unsigned sigs for ${SIGNER}" - git push # Assuming you can push to the gitian.sigs tree - popd +```sh +pushd ./guix.sigs +git add "${VERSION}/${SIGNER}"/noncodesigned.SHA256SUMS{,.asc} +git commit -m "Add attestations by ${SIGNER} for ${VERSION} non-codesigned" +git push # Assuming you can push to the guix.sigs tree +popd +``` Codesigner only: Create Windows/macOS detached signatures: - Only one person handles codesigning. Everyone else should skip to the next step. @@ -201,7 +151,7 @@ Codesigner only: Sign the macOS binary: tar xf bitcoin-osx-unsigned.tar.gz ./detached-sig-create.sh -s "Key ID" Enter the keychain password and authorize the signature - Move signature-osx.tar.gz back to the gitian host + Move signature-osx.tar.gz back to the guix-build host Codesigner only: Sign the windows binaries: @@ -210,110 +160,91 @@ Codesigner only: Sign the windows binaries: Enter the passphrase for the key when prompted signature-win.tar.gz will be created +Code-signer only: It is advised to test that the code signature attaches properly prior to tagging by performing the `guix-codesign` step. +However if this is done, once the release has been tagged in the bitcoin-detached-sigs repo, the `guix-codesign` step must be performed again in order for the guix attestation to be valid when compared against the attestations of non-codesigner builds. + Codesigner only: Commit the detached codesign payloads: - cd ~/bitcoin-detached-sigs - checkout the appropriate branch for this release series - rm -rf * - tar xf signature-osx.tar.gz - tar xf signature-win.tar.gz - git add -A - git commit -m "point to ${VERSION}" - git tag -s v${VERSION} HEAD - git push the current branch and new tag +```sh +pushd ./bitcoin-detached-sigs +# checkout the appropriate branch for this release series +rm -rf ./* +tar xf signature-osx.tar.gz +tar xf signature-win.tar.gz +git add -A +git commit -m "point to ${VERSION}" +git tag -s "v${VERSION}" HEAD +git push the current branch and new tag +popd +``` Non-codesigners: wait for Windows/macOS detached signatures: - Once the Windows/macOS builds each have 3 matching signatures, they will be signed with their respective release keys. - Detached signatures will then be committed to the [bitcoin-detached-sigs](https://github.com/bitcoin-core/bitcoin-detached-sigs) repository, which can be combined with the unsigned apps to create signed binaries. -Create (and optionally verify) the signed macOS binary: - - pushd ./gitian-builder - ./bin/gbuild -i --commit signature=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml - ./bin/gsign --signer "$SIGNER" --release ${VERSION}-osx-signed --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-osx-signed ../bitcoin/contrib/gitian-descriptors/gitian-osx-signer.yml - mv build/out/bitcoin-osx-signed.dmg ../bitcoin-${VERSION}-osx.dmg - popd +Create (and optionally verify) the codesigned outputs: -Create (and optionally verify) the signed Windows binaries: - - pushd ./gitian-builder - ./bin/gbuild -i --commit signature=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml - ./bin/gsign --signer "$SIGNER" --release ${VERSION}-win-signed --destination ../gitian.sigs/ ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml - ./bin/gverify -v -d ../gitian.sigs/ -r ${VERSION}-win-signed ../bitcoin/contrib/gitian-descriptors/gitian-win-signer.yml - mv build/out/bitcoin-*win64-setup.exe ../bitcoin-${VERSION}-win64-setup.exe - popd +- [Codesigning](/contrib/guix/README.md#codesigning) Commit your signature for the signed macOS/Windows binaries: - pushd gitian.sigs - git add ${VERSION}-osx-signed/"${SIGNER}" - git add ${VERSION}-win-signed/"${SIGNER}" - git commit -m "Add ${SIGNER} ${VERSION} signed binaries signatures" - git push # Assuming you can push to the gitian.sigs tree - popd +```sh +pushd ./guix.sigs +git add "${VERSION}/${SIGNER}"/all.SHA256SUMS{,.asc} +git commit -m "Add attestations by ${SIGNER} for ${VERSION} codesigned" +git push # Assuming you can push to the guix.sigs tree +popd +``` -### After 3 or more people have gitian-built and their results match: +### After 3 or more people have guix-built and their results match: -- Create `SHA256SUMS.asc` for the builds, and GPG-sign it: +Combine the `all.SHA256SUMS.asc` file from all signers into `SHA256SUMS.asc`: ```bash -sha256sum * > SHA256SUMS +cat "$VERSION"/*/all.SHA256SUMS.asc > SHA256SUMS.asc ``` -The list of files should be: -``` -bitcoin-${VERSION}-aarch64-linux-gnu.tar.gz -bitcoin-${VERSION}-arm-linux-gnueabihf.tar.gz -bitcoin-${VERSION}-riscv64-linux-gnu.tar.gz -bitcoin-${VERSION}-x86_64-linux-gnu.tar.gz -bitcoin-${VERSION}-osx64.tar.gz -bitcoin-${VERSION}-osx.dmg -bitcoin-${VERSION}.tar.gz -bitcoin-${VERSION}-win64-setup.exe -bitcoin-${VERSION}-win64.zip -``` -The `*-debug*` files generated by the gitian build contain debug symbols -for troubleshooting by developers. It is assumed that anyone that is interested -in debugging can run gitian to generate the files for themselves. To avoid -end-user confusion about which file to pick, as well as save storage -space *do not upload these to the bitcoin.org server, nor put them in the torrent*. -- GPG-sign it, delete the unsigned file: -``` -gpg --digest-algo sha256 --clearsign SHA256SUMS # outputs SHA256SUMS.asc -rm SHA256SUMS -``` -(the digest algorithm is forced to sha256 to avoid confusion of the `Hash:` header that GPG adds with the SHA256 used for the files) -Note: check that SHA256SUMS itself doesn't end up in SHA256SUMS, which is a spurious/nonsensical entry. +- Upload to the bitcoincore.org server (`/var/www/bin/bitcoin-core-${VERSION}/`): + 1. The contents of each `./bitcoin/guix-build-${VERSION}/output/${HOST}/` directory, except for + `*-debug*` files. -- Upload zips and installers, as well as `SHA256SUMS.asc` from last step, to the bitcoin.org server - into `/var/www/bin/bitcoin-core-${VERSION}` + Guix will output all of the results into host subdirectories, but the SHA256SUMS + file does not include these subdirectories. In order for downloads via torrent + to verify without directory structure modification, all of the uploaded files + need to be in the same directory as the SHA256SUMS file. -- A `.torrent` will appear in the directory after a few minutes. Optionally help seed this torrent. To get the `magnet:` URI use: -```bash -transmission-show -m <torrent file> -``` -Insert the magnet URI into the announcement sent to mailing lists. This permits -people without access to `bitcoin.org` to download the binary distribution. -Also put it into the `optional_magnetlink:` slot in the YAML file for -bitcoin.org (see below for bitcoin.org update instructions). + The `*-debug*` files generated by the guix build contain debug symbols + for troubleshooting by developers. It is assumed that anyone that is + interested in debugging can run guix to generate the files for + themselves. To avoid end-user confusion about which file to pick, as well + as save storage space *do not upload these to the bitcoincore.org server, + nor put them in the torrent*. + + ```sh + find guix-build-${VERSION}/output/ -maxdepth 2 -type f -not -name "SHA256SUMS.part" -and -not -name "*debug*" -exec scp {} user@bitcoincore.org:/var/www/bin/bitcoin-core-${VERSION} \; + ``` + + 2. The `SHA256SUMS` file -- Update bitcoin.org version + 3. The `SHA256SUMS.asc` combined signature file you just created - - First, check to see if the Bitcoin.org maintainers have prepared a - release: https://github.com/bitcoin-dot-org/bitcoin.org/labels/Core +- Create a torrent of the `/var/www/bin/bitcoin-core-${VERSION}` directory such + that at the top level there is only one file: the `bitcoin-core-${VERSION}` + directory containing everything else. Name the torrent + `bitcoin-${VERSION}.torrent` (note that there is no `-core-` in this name). - - If they have, it will have previously failed their CI - checks because the final release files weren't uploaded. - Trigger a CI rebuild---if it passes, merge. + Optionally help seed this torrent. To get the `magnet:` URI use: - - If they have not prepared a release, follow the Bitcoin.org release - instructions: https://github.com/bitcoin-dot-org/bitcoin.org/blob/master/docs/adding-events-release-notes-and-alerts.md#release-notes + ```sh + transmission-show -m <torrent file> + ``` - - After the pull request is merged, the website will automatically show the newest version within 15 minutes, as well - as update the OS download links. + Insert the magnet URI into the announcement sent to mailing lists. This permits + people without access to `bitcoincore.org` to download the binary distribution. + Also put it into the `optional_magnetlink:` slot in the YAML file for + bitcoincore.org. - Update other repositories and websites for new version @@ -340,26 +271,7 @@ bitcoin.org (see below for bitcoin.org update instructions). - Push the flatpak to flathub, e.g. https://github.com/flathub/org.bitcoincore.bitcoin-qt/pull/2 - - Push the latest version to master (if applicable), e.g. https://github.com/bitcoin-core/packaging/pull/32 - - - Create a new branch for the major release "0.xx" from master (used to build the snap package) and request the - track (if applicable), e.g. https://forum.snapcraft.io/t/track-request-for-bitcoin-core-snap/10112/7 - - - Notify MarcoFalke so that he can start building the snap package - - - https://code.launchpad.net/~bitcoin-core/bitcoin-core-snap/+git/packaging (Click "Import Now" to fetch the branch) - - https://code.launchpad.net/~bitcoin-core/bitcoin-core-snap/+git/packaging/+ref/0.xx (Click "Create snap package") - - Name it "bitcoin-core-snap-0.xx" - - Leave owner and series as-is - - Select architectures that are compiled via gitian - - Leave "automatically build when branch changes" unticked - - Tick "automatically upload to store" - - Put "bitcoin-core" in the registered store package name field - - Tick the "edge" box - - Put "0.xx" in the track field - - Click "create snap package" - - Click "Request builds" for every new release on this branch (after updating the snapcraft.yml in the branch to reflect the latest gitian results) - - Promote release on https://snapcraft.io/bitcoin-core/releases if it passes sanity checks + - Push the snap, see https://github.com/bitcoin-core/packaging/blob/master/snap/build.md - This repo diff --git a/doc/tor.md b/doc/tor.md index 7d134b64e0..8dc82ca91e 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -23,10 +23,9 @@ There are several ways to see your local onion address in Bitcoin Core: You may set the `-debug=tor` config logging option to have additional information in the debug log about your Tor configuration. -CLI `-addrinfo` returns the number of addresses known to your node per network -type, including Tor v2 and v3. This is useful to see how many onion addresses -are known to your node for `-onlynet=onion` and how many Tor v3 addresses it -knows when upgrading to Bitcoin Core v22.0 and up that supports Tor v3 only. +CLI `-addrinfo` returns the number of addresses known to your node per +network. This can be useful to see how many onion peers your node knows, +e.g. for `-onlynet=onion`. ## 1. Run Bitcoin Core behind a Tor proxy @@ -57,11 +56,11 @@ outgoing connections, but more is possible. -onlynet=onion Make outgoing connections only to .onion addresses. Incoming connections are not affected by this option. This option can be specified multiple times to allow multiple network types, e.g. - ipv4, ipv6 or onion. If you use this option with values other - than onion you *cannot* disable onion connections; outgoing onion - connections will be enabled when you use -proxy or -onion. Use - -noonion or -onion=0 if you want to be sure there are no outbound - onion connections over the default proxy or your defined -proxy. + onlynet=ipv4, onlynet=ipv6, onlynet=onion, onlynet=i2p. + Warning: if you use -onlynet with values other than onion, and + the -onion or -proxy option is set, then outgoing onion + connections will still be made; use -noonion or -onion=0 to + disable outbound onion connections in this case. In a typical situation, this suffices to run behind a Tor proxy: @@ -134,7 +133,7 @@ You can also check the group of the cookie file. On most Linux systems, the Tor auth cookie will usually be `/run/tor/control.authcookie`: ``` -stat -c '%G' /run/tor/control.authcookie +TORGROUP=$(stat -c '%G' /run/tor/control.authcookie) ``` Once you have determined the `${TORGROUP}` and selected the `${USER}` that will diff --git a/doc/tracing.md b/doc/tracing.md new file mode 100644 index 0000000000..87fc9603fe --- /dev/null +++ b/doc/tracing.md @@ -0,0 +1,266 @@ +# User-space, Statically Defined Tracing (USDT) for Bitcoin Core + +Bitcoin Core includes statically defined tracepoints to allow for more +observability during development, debugging, code review, and production usage. +These tracepoints make it possible to keep track of custom statistics and +enable detailed monitoring of otherwise hidden internals. They have +little to no performance impact when unused. + +``` +eBPF and USDT Overview +====================== + + ┌──────────────────┐ ┌──────────────┐ + │ tracing script │ │ bitcoind │ + │==================│ 2. │==============│ + │ eBPF │ tracing │ hooks │ │ + │ code │ logic │ into┌─┤►tracepoint 1─┼───┐ 3. + └────┬───┴──▲──────┘ ├─┤►tracepoint 2 │ │ pass args + 1. │ │ 4. │ │ ... │ │ to eBPF + User compiles │ │ pass data to │ └──────────────┘ │ program + Space & loads │ │ tracing script │ │ + ─────────────────┼──────┼─────────────────┼────────────────────┼─── + Kernel │ │ │ │ + Space ┌──┬─▼──────┴─────────────────┴────────────┐ │ + │ │ eBPF program │◄──────┘ + │ └───────────────────────────────────────┤ + │ eBPF kernel Virtual Machine (sandboxed) │ + └──────────────────────────────────────────┘ + +1. The tracing script compiles the eBPF code and loads the eBPF program into a kernel VM +2. The eBPF program hooks into one or more tracepoints +3. When the tracepoint is called, the arguments are passed to the eBPF program +4. The eBPF program processes the arguments and returns data to the tracing script +``` + +The Linux kernel can hook into the tracepoints during runtime and pass data to +sandboxed [eBPF] programs running in the kernel. These eBPF programs can, for +example, collect statistics or pass data back to user-space scripts for further +processing. + +[eBPF]: https://ebpf.io/ + +The two main eBPF front-ends with support for USDT are [bpftrace] and +[BPF Compiler Collection (BCC)]. BCC is used for complex tools and daemons and +`bpftrace` is preferred for one-liners and shorter scripts. Examples for both can +be found in [contrib/tracing]. + +[bpftrace]: https://github.com/iovisor/bpftrace +[BPF Compiler Collection (BCC)]: https://github.com/iovisor/bcc +[contrib/tracing]: ../contrib/tracing/ + +## Tracepoint documentation + +The currently available tracepoints are listed here. + +### Context `net` + +#### Tracepoint `net:inbound_message` + +Is called when a message is received from a peer over the P2P network. Passes +information about our peer, the connection and the message as arguments. + +Arguments passed: +1. Peer ID as `int64` +2. Peer Address and Port (IPv4, IPv6, Tor v3, I2P, ...) as `pointer to C-style String` (max. length 68 characters) +3. Connection Type (inbound, feeler, outbound-full-relay, ...) as `pointer to C-style String` (max. length 20 characters) +4. Message Type (inv, ping, getdata, addrv2, ...) as `pointer to C-style String` (max. length 20 characters) +5. Message Size in bytes as `uint64` +6. Message Bytes as `pointer to unsigned chars` (i.e. bytes) + +Note: The message is passed to the tracepoint in full, however, due to space +limitations in the eBPF kernel VM it might not be possible to pass the message +to user-space in full. Messages longer than a 32kb might be cut off. This can +be detected in tracing scripts by comparing the message size to the length of +the passed message. + +#### Tracepoint `net:outbound_message` + +Is called when a message is send to a peer over the P2P network. Passes +information about our peer, the connection and the message as arguments. + +Arguments passed: +1. Peer ID as `int64` +2. Peer Address and Port (IPv4, IPv6, Tor v3, I2P, ...) as `pointer to C-style String` (max. length 68 characters) +3. Connection Type (inbound, feeler, outbound-full-relay, ...) as `pointer to C-style String` (max. length 20 characters) +4. Message Type (inv, ping, getdata, addrv2, ...) as `pointer to C-style String` (max. length 20 characters) +5. Message Size in bytes as `uint64` +6. Message Bytes as `pointer to unsigned chars` (i.e. bytes) + +Note: The message is passed to the tracepoint in full, however, due to space +limitations in the eBPF kernel VM it might not be possible to pass the message +to user-space in full. Messages longer than a 32kb might be cut off. This can +be detected in tracing scripts by comparing the message size to the length of +the passed message. + +### Context `validation` + +#### Tracepoint `validation:block_connected` + +Is called *after* a block is connected to the chain. Can, for example, be used +to benchmark block connections together with `-reindex`. + +Arguments passed: +1. Block Header Hash as `pointer to C-style String` (64 characters) +2. Block Height as `int32` +3. Transactions in the Block as `uint64` +4. Inputs spend in the Block as `int32` +5. SigOps in the Block (excluding coinbase SigOps) `uint64` +6. Time it took to connect the Block in microseconds (µs) as `uint64` +7. Block Header Hash as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) + +Note: The 7th argument can't be accessed by bpftrace and is purposefully chosen +to be the block header hash as bytes. See [bpftrace argument limit] for more +details. + +[bpftrace argument limit]: #bpftrace-argument-limit + +## Adding tracepoints to Bitcoin Core + +To add a new tracepoint, `#include <util/trace.h>` in the compilation unit where +the tracepoint is inserted. Use one of the `TRACEx` macros listed below +depending on the number of arguments passed to the tracepoint. Up to 12 +arguments can be provided. The `context` and `event` specify the names by which +the tracepoint is referred to. Please use `snake_case` and try to make sure that +the tracepoint names make sense even without detailed knowledge of the +implementation details. Do not forget to update the tracepoint list in this +document. + +```c +#define TRACE(context, event) +#define TRACE1(context, event, a) +#define TRACE2(context, event, a, b) +#define TRACE3(context, event, a, b, c) +#define TRACE4(context, event, a, b, c, d) +#define TRACE5(context, event, a, b, c, d, e) +#define TRACE6(context, event, a, b, c, d, e, f) +#define TRACE7(context, event, a, b, c, d, e, f, g) +#define TRACE8(context, event, a, b, c, d, e, f, g, h) +#define TRACE9(context, event, a, b, c, d, e, f, g, h, i) +#define TRACE10(context, event, a, b, c, d, e, f, g, h, i, j) +#define TRACE11(context, event, a, b, c, d, e, f, g, h, i, j, k) +#define TRACE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) +``` + +For example: + +```C++ +TRACE6(net, inbound_message, + pnode->GetId(), + pnode->m_addr_name.c_str(), + pnode->ConnectionTypeAsString().c_str(), + sanitizedType.c_str(), + msg.data.size(), + msg.data.data() +); +``` + +### Guidelines and best practices + +#### Clear motivation and use-case +Tracepoints need a clear motivation and use-case. The motivation should +outweigh the impact on, for example, code readability. There is no point in +adding tracepoints that don't end up being used. + +#### Provide an example +When adding a new tracepoint, provide an example. Examples can show the use case +and help reviewers testing that the tracepoint works as intended. The examples +can be kept simple but should give others a starting point when working with +the tracepoint. See existing examples in [contrib/tracing/]. + +[contrib/tracing/]: ../contrib/tracing/ + +#### No expensive computations for tracepoints +Data passed to the tracepoint should be inexpensive to compute. Although the +tracepoint itself only has overhead when enabled, the code to compute arguments +is always run - even if the tracepoint is not used. For example, avoid +serialization and parsing. + +#### Semi-stable API +Tracepoints should have a semi-stable API. Users should be able to rely on the +tracepoints for scripting. This means tracepoints need to be documented, and the +argument order ideally should not change. If there is an important reason to +change argument order, make sure to document the change and update the examples +using the tracepoint. + +#### eBPF Virtual Machine limits +Keep the eBPF Virtual Machine limits in mind. eBPF programs receiving data from +the tracepoints run in a sandboxed Linux kernel VM. This VM has a limited stack +size of 512 bytes. Check if it makes sense to pass larger amounts of data, for +example, with a tracing script that can handle the passed data. + +#### `bpftrace` argument limit +While tracepoints can have up to 12 arguments, bpftrace scripts currently only +support reading from the first six arguments (`arg0` till `arg5`) on `x86_64`. +bpftrace currently lacks real support for handling and printing binary data, +like block header hashes and txids. When a tracepoint passes more than six +arguments, then string and integer arguments should preferably be placed in the +first six argument fields. Binary data can be placed in later arguments. The BCC +supports reading from all 12 arguments. + +#### Strings as C-style String +Generally, strings should be passed into the `TRACEx` macros as pointers to +C-style strings (a null-terminated sequence of characters). For C++ +`std::strings`, [`c_str()`] can be used. It's recommended to document the +maximum expected string size if known. + + +[`c_str()`]: https://www.cplusplus.com/reference/string/string/c_str/ + + +## Listing available tracepoints + +Multiple tools can list the available tracepoints in a `bitcoind` binary with +USDT support. + +### GDB - GNU Project Debugger + +To list probes in Bitcoin Core, use `info probes` in `gdb`: + +``` +$ gdb ./src/bitcoind +… +(gdb) info probes +Type Provider Name Where Semaphore Object +stap net inbound_message 0x000000000014419e /src/bitcoind +stap net outbound_message 0x0000000000107c05 /src/bitcoind +stap validation block_connected 0x00000000002fb10c /src/bitcoind +… +``` + +### With `readelf` + +The `readelf` tool can be used to display the USDT tracepoints in Bitcoin Core. +Look for the notes with the description `NT_STAPSDT`. + +``` +$ readelf -n ./src/bitcoind | grep NT_STAPSDT -A 4 -B 2 +Displaying notes found in: .note.stapsdt + Owner Data size Description + stapsdt 0x0000005d NT_STAPSDT (SystemTap probe descriptors) + Provider: net + Name: outbound_message + Location: 0x0000000000107c05, Base: 0x0000000000579c90, Semaphore: 0x0000000000000000 + Arguments: -8@%r12 8@%rbx 8@%rdi 8@192(%rsp) 8@%rax 8@%rdx +… +``` + +### With `tplist` + +The `tplist` tool is provided by BCC (see [Installing BCC]). It displays kernel +tracepoints or USDT probes and their formats (for more information, see the +[`tplist` usage demonstration]). There are slight binary naming differences +between distributions. For example, on +[Ubuntu the binary is called `tplist-bpfcc`][ubuntu binary]. + +[Installing BCC]: https://github.com/iovisor/bcc/blob/master/INSTALL.md +[`tplist` usage demonstration]: https://github.com/iovisor/bcc/blob/master/tools/tplist_example.txt +[ubuntu binary]: https://github.com/iovisor/bcc/blob/master/INSTALL.md#ubuntu---binary + +``` +$ tplist -l ./src/bitcoind -v +b'net':b'outbound_message' [sema 0x0] + 1 location(s) + 6 argument(s) +… +``` diff --git a/doc/translation_process.md b/doc/translation_process.md index f132693264..97a8fbfff2 100644 --- a/doc/translation_process.md +++ b/doc/translation_process.md @@ -63,17 +63,12 @@ username = USERNAME The Transifex Bitcoin project config file is included as part of the repo. It can be found at `.tx/config`, however you shouldn’t need to change anything. ### Synchronising translations -To assist in updating translations, a helper script is available in the [maintainer-tools repo](https://github.com/bitcoin-core/bitcoin-maintainer-tools). -1. `python3 ../bitcoin-maintainer-tools/update-translations.py` -2. `git add` new translations from `src/qt/locale/` -3. Update `src/qt/bitcoin_locale.qrc` manually or via -```bash -git ls-files src/qt/locale/*ts|xargs -n1 basename|sed 's/\(bitcoin_\(.*\)\).ts/ <file alias="\2">locale\/\1.qm<\/file>/' +To assist in updating translations, a helper script is available in the [maintainer-tools repo](https://github.com/bitcoin-core/bitcoin-maintainer-tools). To use it and commit the result, simply do: + ``` -4. Update `src/Makefile.qt_locale.include` manually or via -```bash -git ls-files src/qt/locale/*ts|xargs -n1 basename|sed 's/\(bitcoin_\(.*\)\).ts/ qt\/locale\/\1.ts \\/' +python3 ../bitcoin-maintainer-tools/update-translations.py +git commit -a ``` **Do not directly download translations** one by one from the Transifex website, as we do a few post-processing steps before committing the translations. diff --git a/doc/zmq.md b/doc/zmq.md index 85f3370130..0521fe08d8 100644 --- a/doc/zmq.md +++ b/doc/zmq.md @@ -84,6 +84,7 @@ For instance: $ bitcoind -zmqpubhashtx=tcp://127.0.0.1:28332 \ -zmqpubhashtx=tcp://192.168.1.2:28332 \ + -zmqpubhashblock="tcp://[::1]:28333" \ -zmqpubrawtx=ipc:///tmp/bitcoind.tx.raw \ -zmqpubhashtxhwm=10000 @@ -125,6 +126,9 @@ Setting the keepalive values appropriately for your operating environment may improve connectivity in situations where long-lived connections are silently dropped by network middle boxes. +Also, the socket's ZMQ_IPV6 option is enabled to accept connections from IPv6 +hosts as well. If needed, this option has to be set on the client side too. + ## Remarks From the perspective of bitcoind, the ZeroMQ socket is write-only; PUB diff --git a/share/examples/bitcoin.conf b/share/examples/bitcoin.conf index 4a947001fa..c5b79709c7 100644 --- a/share/examples/bitcoin.conf +++ b/share/examples/bitcoin.conf @@ -157,7 +157,7 @@ #coinstatsindex=1 # Enable pruning to reduce storage requirements by deleting old blocks. -# This mode is incompatible with -txindex, -coinstatsindex and -rescan. +# This mode is incompatible with -txindex and -coinstatsindex. # 0 = default (no pruning). # 1 = allows manual pruning via RPC. # >=550 = target to stay under in MiB. diff --git a/share/qt/Info.plist.in b/share/qt/Info.plist.in index da10dbb3be..053359e0a8 100644 --- a/share/qt/Info.plist.in +++ b/share/qt/Info.plist.in @@ -3,7 +3,7 @@ <plist version="0.9"> <dict> <key>LSMinimumSystemVersion</key> - <string>10.14.0</string> + <string>10.15.0</string> <key>LSArchitecturePriority</key> <array> @@ -16,6 +16,11 @@ <key>CFBundlePackageType</key> <string>APPL</string> + <key>CFBundleSupportedPlatforms</key> + <array> + <string>MacOSX</string> + </array> + <key>NSHumanReadableCopyright</key> <string>@CLIENT_VERSION_MAJOR@.@CLIENT_VERSION_MINOR@.@CLIENT_VERSION_BUILD@, Copyright © 2009-@COPYRIGHT_YEAR@ @COPYRIGHT_HOLDERS_FINAL@</string> diff --git a/share/rpcauth/rpcauth.py b/share/rpcauth/rpcauth.py index b14c80171e..c6d9b652b8 100755 --- a/share/rpcauth/rpcauth.py +++ b/share/rpcauth/rpcauth.py @@ -5,7 +5,6 @@ from argparse import ArgumentParser from base64 import urlsafe_b64encode -from binascii import hexlify from getpass import getpass from os import urandom @@ -13,7 +12,7 @@ import hmac def generate_salt(size): """Create size byte hex salt""" - return hexlify(urandom(size)).decode() + return urandom(size).hex() def generate_password(): """Create 32 byte b64 password""" diff --git a/src/.clang-tidy b/src/.clang-tidy new file mode 100644 index 0000000000..27616ad072 --- /dev/null +++ b/src/.clang-tidy @@ -0,0 +1,2 @@ +Checks: '-*,bugprone-argument-comment' +WarningsAsErrors: bugprone-argument-comment diff --git a/src/Makefile.am b/src/Makefile.am index e2ed70556d..12fdc9ad75 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -3,7 +3,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. # Pattern rule to print variables, e.g. make print-top_srcdir -print-%: +print-%: FORCE @echo '$*'='$($*)' DIST_SUBDIRS = secp256k1 univalue @@ -117,6 +117,7 @@ endif BITCOIN_CORE_H = \ addrdb.h \ addrman.h \ + addrman_impl.h \ attributes.h \ banman.h \ base58.h \ @@ -145,6 +146,8 @@ BITCOIN_CORE_H = \ core_memusage.h \ cuckoocache.h \ dbwrapper.h \ + deploymentinfo.h \ + deploymentstatus.h \ external_signer.h \ flatfile.h \ fs.h \ @@ -259,6 +262,7 @@ BITCOIN_CORE_H = \ util/sock.h \ util/spanparsing.h \ util/string.h \ + util/syscall_sandbox.h \ util/system.h \ util/thread.h \ util/threadnames.h \ @@ -266,13 +270,13 @@ BITCOIN_CORE_H = \ util/tokenpipe.h \ util/trace.h \ util/translation.h \ + util/types.h \ util/ui_change_type.h \ util/url.h \ util/vector.h \ validation.h \ validationinterface.h \ versionbits.h \ - versionbitsinfo.h \ wallet/bdb.h \ wallet/coincontrol.h \ wallet/coinselection.h \ @@ -328,6 +332,7 @@ libbitcoin_server_a_SOURCES = \ chain.cpp \ consensus/tx_verify.cpp \ dbwrapper.cpp \ + deploymentstatus.cpp \ flatfile.cpp \ httprpc.cpp \ httpserver.cpp \ @@ -495,9 +500,9 @@ crypto_libbitcoin_crypto_shani_a_SOURCES = crypto/sha256_shani.cpp libbitcoin_consensus_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_consensus_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_consensus_a_SOURCES = \ - amount.h \ arith_uint256.cpp \ arith_uint256.h \ + consensus/amount.h \ consensus/merkle.cpp \ consensus/merkle.h \ consensus/params.h \ @@ -540,11 +545,13 @@ libbitcoin_common_a_SOURCES = \ compressor.cpp \ core_read.cpp \ core_write.cpp \ + deploymentinfo.cpp \ external_signer.cpp \ init/common.cpp \ key.cpp \ key_io.cpp \ merkleblock.cpp \ + net_types.cpp \ netaddress.cpp \ netbase.cpp \ net_permissions.cpp \ @@ -561,13 +568,12 @@ libbitcoin_common_a_SOURCES = \ script/sign.cpp \ script/signingprovider.cpp \ script/standard.cpp \ - versionbitsinfo.cpp \ warnings.cpp \ $(BITCOIN_CORE_H) # util: shared between all executables. # This library *must* be included to make sure that the glibc -# backward-compatibility objects and their sanity checks are linked. +# sanity checks are linked. libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_util_a_SOURCES = \ @@ -607,6 +613,7 @@ libbitcoin_util_a_SOURCES = \ util/spanparsing.cpp \ util/strencodings.cpp \ util/string.cpp \ + util/syscall_sandbox.cpp \ util/time.cpp \ util/tokenpipe.cpp \ $(BITCOIN_CORE_H) @@ -615,11 +622,6 @@ if USE_LIBEVENT libbitcoin_util_a_SOURCES += util/url.cpp endif -if GLIBC_BACK_COMPAT -libbitcoin_util_a_SOURCES += compat/glibc_compat.cpp -AM_LDFLAGS += $(COMPAT_LDFLAGS) -endif - # cli: shared between bitcoin-cli and bitcoin-qt libbitcoin_cli_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_cli_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) @@ -747,10 +749,6 @@ if BUILD_BITCOIN_LIBS include_HEADERS = script/bitcoinconsensus.h libbitcoinconsensus_la_SOURCES = support/cleanse.cpp $(crypto_libbitcoin_crypto_base_a_SOURCES) $(libbitcoin_consensus_a_SOURCES) -if GLIBC_BACK_COMPAT - libbitcoinconsensus_la_SOURCES += compat/glibc_compat.cpp -endif - libbitcoinconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) libbitcoinconsensus_la_LIBADD = $(LIBSECP256K1) libbitcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL @@ -812,23 +810,23 @@ clean-local: check-symbols: $(bin_PROGRAMS) if TARGET_DARWIN @echo "Checking macOS dynamic libraries..." - $(AM_V_at) OTOOL=$(OTOOL) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) + $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) endif if TARGET_WINDOWS @echo "Checking Windows dynamic libraries..." - $(AM_V_at) OBJDUMP=$(OBJDUMP) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) + $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) endif -if GLIBC_BACK_COMPAT +if TARGET_LINUX @echo "Checking glibc back compat..." - $(AM_V_at) CPPFILT=$(CPPFILT) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) + $(AM_V_at) CPPFILT='$(CPPFILT)' $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) endif check-security: $(bin_PROGRAMS) if HARDEN @echo "Checking binary security..." - $(AM_V_at) OBJDUMP=$(OBJDUMP) OTOOL=$(OTOOL) $(PYTHON) $(top_srcdir)/contrib/devtools/security-check.py $(bin_PROGRAMS) + $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/security-check.py $(bin_PROGRAMS) endif libbitcoin_ipc_mpgen_input = \ @@ -840,9 +838,11 @@ EXTRA_DIST += $(libbitcoin_ipc_mpgen_input) if BUILD_MULTIPROCESS LIBBITCOIN_IPC=libbitcoin_ipc.a libbitcoin_ipc_a_SOURCES = \ + ipc/capnp/context.h \ ipc/capnp/init-types.h \ ipc/capnp/protocol.cpp \ ipc/capnp/protocol.h \ + ipc/context.h \ ipc/exception.h \ ipc/interfaces.cpp \ ipc/process.cpp \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 56b8ca8ce6..2a8e4a0aac 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -35,6 +35,7 @@ bench_bench_bitcoin_SOURCES = \ bench/mempool_stress.cpp \ bench/nanobench.h \ bench/nanobench.cpp \ + bench/peer_eviction.cpp \ bench/rpc_blockchain.cpp \ bench/rpc_mempool.cpp \ bench/util_time.cpp \ diff --git a/src/Makefile.crc32c.include b/src/Makefile.crc32c.include index 113272e65e..3cbe71792c 100644 --- a/src/Makefile.crc32c.include +++ b/src/Makefile.crc32c.include @@ -14,7 +14,6 @@ CRC32C_CPPFLAGS_INT += -I$(srcdir)/crc32c/include CRC32C_CPPFLAGS_INT += -DHAVE_BUILTIN_PREFETCH=@HAVE_BUILTIN_PREFETCH@ CRC32C_CPPFLAGS_INT += -DHAVE_MM_PREFETCH=@HAVE_MM_PREFETCH@ CRC32C_CPPFLAGS_INT += -DHAVE_STRONG_GETAUXVAL=@HAVE_STRONG_GETAUXVAL@ -CRC32C_CPPFLAGS_INT += -DHAVE_WEAK_GETAUXVAL=@HAVE_WEAK_GETAUXVAL@ CRC32C_CPPFLAGS_INT += -DCRC32C_TESTS_BUILT_WITH_GLOG=0 if ENABLE_SSE42 diff --git a/src/Makefile.leveldb.include b/src/Makefile.leveldb.include index 8a28f4f249..ce1f93f11f 100644 --- a/src/Makefile.leveldb.include +++ b/src/Makefile.leveldb.include @@ -22,6 +22,7 @@ LEVELDB_CPPFLAGS_INT += -DHAVE_SNAPPY=0 -DHAVE_CRC32C=1 LEVELDB_CPPFLAGS_INT += -DHAVE_FDATASYNC=@HAVE_FDATASYNC@ LEVELDB_CPPFLAGS_INT += -DHAVE_FULLFSYNC=@HAVE_FULLFSYNC@ LEVELDB_CPPFLAGS_INT += -DHAVE_O_CLOEXEC=@HAVE_O_CLOEXEC@ +LEVELDB_CPPFLAGS_INT += -DFALLTHROUGH_INTENDED=[[fallthrough]] if WORDS_BIGENDIAN LEVELDB_CPPFLAGS_INT += -DLEVELDB_IS_BIG_ENDIAN=1 diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index a1821cafe3..f4b0b3adbe 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -40,9 +40,9 @@ QT_MOC_CPP = \ qt/moc_askpassphrasedialog.cpp \ qt/moc_createwalletdialog.cpp \ qt/moc_bantablemodel.cpp \ + qt/moc_bitcoin.cpp \ qt/moc_bitcoinaddressvalidator.cpp \ qt/moc_bitcoinamountfield.cpp \ - qt/moc_bitcoin.cpp \ qt/moc_bitcoingui.cpp \ qt/moc_bitcoinunits.cpp \ qt/moc_clientmodel.cpp \ @@ -51,6 +51,7 @@ QT_MOC_CPP = \ qt/moc_csvmodelwriter.cpp \ qt/moc_editaddressdialog.cpp \ qt/moc_guiutil.cpp \ + qt/moc_initexecutor.cpp \ qt/moc_intro.cpp \ qt/moc_macdockiconhandler.cpp \ qt/moc_macnotificationhandler.cpp \ @@ -109,9 +110,9 @@ BITCOIN_QT_H = \ qt/addresstablemodel.h \ qt/askpassphrasedialog.h \ qt/bantablemodel.h \ + qt/bitcoin.h \ qt/bitcoinaddressvalidator.h \ qt/bitcoinamountfield.h \ - qt/bitcoin.h \ qt/bitcoingui.h \ qt/bitcoinunits.h \ qt/clientmodel.h \ @@ -122,6 +123,7 @@ BITCOIN_QT_H = \ qt/editaddressdialog.h \ qt/guiconstants.h \ qt/guiutil.h \ + qt/initexecutor.h \ qt/intro.h \ qt/macdockiconhandler.h \ qt/macnotificationhandler.h \ @@ -227,6 +229,7 @@ BITCOIN_QT_BASE_CPP = \ qt/clientmodel.cpp \ qt/csvmodelwriter.cpp \ qt/guiutil.cpp \ + qt/initexecutor.cpp \ qt/intro.cpp \ qt/modaloverlay.cpp \ qt/networkstyle.cpp \ @@ -335,14 +338,14 @@ bitcoin_qt_libtoolflags = $(AM_LIBTOOLFLAGS) --tag CXX qt_bitcoin_qt_CPPFLAGS = $(bitcoin_qt_cppflags) qt_bitcoin_qt_CXXFLAGS = $(bitcoin_qt_cxxflags) -qt_bitcoin_qt_SOURCES = $(bitcoin_qt_sources) +qt_bitcoin_qt_SOURCES = $(bitcoin_qt_sources) init/bitcoind.cpp qt_bitcoin_qt_LDADD = $(bitcoin_qt_ldadd) qt_bitcoin_qt_LDFLAGS = $(bitcoin_qt_ldflags) qt_bitcoin_qt_LIBTOOLFLAGS = $(bitcoin_qt_libtoolflags) bitcoin_gui_CPPFLAGS = $(bitcoin_qt_cppflags) bitcoin_gui_CXXFLAGS = $(bitcoin_qt_cxxflags) -bitcoin_gui_SOURCES = $(bitcoin_qt_sources) +bitcoin_gui_SOURCES = $(bitcoin_qt_sources) init/bitcoind.cpp bitcoin_gui_LDADD = $(bitcoin_qt_ldadd) bitcoin_gui_LDFLAGS = $(bitcoin_qt_ldflags) bitcoin_gui_LIBTOOLFLAGS = $(bitcoin_qt_libtoolflags) diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index 91a5e9fd9b..8a5521eeb5 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -28,6 +28,7 @@ qt_test_test_bitcoin_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_ $(QT_INCLUDES) $(QT_TEST_INCLUDES) qt_test_test_bitcoin_qt_SOURCES = \ + init/bitcoind.cpp \ qt/test/apptests.cpp \ qt/test/rpcnestedtests.cpp \ qt/test/test_main.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index fc2fd80166..d70793ffa9 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -119,8 +119,9 @@ BITCOIN_TESTS =\ test/sanity_tests.cpp \ test/scheduler_tests.cpp \ test/script_p2sh_tests.cpp \ - test/script_tests.cpp \ + test/script_parse_tests.cpp \ test/script_standard_tests.cpp \ + test/script_tests.cpp \ test/scriptnum_tests.cpp \ test/serfloat_tests.cpp \ test/serialize_tests.cpp \ @@ -152,6 +153,7 @@ BITCOIN_TESTS =\ if ENABLE_WALLET BITCOIN_TESTS += \ wallet/test/psbt_wallet_tests.cpp \ + wallet/test/spend_tests.cpp \ wallet/test/wallet_tests.cpp \ wallet/test/walletdb_tests.cpp \ wallet/test/wallet_crypto_tests.cpp \ @@ -170,6 +172,8 @@ endif BITCOIN_TEST_SUITE += \ + wallet/test/util.cpp \ + wallet/test/util.h \ wallet/test/wallet_test_fixture.cpp \ wallet/test/wallet_test_fixture.h \ wallet/test/init_test_fixture.cpp \ @@ -201,10 +205,9 @@ if ENABLE_FUZZ_BINARY test_fuzz_fuzz_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_fuzz_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_fuzz_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_fuzz_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) +test_fuzz_fuzz_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) $(RUNTIME_LDFLAGS) test_fuzz_fuzz_SOURCES = \ test/fuzz/addition_overflow.cpp \ - test/fuzz/addrdb.cpp \ test/fuzz/addrman.cpp \ test/fuzz/asmap.cpp \ test/fuzz/asmap_direct.cpp \ @@ -230,7 +233,6 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp \ test/fuzz/crypto_poly1305.cpp \ test/fuzz/cuckoocache.cpp \ - test/fuzz/data_stream.cpp \ test/fuzz/decode_tx.cpp \ test/fuzz/descriptor_parse.cpp \ test/fuzz/deserialize.cpp \ @@ -335,8 +337,8 @@ bitcoin_test_clean : FORCE check-local: $(BITCOIN_TESTS:.cpp=.cpp.test) if BUILD_BITCOIN_TX - @echo "Running test/util/bitcoin-util-test.py..." - $(PYTHON) $(top_builddir)/test/util/bitcoin-util-test.py + @echo "Running test/util/test_runner.py..." + $(PYTHON) $(top_builddir)/test/util/test_runner.py endif @echo "Running test/util/rpcauth-test.py..." $(PYTHON) $(top_builddir)/test/util/rpcauth-test.py diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index 85e50ebf70..0a3b99e7d2 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -9,6 +9,7 @@ EXTRA_LIBRARIES += \ TEST_UTIL_H = \ test/util/blockfilter.h \ + test/util/chainstate.h \ test/util/logging.h \ test/util/mining.h \ test/util/net.h \ diff --git a/src/addrdb.cpp b/src/addrdb.cpp index b8fd019bab..50fd09101e 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -18,64 +18,14 @@ #include <univalue.h> #include <util/settings.h> #include <util/system.h> - -CBanEntry::CBanEntry(const UniValue& json) - : nVersion(json["version"].get_int()), nCreateTime(json["ban_created"].get_int64()), - nBanUntil(json["banned_until"].get_int64()) -{ -} - -UniValue CBanEntry::ToJson() const -{ - UniValue json(UniValue::VOBJ); - json.pushKV("version", nVersion); - json.pushKV("ban_created", nCreateTime); - json.pushKV("banned_until", nBanUntil); - return json; -} +#include <util/translation.h> namespace { -static const char* BANMAN_JSON_ADDR_KEY = "address"; - -/** - * Convert a `banmap_t` object to a JSON array. - * @param[in] bans Bans list to convert. - * @return a JSON array, similar to the one returned by the `listbanned` RPC. Suitable for - * passing to `BanMapFromJson()`. - */ -UniValue BanMapToJson(const banmap_t& bans) -{ - UniValue bans_json(UniValue::VARR); - for (const auto& it : bans) { - const auto& address = it.first; - const auto& ban_entry = it.second; - UniValue j = ban_entry.ToJson(); - j.pushKV(BANMAN_JSON_ADDR_KEY, address.ToString()); - bans_json.push_back(j); - } - return bans_json; -} - -/** - * Convert a JSON array to a `banmap_t` object. - * @param[in] bans_json JSON to convert, must be as returned by `BanMapToJson()`. - * @param[out] bans Bans list to create from the JSON. - * @throws std::runtime_error if the JSON does not have the expected fields or they contain - * unparsable values. - */ -void BanMapFromJson(const UniValue& bans_json, banmap_t& bans) +class DbNotFoundError : public std::exception { - for (const auto& ban_entry_json : bans_json.getValues()) { - CSubNet subnet; - const auto& subnet_str = ban_entry_json[BANMAN_JSON_ADDR_KEY].get_str(); - if (!LookupSubNet(subnet_str, subnet)) { - throw std::runtime_error( - strprintf("Cannot parse banned address or subnet: %s", subnet_str)); - } - bans.insert_or_assign(subnet, CBanEntry{ban_entry_json}); - } -} + using std::exception::exception; +}; template <typename Stream, typename Data> bool SerializeDB(Stream& stream, const Data& data) @@ -134,47 +84,40 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data } template <typename Stream, typename Data> -bool DeserializeDB(Stream& stream, Data& data, bool fCheckSum = true) -{ - try { - CHashVerifier<Stream> verifier(&stream); - // de-serialize file header (network specific magic number) and .. - unsigned char pchMsgTmp[4]; - verifier >> pchMsgTmp; - // ... verify the network matches ours - if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp))) - return error("%s: Invalid network magic number", __func__); - - // de-serialize data - verifier >> data; - - // verify checksum - if (fCheckSum) { - uint256 hashTmp; - stream >> hashTmp; - if (hashTmp != verifier.GetHash()) { - return error("%s: Checksum mismatch, data corrupted", __func__); - } +void DeserializeDB(Stream& stream, Data& data, bool fCheckSum = true) +{ + CHashVerifier<Stream> verifier(&stream); + // de-serialize file header (network specific magic number) and .. + unsigned char pchMsgTmp[4]; + verifier >> pchMsgTmp; + // ... verify the network matches ours + if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp))) { + throw std::runtime_error{"Invalid network magic number"}; + } + + // de-serialize data + verifier >> data; + + // verify checksum + if (fCheckSum) { + uint256 hashTmp; + stream >> hashTmp; + if (hashTmp != verifier.GetHash()) { + throw std::runtime_error{"Checksum mismatch, data corrupted"}; } } - catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); - } - - return true; } template <typename Data> -bool DeserializeFileDB(const fs::path& path, Data& data, int version) +void DeserializeFileDB(const fs::path& path, Data& data, int version) { // open input file, and associate with CAutoFile FILE* file = fsbridge::fopen(path, "rb"); CAutoFile filein(file, SER_DISK, version); if (filein.IsNull()) { - LogPrintf("Missing or invalid file %s\n", path.string()); - return false; + throw DbNotFoundError{}; } - return DeserializeDB(filein, data); + DeserializeDB(filein, data); } } // namespace @@ -197,17 +140,16 @@ bool CBanDB::Write(const banmap_t& banSet) return false; } -bool CBanDB::Read(banmap_t& banSet, bool& dirty) +bool CBanDB::Read(banmap_t& banSet) { - // If the JSON banlist does not exist, then try to read the non-upgraded banlist.dat. + if (fs::exists(m_banlist_dat)) { + LogPrintf("banlist.dat ignored because it can only be read by " PACKAGE_NAME " version 22.x. Remove %s to silence this warning.\n", m_banlist_dat); + } + // If the JSON banlist does not exist, then recreate it if (!fs::exists(m_banlist_json)) { - // If this succeeds then we need to flush to disk in order to create the JSON banlist. - dirty = true; - return DeserializeFileDB(m_banlist_dat, banSet, CLIENT_VERSION); + return false; } - dirty = false; - std::map<std::string, util::SettingsValue> settings; std::vector<std::string> errors; @@ -228,29 +170,38 @@ bool CBanDB::Read(banmap_t& banSet, bool& dirty) return true; } -CAddrDB::CAddrDB() -{ - pathAddr = gArgs.GetDataDirNet() / "peers.dat"; -} - -bool CAddrDB::Write(const CAddrMan& addr) +bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr) { + const auto pathAddr = args.GetDataDirNet() / "peers.dat"; return SerializeFileDB("peers", pathAddr, addr, CLIENT_VERSION); } -bool CAddrDB::Read(CAddrMan& addr) +void ReadFromStream(AddrMan& addr, CDataStream& ssPeers) { - return DeserializeFileDB(pathAddr, addr, CLIENT_VERSION); + DeserializeDB(ssPeers, addr, false); } -bool CAddrDB::Read(CAddrMan& addr, CDataStream& ssPeers) +std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman) { - bool ret = DeserializeDB(ssPeers, addr, false); - if (!ret) { - // Ensure addrman is left in a clean state - addr.Clear(); + auto check_addrman = std::clamp<int32_t>(args.GetIntArg("-checkaddrman", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), 0, 1000000); + addrman = std::make_unique<AddrMan>(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); + + int64_t nStart = GetTimeMillis(); + const auto path_addr{args.GetDataDirNet() / "peers.dat"}; + try { + DeserializeFileDB(path_addr, *addrman, CLIENT_VERSION); + LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman->size(), GetTimeMillis() - nStart); + } catch (const DbNotFoundError&) { + // Addrman can be in an inconsistent state after failure, reset it + addrman = std::make_unique<AddrMan>(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); + LogPrintf("Creating peers.dat because the file was not found (%s)\n", path_addr); + DumpPeerAddresses(args, *addrman); + } catch (const std::exception& e) { + addrman = nullptr; + return strprintf(_("Invalid or corrupt peers.dat (%s). If you believe this is a bug, please report it to %s. As a workaround, you can move the file (%s) out of the way (rename, move, or delete) to have a new one created on the next start."), + e.what(), PACKAGE_BUGREPORT, path_addr); } - return ret; + return std::nullopt; } void DumpAnchors(const fs::path& anchors_db_path, const std::vector<CAddress>& anchors) @@ -262,9 +213,10 @@ void DumpAnchors(const fs::path& anchors_db_path, const std::vector<CAddress>& a std::vector<CAddress> ReadAnchors(const fs::path& anchors_db_path) { std::vector<CAddress> anchors; - if (DeserializeFileDB(anchors_db_path, anchors, CLIENT_VERSION | ADDRV2_FORMAT)) { + try { + DeserializeFileDB(anchors_db_path, anchors, CLIENT_VERSION | ADDRV2_FORMAT); LogPrintf("Loaded %i addresses from %s\n", anchors.size(), anchors_db_path.filename()); - } else { + } catch (const std::exception&) { anchors.clear(); } diff --git a/src/addrdb.h b/src/addrdb.h index 399103c991..19be4b5bb4 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -8,75 +8,22 @@ #include <fs.h> #include <net_types.h> // For banmap_t -#include <serialize.h> #include <univalue.h> -#include <string> +#include <optional> #include <vector> +class ArgsManager; +class AddrMan; class CAddress; -class CAddrMan; class CDataStream; +struct bilingual_str; -class CBanEntry -{ -public: - static const int CURRENT_VERSION=1; - int nVersion; - int64_t nCreateTime; - int64_t nBanUntil; - - CBanEntry() - { - SetNull(); - } - - explicit CBanEntry(int64_t nCreateTimeIn) - { - SetNull(); - nCreateTime = nCreateTimeIn; - } - - /** - * Create a ban entry from JSON. - * @param[in] json A JSON representation of a ban entry, as created by `ToJson()`. - * @throw std::runtime_error if the JSON does not have the expected fields. - */ - explicit CBanEntry(const UniValue& json); - - SERIALIZE_METHODS(CBanEntry, obj) - { - uint8_t ban_reason = 2; //! For backward compatibility - READWRITE(obj.nVersion, obj.nCreateTime, obj.nBanUntil, ban_reason); - } +bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr); +/** Only used by tests. */ +void ReadFromStream(AddrMan& addr, CDataStream& ssPeers); - void SetNull() - { - nVersion = CBanEntry::CURRENT_VERSION; - nCreateTime = 0; - nBanUntil = 0; - } - - /** - * Generate a JSON representation of this ban entry. - * @return JSON suitable for passing to the `CBanEntry(const UniValue&)` constructor. - */ - UniValue ToJson() const; -}; - -/** Access to the (IP) address database (peers.dat) */ -class CAddrDB -{ -private: - fs::path pathAddr; -public: - CAddrDB(); - bool Write(const CAddrMan& addr); - bool Read(CAddrMan& addr); - static bool Read(CAddrMan& addr, CDataStream& ssPeers); -}; - -/** Access to the banlist databases (banlist.json and banlist.dat) */ +/** Access to the banlist database (banlist.json) */ class CBanDB { private: @@ -95,13 +42,14 @@ public: * Read the banlist from disk. * @param[out] banSet The loaded list. Set if `true` is returned, otherwise it is left * in an undefined state. - * @param[out] dirty Indicates whether the loaded list needs flushing to disk. Set if - * `true` is returned, otherwise it is left in an undefined state. * @return true on success */ - bool Read(banmap_t& banSet, bool& dirty); + bool Read(banmap_t& banSet); }; +/** Returns an error string on failure */ +std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman); + /** * Dump the anchor IP address database (anchors.dat) * diff --git a/src/addrman.cpp b/src/addrman.cpp index 8f702b5a8c..c364a7710b 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -4,18 +4,44 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <addrman.h> +#include <addrman_impl.h> #include <hash.h> -#include <logging.h> #include <netaddress.h> +#include <protocol.h> +#include <random.h> #include <serialize.h> +#include <streams.h> +#include <timedata.h> +#include <tinyformat.h> +#include <uint256.h> +#include <util/check.h> #include <cmath> #include <optional> -#include <unordered_map> -#include <unordered_set> -int CAddrInfo::GetTriedBucket(const uint256& nKey, const std::vector<bool> &asmap) const +/** Over how many buckets entries with tried addresses from a single group (/16 for IPv4) are spread */ +static constexpr uint32_t ADDRMAN_TRIED_BUCKETS_PER_GROUP{8}; +/** Over how many buckets entries with new addresses originating from a single group are spread */ +static constexpr uint32_t ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP{64}; +/** Maximum number of times an address can occur in the new table */ +static constexpr int32_t ADDRMAN_NEW_BUCKETS_PER_ADDRESS{8}; +/** How old addresses can maximally be */ +static constexpr int64_t ADDRMAN_HORIZON_DAYS{30}; +/** After how many failed attempts we give up on a new node */ +static constexpr int32_t ADDRMAN_RETRIES{3}; +/** How many successive failures are allowed ... */ +static constexpr int32_t ADDRMAN_MAX_FAILURES{10}; +/** ... in at least this many days */ +static constexpr int64_t ADDRMAN_MIN_FAIL_DAYS{7}; +/** How recent a successful connection should be before we allow an address to be evicted from tried */ +static constexpr int64_t ADDRMAN_REPLACEMENT_HOURS{4}; +/** The maximum number of tried addr collisions to store */ +static constexpr size_t ADDRMAN_SET_TRIED_COLLISION_SIZE{10}; +/** The maximum time we'll spend trying to resolve a tried table collision, in seconds */ +static constexpr int64_t ADDRMAN_TEST_WINDOW{40*60}; // 40 minutes + +int AddrInfo::GetTriedBucket(const uint256& nKey, const std::vector<bool>& asmap) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetCheapHash(); uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); @@ -25,7 +51,7 @@ int CAddrInfo::GetTriedBucket(const uint256& nKey, const std::vector<bool> &asma return tried_bucket; } -int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std::vector<bool> &asmap) const +int AddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std::vector<bool>& asmap) const { std::vector<unsigned char> vchSourceGroupKey = src.GetGroup(asmap); uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << vchSourceGroupKey).GetCheapHash(); @@ -36,13 +62,13 @@ int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std: return new_bucket; } -int CAddrInfo::GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const +int AddrInfo::GetBucketPosition(const uint256& nKey, bool fNew, int nBucket) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << (fNew ? uint8_t{'N'} : uint8_t{'K'}) << nBucket << GetKey()).GetCheapHash(); return hash1 % ADDRMAN_BUCKET_SIZE; } -bool CAddrInfo::IsTerrible(int64_t nNow) const +bool AddrInfo::IsTerrible(int64_t nNow) const { if (nLastTry && nLastTry >= nNow - 60) // never remove things tried in the last minute return false; @@ -62,7 +88,7 @@ bool CAddrInfo::IsTerrible(int64_t nNow) const return false; } -double CAddrInfo::GetChance(int64_t nNow) const +double AddrInfo::GetChance(int64_t nNow) const { double fChance = 1.0; int64_t nSinceLastTry = std::max<int64_t>(nNow - nLastTry, 0); @@ -77,7 +103,305 @@ double CAddrInfo::GetChance(int64_t nNow) const return fChance; } -CAddrInfo* CAddrMan::Find(const CNetAddr& addr, int* pnId) +AddrManImpl::AddrManImpl(std::vector<bool>&& asmap, bool deterministic, int32_t consistency_check_ratio) + : insecure_rand{deterministic} + , nKey{deterministic ? uint256{1} : insecure_rand.rand256()} + , m_consistency_check_ratio{consistency_check_ratio} + , m_asmap{std::move(asmap)} +{ + for (auto& bucket : vvNew) { + for (auto& entry : bucket) { + entry = -1; + } + } + for (auto& bucket : vvTried) { + for (auto& entry : bucket) { + entry = -1; + } + } +} + +AddrManImpl::~AddrManImpl() +{ + nKey.SetNull(); +} + +template <typename Stream> +void AddrManImpl::Serialize(Stream& s_) const +{ + LOCK(cs); + + /** + * Serialized format. + * * format version byte (@see `Format`) + * * lowest compatible format version byte. This is used to help old software decide + * whether to parse the file. For example: + * * Bitcoin Core version N knows how to parse up to format=3. If a new format=4 is + * introduced in version N+1 that is compatible with format=3 and it is known that + * version N will be able to parse it, then version N+1 will write + * (format=4, lowest_compatible=3) in the first two bytes of the file, and so + * version N will still try to parse it. + * * Bitcoin Core version N+2 introduces a new incompatible format=5. It will write + * (format=5, lowest_compatible=5) and so any versions that do not know how to parse + * format=5 will not try to read the file. + * * nKey + * * nNew + * * nTried + * * number of "new" buckets XOR 2**30 + * * all new addresses (total count: nNew) + * * all tried addresses (total count: nTried) + * * for each new bucket: + * * number of elements + * * for each element: index in the serialized "all new addresses" + * * asmap checksum + * + * 2**30 is xorred with the number of buckets to make addrman deserializer v0 detect it + * as incompatible. This is necessary because it did not check the version number on + * deserialization. + * + * vvNew, vvTried, mapInfo, mapAddr and vRandom are never encoded explicitly; + * they are instead reconstructed from the other information. + * + * This format is more complex, but significantly smaller (at most 1.5 MiB), and supports + * changes to the ADDRMAN_ parameters without breaking the on-disk structure. + * + * We don't use SERIALIZE_METHODS since the serialization and deserialization code has + * very little in common. + */ + + // Always serialize in the latest version (FILE_FORMAT). + + OverrideStream<Stream> s(&s_, s_.GetType(), s_.GetVersion() | ADDRV2_FORMAT); + + s << static_cast<uint8_t>(FILE_FORMAT); + + // Increment `lowest_compatible` iff a newly introduced format is incompatible with + // the previous one. + static constexpr uint8_t lowest_compatible = Format::V3_BIP155; + s << static_cast<uint8_t>(INCOMPATIBILITY_BASE + lowest_compatible); + + s << nKey; + s << nNew; + s << nTried; + + int nUBuckets = ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30); + s << nUBuckets; + std::unordered_map<int, int> mapUnkIds; + int nIds = 0; + for (const auto& entry : mapInfo) { + mapUnkIds[entry.first] = nIds; + const AddrInfo& info = entry.second; + if (info.nRefCount) { + assert(nIds != nNew); // this means nNew was wrong, oh ow + s << info; + nIds++; + } + } + nIds = 0; + for (const auto& entry : mapInfo) { + const AddrInfo& info = entry.second; + if (info.fInTried) { + assert(nIds != nTried); // this means nTried was wrong, oh ow + s << info; + nIds++; + } + } + for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; bucket++) { + int nSize = 0; + for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { + if (vvNew[bucket][i] != -1) + nSize++; + } + s << nSize; + for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { + if (vvNew[bucket][i] != -1) { + int nIndex = mapUnkIds[vvNew[bucket][i]]; + s << nIndex; + } + } + } + // Store asmap checksum after bucket entries so that it + // can be ignored by older clients for backward compatibility. + uint256 asmap_checksum; + if (m_asmap.size() != 0) { + asmap_checksum = SerializeHash(m_asmap); + } + s << asmap_checksum; +} + +template <typename Stream> +void AddrManImpl::Unserialize(Stream& s_) +{ + LOCK(cs); + + assert(vRandom.empty()); + + Format format; + s_ >> Using<CustomUintFormatter<1>>(format); + + int stream_version = s_.GetVersion(); + if (format >= Format::V3_BIP155) { + // Add ADDRV2_FORMAT to the version so that the CNetAddr and CAddress + // unserialize methods know that an address in addrv2 format is coming. + stream_version |= ADDRV2_FORMAT; + } + + OverrideStream<Stream> s(&s_, s_.GetType(), stream_version); + + uint8_t compat; + s >> compat; + const uint8_t lowest_compatible = compat - INCOMPATIBILITY_BASE; + if (lowest_compatible > FILE_FORMAT) { + throw std::ios_base::failure(strprintf( + "Unsupported format of addrman database: %u. It is compatible with formats >=%u, " + "but the maximum supported by this version of %s is %u.", + uint8_t{format}, uint8_t{lowest_compatible}, PACKAGE_NAME, uint8_t{FILE_FORMAT})); + } + + s >> nKey; + s >> nNew; + s >> nTried; + int nUBuckets = 0; + s >> nUBuckets; + if (format >= Format::V1_DETERMINISTIC) { + nUBuckets ^= (1 << 30); + } + + if (nNew > ADDRMAN_NEW_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE || nNew < 0) { + throw std::ios_base::failure( + strprintf("Corrupt AddrMan serialization: nNew=%d, should be in [0, %d]", + nNew, + ADDRMAN_NEW_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE)); + } + + if (nTried > ADDRMAN_TRIED_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE || nTried < 0) { + throw std::ios_base::failure( + strprintf("Corrupt AddrMan serialization: nTried=%d, should be in [0, %d]", + nTried, + ADDRMAN_TRIED_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE)); + } + + // Deserialize entries from the new table. + for (int n = 0; n < nNew; n++) { + AddrInfo& info = mapInfo[n]; + s >> info; + mapAddr[info] = n; + info.nRandomPos = vRandom.size(); + vRandom.push_back(n); + } + nIdCount = nNew; + + // Deserialize entries from the tried table. + int nLost = 0; + for (int n = 0; n < nTried; n++) { + AddrInfo info; + s >> info; + int nKBucket = info.GetTriedBucket(nKey, m_asmap); + int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); + if (info.IsValid() + && vvTried[nKBucket][nKBucketPos] == -1) { + info.nRandomPos = vRandom.size(); + info.fInTried = true; + vRandom.push_back(nIdCount); + mapInfo[nIdCount] = info; + mapAddr[info] = nIdCount; + vvTried[nKBucket][nKBucketPos] = nIdCount; + nIdCount++; + } else { + nLost++; + } + } + nTried -= nLost; + + // Store positions in the new table buckets to apply later (if possible). + // An entry may appear in up to ADDRMAN_NEW_BUCKETS_PER_ADDRESS buckets, + // so we store all bucket-entry_index pairs to iterate through later. + std::vector<std::pair<int, int>> bucket_entries; + + for (int bucket = 0; bucket < nUBuckets; ++bucket) { + int num_entries{0}; + s >> num_entries; + for (int n = 0; n < num_entries; ++n) { + int entry_index{0}; + s >> entry_index; + if (entry_index >= 0 && entry_index < nNew) { + bucket_entries.emplace_back(bucket, entry_index); + } + } + } + + // If the bucket count and asmap checksum haven't changed, then attempt + // to restore the entries to the buckets/positions they were in before + // serialization. + uint256 supplied_asmap_checksum; + if (m_asmap.size() != 0) { + supplied_asmap_checksum = SerializeHash(m_asmap); + } + uint256 serialized_asmap_checksum; + if (format >= Format::V2_ASMAP) { + s >> serialized_asmap_checksum; + } + const bool restore_bucketing{nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && + serialized_asmap_checksum == supplied_asmap_checksum}; + + if (!restore_bucketing) { + LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n"); + } + + for (auto bucket_entry : bucket_entries) { + int bucket{bucket_entry.first}; + const int entry_index{bucket_entry.second}; + AddrInfo& info = mapInfo[entry_index]; + + // Don't store the entry in the new bucket if it's not a valid address for our addrman + if (!info.IsValid()) continue; + + // The entry shouldn't appear in more than + // ADDRMAN_NEW_BUCKETS_PER_ADDRESS. If it has already, just skip + // this bucket_entry. + if (info.nRefCount >= ADDRMAN_NEW_BUCKETS_PER_ADDRESS) continue; + + int bucket_position = info.GetBucketPosition(nKey, true, bucket); + if (restore_bucketing && vvNew[bucket][bucket_position] == -1) { + // Bucketing has not changed, using existing bucket positions for the new table + vvNew[bucket][bucket_position] = entry_index; + ++info.nRefCount; + } else { + // In case the new table data cannot be used (bucket count wrong or new asmap), + // try to give them a reference based on their primary source address. + bucket = info.GetNewBucket(nKey, m_asmap); + bucket_position = info.GetBucketPosition(nKey, true, bucket); + if (vvNew[bucket][bucket_position] == -1) { + vvNew[bucket][bucket_position] = entry_index; + ++info.nRefCount; + } + } + } + + // Prune new entries with refcount 0 (as a result of collisions or invalid address). + int nLostUnk = 0; + for (auto it = mapInfo.cbegin(); it != mapInfo.cend(); ) { + if (it->second.fInTried == false && it->second.nRefCount == 0) { + const auto itCopy = it++; + Delete(itCopy->first); + ++nLostUnk; + } else { + ++it; + } + } + if (nLost + nLostUnk > 0) { + LogPrint(BCLog::ADDRMAN, "addrman lost %i new and %i tried addresses due to collisions or invalid addresses\n", nLostUnk, nLost); + } + + const int check_code{ForceCheckAddrman()}; + if (check_code != 0) { + throw std::ios_base::failure(strprintf( + "Corrupt data. Consistency check failed with code %s", + check_code)); + } +} + +AddrInfo* AddrManImpl::Find(const CNetAddr& addr, int* pnId) { AssertLockHeld(cs); @@ -92,12 +416,12 @@ CAddrInfo* CAddrMan::Find(const CNetAddr& addr, int* pnId) return nullptr; } -CAddrInfo* CAddrMan::Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId) +AddrInfo* AddrManImpl::Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId) { AssertLockHeld(cs); int nId = nIdCount++; - mapInfo[nId] = CAddrInfo(addr, addrSource); + mapInfo[nId] = AddrInfo(addr, addrSource); mapAddr[addr] = nId; mapInfo[nId].nRandomPos = vRandom.size(); vRandom.push_back(nId); @@ -106,7 +430,7 @@ CAddrInfo* CAddrMan::Create(const CAddress& addr, const CNetAddr& addrSource, in return &mapInfo[nId]; } -void CAddrMan::SwapRandom(unsigned int nRndPos1, unsigned int nRndPos2) +void AddrManImpl::SwapRandom(unsigned int nRndPos1, unsigned int nRndPos2) const { AssertLockHeld(cs); @@ -118,22 +442,24 @@ void CAddrMan::SwapRandom(unsigned int nRndPos1, unsigned int nRndPos2) int nId1 = vRandom[nRndPos1]; int nId2 = vRandom[nRndPos2]; - assert(mapInfo.count(nId1) == 1); - assert(mapInfo.count(nId2) == 1); + const auto it_1{mapInfo.find(nId1)}; + const auto it_2{mapInfo.find(nId2)}; + assert(it_1 != mapInfo.end()); + assert(it_2 != mapInfo.end()); - mapInfo[nId1].nRandomPos = nRndPos2; - mapInfo[nId2].nRandomPos = nRndPos1; + it_1->second.nRandomPos = nRndPos2; + it_2->second.nRandomPos = nRndPos1; vRandom[nRndPos1] = nId2; vRandom[nRndPos2] = nId1; } -void CAddrMan::Delete(int nId) +void AddrManImpl::Delete(int nId) { AssertLockHeld(cs); assert(mapInfo.count(nId) != 0); - CAddrInfo& info = mapInfo[nId]; + AddrInfo& info = mapInfo[nId]; assert(!info.fInTried); assert(info.nRefCount == 0); @@ -144,14 +470,14 @@ void CAddrMan::Delete(int nId) nNew--; } -void CAddrMan::ClearNew(int nUBucket, int nUBucketPos) +void AddrManImpl::ClearNew(int nUBucket, int nUBucketPos) { AssertLockHeld(cs); // if there is an entry in the specified bucket, delete it. if (vvNew[nUBucket][nUBucketPos] != -1) { int nIdDelete = vvNew[nUBucket][nUBucketPos]; - CAddrInfo& infoDelete = mapInfo[nIdDelete]; + AddrInfo& infoDelete = mapInfo[nIdDelete]; assert(infoDelete.nRefCount > 0); infoDelete.nRefCount--; vvNew[nUBucket][nUBucketPos] = -1; @@ -161,16 +487,19 @@ void CAddrMan::ClearNew(int nUBucket, int nUBucketPos) } } -void CAddrMan::MakeTried(CAddrInfo& info, int nId) +void AddrManImpl::MakeTried(AddrInfo& info, int nId) { AssertLockHeld(cs); // remove the entry from all new buckets - for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; bucket++) { - int pos = info.GetBucketPosition(nKey, true, bucket); + const int start_bucket{info.GetNewBucket(nKey, m_asmap)}; + for (int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; ++n) { + const int bucket{(start_bucket + n) % ADDRMAN_NEW_BUCKET_COUNT}; + const int pos{info.GetBucketPosition(nKey, true, bucket)}; if (vvNew[bucket][pos] == nId) { vvNew[bucket][pos] = -1; info.nRefCount--; + if (info.nRefCount == 0) break; } } nNew--; @@ -186,7 +515,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) // find an item to evict int nIdEvict = vvTried[nKBucket][nKBucketPos]; assert(mapInfo.count(nIdEvict) == 1); - CAddrInfo& infoOld = mapInfo[nIdEvict]; + AddrInfo& infoOld = mapInfo[nIdEvict]; // Remove the to-be-evicted item from the tried set. infoOld.fInTried = false; @@ -211,7 +540,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) info.fInTried = true; } -void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime) +void AddrManImpl::Good_(const CService& addr, bool test_before_evict, int64_t nTime) { AssertLockHeld(cs); @@ -219,13 +548,13 @@ void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime nLastGood = nTime; - CAddrInfo* pinfo = Find(addr, &nId); + AddrInfo* pinfo = Find(addr, &nId); // if not found, bail out if (!pinfo) return; - CAddrInfo& info = *pinfo; + AddrInfo& info = *pinfo; // check whether we are talking about the exact same CService (including same port) if (info != addr) @@ -242,22 +571,10 @@ void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime if (info.fInTried) return; - // find a bucket it is in now - int nRnd = insecure_rand.randrange(ADDRMAN_NEW_BUCKET_COUNT); - int nUBucket = -1; - for (unsigned int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; n++) { - int nB = (n + nRnd) % ADDRMAN_NEW_BUCKET_COUNT; - int nBpos = info.GetBucketPosition(nKey, true, nB); - if (vvNew[nB][nBpos] == nId) { - nUBucket = nB; - break; - } - } - - // if no bucket is found, something bad happened; - // TODO: maybe re-add the node, but for now, just bail out - if (nUBucket == -1) + // if it is not in new, something bad happened + if (!Assume(info.nRefCount > 0)) { return; + } // which tried bucket to move the entry to int tried_bucket = info.GetTriedBucket(nKey, m_asmap); @@ -279,7 +596,7 @@ void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime } } -bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimePenalty) +bool AddrManImpl::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimePenalty) { AssertLockHeld(cs); @@ -288,7 +605,7 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP bool fNew = false; int nId; - CAddrInfo* pinfo = Find(addr, &nId); + AddrInfo* pinfo = Find(addr, &nId); // Do not set a penalty for a source's self-announcement if (addr == source) { @@ -335,7 +652,7 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP if (vvNew[nUBucket][nUBucketPos] != nId) { bool fInsert = vvNew[nUBucket][nUBucketPos] == -1; if (!fInsert) { - CAddrInfo& infoExisting = mapInfo[vvNew[nUBucket][nUBucketPos]]; + AddrInfo& infoExisting = mapInfo[vvNew[nUBucket][nUBucketPos]]; if (infoExisting.IsTerrible() || (infoExisting.nRefCount > 1 && pinfo->nRefCount == 0)) { // Overwrite the existing new table entry. fInsert = true; @@ -354,17 +671,17 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP return fNew; } -void CAddrMan::Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) +void AddrManImpl::Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) { AssertLockHeld(cs); - CAddrInfo* pinfo = Find(addr); + AddrInfo* pinfo = Find(addr); // if not found, bail out if (!pinfo) return; - CAddrInfo& info = *pinfo; + AddrInfo& info = *pinfo; // check whether we are talking about the exact same CService (including same port) if (info != addr) @@ -378,15 +695,13 @@ void CAddrMan::Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) } } -CAddrInfo CAddrMan::Select_(bool newOnly) +std::pair<CAddress, int64_t> AddrManImpl::Select_(bool newOnly) const { AssertLockHeld(cs); - if (vRandom.empty()) - return CAddrInfo(); + if (vRandom.empty()) return {}; - if (newOnly && nNew == 0) - return CAddrInfo(); + if (newOnly && nNew == 0) return {}; // Use a 50% chance for choosing between tried and new table entries. if (!newOnly && @@ -401,10 +716,12 @@ CAddrInfo CAddrMan::Select_(bool newOnly) nKBucketPos = (nKBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; } int nId = vvTried[nKBucket][nKBucketPos]; - assert(mapInfo.count(nId) == 1); - CAddrInfo& info = mapInfo[nId]; - if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) - return info; + const auto it_found{mapInfo.find(nId)}; + assert(it_found != mapInfo.end()); + const AddrInfo& info{it_found->second}; + if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) { + return {info, info.nLastTry}; + } fChanceFactor *= 1.2; } } else { @@ -418,96 +735,18 @@ CAddrInfo CAddrMan::Select_(bool newOnly) nUBucketPos = (nUBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; } int nId = vvNew[nUBucket][nUBucketPos]; - assert(mapInfo.count(nId) == 1); - CAddrInfo& info = mapInfo[nId]; - if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) - return info; - fChanceFactor *= 1.2; - } - } -} - -#ifdef DEBUG_ADDRMAN -int CAddrMan::Check_() -{ - AssertLockHeld(cs); - - std::unordered_set<int> setTried; - std::unordered_map<int, int> mapNew; - - if (vRandom.size() != (size_t)(nTried + nNew)) - return -7; - - for (const auto& entry : mapInfo) { - int n = entry.first; - const CAddrInfo& info = entry.second; - if (info.fInTried) { - if (!info.nLastSuccess) - return -1; - if (info.nRefCount) - return -2; - setTried.insert(n); - } else { - if (info.nRefCount < 0 || info.nRefCount > ADDRMAN_NEW_BUCKETS_PER_ADDRESS) - return -3; - if (!info.nRefCount) - return -4; - mapNew[n] = info.nRefCount; - } - if (mapAddr[info] != n) - return -5; - if (info.nRandomPos < 0 || (size_t)info.nRandomPos >= vRandom.size() || vRandom[info.nRandomPos] != n) - return -14; - if (info.nLastTry < 0) - return -6; - if (info.nLastSuccess < 0) - return -8; - } - - if (setTried.size() != (size_t)nTried) - return -9; - if (mapNew.size() != (size_t)nNew) - return -10; - - for (int n = 0; n < ADDRMAN_TRIED_BUCKET_COUNT; n++) { - for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { - if (vvTried[n][i] != -1) { - if (!setTried.count(vvTried[n][i])) - return -11; - if (mapInfo[vvTried[n][i]].GetTriedBucket(nKey, m_asmap) != n) - return -17; - if (mapInfo[vvTried[n][i]].GetBucketPosition(nKey, false, n) != i) - return -18; - setTried.erase(vvTried[n][i]); - } - } - } - - for (int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; n++) { - for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { - if (vvNew[n][i] != -1) { - if (!mapNew.count(vvNew[n][i])) - return -12; - if (mapInfo[vvNew[n][i]].GetBucketPosition(nKey, true, n) != i) - return -19; - if (--mapNew[vvNew[n][i]] == 0) - mapNew.erase(vvNew[n][i]); + const auto it_found{mapInfo.find(nId)}; + assert(it_found != mapInfo.end()); + const AddrInfo& info{it_found->second}; + if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) { + return {info, info.nLastTry}; } + fChanceFactor *= 1.2; } } - - if (setTried.size()) - return -13; - if (mapNew.size()) - return -15; - if (nKey.IsNull()) - return -16; - - return 0; } -#endif -void CAddrMan::GetAddr_(std::vector<CAddress>& vAddr, size_t max_addresses, size_t max_pct, std::optional<Network> network) +std::vector<CAddress> AddrManImpl::GetAddr_(size_t max_addresses, size_t max_pct, std::optional<Network> network) const { AssertLockHeld(cs); @@ -521,15 +760,17 @@ void CAddrMan::GetAddr_(std::vector<CAddress>& vAddr, size_t max_addresses, size // gather a list of random nodes, skipping those of low quality const int64_t now{GetAdjustedTime()}; + std::vector<CAddress> addresses; for (unsigned int n = 0; n < vRandom.size(); n++) { - if (vAddr.size() >= nNodes) + if (addresses.size() >= nNodes) break; int nRndPos = insecure_rand.randrange(vRandom.size() - n) + n; SwapRandom(n, nRndPos); - assert(mapInfo.count(vRandom[n]) == 1); + const auto it{mapInfo.find(vRandom[n])}; + assert(it != mapInfo.end()); - const CAddrInfo& ai = mapInfo[vRandom[n]]; + const AddrInfo& ai{it->second}; // Filter by network (optional) if (network != std::nullopt && ai.GetNetClass() != network) continue; @@ -537,21 +778,23 @@ void CAddrMan::GetAddr_(std::vector<CAddress>& vAddr, size_t max_addresses, size // Filter for quality if (ai.IsTerrible(now)) continue; - vAddr.push_back(ai); + addresses.push_back(ai); } + + return addresses; } -void CAddrMan::Connected_(const CService& addr, int64_t nTime) +void AddrManImpl::Connected_(const CService& addr, int64_t nTime) { AssertLockHeld(cs); - CAddrInfo* pinfo = Find(addr); + AddrInfo* pinfo = Find(addr); // if not found, bail out if (!pinfo) return; - CAddrInfo& info = *pinfo; + AddrInfo& info = *pinfo; // check whether we are talking about the exact same CService (including same port) if (info != addr) @@ -563,17 +806,17 @@ void CAddrMan::Connected_(const CService& addr, int64_t nTime) info.nTime = nTime; } -void CAddrMan::SetServices_(const CService& addr, ServiceFlags nServices) +void AddrManImpl::SetServices_(const CService& addr, ServiceFlags nServices) { AssertLockHeld(cs); - CAddrInfo* pinfo = Find(addr); + AddrInfo* pinfo = Find(addr); // if not found, bail out if (!pinfo) return; - CAddrInfo& info = *pinfo; + AddrInfo& info = *pinfo; // check whether we are talking about the exact same CService (including same port) if (info != addr) @@ -583,7 +826,7 @@ void CAddrMan::SetServices_(const CService& addr, ServiceFlags nServices) info.nServices = nServices; } -void CAddrMan::ResolveCollisions_() +void AddrManImpl::ResolveCollisions_() { AssertLockHeld(cs); @@ -596,7 +839,7 @@ void CAddrMan::ResolveCollisions_() if (mapInfo.count(id_new) != 1) { erase_collision = true; } else { - CAddrInfo& info_new = mapInfo[id_new]; + AddrInfo& info_new = mapInfo[id_new]; // Which tried bucket to move the entry to. int tried_bucket = info_new.GetTriedBucket(nKey, m_asmap); @@ -607,7 +850,7 @@ void CAddrMan::ResolveCollisions_() // Get the to-be-evicted address that is being tested int id_old = vvTried[tried_bucket][tried_bucket_pos]; - CAddrInfo& info_old = mapInfo[id_old]; + AddrInfo& info_old = mapInfo[id_old]; // Has successfully connected in last X hours if (GetAdjustedTime() - info_old.nLastSuccess < ADDRMAN_REPLACEMENT_HOURS*(60*60)) { @@ -644,11 +887,11 @@ void CAddrMan::ResolveCollisions_() } } -CAddrInfo CAddrMan::SelectTriedCollision_() +std::pair<CAddress, int64_t> AddrManImpl::SelectTriedCollision_() { AssertLockHeld(cs); - if (m_tried_collisions.size() == 0) return CAddrInfo(); + if (m_tried_collisions.size() == 0) return {}; std::set<int>::iterator it = m_tried_collisions.begin(); @@ -659,43 +902,291 @@ CAddrInfo CAddrMan::SelectTriedCollision_() // If id_new not found in mapInfo remove it from m_tried_collisions if (mapInfo.count(id_new) != 1) { m_tried_collisions.erase(it); - return CAddrInfo(); + return {}; } - const CAddrInfo& newInfo = mapInfo[id_new]; + const AddrInfo& newInfo = mapInfo[id_new]; // which tried bucket to move the entry to int tried_bucket = newInfo.GetTriedBucket(nKey, m_asmap); int tried_bucket_pos = newInfo.GetBucketPosition(nKey, false, tried_bucket); - int id_old = vvTried[tried_bucket][tried_bucket_pos]; + const AddrInfo& info_old = mapInfo[vvTried[tried_bucket][tried_bucket_pos]]; + return {info_old, info_old.nLastTry}; +} + +void AddrManImpl::Check() const +{ + AssertLockHeld(cs); + + // Run consistency checks 1 in m_consistency_check_ratio times if enabled + if (m_consistency_check_ratio == 0) return; + if (insecure_rand.randrange(m_consistency_check_ratio) >= 1) return; - return mapInfo[id_old]; + const int err{ForceCheckAddrman()}; + if (err) { + LogPrintf("ADDRMAN CONSISTENCY CHECK FAILED!!! err=%i\n", err); + assert(false); + } } -std::vector<bool> CAddrMan::DecodeAsmap(fs::path path) +int AddrManImpl::ForceCheckAddrman() const { - std::vector<bool> bits; - FILE *filestr = fsbridge::fopen(path, "rb"); - CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); - if (file.IsNull()) { - LogPrintf("Failed to open asmap file from disk\n"); - return bits; + AssertLockHeld(cs); + + LogPrint(BCLog::ADDRMAN, "Addrman checks started: new %i, tried %i, total %u\n", nNew, nTried, vRandom.size()); + + std::unordered_set<int> setTried; + std::unordered_map<int, int> mapNew; + + if (vRandom.size() != (size_t)(nTried + nNew)) + return -7; + + for (const auto& entry : mapInfo) { + int n = entry.first; + const AddrInfo& info = entry.second; + if (info.fInTried) { + if (!info.nLastSuccess) + return -1; + if (info.nRefCount) + return -2; + setTried.insert(n); + } else { + if (info.nRefCount < 0 || info.nRefCount > ADDRMAN_NEW_BUCKETS_PER_ADDRESS) + return -3; + if (!info.nRefCount) + return -4; + mapNew[n] = info.nRefCount; + } + const auto it{mapAddr.find(info)}; + if (it == mapAddr.end() || it->second != n) { + return -5; + } + if (info.nRandomPos < 0 || (size_t)info.nRandomPos >= vRandom.size() || vRandom[info.nRandomPos] != n) + return -14; + if (info.nLastTry < 0) + return -6; + if (info.nLastSuccess < 0) + return -8; } - fseek(filestr, 0, SEEK_END); - int length = ftell(filestr); - LogPrintf("Opened asmap file %s (%d bytes) from disk\n", path, length); - fseek(filestr, 0, SEEK_SET); - uint8_t cur_byte; - for (int i = 0; i < length; ++i) { - file >> cur_byte; - for (int bit = 0; bit < 8; ++bit) { - bits.push_back((cur_byte >> bit) & 1); + + if (setTried.size() != (size_t)nTried) + return -9; + if (mapNew.size() != (size_t)nNew) + return -10; + + for (int n = 0; n < ADDRMAN_TRIED_BUCKET_COUNT; n++) { + for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { + if (vvTried[n][i] != -1) { + if (!setTried.count(vvTried[n][i])) + return -11; + const auto it{mapInfo.find(vvTried[n][i])}; + if (it == mapInfo.end() || it->second.GetTriedBucket(nKey, m_asmap) != n) { + return -17; + } + if (it->second.GetBucketPosition(nKey, false, n) != i) { + return -18; + } + setTried.erase(vvTried[n][i]); + } } } - if (!SanityCheckASMap(bits)) { - LogPrintf("Sanity check of asmap file %s failed\n", path); - return {}; + + for (int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; n++) { + for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { + if (vvNew[n][i] != -1) { + if (!mapNew.count(vvNew[n][i])) + return -12; + const auto it{mapInfo.find(vvNew[n][i])}; + if (it == mapInfo.end() || it->second.GetBucketPosition(nKey, true, n) != i) { + return -19; + } + if (--mapNew[vvNew[n][i]] == 0) + mapNew.erase(vvNew[n][i]); + } + } } - return bits; + + if (setTried.size()) + return -13; + if (mapNew.size()) + return -15; + if (nKey.IsNull()) + return -16; + + LogPrint(BCLog::ADDRMAN, "Addrman checks completed successfully\n"); + return 0; +} + +size_t AddrManImpl::size() const +{ + LOCK(cs); // TODO: Cache this in an atomic to avoid this overhead + return vRandom.size(); +} + +bool AddrManImpl::Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, int64_t nTimePenalty) +{ + LOCK(cs); + int nAdd = 0; + Check(); + for (std::vector<CAddress>::const_iterator it = vAddr.begin(); it != vAddr.end(); it++) + nAdd += Add_(*it, source, nTimePenalty) ? 1 : 0; + Check(); + if (nAdd) { + LogPrint(BCLog::ADDRMAN, "Added %i addresses from %s: %i tried, %i new\n", nAdd, source.ToString(), nTried, nNew); + } + return nAdd > 0; +} + +void AddrManImpl::Good(const CService& addr, int64_t nTime) +{ + LOCK(cs); + Check(); + Good_(addr, /* test_before_evict */ true, nTime); + Check(); +} + +void AddrManImpl::Attempt(const CService& addr, bool fCountFailure, int64_t nTime) +{ + LOCK(cs); + Check(); + Attempt_(addr, fCountFailure, nTime); + Check(); +} + +void AddrManImpl::ResolveCollisions() +{ + LOCK(cs); + Check(); + ResolveCollisions_(); + Check(); +} + +std::pair<CAddress, int64_t> AddrManImpl::SelectTriedCollision() +{ + LOCK(cs); + Check(); + const auto ret = SelectTriedCollision_(); + Check(); + return ret; +} + +std::pair<CAddress, int64_t> AddrManImpl::Select(bool newOnly) const +{ + LOCK(cs); + Check(); + const auto addrRet = Select_(newOnly); + Check(); + return addrRet; +} + +std::vector<CAddress> AddrManImpl::GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const +{ + LOCK(cs); + Check(); + const auto addresses = GetAddr_(max_addresses, max_pct, network); + Check(); + return addresses; +} + +void AddrManImpl::Connected(const CService& addr, int64_t nTime) +{ + LOCK(cs); + Check(); + Connected_(addr, nTime); + Check(); +} + +void AddrManImpl::SetServices(const CService& addr, ServiceFlags nServices) +{ + LOCK(cs); + Check(); + SetServices_(addr, nServices); + Check(); +} + +const std::vector<bool>& AddrManImpl::GetAsmap() const +{ + return m_asmap; +} + +AddrMan::AddrMan(std::vector<bool> asmap, bool deterministic, int32_t consistency_check_ratio) + : m_impl(std::make_unique<AddrManImpl>(std::move(asmap), deterministic, consistency_check_ratio)) {} + +AddrMan::~AddrMan() = default; + +template <typename Stream> +void AddrMan::Serialize(Stream& s_) const +{ + m_impl->Serialize<Stream>(s_); +} + +template <typename Stream> +void AddrMan::Unserialize(Stream& s_) +{ + m_impl->Unserialize<Stream>(s_); +} + +// explicit instantiation +template void AddrMan::Serialize(CHashWriter& s) const; +template void AddrMan::Serialize(CAutoFile& s) const; +template void AddrMan::Serialize(CDataStream& s) const; +template void AddrMan::Unserialize(CAutoFile& s); +template void AddrMan::Unserialize(CHashVerifier<CAutoFile>& s); +template void AddrMan::Unserialize(CDataStream& s); +template void AddrMan::Unserialize(CHashVerifier<CDataStream>& s); + +size_t AddrMan::size() const +{ + return m_impl->size(); +} + +bool AddrMan::Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, int64_t nTimePenalty) +{ + return m_impl->Add(vAddr, source, nTimePenalty); +} + +void AddrMan::Good(const CService& addr, int64_t nTime) +{ + m_impl->Good(addr, nTime); +} + +void AddrMan::Attempt(const CService& addr, bool fCountFailure, int64_t nTime) +{ + m_impl->Attempt(addr, fCountFailure, nTime); +} + +void AddrMan::ResolveCollisions() +{ + m_impl->ResolveCollisions(); +} + +std::pair<CAddress, int64_t> AddrMan::SelectTriedCollision() +{ + return m_impl->SelectTriedCollision(); +} + +std::pair<CAddress, int64_t> AddrMan::Select(bool newOnly) const +{ + return m_impl->Select(newOnly); +} + +std::vector<CAddress> AddrMan::GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const +{ + return m_impl->GetAddr(max_addresses, max_pct, network); +} + +void AddrMan::Connected(const CService& addr, int64_t nTime) +{ + m_impl->Connected(addr, nTime); +} + +void AddrMan::SetServices(const CService& addr, ServiceFlags nServices) +{ + m_impl->SetServices(addr, nServices); +} + +const std::vector<bool>& AddrMan::GetAsmap() const +{ + return m_impl->GetAsmap(); } diff --git a/src/addrman.h b/src/addrman.h index 665e253192..174ab4f811 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -6,96 +6,21 @@ #ifndef BITCOIN_ADDRMAN_H #define BITCOIN_ADDRMAN_H -#include <clientversion.h> -#include <config/bitcoin-config.h> -#include <fs.h> -#include <hash.h> #include <netaddress.h> #include <protocol.h> -#include <random.h> #include <streams.h> -#include <sync.h> #include <timedata.h> -#include <tinyformat.h> -#include <util/system.h> -#include <iostream> +#include <cstdint> +#include <memory> #include <optional> -#include <set> -#include <stdint.h> -#include <unordered_map> +#include <utility> #include <vector> -/** - * Extended statistics about a CAddress - */ -class CAddrInfo : public CAddress -{ -public: - //! last try whatsoever by us (memory only) - int64_t nLastTry{0}; - - //! last counted attempt (memory only) - int64_t nLastCountAttempt{0}; - -private: - //! where knowledge about this address first came from - CNetAddr source; - - //! last successful connection by us - int64_t nLastSuccess{0}; - - //! connection attempts since last successful attempt - int nAttempts{0}; - - //! reference count in new sets (memory only) - int nRefCount{0}; - - //! in tried set? (memory only) - bool fInTried{false}; - - //! position in vRandom - int nRandomPos{-1}; - - friend class CAddrMan; - -public: - - SERIALIZE_METHODS(CAddrInfo, obj) - { - READWRITEAS(CAddress, obj); - READWRITE(obj.source, obj.nLastSuccess, obj.nAttempts); - } - - CAddrInfo(const CAddress &addrIn, const CNetAddr &addrSource) : CAddress(addrIn), source(addrSource) - { - } +class AddrManImpl; - CAddrInfo() : CAddress(), source() - { - } - - //! Calculate in which "tried" bucket this entry belongs - int GetTriedBucket(const uint256 &nKey, const std::vector<bool> &asmap) const; - - //! Calculate in which "new" bucket this entry belongs, given a certain source - int GetNewBucket(const uint256 &nKey, const CNetAddr& src, const std::vector<bool> &asmap) const; - - //! Calculate in which "new" bucket this entry belongs, using its default source - int GetNewBucket(const uint256 &nKey, const std::vector<bool> &asmap) const - { - return GetNewBucket(nKey, source, asmap); - } - - //! Calculate in which position of a bucket to store this entry. - int GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const; - - //! Determine whether the statistics about this entry are bad enough so that it can just be deleted - bool IsTerrible(int64_t nNow = GetAdjustedTime()) const; - - //! Calculate the relative chance this entry should be given when selecting nodes to connect to - double GetChance(int64_t nNow = GetAdjustedTime()) const; -}; +/** Default for -checkaddrman */ +static constexpr int32_t DEFAULT_ADDRMAN_CONSISTENCY_CHECKS{0}; /** Stochastic address manager * @@ -104,647 +29,86 @@ public: * * Make sure no (localized) attacker can fill the entire table with his nodes/addresses. * * To that end: - * * Addresses are organized into buckets. - * * Addresses that have not yet been tried go into 1024 "new" buckets. - * * Based on the address range (/16 for IPv4) of the source of information, 64 buckets are selected at random. + * * Addresses are organized into buckets that can each store up to 64 entries. + * * Addresses to which our node has not successfully connected go into 1024 "new" buckets. + * * Based on the address range (/16 for IPv4) of the source of information, or if an asmap is provided, + * the AS it belongs to (for IPv4/IPv6), 64 buckets are selected at random. * * The actual bucket is chosen from one of these, based on the range in which the address itself is located. + * * The position in the bucket is chosen based on the full address. * * One single address can occur in up to 8 different buckets to increase selection chances for addresses that * are seen frequently. The chance for increasing this multiplicity decreases exponentially. - * * When adding a new address to a full bucket, a randomly chosen entry (with a bias favoring less recently seen - * ones) is removed from it first. + * * When adding a new address to an occupied position of a bucket, it will not replace the existing entry + * unless that address is also stored in another bucket or it doesn't meet one of several quality criteria + * (see IsTerrible for exact criteria). * * Addresses of nodes that are known to be accessible go into 256 "tried" buckets. * * Each address range selects at random 8 of these buckets. * * The actual bucket is chosen from one of these, based on the full address. - * * When adding a new good address to a full bucket, a randomly chosen entry (with a bias favoring less recently - * tried ones) is evicted from it, back to the "new" buckets. + * * When adding a new good address to an occupied position of a bucket, a FEELER connection to the + * old address is attempted. The old entry is only replaced and moved back to the "new" buckets if this + * attempt was unsuccessful. * * Bucket selection is based on cryptographic hashing, using a randomly-generated 256-bit key, which should not * be observable by adversaries. - * * Several indexes are kept for high performance. Defining DEBUG_ADDRMAN will introduce frequent (and expensive) - * consistency checks for the entire data structure. + * * Several indexes are kept for high performance. Setting m_consistency_check_ratio with the -checkaddrman + * configuration option will introduce (expensive) consistency checks for the entire data structure. */ - -//! total number of buckets for tried addresses -#define ADDRMAN_TRIED_BUCKET_COUNT_LOG2 8 - -//! total number of buckets for new addresses -#define ADDRMAN_NEW_BUCKET_COUNT_LOG2 10 - -//! maximum allowed number of entries in buckets for new and tried addresses -#define ADDRMAN_BUCKET_SIZE_LOG2 6 - -//! over how many buckets entries with tried addresses from a single group (/16 for IPv4) are spread -#define ADDRMAN_TRIED_BUCKETS_PER_GROUP 8 - -//! over how many buckets entries with new addresses originating from a single group are spread -#define ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP 64 - -//! in how many buckets for entries with new addresses a single address may occur -#define ADDRMAN_NEW_BUCKETS_PER_ADDRESS 8 - -//! how old addresses can maximally be -#define ADDRMAN_HORIZON_DAYS 30 - -//! after how many failed attempts we give up on a new node -#define ADDRMAN_RETRIES 3 - -//! how many successive failures are allowed ... -#define ADDRMAN_MAX_FAILURES 10 - -//! ... in at least this many days -#define ADDRMAN_MIN_FAIL_DAYS 7 - -//! how recent a successful connection should be before we allow an address to be evicted from tried -#define ADDRMAN_REPLACEMENT_HOURS 4 - -//! Convenience -#define ADDRMAN_TRIED_BUCKET_COUNT (1 << ADDRMAN_TRIED_BUCKET_COUNT_LOG2) -#define ADDRMAN_NEW_BUCKET_COUNT (1 << ADDRMAN_NEW_BUCKET_COUNT_LOG2) -#define ADDRMAN_BUCKET_SIZE (1 << ADDRMAN_BUCKET_SIZE_LOG2) - -//! the maximum number of tried addr collisions to store -#define ADDRMAN_SET_TRIED_COLLISION_SIZE 10 - -//! the maximum time we'll spend trying to resolve a tried table collision, in seconds -static const int64_t ADDRMAN_TEST_WINDOW = 40*60; // 40 minutes - -/** - * Stochastical (IP) address manager - */ -class CAddrMan +class AddrMan { + const std::unique_ptr<AddrManImpl> m_impl; + public: - // Compressed IP->ASN mapping, loaded from a file when a node starts. - // Should be always empty if no file was provided. - // This mapping is then used for bucketing nodes in Addrman. - // - // If asmap is provided, nodes will be bucketed by - // AS they belong to, in order to make impossible for a node - // to connect to several nodes hosted in a single AS. - // This is done in response to Erebus attack, but also to generally - // diversify the connections every node creates, - // especially useful when a large fraction of nodes - // operate under a couple of cloud providers. - // - // If a new asmap was provided, the existing records - // would be re-bucketed accordingly. - std::vector<bool> m_asmap; + explicit AddrMan(std::vector<bool> asmap, bool deterministic, int32_t consistency_check_ratio); - // Read asmap from provided binary file - static std::vector<bool> DecodeAsmap(fs::path path); + ~AddrMan(); - /** - * Serialized format. - * * format version byte (@see `Format`) - * * lowest compatible format version byte. This is used to help old software decide - * whether to parse the file. For example: - * * Bitcoin Core version N knows how to parse up to format=3. If a new format=4 is - * introduced in version N+1 that is compatible with format=3 and it is known that - * version N will be able to parse it, then version N+1 will write - * (format=4, lowest_compatible=3) in the first two bytes of the file, and so - * version N will still try to parse it. - * * Bitcoin Core version N+2 introduces a new incompatible format=5. It will write - * (format=5, lowest_compatible=5) and so any versions that do not know how to parse - * format=5 will not try to read the file. - * * nKey - * * nNew - * * nTried - * * number of "new" buckets XOR 2**30 - * * all new addresses (total count: nNew) - * * all tried addresses (total count: nTried) - * * for each new bucket: - * * number of elements - * * for each element: index in the serialized "all new addresses" - * * asmap checksum - * - * 2**30 is xorred with the number of buckets to make addrman deserializer v0 detect it - * as incompatible. This is necessary because it did not check the version number on - * deserialization. - * - * vvNew, vvTried, mapInfo, mapAddr and vRandom are never encoded explicitly; - * they are instead reconstructed from the other information. - * - * This format is more complex, but significantly smaller (at most 1.5 MiB), and supports - * changes to the ADDRMAN_ parameters without breaking the on-disk structure. - * - * We don't use SERIALIZE_METHODS since the serialization and deserialization code has - * very little in common. - */ template <typename Stream> - void Serialize(Stream& s_) const - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - - // Always serialize in the latest version (FILE_FORMAT). - - OverrideStream<Stream> s(&s_, s_.GetType(), s_.GetVersion() | ADDRV2_FORMAT); - - s << static_cast<uint8_t>(FILE_FORMAT); - - // Increment `lowest_compatible` iff a newly introduced format is incompatible with - // the previous one. - static constexpr uint8_t lowest_compatible = Format::V3_BIP155; - s << static_cast<uint8_t>(INCOMPATIBILITY_BASE + lowest_compatible); - - s << nKey; - s << nNew; - s << nTried; - - int nUBuckets = ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30); - s << nUBuckets; - std::unordered_map<int, int> mapUnkIds; - int nIds = 0; - for (const auto& entry : mapInfo) { - mapUnkIds[entry.first] = nIds; - const CAddrInfo &info = entry.second; - if (info.nRefCount) { - assert(nIds != nNew); // this means nNew was wrong, oh ow - s << info; - nIds++; - } - } - nIds = 0; - for (const auto& entry : mapInfo) { - const CAddrInfo &info = entry.second; - if (info.fInTried) { - assert(nIds != nTried); // this means nTried was wrong, oh ow - s << info; - nIds++; - } - } - for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; bucket++) { - int nSize = 0; - for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { - if (vvNew[bucket][i] != -1) - nSize++; - } - s << nSize; - for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { - if (vvNew[bucket][i] != -1) { - int nIndex = mapUnkIds[vvNew[bucket][i]]; - s << nIndex; - } - } - } - // Store asmap checksum after bucket entries so that it - // can be ignored by older clients for backward compatibility. - uint256 asmap_checksum; - if (m_asmap.size() != 0) { - asmap_checksum = SerializeHash(m_asmap); - } - s << asmap_checksum; - } + void Serialize(Stream& s_) const; template <typename Stream> - void Unserialize(Stream& s_) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - - assert(vRandom.empty()); - - Format format; - s_ >> Using<CustomUintFormatter<1>>(format); - - int stream_version = s_.GetVersion(); - if (format >= Format::V3_BIP155) { - // Add ADDRV2_FORMAT to the version so that the CNetAddr and CAddress - // unserialize methods know that an address in addrv2 format is coming. - stream_version |= ADDRV2_FORMAT; - } - - OverrideStream<Stream> s(&s_, s_.GetType(), stream_version); - - uint8_t compat; - s >> compat; - const uint8_t lowest_compatible = compat - INCOMPATIBILITY_BASE; - if (lowest_compatible > FILE_FORMAT) { - throw std::ios_base::failure(strprintf( - "Unsupported format of addrman database: %u. It is compatible with formats >=%u, " - "but the maximum supported by this version of %s is %u.", - format, lowest_compatible, PACKAGE_NAME, static_cast<uint8_t>(FILE_FORMAT))); - } - - s >> nKey; - s >> nNew; - s >> nTried; - int nUBuckets = 0; - s >> nUBuckets; - if (format >= Format::V1_DETERMINISTIC) { - nUBuckets ^= (1 << 30); - } - - if (nNew > ADDRMAN_NEW_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE) { - throw std::ios_base::failure("Corrupt CAddrMan serialization, nNew exceeds limit."); - } - - if (nTried > ADDRMAN_TRIED_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE) { - throw std::ios_base::failure("Corrupt CAddrMan serialization, nTried exceeds limit."); - } - - // Deserialize entries from the new table. - for (int n = 0; n < nNew; n++) { - CAddrInfo &info = mapInfo[n]; - s >> info; - mapAddr[info] = n; - info.nRandomPos = vRandom.size(); - vRandom.push_back(n); - } - nIdCount = nNew; - - // Deserialize entries from the tried table. - int nLost = 0; - for (int n = 0; n < nTried; n++) { - CAddrInfo info; - s >> info; - int nKBucket = info.GetTriedBucket(nKey, m_asmap); - int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); - if (vvTried[nKBucket][nKBucketPos] == -1) { - info.nRandomPos = vRandom.size(); - info.fInTried = true; - vRandom.push_back(nIdCount); - mapInfo[nIdCount] = info; - mapAddr[info] = nIdCount; - vvTried[nKBucket][nKBucketPos] = nIdCount; - nIdCount++; - } else { - nLost++; - } - } - nTried -= nLost; - - // Store positions in the new table buckets to apply later (if possible). - // An entry may appear in up to ADDRMAN_NEW_BUCKETS_PER_ADDRESS buckets, - // so we store all bucket-entry_index pairs to iterate through later. - std::vector<std::pair<int, int>> bucket_entries; - - for (int bucket = 0; bucket < nUBuckets; ++bucket) { - int num_entries{0}; - s >> num_entries; - for (int n = 0; n < num_entries; ++n) { - int entry_index{0}; - s >> entry_index; - if (entry_index >= 0 && entry_index < nNew) { - bucket_entries.emplace_back(bucket, entry_index); - } - } - } - - // If the bucket count and asmap checksum haven't changed, then attempt - // to restore the entries to the buckets/positions they were in before - // serialization. - uint256 supplied_asmap_checksum; - if (m_asmap.size() != 0) { - supplied_asmap_checksum = SerializeHash(m_asmap); - } - uint256 serialized_asmap_checksum; - if (format >= Format::V2_ASMAP) { - s >> serialized_asmap_checksum; - } - const bool restore_bucketing{nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && - serialized_asmap_checksum == supplied_asmap_checksum}; - - if (!restore_bucketing) { - LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n"); - } - - for (auto bucket_entry : bucket_entries) { - int bucket{bucket_entry.first}; - const int entry_index{bucket_entry.second}; - CAddrInfo& info = mapInfo[entry_index]; - - // The entry shouldn't appear in more than - // ADDRMAN_NEW_BUCKETS_PER_ADDRESS. If it has already, just skip - // this bucket_entry. - if (info.nRefCount >= ADDRMAN_NEW_BUCKETS_PER_ADDRESS) continue; - - int bucket_position = info.GetBucketPosition(nKey, true, bucket); - if (restore_bucketing && vvNew[bucket][bucket_position] == -1) { - // Bucketing has not changed, using existing bucket positions for the new table - vvNew[bucket][bucket_position] = entry_index; - ++info.nRefCount; - } else { - // In case the new table data cannot be used (bucket count wrong or new asmap), - // try to give them a reference based on their primary source address. - bucket = info.GetNewBucket(nKey, m_asmap); - bucket_position = info.GetBucketPosition(nKey, true, bucket); - if (vvNew[bucket][bucket_position] == -1) { - vvNew[bucket][bucket_position] = entry_index; - ++info.nRefCount; - } - } - } - - // Prune new entries with refcount 0 (as a result of collisions). - int nLostUnk = 0; - for (auto it = mapInfo.cbegin(); it != mapInfo.cend(); ) { - if (it->second.fInTried == false && it->second.nRefCount == 0) { - const auto itCopy = it++; - Delete(itCopy->first); - ++nLostUnk; - } else { - ++it; - } - } - if (nLost + nLostUnk > 0) { - LogPrint(BCLog::ADDRMAN, "addrman lost %i new and %i tried addresses due to collisions\n", nLostUnk, nLost); - } - - Check(); - } - - void Clear() - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - std::vector<int>().swap(vRandom); - nKey = insecure_rand.rand256(); - for (size_t bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; bucket++) { - for (size_t entry = 0; entry < ADDRMAN_BUCKET_SIZE; entry++) { - vvNew[bucket][entry] = -1; - } - } - for (size_t bucket = 0; bucket < ADDRMAN_TRIED_BUCKET_COUNT; bucket++) { - for (size_t entry = 0; entry < ADDRMAN_BUCKET_SIZE; entry++) { - vvTried[bucket][entry] = -1; - } - } - - nIdCount = 0; - nTried = 0; - nNew = 0; - nLastGood = 1; //Initially at 1 so that "never" is strictly worse. - mapInfo.clear(); - mapAddr.clear(); - } - - CAddrMan() - { - Clear(); - } - - ~CAddrMan() - { - nKey.SetNull(); - } + void Unserialize(Stream& s_); //! Return the number of (unique) addresses in all tables. - size_t size() const - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); // TODO: Cache this in an atomic to avoid this overhead - return vRandom.size(); - } + size_t size() const; - //! Add a single address. - bool Add(const CAddress &addr, const CNetAddr& source, int64_t nTimePenalty = 0) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - bool fRet = false; - Check(); - fRet |= Add_(addr, source, nTimePenalty); - Check(); - if (fRet) { - LogPrint(BCLog::ADDRMAN, "Added %s from %s: %i tried, %i new\n", addr.ToStringIPPort(), source.ToString(), nTried, nNew); - } - return fRet; - } + //! Add addresses to addrman's new table. + bool Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, int64_t nTimePenalty = 0); - //! Add multiple addresses. - bool Add(const std::vector<CAddress> &vAddr, const CNetAddr& source, int64_t nTimePenalty = 0) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - int nAdd = 0; - Check(); - for (std::vector<CAddress>::const_iterator it = vAddr.begin(); it != vAddr.end(); it++) - nAdd += Add_(*it, source, nTimePenalty) ? 1 : 0; - Check(); - if (nAdd) { - LogPrint(BCLog::ADDRMAN, "Added %i addresses from %s: %i tried, %i new\n", nAdd, source.ToString(), nTried, nNew); - } - return nAdd > 0; - } - - //! Mark an entry as accessible. - void Good(const CService &addr, bool test_before_evict = true, int64_t nTime = GetAdjustedTime()) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - Good_(addr, test_before_evict, nTime); - Check(); - } + //! Mark an entry as accessible, possibly moving it from "new" to "tried". + void Good(const CService& addr, int64_t nTime = GetAdjustedTime()); //! Mark an entry as connection attempted to. - void Attempt(const CService &addr, bool fCountFailure, int64_t nTime = GetAdjustedTime()) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - Attempt_(addr, fCountFailure, nTime); - Check(); - } + void Attempt(const CService& addr, bool fCountFailure, int64_t nTime = GetAdjustedTime()); //! See if any to-be-evicted tried table entries have been tested and if so resolve the collisions. - void ResolveCollisions() - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - ResolveCollisions_(); - Check(); - } - - //! Randomly select an address in tried that another address is attempting to evict. - CAddrInfo SelectTriedCollision() - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - const CAddrInfo ret = SelectTriedCollision_(); - Check(); - return ret; - } + void ResolveCollisions(); /** - * Choose an address to connect to. + * Randomly select an address in the tried table that another address is + * attempting to evict. + * + * @return CAddress The record for the selected tried peer. + * int64_t The last time we attempted to connect to that peer. */ - CAddrInfo Select(bool newOnly = false) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - const CAddrInfo addrRet = Select_(newOnly); - Check(); - return addrRet; - } + std::pair<CAddress, int64_t> SelectTriedCollision(); /** - * Return all or many randomly selected addresses, optionally by network. + * Choose an address to connect to. * - * @param[in] max_addresses Maximum number of addresses to return (0 = all). - * @param[in] max_pct Maximum percentage of addresses to return (0 = all). - * @param[in] network Select only addresses of this network (nullopt = all). + * @param[in] newOnly Whether to only select addresses from the new table. + * @return CAddress The record for the selected peer. + * int64_t The last time we attempted to connect to that peer. */ - std::vector<CAddress> GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - std::vector<CAddress> vAddr; - GetAddr_(vAddr, max_addresses, max_pct, network); - Check(); - return vAddr; - } - - //! Outer function for Connected_() - void Connected(const CService &addr, int64_t nTime = GetAdjustedTime()) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - Connected_(addr, nTime); - Check(); - } - - void SetServices(const CService &addr, ServiceFlags nServices) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - SetServices_(addr, nServices); - Check(); - } - -protected: - //! secret key to randomize bucket select with - uint256 nKey; - - //! Source of random numbers for randomization in inner loops - FastRandomContext insecure_rand; - -private: - //! A mutex to protect the inner data structures. - mutable Mutex cs; - - //! Serialization versions. - enum Format : uint8_t { - V0_HISTORICAL = 0, //!< historic format, before commit e6b343d88 - V1_DETERMINISTIC = 1, //!< for pre-asmap files - V2_ASMAP = 2, //!< for files including asmap version - V3_BIP155 = 3, //!< same as V2_ASMAP plus addresses are in BIP155 format - }; - - //! The maximum format this software knows it can unserialize. Also, we always serialize - //! in this format. - //! The format (first byte in the serialized stream) can be higher than this and - //! still this software may be able to unserialize the file - if the second byte - //! (see `lowest_compatible` in `Unserialize()`) is less or equal to this. - static constexpr Format FILE_FORMAT = Format::V3_BIP155; - - //! The initial value of a field that is incremented every time an incompatible format - //! change is made (such that old software versions would not be able to parse and - //! understand the new file format). This is 32 because we overtook the "key size" - //! field which was 32 historically. - //! @note Don't increment this. Increment `lowest_compatible` in `Serialize()` instead. - static constexpr uint8_t INCOMPATIBILITY_BASE = 32; - - //! last used nId - int nIdCount GUARDED_BY(cs); - - //! table with information about all nIds - std::unordered_map<int, CAddrInfo> mapInfo GUARDED_BY(cs); - - //! find an nId based on its network address - std::unordered_map<CNetAddr, int, CNetAddrHash> mapAddr GUARDED_BY(cs); - - //! randomly-ordered vector of all nIds - std::vector<int> vRandom GUARDED_BY(cs); - - // number of "tried" entries - int nTried GUARDED_BY(cs); - - //! list of "tried" buckets - int vvTried[ADDRMAN_TRIED_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); - - //! number of (unique) "new" entries - int nNew GUARDED_BY(cs); - - //! list of "new" buckets - int vvNew[ADDRMAN_NEW_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); - - //! last time Good was called (memory only) - int64_t nLastGood GUARDED_BY(cs); - - //! Holds addrs inserted into tried table that collide with existing entries. Test-before-evict discipline used to resolve these collisions. - std::set<int> m_tried_collisions; - - //! Find an entry. - CAddrInfo* Find(const CNetAddr& addr, int *pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! find an entry, creating it if necessary. - //! nTime and nServices of the found node are updated, if necessary. - CAddrInfo* Create(const CAddress &addr, const CNetAddr &addrSource, int *pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Swap two elements in vRandom. - void SwapRandom(unsigned int nRandomPos1, unsigned int nRandomPos2) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Move an entry from the "new" table(s) to the "tried" table - void MakeTried(CAddrInfo& info, int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Delete an entry. It must not be in tried, and have refcount 0. - void Delete(int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Clear a position in a "new" table. This is the only place where entries are actually deleted. - void ClearNew(int nUBucket, int nUBucketPos) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Mark an entry "good", possibly moving it from "new" to "tried". - void Good_(const CService &addr, bool test_before_evict, int64_t time) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Add an entry to the "new" table. - bool Add_(const CAddress &addr, const CNetAddr& source, int64_t nTimePenalty) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Mark an entry as attempted to connect. - void Attempt_(const CService &addr, bool fCountFailure, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Select an address to connect to, if newOnly is set to true, only the new table is selected from. - CAddrInfo Select_(bool newOnly) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! See if any to-be-evicted tried table entries have been tested and if so resolve the collisions. - void ResolveCollisions_() EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Return a random to-be-evicted tried table address. - CAddrInfo SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Consistency check - void Check() - EXCLUSIVE_LOCKS_REQUIRED(cs) - { -#ifdef DEBUG_ADDRMAN - AssertLockHeld(cs); - const int err = Check_(); - if (err) { - LogPrintf("ADDRMAN CONSISTENCY CHECK FAILED!!! err=%i\n", err); - } -#endif - } - -#ifdef DEBUG_ADDRMAN - //! Perform consistency check. Returns an error code or zero. - int Check_() EXCLUSIVE_LOCKS_REQUIRED(cs); -#endif + std::pair<CAddress, int64_t> Select(bool newOnly = false) const; /** * Return all or many randomly selected addresses, optionally by network. * - * @param[out] vAddr Vector of randomly selected addresses from vRandom. * @param[in] max_addresses Maximum number of addresses to return (0 = all). * @param[in] max_pct Maximum percentage of addresses to return (0 = all). * @param[in] network Select only addresses of this network (nullopt = all). + * + * @return A vector of randomly selected addresses from vRandom. */ - void GetAddr_(std::vector<CAddress>& vAddr, size_t max_addresses, size_t max_pct, std::optional<Network> network) EXCLUSIVE_LOCKS_REQUIRED(cs); + std::vector<CAddress> GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const; /** We have successfully connected to this peer. Calling this function * updates the CAddress's nTime, which is used in our IsTerrible() @@ -757,12 +121,15 @@ private: * @param[in] addr The address of the peer we were connected to * @param[in] nTime The time that we were last connected to this peer */ - void Connected_(const CService& addr, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); + void Connected(const CService& addr, int64_t nTime = GetAdjustedTime()); //! Update an entry's service bits. - void SetServices_(const CService &addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); + void SetServices(const CService& addr, ServiceFlags nServices); + + const std::vector<bool>& GetAsmap() const; - friend class CAddrManTest; + friend class AddrManTest; + friend class AddrManDeterministic; }; #endif // BITCOIN_ADDRMAN_H diff --git a/src/addrman_impl.h b/src/addrman_impl.h new file mode 100644 index 0000000000..1dc7f25f9c --- /dev/null +++ b/src/addrman_impl.h @@ -0,0 +1,271 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_ADDRMAN_IMPL_H +#define BITCOIN_ADDRMAN_IMPL_H + +#include <logging.h> +#include <netaddress.h> +#include <protocol.h> +#include <serialize.h> +#include <sync.h> +#include <uint256.h> + +#include <cstdint> +#include <optional> +#include <set> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +/** Total number of buckets for tried addresses */ +static constexpr int32_t ADDRMAN_TRIED_BUCKET_COUNT_LOG2{8}; +static constexpr int ADDRMAN_TRIED_BUCKET_COUNT{1 << ADDRMAN_TRIED_BUCKET_COUNT_LOG2}; +/** Total number of buckets for new addresses */ +static constexpr int32_t ADDRMAN_NEW_BUCKET_COUNT_LOG2{10}; +static constexpr int ADDRMAN_NEW_BUCKET_COUNT{1 << ADDRMAN_NEW_BUCKET_COUNT_LOG2}; +/** Maximum allowed number of entries in buckets for new and tried addresses */ +static constexpr int32_t ADDRMAN_BUCKET_SIZE_LOG2{6}; +static constexpr int ADDRMAN_BUCKET_SIZE{1 << ADDRMAN_BUCKET_SIZE_LOG2}; + +/** + * Extended statistics about a CAddress + */ +class AddrInfo : public CAddress +{ +public: + //! last try whatsoever by us (memory only) + int64_t nLastTry{0}; + + //! last counted attempt (memory only) + int64_t nLastCountAttempt{0}; + + //! where knowledge about this address first came from + CNetAddr source; + + //! last successful connection by us + int64_t nLastSuccess{0}; + + //! connection attempts since last successful attempt + int nAttempts{0}; + + //! reference count in new sets (memory only) + int nRefCount{0}; + + //! in tried set? (memory only) + bool fInTried{false}; + + //! position in vRandom + mutable int nRandomPos{-1}; + + SERIALIZE_METHODS(AddrInfo, obj) + { + READWRITEAS(CAddress, obj); + READWRITE(obj.source, obj.nLastSuccess, obj.nAttempts); + } + + AddrInfo(const CAddress &addrIn, const CNetAddr &addrSource) : CAddress(addrIn), source(addrSource) + { + } + + AddrInfo() : CAddress(), source() + { + } + + //! Calculate in which "tried" bucket this entry belongs + int GetTriedBucket(const uint256 &nKey, const std::vector<bool> &asmap) const; + + //! Calculate in which "new" bucket this entry belongs, given a certain source + int GetNewBucket(const uint256 &nKey, const CNetAddr& src, const std::vector<bool> &asmap) const; + + //! Calculate in which "new" bucket this entry belongs, using its default source + int GetNewBucket(const uint256 &nKey, const std::vector<bool> &asmap) const + { + return GetNewBucket(nKey, source, asmap); + } + + //! Calculate in which position of a bucket to store this entry. + int GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const; + + //! Determine whether the statistics about this entry are bad enough so that it can just be deleted + bool IsTerrible(int64_t nNow = GetAdjustedTime()) const; + + //! Calculate the relative chance this entry should be given when selecting nodes to connect to + double GetChance(int64_t nNow = GetAdjustedTime()) const; +}; + +class AddrManImpl +{ +public: + AddrManImpl(std::vector<bool>&& asmap, bool deterministic, int32_t consistency_check_ratio); + + ~AddrManImpl(); + + template <typename Stream> + void Serialize(Stream& s_) const EXCLUSIVE_LOCKS_REQUIRED(!cs); + + template <typename Stream> + void Unserialize(Stream& s_) EXCLUSIVE_LOCKS_REQUIRED(!cs); + + size_t size() const EXCLUSIVE_LOCKS_REQUIRED(!cs); + + bool Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, int64_t nTimePenalty) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void Good(const CService& addr, int64_t nTime) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void Attempt(const CService& addr, bool fCountFailure, int64_t nTime) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void ResolveCollisions() EXCLUSIVE_LOCKS_REQUIRED(!cs); + + std::pair<CAddress, int64_t> SelectTriedCollision() EXCLUSIVE_LOCKS_REQUIRED(!cs); + + std::pair<CAddress, int64_t> Select(bool newOnly) const + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + std::vector<CAddress> GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void Connected(const CService& addr, int64_t nTime) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void SetServices(const CService& addr, ServiceFlags nServices) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + const std::vector<bool>& GetAsmap() const; + + friend class AddrManTest; + friend class AddrManDeterministic; + +private: + //! A mutex to protect the inner data structures. + mutable Mutex cs; + + //! Source of random numbers for randomization in inner loops + mutable FastRandomContext insecure_rand GUARDED_BY(cs); + + //! secret key to randomize bucket select with + uint256 nKey; + + //! Serialization versions. + enum Format : uint8_t { + V0_HISTORICAL = 0, //!< historic format, before commit e6b343d88 + V1_DETERMINISTIC = 1, //!< for pre-asmap files + V2_ASMAP = 2, //!< for files including asmap version + V3_BIP155 = 3, //!< same as V2_ASMAP plus addresses are in BIP155 format + }; + + //! The maximum format this software knows it can unserialize. Also, we always serialize + //! in this format. + //! The format (first byte in the serialized stream) can be higher than this and + //! still this software may be able to unserialize the file - if the second byte + //! (see `lowest_compatible` in `Unserialize()`) is less or equal to this. + static constexpr Format FILE_FORMAT = Format::V3_BIP155; + + //! The initial value of a field that is incremented every time an incompatible format + //! change is made (such that old software versions would not be able to parse and + //! understand the new file format). This is 32 because we overtook the "key size" + //! field which was 32 historically. + //! @note Don't increment this. Increment `lowest_compatible` in `Serialize()` instead. + static constexpr uint8_t INCOMPATIBILITY_BASE = 32; + + //! last used nId + int nIdCount GUARDED_BY(cs){0}; + + //! table with information about all nIds + std::unordered_map<int, AddrInfo> mapInfo GUARDED_BY(cs); + + //! find an nId based on its network address + std::unordered_map<CNetAddr, int, CNetAddrHash> mapAddr GUARDED_BY(cs); + + //! randomly-ordered vector of all nIds + //! This is mutable because it is unobservable outside the class, so any + //! changes to it (even in const methods) are also unobservable. + mutable std::vector<int> vRandom GUARDED_BY(cs); + + // number of "tried" entries + int nTried GUARDED_BY(cs){0}; + + //! list of "tried" buckets + int vvTried[ADDRMAN_TRIED_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); + + //! number of (unique) "new" entries + int nNew GUARDED_BY(cs){0}; + + //! list of "new" buckets + int vvNew[ADDRMAN_NEW_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); + + //! last time Good was called (memory only). Initially set to 1 so that "never" is strictly worse. + int64_t nLastGood GUARDED_BY(cs){1}; + + //! Holds addrs inserted into tried table that collide with existing entries. Test-before-evict discipline used to resolve these collisions. + std::set<int> m_tried_collisions; + + /** Perform consistency checks every m_consistency_check_ratio operations (if non-zero). */ + const int32_t m_consistency_check_ratio; + + // Compressed IP->ASN mapping, loaded from a file when a node starts. + // Should be always empty if no file was provided. + // This mapping is then used for bucketing nodes in Addrman. + // + // If asmap is provided, nodes will be bucketed by + // AS they belong to, in order to make impossible for a node + // to connect to several nodes hosted in a single AS. + // This is done in response to Erebus attack, but also to generally + // diversify the connections every node creates, + // especially useful when a large fraction of nodes + // operate under a couple of cloud providers. + // + // If a new asmap was provided, the existing records + // would be re-bucketed accordingly. + const std::vector<bool> m_asmap; + + //! Find an entry. + AddrInfo* Find(const CNetAddr& addr, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Create a new entry and add it to the internal data structures mapInfo, mapAddr and vRandom. + AddrInfo* Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Swap two elements in vRandom. + void SwapRandom(unsigned int nRandomPos1, unsigned int nRandomPos2) const EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Delete an entry. It must not be in tried, and have refcount 0. + void Delete(int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Clear a position in a "new" table. This is the only place where entries are actually deleted. + void ClearNew(int nUBucket, int nUBucketPos) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Move an entry from the "new" table(s) to the "tried" table + void MakeTried(AddrInfo& info, int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); + + void Good_(const CService& addr, bool test_before_evict, int64_t time) EXCLUSIVE_LOCKS_REQUIRED(cs); + + bool Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimePenalty) EXCLUSIVE_LOCKS_REQUIRED(cs); + + void Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); + + std::pair<CAddress, int64_t> Select_(bool newOnly) const EXCLUSIVE_LOCKS_REQUIRED(cs); + + std::vector<CAddress> GetAddr_(size_t max_addresses, size_t max_pct, std::optional<Network> network) const EXCLUSIVE_LOCKS_REQUIRED(cs); + + void Connected_(const CService& addr, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); + + void SetServices_(const CService& addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); + + void ResolveCollisions_() EXCLUSIVE_LOCKS_REQUIRED(cs); + + std::pair<CAddress, int64_t> SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Consistency check, taking into account m_consistency_check_ratio. Will std::abort if an inconsistency is detected. + void Check() const EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Perform consistency check, regardless of m_consistency_check_ratio. + //! @returns an error code or zero. + int ForceCheckAddrman() const EXCLUSIVE_LOCKS_REQUIRED(cs); +}; + +#endif // BITCOIN_ADDRMAN_IMPL_H diff --git a/src/banman.cpp b/src/banman.cpp index d2437e6733..c64a48a05a 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -18,7 +18,7 @@ BanMan::BanMan(fs::path ban_file, CClientUIInterface* client_interface, int64_t if (m_client_interface) m_client_interface->InitMessage(_("Loading banlist…").translated); int64_t n_start = GetTimeMillis(); - if (m_ban_db.Read(m_banned, m_is_dirty)) { + if (m_ban_db.Read(m_banned)) { SweepBanned(); // sweep out unused entries LogPrint(BCLog::NET, "Loaded %d banned node addresses/subnets %dms\n", m_banned.size(), diff --git a/src/banman.h b/src/banman.h index 8c75d4037e..8a03a9e3fc 100644 --- a/src/banman.h +++ b/src/banman.h @@ -88,7 +88,7 @@ private: RecursiveMutex m_cs_banned; banmap_t m_banned GUARDED_BY(m_cs_banned); - bool m_is_dirty GUARDED_BY(m_cs_banned); + bool m_is_dirty GUARDED_BY(m_cs_banned){false}; CClientUIInterface* m_client_interface = nullptr; CBanDB m_ban_db; const int64_t m_default_ban_time; diff --git a/src/bech32.cpp b/src/bech32.cpp index 288b14e023..9da2488ef2 100644 --- a/src/bech32.cpp +++ b/src/bech32.cpp @@ -66,6 +66,26 @@ uint32_t PolyMod(const data& v) // the above example, `c` initially corresponds to 1 mod g(x), and after processing 2 inputs of // v, it corresponds to x^2 + v0*x + v1 mod g(x). As 1 mod g(x) = 1, that is the starting value // for `c`. + + // The following Sage code constructs the generator used: + // + // B = GF(2) # Binary field + // BP.<b> = B[] # Polynomials over the binary field + // F_mod = b**5 + b**3 + 1 + // F.<f> = GF(32, modulus=F_mod, repr='int') # GF(32) definition + // FP.<x> = F[] # Polynomials over GF(32) + // E_mod = x**2 + F.fetch_int(9)*x + F.fetch_int(23) + // E.<e> = F.extension(E_mod) # GF(1024) extension field definition + // for p in divisors(E.order() - 1): # Verify e has order 1023. + // assert((e**p == 1) == (p % 1023 == 0)) + // G = lcm([(e**i).minpoly() for i in range(997,1000)]) + // print(G) # Print out the generator + // + // It demonstrates that g(x) is the least common multiple of the minimal polynomials + // of 3 consecutive powers (997,998,999) of a primitive element (e) of GF(1024). + // That guarantees it is, in fact, the generator of a primitive BCH code with cycle + // length 1023 and distance 4. See https://en.wikipedia.org/wiki/BCH_code for more details. + uint32_t c = 1; for (const auto v_i : v) { // We want to update `c` to correspond to a polynomial with one extra term. If the initial @@ -88,12 +108,21 @@ uint32_t PolyMod(const data& v) // Then compute c1*x^5 + c2*x^4 + c3*x^3 + c4*x^2 + c5*x + v_i: c = ((c & 0x1ffffff) << 5) ^ v_i; - // Finally, for each set bit n in c0, conditionally add {2^n}k(x): + // Finally, for each set bit n in c0, conditionally add {2^n}k(x). These constants can be + // computed using the following Sage code (continuing the code above): + // + // for i in [1,2,4,8,16]: # Print out {1,2,4,8,16}*(g(x) mod x^6), packed in hex integers. + // v = 0 + // for coef in reversed((F.fetch_int(i)*(G % x**6)).coefficients(sparse=True)): + // v = v*32 + coef.integer_representation() + // print("0x%x" % v) + // if (c0 & 1) c ^= 0x3b6a57b2; // k(x) = {29}x^5 + {22}x^4 + {20}x^3 + {21}x^2 + {29}x + {18} if (c0 & 2) c ^= 0x26508e6d; // {2}k(x) = {19}x^5 + {5}x^4 + x^3 + {3}x^2 + {19}x + {13} if (c0 & 4) c ^= 0x1ea119fa; // {4}k(x) = {15}x^5 + {10}x^4 + {2}x^3 + {6}x^2 + {15}x + {26} if (c0 & 8) c ^= 0x3d4233dd; // {8}k(x) = {30}x^5 + {20}x^4 + {4}x^3 + {12}x^2 + {30}x + {29} if (c0 & 16) c ^= 0x2a1462b3; // {16}k(x) = {21}x^5 + x^4 + {8}x^3 + {24}x^2 + {21}x + {19} + } return c; } @@ -125,7 +154,8 @@ Encoding VerifyChecksum(const std::string& hrp, const data& values) // PolyMod computes what value to xor into the final values to make the checksum 0. However, // if we required that the checksum was 0, it would be the case that appending a 0 to a valid // list of values would result in a new valid list. For that reason, Bech32 requires the - // resulting checksum to be 1 instead. In Bech32m, this constant was amended. + // resulting checksum to be 1 instead. In Bech32m, this constant was amended. See + // https://gist.github.com/sipa/14c248c288c3880a3b191f978a34508e for details. const uint32_t check = PolyMod(Cat(ExpandHRP(hrp), values)); if (check == EncodingConstant(Encoding::BECH32)) return Encoding::BECH32; if (check == EncodingConstant(Encoding::BECH32M)) return Encoding::BECH32M; diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index b7bd8a3261..d6834a239b 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -5,6 +5,7 @@ #include <addrman.h> #include <bench/bench.h> #include <random.h> +#include <util/check.h> #include <util/time.h> #include <optional> @@ -52,14 +53,14 @@ static void CreateAddresses() } } -static void AddAddressesToAddrMan(CAddrMan& addrman) +static void AddAddressesToAddrMan(AddrMan& addrman) { for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { addrman.Add(g_addresses[source_i], g_sources[source_i]); } } -static void FillAddrMan(CAddrMan& addrman) +static void FillAddrMan(AddrMan& addrman) { CreateAddresses(); @@ -72,29 +73,27 @@ static void AddrManAdd(benchmark::Bench& bench) { CreateAddresses(); - CAddrMan addrman; - bench.run([&] { + AddrMan addrman{/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0}; AddAddressesToAddrMan(addrman); - addrman.Clear(); }); } static void AddrManSelect(benchmark::Bench& bench) { - CAddrMan addrman; + AddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); FillAddrMan(addrman); bench.run([&] { const auto& address = addrman.Select(); - assert(address.GetPort() > 0); + assert(address.first.GetPort() > 0); }); } static void AddrManGetAddr(benchmark::Bench& bench) { - CAddrMan addrman; + AddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); FillAddrMan(addrman); @@ -104,38 +103,33 @@ static void AddrManGetAddr(benchmark::Bench& bench) }); } -static void AddrManGood(benchmark::Bench& bench) +static void AddrManAddThenGood(benchmark::Bench& bench) { - /* Create many CAddrMan objects - one to be modified at each loop iteration. - * This is necessary because the CAddrMan::Good() method modifies the - * object, affecting the timing of subsequent calls to the same method and - * we want to do the same amount of work in every loop iteration. */ - - bench.epochs(5).epochIterations(1); - - std::vector<CAddrMan> addrmans(bench.epochs() * bench.epochIterations()); - for (auto& addrman : addrmans) { - FillAddrMan(addrman); - } - - auto markSomeAsGood = [](CAddrMan& addrman) { + auto markSomeAsGood = [](AddrMan& addrman) { for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { for (size_t addr_i = 0; addr_i < NUM_ADDRESSES_PER_SOURCE; ++addr_i) { - if (addr_i % 32 == 0) { - addrman.Good(g_addresses[source_i][addr_i]); - } + addrman.Good(g_addresses[source_i][addr_i]); } } }; - uint64_t i = 0; + CreateAddresses(); + bench.run([&] { - markSomeAsGood(addrmans.at(i)); - ++i; + // To make the benchmark independent of the number of evaluations, we always prepare a new addrman. + // This is necessary because AddrMan::Good() method modifies the object, affecting the timing of subsequent calls + // to the same method and we want to do the same amount of work in every loop iteration. + // + // This has some overhead (exactly the result of AddrManAdd benchmark), but that overhead is constant so improvements in + // AddrMan::Good() will still be noticeable. + AddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + AddAddressesToAddrMan(addrman); + + markSomeAsGood(addrman); }); } BENCHMARK(AddrManAdd); BENCHMARK(AddrManSelect); BENCHMARK(AddrManGetAddr); -BENCHMARK(AddrManGood); +BENCHMARK(AddrManAddThenGood); diff --git a/src/bench/bech32.cpp b/src/bench/bech32.cpp index 8e10862a37..bc3685818e 100644 --- a/src/bench/bech32.cpp +++ b/src/bench/bech32.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> -#include <bench/nanobench.h> #include <bech32.h> #include <util/strencodings.h> diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 012057e792..030bc43396 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -4,11 +4,18 @@ #include <bench/bench.h> -#include <chainparams.h> #include <test/util/setup_common.h> -#include <validation.h> +#include <chrono> +#include <fstream> +#include <functional> +#include <iostream> +#include <map> #include <regex> +#include <string> +#include <vector> + +using namespace std::chrono_literals; const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; @@ -61,6 +68,12 @@ void benchmark::BenchRunner::RunAll(const Args& args) Bench bench; bench.name(p.first); + if (args.min_time > 0ms) { + // convert to nanos before dividing to reduce rounding errors + std::chrono::nanoseconds min_time_ns = args.min_time; + bench.minEpochTime(min_time_ns / bench.epochs()); + } + if (args.asymptote.empty()) { p.second(bench); } else { diff --git a/src/bench/bench.h b/src/bench/bench.h index 22f06d8cb8..b0e4006ee3 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -18,16 +18,19 @@ /* * Usage: -static void CODE_TO_TIME(benchmark::Bench& bench) +static void NameOfYourBenchmarkFunction(benchmark::Bench& bench) { - ... do any setup needed... - nanobench::Config().run([&] { - ... do stuff you want to time... + ...do any setup needed... + + bench.run([&] { + ...do stuff you want to time; refer to src/bench/nanobench.h + for more information and the options that can be passed here... }); - ... do any cleanup needed... + + ...do any cleanup needed... } -BENCHMARK(CODE_TO_TIME); +BENCHMARK(NameOfYourBenchmarkFunction); */ @@ -38,11 +41,12 @@ using ankerl::nanobench::Bench; typedef std::function<void(Bench&)> BenchFunction; struct Args { - std::string regex_filter; bool is_list_only; + std::chrono::milliseconds min_time; std::vector<double> asymptote; std::string output_csv; std::string output_json; + std::string regex_filter; }; class BenchRunner @@ -55,7 +59,8 @@ public: static void RunAll(const Args& args); }; -} +} // namespace benchmark + // BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo); #define BENCHMARK(n) \ benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n); diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 135659f87f..0b43ea1fd5 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -4,23 +4,30 @@ #include <bench/bench.h> +#include <clientversion.h> #include <crypto/sha256.h> #include <util/strencodings.h> #include <util/system.h> -#include <memory> +#include <chrono> +#include <cstdint> +#include <iostream> +#include <sstream> +#include <vector> static const char* DEFAULT_BENCH_FILTER = ".*"; +static constexpr int64_t DEFAULT_MIN_TIME_MS{10}; static void SetupBenchArgs(ArgsManager& argsman) { SetupHelpOptions(argsman); - argsman.AddArg("-list", "List benchmarks without executing them", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-asymptote=<n1,n2,n3,...>", "Test asymptotic growth of the runtime of an algorithm, if supported by the benchmark", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-filter=<regex>", strprintf("Regular expression filter to select benchmark by name (default: %s)", DEFAULT_BENCH_FILTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-asymptote=n1,n2,n3,...", strprintf("Test asymptotic growth of the runtime of an algorithm, if supported by the benchmark"), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-output_csv=<output.csv>", "Generate CSV file with the most important benchmark results.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-output_json=<output.json>", "Generate JSON file with all benchmark results.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-list", "List benchmarks without executing them", ArgsManager::ALLOW_BOOL, OptionsCategory::OPTIONS); + argsman.AddArg("-min_time=<milliseconds>", strprintf("Minimum runtime per benchmark, in milliseconds (default: %d)", DEFAULT_MIN_TIME_MS), ArgsManager::ALLOW_INT, OptionsCategory::OPTIONS); + argsman.AddArg("-output_csv=<output.csv>", "Generate CSV file with the most important benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-output_json=<output.json>", "Generate JSON file with all benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); } // parses a comma separated list like "10,20,30,50" @@ -48,17 +55,62 @@ int main(int argc, char** argv) } if (HelpRequested(argsman)) { - std::cout << argsman.GetHelpMessage(); + std::cout << "Usage: bench_bitcoin [options]\n" + "\n" + << argsman.GetHelpMessage() + << "Description:\n" + "\n" + " bench_bitcoin executes microbenchmarks. The quality of the benchmark results\n" + " highly depend on the stability of the machine. It can sometimes be difficult\n" + " to get stable, repeatable results, so here are a few tips:\n" + "\n" + " * Use pyperf [1] to disable frequency scaling, turbo boost etc. For best\n" + " results, use CPU pinning and CPU isolation (see [2]).\n" + "\n" + " * Each call of run() should do exactly the same work. E.g. inserting into\n" + " a std::vector doesn't do that as it will reallocate on certain calls. Make\n" + " sure each run has exactly the same preconditions.\n" + "\n" + " * If results are still not reliable, increase runtime with e.g.\n" + " -min_time=5000 to let a benchmark run for at least 5 seconds.\n" + "\n" + " * bench_bitcoin uses nanobench [3] for which there is extensive\n" + " documentation available online.\n" + "\n" + "Environment Variables:\n" + "\n" + " To attach a profiler you can run a benchmark in endless mode. This can be\n" + " done with the environment variable NANOBENCH_ENDLESS. E.g. like so:\n" + "\n" + " NANOBENCH_ENDLESS=MuHash ./bench_bitcoin -filter=MuHash\n" + "\n" + " In rare cases it can be useful to suppress stability warnings. This can be\n" + " done with the environment variable NANOBENCH_SUPPRESS_WARNINGS, e.g:\n" + "\n" + " NANOBENCH_SUPPRESS_WARNINGS=1 ./bench_bitcoin\n" + "\n" + "Notes:\n" + "\n" + " 1. pyperf\n" + " https://github.com/psf/pyperf\n" + "\n" + " 2. CPU pinning & isolation\n" + " https://pyperf.readthedocs.io/en/latest/system.html\n" + "\n" + " 3. nanobench\n" + " https://github.com/martinus/nanobench\n" + "\n"; return EXIT_SUCCESS; } benchmark::Args args; - args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER); - args.is_list_only = argsman.GetBoolArg("-list", false); args.asymptote = parseAsymptote(argsman.GetArg("-asymptote", "")); + args.is_list_only = argsman.GetBoolArg("-list", false); + args.min_time = std::chrono::milliseconds(argsman.GetIntArg("-min_time", DEFAULT_MIN_TIME_MS)); args.output_csv = argsman.GetArg("-output_csv", ""); args.output_json = argsman.GetArg("-output_json", ""); + args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER); benchmark::BenchRunner::RunAll(args); diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 5beb833b48..934b574f8b 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -6,6 +6,7 @@ #include <interfaces/chain.h> #include <node/context.h> #include <wallet/coinselection.h> +#include <wallet/spend.h> #include <wallet/wallet.h> #include <set> @@ -17,7 +18,7 @@ static void addCoin(const CAmount& nValue, const CWallet& wallet, std::vector<st tx.nLockTime = nextLockTime++; // so all transactions get different hashes tx.vout.resize(1); tx.vout[0].nValue = nValue; - wtxs.push_back(std::make_unique<CWalletTx>(&wallet, MakeTransactionRef(std::move(tx)))); + wtxs.push_back(std::make_unique<CWalletTx>(MakeTransactionRef(std::move(tx)))); } // Simple benchmark for wallet coin selection. Note that it maybe be necessary @@ -45,18 +46,18 @@ static void CoinSelection(benchmark::Bench& bench) // Create coins std::vector<COutput> coins; for (const auto& wtx : wtxs) { - coins.emplace_back(wtx.get(), 0 /* iIn */, 6 * 24 /* nDepthIn */, true /* spendable */, true /* solvable */, true /* safe */); + coins.emplace_back(wallet, *wtx, 0 /* iIn */, 6 * 24 /* nDepthIn */, true /* spendable */, true /* solvable */, true /* safe */); } const CoinEligibilityFilter filter_standard(1, 6, 0); const CoinSelectionParams coin_selection_params(/* change_output_size= */ 34, /* change_spend_size= */ 148, /* effective_feerate= */ CFeeRate(0), /* long_term_feerate= */ CFeeRate(0), /* discard_feerate= */ CFeeRate(0), - /* tx_no_inputs_size= */ 0, /* avoid_partial= */ false); + /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); bench.run([&] { std::set<CInputCoin> setCoinsRet; CAmount nValueRet; - bool success = wallet.AttemptSelection(1003 * COIN, filter_standard, coins, setCoinsRet, nValueRet, coin_selection_params); + bool success = AttemptSelection(wallet, 1003 * COIN, filter_standard, coins, setCoinsRet, nValueRet, coin_selection_params); assert(success); assert(nValueRet == 1003 * COIN); assert(setCoinsRet.size() == 2); @@ -75,9 +76,9 @@ static void add_coin(const CAmount& nValue, int nInput, std::vector<OutputGroup> CMutableTransaction tx; tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; - std::unique_ptr<CWalletTx> wtx = std::make_unique<CWalletTx>(&testWallet, MakeTransactionRef(std::move(tx))); + std::unique_ptr<CWalletTx> wtx = std::make_unique<CWalletTx>(MakeTransactionRef(std::move(tx))); set.emplace_back(); - set.back().Insert(COutput(wtx.get(), nInput, 0, true, true, true).GetInputCoin(), 0, true, 0, 0, false); + set.back().Insert(COutput(testWallet, *wtx, nInput, 0, true, true, true).GetInputCoin(), 0, true, 0, 0, false); wtxn.emplace_back(std::move(wtx)); } // Copied from src/wallet/test/coinselector_tests.cpp diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index 30fe11be6b..d36e504bfc 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -110,9 +110,9 @@ static void MuHash(benchmark::Bench& bench) { MuHash3072 acc; unsigned char key[32] = {0}; - int i = 0; + uint32_t i = 0; bench.run([&] { - key[0] = ++i; + key[0] = ++i & 0xFF; acc *= MuHash3072(key); }); } @@ -134,10 +134,6 @@ static void MuHashDiv(benchmark::Bench& bench) FastRandomContext rng(true); MuHash3072 muhash{rng.randbytes(32)}; - for (size_t i = 0; i < bench.epochIterations(); ++i) { - acc *= muhash; - } - bench.run([&] { acc /= muhash; }); diff --git a/src/bench/nanobench.h b/src/bench/nanobench.h index 030d6ebf6a..27df08fb69 100644 --- a/src/bench/nanobench.h +++ b/src/bench/nanobench.h @@ -33,7 +33,7 @@ // see https://semver.org/ #define ANKERL_NANOBENCH_VERSION_MAJOR 4 // incompatible API changes #define ANKERL_NANOBENCH_VERSION_MINOR 3 // backwards-compatible changes -#define ANKERL_NANOBENCH_VERSION_PATCH 4 // backwards-compatible bug fixes +#define ANKERL_NANOBENCH_VERSION_PATCH 6 // backwards-compatible bug fixes /////////////////////////////////////////////////////////////////////////////////////////////////// // public facing api - as minimal as possible @@ -88,13 +88,15 @@ } while (0) #endif -#if defined(__linux__) && defined(PERF_EVENT_IOC_ID) && defined(PERF_COUNT_HW_REF_CPU_CYCLES) && defined(PERF_FLAG_FD_CLOEXEC) && \ - !defined(ANKERL_NANOBENCH_DISABLE_PERF_COUNTERS) -// only enable perf counters on kernel 3.14 which seems to have all the necessary defines. The three PERF_... defines are not in -// kernel 2.6.32 (all others are). -# define ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS() 1 -#else -# define ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS() 0 +#define ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS() 0 +#if defined(__linux__) && !defined(ANKERL_NANOBENCH_DISABLE_PERF_COUNTERS) +# include <linux/version.h> +# if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 14, 0) +// PERF_COUNT_HW_REF_CPU_CYCLES only available since kernel 3.3 +// PERF_FLAG_FD_CLOEXEC since kernel 3.14 +# undef ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS +# define ANKERL_NANOBENCH_PRIVATE_PERF_COUNTERS() 1 +# endif #endif #if defined(__clang__) @@ -2210,20 +2212,20 @@ struct IterationLogic::Impl { columns.emplace_back(10, 1, "err%", "%", rErrorMedian * 100.0); double rInsMedian = -1.0; - if (mResult.has(Result::Measure::instructions)) { + if (mBench.performanceCounters() && mResult.has(Result::Measure::instructions)) { rInsMedian = mResult.median(Result::Measure::instructions); columns.emplace_back(18, 2, "ins/" + mBench.unit(), "", rInsMedian / mBench.batch()); } double rCycMedian = -1.0; - if (mResult.has(Result::Measure::cpucycles)) { + if (mBench.performanceCounters() && mResult.has(Result::Measure::cpucycles)) { rCycMedian = mResult.median(Result::Measure::cpucycles); columns.emplace_back(18, 2, "cyc/" + mBench.unit(), "", rCycMedian / mBench.batch()); } if (rInsMedian > 0.0 && rCycMedian > 0.0) { columns.emplace_back(9, 3, "IPC", "", rCycMedian <= 0.0 ? 0.0 : rInsMedian / rCycMedian); } - if (mResult.has(Result::Measure::branchinstructions)) { + if (mBench.performanceCounters() && mResult.has(Result::Measure::branchinstructions)) { double rBraMedian = mResult.median(Result::Measure::branchinstructions); columns.emplace_back(17, 2, "bra/" + mBench.unit(), "", rBraMedian / mBench.batch()); if (mResult.has(Result::Measure::branchmisses)) { @@ -2402,6 +2404,14 @@ public: return (a + divisor / 2) / divisor; } + ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") + static inline uint32_t mix(uint32_t x) noexcept { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + return x; + } + template <typename Op> ANKERL_NANOBENCH_NO_SANITIZE("integer", "undefined") void calibrate(Op&& op) { @@ -2441,15 +2451,10 @@ public: uint64_t const numIters = 100000U + (std::random_device{}() & 3); uint64_t n = numIters; uint32_t x = 1234567; - auto fn = [&]() { - x ^= x << 13; - x ^= x >> 17; - x ^= x << 5; - }; beginMeasure(); while (n-- > 0) { - fn(); + x = mix(x); } endMeasure(); detail::doNotOptimizeAway(x); @@ -2459,8 +2464,8 @@ public: beginMeasure(); while (n-- > 0) { // we now run *twice* so we can easily calculate the overhead - fn(); - fn(); + x = mix(x); + x = mix(x); } endMeasure(); detail::doNotOptimizeAway(x); diff --git a/src/bench/peer_eviction.cpp b/src/bench/peer_eviction.cpp new file mode 100644 index 0000000000..8429f18613 --- /dev/null +++ b/src/bench/peer_eviction.cpp @@ -0,0 +1,155 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <net.h> +#include <netaddress.h> +#include <random.h> +#include <test/util/net.h> +#include <test/util/setup_common.h> + +#include <algorithm> +#include <functional> +#include <vector> + +static void EvictionProtectionCommon( + benchmark::Bench& bench, + int num_candidates, + std::function<void(NodeEvictionCandidate&)> candidate_setup_fn) +{ + using Candidates = std::vector<NodeEvictionCandidate>; + FastRandomContext random_context{true}; + + Candidates candidates{GetRandomNodeEvictionCandidates(num_candidates, random_context)}; + for (auto& c : candidates) { + candidate_setup_fn(c); + } + + + bench.run([&] { + // creating a copy has an overhead of about 3%, so it does not influence the benchmark results much. + auto copy = candidates; + ProtectEvictionCandidatesByRatio(copy); + }); +} + +/* Benchmarks */ + +static void EvictionProtection0Networks250Candidates(benchmark::Bench& bench) +{ + EvictionProtectionCommon( + bench, + 250 /* num_candidates */, + [](NodeEvictionCandidate& c) { + c.nTimeConnected = c.id; + c.m_network = NET_IPV4; + }); +} + +static void EvictionProtection1Networks250Candidates(benchmark::Bench& bench) +{ + EvictionProtectionCommon( + bench, + 250 /* num_candidates */, + [](NodeEvictionCandidate& c) { + c.nTimeConnected = c.id; + c.m_is_local = false; + if (c.id >= 130 && c.id < 240) { // 110 Tor + c.m_network = NET_ONION; + } else { + c.m_network = NET_IPV4; + } + }); +} + +static void EvictionProtection2Networks250Candidates(benchmark::Bench& bench) +{ + EvictionProtectionCommon( + bench, + 250 /* num_candidates */, + [](NodeEvictionCandidate& c) { + c.nTimeConnected = c.id; + c.m_is_local = false; + if (c.id >= 90 && c.id < 160) { // 70 Tor + c.m_network = NET_ONION; + } else if (c.id >= 170 && c.id < 250) { // 80 I2P + c.m_network = NET_I2P; + } else { + c.m_network = NET_IPV4; + } + }); +} + +static void EvictionProtection3Networks050Candidates(benchmark::Bench& bench) +{ + EvictionProtectionCommon( + bench, + 50 /* num_candidates */, + [](NodeEvictionCandidate& c) { + c.nTimeConnected = c.id; + c.m_is_local = (c.id == 28 || c.id == 47); // 2 localhost + if (c.id >= 30 && c.id < 47) { // 17 I2P + c.m_network = NET_I2P; + } else if (c.id >= 24 && c.id < 28) { // 4 Tor + c.m_network = NET_ONION; + } else { + c.m_network = NET_IPV4; + } + }); +} + +static void EvictionProtection3Networks100Candidates(benchmark::Bench& bench) +{ + EvictionProtectionCommon( + bench, + 100 /* num_candidates */, + [](NodeEvictionCandidate& c) { + c.nTimeConnected = c.id; + c.m_is_local = (c.id >= 55 && c.id < 60); // 5 localhost + if (c.id >= 70 && c.id < 80) { // 10 I2P + c.m_network = NET_I2P; + } else if (c.id >= 80 && c.id < 96) { // 16 Tor + c.m_network = NET_ONION; + } else { + c.m_network = NET_IPV4; + } + }); +} + +static void EvictionProtection3Networks250Candidates(benchmark::Bench& bench) +{ + EvictionProtectionCommon( + bench, + 250 /* num_candidates */, + [](NodeEvictionCandidate& c) { + c.nTimeConnected = c.id; + c.m_is_local = (c.id >= 140 && c.id < 160); // 20 localhost + if (c.id >= 170 && c.id < 180) { // 10 I2P + c.m_network = NET_I2P; + } else if (c.id >= 190 && c.id < 240) { // 50 Tor + c.m_network = NET_ONION; + } else { + c.m_network = NET_IPV4; + } + }); +} + +// Candidate numbers used for the benchmarks: +// - 50 candidates simulates a possible use of -maxconnections +// - 100 candidates approximates an average node with default settings +// - 250 candidates is the number of peers reported by operators of busy nodes + +// No disadvantaged networks, with 250 eviction candidates. +BENCHMARK(EvictionProtection0Networks250Candidates); + +// 1 disadvantaged network (Tor) with 250 eviction candidates. +BENCHMARK(EvictionProtection1Networks250Candidates); + +// 2 disadvantaged networks (I2P, Tor) with 250 eviction candidates. +BENCHMARK(EvictionProtection2Networks250Candidates); + +// 3 disadvantaged networks (I2P/localhost/Tor) with 50/100/250 eviction candidates. +BENCHMARK(EvictionProtection3Networks050Candidates); +BENCHMARK(EvictionProtection3Networks100Candidates); +BENCHMARK(EvictionProtection3Networks250Candidates); diff --git a/src/bench/rollingbloom.cpp b/src/bench/rollingbloom.cpp index 997ab56549..28167767db 100644 --- a/src/bench/rollingbloom.cpp +++ b/src/bench/rollingbloom.cpp @@ -13,16 +13,16 @@ static void RollingBloom(benchmark::Bench& bench) uint32_t count = 0; bench.run([&] { count++; - data[0] = count; - data[1] = count >> 8; - data[2] = count >> 16; - data[3] = count >> 24; + data[0] = count & 0xFF; + data[1] = (count >> 8) & 0xFF; + data[2] = (count >> 16) & 0xFF; + data[3] = (count >> 24) & 0xFF; filter.insert(data); - data[0] = count >> 24; - data[1] = count >> 16; - data[2] = count >> 8; - data[3] = count; + data[0] = (count >> 24) & 0xFF; + data[1] = (count >> 16) & 0xFF; + data[2] = (count >> 8) & 0xFF; + data[3] = count & 0xFF; filter.contains(data); }); } diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index 39e74b9b2b..928aa7573c 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -21,7 +21,7 @@ static void VerifyScriptBench(benchmark::Bench& bench) const ECCVerifyHandle verify_handle; ECC_Start(); - const int flags = SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH; + const uint32_t flags{SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH}; const int witnessversion = 0; // Key pair. diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index 362b7c1e15..a205d8b6e7 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -9,6 +9,7 @@ #include <test/util/setup_common.h> #include <test/util/wallet.h> #include <validationinterface.h> +#include <wallet/receive.h> #include <wallet/wallet.h> #include <optional> @@ -35,11 +36,11 @@ static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const b } SyncWithValidationInterfaceQueue(); - auto bal = wallet.GetBalance(); // Cache + auto bal = GetBalance(wallet); // Cache bench.run([&] { if (set_dirty) wallet.MarkDirty(); - bal = wallet.GetBalance(); + bal = GetBalance(wallet); if (add_mine) assert(bal.m_mine_trusted > 0); if (add_watchonly) assert(bal.m_watchonly_trusted > 0); }); diff --git a/src/bitcoin-cli-res.rc b/src/bitcoin-cli-res.rc index 405a302261..d9e5dcf7fd 100644 --- a/src/bitcoin-cli-res.rc +++ b/src/bitcoin-cli-res.rc @@ -2,9 +2,7 @@ #include "clientversion.h" // holds the needed client version information #define VER_PRODUCTVERSION CLIENT_VERSION_MAJOR,CLIENT_VERSION_MINOR,CLIENT_VERSION_BUILD -#define VER_PRODUCTVERSION_STR STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) "." STRINGIZE(CLIENT_VERSION_BUILD) #define VER_FILEVERSION VER_PRODUCTVERSION -#define VER_FILEVERSION_STR VER_PRODUCTVERSION_STR VS_VERSION_INFO VERSIONINFO FILEVERSION VER_FILEVERSION @@ -18,13 +16,13 @@ BEGIN BEGIN VALUE "CompanyName", "Bitcoin" VALUE "FileDescription", "bitcoin-cli (JSON-RPC client for " PACKAGE_NAME ")" - VALUE "FileVersion", VER_FILEVERSION_STR + VALUE "FileVersion", PACKAGE_VERSION VALUE "InternalName", "bitcoin-cli" VALUE "LegalCopyright", COPYRIGHT_STR VALUE "LegalTrademarks1", "Distributed under the MIT software license, see the accompanying file COPYING or http://www.opensource.org/licenses/mit-license.php." VALUE "OriginalFilename", "bitcoin-cli.exe" VALUE "ProductName", "bitcoin-cli" - VALUE "ProductVersion", VER_PRODUCTVERSION_STR + VALUE "ProductVersion", PACKAGE_VERSION END END diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 7a5f945511..3c22ee0f67 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -9,6 +9,7 @@ #include <chainparamsbase.h> #include <clientversion.h> +#include <policy/feerate.h> #include <rpc/client.h> #include <rpc/mining.h> #include <rpc/protocol.h> @@ -28,6 +29,10 @@ #include <string> #include <tuple> +#ifndef WIN32 +#include <unistd.h> +#endif + #include <event2/buffer.h> #include <event2/keyvalq_struct.h> #include <support/events.h> @@ -48,6 +53,9 @@ static constexpr int8_t UNKNOWN_NETWORK{-1}; /** Default number of blocks to generate for RPC generatetoaddress. */ static const std::string DEFAULT_NBLOCKS = "1"; +/** Default -color setting. */ +static const std::string DEFAULT_COLOR_SETTING{"auto"}; + static void SetupCliArgs(ArgsManager& argsman) { SetupHelpOptions(argsman); @@ -66,6 +74,7 @@ static void SetupCliArgs(ArgsManager& argsman) argsman.AddArg("-netinfo", "Get network peer connection information from the remote server. An optional integer argument from 0 to 4 can be passed for different peers listings (default: 0). Pass \"help\" for detailed help documentation.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); SetupChainParamsBaseOptions(argsman); + argsman.AddArg("-color=<when>", strprintf("Color setting for CLI output (default: %s). Valid values: always, auto (add color codes when standard output is connected to a terminal and OS is not WIN32), never.", DEFAULT_COLOR_SETTING), ArgsManager::ALLOW_STRING, OptionsCategory::OPTIONS); argsman.AddArg("-named", strprintf("Pass named instead of positional arguments (default: %s)", DEFAULT_NAMED), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-rpcclienttimeout=<n>", strprintf("Timeout in seconds during HTTP requests, or 0 for no timeout. (default: %d)", DEFAULT_HTTP_CLIENT_TIMEOUT), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-rpcconnect=<ip>", strprintf("Send commands to node running on <ip> (default: %s)", DEFAULT_RPCCONNECT), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -84,9 +93,6 @@ static void SetupCliArgs(ArgsManager& argsman) /** libevent event log callback */ static void libevent_log_cb(int severity, const char *msg) { -#ifndef EVENT_LOG_ERR // EVENT_LOG_ERR was added in 2.0.19; but before then _EVENT_LOG_ERR existed. -# define EVENT_LOG_ERR _EVENT_LOG_ERR -#endif // Ignore everything other than errors if (severity >= EVENT_LOG_ERR) { throw std::runtime_error(strprintf("libevent error: %s", msg)); @@ -236,7 +242,7 @@ public: class AddrinfoRequestHandler : public BaseRequestHandler { private: - static constexpr std::array m_networks{"ipv4", "ipv6", "torv2", "torv3", "i2p"}; + static constexpr std::array m_networks{"ipv4", "ipv6", "onion", "i2p"}; int8_t NetworkStringToId(const std::string& str) const { for (size_t i = 0; i < m_networks.size(); ++i) { @@ -262,13 +268,10 @@ public: if (!nodes.empty() && nodes.at(0)["network"].isNull()) { throw std::runtime_error("-addrinfo requires bitcoind server to be running v22.0 and up"); } - // Count the number of peers we know by network, including torv2 versus torv3. + // Count the number of peers known to our node, by network. std::array<uint64_t, m_networks.size()> counts{{}}; for (const UniValue& node : nodes) { std::string network_name{node["network"].get_str()}; - if (network_name == "onion") { - network_name = node["address"].get_str().size() > 22 ? "torv3" : "torv2"; - } const int8_t network_id{NetworkStringToId(network_name)}; if (network_id == UNKNOWN_NETWORK) continue; ++counts.at(network_id); @@ -334,11 +337,13 @@ public: connections.pushKV("total", batch[ID_NETWORKINFO]["result"]["connections"]); result.pushKV("connections", connections); - result.pushKV("proxy", batch[ID_NETWORKINFO]["result"]["networks"][0]["proxy"]); + result.pushKV("networks", batch[ID_NETWORKINFO]["result"]["networks"]); result.pushKV("difficulty", batch[ID_BLOCKCHAININFO]["result"]["difficulty"]); result.pushKV("chain", UniValue(batch[ID_BLOCKCHAININFO]["result"]["chain"])); if (!batch[ID_WALLETINFO]["result"].isNull()) { + result.pushKV("has_wallet", true); result.pushKV("keypoolsize", batch[ID_WALLETINFO]["result"]["keypoolsize"]); + result.pushKV("walletname", batch[ID_WALLETINFO]["result"]["walletname"]); if (!batch[ID_WALLETINFO]["result"]["unlocked_until"].isNull()) { result.pushKV("unlocked_until", batch[ID_WALLETINFO]["result"]["unlocked_until"]); } @@ -375,7 +380,9 @@ private: bool IsVersionSelected() const { return m_details_level == 3 || m_details_level == 4; } bool m_is_asmap_on{false}; size_t m_max_addr_length{0}; - size_t m_max_age_length{3}; + size_t m_max_addr_processed_length{5}; + size_t m_max_addr_rate_limited_length{6}; + size_t m_max_age_length{5}; size_t m_max_id_length{2}; struct Peer { std::string addr; @@ -385,6 +392,8 @@ private: std::string age; double min_ping; double ping; + int64_t addr_processed; + int64_t addr_rate_limited; int64_t last_blck; int64_t last_recv; int64_t last_send; @@ -392,6 +401,7 @@ private: int id; int mapped_as; int version; + bool is_addr_relay_enabled; bool is_bip152_hb_from; bool is_bip152_hb_to; bool is_block_relay; @@ -472,6 +482,8 @@ public: const int peer_id{peer["id"].get_int()}; const int mapped_as{peer["mapped_as"].isNull() ? 0 : peer["mapped_as"].get_int()}; const int version{peer["version"].get_int()}; + const int64_t addr_processed{peer["addr_processed"].isNull() ? 0 : peer["addr_processed"].get_int64()}; + const int64_t addr_rate_limited{peer["addr_rate_limited"].isNull() ? 0 : peer["addr_rate_limited"].get_int64()}; const int64_t conn_time{peer["conntime"].get_int64()}; const int64_t last_blck{peer["last_block"].get_int64()}; const int64_t last_recv{peer["lastrecv"].get_int64()}; @@ -482,10 +494,13 @@ public: const std::string addr{peer["addr"].get_str()}; const std::string age{conn_time == 0 ? "" : ToString((m_time_now - conn_time) / 60)}; const std::string sub_version{peer["subver"].get_str()}; + const bool is_addr_relay_enabled{peer["addr_relay_enabled"].isNull() ? false : peer["addr_relay_enabled"].get_bool()}; const bool is_bip152_hb_from{peer["bip152_hb_from"].get_bool()}; const bool is_bip152_hb_to{peer["bip152_hb_to"].get_bool()}; - m_peers.push_back({addr, sub_version, conn_type, network, age, min_ping, ping, last_blck, last_recv, last_send, last_trxn, peer_id, mapped_as, version, is_bip152_hb_from, is_bip152_hb_to, is_block_relay, is_outbound}); + m_peers.push_back({addr, sub_version, conn_type, network, age, min_ping, ping, addr_processed, addr_rate_limited, last_blck, last_recv, last_send, last_trxn, peer_id, mapped_as, version, is_addr_relay_enabled, is_bip152_hb_from, is_bip152_hb_to, is_block_relay, is_outbound}); m_max_addr_length = std::max(addr.length() + 1, m_max_addr_length); + m_max_addr_processed_length = std::max(ToString(addr_processed).length(), m_max_addr_processed_length); + m_max_addr_rate_limited_length = std::max(ToString(addr_rate_limited).length(), m_max_addr_rate_limited_length); m_max_age_length = std::max(age.length(), m_max_age_length); m_max_id_length = std::max(ToString(peer_id).length(), m_max_id_length); m_is_asmap_on |= (mapped_as != 0); @@ -493,39 +508,46 @@ public: } // Generate report header. - std::string result{strprintf("%s %s%s - %i%s\n\n", PACKAGE_NAME, FormatFullVersion(), ChainToString(), networkinfo["protocolversion"].get_int(), networkinfo["subversion"].get_str())}; + std::string result{strprintf("%s client %s%s - server %i%s\n\n", PACKAGE_NAME, FormatFullVersion(), ChainToString(), networkinfo["protocolversion"].get_int(), networkinfo["subversion"].get_str())}; // Report detailed peer connections list sorted by direction and minimum ping time. if (DetailsRequested() && !m_peers.empty()) { std::sort(m_peers.begin(), m_peers.end()); - result += strprintf("<-> type net mping ping send recv txn blk hb %*s ", m_max_age_length, "age"); + result += strprintf("<-> type net mping ping send recv txn blk hb %*s%*s%*s ", + m_max_addr_processed_length, "addrp", + m_max_addr_rate_limited_length, "addrl", + m_max_age_length, "age"); if (m_is_asmap_on) result += " asmap "; result += strprintf("%*s %-*s%s\n", m_max_id_length, "id", IsAddressSelected() ? m_max_addr_length : 0, IsAddressSelected() ? "address" : "", IsVersionSelected() ? "version" : ""); for (const Peer& peer : m_peers) { std::string version{ToString(peer.version) + peer.sub_version}; result += strprintf( - "%3s %6s %5s%7s%7s%5s%5s%5s%5s %2s %*s%*i %*s %-*s%s\n", + "%3s %6s %5s%7s%7s%5s%5s%5s%5s %2s %*s%*s%*s%*i %*s %-*s%s\n", peer.is_outbound ? "out" : "in", ConnectionTypeForNetinfo(peer.conn_type), peer.network, PingTimeToString(peer.min_ping), PingTimeToString(peer.ping), - peer.last_send == 0 ? "" : ToString(m_time_now - peer.last_send), - peer.last_recv == 0 ? "" : ToString(m_time_now - peer.last_recv), - peer.last_trxn == 0 ? "" : ToString((m_time_now - peer.last_trxn) / 60), - peer.last_blck == 0 ? "" : ToString((m_time_now - peer.last_blck) / 60), + peer.last_send ? ToString(m_time_now - peer.last_send) : "", + peer.last_recv ? ToString(m_time_now - peer.last_recv) : "", + peer.last_trxn ? ToString((m_time_now - peer.last_trxn) / 60) : peer.is_block_relay ? "*" : "", + peer.last_blck ? ToString((m_time_now - peer.last_blck) / 60) : "", strprintf("%s%s", peer.is_bip152_hb_to ? "." : " ", peer.is_bip152_hb_from ? "*" : " "), + m_max_addr_processed_length, // variable spacing + peer.addr_processed ? ToString(peer.addr_processed) : peer.is_addr_relay_enabled ? "" : ".", + m_max_addr_rate_limited_length, // variable spacing + peer.addr_rate_limited ? ToString(peer.addr_rate_limited) : "", m_max_age_length, // variable spacing peer.age, m_is_asmap_on ? 7 : 0, // variable spacing - m_is_asmap_on && peer.mapped_as != 0 ? ToString(peer.mapped_as) : "", + m_is_asmap_on && peer.mapped_as ? ToString(peer.mapped_as) : "", m_max_id_length, // variable spacing peer.id, IsAddressSelected() ? m_max_addr_length : 0, // variable spacing IsAddressSelected() ? peer.addr : "", IsVersionSelected() && version != "0" ? version : ""); } - result += strprintf(" ms ms sec sec min min %*s\n\n", m_max_age_length, "min"); + result += strprintf(" ms ms sec sec min min %*s\n\n", m_max_age_length, "min"); } // Report peer connection totals by type. @@ -599,10 +621,14 @@ public: " send Time since last message sent to the peer, in seconds\n" " recv Time since last message received from the peer, in seconds\n" " txn Time since last novel transaction received from the peer and accepted into our mempool, in minutes\n" + " \"*\" - the peer requested we not relay transactions to it (relaytxes is false)\n" " blk Time since last novel block passing initial validity checks received from the peer, in minutes\n" " hb High-bandwidth BIP152 compact block relay\n" " \".\" (to) - we selected the peer as a high-bandwidth peer\n" " \"*\" (from) - the peer selected us as a high-bandwidth peer\n" + " addrp Total number of addresses processed, excluding those dropped due to rate limiting\n" + " \".\" - we do not relay addresses to this peer (addr_relay_enabled is false)\n" + " addrl Total number of addresses dropped due to rate limiting\n" " age Duration of connection to the peer, in minutes\n" " asmap Mapped AS (Autonomous System) number in the BGP route to the peer, used for diversifying\n" " peer selection (only displayed if the -asmap config option is set)\n" @@ -676,7 +702,7 @@ static UniValue CallRPC(BaseRequestHandler* rh, const std::string& strMethod, co // 3. default port for chain uint16_t port{BaseParams().RPCPort()}; SplitHostPort(gArgs.GetArg("-rpcconnect", DEFAULT_RPCCONNECT), port, host); - port = static_cast<uint16_t>(gArgs.GetArg("-rpcport", port)); + port = static_cast<uint16_t>(gArgs.GetIntArg("-rpcport", port)); // Obtain event base raii_event_base base = obtain_event_base(); @@ -686,7 +712,7 @@ static UniValue CallRPC(BaseRequestHandler* rh, const std::string& strMethod, co // Set connection timeout { - const int timeout = gArgs.GetArg("-rpcclienttimeout", DEFAULT_HTTP_CLIENT_TIMEOUT); + const int timeout = gArgs.GetIntArg("-rpcclienttimeout", DEFAULT_HTTP_CLIENT_TIMEOUT); if (timeout > 0) { evhttp_connection_set_timeout(evcon.get(), timeout); } else { @@ -796,7 +822,7 @@ static UniValue ConnectAndCallRPC(BaseRequestHandler* rh, const std::string& str UniValue response(UniValue::VOBJ); // Execute and handle connection failures with -rpcwait. const bool fWait = gArgs.GetBoolArg("-rpcwait", false); - const int timeout = gArgs.GetArg("-rpcwaittimeout", DEFAULT_WAIT_CLIENT_TIMEOUT); + const int timeout = gArgs.GetIntArg("-rpcwaittimeout", DEFAULT_WAIT_CLIENT_TIMEOUT); const auto deadline{GetTime<std::chrono::microseconds>() + 1s * timeout}; do { @@ -874,6 +900,151 @@ static void GetWalletBalances(UniValue& result) } /** + * GetProgressBar constructs a progress bar with 5% intervals. + * + * @param[in] progress The proportion of the progress bar to be filled between 0 and 1. + * @param[out] progress_bar String representation of the progress bar. + */ +static void GetProgressBar(double progress, std::string& progress_bar) +{ + if (progress < 0 || progress > 1) return; + + static constexpr double INCREMENT{0.05}; + static const std::string COMPLETE_BAR{"\u2592"}; + static const std::string INCOMPLETE_BAR{"\u2591"}; + + for (int i = 0; i < progress / INCREMENT; ++i) { + progress_bar += COMPLETE_BAR; + } + + for (int i = 0; i < (1 - progress) / INCREMENT; ++i) { + progress_bar += INCOMPLETE_BAR; + } +} + +/** + * ParseGetInfoResult takes in -getinfo result in UniValue object and parses it + * into a user friendly UniValue string to be printed on the console. + * @param[out] result Reference to UniValue result containing the -getinfo output. + */ +static void ParseGetInfoResult(UniValue& result) +{ + if (!find_value(result, "error").isNull()) return; + + std::string RESET, GREEN, BLUE, YELLOW, MAGENTA, CYAN; + bool should_colorize = false; + +#ifndef WIN32 + if (isatty(fileno(stdout))) { + // By default, only print colored text if OS is not WIN32 and stdout is connected to a terminal. + should_colorize = true; + } +#endif + + if (gArgs.IsArgSet("-color")) { + const std::string color{gArgs.GetArg("-color", DEFAULT_COLOR_SETTING)}; + if (color == "always") { + should_colorize = true; + } else if (color == "never") { + should_colorize = false; + } else if (color != "auto") { + throw std::runtime_error("Invalid value for -color option. Valid values: always, auto, never."); + } + } + + if (should_colorize) { + RESET = "\x1B[0m"; + GREEN = "\x1B[32m"; + BLUE = "\x1B[34m"; + YELLOW = "\x1B[33m"; + MAGENTA = "\x1B[35m"; + CYAN = "\x1B[36m"; + } + + std::string result_string = strprintf("%sChain: %s%s\n", BLUE, result["chain"].getValStr(), RESET); + result_string += strprintf("Blocks: %s\n", result["blocks"].getValStr()); + result_string += strprintf("Headers: %s\n", result["headers"].getValStr()); + + const double ibd_progress{result["verificationprogress"].get_real()}; + std::string ibd_progress_bar; + // Display the progress bar only if IBD progress is less than 99% + if (ibd_progress < 0.99) { + GetProgressBar(ibd_progress, ibd_progress_bar); + // Add padding between progress bar and IBD progress + ibd_progress_bar += " "; + } + + result_string += strprintf("Verification progress: %s%.4f%%\n", ibd_progress_bar, ibd_progress * 100); + result_string += strprintf("Difficulty: %s\n\n", result["difficulty"].getValStr()); + + result_string += strprintf( + "%sNetwork: in %s, out %s, total %s%s\n", + GREEN, + result["connections"]["in"].getValStr(), + result["connections"]["out"].getValStr(), + result["connections"]["total"].getValStr(), + RESET); + result_string += strprintf("Version: %s\n", result["version"].getValStr()); + result_string += strprintf("Time offset (s): %s\n", result["timeoffset"].getValStr()); + + // proxies + std::map<std::string, std::vector<std::string>> proxy_networks; + std::vector<std::string> ordered_proxies; + + for (const UniValue& network : result["networks"].getValues()) { + const std::string proxy = network["proxy"].getValStr(); + if (proxy.empty()) continue; + // Add proxy to ordered_proxy if has not been processed + if (proxy_networks.find(proxy) == proxy_networks.end()) ordered_proxies.push_back(proxy); + + proxy_networks[proxy].push_back(network["name"].getValStr()); + } + + std::vector<std::string> formatted_proxies; + for (const std::string& proxy : ordered_proxies) { + formatted_proxies.emplace_back(strprintf("%s (%s)", proxy, Join(proxy_networks.find(proxy)->second, ", "))); + } + result_string += strprintf("Proxies: %s\n", formatted_proxies.empty() ? "n/a" : Join(formatted_proxies, ", ")); + + result_string += strprintf("Min tx relay fee rate (%s/kvB): %s\n\n", CURRENCY_UNIT, result["relayfee"].getValStr()); + + if (!result["has_wallet"].isNull()) { + const std::string walletname = result["walletname"].getValStr(); + result_string += strprintf("%sWallet: %s%s\n", MAGENTA, walletname.empty() ? "\"\"" : walletname, RESET); + + result_string += strprintf("Keypool size: %s\n", result["keypoolsize"].getValStr()); + if (!result["unlocked_until"].isNull()) { + result_string += strprintf("Unlocked until: %s\n", result["unlocked_until"].getValStr()); + } + result_string += strprintf("Transaction fee rate (-paytxfee) (%s/kvB): %s\n\n", CURRENCY_UNIT, result["paytxfee"].getValStr()); + } + if (!result["balance"].isNull()) { + result_string += strprintf("%sBalance:%s %s\n\n", CYAN, RESET, result["balance"].getValStr()); + } + + if (!result["balances"].isNull()) { + result_string += strprintf("%sBalances%s\n", CYAN, RESET); + + size_t max_balance_length{10}; + + for (const std::string& wallet : result["balances"].getKeys()) { + max_balance_length = std::max(result["balances"][wallet].getValStr().length(), max_balance_length); + } + + for (const std::string& wallet : result["balances"].getKeys()) { + result_string += strprintf("%*s %s\n", + max_balance_length, + result["balances"][wallet].getValStr(), + wallet.empty() ? "\"\"" : wallet); + } + result_string += "\n"; + } + + result_string += strprintf("%sWarnings:%s %s", YELLOW, RESET, result["warnings"].getValStr()); + result.setStr(result_string); +} + +/** * Call RPC getnewaddress. * @returns getnewaddress response as a UniValue object. */ @@ -994,9 +1165,13 @@ static int CommandLineRPC(int argc, char *argv[]) UniValue result = find_value(reply, "result"); const UniValue& error = find_value(reply, "error"); if (error.isNull()) { - if (gArgs.IsArgSet("-getinfo") && !gArgs.IsArgSet("-rpcwallet")) { - GetWalletBalances(result); // fetch multiwallet balances and append to result + if (gArgs.GetBoolArg("-getinfo", false)) { + if (!gArgs.IsArgSet("-rpcwallet")) { + GetWalletBalances(result); // fetch multiwallet balances and append to result + } + ParseGetInfoResult(result); } + ParseResult(result, strPrint); } else { ParseError(error, strPrint, nRet); diff --git a/src/bitcoin-tx-res.rc b/src/bitcoin-tx-res.rc index b545ce9dbe..46e4fc9274 100644 --- a/src/bitcoin-tx-res.rc +++ b/src/bitcoin-tx-res.rc @@ -2,9 +2,7 @@ #include "clientversion.h" // holds the needed client version information #define VER_PRODUCTVERSION CLIENT_VERSION_MAJOR,CLIENT_VERSION_MINOR,CLIENT_VERSION_BUILD -#define VER_PRODUCTVERSION_STR STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) "." STRINGIZE(CLIENT_VERSION_BUILD) #define VER_FILEVERSION VER_PRODUCTVERSION -#define VER_FILEVERSION_STR VER_PRODUCTVERSION_STR VS_VERSION_INFO VERSIONINFO FILEVERSION VER_FILEVERSION @@ -18,13 +16,13 @@ BEGIN BEGIN VALUE "CompanyName", "Bitcoin" VALUE "FileDescription", "bitcoin-tx (CLI Bitcoin transaction editor utility)" - VALUE "FileVersion", VER_FILEVERSION_STR + VALUE "FileVersion", PACKAGE_VERSION VALUE "InternalName", "bitcoin-tx" VALUE "LegalCopyright", COPYRIGHT_STR VALUE "LegalTrademarks1", "Distributed under the MIT software license, see the accompanying file COPYING or http://www.opensource.org/licenses/mit-license.php." VALUE "OriginalFilename", "bitcoin-tx.exe" VALUE "ProductName", "bitcoin-tx" - VALUE "ProductVersion", VER_PRODUCTVERSION_STR + VALUE "ProductVersion", PACKAGE_VERSION END END diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 3fc87ae1ff..fc3bc6aa71 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -8,6 +8,7 @@ #include <clientversion.h> #include <coins.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <core_io.h> #include <key_io.h> @@ -188,10 +189,11 @@ static void RegisterLoad(const std::string& strInput) static CAmount ExtractAndValidateValue(const std::string& strValue) { - CAmount value; - if (!ParseMoney(strValue, value)) + if (std::optional<CAmount> parsed = ParseMoney(strValue)) { + return parsed.value(); + } else { throw std::runtime_error("invalid TX output value"); - return value; + } } static void MutateTxVersion(CMutableTransaction& tx, const std::string& cmdVal) @@ -726,7 +728,7 @@ static void MutateTx(CMutableTransaction& tx, const std::string& command, static void OutputTxJSON(const CTransaction& tx) { UniValue entry(UniValue::VOBJ); - TxToUniv(tx, uint256(), /* include_addresses */ false, entry); + TxToUniv(tx, uint256(), entry); std::string jsonOutput = entry.write(4); tfm::format(std::cout, "%s\n", jsonOutput); @@ -771,9 +773,7 @@ static std::string readStdin() if (ferror(stdin)) throw std::runtime_error("error reading stdin"); - boost::algorithm::trim_right(ret); - - return ret; + return TrimString(ret); } static int CommandLineRawTx(int argc, char* argv[]) diff --git a/src/bitcoin-util-res.rc b/src/bitcoin-util-res.rc index 3f0fa8ab6d..0de8c5befa 100644 --- a/src/bitcoin-util-res.rc +++ b/src/bitcoin-util-res.rc @@ -2,9 +2,7 @@ #include "clientversion.h" // holds the needed client version information #define VER_PRODUCTVERSION CLIENT_VERSION_MAJOR,CLIENT_VERSION_MINOR,CLIENT_VERSION_BUILD -#define VER_PRODUCTVERSION_STR STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) "." STRINGIZE(CLIENT_VERSION_BUILD) #define VER_FILEVERSION VER_PRODUCTVERSION -#define VER_FILEVERSION_STR VER_PRODUCTVERSION_STR VS_VERSION_INFO VERSIONINFO FILEVERSION VER_FILEVERSION @@ -18,13 +16,13 @@ BEGIN BEGIN VALUE "CompanyName", "Bitcoin" VALUE "FileDescription", "bitcoin-util (CLI Bitcoin utility)" - VALUE "FileVersion", VER_FILEVERSION_STR + VALUE "FileVersion", PACKAGE_VERSION VALUE "InternalName", "bitcoin-util" VALUE "LegalCopyright", COPYRIGHT_STR VALUE "LegalTrademarks1", "Distributed under the MIT software license, see the accompanying file COPYING or http://www.opensource.org/licenses/mit-license.php." VALUE "OriginalFilename", "bitcoin-util.exe" VALUE "ProductName", "bitcoin-util" - VALUE "ProductVersion", VER_PRODUCTVERSION_STR + VALUE "ProductVersion", PACKAGE_VERSION END END diff --git a/src/bitcoin-wallet-res.rc b/src/bitcoin-wallet-res.rc index 59346ab8f6..d86ffbd9f1 100644 --- a/src/bitcoin-wallet-res.rc +++ b/src/bitcoin-wallet-res.rc @@ -2,9 +2,7 @@ #include "clientversion.h" // holds the needed client version information #define VER_PRODUCTVERSION CLIENT_VERSION_MAJOR,CLIENT_VERSION_MINOR,CLIENT_VERSION_BUILD -#define VER_PRODUCTVERSION_STR STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) "." STRINGIZE(CLIENT_VERSION_BUILD) #define VER_FILEVERSION VER_PRODUCTVERSION -#define VER_FILEVERSION_STR VER_PRODUCTVERSION_STR VS_VERSION_INFO VERSIONINFO FILEVERSION VER_FILEVERSION @@ -18,13 +16,13 @@ BEGIN BEGIN VALUE "CompanyName", "Bitcoin" VALUE "FileDescription", "bitcoin-wallet (CLI tool for " PACKAGE_NAME " wallets)" - VALUE "FileVersion", VER_FILEVERSION_STR + VALUE "FileVersion", PACKAGE_VERSION VALUE "InternalName", "bitcoin-wallet" VALUE "LegalCopyright", COPYRIGHT_STR VALUE "LegalTrademarks1", "Distributed under the MIT software license, see the accompanying file COPYING or http://www.opensource.org/licenses/mit-license.php." VALUE "OriginalFilename", "bitcoin-wallet.exe" VALUE "ProductName", "bitcoin-wallet" - VALUE "ProductVersion", VER_PRODUCTVERSION_STR + VALUE "ProductVersion", PACKAGE_VERSION END END diff --git a/src/bitcoind-res.rc b/src/bitcoind-res.rc index a98b50c899..353761dfa7 100644 --- a/src/bitcoind-res.rc +++ b/src/bitcoind-res.rc @@ -2,9 +2,7 @@ #include "clientversion.h" // holds the needed client version information #define VER_PRODUCTVERSION CLIENT_VERSION_MAJOR,CLIENT_VERSION_MINOR,CLIENT_VERSION_BUILD -#define VER_PRODUCTVERSION_STR STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) "." STRINGIZE(CLIENT_VERSION_BUILD) #define VER_FILEVERSION VER_PRODUCTVERSION -#define VER_FILEVERSION_STR VER_PRODUCTVERSION_STR VS_VERSION_INFO VERSIONINFO FILEVERSION VER_FILEVERSION @@ -18,13 +16,13 @@ BEGIN BEGIN VALUE "CompanyName", "Bitcoin" VALUE "FileDescription", "bitcoind (Bitcoin node with a JSON-RPC server)" - VALUE "FileVersion", VER_FILEVERSION_STR + VALUE "FileVersion", PACKAGE_VERSION VALUE "InternalName", "bitcoind" VALUE "LegalCopyright", COPYRIGHT_STR VALUE "LegalTrademarks1", "Distributed under the MIT software license, see the accompanying file COPYING or http://www.opensource.org/licenses/mit-license.php." VALUE "OriginalFilename", "bitcoind.exe" VALUE "ProductName", "bitcoind" - VALUE "ProductVersion", VER_PRODUCTVERSION_STR + VALUE "ProductVersion", PACKAGE_VERSION END END diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 654679af27..25ec2809e9 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -19,6 +19,7 @@ #include <shutdown.h> #include <util/check.h> #include <util/strencodings.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <util/threadnames.h> #include <util/tokenpipe.h> @@ -238,6 +239,7 @@ static bool AppInit(NodeContext& node, int argc, char* argv[]) daemon_ep.Close(); } #endif + SetSyscallSandboxPolicy(SyscallSandboxPolicy::SHUTOFF); if (fRet) { WaitForShutdown(); } diff --git a/src/bloom.cpp b/src/bloom.cpp index d0128a26d7..15e06389de 100644 --- a/src/bloom.cpp +++ b/src/bloom.cpp @@ -4,20 +4,22 @@ #include <bloom.h> -#include <primitives/transaction.h> #include <hash.h> +#include <primitives/transaction.h> +#include <random.h> #include <script/script.h> #include <script/standard.h> -#include <random.h> +#include <span.h> #include <streams.h> -#include <math.h> -#include <stdlib.h> - #include <algorithm> +#include <cmath> +#include <cstdlib> +#include <limits> +#include <vector> -#define LN2SQUARED 0.4804530139182014246671025263266649717305529515945455 -#define LN2 0.6931471805599453094172321214581765680755001343602552 +static constexpr double LN2SQUARED = 0.4804530139182014246671025263266649717305529515945455; +static constexpr double LN2 = 0.6931471805599453094172321214581765680755001343602552; CBloomFilter::CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweakIn, unsigned char nFlagsIn) : /** @@ -37,13 +39,13 @@ CBloomFilter::CBloomFilter(const unsigned int nElements, const double nFPRate, c { } -inline unsigned int CBloomFilter::Hash(unsigned int nHashNum, const std::vector<unsigned char>& vDataToHash) const +inline unsigned int CBloomFilter::Hash(unsigned int nHashNum, Span<const unsigned char> vDataToHash) const { // 0xFBA4C795 chosen as it guarantees a reasonable bit difference between nHashNum values. return MurmurHash3(nHashNum * 0xFBA4C795 + nTweak, vDataToHash) % (vData.size() * 8); } -void CBloomFilter::insert(const std::vector<unsigned char>& vKey) +void CBloomFilter::insert(Span<const unsigned char> vKey) { if (vData.empty()) // Avoid divide-by-zero (CVE-2013-5700) return; @@ -59,17 +61,10 @@ void CBloomFilter::insert(const COutPoint& outpoint) { CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << outpoint; - std::vector<unsigned char> data(stream.begin(), stream.end()); - insert(data); + insert(stream); } -void CBloomFilter::insert(const uint256& hash) -{ - std::vector<unsigned char> data(hash.begin(), hash.end()); - insert(data); -} - -bool CBloomFilter::contains(const std::vector<unsigned char>& vKey) const +bool CBloomFilter::contains(Span<const unsigned char> vKey) const { if (vData.empty()) // Avoid divide-by-zero (CVE-2013-5700) return true; @@ -87,14 +82,7 @@ bool CBloomFilter::contains(const COutPoint& outpoint) const { CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << outpoint; - std::vector<unsigned char> data(stream.begin(), stream.end()); - return contains(data); -} - -bool CBloomFilter::contains(const uint256& hash) const -{ - std::vector<unsigned char> data(hash.begin(), hash.end()); - return contains(data); + return contains(MakeUCharSpan(stream)); } bool CBloomFilter::IsWithinSizeConstraints() const @@ -198,7 +186,8 @@ CRollingBloomFilter::CRollingBloomFilter(const unsigned int nElements, const dou } /* Similar to CBloomFilter::Hash */ -static inline uint32_t RollingBloomHash(unsigned int nHashNum, uint32_t nTweak, const std::vector<unsigned char>& vDataToHash) { +static inline uint32_t RollingBloomHash(unsigned int nHashNum, uint32_t nTweak, Span<const unsigned char> vDataToHash) +{ return MurmurHash3(nHashNum * 0xFBA4C795 + nTweak, vDataToHash); } @@ -210,7 +199,7 @@ static inline uint32_t FastMod(uint32_t x, size_t n) { return ((uint64_t)x * (uint64_t)n) >> 32; } -void CRollingBloomFilter::insert(const std::vector<unsigned char>& vKey) +void CRollingBloomFilter::insert(Span<const unsigned char> vKey) { if (nEntriesThisGeneration == nEntriesPerGeneration) { nEntriesThisGeneration = 0; @@ -241,13 +230,7 @@ void CRollingBloomFilter::insert(const std::vector<unsigned char>& vKey) } } -void CRollingBloomFilter::insert(const uint256& hash) -{ - std::vector<unsigned char> vData(hash.begin(), hash.end()); - insert(vData); -} - -bool CRollingBloomFilter::contains(const std::vector<unsigned char>& vKey) const +bool CRollingBloomFilter::contains(Span<const unsigned char> vKey) const { for (int n = 0; n < nHashFuncs; n++) { uint32_t h = RollingBloomHash(n, nTweak, vKey); @@ -261,12 +244,6 @@ bool CRollingBloomFilter::contains(const std::vector<unsigned char>& vKey) const return true; } -bool CRollingBloomFilter::contains(const uint256& hash) const -{ - std::vector<unsigned char> vData(hash.begin(), hash.end()); - return contains(vData); -} - void CRollingBloomFilter::reset() { nTweak = GetRand(std::numeric_limits<unsigned int>::max()); diff --git a/src/bloom.h b/src/bloom.h index fdaa8abfb2..422646d8b9 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -6,16 +6,16 @@ #define BITCOIN_BLOOM_H #include <serialize.h> +#include <span.h> #include <vector> class COutPoint; class CTransaction; -class uint256; //! 20,000 items with fp rate < 0.1% or 10,000 items and <0.0001% -static const unsigned int MAX_BLOOM_FILTER_SIZE = 36000; // bytes -static const unsigned int MAX_HASH_FUNCS = 50; +static constexpr unsigned int MAX_BLOOM_FILTER_SIZE = 36000; // bytes +static constexpr unsigned int MAX_HASH_FUNCS = 50; /** * First two bits of nFlags control how much IsRelevantAndUpdate actually updates @@ -49,7 +49,7 @@ private: unsigned int nTweak; unsigned char nFlags; - unsigned int Hash(unsigned int nHashNum, const std::vector<unsigned char>& vDataToHash) const; + unsigned int Hash(unsigned int nHashNum, Span<const unsigned char> vDataToHash) const; public: /** @@ -66,13 +66,11 @@ public: SERIALIZE_METHODS(CBloomFilter, obj) { READWRITE(obj.vData, obj.nHashFuncs, obj.nTweak, obj.nFlags); } - void insert(const std::vector<unsigned char>& vKey); + void insert(Span<const unsigned char> vKey); void insert(const COutPoint& outpoint); - void insert(const uint256& hash); - bool contains(const std::vector<unsigned char>& vKey) const; + bool contains(Span<const unsigned char> vKey) const; bool contains(const COutPoint& outpoint) const; - bool contains(const uint256& hash) const; //! True if the size is <= MAX_BLOOM_FILTER_SIZE and the number of hash functions is <= MAX_HASH_FUNCS //! (catch a filter which was just deserialized which was too big) @@ -112,10 +110,8 @@ class CRollingBloomFilter public: CRollingBloomFilter(const unsigned int nElements, const double nFPRate); - void insert(const std::vector<unsigned char>& vKey); - void insert(const uint256& hash); - bool contains(const std::vector<unsigned char>& vKey) const; - bool contains(const uint256& hash) const; + void insert(Span<const unsigned char> vKey); + bool contains(Span<const unsigned char> vKey) const; void reset(); diff --git a/src/chain.h b/src/chain.h index 84a3a4e1e7..365a7f79b6 100644 --- a/src/chain.h +++ b/src/chain.h @@ -126,7 +126,15 @@ enum BlockStatus: uint32_t { BLOCK_FAILED_CHILD = 64, //!< descends from failed block BLOCK_FAILED_MASK = BLOCK_FAILED_VALID | BLOCK_FAILED_CHILD, - BLOCK_OPT_WITNESS = 128, //!< block data in blk*.data was received with a witness-enforcing client + BLOCK_OPT_WITNESS = 128, //!< block data in blk*.dat was received with a witness-enforcing client + + /** + * If set, this indicates that the block index entry is assumed-valid. + * Certain diagnostics will be skipped in e.g. CheckBlockIndex(). + * It almost certainly means that the block's full validation is pending + * on a background chainstate. See `doc/assumeutxo.md`. + */ + BLOCK_ASSUMED_VALID = 256, }; /** The block chain is a tree shaped structure starting with the @@ -300,14 +308,24 @@ public: return ((nStatus & BLOCK_VALID_MASK) >= nUpTo); } + //! @returns true if the block is assumed-valid; this means it is queued to be + //! validated by a background chainstate. + bool IsAssumedValid() const { return nStatus & BLOCK_ASSUMED_VALID; } + //! Raise the validity level of this block index entry. //! Returns true if the validity was changed. bool RaiseValidity(enum BlockStatus nUpTo) { assert(!(nUpTo & ~BLOCK_VALID_MASK)); // Only validity flags allowed. - if (nStatus & BLOCK_FAILED_MASK) - return false; + if (nStatus & BLOCK_FAILED_MASK) return false; + if ((nStatus & BLOCK_VALID_MASK) < nUpTo) { + // If this block had been marked assumed-valid and we're raising + // its validity to a certain point, there is no longer an assumption. + if (nStatus & BLOCK_ASSUMED_VALID && nUpTo >= BLOCK_VALID_SCRIPTS) { + nStatus &= ~BLOCK_ASSUMED_VALID; + } + nStatus = (nStatus & ~BLOCK_VALID_MASK) | nUpTo; return true; } diff --git a/src/chainparams.cpp b/src/chainparams.cpp index fdaadeed4a..b155745794 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -7,9 +7,9 @@ #include <chainparamsseeds.h> #include <consensus/merkle.h> +#include <deploymentinfo.h> #include <hash.h> // for signet block challenge hash #include <util/system.h> -#include <versionbitsinfo.h> #include <assert.h> @@ -91,8 +91,8 @@ public: consensus.vDeployments[Consensus::DEPLOYMENT_TAPROOT].nTimeout = 1628640000; // August 11th, 2021 consensus.vDeployments[Consensus::DEPLOYMENT_TAPROOT].min_activation_height = 709632; // Approximately November 12th, 2021 - consensus.nMinimumChainWork = uint256S("0x00000000000000000000000000000000000000001533efd8d716a517fe2c5008"); - consensus.defaultAssumeValid = uint256S("0x0000000000000000000b9d2ec5a352ecba0592946514a92f14319dc2b367fc72"); // 654683 + consensus.nMinimumChainWork = uint256S("0x00000000000000000000000000000000000000001fa4663bbbe19f82de910280"); + consensus.defaultAssumeValid = uint256S("0x00000000000000000008a89e854d57e5667df88f1cdef6fde2fbca1de5b639ad"); // 691719 /** * The message start string is designed to be unlikely to occur in normal data. @@ -105,7 +105,7 @@ public: pchMessageStart[3] = 0xd9; nDefaultPort = 8333; nPruneAfterHeight = 100000; - m_assumed_blockchain_size = 350; + m_assumed_blockchain_size = 420; m_assumed_chain_state_size = 6; genesis = CreateGenesisBlock(1231006505, 2083236893, 0x1d00ffff, 1, 50 * COIN); @@ -166,10 +166,10 @@ public: }; chainTxData = ChainTxData{ - // Data from RPC: getchaintxstats 4096 0000000000000000000b9d2ec5a352ecba0592946514a92f14319dc2b367fc72 - /* nTime */ 1603995752, - /* nTxCount */ 582083445, - /* dTxRate */ 3.508976121410527, + // Data from RPC: getchaintxstats 4096 00000000000000000008a89e854d57e5667df88f1cdef6fde2fbca1de5b639ad + /* nTime */ 1626697539, + /* nTxCount */ 656509474, + /* dTxRate */ 2.424920418708139, }; } }; @@ -210,8 +210,8 @@ public: consensus.vDeployments[Consensus::DEPLOYMENT_TAPROOT].nTimeout = 1628640000; // August 11th, 2021 consensus.vDeployments[Consensus::DEPLOYMENT_TAPROOT].min_activation_height = 0; // No activation delay - consensus.nMinimumChainWork = uint256S("0x0000000000000000000000000000000000000000000001db6ec4ac88cf2272c6"); - consensus.defaultAssumeValid = uint256S("0x000000000000006433d1efec504c53ca332b64963c425395515b01977bd7b3b0"); // 1864000 + consensus.nMinimumChainWork = uint256S("0x0000000000000000000000000000000000000000000005180c3bd8290da33a1a"); + consensus.defaultAssumeValid = uint256S("0x0000000000004ae2f3896ca8ecd41c460a35bf6184e145d91558cece1c688a76"); // 2010000 pchMessageStart[0] = 0x0b; pchMessageStart[1] = 0x11; @@ -261,10 +261,10 @@ public: }; chainTxData = ChainTxData{ - // Data from RPC: getchaintxstats 4096 000000000000006433d1efec504c53ca332b64963c425395515b01977bd7b3b0 - /* nTime */ 1603359686, - /* nTxCount */ 58090238, - /* dTxRate */ 0.1232886622799463, + // Data from RPC: getchaintxstats 4096 0000000000004ae2f3896ca8ecd41c460a35bf6184e145d91558cece1c688a76 + /* nTime */ 1625727096, + /* nTxCount */ 60408943, + /* dTxRate */ 0.08379062270367649, }; } }; @@ -280,19 +280,21 @@ public: if (!args.IsArgSet("-signetchallenge")) { bin = ParseHex("512103ad5e0edad18cb1f0fc0d28a3d4f1f3e445640337489abb10404f2d1e086be430210359ef5021964fe22d6f8e05b2463c9540ce96883fe3b278760f048f5189f2e6c452ae"); + vSeeds.emplace_back("seed.signet.bitcoin.sprovoost.nl"); + + // Hardcoded nodes can be removed once there are more DNS seeds vSeeds.emplace_back("178.128.221.177"); - vSeeds.emplace_back("2a01:7c8:d005:390::5"); vSeeds.emplace_back("v7ajjeirttkbnt32wpy3c6w3emwnfr3fkla7hpxcfokr3ysd3kqtzmqd.onion:38333"); - consensus.nMinimumChainWork = uint256S("0x00000000000000000000000000000000000000000000000000000019fd16269a"); - consensus.defaultAssumeValid = uint256S("0x0000002a1de0f46379358c1fd09906f7ac59adf3712323ed90eb59e4c183c020"); // 9434 + consensus.nMinimumChainWork = uint256S("0x0000000000000000000000000000000000000000000000000000008546553c03"); + consensus.defaultAssumeValid = uint256S("0x000000187d4440e5bff91488b700a140441e089a8aaea707414982460edbfe54"); // 47200 m_assumed_blockchain_size = 1; m_assumed_chain_state_size = 0; chainTxData = ChainTxData{ - // Data from RPC: getchaintxstats 4096 0000002a1de0f46379358c1fd09906f7ac59adf3712323ed90eb59e4c183c020 - /* nTime */ 1603986000, - /* nTxCount */ 9582, - /* dTxRate */ 0.00159272030651341, + // Data from RPC: getchaintxstats 4096 000000187d4440e5bff91488b700a140441e089a8aaea707414982460edbfe54 + /* nTime */ 1626696658, + /* nTxCount */ 387761, + /* dTxRate */ 0.04035946932424404, }; } else { const auto signet_challenge = args.GetArgs("-signetchallenge"); @@ -390,12 +392,12 @@ public: consensus.signet_challenge.clear(); consensus.nSubsidyHalvingInterval = 150; consensus.BIP16Exception = uint256(); - consensus.BIP34Height = 500; // BIP34 activated on regtest (Used in functional tests) + consensus.BIP34Height = 1; // Always active unless overridden consensus.BIP34Hash = uint256(); - consensus.BIP65Height = 1351; // BIP65 activated on regtest (Used in functional tests) - consensus.BIP66Height = 1251; // BIP66 activated on regtest (Used in functional tests) - consensus.CSVHeight = 432; // CSV activated on regtest (Used in rpc activation tests) - consensus.SegwitHeight = 0; // SEGWIT is always activated on regtest unless overridden + consensus.BIP65Height = 1; // Always active unless overridden + consensus.BIP66Height = 1; // Always active unless overridden + consensus.CSVHeight = 1; // Always active unless overridden + consensus.SegwitHeight = 1; // Always active unless overridden consensus.MinBIP9WarningHeight = 0; consensus.powLimit = uint256S("7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); consensus.nPowTargetTimespan = 14 * 24 * 60 * 60; // two weeks @@ -435,7 +437,8 @@ public: assert(genesis.hashMerkleRoot == uint256S("0x4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b")); vFixedSeeds.clear(); //!< Regtest mode doesn't have any fixed seeds. - vSeeds.clear(); //!< Regtest mode doesn't have any DNS seeds. + vSeeds.clear(); + vSeeds.emplace_back("dummySeed.invalid."); fDefaultConsistencyChecks = true; fRequireStandard = true; @@ -486,18 +489,38 @@ public: void UpdateActivationParametersFromArgs(const ArgsManager& args); }; -void CRegTestParams::UpdateActivationParametersFromArgs(const ArgsManager& args) +static void MaybeUpdateHeights(const ArgsManager& args, Consensus::Params& consensus) { - if (args.IsArgSet("-segwitheight")) { - int64_t height = args.GetArg("-segwitheight", consensus.SegwitHeight); - if (height < -1 || height >= std::numeric_limits<int>::max()) { - throw std::runtime_error(strprintf("Activation height %ld for segwit is out of valid range. Use -1 to disable segwit.", height)); - } else if (height == -1) { - LogPrintf("Segwit disabled for testing\n"); - height = std::numeric_limits<int>::max(); + for (const std::string& arg : args.GetArgs("-testactivationheight")) { + const auto found{arg.find('@')}; + if (found == std::string::npos) { + throw std::runtime_error(strprintf("Invalid format (%s) for -testactivationheight=name@height.", arg)); + } + const auto name{arg.substr(0, found)}; + const auto value{arg.substr(found + 1)}; + int32_t height; + if (!ParseInt32(value, &height) || height < 0 || height >= std::numeric_limits<int>::max()) { + throw std::runtime_error(strprintf("Invalid height value (%s) for -testactivationheight=name@height.", arg)); + } + if (name == "segwit") { + consensus.SegwitHeight = int{height}; + } else if (name == "bip34") { + consensus.BIP34Height = int{height}; + } else if (name == "dersig") { + consensus.BIP66Height = int{height}; + } else if (name == "cltv") { + consensus.BIP65Height = int{height}; + } else if (name == "csv") { + consensus.CSVHeight = int{height}; + } else { + throw std::runtime_error(strprintf("Invalid name (%s) for -testactivationheight=name@height.", arg)); } - consensus.SegwitHeight = static_cast<int>(height); } +} + +void CRegTestParams::UpdateActivationParametersFromArgs(const ArgsManager& args) +{ + MaybeUpdateHeights(args, consensus); if (!args.IsArgSet("-vbparams")) return; diff --git a/src/chainparams.h b/src/chainparams.h index 5c2351eea6..4faa6f8d06 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -8,11 +8,13 @@ #include <chainparamsbase.h> #include <consensus/params.h> +#include <netaddress.h> #include <primitives/block.h> #include <protocol.h> #include <util/hash_type.h> #include <memory> +#include <string> #include <vector> typedef std::map<int, uint256> MapCheckpoints; @@ -80,6 +82,15 @@ public: const Consensus::Params& GetConsensus() const { return consensus; } const CMessageHeader::MessageStartChars& MessageStart() const { return pchMessageStart; } uint16_t GetDefaultPort() const { return nDefaultPort; } + uint16_t GetDefaultPort(Network net) const + { + return net == NET_I2P ? I2P_SAM31_PORT : GetDefaultPort(); + } + uint16_t GetDefaultPort(const std::string& addr) const + { + CNetAddr a; + return a.SetSpecial(addr) ? GetDefaultPort(a.GetNetwork()) : GetDefaultPort(); + } const CBlock& GenesisBlock() const { return genesis; } /** Default value for -checkmempool and -checkblockindex argument */ diff --git a/src/chainparamsbase.cpp b/src/chainparamsbase.cpp index e71b4bc859..dc484f5c03 100644 --- a/src/chainparamsbase.cpp +++ b/src/chainparamsbase.cpp @@ -20,7 +20,7 @@ void SetupChainParamsBaseOptions(ArgsManager& argsman) argsman.AddArg("-chain=<chain>", "Use the chain <chain> (default: main). Allowed values: main, test, signet, regtest", ArgsManager::ALLOW_ANY, OptionsCategory::CHAINPARAMS); argsman.AddArg("-regtest", "Enter regression test mode, which uses a special chain in which blocks can be solved instantly. " "This is intended for regression testing tools and app development. Equivalent to -chain=regtest.", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CHAINPARAMS); - argsman.AddArg("-segwitheight=<n>", "Set the activation height of segwit. -1 to disable. (regtest-only)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-testactivationheight=name@height.", "Set the activation height of 'name' (segwit, bip34, dersig, cltv, csv). (regtest-only)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-testnet", "Use the test chain. Equivalent to -chain=test.", ArgsManager::ALLOW_ANY, OptionsCategory::CHAINPARAMS); argsman.AddArg("-vbparams=deployment:start:end[:min_activation_height]", "Use given start/end times and min_activation_height for specified version bits deployment (regtest-only)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CHAINPARAMS); argsman.AddArg("-signet", "Use the signet chain. Equivalent to -chain=signet. Note that the network is defined by the -signetchallenge parameter", ArgsManager::ALLOW_ANY, OptionsCategory::CHAINPARAMS); diff --git a/src/chainparamsseeds.h b/src/chainparamsseeds.h index 08587e2b37..953a09d5e7 100644 --- a/src/chainparamsseeds.h +++ b/src/chainparamsseeds.h @@ -683,14 +683,22 @@ static const uint8_t chainparams_seed_main[] = { 0x04,0x20,0x98,0xc6,0x44,0x27,0x90,0x41,0xa6,0x98,0xf9,0x25,0x6c,0x59,0x0f,0x06,0x6d,0x44,0x59,0x0e,0xb2,0x46,0xb0,0xa4,0x37,0x88,0x69,0x8f,0xc1,0x32,0xcd,0x9f,0x15,0xd7,0x20,0x8d, 0x04,0x20,0xaa,0x3a,0x16,0x86,0xea,0x59,0x09,0x04,0x78,0xe5,0x10,0x92,0xe1,0x1d,0xad,0xf7,0x56,0x2b,0xac,0xb0,0x97,0x29,0x63,0x30,0xf4,0x1b,0xcf,0xde,0xf3,0x28,0x0a,0x29,0x20,0x8d, 0x04,0x20,0xbc,0x27,0xae,0x89,0xc1,0x67,0x73,0x0a,0x08,0x02,0xdf,0xb7,0xcc,0x94,0xc7,0x9f,0xf4,0x72,0x7a,0x9b,0x20,0x0c,0x5c,0x11,0x3d,0x22,0xd6,0x13,0x88,0x66,0x74,0xbf,0x20,0x8d, - 0x05,0x20,0xfe,0x97,0xba,0x09,0x2a,0xa4,0x85,0x10,0xa1,0x04,0x7b,0x88,0x7a,0x5a,0x06,0x53,0x71,0x93,0x3b,0xf9,0xa2,0x2f,0xd9,0xe3,0x8f,0xa5,0xa2,0xac,0x1e,0x6c,0x6c,0x8c,0x20,0x8d, - 0x05,0x20,0x17,0x0c,0x56,0xce,0x72,0xa5,0xa0,0xe6,0x23,0x06,0xa3,0xc7,0x08,0x43,0x18,0xee,0x3a,0x46,0x35,0x5d,0x17,0xf6,0x78,0x96,0xa0,0x9c,0x51,0xef,0xbe,0x23,0xfd,0x71,0x20,0x8d, - 0x05,0x20,0x31,0x0f,0x30,0x0b,0x9d,0x70,0x0c,0x7c,0xf7,0x98,0x7e,0x1c,0xf4,0x33,0xdc,0x64,0x17,0xf7,0x00,0x7a,0x0c,0x04,0xb5,0x83,0xfc,0x5f,0xa6,0x52,0x39,0x79,0x63,0x87,0x20,0x8d, - 0x05,0x20,0x3e,0xe3,0xe0,0xa9,0xbc,0xf4,0x2e,0x59,0xd9,0x20,0xee,0xdf,0x74,0x61,0x4d,0x99,0x0c,0x5c,0x15,0x30,0x9b,0x72,0x16,0x79,0x15,0xf4,0x7a,0xca,0x34,0xcc,0x81,0x99,0x20,0x8d, - 0x05,0x20,0x3b,0x42,0x1c,0x25,0xf7,0xbf,0x79,0xed,0x6d,0x7d,0xef,0x65,0x30,0x7d,0xee,0x16,0x37,0x22,0x72,0x43,0x33,0x28,0x40,0xa3,0xaa,0xf4,0x48,0x49,0x67,0xb1,0x4b,0xfd,0x20,0x8d, - 0x05,0x20,0x7a,0x65,0xf7,0x47,0x42,0x9d,0x66,0x42,0x3b,0xb3,0xa7,0x03,0x6c,0x46,0x78,0x19,0x28,0x78,0x1e,0xa3,0x7c,0x67,0x44,0xb7,0x83,0x05,0xe3,0xfe,0xa5,0xe4,0x0a,0x6e,0x20,0x8d, - 0x05,0x20,0xb5,0x83,0x6f,0xb6,0x11,0xd8,0x0e,0xa8,0x57,0xda,0x15,0x20,0x5b,0x1a,0x6d,0x21,0x15,0x5a,0xbd,0xb4,0x17,0x11,0xc2,0xfb,0x0e,0xfc,0xde,0xe8,0x26,0x56,0xa8,0xac,0x20,0x8d, - 0x05,0x20,0xcc,0xaf,0x6c,0x3b,0xd0,0x13,0x76,0x23,0xc3,0x36,0xbb,0x64,0x4a,0x4a,0x06,0x93,0x69,0x6d,0xb0,0x10,0x6e,0x66,0xa4,0x61,0xf8,0x2d,0xe7,0x80,0x72,0x4d,0x53,0x94,0x20,0x8d, + 0x05,0x20,0xe1,0xd6,0xb8,0xfa,0xdd,0xeb,0x03,0x32,0x30,0x3b,0x20,0x6a,0xbc,0xaf,0x99,0x4f,0xa0,0xa2,0x72,0x48,0xfe,0x44,0xe0,0xf6,0x03,0xc1,0xbd,0xb6,0x24,0xd0,0xf6,0xb8,0x00,0x00, + 0x05,0x20,0xf4,0xb7,0xb4,0xcd,0xf5,0xb6,0x54,0x82,0x27,0x6d,0x29,0x7b,0x06,0x7f,0x52,0x59,0xa0,0xb4,0xdc,0xf7,0x6f,0xb4,0x71,0xcf,0xcc,0xfb,0x6b,0x86,0xc2,0x57,0x80,0xc6,0x00,0x00, + 0x05,0x20,0x07,0x61,0x26,0xd7,0x6c,0x05,0xbf,0xf6,0x2d,0x8c,0xca,0xc4,0x65,0xd3,0xd3,0xb2,0x49,0xe9,0xcc,0x53,0x1e,0xca,0x77,0x84,0xb6,0x10,0x5e,0xc2,0x5a,0xfe,0x28,0xb3,0x00,0x00, + 0x05,0x20,0x0a,0x26,0x27,0x45,0xb1,0x1e,0xfc,0x27,0x03,0x32,0x0e,0x65,0x9e,0x3c,0x64,0x0e,0x33,0x50,0x3d,0x6c,0x90,0x17,0x0e,0x29,0xee,0x5a,0x58,0xdf,0x08,0xde,0xbf,0x73,0x00,0x00, + 0x05,0x20,0x17,0x0c,0x56,0xce,0x72,0xa5,0xa0,0xe6,0x23,0x06,0xa3,0xc7,0x08,0x43,0x18,0xee,0x3a,0x46,0x35,0x5d,0x17,0xf6,0x78,0x96,0xa0,0x9c,0x51,0xef,0xbe,0x23,0xfd,0x71,0x00,0x00, + 0x05,0x20,0x19,0xe7,0x0d,0x3f,0xfe,0x9e,0x0e,0x8e,0x73,0x40,0x40,0xc3,0xba,0x8f,0x41,0xaf,0xf1,0x7b,0xa6,0x83,0x1b,0xc3,0xa4,0xe0,0x6d,0x6c,0x57,0xa7,0x36,0x5d,0x09,0xce,0x00,0x00, + 0x05,0x20,0x31,0x0f,0x30,0x0b,0x9d,0x70,0x0c,0x7c,0xf7,0x98,0x7e,0x1c,0xf4,0x33,0xdc,0x64,0x17,0xf7,0x00,0x7a,0x0c,0x04,0xb5,0x83,0xfc,0x5f,0xa6,0x52,0x39,0x79,0x63,0x87,0x00,0x00, + 0x05,0x20,0x3e,0xe3,0xe0,0xa9,0xbc,0xf4,0x2e,0x59,0xd9,0x20,0xee,0xdf,0x74,0x61,0x4d,0x99,0x0c,0x5c,0x15,0x30,0x9b,0x72,0x16,0x79,0x15,0xf4,0x7a,0xca,0x34,0xcc,0x81,0x99,0x00,0x00, + 0x05,0x20,0x3b,0x42,0x1c,0x25,0xf7,0xbf,0x79,0xed,0x6d,0x7d,0xef,0x65,0x30,0x7d,0xee,0x16,0x37,0x22,0x72,0x43,0x33,0x28,0x40,0xa3,0xaa,0xf4,0x48,0x49,0x67,0xb1,0x4b,0xfd,0x00,0x00, + 0x05,0x20,0x46,0xce,0x21,0x81,0x48,0xce,0xa7,0x8a,0x98,0xca,0xb1,0x0b,0x51,0xa5,0xc8,0xff,0x39,0xc5,0x1a,0xa3,0xd3,0x02,0x32,0xa3,0x29,0xad,0x79,0xb8,0x7f,0x34,0x51,0x33,0x00,0x00, + 0x05,0x20,0x4e,0x77,0x2e,0x12,0x91,0x67,0x6b,0x94,0xc4,0x92,0x2f,0x19,0x67,0x7d,0xcd,0x47,0x02,0xad,0xf8,0x60,0x72,0xed,0x73,0xf1,0x10,0x99,0x2c,0x05,0x61,0x66,0x55,0xd9,0x00,0x00, + 0x05,0x20,0x53,0x94,0xa6,0x3e,0x14,0x82,0xd4,0xf9,0xd3,0xa7,0x53,0x33,0x05,0xce,0x72,0x64,0xed,0x74,0x09,0x63,0x8f,0x24,0xef,0xda,0x12,0xa1,0x55,0xe0,0xd8,0xbb,0xd3,0x58,0x00,0x00, + 0x05,0x20,0x55,0x62,0x32,0x7d,0x82,0x32,0x4f,0x9d,0xdf,0x24,0x5c,0xed,0x8e,0x1a,0x5a,0x8d,0xc6,0x50,0xb4,0x32,0xd5,0x85,0xef,0xb0,0xfa,0x7c,0xf9,0xbb,0x25,0x89,0x6b,0x03,0x00,0x00, + 0x05,0x20,0x91,0xcf,0xa2,0x5b,0x04,0x33,0x69,0x66,0xb0,0x72,0x27,0x54,0xbe,0xcd,0xd8,0x08,0xeb,0x95,0x55,0x5a,0xc2,0x79,0x91,0x3a,0xd9,0xf2,0x2c,0x73,0x9f,0x78,0x50,0xca,0x00,0x00, + 0x05,0x20,0xb5,0x83,0x6f,0xb6,0x11,0xd8,0x0e,0xa8,0x57,0xda,0x15,0x20,0x5b,0x1a,0x6d,0x21,0x15,0x5a,0xbd,0xb4,0x17,0x11,0xc2,0xfb,0x0e,0xfc,0xde,0xe8,0x26,0x56,0xa8,0xac,0x00,0x00, + 0x05,0x20,0xcc,0xaf,0x6c,0x3b,0xd0,0x13,0x76,0x23,0xc3,0x36,0xbb,0x64,0x4a,0x4a,0x06,0x93,0x69,0x6d,0xb0,0x10,0x6e,0x66,0xa4,0x61,0xf8,0x2d,0xe7,0x80,0x72,0x4d,0x53,0x94,0x00,0x00, }; static const uint8_t chainparams_seed_test[] = { diff --git a/src/checkqueue.h b/src/checkqueue.h index 4ceeb3600a..7c20e2013c 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -7,6 +7,7 @@ #include <sync.h> #include <tinyformat.h> +#include <util/syscall_sandbox.h> #include <util/threadnames.h> #include <algorithm> @@ -151,6 +152,7 @@ public: for (int n = 0; n < threads_num; ++n) { m_worker_threads.emplace_back([this, n]() { util::ThreadRename(strprintf("scriptch.%i", n)); + SetSyscallSandboxPolicy(SyscallSandboxPolicy::VALIDATION_SCRIPT_CHECK); Loop(false /* worker thread */); }); } diff --git a/src/clientversion.cpp b/src/clientversion.cpp index 29c38e2d3b..f97e4097e8 100644 --- a/src/clientversion.cpp +++ b/src/clientversion.cpp @@ -30,8 +30,10 @@ const std::string CLIENT_NAME("Satoshi"); #define BUILD_DESC BUILD_GIT_TAG #define BUILD_SUFFIX "" #else - #define BUILD_DESC "v" STRINGIZE(CLIENT_VERSION_MAJOR) "." STRINGIZE(CLIENT_VERSION_MINOR) "." STRINGIZE(CLIENT_VERSION_BUILD) - #ifdef BUILD_GIT_COMMIT + #define BUILD_DESC "v" PACKAGE_VERSION + #if CLIENT_VERSION_IS_RELEASE + #define BUILD_SUFFIX "" + #elif defined(BUILD_GIT_COMMIT) #define BUILD_SUFFIX "-" BUILD_GIT_COMMIT #elif defined(GIT_COMMIT_ID) #define BUILD_SUFFIX "-g" GIT_COMMIT_ID @@ -40,8 +42,6 @@ const std::string CLIENT_NAME("Satoshi"); #endif #endif -const std::string CLIENT_BUILD(BUILD_DESC BUILD_SUFFIX); - static std::string FormatVersion(int nVersion) { return strprintf("%d.%d.%d", nVersion / 10000, (nVersion / 100) % 100, nVersion % 100); @@ -49,6 +49,7 @@ static std::string FormatVersion(int nVersion) std::string FormatFullVersion() { + static const std::string CLIENT_BUILD(BUILD_DESC BUILD_SUFFIX); return CLIENT_BUILD; } diff --git a/src/clientversion.h b/src/clientversion.h index 0ed3f68094..a3e6233437 100644 --- a/src/clientversion.h +++ b/src/clientversion.h @@ -36,7 +36,6 @@ static const int CLIENT_VERSION = + 1 * CLIENT_VERSION_BUILD; extern const std::string CLIENT_NAME; -extern const std::string CLIENT_BUILD; std::string FormatFullVersion(); diff --git a/src/compat/glibc_compat.cpp b/src/compat/glibc_compat.cpp deleted file mode 100644 index ff581d4a9e..0000000000 --- a/src/compat/glibc_compat.cpp +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) 2009-2020 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - -#include <cstddef> -#include <cstdint> - -#if defined(__i386__) || defined(__arm__) - -extern "C" int64_t __udivmoddi4(uint64_t u, uint64_t v, uint64_t* rp); - -extern "C" int64_t __wrap___divmoddi4(int64_t u, int64_t v, int64_t* rp) -{ - int32_t c1 = 0, c2 = 0; - int64_t uu = u, vv = v; - int64_t w; - int64_t r; - - if (uu < 0) { - c1 = ~c1, c2 = ~c2, uu = -uu; - } - if (vv < 0) { - c1 = ~c1, vv = -vv; - } - - w = __udivmoddi4(uu, vv, (uint64_t*)&r); - if (c1) - w = -w; - if (c2) - r = -r; - - *rp = r; - return w; -} -#endif - -extern "C" float log2f_old(float x); -#ifdef __i386__ -__asm(".symver log2f_old,log2f@GLIBC_2.1"); -#elif defined(__amd64__) -__asm(".symver log2f_old,log2f@GLIBC_2.2.5"); -#elif defined(__arm__) -__asm(".symver log2f_old,log2f@GLIBC_2.4"); -#elif defined(__aarch64__) -__asm(".symver log2f_old,log2f@GLIBC_2.17"); -#elif defined(__powerpc64__) -# ifdef WORDS_BIGENDIAN -__asm(".symver log2f_old,log2f@GLIBC_2.3"); -# else -__asm(".symver log2f_old,log2f@GLIBC_2.17"); -# endif -#elif defined(__riscv) -__asm(".symver log2f_old,log2f@GLIBC_2.27"); -#endif -extern "C" float __wrap_log2f(float x) -{ - return log2f_old(x); -} diff --git a/src/amount.h b/src/consensus/amount.h index 47968e80b1..8b41a2277d 100644 --- a/src/amount.h +++ b/src/consensus/amount.h @@ -3,15 +3,15 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_AMOUNT_H -#define BITCOIN_AMOUNT_H +#ifndef BITCOIN_CONSENSUS_AMOUNT_H +#define BITCOIN_CONSENSUS_AMOUNT_H -#include <stdint.h> +#include <cstdint> /** Amount in satoshis (Can be negative) */ typedef int64_t CAmount; -static const CAmount COIN = 100000000; +static constexpr CAmount COIN = 100000000; /** No amount larger than this (in satoshi) is valid. * @@ -22,7 +22,7 @@ static const CAmount COIN = 100000000; * critical; in unusual circumstances like a(nother) overflow bug that allowed * for the creation of coins out of thin air modification could lead to a fork. * */ -static const CAmount MAX_MONEY = 21000000 * COIN; +static constexpr CAmount MAX_MONEY = 21000000 * COIN; inline bool MoneyRange(const CAmount& nValue) { return (nValue >= 0 && nValue <= MAX_MONEY); } -#endif // BITCOIN_AMOUNT_H +#endif // BITCOIN_CONSENSUS_AMOUNT_H diff --git a/src/consensus/params.h b/src/consensus/params.h index 28c95e0884..77bf7fd0d8 100644 --- a/src/consensus/params.h +++ b/src/consensus/params.h @@ -11,13 +11,27 @@ namespace Consensus { -enum DeploymentPos -{ +/** + * A buried deployment is one where the height of the activation has been hardcoded into + * the client implementation long after the consensus change has activated. See BIP 90. + */ +enum BuriedDeployment : int16_t { + // buried deployments get negative values to avoid overlap with DeploymentPos + DEPLOYMENT_HEIGHTINCB = std::numeric_limits<int16_t>::min(), + DEPLOYMENT_CLTV, + DEPLOYMENT_DERSIG, + DEPLOYMENT_CSV, + DEPLOYMENT_SEGWIT, +}; +constexpr bool ValidDeployment(BuriedDeployment dep) { return dep <= DEPLOYMENT_SEGWIT; } + +enum DeploymentPos : uint16_t { DEPLOYMENT_TESTDUMMY, DEPLOYMENT_TAPROOT, // Deployment of Schnorr/Taproot (BIPs 340-342) - // NOTE: Also add new deployments to VersionBitsDeploymentInfo in versionbits.cpp + // NOTE: Also add new deployments to VersionBitsDeploymentInfo in deploymentinfo.cpp MAX_VERSION_BITS_DEPLOYMENTS }; +constexpr bool ValidDeployment(DeploymentPos dep) { return dep < MAX_VERSION_BITS_DEPLOYMENTS; } /** * Struct for each individual consensus rule change using BIP9. @@ -100,7 +114,25 @@ struct Params { */ bool signet_blocks{false}; std::vector<uint8_t> signet_challenge; + + int DeploymentHeight(BuriedDeployment dep) const + { + switch (dep) { + case DEPLOYMENT_HEIGHTINCB: + return BIP34Height; + case DEPLOYMENT_CLTV: + return BIP65Height; + case DEPLOYMENT_DERSIG: + return BIP66Height; + case DEPLOYMENT_CSV: + return CSVHeight; + case DEPLOYMENT_SEGWIT: + return SegwitHeight; + } // no default case, so the compiler can warn about missing cases + return std::numeric_limits<int>::max(); + } }; + } // namespace Consensus #endif // BITCOIN_CONSENSUS_PARAMS_H diff --git a/src/consensus/tx_check.cpp b/src/consensus/tx_check.cpp index bb8cd10c63..de4824fadc 100644 --- a/src/consensus/tx_check.cpp +++ b/src/consensus/tx_check.cpp @@ -4,6 +4,7 @@ #include <consensus/tx_check.h> +#include <consensus/amount.h> #include <primitives/transaction.h> #include <consensus/validation.h> diff --git a/src/consensus/tx_verify.cpp b/src/consensus/tx_verify.cpp index f595f16eab..a07adae536 100644 --- a/src/consensus/tx_verify.cpp +++ b/src/consensus/tx_verify.cpp @@ -4,6 +4,7 @@ #include <consensus/tx_verify.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <primitives/transaction.h> #include <script/interpreter.h> @@ -20,6 +21,15 @@ bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime) return true; if ((int64_t)tx.nLockTime < ((int64_t)tx.nLockTime < LOCKTIME_THRESHOLD ? (int64_t)nBlockHeight : nBlockTime)) return true; + + // Even if tx.nLockTime isn't satisfied by nBlockHeight/nBlockTime, a + // transaction is still considered final if all inputs' nSequence == + // SEQUENCE_FINAL (0xffffffff), in which case nLockTime is ignored. + // + // Because of this behavior OP_CHECKLOCKTIMEVERIFY/CheckLockTime() will + // also check that the spending input's nSequence != SEQUENCE_FINAL, + // ensuring that an unsatisfied nLockTime value will actually cause + // IsFinalTx() to return false here: for (const auto& txin : tx.vin) { if (!(txin.nSequence == CTxIn::SEQUENCE_FINAL)) return false; @@ -135,7 +145,7 @@ unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& in return nSigOps; } -int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& inputs, int flags) +int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& inputs, uint32_t flags) { int64_t nSigOps = GetLegacySigOpCount(tx) * WITNESS_SCALE_FACTOR; diff --git a/src/consensus/tx_verify.h b/src/consensus/tx_verify.h index d5fd43e131..777556808a 100644 --- a/src/consensus/tx_verify.h +++ b/src/consensus/tx_verify.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_CONSENSUS_TX_VERIFY_H #define BITCOIN_CONSENSUS_TX_VERIFY_H -#include <amount.h> +#include <consensus/amount.h> #include <stdint.h> #include <vector> @@ -49,10 +49,10 @@ unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& ma * Compute total signature operation cost of a transaction. * @param[in] tx Transaction for which we are computing the cost * @param[in] inputs Map of previous transactions that have outputs we're spending - * @param[out] flags Script verification flags + * @param[in] flags Script verification flags * @return Total signature operation cost of tx */ -int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& inputs, int flags); +int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& inputs, uint32_t flags); /** * Check if transaction is final and can be included in a block with the diff --git a/src/core_io.h b/src/core_io.h index 3b9b66574c..be93a17efe 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_CORE_IO_H #define BITCOIN_CORE_IO_H -#include <amount.h> +#include <consensus/amount.h> #include <attributes.h> #include <string> @@ -44,8 +44,8 @@ UniValue ValueFromAmount(const CAmount amount); std::string FormatScript(const CScript& script); std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags = 0); std::string SighashToStr(unsigned char sighash_type); -void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex, bool include_addresses); -void ScriptToUniv(const CScript& script, UniValue& out, bool include_address); -void TxToUniv(const CTransaction& tx, const uint256& hashBlock, bool include_addresses, UniValue& entry, bool include_hex = true, int serialize_flags = 0, const CTxUndo* txundo = nullptr); +void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool include_hex, bool include_address = true); +void ScriptToUniv(const CScript& script, UniValue& out); +void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex = true, int serialize_flags = 0, const CTxUndo* txundo = nullptr); #endif // BITCOIN_CORE_IO_H diff --git a/src/core_read.cpp b/src/core_read.cpp index 6108961010..320811b9e9 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -69,7 +69,7 @@ CScript ParseScript(const std::string& s) (w->front() == '-' && w->size() > 1 && std::all_of(w->begin()+1, w->end(), ::IsDigit))) { // Number - int64_t n = atoi64(*w); + int64_t n = LocaleIndependentAtoi<int64_t>(*w); //limit the range of numbers ParseScript accepts in decimal //since numbers outside -0xFFFFFFFF...0xFFFFFFFF are illegal in scripts diff --git a/src/core_write.cpp b/src/core_write.cpp index b35f835f42..6b13e4c586 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -4,6 +4,7 @@ #include <core_io.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/validation.h> #include <key_io.h> @@ -141,56 +142,28 @@ std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags) return HexStr(ssTx); } -void ScriptToUniv(const CScript& script, UniValue& out, bool include_address) +void ScriptToUniv(const CScript& script, UniValue& out) { - out.pushKV("asm", ScriptToAsmStr(script)); - out.pushKV("hex", HexStr(script)); - - std::vector<std::vector<unsigned char>> solns; - TxoutType type = Solver(script, solns); - out.pushKV("type", GetTxnOutputType(type)); - - CTxDestination address; - if (include_address && ExtractDestination(script, address) && type != TxoutType::PUBKEY) { - out.pushKV("address", EncodeDestination(address)); - } + ScriptPubKeyToUniv(script, out, /* include_hex */ true, /* include_address */ false); } -// TODO: from v23 ("addresses" and "reqSigs" deprecated) this method should be refactored to remove the `include_addresses` option -// this method can also be combined with `ScriptToUniv` as they will overlap -void ScriptPubKeyToUniv(const CScript& scriptPubKey, - UniValue& out, bool fIncludeHex, bool include_addresses) +void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool include_hex, bool include_address) { - TxoutType type; CTxDestination address; - std::vector<CTxDestination> addresses; - int nRequired; out.pushKV("asm", ScriptToAsmStr(scriptPubKey)); - if (fIncludeHex) - out.pushKV("hex", HexStr(scriptPubKey)); + if (include_hex) out.pushKV("hex", HexStr(scriptPubKey)); - if (!ExtractDestinations(scriptPubKey, type, addresses, nRequired) || type == TxoutType::PUBKEY) { - out.pushKV("type", GetTxnOutputType(type)); - return; - } + std::vector<std::vector<unsigned char>> solns; + const TxoutType type{Solver(scriptPubKey, solns)}; - if (ExtractDestination(scriptPubKey, address)) { + if (include_address && ExtractDestination(scriptPubKey, address) && type != TxoutType::PUBKEY) { out.pushKV("address", EncodeDestination(address)); } out.pushKV("type", GetTxnOutputType(type)); - - if (include_addresses) { - UniValue a(UniValue::VARR); - for (const CTxDestination& addr : addresses) { - a.push_back(EncodeDestination(addr)); - } - out.pushKV("addresses", a); - out.pushKV("reqSigs", nRequired); - } } -void TxToUniv(const CTransaction& tx, const uint256& hashBlock, bool include_addresses, UniValue& entry, bool include_hex, int serialize_flags, const CTxUndo* txundo) +void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex, int serialize_flags, const CTxUndo* txundo) { entry.pushKV("txid", tx.GetHash().GetHex()); entry.pushKV("hash", tx.GetWitnessHash().GetHex()); @@ -249,7 +222,7 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, bool include_add out.pushKV("n", (int64_t)i); UniValue o(UniValue::VOBJ); - ScriptPubKeyToUniv(txout.scriptPubKey, o, true, include_addresses); + ScriptPubKeyToUniv(txout.scriptPubKey, o, true); out.pushKV("scriptPubKey", o); vout.push_back(out); diff --git a/src/crc32c/.travis.yml b/src/crc32c/.travis.yml index d990a89f07..183a5fba45 100644 --- a/src/crc32c/.travis.yml +++ b/src/crc32c/.travis.yml @@ -4,7 +4,7 @@ language: cpp dist: bionic -osx_image: xcode10.3 +osx_image: xcode12.5 compiler: - gcc @@ -24,20 +24,20 @@ env: addons: apt: sources: - - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main' + - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main' key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' - sourceline: 'ppa:ubuntu-toolchain-r/test' packages: - - clang-9 + - clang-12 - cmake - - gcc-9 - - g++-9 + - gcc-11 + - g++-11 - ninja-build homebrew: packages: - cmake - - gcc@9 - - llvm@9 + - gcc@11 + - llvm@12 - ninja update: true @@ -48,14 +48,14 @@ install: export PATH="$(brew --prefix llvm)/bin:$PATH"; fi # /usr/bin/gcc points to an older compiler on both Linux and macOS. -- if [ "$CXX" = "g++" ]; then export CXX="g++-9" CC="gcc-9"; fi +- if [ "$CXX" = "g++" ]; then export CXX="g++-11" CC="gcc-11"; fi # /usr/bin/clang points to an older compiler on both Linux and macOS. # # Homebrew's llvm package doesn't ship a versioned clang++ binary, so the values # below don't work on macOS. Fortunately, the path change above makes the # default values (clang and clang++) resolve to the correct compiler on macOS. - if [ "$TRAVIS_OS_NAME" = "linux" ]; then - if [ "$CXX" = "clang++" ]; then export CXX="clang++-9" CC="clang-9"; fi; + if [ "$CXX" = "clang++" ]; then export CXX="clang++-12" CC="clang-12"; fi; fi - echo ${CC} - echo ${CXX} diff --git a/src/crc32c/.ycm_extra_conf.py b/src/crc32c/.ycm_extra_conf.py index 536aadcec8..62daa8a4ac 100644 --- a/src/crc32c/.ycm_extra_conf.py +++ b/src/crc32c/.ycm_extra_conf.py @@ -4,10 +4,10 @@ """YouCompleteMe configuration that interprets a .clang_complete file. This module implementes the YouCompleteMe configuration API documented at: -https://github.com/Valloric/ycmd#ycm_extra_confpy-specification +https://github.com/ycm-core/ycmd#ycm_extra_confpy-specification The implementation loads and processes a .clang_complete file, documented at: -https://github.com/Rip-Rip/clang_complete/blob/master/README.md +https://github.com/xavierd/clang_complete/blob/master/README.md """ import os diff --git a/src/crc32c/README.md b/src/crc32c/README.md index 0bd69f7f09..58ba38e611 100644 --- a/src/crc32c/README.md +++ b/src/crc32c/README.md @@ -65,7 +65,7 @@ apm install autocomplete-clang build build-cmake clang-format language-cmake \ If you don't mind more setup in return for more speed, replace `autocomplete-clang` and `linter-clang` with `you-complete-me`. This requires -[setting up ycmd](https://github.com/Valloric/ycmd#building). +[setting up ycmd](https://github.com/ycm-core/ycmd#building). ```bash apm install autocomplete-plus build build-cmake clang-format language-cmake \ diff --git a/src/crc32c/src/crc32c_arm64_check.h b/src/crc32c/src/crc32c_arm64_check.h index 62a07aba09..6b80f70037 100644 --- a/src/crc32c/src/crc32c_arm64_check.h +++ b/src/crc32c/src/crc32c_arm64_check.h @@ -40,7 +40,15 @@ inline bool CanUseArm64Crc32() { // From 'arch/arm64/include/uapi/asm/hwcap.h' in Linux kernel source code. constexpr unsigned long kHWCAP_PMULL = 1 << 4; constexpr unsigned long kHWCAP_CRC32 = 1 << 7; - unsigned long hwcap = (&getauxval != nullptr) ? getauxval(AT_HWCAP) : 0; + unsigned long hwcap = +#if HAVE_STRONG_GETAUXVAL + // Some compilers warn on (&getauxval != nullptr) in the block below. + getauxval(AT_HWCAP); +#elif HAVE_WEAK_GETAUXVAL + (&getauxval != nullptr) ? getauxval(AT_HWCAP) : 0; +#else +#error This is supposed to be nested inside a check for HAVE_*_GETAUXVAL. +#endif // HAVE_STRONG_GETAUXVAL return (hwcap & (kHWCAP_PMULL | kHWCAP_CRC32)) == (kHWCAP_PMULL | kHWCAP_CRC32); #elif defined(__APPLE__) diff --git a/src/crypto/chacha_poly_aead.cpp b/src/crypto/chacha_poly_aead.cpp index 0582a60c4f..b73b22a2b8 100644 --- a/src/crypto/chacha_poly_aead.cpp +++ b/src/crypto/chacha_poly_aead.cpp @@ -31,8 +31,9 @@ ChaCha20Poly1305AEAD::ChaCha20Poly1305AEAD(const unsigned char* K_1, size_t K_1_ { assert(K_1_len == CHACHA20_POLY1305_AEAD_KEY_LEN); assert(K_2_len == CHACHA20_POLY1305_AEAD_KEY_LEN); - m_chacha_main.SetKey(K_1, CHACHA20_POLY1305_AEAD_KEY_LEN); - m_chacha_header.SetKey(K_2, CHACHA20_POLY1305_AEAD_KEY_LEN); + + m_chacha_header.SetKey(K_1, CHACHA20_POLY1305_AEAD_KEY_LEN); + m_chacha_main.SetKey(K_2, CHACHA20_POLY1305_AEAD_KEY_LEN); // set the cached sequence number to uint64 max which hints for an unset cache. // we can't hit uint64 max since the rekey rule (which resets the sequence number) is 1GB diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 3a1086bf4c..bcaf746167 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -197,13 +197,15 @@ bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) return true; } -size_t CDBWrapper::DynamicMemoryUsage() const { +size_t CDBWrapper::DynamicMemoryUsage() const +{ std::string memory; - if (!pdb->GetProperty("leveldb.approximate-memory-usage", &memory)) { + std::optional<size_t> parsed; + if (!pdb->GetProperty("leveldb.approximate-memory-usage", &memory) || !(parsed = ToIntegral<size_t>(memory))) { LogPrint(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n"); return 0; } - return stoul(memory); + return parsed.value(); } // Prefixed with null character to avoid collisions with other keys diff --git a/src/deploymentinfo.cpp b/src/deploymentinfo.cpp new file mode 100644 index 0000000000..030a7806de --- /dev/null +++ b/src/deploymentinfo.cpp @@ -0,0 +1,36 @@ +// Copyright (c) 2016-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <deploymentinfo.h> + +#include <consensus/params.h> + +const struct VBDeploymentInfo VersionBitsDeploymentInfo[Consensus::MAX_VERSION_BITS_DEPLOYMENTS] = { + { + /*.name =*/ "testdummy", + /*.gbt_force =*/ true, + }, + { + /*.name =*/ "taproot", + /*.gbt_force =*/ true, + }, +}; + +std::string DeploymentName(Consensus::BuriedDeployment dep) +{ + assert(ValidDeployment(dep)); + switch (dep) { + case Consensus::DEPLOYMENT_HEIGHTINCB: + return "bip34"; + case Consensus::DEPLOYMENT_CLTV: + return "bip65"; + case Consensus::DEPLOYMENT_DERSIG: + return "bip66"; + case Consensus::DEPLOYMENT_CSV: + return "csv"; + case Consensus::DEPLOYMENT_SEGWIT: + return "segwit"; + } // no default case, so the compiler can warn about missing cases + return ""; +} diff --git a/src/deploymentinfo.h b/src/deploymentinfo.h new file mode 100644 index 0000000000..63d58a7da2 --- /dev/null +++ b/src/deploymentinfo.h @@ -0,0 +1,29 @@ +// Copyright (c) 2016-2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_DEPLOYMENTINFO_H +#define BITCOIN_DEPLOYMENTINFO_H + +#include <consensus/params.h> + +#include <string> + +struct VBDeploymentInfo { + /** Deployment name */ + const char *name; + /** Whether GBT clients can safely ignore this rule in simplified usage */ + bool gbt_force; +}; + +extern const VBDeploymentInfo VersionBitsDeploymentInfo[Consensus::MAX_VERSION_BITS_DEPLOYMENTS]; + +std::string DeploymentName(Consensus::BuriedDeployment dep); + +inline std::string DeploymentName(Consensus::DeploymentPos pos) +{ + assert(Consensus::ValidDeployment(pos)); + return VersionBitsDeploymentInfo[pos].name; +} + +#endif // BITCOIN_DEPLOYMENTINFO_H diff --git a/src/deploymentstatus.cpp b/src/deploymentstatus.cpp new file mode 100644 index 0000000000..bba86639a3 --- /dev/null +++ b/src/deploymentstatus.cpp @@ -0,0 +1,34 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <deploymentstatus.h> + +#include <consensus/params.h> +#include <versionbits.h> + +#include <type_traits> + +VersionBitsCache g_versionbitscache; + +/* Basic sanity checking for BuriedDeployment/DeploymentPos enums and + * ValidDeployment check */ + +static_assert(ValidDeployment(Consensus::DEPLOYMENT_TESTDUMMY), "sanity check of DeploymentPos failed (TESTDUMMY not valid)"); +static_assert(!ValidDeployment(Consensus::MAX_VERSION_BITS_DEPLOYMENTS), "sanity check of DeploymentPos failed (MAX value considered valid)"); +static_assert(!ValidDeployment(static_cast<Consensus::BuriedDeployment>(Consensus::DEPLOYMENT_TESTDUMMY)), "sanity check of BuriedDeployment failed (overlaps with DeploymentPos)"); + +/* ValidDeployment only checks upper bounds for ensuring validity. + * This checks that the lowest possible value or the type is also a + * (specific) valid deployment so that lower bounds don't need to be checked. + */ + +template<typename T, T x> +static constexpr bool is_minimum() +{ + using U = typename std::underlying_type<T>::type; + return x == std::numeric_limits<U>::min(); +} + +static_assert(is_minimum<Consensus::BuriedDeployment, Consensus::DEPLOYMENT_HEIGHTINCB>(), "heightincb is not minimum value for BuriedDeployment"); +static_assert(is_minimum<Consensus::DeploymentPos, Consensus::DEPLOYMENT_TESTDUMMY>(), "testdummy is not minimum value for DeploymentPos"); diff --git a/src/deploymentstatus.h b/src/deploymentstatus.h new file mode 100644 index 0000000000..f95c5996f5 --- /dev/null +++ b/src/deploymentstatus.h @@ -0,0 +1,55 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_DEPLOYMENTSTATUS_H +#define BITCOIN_DEPLOYMENTSTATUS_H + +#include <chain.h> +#include <versionbits.h> + +#include <limits> + +/** Global cache for versionbits deployment status */ +extern VersionBitsCache g_versionbitscache; + +/** Determine if a deployment is active for the next block */ +inline bool DeploymentActiveAfter(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::BuriedDeployment dep) +{ + assert(Consensus::ValidDeployment(dep)); + return (pindexPrev == nullptr ? 0 : pindexPrev->nHeight + 1) >= params.DeploymentHeight(dep); +} + +inline bool DeploymentActiveAfter(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos dep) +{ + assert(Consensus::ValidDeployment(dep)); + return ThresholdState::ACTIVE == g_versionbitscache.State(pindexPrev, params, dep); +} + +/** Determine if a deployment is active for this block */ +inline bool DeploymentActiveAt(const CBlockIndex& index, const Consensus::Params& params, Consensus::BuriedDeployment dep) +{ + assert(Consensus::ValidDeployment(dep)); + return index.nHeight >= params.DeploymentHeight(dep); +} + +inline bool DeploymentActiveAt(const CBlockIndex& index, const Consensus::Params& params, Consensus::DeploymentPos dep) +{ + assert(Consensus::ValidDeployment(dep)); + return DeploymentActiveAfter(index.pprev, params, dep); +} + +/** Determine if a deployment is enabled (can ever be active) */ +inline bool DeploymentEnabled(const Consensus::Params& params, Consensus::BuriedDeployment dep) +{ + assert(Consensus::ValidDeployment(dep)); + return params.DeploymentHeight(dep) != std::numeric_limits<int>::max(); +} + +inline bool DeploymentEnabled(const Consensus::Params& params, Consensus::DeploymentPos dep) +{ + assert(Consensus::ValidDeployment(dep)); + return params.vDeployments[dep].nStartTime != Consensus::BIP9Deployment::NEVER_ACTIVE; +} + +#endif // BITCOIN_DEPLOYMENTSTATUS_H diff --git a/src/dummywallet.cpp b/src/dummywallet.cpp index 95886d3138..7f6471740f 100644 --- a/src/dummywallet.cpp +++ b/src/dummywallet.cpp @@ -5,12 +5,14 @@ #include <util/system.h> #include <walletinitinterface.h> +class ArgsManager; class CWallet; namespace interfaces { class Chain; class Handler; class Wallet; +class WalletClient; } class DummyWalletInit : public WalletInitInterface { @@ -28,6 +30,7 @@ void DummyWalletInit::AddWalletOptions(ArgsManager& argsman) const "-addresstype", "-avoidpartialspends", "-changetype", + "-consolidatefeerate=<amt>", "-disablewallet", "-discardfee=<amt>", "-fallbackfee=<amt>", @@ -36,8 +39,6 @@ void DummyWalletInit::AddWalletOptions(ArgsManager& argsman) const "-maxtxfee=<amt>", "-mintxfee=<amt>", "-paytxfee=<amt>", - "-rescan", - "-salvagewallet", "-signer=<cmd>", "-spendzeroconfchange", "-txconfirmtarget=<n>", @@ -63,4 +64,9 @@ std::unique_ptr<Wallet> MakeWallet(const std::shared_ptr<CWallet>& wallet) throw std::logic_error("Wallet function called in non-wallet build."); } +std::unique_ptr<WalletClient> MakeWalletClient(Chain& chain, ArgsManager& args) +{ + throw std::logic_error("Wallet function called in non-wallet build."); +} + } // namespace interfaces diff --git a/src/fs.cpp b/src/fs.cpp index 4f20ca4d28..b9b3c46d8d 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -154,7 +154,10 @@ std::string get_filesystem_error_message(const fs::filesystem_error& e) #ifdef __GLIBCXX__ // reference: https://github.com/gcc-mirror/gcc/blob/gcc-7_3_0-release/libstdc%2B%2B-v3/include/std/fstream#L270 - +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch" +#endif static std::string openmodeToStr(std::ios_base::openmode mode) { switch (mode & ~std::ios_base::ate) { @@ -192,6 +195,9 @@ static std::string openmodeToStr(std::ios_base::openmode mode) return std::string(); } } +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif void ifstream::open(const fs::path& p, std::ios_base::openmode mode) { @@ -242,7 +248,11 @@ void ofstream::close() } #else // __GLIBCXX__ +#if BOOST_VERSION >= 107700 +static_assert(sizeof(*BOOST_FILESYSTEM_C_STR(fs::path())) == sizeof(wchar_t), +#else static_assert(sizeof(*fs::path().BOOST_FILESYSTEM_C_STR) == sizeof(wchar_t), +#endif // BOOST_VERSION >= 107700 "Warning: This build is using boost::filesystem ofstream and ifstream " "implementations which will fail to open paths containing multibyte " "characters. You should delete this static_assert to ignore this warning, " diff --git a/src/hash.cpp b/src/hash.cpp index cc46043c2b..0e5bd975e4 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <hash.h> +#include <span.h> #include <crypto/common.h> #include <crypto/hmac_sha512.h> @@ -47,8 +48,10 @@ unsigned int MurmurHash3(unsigned int nHashSeed, Span<const unsigned char> vData switch (vDataToHash.size() & 3) { case 3: k1 ^= tail[2] << 16; + [[fallthrough]]; case 2: k1 ^= tail[1] << 8; + [[fallthrough]]; case 1: k1 ^= tail[0]; k1 *= c1; @@ -72,10 +75,7 @@ unsigned int MurmurHash3(unsigned int nHashSeed, Span<const unsigned char> vData void BIP32Hash(const ChainCode &chainCode, unsigned int nChild, unsigned char header, const unsigned char data[32], unsigned char output[64]) { unsigned char num[4]; - num[0] = (nChild >> 24) & 0xFF; - num[1] = (nChild >> 16) & 0xFF; - num[2] = (nChild >> 8) & 0xFF; - num[3] = (nChild >> 0) & 0xFF; + WriteBE32(num, nChild); CHMAC_SHA512(chainCode.begin(), chainCode.size()).Write(&header, 1).Write(data, 32).Write(num, 4).Finalize(output); } diff --git a/src/httprpc.cpp b/src/httprpc.cpp index e11e4acb5c..9ae592be79 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -10,6 +10,7 @@ #include <rpc/protocol.h> #include <rpc/server.h> #include <util/strencodings.h> +#include <util/string.h> #include <util/system.h> #include <util/translation.h> #include <walletinitinterface.h> @@ -22,7 +23,7 @@ #include <set> #include <string> -#include <boost/algorithm/string.hpp> // boost::trim +#include <boost/algorithm/string.hpp> /** WWW-Authenticate to present with 401 Unauthorized response */ static const char* WWW_AUTH_HEADER_DATA = "Basic realm=\"jsonrpc\""; @@ -130,8 +131,7 @@ static bool RPCAuthorized(const std::string& strAuth, std::string& strAuthUserna return false; if (strAuth.substr(0, 6) != "Basic ") return false; - std::string strUserPass64 = strAuth.substr(6); - boost::trim(strUserPass64); + std::string strUserPass64 = TrimString(strAuth.substr(6)); std::string strUserPass = DecodeBase64(strUserPass64); if (strUserPass.find(':') != std::string::npos) diff --git a/src/httpserver.cpp b/src/httpserver.cpp index b3984a43bb..6e75e28596 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -12,6 +12,7 @@ #include <shutdown.h> #include <sync.h> #include <util/strencodings.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <util/threadnames.h> #include <util/translation.h> @@ -136,7 +137,7 @@ static struct evhttp* eventHTTP = nullptr; //! List of subnets to allow RPC connections from static std::vector<CSubNet> rpc_allow_subnets; //! Work queue for handling longer requests off the event loop thread -static WorkQueue<HTTPClosure>* workQueue = nullptr; +static std::unique_ptr<WorkQueue<HTTPClosure>> g_work_queue{nullptr}; //! Handlers for (sub)paths static std::vector<HTTPPathHandler> pathHandlers; //! Bound listening sockets @@ -256,10 +257,10 @@ static void http_request_cb(struct evhttp_request* req, void* arg) // Dispatch to worker thread if (i != iend) { std::unique_ptr<HTTPWorkItem> item(new HTTPWorkItem(std::move(hreq), path, i->handler)); - assert(workQueue); - if (workQueue->Enqueue(item.get())) + assert(g_work_queue); + if (g_work_queue->Enqueue(item.get())) { item.release(); /* if true, queue took ownership */ - else { + } else { LogPrintf("WARNING: request rejected because http work queue depth exceeded, it can be increased with the -rpcworkqueue= setting\n"); item->req->WriteReply(HTTP_SERVICE_UNAVAILABLE, "Work queue depth exceeded"); } @@ -279,6 +280,7 @@ static void http_reject_request_cb(struct evhttp_request* req, void*) static bool ThreadHTTP(struct event_base* base) { util::ThreadRename("http"); + SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET_HTTP_SERVER); LogPrint(BCLog::HTTP, "Entering http event loop\n"); event_base_dispatch(base); // Event loop will be interrupted by InterruptHTTPServer() @@ -289,7 +291,7 @@ static bool ThreadHTTP(struct event_base* base) /** Bind HTTP server to specified addresses */ static bool HTTPBindAddresses(struct evhttp* http) { - uint16_t http_port{static_cast<uint16_t>(gArgs.GetArg("-rpcport", BaseParams().RPCPort()))}; + uint16_t http_port{static_cast<uint16_t>(gArgs.GetIntArg("-rpcport", BaseParams().RPCPort()))}; std::vector<std::pair<std::string, uint16_t>> endpoints; // Determine what addresses to bind to @@ -332,16 +334,13 @@ static bool HTTPBindAddresses(struct evhttp* http) static void HTTPWorkQueueRun(WorkQueue<HTTPClosure>* queue, int worker_num) { util::ThreadRename(strprintf("httpworker.%i", worker_num)); + SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET_HTTP_SERVER_WORKER); queue->Run(); } /** libevent event log callback */ static void libevent_log_cb(int severity, const char *msg) { -#ifndef EVENT_LOG_WARN -// EVENT_LOG_WARN was added in 2.0.19; but before then _EVENT_LOG_WARN existed. -# define EVENT_LOG_WARN _EVENT_LOG_WARN -#endif if (severity >= EVENT_LOG_WARN) // Log warn messages and higher without debug category LogPrintf("libevent: %s\n", msg); else @@ -378,7 +377,7 @@ bool InitHTTPServer() return false; } - evhttp_set_timeout(http, gArgs.GetArg("-rpcservertimeout", DEFAULT_HTTP_SERVER_TIMEOUT)); + evhttp_set_timeout(http, gArgs.GetIntArg("-rpcservertimeout", DEFAULT_HTTP_SERVER_TIMEOUT)); evhttp_set_max_headers_size(http, MAX_HEADERS_SIZE); evhttp_set_max_body_size(http, MAX_SIZE); evhttp_set_gencb(http, http_request_cb, nullptr); @@ -389,10 +388,10 @@ bool InitHTTPServer() } LogPrint(BCLog::HTTP, "Initialized HTTP server\n"); - int workQueueDepth = std::max((long)gArgs.GetArg("-rpcworkqueue", DEFAULT_HTTP_WORKQUEUE), 1L); + int workQueueDepth = std::max((long)gArgs.GetIntArg("-rpcworkqueue", DEFAULT_HTTP_WORKQUEUE), 1L); LogPrintf("HTTP: creating work queue of depth %d\n", workQueueDepth); - workQueue = new WorkQueue<HTTPClosure>(workQueueDepth); + g_work_queue = std::make_unique<WorkQueue<HTTPClosure>>(workQueueDepth); // transfer ownership to eventBase/HTTP via .release() eventBase = base_ctr.release(); eventHTTP = http_ctr.release(); @@ -419,12 +418,12 @@ static std::vector<std::thread> g_thread_http_workers; void StartHTTPServer() { LogPrint(BCLog::HTTP, "Starting HTTP server\n"); - int rpcThreads = std::max((long)gArgs.GetArg("-rpcthreads", DEFAULT_HTTP_THREADS), 1L); + int rpcThreads = std::max((long)gArgs.GetIntArg("-rpcthreads", DEFAULT_HTTP_THREADS), 1L); LogPrintf("HTTP: starting %d worker threads\n", rpcThreads); g_thread_http = std::thread(ThreadHTTP, eventBase); for (int i = 0; i < rpcThreads; i++) { - g_thread_http_workers.emplace_back(HTTPWorkQueueRun, workQueue, i); + g_thread_http_workers.emplace_back(HTTPWorkQueueRun, g_work_queue.get(), i); } } @@ -435,16 +434,17 @@ void InterruptHTTPServer() // Reject requests on current connections evhttp_set_gencb(eventHTTP, http_reject_request_cb, nullptr); } - if (workQueue) - workQueue->Interrupt(); + if (g_work_queue) { + g_work_queue->Interrupt(); + } } void StopHTTPServer() { LogPrint(BCLog::HTTP, "Stopping HTTP server\n"); - if (workQueue) { + if (g_work_queue) { LogPrint(BCLog::HTTP, "Waiting for HTTP worker threads to exit\n"); - for (auto& thread: g_thread_http_workers) { + for (auto& thread : g_thread_http_workers) { thread.join(); } g_thread_http_workers.clear(); @@ -467,10 +467,7 @@ void StopHTTPServer() event_base_free(eventBase); eventBase = nullptr; } - if (workQueue) { - delete workQueue; - workQueue = nullptr; - } + g_work_queue.reset(); LogPrint(BCLog::HTTP, "Stopped HTTP server\n"); } diff --git a/src/i2p.cpp b/src/i2p.cpp index 2ae164633b..5e7e42fb77 100644 --- a/src/i2p.cpp +++ b/src/i2p.cpp @@ -159,7 +159,7 @@ bool Session::Accept(Connection& conn) const std::string& peer_dest = conn.sock->RecvUntilTerminator('\n', MAX_WAIT_FOR_IO, *m_interrupt, MAX_MSG_SIZE); - conn.peer = CService(DestB64ToAddr(peer_dest), Params().GetDefaultPort()); + conn.peer = CService(DestB64ToAddr(peer_dest), I2P_SAM31_PORT); return true; } @@ -172,6 +172,13 @@ bool Session::Accept(Connection& conn) bool Session::Connect(const CService& to, Connection& conn, bool& proxy_error) { + // Refuse connecting to arbitrary ports. We don't specify any destination port to the SAM proxy + // when connecting (SAM 3.1 does not use ports) and it forces/defaults it to I2P_SAM31_PORT. + if (to.GetPort() != I2P_SAM31_PORT) { + proxy_error = false; + return false; + } + proxy_error = true; std::string session_id; @@ -366,7 +373,7 @@ void Session::CreateIfNotCreatedAlready() SendRequestAndGetReply(*sock, strprintf("SESSION CREATE STYLE=STREAM ID=%s DESTINATION=%s", session_id, private_key_b64)); - m_my_addr = CService(DestBinToAddr(MyDestination()), Params().GetDefaultPort()); + m_my_addr = CService(DestBinToAddr(MyDestination()), I2P_SAM31_PORT); m_session_id = session_id; m_control_sock = std::move(sock); diff --git a/src/index/base.cpp b/src/index/base.cpp index 6fd2701e2e..3ca86a310e 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -8,6 +8,7 @@ #include <node/ui_interface.h> #include <shutdown.h> #include <tinyformat.h> +#include <util/syscall_sandbox.h> #include <util/thread.h> #include <util/translation.h> #include <validation.h> // For g_chainman @@ -123,6 +124,7 @@ static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev, CChain& void BaseIndex::ThreadSync() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::TX_INDEX); const CBlockIndex* pindex = m_best_block_index.load(); if (!m_synced) { auto& consensus_params = Params().GetConsensus(); diff --git a/src/index/base.h b/src/index/base.h index df4bdff1ea..1390e3e570 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -6,11 +6,10 @@ #define BITCOIN_INDEX_BASE_H #include <dbwrapper.h> -#include <primitives/block.h> -#include <primitives/transaction.h> #include <threadinterrupt.h> #include <validationinterface.h> +class CBlock; class CBlockIndex; class CChainState; diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index 8c6046489b..9ab9209ca4 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -24,14 +24,14 @@ struct DBVal { uint64_t bogo_size; CAmount total_amount; CAmount total_subsidy; - CAmount block_unspendable_amount; - CAmount block_prevout_spent_amount; - CAmount block_new_outputs_ex_coinbase_amount; - CAmount block_coinbase_amount; - CAmount unspendables_genesis_block; - CAmount unspendables_bip30; - CAmount unspendables_scripts; - CAmount unspendables_unclaimed_rewards; + CAmount total_unspendable_amount; + CAmount total_prevout_spent_amount; + CAmount total_new_outputs_ex_coinbase_amount; + CAmount total_coinbase_amount; + CAmount total_unspendables_genesis_block; + CAmount total_unspendables_bip30; + CAmount total_unspendables_scripts; + CAmount total_unspendables_unclaimed_rewards; SERIALIZE_METHODS(DBVal, obj) { @@ -40,14 +40,14 @@ struct DBVal { READWRITE(obj.bogo_size); READWRITE(obj.total_amount); READWRITE(obj.total_subsidy); - READWRITE(obj.block_unspendable_amount); - READWRITE(obj.block_prevout_spent_amount); - READWRITE(obj.block_new_outputs_ex_coinbase_amount); - READWRITE(obj.block_coinbase_amount); - READWRITE(obj.unspendables_genesis_block); - READWRITE(obj.unspendables_bip30); - READWRITE(obj.unspendables_scripts); - READWRITE(obj.unspendables_unclaimed_rewards); + READWRITE(obj.total_unspendable_amount); + READWRITE(obj.total_prevout_spent_amount); + READWRITE(obj.total_new_outputs_ex_coinbase_amount); + READWRITE(obj.total_coinbase_amount); + READWRITE(obj.total_unspendables_genesis_block); + READWRITE(obj.total_unspendables_bip30); + READWRITE(obj.total_unspendables_scripts); + READWRITE(obj.total_unspendables_unclaimed_rewards); } }; @@ -122,9 +122,12 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) uint256 expected_block_hash{pindex->pprev->GetBlockHash()}; if (read_out.first != expected_block_hash) { + LogPrintf("WARNING: previous block header belongs to unexpected block %s; expected %s\n", + read_out.first.ToString(), expected_block_hash.ToString()); + if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) { - return error("%s: previous block header belongs to unexpected block %s; expected %s", - __func__, read_out.first.ToString(), expected_block_hash.ToString()); + return error("%s: previous block header not found; expected %s", + __func__, expected_block_hash.ToString()); } } @@ -138,29 +141,29 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) // Skip duplicate txid coinbase transactions (BIP30). if (is_bip30_block && tx->IsCoinBase()) { - m_block_unspendable_amount += block_subsidy; - m_unspendables_bip30 += block_subsidy; + m_total_unspendable_amount += block_subsidy; + m_total_unspendables_bip30 += block_subsidy; continue; } - for (size_t j = 0; j < tx->vout.size(); ++j) { + for (uint32_t j = 0; j < tx->vout.size(); ++j) { const CTxOut& out{tx->vout[j]}; Coin coin{out, pindex->nHeight, tx->IsCoinBase()}; - COutPoint outpoint{tx->GetHash(), static_cast<uint32_t>(j)}; + COutPoint outpoint{tx->GetHash(), j}; // Skip unspendable coins if (coin.out.scriptPubKey.IsUnspendable()) { - m_block_unspendable_amount += coin.out.nValue; - m_unspendables_scripts += coin.out.nValue; + m_total_unspendable_amount += coin.out.nValue; + m_total_unspendables_scripts += coin.out.nValue; continue; } m_muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin))); if (tx->IsCoinBase()) { - m_block_coinbase_amount += coin.out.nValue; + m_total_coinbase_amount += coin.out.nValue; } else { - m_block_new_outputs_ex_coinbase_amount += coin.out.nValue; + m_total_new_outputs_ex_coinbase_amount += coin.out.nValue; } ++m_transaction_output_count; @@ -178,7 +181,7 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) m_muhash.Remove(MakeUCharSpan(TxOutSer(outpoint, coin))); - m_block_prevout_spent_amount += coin.out.nValue; + m_total_prevout_spent_amount += coin.out.nValue; --m_transaction_output_count; m_total_amount -= coin.out.nValue; @@ -188,17 +191,17 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) } } else { // genesis block - m_block_unspendable_amount += block_subsidy; - m_unspendables_genesis_block += block_subsidy; + m_total_unspendable_amount += block_subsidy; + m_total_unspendables_genesis_block += block_subsidy; } // If spent prevouts + block subsidy are still a higher amount than // new outputs + coinbase + current unspendable amount this means // the miner did not claim the full block reward. Unclaimed block // rewards are also unspendable. - const CAmount unclaimed_rewards{(m_block_prevout_spent_amount + m_total_subsidy) - (m_block_new_outputs_ex_coinbase_amount + m_block_coinbase_amount + m_block_unspendable_amount)}; - m_block_unspendable_amount += unclaimed_rewards; - m_unspendables_unclaimed_rewards += unclaimed_rewards; + const CAmount unclaimed_rewards{(m_total_prevout_spent_amount + m_total_subsidy) - (m_total_new_outputs_ex_coinbase_amount + m_total_coinbase_amount + m_total_unspendable_amount)}; + m_total_unspendable_amount += unclaimed_rewards; + m_total_unspendables_unclaimed_rewards += unclaimed_rewards; std::pair<uint256, DBVal> value; value.first = pindex->GetBlockHash(); @@ -206,20 +209,23 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) value.second.bogo_size = m_bogo_size; value.second.total_amount = m_total_amount; value.second.total_subsidy = m_total_subsidy; - value.second.block_unspendable_amount = m_block_unspendable_amount; - value.second.block_prevout_spent_amount = m_block_prevout_spent_amount; - value.second.block_new_outputs_ex_coinbase_amount = m_block_new_outputs_ex_coinbase_amount; - value.second.block_coinbase_amount = m_block_coinbase_amount; - value.second.unspendables_genesis_block = m_unspendables_genesis_block; - value.second.unspendables_bip30 = m_unspendables_bip30; - value.second.unspendables_scripts = m_unspendables_scripts; - value.second.unspendables_unclaimed_rewards = m_unspendables_unclaimed_rewards; + value.second.total_unspendable_amount = m_total_unspendable_amount; + value.second.total_prevout_spent_amount = m_total_prevout_spent_amount; + value.second.total_new_outputs_ex_coinbase_amount = m_total_new_outputs_ex_coinbase_amount; + value.second.total_coinbase_amount = m_total_coinbase_amount; + value.second.total_unspendables_genesis_block = m_total_unspendables_genesis_block; + value.second.total_unspendables_bip30 = m_total_unspendables_bip30; + value.second.total_unspendables_scripts = m_total_unspendables_scripts; + value.second.total_unspendables_unclaimed_rewards = m_total_unspendables_unclaimed_rewards; uint256 out; m_muhash.Finalize(out); value.second.muhash = out; - return m_db->Write(DBHeightKey(pindex->nHeight), value) && m_db->Write(DB_MUHASH, m_muhash); + CDBBatch batch(*m_db); + batch.Write(DBHeightKey(pindex->nHeight), value); + batch.Write(DB_MUHASH, m_muhash); + return m_db->WriteBatch(batch); } static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch, @@ -317,14 +323,14 @@ bool CoinStatsIndex::LookUpStats(const CBlockIndex* block_index, CCoinsStats& co coins_stats.nBogoSize = entry.bogo_size; coins_stats.nTotalAmount = entry.total_amount; coins_stats.total_subsidy = entry.total_subsidy; - coins_stats.block_unspendable_amount = entry.block_unspendable_amount; - coins_stats.block_prevout_spent_amount = entry.block_prevout_spent_amount; - coins_stats.block_new_outputs_ex_coinbase_amount = entry.block_new_outputs_ex_coinbase_amount; - coins_stats.block_coinbase_amount = entry.block_coinbase_amount; - coins_stats.unspendables_genesis_block = entry.unspendables_genesis_block; - coins_stats.unspendables_bip30 = entry.unspendables_bip30; - coins_stats.unspendables_scripts = entry.unspendables_scripts; - coins_stats.unspendables_unclaimed_rewards = entry.unspendables_unclaimed_rewards; + coins_stats.total_unspendable_amount = entry.total_unspendable_amount; + coins_stats.total_prevout_spent_amount = entry.total_prevout_spent_amount; + coins_stats.total_new_outputs_ex_coinbase_amount = entry.total_new_outputs_ex_coinbase_amount; + coins_stats.total_coinbase_amount = entry.total_coinbase_amount; + coins_stats.total_unspendables_genesis_block = entry.total_unspendables_genesis_block; + coins_stats.total_unspendables_bip30 = entry.total_unspendables_bip30; + coins_stats.total_unspendables_scripts = entry.total_unspendables_scripts; + coins_stats.total_unspendables_unclaimed_rewards = entry.total_unspendables_unclaimed_rewards; return true; } @@ -341,33 +347,31 @@ bool CoinStatsIndex::Init() } } - if (BaseIndex::Init()) { - const CBlockIndex* pindex{CurrentIndex()}; + if (!BaseIndex::Init()) return false; - if (pindex) { - DBVal entry; - if (!LookUpOne(*m_db, pindex, entry)) { - return false; - } + const CBlockIndex* pindex{CurrentIndex()}; - m_transaction_output_count = entry.transaction_output_count; - m_bogo_size = entry.bogo_size; - m_total_amount = entry.total_amount; - m_total_subsidy = entry.total_subsidy; - m_block_unspendable_amount = entry.block_unspendable_amount; - m_block_prevout_spent_amount = entry.block_prevout_spent_amount; - m_block_new_outputs_ex_coinbase_amount = entry.block_new_outputs_ex_coinbase_amount; - m_block_coinbase_amount = entry.block_coinbase_amount; - m_unspendables_genesis_block = entry.unspendables_genesis_block; - m_unspendables_bip30 = entry.unspendables_bip30; - m_unspendables_scripts = entry.unspendables_scripts; - m_unspendables_unclaimed_rewards = entry.unspendables_unclaimed_rewards; + if (pindex) { + DBVal entry; + if (!LookUpOne(*m_db, pindex, entry)) { + return false; } - return true; + m_transaction_output_count = entry.transaction_output_count; + m_bogo_size = entry.bogo_size; + m_total_amount = entry.total_amount; + m_total_subsidy = entry.total_subsidy; + m_total_unspendable_amount = entry.total_unspendable_amount; + m_total_prevout_spent_amount = entry.total_prevout_spent_amount; + m_total_new_outputs_ex_coinbase_amount = entry.total_new_outputs_ex_coinbase_amount; + m_total_coinbase_amount = entry.total_coinbase_amount; + m_total_unspendables_genesis_block = entry.total_unspendables_genesis_block; + m_total_unspendables_bip30 = entry.total_unspendables_bip30; + m_total_unspendables_scripts = entry.total_unspendables_scripts; + m_total_unspendables_unclaimed_rewards = entry.total_unspendables_unclaimed_rewards; } - return false; + return true; } // Reverse a single block as part of a reorg @@ -391,9 +395,12 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex uint256 expected_block_hash{pindex->pprev->GetBlockHash()}; if (read_out.first != expected_block_hash) { + LogPrintf("WARNING: previous block header belongs to unexpected block %s; expected %s\n", + read_out.first.ToString(), expected_block_hash.ToString()); + if (!m_db->Read(DBHashKey(expected_block_hash), read_out)) { - return error("%s: previous block header belongs to unexpected block %s; expected %s", - __func__, read_out.first.ToString(), expected_block_hash.ToString()); + return error("%s: previous block header not found; expected %s", + __func__, expected_block_hash.ToString()); } } } @@ -402,24 +409,24 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex for (size_t i = 0; i < block.vtx.size(); ++i) { const auto& tx{block.vtx.at(i)}; - for (size_t j = 0; j < tx->vout.size(); ++j) { + for (uint32_t j = 0; j < tx->vout.size(); ++j) { const CTxOut& out{tx->vout[j]}; - COutPoint outpoint{tx->GetHash(), static_cast<uint32_t>(j)}; + COutPoint outpoint{tx->GetHash(), j}; Coin coin{out, pindex->nHeight, tx->IsCoinBase()}; // Skip unspendable coins if (coin.out.scriptPubKey.IsUnspendable()) { - m_block_unspendable_amount -= coin.out.nValue; - m_unspendables_scripts -= coin.out.nValue; + m_total_unspendable_amount -= coin.out.nValue; + m_total_unspendables_scripts -= coin.out.nValue; continue; } m_muhash.Remove(MakeUCharSpan(TxOutSer(outpoint, coin))); if (tx->IsCoinBase()) { - m_block_coinbase_amount -= coin.out.nValue; + m_total_coinbase_amount -= coin.out.nValue; } else { - m_block_new_outputs_ex_coinbase_amount -= coin.out.nValue; + m_total_new_outputs_ex_coinbase_amount -= coin.out.nValue; } --m_transaction_output_count; @@ -437,7 +444,7 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex m_muhash.Insert(MakeUCharSpan(TxOutSer(outpoint, coin))); - m_block_prevout_spent_amount -= coin.out.nValue; + m_total_prevout_spent_amount -= coin.out.nValue; m_transaction_output_count++; m_total_amount += coin.out.nValue; @@ -446,9 +453,9 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex } } - const CAmount unclaimed_rewards{(m_block_new_outputs_ex_coinbase_amount + m_block_coinbase_amount + m_block_unspendable_amount) - (m_block_prevout_spent_amount + m_total_subsidy)}; - m_block_unspendable_amount -= unclaimed_rewards; - m_unspendables_unclaimed_rewards -= unclaimed_rewards; + const CAmount unclaimed_rewards{(m_total_new_outputs_ex_coinbase_amount + m_total_coinbase_amount + m_total_unspendable_amount) - (m_total_prevout_spent_amount + m_total_subsidy)}; + m_total_unspendable_amount -= unclaimed_rewards; + m_total_unspendables_unclaimed_rewards -= unclaimed_rewards; // Check that the rolled back internal values are consistent with the DB read out uint256 out; @@ -459,14 +466,14 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex Assert(m_total_amount == read_out.second.total_amount); Assert(m_bogo_size == read_out.second.bogo_size); Assert(m_total_subsidy == read_out.second.total_subsidy); - Assert(m_block_unspendable_amount == read_out.second.block_unspendable_amount); - Assert(m_block_prevout_spent_amount == read_out.second.block_prevout_spent_amount); - Assert(m_block_new_outputs_ex_coinbase_amount == read_out.second.block_new_outputs_ex_coinbase_amount); - Assert(m_block_coinbase_amount == read_out.second.block_coinbase_amount); - Assert(m_unspendables_genesis_block == read_out.second.unspendables_genesis_block); - Assert(m_unspendables_bip30 == read_out.second.unspendables_bip30); - Assert(m_unspendables_scripts == read_out.second.unspendables_scripts); - Assert(m_unspendables_unclaimed_rewards == read_out.second.unspendables_unclaimed_rewards); + Assert(m_total_unspendable_amount == read_out.second.total_unspendable_amount); + Assert(m_total_prevout_spent_amount == read_out.second.total_prevout_spent_amount); + Assert(m_total_new_outputs_ex_coinbase_amount == read_out.second.total_new_outputs_ex_coinbase_amount); + Assert(m_total_coinbase_amount == read_out.second.total_coinbase_amount); + Assert(m_total_unspendables_genesis_block == read_out.second.total_unspendables_genesis_block); + Assert(m_total_unspendables_bip30 == read_out.second.total_unspendables_bip30); + Assert(m_total_unspendables_scripts == read_out.second.total_unspendables_scripts); + Assert(m_total_unspendables_unclaimed_rewards == read_out.second.total_unspendables_unclaimed_rewards); return m_db->Write(DB_MUHASH, m_muhash); } diff --git a/src/index/coinstatsindex.h b/src/index/coinstatsindex.h index 6149f9b4b3..a575b37c7c 100644 --- a/src/index/coinstatsindex.h +++ b/src/index/coinstatsindex.h @@ -25,14 +25,14 @@ private: uint64_t m_bogo_size{0}; CAmount m_total_amount{0}; CAmount m_total_subsidy{0}; - CAmount m_block_unspendable_amount{0}; - CAmount m_block_prevout_spent_amount{0}; - CAmount m_block_new_outputs_ex_coinbase_amount{0}; - CAmount m_block_coinbase_amount{0}; - CAmount m_unspendables_genesis_block{0}; - CAmount m_unspendables_bip30{0}; - CAmount m_unspendables_scripts{0}; - CAmount m_unspendables_unclaimed_rewards{0}; + CAmount m_total_unspendable_amount{0}; + CAmount m_total_prevout_spent_amount{0}; + CAmount m_total_new_outputs_ex_coinbase_amount{0}; + CAmount m_total_coinbase_amount{0}; + CAmount m_total_unspendables_genesis_block{0}; + CAmount m_total_unspendables_bip30{0}; + CAmount m_total_unspendables_scripts{0}; + CAmount m_total_unspendables_unclaimed_rewards{0}; bool ReverseBlock(const CBlock& block, const CBlockIndex* pindex); diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 782e557478..209785d487 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -2,18 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <index/disktxpos.h> #include <index/txindex.h> + +#include <index/disktxpos.h> #include <node/blockstorage.h> -#include <node/ui_interface.h> -#include <shutdown.h> #include <util/system.h> -#include <util/translation.h> #include <validation.h> -constexpr uint8_t DB_BEST_BLOCK{'B'}; constexpr uint8_t DB_TXINDEX{'t'}; -constexpr uint8_t DB_TXINDEX_BLOCK{'T'}; std::unique_ptr<TxIndex> g_txindex; @@ -30,10 +26,6 @@ public: /// Write a batch of transaction positions to the DB. bool WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos); - - /// Migrate txindex data from the block tree DB, where it may be for older nodes that have not - /// been upgraded yet to the new database. - bool MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator); }; TxIndex::DB::DB(size_t n_cache_size, bool f_memory, bool f_wipe) : @@ -54,163 +46,12 @@ bool TxIndex::DB::WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_ return WriteBatch(batch); } -/* - * Safely persist a transfer of data from the old txindex database to the new one, and compact the - * range of keys updated. This is used internally by MigrateData. - */ -static void WriteTxIndexMigrationBatches(CDBWrapper& newdb, CDBWrapper& olddb, - CDBBatch& batch_newdb, CDBBatch& batch_olddb, - const std::pair<uint8_t, uint256>& begin_key, - const std::pair<uint8_t, uint256>& end_key) -{ - // Sync new DB changes to disk before deleting from old DB. - newdb.WriteBatch(batch_newdb, /*fSync=*/ true); - olddb.WriteBatch(batch_olddb); - olddb.CompactRange(begin_key, end_key); - - batch_newdb.Clear(); - batch_olddb.Clear(); -} - -bool TxIndex::DB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator) -{ - // The prior implementation of txindex was always in sync with block index - // and presence was indicated with a boolean DB flag. If the flag is set, - // this means the txindex from a previous version is valid and in sync with - // the chain tip. The first step of the migration is to unset the flag and - // write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the - // index entries are copied over in batches to the new database. Finally, - // DB_TXINDEX_BLOCK is erased from the old database and the block hash is - // written to the new database. - // - // Unsetting the boolean flag ensures that if the node is downgraded to a - // previous version, it will not see a corrupted, partially migrated index - // -- it will see that the txindex is disabled. When the node is upgraded - // again, the migration will pick up where it left off and sync to the block - // with hash DB_TXINDEX_BLOCK. - bool f_legacy_flag = false; - block_tree_db.ReadFlag("txindex", f_legacy_flag); - if (f_legacy_flag) { - if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) { - return error("%s: cannot write block indicator", __func__); - } - if (!block_tree_db.WriteFlag("txindex", false)) { - return error("%s: cannot write block index db flag", __func__); - } - } - - CBlockLocator locator; - if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) { - return true; - } - - int64_t count = 0; - LogPrintf("Upgrading txindex database... [0%%]\n"); - uiInterface.ShowProgress(_("Upgrading txindex database").translated, 0, true); - int report_done = 0; - const size_t batch_size = 1 << 24; // 16 MiB - - CDBBatch batch_newdb(*this); - CDBBatch batch_olddb(block_tree_db); - - std::pair<uint8_t, uint256> key; - std::pair<uint8_t, uint256> begin_key{DB_TXINDEX, uint256()}; - std::pair<uint8_t, uint256> prev_key = begin_key; - - bool interrupted = false; - std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator()); - for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) { - if (ShutdownRequested()) { - interrupted = true; - break; - } - - if (!cursor->GetKey(key)) { - return error("%s: cannot get key from valid cursor", __func__); - } - if (key.first != DB_TXINDEX) { - break; - } - - // Log progress every 10%. - if (++count % 256 == 0) { - // Since txids are uniformly random and traversed in increasing order, the high 16 bits - // of the hash can be used to estimate the current progress. - const uint256& txid = key.second; - uint32_t high_nibble = - (static_cast<uint32_t>(*(txid.begin() + 0)) << 8) + - (static_cast<uint32_t>(*(txid.begin() + 1)) << 0); - int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5); - - uiInterface.ShowProgress(_("Upgrading txindex database").translated, percentage_done, true); - if (report_done < percentage_done/10) { - LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done); - report_done = percentage_done/10; - } - } - - CDiskTxPos value; - if (!cursor->GetValue(value)) { - return error("%s: cannot parse txindex record", __func__); - } - batch_newdb.Write(key, value); - batch_olddb.Erase(key); - - if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) { - // NOTE: it's OK to delete the key pointed at by the current DB cursor while iterating - // because LevelDB iterators are guaranteed to provide a consistent view of the - // underlying data, like a lightweight snapshot. - WriteTxIndexMigrationBatches(*this, block_tree_db, - batch_newdb, batch_olddb, - prev_key, key); - prev_key = key; - } - } - - // If these final DB batches complete the migration, write the best block - // hash marker to the new database and delete from the old one. This signals - // that the former is fully caught up to that point in the blockchain and - // that all txindex entries have been removed from the latter. - if (!interrupted) { - batch_olddb.Erase(DB_TXINDEX_BLOCK); - batch_newdb.Write(DB_BEST_BLOCK, locator); - } - - WriteTxIndexMigrationBatches(*this, block_tree_db, - batch_newdb, batch_olddb, - begin_key, key); - - if (interrupted) { - LogPrintf("[CANCELLED].\n"); - return false; - } - - uiInterface.ShowProgress("", 100, false); - - LogPrintf("[DONE].\n"); - return true; -} - TxIndex::TxIndex(size_t n_cache_size, bool f_memory, bool f_wipe) : m_db(std::make_unique<TxIndex::DB>(n_cache_size, f_memory, f_wipe)) {} TxIndex::~TxIndex() {} -bool TxIndex::Init() -{ - LOCK(cs_main); - - // Attempt to migrate txindex from the old database to the new one. Even if - // chain_tip is null, the node could be reindexing and we still want to - // delete txindex records in the old database. - if (!m_db->MigrateData(*pblocktree, m_chainstate->m_chain.GetLocator())) { - return false; - } - - return BaseIndex::Init(); -} - bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) { // Exclude genesis block transaction because outputs are not spendable. diff --git a/src/index/txindex.h b/src/index/txindex.h index 8202c3c951..59375bc204 100644 --- a/src/index/txindex.h +++ b/src/index/txindex.h @@ -5,9 +5,7 @@ #ifndef BITCOIN_INDEX_TXINDEX_H #define BITCOIN_INDEX_TXINDEX_H -#include <chain.h> #include <index/base.h> -#include <txdb.h> /** * TxIndex is used to look up transactions included in the blockchain by hash. @@ -23,9 +21,6 @@ private: const std::unique_ptr<DB> m_db; protected: - /// Override base class init to migrate from old database. - bool Init() override; - bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; BaseIndex::DB& GetDB() const override; diff --git a/src/init.cpp b/src/init.cpp index da0447ca79..4495ded64d 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -10,12 +10,13 @@ #include <init.h> #include <addrman.h> -#include <amount.h> #include <banman.h> #include <blockfilter.h> #include <chain.h> #include <chainparams.h> #include <compat/sanity.h> +#include <consensus/amount.h> +#include <deploymentstatus.h> #include <fs.h> #include <hash.h> #include <httprpc.h> @@ -25,6 +26,7 @@ #include <index/txindex.h> #include <init/common.h> #include <interfaces/chain.h> +#include <interfaces/init.h> #include <interfaces/node.h> #include <mapport.h> #include <miner.h> @@ -58,6 +60,7 @@ #include <util/check.h> #include <util/moneystr.h> #include <util/string.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <util/thread.h> #include <util/threadnames.h> @@ -216,7 +219,7 @@ void Shutdown(NodeContext& node) node.banman.reset(); node.addrman.reset(); - if (node.mempool && node.mempool->IsLoaded() && node.args->GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { + if (node.mempool && node.mempool->IsLoaded() && node.args->GetBoolArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { DumpMempool(*node.mempool); } @@ -263,7 +266,6 @@ void Shutdown(NodeContext& node) chainstate->ResetCoinsViews(); } } - pblocktree.reset(); } for (const auto& client : node.chain_clients) { client->stop(); @@ -396,7 +398,7 @@ void SetupServerArgs(ArgsManager& argsman) -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-persistmempool", strprintf("Whether to save the mempool on shutdown and load on restart (default: %u)", DEFAULT_PERSIST_MEMPOOL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-pid=<file>", strprintf("Specify pid file. Relative paths will be prefixed by a net-specific datadir location. (default: %s)", BITCOIN_PID_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex, -coinstatsindex and -rescan. " + argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex and -coinstatsindex. " "Warning: Reverting this setting requires re-downloading the entire blockchain. " "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >=%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-reindex", "Rebuild chain state and block index from the blk*.dat files on disk", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -426,7 +428,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-dnsseed", strprintf("Query for peer addresses via DNS lookup, if low on addresses (default: %u unless -connect used)", DEFAULT_DNSSEED), ArgsManager::ALLOW_BOOL, OptionsCategory::CONNECTION); argsman.AddArg("-externalip=<ip>", "Specify your own public address", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-fixedseeds", strprintf("Allow fixed seeds if DNS seeds don't provide peers (default: %u)", DEFAULT_FIXEDSEEDS), ArgsManager::ALLOW_BOOL, OptionsCategory::CONNECTION); - argsman.AddArg("-forcednsseed", strprintf("Always query for peer addresses via DNS lookup (default: %u)", DEFAULT_FORCEDNSSEED), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-forcednsseed", strprintf("Always query for peer addresses via DNS lookup (default: %u)", DEFAULT_FORCEDNSSEED), ArgsManager::ALLOW_BOOL, OptionsCategory::CONNECTION); argsman.AddArg("-listen", "Accept connections from outside (default: 1 if no -proxy or -connect)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-listenonion", strprintf("Automatically create Tor onion service (default: %d)", DEFAULT_LISTEN_ONION), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxconnections=<n>", strprintf("Maintain at most <n> connections to peers (default: %u). This limit does not apply to connections manually added via -addnode or the addnode RPC, which have a separate limit of %u.", DEFAULT_MAX_PEER_CONNECTIONS, MAX_ADDNODE_CONNECTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -441,7 +443,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-peerbloomfilters", strprintf("Support filtering of blocks and transaction with bloom filters (default: %u)", DEFAULT_PEERBLOOMFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerblockfilters", strprintf("Serve compact block filters to peers per BIP 157 (default: %u)", DEFAULT_PEERBLOCKFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-permitbaremultisig", strprintf("Relay non-P2SH multisig (default: %u)", DEFAULT_PERMIT_BAREMULTISIG), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-port=<port>", strprintf("Listen for connections on <port>. Nodes not using the default ports (default: %u, testnet: %u, signet: %u, regtest: %u) are unlikely to get incoming connections.", defaultChainParams->GetDefaultPort(), testnetChainParams->GetDefaultPort(), signetChainParams->GetDefaultPort(), regtestChainParams->GetDefaultPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); + argsman.AddArg("-port=<port>", strprintf("Listen for connections on <port>. Nodes not using the default ports (default: %u, testnet: %u, signet: %u, regtest: %u) are unlikely to get incoming connections. Not relevant for I2P (see doc/i2p.md).", defaultChainParams->GetDefaultPort(), testnetChainParams->GetDefaultPort(), signetChainParams->GetDefaultPort(), regtestChainParams->GetDefaultPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-proxy=<ip:port>", "Connect through SOCKS5 proxy, set -noproxy to disable (default: disabled)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-proxyrandomize", strprintf("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)", DEFAULT_PROXYRANDOMIZE), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-seednode=<ip>", "Connect to a node to retrieve peer addresses, and disconnect. This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -501,7 +503,8 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-checkblocks=<n>", strprintf("How many blocks to check at startup (default: %u, 0 = all)", DEFAULT_CHECKBLOCKS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checklevel=<n>", strprintf("How thorough the block verification of -checkblocks is: %s (0-4, default: %u)", Join(CHECKLEVEL_DOC, ", "), DEFAULT_CHECKLEVEL), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checkblockindex", strprintf("Do a consistency check for the block tree, chainstate, and other validation data structures occasionally. (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-checkmempool=<n>", strprintf("Run checks every <n> transactions (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-checkaddrman=<n>", strprintf("Run addrman consistency checks every <n> operations. Use 0 to disable. (default: %u)", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-checkmempool=<n>", strprintf("Run mempool consistency checks every <n> transactions. Use 0 to disable. (default: %u, regtest: %u)", defaultChainParams->DefaultConsistencyChecks(), regtestChainParams->DefaultConsistencyChecks()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-checkpoints", strprintf("Enable rejection of any forks from the known historical chain until block %s (default: %u)", defaultChainParams->Checkpoints().GetHeight(), DEFAULT_CHECKPOINTS_ENABLED), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-deprecatedrpc=<method>", "Allows deprecated RPC method(s) to be used", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-stopafterblockimport", strprintf("Stop running after importing blocks from disk (default: %u)", DEFAULT_STOPAFTERBLOCKIMPORT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); @@ -560,6 +563,10 @@ void SetupServerArgs(ArgsManager& argsman) hidden_args.emplace_back("-daemonwait"); #endif +#if defined(USE_SYSCALL_SANDBOX) + argsman.AddArg("-sandbox=<mode>", "Use the experimental syscall sandbox in the specified mode (-sandbox=log-and-abort or -sandbox=abort). Allow only expected syscalls to be used by bitcoind. Note that this is an experimental new feature that may cause bitcoind to exit or crash unexpectedly: use with caution. In the \"log-and-abort\" mode the invocation of an unexpected syscall results in a debug handler being invoked which will log the incident and terminate the program (without executing the unexpected syscall). In the \"abort\" mode the invocation of an unexpected syscall results in the entire process being killed immediately by the kernel without executing the unexpected syscall.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); +#endif // USE_SYSCALL_SANDBOX + // Add the hidden options argsman.AddHiddenArgs(hidden_args); } @@ -715,7 +722,7 @@ namespace { // Variables internal to initialization process only int nMaxConnections; int nUserMaxConnections; int nFD; -ServiceFlags nLocalServices = ServiceFlags(NODE_NETWORK | NODE_NETWORK_LIMITED); +ServiceFlags nLocalServices = ServiceFlags(NODE_NETWORK | NODE_NETWORK_LIMITED | NODE_WITNESS); int64_t peer_connect_timeout; std::set<BlockFilterType> g_enabled_filter_types; @@ -841,26 +848,34 @@ bool AppInitParameterInteraction(const ArgsManager& args) } // if using block pruning, then disallow txindex and coinstatsindex - if (args.GetArg("-prune", 0)) { + if (args.GetIntArg("-prune", 0)) { if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) return InitError(_("Prune mode is incompatible with -txindex.")); if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) return InitError(_("Prune mode is incompatible with -coinstatsindex.")); } + // If -forcednsseed is set to true, ensure -dnsseed has not been set to false + if (args.GetBoolArg("-forcednsseed", DEFAULT_FORCEDNSSEED) && !args.GetBoolArg("-dnsseed", DEFAULT_DNSSEED)){ + return InitError(_("Cannot set -forcednsseed to true when setting -dnsseed to false.")); + } + // -bind and -whitebind can't be set when not listening size_t nUserBind = args.GetArgs("-bind").size() + args.GetArgs("-whitebind").size(); if (nUserBind != 0 && !args.GetBoolArg("-listen", DEFAULT_LISTEN)) { return InitError(Untranslated("Cannot set -bind or -whitebind together with -listen=0")); } + // if listen=0, then disallow listenonion=1 + if (!args.GetBoolArg("-listen", DEFAULT_LISTEN) && args.GetBoolArg("-listenonion", DEFAULT_LISTEN_ONION)) { + return InitError(Untranslated("Cannot set -listen=0 together with -listenonion=1")); + } + // Make sure enough file descriptors are available int nBind = std::max(nUserBind, size_t(1)); - nUserMaxConnections = args.GetArg("-maxconnections", DEFAULT_MAX_PEER_CONNECTIONS); + nUserMaxConnections = args.GetIntArg("-maxconnections", DEFAULT_MAX_PEER_CONNECTIONS); nMaxConnections = std::max(nUserMaxConnections, 0); - // Trim requested connection counts, to fit into system limitations - // <int> in std::min<int>(...) to work around FreeBSD compilation issue described in #2695 nFD = RaiseFileDescriptorLimit(nMaxConnections + MIN_CORE_FILEDESCRIPTORS + MAX_ADDNODE_CONNECTIONS + nBind + NUM_FDS_MESSAGE_CAPTURE); #ifdef USE_POLL @@ -868,6 +883,8 @@ bool AppInitParameterInteraction(const ArgsManager& args) #else int fd_max = FD_SETSIZE; #endif + // Trim requested connection counts, to fit into system limitations + // <int> in std::min<int>(...) to work around FreeBSD compilation issue described in #2695 nMaxConnections = std::max(std::min<int>(nMaxConnections, fd_max - nBind - MIN_CORE_FILEDESCRIPTORS - MAX_ADDNODE_CONNECTIONS - NUM_FDS_MESSAGE_CAPTURE), 0); if (nFD < MIN_CORE_FILEDESCRIPTORS) return InitError(_("Not enough file descriptors available.")); @@ -903,21 +920,22 @@ bool AppInitParameterInteraction(const ArgsManager& args) } // mempool limits - int64_t nMempoolSizeMax = args.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; - int64_t nMempoolSizeMin = args.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000 * 40; + int64_t nMempoolSizeMax = args.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; + int64_t nMempoolSizeMin = args.GetIntArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000 * 40; if (nMempoolSizeMax < 0 || nMempoolSizeMax < nMempoolSizeMin) return InitError(strprintf(_("-maxmempool must be at least %d MB"), std::ceil(nMempoolSizeMin / 1000000.0))); // incremental relay fee sets the minimum feerate increase necessary for BIP 125 replacement in the mempool // and the amount the mempool min fee increases above the feerate of txs evicted due to mempool limiting. if (args.IsArgSet("-incrementalrelayfee")) { - CAmount n = 0; - if (!ParseMoney(args.GetArg("-incrementalrelayfee", ""), n)) + if (std::optional<CAmount> inc_relay_fee = ParseMoney(args.GetArg("-incrementalrelayfee", ""))) { + ::incrementalRelayFee = CFeeRate{inc_relay_fee.value()}; + } else { return InitError(AmountErrMsg("incrementalrelayfee", args.GetArg("-incrementalrelayfee", ""))); - incrementalRelayFee = CFeeRate(n); + } } // block pruning; get the amount of disk space (in MiB) to allot for block & undo files - int64_t nPruneArg = args.GetArg("-prune", 0); + int64_t nPruneArg = args.GetIntArg("-prune", 0); if (nPruneArg < 0) { return InitError(_("Prune cannot be configured with a negative value.")); } @@ -934,23 +952,23 @@ bool AppInitParameterInteraction(const ArgsManager& args) fPruneMode = true; } - nConnectTimeout = args.GetArg("-timeout", DEFAULT_CONNECT_TIMEOUT); + nConnectTimeout = args.GetIntArg("-timeout", DEFAULT_CONNECT_TIMEOUT); if (nConnectTimeout <= 0) { nConnectTimeout = DEFAULT_CONNECT_TIMEOUT; } - peer_connect_timeout = args.GetArg("-peertimeout", DEFAULT_PEER_CONNECT_TIMEOUT); + peer_connect_timeout = args.GetIntArg("-peertimeout", DEFAULT_PEER_CONNECT_TIMEOUT); if (peer_connect_timeout <= 0) { return InitError(Untranslated("peertimeout cannot be configured with a negative value.")); } if (args.IsArgSet("-minrelaytxfee")) { - CAmount n = 0; - if (!ParseMoney(args.GetArg("-minrelaytxfee", ""), n)) { + if (std::optional<CAmount> min_relay_fee = ParseMoney(args.GetArg("-minrelaytxfee", ""))) { + // High fee check is done afterward in CWallet::Create() + ::minRelayTxFee = CFeeRate{min_relay_fee.value()}; + } else { return InitError(AmountErrMsg("minrelaytxfee", args.GetArg("-minrelaytxfee", ""))); } - // High fee check is done afterward in CWallet::Create() - ::minRelayTxFee = CFeeRate(n); } else if (incrementalRelayFee > ::minRelayTxFee) { // Allow only setting incrementalRelayFee to control both ::minRelayTxFee = incrementalRelayFee; @@ -960,50 +978,82 @@ bool AppInitParameterInteraction(const ArgsManager& args) // Sanity check argument for min fee for including tx in block // TODO: Harmonize which arguments need sanity checking and where that happens if (args.IsArgSet("-blockmintxfee")) { - CAmount n = 0; - if (!ParseMoney(args.GetArg("-blockmintxfee", ""), n)) + if (!ParseMoney(args.GetArg("-blockmintxfee", ""))) { return InitError(AmountErrMsg("blockmintxfee", args.GetArg("-blockmintxfee", ""))); + } } // Feerate used to define dust. Shouldn't be changed lightly as old // implementations may inadvertently create non-standard transactions if (args.IsArgSet("-dustrelayfee")) { - CAmount n = 0; - if (!ParseMoney(args.GetArg("-dustrelayfee", ""), n)) + if (std::optional<CAmount> parsed = ParseMoney(args.GetArg("-dustrelayfee", ""))) { + dustRelayFee = CFeeRate{parsed.value()}; + } else { return InitError(AmountErrMsg("dustrelayfee", args.GetArg("-dustrelayfee", ""))); - dustRelayFee = CFeeRate(n); + } } fRequireStandard = !args.GetBoolArg("-acceptnonstdtxn", !chainparams.RequireStandard()); if (!chainparams.IsTestChain() && !fRequireStandard) { return InitError(strprintf(Untranslated("acceptnonstdtxn is not currently supported for %s chain"), chainparams.NetworkIDString())); } - nBytesPerSigOp = args.GetArg("-bytespersigop", nBytesPerSigOp); + nBytesPerSigOp = args.GetIntArg("-bytespersigop", nBytesPerSigOp); if (!g_wallet_init_interface.ParameterInteraction()) return false; fIsBareMultisigStd = args.GetBoolArg("-permitbaremultisig", DEFAULT_PERMIT_BAREMULTISIG); fAcceptDatacarrier = args.GetBoolArg("-datacarrier", DEFAULT_ACCEPT_DATACARRIER); - nMaxDatacarrierBytes = args.GetArg("-datacarriersize", nMaxDatacarrierBytes); + nMaxDatacarrierBytes = args.GetIntArg("-datacarriersize", nMaxDatacarrierBytes); // Option to startup with mocktime set (used for regression testing): - SetMockTime(args.GetArg("-mocktime", 0)); // SetMockTime(0) is a no-op + SetMockTime(args.GetIntArg("-mocktime", 0)); // SetMockTime(0) is a no-op if (args.GetBoolArg("-peerbloomfilters", DEFAULT_PEERBLOOMFILTERS)) nLocalServices = ServiceFlags(nLocalServices | NODE_BLOOM); - if (args.GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) < 0) + if (args.GetIntArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) < 0) return InitError(Untranslated("rpcserialversion must be non-negative.")); - if (args.GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) > 1) + if (args.GetIntArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) > 1) return InitError(Untranslated("Unknown rpcserialversion requested.")); - nMaxTipAge = args.GetArg("-maxtipage", DEFAULT_MAX_TIP_AGE); + nMaxTipAge = args.GetIntArg("-maxtipage", DEFAULT_MAX_TIP_AGE); if (args.IsArgSet("-proxy") && args.GetArg("-proxy", "").empty()) { return InitError(_("No proxy server specified. Use -proxy=<ip> or -proxy=<ip:port>.")); } +#if defined(USE_SYSCALL_SANDBOX) + if (args.IsArgSet("-sandbox") && !args.IsArgNegated("-sandbox")) { + const std::string sandbox_arg{args.GetArg("-sandbox", "")}; + bool log_syscall_violation_before_terminating{false}; + if (sandbox_arg == "log-and-abort") { + log_syscall_violation_before_terminating = true; + } else if (sandbox_arg == "abort") { + // log_syscall_violation_before_terminating is false by default. + } else { + return InitError(Untranslated("Unknown syscall sandbox mode (-sandbox=<mode>). Available modes are \"log-and-abort\" and \"abort\".")); + } + // execve(...) is not allowed by the syscall sandbox. + const std::vector<std::string> features_using_execve{ + "-alertnotify", + "-blocknotify", + "-signer", + "-startupnotify", + "-walletnotify", + }; + for (const std::string& feature_using_execve : features_using_execve) { + if (!args.GetArg(feature_using_execve, "").empty()) { + return InitError(Untranslated(strprintf("The experimental syscall sandbox feature (-sandbox=<mode>) is incompatible with %s (which uses execve).", feature_using_execve))); + } + } + if (!SetupSyscallSandbox(log_syscall_violation_before_terminating)) { + return InitError(Untranslated("Installation of the syscall sandbox failed.")); + } + LogPrintf("Experimental syscall sandbox enabled (-sandbox=%s): bitcoind will terminate if an unexpected (not allowlisted) syscall is invoked.\n", sandbox_arg); + } +#endif // USE_SYSCALL_SANDBOX + return true; } @@ -1050,7 +1100,7 @@ bool AppInitLockDataDirectory() bool AppInitInterfaces(NodeContext& node) { - node.chain = interfaces::MakeChain(node); + node.chain = node.init->makeChain(); // Create client interfaces for wallets that are supposed to be loaded // according to -wallet and -disablewallet options. This only constructs // the interfaces, it doesn't load wallet data. Wallets actually get loaded @@ -1087,7 +1137,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) InitSignatureCache(); InitScriptExecutionCache(); - int script_threads = args.GetArg("-par", DEFAULT_SCRIPTCHECK_THREADS); + int script_threads = args.GetIntArg("-par", DEFAULT_SCRIPTCHECK_THREADS); if (script_threads <= 0) { // -par=0 means autodetect (number of cores - 1 script threads) // -par=-n means "leave n cores free" (number of cores - n - 1 script threads) @@ -1158,10 +1208,43 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) fDiscover = args.GetBoolArg("-discover", true); const bool ignores_incoming_txs{args.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY)}; - assert(!node.addrman); - node.addrman = std::make_unique<CAddrMan>(); + { + // Initialize addrman + assert(!node.addrman); + + // Read asmap file if configured + std::vector<bool> asmap; + if (args.IsArgSet("-asmap")) { + fs::path asmap_path = fs::path(args.GetArg("-asmap", "")); + if (asmap_path.empty()) { + asmap_path = DEFAULT_ASMAP_FILENAME; + } + if (!asmap_path.is_absolute()) { + asmap_path = gArgs.GetDataDirNet() / asmap_path; + } + if (!fs::exists(asmap_path)) { + InitError(strprintf(_("Could not find asmap file %s"), asmap_path)); + return false; + } + asmap = DecodeAsmap(asmap_path); + if (asmap.size() == 0) { + InitError(strprintf(_("Could not parse asmap file %s"), asmap_path)); + return false; + } + const uint256 asmap_version = SerializeHash(asmap); + LogPrintf("Using asmap version %s for IP bucketing\n", asmap_version.ToString()); + } else { + LogPrintf("Using /16 prefix for IP bucketing\n"); + } + + uiInterface.InitMessage(_("Loading P2P addresses…").translated); + if (const auto error{LoadAddrman(asmap, args, node.addrman)}) { + return InitError(*error); + } + } + assert(!node.banman); - node.banman = std::make_unique<BanMan>(gArgs.GetDataDirNet() / "banlist", &uiInterface, args.GetArg("-bantime", DEFAULT_MISBEHAVING_BANTIME)); + node.banman = std::make_unique<BanMan>(gArgs.GetDataDirNet() / "banlist", &uiInterface, args.GetIntArg("-bantime", DEFAULT_MISBEHAVING_BANTIME)); assert(!node.connman); node.connman = std::make_unique<CConnman>(GetRand(std::numeric_limits<uint64_t>::max()), GetRand(std::numeric_limits<uint64_t>::max()), *node.addrman, args.GetBoolArg("-networkactive", true)); @@ -1171,7 +1254,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) if (!ignores_incoming_txs) node.fee_estimator = std::make_unique<CBlockPolicyEstimator>(); assert(!node.mempool); - int check_ratio = std::min<int>(std::max<int>(args.GetArg("-checkmempool", chainparams.DefaultConsistencyChecks() ? 1 : 0), 0), 1000000); + int check_ratio = std::min<int>(std::max<int>(args.GetIntArg("-checkmempool", chainparams.DefaultConsistencyChecks() ? 1 : 0), 0), 1000000); node.mempool = std::make_unique<CTxMemPool>(node.fee_estimator.get(), check_ratio); assert(!node.chainman); @@ -1180,7 +1263,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) assert(!node.peerman); node.peerman = PeerManager::make(chainparams, *node.connman, *node.addrman, node.banman.get(), - *node.scheduler, chainman, *node.mempool, ignores_incoming_txs); + chainman, *node.mempool, ignores_incoming_txs); RegisterValidationInterface(node.peerman.get()); // sanitize comments per BIP-0014, format user agent and check total size @@ -1264,31 +1347,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) return InitError(ResolveErrMsg("externalip", strAddr)); } - // Read asmap file if configured - if (args.IsArgSet("-asmap")) { - fs::path asmap_path = fs::path(args.GetArg("-asmap", "")); - if (asmap_path.empty()) { - asmap_path = DEFAULT_ASMAP_FILENAME; - } - if (!asmap_path.is_absolute()) { - asmap_path = gArgs.GetDataDirNet() / asmap_path; - } - if (!fs::exists(asmap_path)) { - InitError(strprintf(_("Could not find asmap file %s"), asmap_path)); - return false; - } - std::vector<bool> asmap = CAddrMan::DecodeAsmap(asmap_path); - if (asmap.size() == 0) { - InitError(strprintf(_("Could not parse asmap file %s"), asmap_path)); - return false; - } - const uint256 asmap_version = SerializeHash(asmap); - node.connman->SetAsmap(std::move(asmap)); - LogPrintf("Using asmap version %s for IP bucketing\n", asmap_version.ToString()); - } else { - LogPrintf("Using /16 prefix for IP bucketing\n"); - } - #if ENABLE_ZMQ g_zmq_notification_interface = CZMQNotificationInterface::Create(); @@ -1303,7 +1361,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) bool fReindexChainState = args.GetBoolArg("-reindex-chainstate", false); // cache size calculations - int64_t nTotalCache = (args.GetArg("-dbcache", nDefaultDbCache) << 20); + int64_t nTotalCache = (args.GetIntArg("-dbcache", nDefaultDbCache) << 20); nTotalCache = std::max(nTotalCache, nMinDbCache << 20); // total cache cannot be less than nMinDbCache nTotalCache = std::min(nTotalCache, nMaxDbCache << 20); // total cache cannot be greater than nMaxDbcache int64_t nBlockTreeDBCache = std::min(nTotalCache / 8, nMaxBlockDBCache << 20); @@ -1321,7 +1379,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache nTotalCache -= nCoinDBCache; int64_t nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache - int64_t nMempoolSizeMax = args.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; + int64_t nMempoolSizeMax = args.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; LogPrintf("Cache configuration:\n"); LogPrintf("* Using %.1f MiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024)); if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { @@ -1348,12 +1406,13 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) const int64_t load_block_index_start_time = GetTimeMillis(); try { LOCK(cs_main); - chainman.InitializeChainstate(*Assert(node.mempool)); + chainman.InitializeChainstate(Assert(node.mempool.get())); chainman.m_total_coinstip_cache = nCoinCacheUsage; chainman.m_total_coinsdb_cache = nCoinDBCache; UnloadBlockIndex(node.mempool.get(), chainman); + auto& pblocktree{chainman.m_blockman.m_block_tree_db}; // new CBlockTreeDB tries to delete the existing file, which // fails if it's still open from the previous loop. Close it first: pblocktree.reset(); @@ -1372,7 +1431,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // block file from disk. // Note that it also sets fReindex based on the disk flag! // From here on out fReindex and fReset mean something different! - if (!chainman.LoadBlockIndex(chainparams)) { + if (!chainman.LoadBlockIndex()) { if (ShutdownRequested()) break; strLoadError = _("Error loading block database"); break; @@ -1396,7 +1455,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // If we're not mid-reindex (based on disk + args), add a genesis block on disk // (otherwise we use the one already on disk). // This is called again in ThreadImport after the reindex completes. - if (!fReindex && !chainman.ActiveChainstate().LoadGenesisBlock(chainparams)) { + if (!fReindex && !chainman.ActiveChainstate().LoadGenesisBlock()) { strLoadError = _("Error initializing block database"); break; } @@ -1427,7 +1486,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } // ReplayBlocks is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate - if (!chainstate->ReplayBlocks(chainparams)) { + if (!chainstate->ReplayBlocks()) { strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate."); failed_chainstate_init = true; break; @@ -1439,7 +1498,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) if (!is_coinsview_empty(chainstate)) { // LoadChainTip initializes the chain based on CoinsTip()'s best block - if (!chainstate->LoadChainTip(chainparams)) { + if (!chainstate->LoadChainTip()) { strLoadError = _("Error initializing block database"); failed_chainstate_init = true; break; // out of the per-chainstate loop @@ -1461,7 +1520,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) LOCK(cs_main); auto chainstates{chainman.GetAll()}; if (std::any_of(chainstates.begin(), chainstates.end(), - [&chainparams](const CChainState* cs) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return cs->NeedsRedownload(chainparams); })) { + [](const CChainState* cs) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return cs->NeedsRedownload(); })) { strLoadError = strprintf(_("Witness data for blocks after height %d requires validation. Please restart with -reindex."), chainparams.GetConsensus().SegwitHeight); break; @@ -1476,7 +1535,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) for (CChainState* chainstate : chainman.GetAll()) { if (!is_coinsview_empty(chainstate)) { uiInterface.InitMessage(_("Verifying blocks…").translated); - if (fHavePruned && args.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { + if (fHavePruned && args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS) > MIN_BLOCKS_TO_KEEP) { LogPrintf("Prune: pruned datadir may not have more than %d blocks; only checking available blocks\n", MIN_BLOCKS_TO_KEEP); } @@ -1493,8 +1552,8 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) if (!CVerifyDB().VerifyDB( *chainstate, chainparams, chainstate->CoinsDB(), - args.GetArg("-checklevel", DEFAULT_CHECKLEVEL), - args.GetArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { + args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL), + args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS))) { strLoadError = _("Corrupted block database detected"); failed_verification = true; break; @@ -1544,6 +1603,10 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // ********************************************************* Step 8: start indexers if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { + if (const auto error{CheckLegacyTxindex(*Assert(chainman.m_blockman.m_block_tree_db))}) { + return InitError(*error); + } + g_txindex = std::make_unique<TxIndex>(nTxIndexCache, false, fReindex); if (!g_txindex->Start(chainman.ActiveChainstate())) { return false; @@ -1587,12 +1650,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } } - if (chainparams.GetConsensus().SegwitHeight != std::numeric_limits<int>::max()) { - // Advertise witness capabilities. - // The option to not set NODE_WITNESS is only used in the tests and should be removed. - nLocalServices = ServiceFlags(nLocalServices | NODE_WITNESS); - } - // ********************************************************* Step 11: import blocks if (!CheckDiskSpace(gArgs.GetDataDirNet())) { @@ -1688,11 +1745,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) connOptions.uiInterface = &uiInterface; connOptions.m_banman = node.banman.get(); connOptions.m_msgproc = node.peerman.get(); - connOptions.nSendBufferMaxSize = 1000 * args.GetArg("-maxsendbuffer", DEFAULT_MAXSENDBUFFER); - connOptions.nReceiveFloodSize = 1000 * args.GetArg("-maxreceivebuffer", DEFAULT_MAXRECEIVEBUFFER); + connOptions.nSendBufferMaxSize = 1000 * args.GetIntArg("-maxsendbuffer", DEFAULT_MAXSENDBUFFER); + connOptions.nReceiveFloodSize = 1000 * args.GetIntArg("-maxreceivebuffer", DEFAULT_MAXRECEIVEBUFFER); connOptions.m_added_nodes = args.GetArgs("-addnode"); - connOptions.nMaxOutboundLimit = 1024 * 1024 * args.GetArg("-maxuploadtarget", DEFAULT_MAX_UPLOAD_TARGET); + connOptions.nMaxOutboundLimit = 1024 * 1024 * args.GetIntArg("-maxuploadtarget", DEFAULT_MAX_UPLOAD_TARGET); connOptions.m_peer_connect_timeout = peer_connect_timeout; for (const std::string& bind_arg : args.GetArgs("-bind")) { @@ -1716,18 +1773,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) return InitError(ResolveErrMsg("bind", bind_arg)); } - if (connOptions.onion_binds.empty()) { - connOptions.onion_binds.push_back(DefaultOnionServiceTarget()); - } - - if (args.GetBoolArg("-listenonion", DEFAULT_LISTEN_ONION)) { - const auto bind_addr = connOptions.onion_binds.front(); - if (connOptions.onion_binds.size() > 1) { - InitWarning(strprintf(_("More than one onion bind address is provided. Using %s for the automatically created Tor onion service."), bind_addr.ToStringIPPort())); - } - StartTorControl(bind_addr); - } - for (const std::string& strBind : args.GetArgs("-whitebind")) { NetWhitebindPermissions whitebind; bilingual_str error; @@ -1735,6 +1780,27 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) connOptions.vWhiteBinds.push_back(whitebind); } + // If the user did not specify -bind= or -whitebind= then we bind + // on any address - 0.0.0.0 (IPv4) and :: (IPv6). + connOptions.bind_on_any = args.GetArgs("-bind").empty() && args.GetArgs("-whitebind").empty(); + + CService onion_service_target; + if (!connOptions.onion_binds.empty()) { + onion_service_target = connOptions.onion_binds.front(); + } else { + onion_service_target = DefaultOnionServiceTarget(); + connOptions.onion_binds.push_back(onion_service_target); + } + + if (args.GetBoolArg("-listenonion", DEFAULT_LISTEN_ONION)) { + if (connOptions.onion_binds.size() > 1) { + InitWarning(strprintf(_("More than one onion bind address is provided. Using %s " + "for the automatically created Tor onion service."), + onion_service_target.ToStringIPPort())); + } + StartTorControl(onion_service_target); + } + for (const auto& net : args.GetArgs("-whitelist")) { NetWhitelistPermissions subnet; bilingual_str error; @@ -1785,6 +1851,8 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) banman->DumpBanlist(); }, DUMP_BANS_INTERVAL); + if (node.peerman) node.peerman->StartScheduledTasks(*node.scheduler); + #if HAVE_SYSTEM StartupNotify(args); #endif diff --git a/src/init.h b/src/init.h index b856468e5d..5af6930a16 100644 --- a/src/init.h +++ b/src/init.h @@ -20,9 +20,6 @@ struct NodeContext; namespace interfaces { struct BlockAndHeaderTipInfo; } -namespace boost { -class thread_group; -} // namespace boost /** Interrupt threads */ void Interrupt(NodeContext& node); diff --git a/src/init/bitcoin-node.cpp b/src/init/bitcoin-node.cpp index 6b6157c139..fa56153745 100644 --- a/src/init/bitcoin-node.cpp +++ b/src/init/bitcoin-node.cpp @@ -2,9 +2,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <interfaces/chain.h> #include <interfaces/echo.h> #include <interfaces/init.h> #include <interfaces/ipc.h> +#include <interfaces/node.h> +#include <interfaces/wallet.h> #include <node/context.h> #include <util/system.h> @@ -24,6 +27,12 @@ public: m_node.args = &gArgs; m_node.init = this; } + std::unique_ptr<interfaces::Node> makeNode() override { return interfaces::MakeNode(m_node); } + std::unique_ptr<interfaces::Chain> makeChain() override { return interfaces::MakeChain(m_node); } + std::unique_ptr<interfaces::WalletClient> makeWalletClient(interfaces::Chain& chain) override + { + return MakeWalletClient(chain, *Assert(m_node.args)); + } std::unique_ptr<interfaces::Echo> makeEcho() override { return interfaces::MakeEcho(); } interfaces::Ipc* ipc() override { return m_ipc.get(); } NodeContext& m_node; diff --git a/src/init/bitcoind.cpp b/src/init/bitcoind.cpp index 1d4504c24f..9c8d5bd9bb 100644 --- a/src/init/bitcoind.cpp +++ b/src/init/bitcoind.cpp @@ -2,7 +2,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <interfaces/chain.h> +#include <interfaces/echo.h> #include <interfaces/init.h> +#include <interfaces/node.h> +#include <interfaces/wallet.h> #include <node/context.h> #include <util/system.h> @@ -18,6 +22,13 @@ public: m_node.args = &gArgs; m_node.init = this; } + std::unique_ptr<interfaces::Node> makeNode() override { return interfaces::MakeNode(m_node); } + std::unique_ptr<interfaces::Chain> makeChain() override { return interfaces::MakeChain(m_node); } + std::unique_ptr<interfaces::WalletClient> makeWalletClient(interfaces::Chain& chain) override + { + return MakeWalletClient(chain, *Assert(m_node.args)); + } + std::unique_ptr<interfaces::Echo> makeEcho() override { return interfaces::MakeEcho(); } NodeContext& m_node; }; } // namespace diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 7cac435e96..9a97cad1f8 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -35,7 +35,9 @@ namespace interfaces { class Handler; class Wallet; -//! Helper for findBlock to selectively return pieces of block data. +//! Helper for findBlock to selectively return pieces of block data. If block is +//! found, data will be returned by setting specified output variables. If block +//! is not found, output variables will keep their previous values. class FoundBlock { public: @@ -60,6 +62,7 @@ public: bool* m_in_active_chain = nullptr; const FoundBlock* m_next_block = nullptr; CBlock* m_data = nullptr; + mutable bool found = false; }; //! Interface giving clients (wallet processes, maybe other analysis tools in @@ -174,7 +177,7 @@ public: std::string& err_string) = 0; //! Calculate mempool ancestor and descendant counts for the given transaction. - virtual void getTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants) = 0; + virtual void getTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants, size_t* ancestorsize = nullptr, CAmount* ancestorfees = nullptr) = 0; //! Get the node's package limits. //! Currently only returns the ancestor and descendant count limits, but could be enhanced to @@ -262,11 +265,18 @@ public: //! Current RPC serialization flags. virtual int rpcSerializationFlags() = 0; + //! Get settings value. + virtual util::SettingsValue getSetting(const std::string& arg) = 0; + + //! Get list of settings values. + virtual std::vector<util::SettingsValue> getSettingsList(const std::string& arg) = 0; + //! Return <datadir>/settings.json setting value. virtual util::SettingsValue getRwSetting(const std::string& name) = 0; - //! Write a setting to <datadir>/settings.json. - virtual bool updateRwSetting(const std::string& name, const util::SettingsValue& value) = 0; + //! Write a setting to <datadir>/settings.json. Optionally just update the + //! setting in memory and do not write the file. + virtual bool updateRwSetting(const std::string& name, const util::SettingsValue& value, bool write=true) = 0; //! Synchronously send transactionAddedToMempool notifications about all //! current mempool transactions to the specified handler and return after diff --git a/src/interfaces/ipc.h b/src/interfaces/ipc.h index e9e6c78053..963649fc9a 100644 --- a/src/interfaces/ipc.h +++ b/src/interfaces/ipc.h @@ -9,6 +9,10 @@ #include <memory> #include <typeindex> +namespace ipc { +struct Context; +} // namespace ipc + namespace interfaces { class Init; @@ -58,6 +62,9 @@ public: addCleanup(typeid(Interface), &iface, std::move(cleanup)); } + //! IPC context struct accessor (see struct definition for more description). + virtual ipc::Context& context() = 0; + protected: //! Internal implementation of public addCleanup method (above) as a //! type-erased virtual function, since template functions can't be virtual. diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 77129423db..34fdde3774 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_INTERFACES_NODE_H #define BITCOIN_INTERFACES_NODE_H -#include <amount.h> // For CAmount +#include <consensus/amount.h> #include <external_signer.h> #include <net.h> // For NodeId #include <net_types.h> // For banmap_t @@ -230,7 +230,7 @@ public: }; //! Return implementation of Node interface. -std::unique_ptr<Node> MakeNode(NodeContext* context = nullptr); +std::unique_ptr<Node> MakeNode(NodeContext& context); //! Block tip (could be a header or not, depends on the subscribed signal). struct BlockTip { diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index fb1febc11b..490563426c 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_INTERFACES_WALLET_H #define BITCOIN_INTERFACES_WALLET_H -#include <amount.h> // For CAmount +#include <consensus/amount.h> #include <interfaces/chain.h> // For ChainClient #include <pubkey.h> // For CKeyID and CScriptID (definitions needed in CTxDestination instantiation) #include <script/standard.h> // For CTxDestination @@ -122,10 +122,10 @@ public: virtual bool displayAddress(const CTxDestination& dest) = 0; //! Lock coin. - virtual void lockCoin(const COutPoint& output) = 0; + virtual bool lockCoin(const COutPoint& output, const bool write_to_db) = 0; //! Unlock coin. - virtual void unlockCoin(const COutPoint& output) = 0; + virtual bool unlockCoin(const COutPoint& output) = 0; //! Return whether coin is locked. virtual bool isLockedCoin(const COutPoint& output) = 0; @@ -332,6 +332,9 @@ public: //! loaded at startup or by RPC. using LoadWalletFn = std::function<void(std::unique_ptr<Wallet> wallet)>; virtual std::unique_ptr<Handler> handleLoadWallet(LoadWalletFn fn) = 0; + + //! Return pointer to internal context, useful for testing. + virtual WalletContext* context() { return nullptr; } }; //! Information about one wallet address. @@ -410,7 +413,7 @@ struct WalletTxOut //! Return implementation of Wallet interface. This function is defined in //! dummywallet.cpp and throws if the wallet component is not compiled. -std::unique_ptr<Wallet> MakeWallet(const std::shared_ptr<CWallet>& wallet); +std::unique_ptr<Wallet> MakeWallet(WalletContext& context, const std::shared_ptr<CWallet>& wallet); //! Return implementation of ChainClient interface for a wallet client. This //! function will be undefined in builds where ENABLE_WALLET is false. diff --git a/src/ipc/capnp/context.h b/src/ipc/capnp/context.h new file mode 100644 index 0000000000..06e1614494 --- /dev/null +++ b/src/ipc/capnp/context.h @@ -0,0 +1,23 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_IPC_CAPNP_CONTEXT_H +#define BITCOIN_IPC_CAPNP_CONTEXT_H + +#include <ipc/context.h> + +namespace ipc { +namespace capnp { +//! Cap'n Proto context struct. Generally the parent ipc::Context struct should +//! be used instead of this struct to give all IPC protocols access to +//! application state, so there aren't unnecessary differences between IPC +//! protocols. But this specialized struct can be used to pass capnp-specific +//! function and object types to capnp hooks. +struct Context : ipc::Context +{ +}; +} // namespace capnp +} // namespace ipc + +#endif // BITCOIN_IPC_CAPNP_CONTEXT_H diff --git a/src/ipc/capnp/protocol.cpp b/src/ipc/capnp/protocol.cpp index 74c66c899a..37b57a9525 100644 --- a/src/ipc/capnp/protocol.cpp +++ b/src/ipc/capnp/protocol.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <interfaces/init.h> +#include <ipc/capnp/context.h> #include <ipc/capnp/init.capnp.h> #include <ipc/capnp/init.capnp.proxy.h> #include <ipc/capnp/protocol.h> @@ -54,7 +55,7 @@ public: { assert(!m_loop); mp::g_thread_context.thread_name = mp::ThreadName(exe_name); - m_loop.emplace(exe_name, &IpcLogFn, nullptr); + m_loop.emplace(exe_name, &IpcLogFn, &m_context); mp::ServeStream<messages::Init>(*m_loop, fd, init); m_loop->loop(); m_loop.reset(); @@ -63,13 +64,14 @@ public: { mp::ProxyTypeRegister::types().at(type)(iface).cleanup.emplace_back(std::move(cleanup)); } + Context& context() override { return m_context; } void startLoop(const char* exe_name) { if (m_loop) return; std::promise<void> promise; m_loop_thread = std::thread([&] { util::ThreadRename("capnp-loop"); - m_loop.emplace(exe_name, &IpcLogFn, nullptr); + m_loop.emplace(exe_name, &IpcLogFn, &m_context); { std::unique_lock<std::mutex> lock(m_loop->m_mutex); m_loop->addClient(lock); @@ -80,6 +82,7 @@ public: }); promise.get_future().wait(); } + Context m_context; std::thread m_loop_thread; std::optional<mp::EventLoop> m_loop; }; diff --git a/src/ipc/context.h b/src/ipc/context.h new file mode 100644 index 0000000000..924d7d7450 --- /dev/null +++ b/src/ipc/context.h @@ -0,0 +1,19 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_IPC_CONTEXT_H +#define BITCOIN_IPC_CONTEXT_H + +namespace ipc { +//! Context struct used to give IPC protocol implementations or implementation +//! hooks access to application state, in case they need to run extra code that +//! isn't needed within a single process, like code copying global state from an +//! existing process to a new process when it's initialized, or code dealing +//! with shared objects that are created or destroyed remotely. +struct Context +{ +}; +} // namespace ipc + +#endif // BITCOIN_IPC_CONTEXT_H diff --git a/src/ipc/interfaces.cpp b/src/ipc/interfaces.cpp index ad4b78ed81..580590fde9 100644 --- a/src/ipc/interfaces.cpp +++ b/src/ipc/interfaces.cpp @@ -60,6 +60,7 @@ public: { m_protocol->addCleanup(type, iface, std::move(cleanup)); } + Context& context() override { return m_protocol->context(); } const char* m_exe_name; const char* m_process_argv0; interfaces::Init& m_init; diff --git a/src/ipc/protocol.h b/src/ipc/protocol.h index af955b0007..4cd892e411 100644 --- a/src/ipc/protocol.h +++ b/src/ipc/protocol.h @@ -12,6 +12,8 @@ #include <typeindex> namespace ipc { +struct Context; + //! IPC protocol interface for calling IPC methods over sockets. //! //! There may be different implementations of this interface for different IPC @@ -33,6 +35,9 @@ public: //! Add cleanup callback to interface that will run when the interface is //! deleted. virtual void addCleanup(std::type_index type, void* iface, std::function<void()> cleanup) = 0; + + //! Context accessor. + virtual Context& context() = 0; }; } // namespace ipc diff --git a/src/key.cpp b/src/key.cpp index dcad386e77..39155e4311 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -274,7 +274,7 @@ bool CKey::SignSchnorr(const uint256& hash, Span<unsigned char> sig, const uint2 uint256 tweak = XOnlyPubKey(pubkey_bytes).ComputeTapTweakHash(merkle_root->IsNull() ? nullptr : merkle_root); if (!secp256k1_keypair_xonly_tweak_add(GetVerifyContext(), &keypair, tweak.data())) return false; } - bool ret = secp256k1_schnorrsig_sign(secp256k1_context_sign, sig.data(), hash.data(), &keypair, secp256k1_nonce_function_bip340, aux ? (void*)aux->data() : nullptr); + bool ret = secp256k1_schnorrsig_sign(secp256k1_context_sign, sig.data(), hash.data(), &keypair, aux ? (unsigned char*)aux->data() : nullptr); memory_cleanse(&keypair, sizeof(keypair)); return ret; } @@ -343,8 +343,7 @@ CExtPubKey CExtKey::Neuter() const { void CExtKey::Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const { code[0] = nDepth; memcpy(code+1, vchFingerprint, 4); - code[5] = (nChild >> 24) & 0xFF; code[6] = (nChild >> 16) & 0xFF; - code[7] = (nChild >> 8) & 0xFF; code[8] = (nChild >> 0) & 0xFF; + WriteBE32(code+5, nChild); memcpy(code+9, chaincode.begin(), 32); code[41] = 0; assert(key.size() == 32); @@ -354,9 +353,10 @@ void CExtKey::Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const { void CExtKey::Decode(const unsigned char code[BIP32_EXTKEY_SIZE]) { nDepth = code[0]; memcpy(vchFingerprint, code+1, 4); - nChild = (code[5] << 24) | (code[6] << 16) | (code[7] << 8) | code[8]; + nChild = ReadBE32(code+5); memcpy(chaincode.begin(), code+9, 32); key.Set(code+42, code+BIP32_EXTKEY_SIZE, true); + if ((nDepth == 0 && (nChild != 0 || ReadLE32(vchFingerprint) != 0)) || code[41] != 0) key = CKey(); } bool ECC_InitSanityCheck() { @@ -17,7 +17,6 @@ /** - * secure_allocator is defined in allocators.h * CPrivKey is a serialized private key, with all parameters included * (SIZE bytes) */ @@ -133,10 +132,15 @@ public: * optionally tweaked by *merkle_root. Additional nonce entropy can be provided through * aux. * - * When merkle_root is not nullptr, this results in a signature with a modified key as - * specified in BIP341: - * - If merkle_root->IsNull(): key + H_TapTweak(pubkey)*G - * - Otherwise: key + H_TapTweak(pubkey || *merkle_root) + * merkle_root is used to optionally perform tweaking of the private key, as specified + * in BIP341: + * - If merkle_root == nullptr: no tweaking is done, sign with key directly (this is + * used for signatures in BIP342 script). + * - If merkle_root->IsNull(): sign with key + H_TapTweak(pubkey) (this is used for + * key path spending when no scripts are present). + * - Otherwise: sign with key + H_TapTweak(pubkey || *merkle_root) + * (this is used for key path spending, with specific + * Merkle root of the script tree). */ bool SignSchnorr(const uint256& hash, Span<unsigned char> sig, const uint256* merkle_root = nullptr, const uint256* aux = nullptr) const; diff --git a/src/logging.cpp b/src/logging.cpp index e5187fd596..a352e106e5 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -8,6 +8,8 @@ #include <util/string.h> #include <util/time.h> +#include <algorithm> +#include <array> #include <mutex> const char * const DEFAULT_DEBUGLOGFILE = "debug.log"; @@ -124,8 +126,7 @@ bool BCLog::Logger::DefaultShrinkDebugFile() const return m_categories == BCLog::NONE; } -struct CLogCategoryDesc -{ +struct CLogCategoryDesc { BCLog::LogFlags flag; std::string category; }; @@ -158,6 +159,8 @@ const CLogCategoryDesc LogCategories[] = {BCLog::VALIDATION, "validation"}, {BCLog::I2P, "i2p"}, {BCLog::IPC, "ipc"}, + {BCLog::LOCK, "lock"}, + {BCLog::UTIL, "util"}, {BCLog::ALL, "1"}, {BCLog::ALL, "all"}, }; @@ -179,15 +182,18 @@ bool GetLogCategory(BCLog::LogFlags& flag, const std::string& str) std::vector<LogCategory> BCLog::Logger::LogCategoriesList() const { + // Sort log categories by alphabetical order. + std::array<CLogCategoryDesc, std::size(LogCategories)> categories; + std::copy(std::begin(LogCategories), std::end(LogCategories), categories.begin()); + std::sort(categories.begin(), categories.end(), [](auto a, auto b) { return a.category < b.category; }); + std::vector<LogCategory> ret; - for (const CLogCategoryDesc& category_desc : LogCategories) { - // Omit the special cases. - if (category_desc.flag != BCLog::NONE && category_desc.flag != BCLog::ALL) { - LogCategory catActive; - catActive.category = category_desc.category; - catActive.active = WillLogCategory(category_desc.flag); - ret.push_back(catActive); - } + for (const CLogCategoryDesc& category_desc : categories) { + if (category_desc.flag == BCLog::NONE || category_desc.flag == BCLog::ALL) continue; + LogCategory catActive; + catActive.category = category_desc.category; + catActive.active = WillLogCategory(category_desc.flag); + ret.push_back(catActive); } return ret; } @@ -237,7 +243,7 @@ namespace BCLog { } return ret; } -} +} // namespace BCLog void BCLog::Logger::LogPrintStr(const std::string& str, const std::string& logging_function, const std::string& source_file, const int source_line) { diff --git a/src/logging.h b/src/logging.h index d04bc99268..02e64a7c48 100644 --- a/src/logging.h +++ b/src/logging.h @@ -59,6 +59,8 @@ namespace BCLog { VALIDATION = (1 << 21), I2P = (1 << 22), IPC = (1 << 23), + LOCK = (1 << 24), + UTIL = (1 << 25), ALL = ~(uint32_t)0, }; @@ -138,9 +140,9 @@ namespace BCLog { bool DisableCategory(const std::string& str); bool WillLogCategory(LogFlags category) const; - /** Returns a vector of the log categories */ + /** Returns a vector of the log categories in alphabetical order. */ std::vector<LogCategory> LogCategoriesList() const; - /** Returns a string with the log categories */ + /** Returns a string with the log categories in alphabetical order. */ std::string LogCategoriesString() const { return Join(LogCategoriesList(), ", ", [&](const LogCategory& i) { return i.category; }); diff --git a/src/logging/timer.h b/src/logging/timer.h index 159920e397..79627b1fe3 100644 --- a/src/logging/timer.h +++ b/src/logging/timer.h @@ -9,6 +9,7 @@ #include <logging.h> #include <util/macros.h> #include <util/time.h> +#include <util/types.h> #include <chrono> #include <string> @@ -58,21 +59,15 @@ public: return strprintf("%s: %s", m_prefix, msg); } - std::string units = ""; - float divisor = 1; - - if (std::is_same<TimeType, std::chrono::microseconds>::value) { - units = "μs"; - } else if (std::is_same<TimeType, std::chrono::milliseconds>::value) { - units = "ms"; - divisor = 1000.; - } else if (std::is_same<TimeType, std::chrono::seconds>::value) { - units = "s"; - divisor = 1000. * 1000.; + if constexpr (std::is_same<TimeType, std::chrono::microseconds>::value) { + return strprintf("%s: %s (%iμs)", m_prefix, msg, end_time.count()); + } else if constexpr (std::is_same<TimeType, std::chrono::milliseconds>::value) { + return strprintf("%s: %s (%.2fms)", m_prefix, msg, end_time.count() * 0.001); + } else if constexpr (std::is_same<TimeType, std::chrono::seconds>::value) { + return strprintf("%s: %s (%.2fs)", m_prefix, msg, end_time.count() * 0.000001); + } else { + static_assert(ALWAYS_FALSE<TimeType>, "Error: unexpected time type"); } - - const float time_ms = end_time.count() / divisor; - return strprintf("%s: %s (%.2f%s)", m_prefix, msg, time_ms, units); } private: @@ -87,12 +82,13 @@ private: //! Forwarded on to LogPrint if specified - has the effect of only //! outputting the timing log when a particular debug= category is specified. const BCLog::LogFlags m_log_category{}; - }; } // namespace BCLog +#define LOG_TIME_MICROS_WITH_CATEGORY(end_msg, log_category) \ + BCLog::Timer<std::chrono::microseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, log_category) #define LOG_TIME_MILLIS_WITH_CATEGORY(end_msg, log_category) \ BCLog::Timer<std::chrono::milliseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, log_category) #define LOG_TIME_SECONDS(end_msg) \ diff --git a/src/mapport.cpp b/src/mapport.cpp index 135efb561e..a2d06c68b4 100644 --- a/src/mapport.cpp +++ b/src/mapport.cpp @@ -14,6 +14,7 @@ #include <netaddress.h> #include <netbase.h> #include <threadinterrupt.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <util/thread.h> @@ -222,6 +223,7 @@ static bool ProcessUpnp() static void ThreadMapPort() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION_MAP_PORT); bool ok; do { ok = false; diff --git a/src/miner.cpp b/src/miner.cpp index 0cf303eb3c..1ef246cd14 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -5,14 +5,15 @@ #include <miner.h> -#include <amount.h> #include <chain.h> #include <chainparams.h> #include <coins.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/merkle.h> #include <consensus/tx_verify.h> #include <consensus/validation.h> +#include <deploymentstatus.h> #include <policy/feerate.h> #include <policy/policy.h> #include <pow.h> @@ -71,12 +72,12 @@ static BlockAssembler::Options DefaultOptions() // Block resource limits // If -blockmaxweight is not given, limit to DEFAULT_BLOCK_MAX_WEIGHT BlockAssembler::Options options; - options.nBlockMaxWeight = gArgs.GetArg("-blockmaxweight", DEFAULT_BLOCK_MAX_WEIGHT); - CAmount n = 0; - if (gArgs.IsArgSet("-blockmintxfee") && ParseMoney(gArgs.GetArg("-blockmintxfee", ""), n)) { - options.blockMinFeeRate = CFeeRate(n); + options.nBlockMaxWeight = gArgs.GetIntArg("-blockmaxweight", DEFAULT_BLOCK_MAX_WEIGHT); + if (gArgs.IsArgSet("-blockmintxfee")) { + std::optional<CAmount> parsed = ParseMoney(gArgs.GetArg("-blockmintxfee", "")); + options.blockMinFeeRate = CFeeRate{parsed.value_or(DEFAULT_BLOCK_MIN_TX_FEE)}; } else { - options.blockMinFeeRate = CFeeRate(DEFAULT_BLOCK_MIN_TX_FEE); + options.blockMinFeeRate = CFeeRate{DEFAULT_BLOCK_MIN_TX_FEE}; } return options; } @@ -120,11 +121,11 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc assert(pindexPrev != nullptr); nHeight = pindexPrev->nHeight + 1; - pblock->nVersion = ComputeBlockVersion(pindexPrev, chainparams.GetConsensus()); + pblock->nVersion = g_versionbitscache.ComputeBlockVersion(pindexPrev, chainparams.GetConsensus()); // -regtest only: allow overriding block.nVersion with // -blockversion=N to test forking scenarios if (chainparams.MineBlocksOnDemand()) - pblock->nVersion = gArgs.GetArg("-blockversion", pblock->nVersion); + pblock->nVersion = gArgs.GetIntArg("-blockversion", pblock->nVersion); pblock->nTime = GetAdjustedTime(); const int64_t nMedianTimePast = pindexPrev->GetMedianTimePast(); @@ -137,12 +138,12 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc // This is only needed in case the witness softfork activation is reverted // (which would require a very deep reorganization). // Note that the mempool would accept transactions with witness data before - // IsWitnessEnabled, but we would only ever mine blocks after IsWitnessEnabled + // the deployment is active, but we would only ever mine blocks after activation // unless there is a massive block reorganization with the witness softfork // not activated. // TODO: replace this with a call to main to assess validity of a mempool // transaction (which in most cases can be a no-op). - fIncludeWitness = IsWitnessEnabled(pindexPrev, chainparams.GetConsensus()); + fIncludeWitness = DeploymentActiveAfter(pindexPrev, chainparams.GetConsensus(), Consensus::DEPLOYMENT_SEGWIT); int nPackagesSelected = 0; int nDescendantsUpdated = 0; @@ -236,7 +237,7 @@ void BlockAssembler::AddToBlock(CTxMemPool::txiter iter) bool fPrintPriority = gArgs.GetBoolArg("-printpriority", DEFAULT_PRINTPRIORITY); if (fPrintPriority) { - LogPrintf("fee %s txid %s\n", + LogPrintf("fee rate %s txid %s\n", CFeeRate(iter->GetModifiedFee(), iter->GetTxSize()).ToString(), iter->GetTx().GetHash().ToString()); } diff --git a/src/net.cpp b/src/net.cpp index 60059249ed..7271ff22b2 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -9,6 +9,7 @@ #include <net.h> +#include <addrdb.h> #include <banman.h> #include <clientversion.h> #include <compat.h> @@ -24,7 +25,10 @@ #include <scheduler.h> #include <util/sock.h> #include <util/strencodings.h> +#include <util/syscall_sandbox.h> +#include <util/system.h> #include <util/thread.h> +#include <util/trace.h> #include <util/translation.h> #ifdef WIN32 @@ -120,7 +124,7 @@ void CConnman::AddAddrFetch(const std::string& strDest) uint16_t GetListenPort() { - return static_cast<uint16_t>(gArgs.GetArg("-port", Params().GetDefaultPort())); + return static_cast<uint16_t>(gArgs.GetIntArg("-port", Params().GetDefaultPort())); } // find 'best' local address for a particular peer @@ -189,8 +193,8 @@ CAddress GetLocalAddress(const CNetAddr *paddrPeer, ServiceFlags nLocalServices) static int GetnScore(const CService& addr) { LOCK(cs_mapLocalHost); - if (mapLocalHost.count(addr) == 0) return 0; - return mapLocalHost[addr].nScore; + const auto it = mapLocalHost.find(addr); + return (it != mapLocalHost.end()) ? it->second.nScore : 0; } // Is our peer's addrLocal potentially useful as an external IP source? @@ -242,10 +246,10 @@ bool AddLocal(const CService& addr, int nScore) { LOCK(cs_mapLocalHost); - bool fAlready = mapLocalHost.count(addr) > 0; - LocalServiceInfo &info = mapLocalHost[addr]; - if (!fAlready || nScore >= info.nScore) { - info.nScore = nScore + (fAlready ? 1 : 0); + const auto [it, is_newly_added] = mapLocalHost.emplace(addr, LocalServiceInfo()); + LocalServiceInfo &info = it->second; + if (is_newly_added || nScore >= info.nScore) { + info.nScore = nScore + (is_newly_added ? 0 : 1); info.nPort = addr.GetPort(); } } @@ -287,12 +291,10 @@ bool IsReachable(const CNetAddr &addr) /** vote for a local address */ bool SeenLocal(const CService& addr) { - { - LOCK(cs_mapLocalHost); - if (mapLocalHost.count(addr) == 0) - return false; - mapLocalHost[addr].nScore++; - } + LOCK(cs_mapLocalHost); + const auto it = mapLocalHost.find(addr); + if (it == mapLocalHost.end()) return false; + ++it->second.nScore; return true; } @@ -330,7 +332,7 @@ CNode* CConnman::FindNode(const std::string& addrName) { LOCK(cs_vNodes); for (CNode* pnode : vNodes) { - if (pnode->GetAddrName() == addrName) { + if (pnode->m_addr_name == addrName) { return pnode; } } @@ -402,7 +404,8 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo pszDest ? 0.0 : (double)(GetAdjustedTime() - addrConnect.nTime)/3600.0); // Resolve - const uint16_t default_port{Params().GetDefaultPort()}; + const uint16_t default_port{pszDest != nullptr ? Params().GetDefaultPort(pszDest) : + Params().GetDefaultPort()}; if (pszDest) { std::vector<CService> resolved; if (Lookup(pszDest, resolved, default_port, fNameLookup && !HaveNameProxy(), 256) && !resolved.empty()) { @@ -412,14 +415,10 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo return nullptr; } // It is possible that we already have a connection to the IP/port pszDest resolved to. - // In that case, drop the connection that was just created, and return the existing CNode instead. - // Also store the name we used to connect in that CNode, so that future FindNode() calls to that - // name catch this early. + // In that case, drop the connection that was just created. LOCK(cs_vNodes); CNode* pnode = FindNode(static_cast<CService>(addrConnect)); - if (pnode) - { - pnode->MaybeSetAddrName(std::string(pszDest)); + if (pnode) { LogPrintf("Failed to open new connection, already connected\n"); return nullptr; } @@ -532,19 +531,8 @@ std::string ConnectionTypeAsString(ConnectionType conn_type) assert(false); } -std::string CNode::GetAddrName() const { - LOCK(cs_addrName); - return addrName; -} - -void CNode::MaybeSetAddrName(const std::string& addrNameIn) { - LOCK(cs_addrName); - if (addrName.empty()) { - addrName = addrNameIn; - } -} - -CService CNode::GetAddrLocal() const { +CService CNode::GetAddrLocal() const +{ LOCK(cs_addrLocal); return addrLocal; } @@ -565,14 +553,13 @@ Network CNode::ConnectedThroughNetwork() const #undef X #define X(name) stats.name = name -void CNode::copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap) +void CNode::CopyStats(CNodeStats& stats) { stats.nodeid = this->GetId(); X(nServices); X(addr); X(addrBind); stats.m_network = ConnectedThroughNetwork(); - stats.m_mapped_as = addr.GetMappedAS(m_asmap); if (m_tx_relay != nullptr) { LOCK(m_tx_relay->cs_filter); stats.fRelayTxes = m_tx_relay->fRelayTxes; @@ -585,7 +572,7 @@ void CNode::copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap) X(nLastBlockTime); X(nTimeConnected); X(nTimeOffset); - stats.addrName = GetAddrName(); + X(m_addr_name); X(nVersion); { LOCK(cs_SubVer); @@ -936,14 +923,17 @@ void ProtectEvictionCandidatesByRatio(std::vector<NodeEvictionCandidate>& evicti size_t num_protected{0}; while (num_protected < max_protect_by_network) { + // Count the number of disadvantaged networks from which we have peers to protect. + auto num_networks = std::count_if(networks.begin(), networks.end(), [](const Net& n) { return n.count; }); + if (num_networks == 0) { + break; + } const size_t disadvantaged_to_protect{max_protect_by_network - num_protected}; - const size_t protect_per_network{ - std::max(disadvantaged_to_protect / networks.size(), static_cast<size_t>(1))}; - + const size_t protect_per_network{std::max(disadvantaged_to_protect / num_networks, static_cast<size_t>(1))}; // Early exit flag if there are no remaining candidates by disadvantaged network. bool protected_at_least_one{false}; - for (const Net& n : networks) { + for (Net& n : networks) { if (n.count == 0) continue; const size_t before = eviction_candidates.size(); EraseLastKElements(eviction_candidates, CompareNodeNetworkTime(n.is_local, n.id), @@ -953,10 +943,12 @@ void ProtectEvictionCandidatesByRatio(std::vector<NodeEvictionCandidate>& evicti const size_t after = eviction_candidates.size(); if (before > after) { protected_at_least_one = true; - num_protected += before - after; + const size_t delta{before - after}; + num_protected += delta; if (num_protected >= max_protect_by_network) { break; } + n.count -= delta; } } if (!protected_at_least_one) { @@ -1206,16 +1198,29 @@ void CConnman::CreateNodeFromAcceptedSocket(SOCKET hSocket, bool CConnman::AddConnection(const std::string& address, ConnectionType conn_type) { - if (conn_type != ConnectionType::OUTBOUND_FULL_RELAY && conn_type != ConnectionType::BLOCK_RELAY) return false; - - const int max_connections = conn_type == ConnectionType::OUTBOUND_FULL_RELAY ? m_max_outbound_full_relay : m_max_outbound_block_relay; + std::optional<int> max_connections; + switch (conn_type) { + case ConnectionType::INBOUND: + case ConnectionType::MANUAL: + case ConnectionType::FEELER: + return false; + case ConnectionType::OUTBOUND_FULL_RELAY: + max_connections = m_max_outbound_full_relay; + break; + case ConnectionType::BLOCK_RELAY: + max_connections = m_max_outbound_block_relay; + break; + // no limit for ADDR_FETCH because -seednode has no limit either + case ConnectionType::ADDR_FETCH: + break; + } // no default case, so the compiler can warn about missing cases // Count existing connections int existing_connections = WITH_LOCK(cs_vNodes, return std::count_if(vNodes.begin(), vNodes.end(), [conn_type](CNode* node) { return node->m_conn_type == conn_type; });); // Max connections of specified type already exist - if (existing_connections >= max_connections) return false; + if (max_connections != std::nullopt && existing_connections >= max_connections) return false; // Max total outbound connections already exist CSemaphoreGrant grant(*semOutbound, true); @@ -1284,8 +1289,9 @@ void CConnman::NotifyNumConnectionsChanged() } if(vNodesSize != nPrevNodeCount) { nPrevNodeCount = vNodesSize; - if(clientInterface) - clientInterface->NotifyNumConnectionsChanged(vNodesSize); + if (m_client_interface) { + m_client_interface->NotifyNumConnectionsChanged(vNodesSize); + } } } @@ -1610,6 +1616,7 @@ void CConnman::SocketHandler() void CConnman::ThreadSocketHandler() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET); while (!interruptNet) { DisconnectNodes(); @@ -1629,6 +1636,7 @@ void CConnman::WakeMessageHandler() void CConnman::ThreadDNSAddressSeed() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION_DNS_SEED); FastRandomContext rng; std::vector<std::string> seeds = Params().DNSSeeds(); Shuffle(seeds.begin(), seeds.end(), rng); @@ -1741,8 +1749,7 @@ void CConnman::DumpAddresses() { int64_t nStart = GetTimeMillis(); - CAddrDB adb; - adb.Write(addrman); + DumpPeerAddresses(::gArgs, addrman); LogPrint(BCLog::NET, "Flushed %d addresses to peers.dat %dms\n", addrman.size(), GetTimeMillis() - nStart); @@ -1812,6 +1819,7 @@ int CConnman::GetExtraBlockRelayCount() const void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET_OPEN_CONNECTION); // Connect to specific addresses if (!connect.empty()) { @@ -1915,7 +1923,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) case ConnectionType::BLOCK_RELAY: case ConnectionType::ADDR_FETCH: case ConnectionType::FEELER: - setConnected.insert(pnode->addr.GetGroup(addrman.m_asmap)); + setConnected.insert(pnode->addr.GetGroup(addrman.GetAsmap())); } // no default case, so the compiler can warn about missing cases } } @@ -1989,7 +1997,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) m_anchors.pop_back(); if (!addr.IsValid() || IsLocal(addr) || !IsReachable(addr) || !HasAllDesirableServiceFlags(addr.nServices) || - setConnected.count(addr.GetGroup(addrman.m_asmap))) continue; + setConnected.count(addr.GetGroup(addrman.GetAsmap()))) continue; addrConnect = addr; LogPrint(BCLog::NET, "Trying to make an anchor connection to %s\n", addrConnect.ToString()); break; @@ -2002,17 +2010,18 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) if (nTries > 100) break; - CAddrInfo addr; + CAddress addr; + int64_t addr_last_try{0}; if (fFeeler) { // First, try to get a tried table collision address. This returns // an empty (invalid) address if there are no collisions to try. - addr = addrman.SelectTriedCollision(); + std::tie(addr, addr_last_try) = addrman.SelectTriedCollision(); if (!addr.IsValid()) { // No tried table collisions. Select a new table address // for our feeler. - addr = addrman.Select(true); + std::tie(addr, addr_last_try) = addrman.Select(true); } else if (AlreadyConnectedToAddress(addr)) { // If test-before-evict logic would have us connect to a // peer that we're already connected to, just mark that @@ -2021,15 +2030,15 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // a currently-connected peer. addrman.Good(addr); // Select a new table address for our feeler instead. - addr = addrman.Select(true); + std::tie(addr, addr_last_try) = addrman.Select(true); } } else { // Not a feeler - addr = addrman.Select(); + std::tie(addr, addr_last_try) = addrman.Select(); } // Require outbound connections, other than feelers, to be to distinct network groups - if (!fFeeler && setConnected.count(addr.GetGroup(addrman.m_asmap))) { + if (!fFeeler && setConnected.count(addr.GetGroup(addrman.GetAsmap()))) { break; } @@ -2042,7 +2051,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) continue; // only consider very recently tried nodes after 30 failed attempts - if (nANow - addr.nLastTry < 600 && nTries < 30) + if (nANow - addr_last_try < 600 && nTries < 30) continue; // for non-feelers, require all the services we'll want, @@ -2059,8 +2068,9 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // from advertising themselves as a service on another host and // port, causing a DoS attack as nodes around the network attempt // to connect to it fruitlessly. - if (addr.GetPort() != Params().GetDefaultPort() && nTries < 50) + if (addr.GetPort() != Params().GetDefaultPort(addr.GetNetwork()) && nTries < 50) { continue; + } addrConnect = addr; break; @@ -2115,7 +2125,7 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const if (pnode->addr.IsValid()) { mapConnected[pnode->addr] = pnode->IsInboundConn(); } - std::string addrName = pnode->GetAddrName(); + std::string addrName{pnode->m_addr_name}; if (!addrName.empty()) { mapConnectedByName[std::move(addrName)] = std::make_pair(pnode->IsInboundConn(), static_cast<const CService&>(pnode->addr)); } @@ -2123,7 +2133,7 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const } for (const std::string& strAddNode : lAddresses) { - CService service(LookupNumeric(strAddNode, Params().GetDefaultPort())); + CService service(LookupNumeric(strAddNode, Params().GetDefaultPort(strAddNode))); AddedNodeInfo addedNode{strAddNode, CService(), false, false}; if (service.IsValid()) { // strAddNode is an IP:port @@ -2150,6 +2160,7 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const void CConnman::ThreadOpenAddedConnections() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET_ADD_CONNECTION); while (true) { CSemaphoreGrant grant(*semAddnode); @@ -2213,6 +2224,7 @@ void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai void CConnman::ThreadMessageHandler() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::MESSAGE_HANDLER); FastRandomContext rng; while (!flagInterruptMsgProc) { @@ -2427,10 +2439,12 @@ void CConnman::SetNetworkActive(bool active) fNetworkActive = active; - uiInterface.NotifyNetworkActiveChanged(fNetworkActive); + if (m_client_interface) { + m_client_interface->NotifyNetworkActiveChanged(fNetworkActive); + } } -CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In, CAddrMan& addrman_in, bool network_active) +CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In, AddrMan& addrman_in, bool network_active) : addrman(addrman_in), nSeed0(nSeed0In), nSeed1(nSeed1In) { SetTryNewOutboundPeer(false); @@ -2452,8 +2466,8 @@ bool CConnman::Bind(const CService &addr, unsigned int flags, NetPermissionFlags } bilingual_str strError; if (!BindListenPort(addr, strError, permissions)) { - if ((flags & BF_REPORT_ERROR) && clientInterface) { - clientInterface->ThreadSafeMessageBox(strError, "", CClientUIInterface::MSG_ERROR); + if ((flags & BF_REPORT_ERROR) && m_client_interface) { + m_client_interface->ThreadSafeMessageBox(strError, "", CClientUIInterface::MSG_ERROR); } return false; } @@ -2465,30 +2479,25 @@ bool CConnman::Bind(const CService &addr, unsigned int flags, NetPermissionFlags return true; } -bool CConnman::InitBinds( - const std::vector<CService>& binds, - const std::vector<NetWhitebindPermissions>& whiteBinds, - const std::vector<CService>& onion_binds) +bool CConnman::InitBinds(const Options& options) { bool fBound = false; - for (const auto& addrBind : binds) { + for (const auto& addrBind : options.vBinds) { fBound |= Bind(addrBind, (BF_EXPLICIT | BF_REPORT_ERROR), NetPermissionFlags::None); } - for (const auto& addrBind : whiteBinds) { + for (const auto& addrBind : options.vWhiteBinds) { fBound |= Bind(addrBind.m_service, (BF_EXPLICIT | BF_REPORT_ERROR), addrBind.m_flags); } - if (binds.empty() && whiteBinds.empty()) { + for (const auto& addr_bind : options.onion_binds) { + fBound |= Bind(addr_bind, BF_EXPLICIT | BF_DONT_ADVERTISE, NetPermissionFlags::None); + } + if (options.bind_on_any) { struct in_addr inaddr_any; inaddr_any.s_addr = htonl(INADDR_ANY); struct in6_addr inaddr6_any = IN6ADDR_ANY_INIT; fBound |= Bind(CService(inaddr6_any, GetListenPort()), BF_NONE, NetPermissionFlags::None); fBound |= Bind(CService(inaddr_any, GetListenPort()), !fBound ? BF_REPORT_ERROR : BF_NONE, NetPermissionFlags::None); } - - for (const auto& addr_bind : onion_binds) { - fBound |= Bind(addr_bind, BF_EXPLICIT | BF_DONT_ADVERTISE, NetPermissionFlags::None); - } - return fBound; } @@ -2496,9 +2505,9 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) { Init(connOptions); - if (fListen && !InitBinds(connOptions.vBinds, connOptions.vWhiteBinds, connOptions.onion_binds)) { - if (clientInterface) { - clientInterface->ThreadSafeMessageBox( + if (fListen && !InitBinds(connOptions)) { + if (m_client_interface) { + m_client_interface->ThreadSafeMessageBox( _("Failed to listen on any port. Use -listen=0 if you want this."), "", CClientUIInterface::MSG_ERROR); } @@ -2515,22 +2524,6 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) AddAddrFetch(strDest); } - if (clientInterface) { - clientInterface->InitMessage(_("Loading P2P addresses…").translated); - } - // Load addresses from peers.dat - int64_t nStart = GetTimeMillis(); - { - CAddrDB adb; - if (adb.Read(addrman)) - LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman.size(), GetTimeMillis() - nStart); - else { - addrman.Clear(); // Addrman can be in an inconsistent state after failure, reset it - LogPrintf("Recreating peers.dat\n"); - DumpAddresses(); - } - } - if (m_use_addrman_outgoing) { // Load addresses from anchors.dat m_anchors = ReadAnchors(gArgs.GetDataDirNet() / ANCHORS_DATABASE_FILENAME); @@ -2540,7 +2533,9 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) LogPrintf("%i block-relay-only anchors will be tried for connections.\n", m_anchors.size()); } - uiInterface.InitMessage(_("Starting network threads…").translated); + if (m_client_interface) { + m_client_interface->InitMessage(_("Starting network threads…").translated); + } fAddressesInitialized = true; @@ -2578,8 +2573,8 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) threadOpenAddedConnections = std::thread(&util::TraceThread, "addcon", [this] { ThreadOpenAddedConnections(); }); if (connOptions.m_use_addrman_outgoing && !connOptions.m_specified_outgoing.empty()) { - if (clientInterface) { - clientInterface->ThreadSafeMessageBox( + if (m_client_interface) { + m_client_interface->ThreadSafeMessageBox( _("Cannot provide specific connections and have addrman find outgoing connections at the same."), "", CClientUIInterface::MSG_ERROR); } @@ -2814,7 +2809,8 @@ void CConnman::GetNodeStats(std::vector<CNodeStats>& vstats) const vstats.reserve(vNodes.size()); for (CNode* pnode : vNodes) { vstats.emplace_back(); - pnode->copyStats(vstats.back(), addrman.m_asmap); + pnode->CopyStats(vstats.back()); + vstats.back().m_mapped_as = pnode->addr.GetMappedAS(addrman.GetAsmap()); } } @@ -2961,6 +2957,7 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, SOCKET hSocketIn, const : nTimeConnected(GetTimeSeconds()), addr(addrIn), addrBind(addrBindIn), + m_addr_name{addrNameIn.empty() ? addr.ToStringIPPort() : addrNameIn}, m_inbound_onion(inbound_onion), nKeyedNetGroup(nKeyedNetGroupIn), id(idIn), @@ -2970,7 +2967,6 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, SOCKET hSocketIn, const { if (inbound_onion) assert(conn_type_in == ConnectionType::INBOUND); hSocket = hSocketIn; - addrName = addrNameIn == "" ? addr.ToStringIPPort() : addrNameIn; if (conn_type_in != ConnectionType::BLOCK_RELAY) { m_tx_relay = std::make_unique<TxRelay>(); } @@ -2980,7 +2976,7 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, SOCKET hSocketIn, const mapRecvBytesPerMsgCmd[NET_MESSAGE_COMMAND_OTHER] = 0; if (fLogIPs) { - LogPrint(BCLog::NET, "Added connection to %s peer=%d\n", addrName, id); + LogPrint(BCLog::NET, "Added connection to %s peer=%d\n", m_addr_name, id); } else { LogPrint(BCLog::NET, "Added connection peer=%d\n", id); } @@ -3002,11 +2998,20 @@ bool CConnman::NodeFullyConnected(const CNode* pnode) void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) { size_t nMessageSize = msg.data.size(); - LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", SanitizeString(msg.m_type), nMessageSize, pnode->GetId()); + LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", msg.m_type, nMessageSize, pnode->GetId()); if (gArgs.GetBoolArg("-capturemessages", false)) { CaptureMessage(pnode->addr, msg.m_type, msg.data, /* incoming */ false); } + TRACE6(net, outbound_message, + pnode->GetId(), + pnode->m_addr_name.c_str(), + pnode->ConnectionTypeAsString().c_str(), + msg.m_type.c_str(), + msg.data.size(), + msg.data.data() + ); + // make sure we use the appropriate network transport format std::vector<unsigned char> serializedHeader; pnode->m_serializer->prepareForTransport(msg, serializedHeader); @@ -3068,7 +3073,7 @@ CSipHasher CConnman::GetDeterministicRandomizer(uint64_t id) const uint64_t CConnman::CalculateKeyedNetGroup(const CAddress& ad) const { - std::vector<unsigned char> vchNetGroup(ad.GetGroup(addrman.m_asmap)); + std::vector<unsigned char> vchNetGroup(ad.GetGroup(addrman.GetAsmap())); return GetDeterministicRandomizer(RANDOMIZER_ID_NETGROUP).Write(vchNetGroup.data(), vchNetGroup.size()).Finalize(); } @@ -6,12 +6,11 @@ #ifndef BITCOIN_NET_H #define BITCOIN_NET_H -#include <addrdb.h> #include <addrman.h> -#include <amount.h> #include <bloom.h> #include <chainparams.h> #include <compat.h> +#include <consensus/amount.h> #include <crypto/siphash.h> #include <hash.h> #include <i2p.h> @@ -79,9 +78,9 @@ static const int64_t DEFAULT_PEER_CONNECT_TIMEOUT = 60; /** Number of file descriptors required for message capture **/ static const int NUM_FDS_MESSAGE_CAPTURE = 1; -static const bool DEFAULT_FORCEDNSSEED = false; -static const bool DEFAULT_DNSSEED = true; -static const bool DEFAULT_FIXEDSEEDS = true; +static constexpr bool DEFAULT_FORCEDNSSEED{false}; +static constexpr bool DEFAULT_DNSSEED{true}; +static constexpr bool DEFAULT_FIXEDSEEDS{true}; static const size_t DEFAULT_MAXRECEIVEBUFFER = 5 * 1000; static const size_t DEFAULT_MAXSENDBUFFER = 1 * 1000; @@ -248,7 +247,7 @@ public: int64_t nLastBlockTime; int64_t nTimeConnected; int64_t nTimeOffset; - std::string addrName; + std::string m_addr_name; int nVersion; std::string cleanSubVer; bool fInbound; @@ -430,6 +429,7 @@ public: const CAddress addr; // Bind address of our side of the connection const CAddress addrBind; + const std::string m_addr_name; //! Whether this peer is an inbound onion, i.e. connected via our Tor onion service. const bool m_inbound_onion; std::atomic<int> nVersion{0}; @@ -651,17 +651,13 @@ public: void CloseSocketDisconnect(); - void copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap); + void CopyStats(CNodeStats& stats); ServiceFlags GetLocalServices() const { return nLocalServices; } - std::string GetAddrName() const; - //! Sets the addrName only if it was not previously set - void MaybeSetAddrName(const std::string& addrNameIn); - std::string ConnectionTypeAsString() const { return ::ConnectionTypeAsString(m_conn_type); } /** A ping-pong round trip has completed successfully. Update latest and minimum ping times. */ @@ -693,10 +689,7 @@ private: //! service advertisements. const ServiceFlags nLocalServices; - std::list<CNetMessage> vRecvMsg; // Used only by SocketHandler thread - - mutable RecursiveMutex cs_addrName; - std::string addrName GUARDED_BY(cs_addrName); + std::list<CNetMessage> vRecvMsg; // Used only by SocketHandler thread // Our address, as reported by the peer CService addrLocal GUARDED_BY(cs_addrLocal); @@ -768,10 +761,12 @@ public: std::vector<NetWhitebindPermissions> vWhiteBinds; std::vector<CService> vBinds; std::vector<CService> onion_binds; + /// True if the user did not specify -bind= or -whitebind= and thus + /// we should bind on `0.0.0.0` (IPv4) and `::` (IPv6). + bool bind_on_any; bool m_use_addrman_outgoing = true; std::vector<std::string> m_specified_outgoing; std::vector<std::string> m_added_nodes; - std::vector<bool> m_asmap; bool m_i2p_accept_incoming; }; @@ -784,7 +779,7 @@ public: nMaxAddnode = connOptions.nMaxAddnode; nMaxFeeler = connOptions.nMaxFeeler; m_max_outbound = m_max_outbound_full_relay + m_max_outbound_block_relay + nMaxFeeler; - clientInterface = connOptions.uiInterface; + m_client_interface = connOptions.uiInterface; m_banman = connOptions.m_banman; m_msgproc = connOptions.m_msgproc; nSendBufferMaxSize = connOptions.nSendBufferMaxSize; @@ -802,7 +797,7 @@ public: m_onion_binds = connOptions.onion_binds; } - CConnman(uint64_t seed0, uint64_t seed1, CAddrMan& addrman, bool network_active = true); + CConnman(uint64_t seed0, uint64_t seed1, AddrMan& addrman, bool network_active = true); ~CConnman(); bool Start(CScheduler& scheduler, const Options& options); @@ -890,6 +885,7 @@ public: * * @param[in] address Address of node to try connecting to * @param[in] conn_type ConnectionType::OUTBOUND or ConnectionType::BLOCK_RELAY + * or ConnectionType::ADDR_FETCH * @return bool Returns false if there are no available * slots for this connection: * - conn_type not a supported ConnectionType @@ -945,8 +941,6 @@ public: */ std::chrono::microseconds PoissonNextSendInbound(std::chrono::microseconds now, std::chrono::seconds average_interval); - void SetAsmap(std::vector<bool> asmap) { addrman.m_asmap = std::move(asmap); } - /** Return true if we should disconnect the peer for failing an inactivity check. */ bool ShouldRunInactivityChecks(const CNode& node, std::optional<int64_t> now=std::nullopt) const; @@ -962,10 +956,7 @@ private: bool BindListenPort(const CService& bindAddr, bilingual_str& strError, NetPermissionFlags permissions); bool Bind(const CService& addr, unsigned int flags, NetPermissionFlags permissions); - bool InitBinds( - const std::vector<CService>& binds, - const std::vector<NetWhitebindPermissions>& whiteBinds, - const std::vector<CService>& onion_binds); + bool InitBinds(const Options& options); void ThreadOpenAddedConnections(); void AddAddrFetch(const std::string& strDest); @@ -1058,7 +1049,7 @@ private: std::vector<ListenSocket> vhListenSocket; std::atomic<bool> fNetworkActive{true}; bool fAddressesInitialized{false}; - CAddrMan& addrman; + AddrMan& addrman; std::deque<std::string> m_addr_fetches GUARDED_BY(m_addr_fetches_mutex); RecursiveMutex m_addr_fetches_mutex; std::vector<std::string> vAddedNodes GUARDED_BY(cs_vAddedNodes); @@ -1125,7 +1116,7 @@ private: int nMaxFeeler; int m_max_outbound; bool m_use_addrman_outgoing; - CClientUIInterface* clientInterface; + CClientUIInterface* m_client_interface; NetEventsInterface* m_msgproc; /** Pointer to this node's banman. May be nullptr - check existence before dereferencing. */ BanMan* m_banman; diff --git a/src/net_permissions.h b/src/net_permissions.h index c00689465e..bc979e3792 100644 --- a/src/net_permissions.h +++ b/src/net_permissions.h @@ -31,7 +31,8 @@ enum class NetPermissionFlags : uint32_t { NoBan = (1U << 4) | Download, // Can query the mempool Mempool = (1U << 5), - // Can request addrs without hitting a privacy-preserving cache + // Can request addrs without hitting a privacy-preserving cache, and send us + // unlimited amounts of addrs. Addr = (1U << 7), // True if the user did not specifically set fine grained permissions diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 2175b76d16..66b99aa2bb 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -10,7 +10,9 @@ #include <blockencodings.h> #include <blockfilter.h> #include <chainparams.h> +#include <consensus/amount.h> #include <consensus/validation.h> +#include <deploymentstatus.h> #include <hash.h> #include <index/blockfilterindex.h> #include <merkleblock.h> @@ -33,6 +35,7 @@ #include <util/check.h> // For NDEBUG compile time check #include <util/strencodings.h> #include <util/system.h> +#include <util/trace.h> #include <validation.h> #include <algorithm> @@ -154,6 +157,13 @@ static constexpr uint32_t MAX_GETCFHEADERS_SIZE = 2000; static constexpr size_t MAX_PCT_ADDR_TO_SEND = 23; /** The maximum number of address records permitted in an ADDR message. */ static constexpr size_t MAX_ADDR_TO_SEND{1000}; +/** The maximum rate of address records we're willing to process on average. Can be bypassed using + * the NetPermissionFlags::Addr permission. */ +static constexpr double MAX_ADDR_RATE_PER_SECOND{0.1}; +/** The soft limit of the address processing token bucket (the regular MAX_ADDR_RATE_PER_SECOND + * based increments won't go above this, but the MAX_ADDR_TO_SEND increment following GETADDR + * is exempt from this limit). */ +static constexpr size_t MAX_ADDR_PROCESSING_TOKEN_BUCKET{MAX_ADDR_TO_SEND}; // Internal stuff namespace { @@ -216,9 +226,31 @@ struct Peer { /** A vector of addresses to send to the peer, limited to MAX_ADDR_TO_SEND. */ std::vector<CAddress> m_addrs_to_send; - /** Probabilistic filter of addresses that this peer already knows. - * Used to avoid relaying addresses to this peer more than once. */ - const std::unique_ptr<CRollingBloomFilter> m_addr_known; + /** Probabilistic filter to track recent addr messages relayed with this + * peer. Used to avoid relaying redundant addresses to this peer. + * + * We initialize this filter for outbound peers (other than + * block-relay-only connections) or when an inbound peer sends us an + * address related message (ADDR, ADDRV2, GETADDR). + * + * Presence of this filter must correlate with m_addr_relay_enabled. + **/ + std::unique_ptr<CRollingBloomFilter> m_addr_known; + /** Whether we are participating in address relay with this connection. + * + * We set this bool to true for outbound peers (other than + * block-relay-only connections), or when an inbound peer sends us an + * address related message (ADDR, ADDRV2, GETADDR). + * + * We use this bool to decide whether a peer is eligible for gossiping + * addr messages. This avoids relaying to peers that are unlikely to + * forward them, effectively blackholing self announcements. Reasons + * peers might support addr relay on the link include that they connected + * to us as a block-relay-only peer or they are a light client. + * + * This field must correlate with whether m_addr_known has been + * initialized.*/ + std::atomic_bool m_addr_relay_enabled{false}; /** Whether a getaddr request to this peer is outstanding. */ bool m_getaddr_sent{false}; /** Guards address sending timers. */ @@ -232,6 +264,15 @@ struct Peer { std::atomic_bool m_wants_addrv2{false}; /** Whether this peer has already sent us a getaddr message. */ bool m_getaddr_recvd{false}; + /** Number of addresses that can be processed from this peer. Start at 1 to + * permit self-announcement. */ + double m_addr_token_bucket{1.0}; + /** When m_addr_token_bucket was last updated */ + std::chrono::microseconds m_addr_token_timestamp{GetTime<std::chrono::microseconds>()}; + /** Total number of addresses that were dropped due to rate limiting. */ + std::atomic<uint64_t> m_addr_rate_limited{0}; + /** Total number of addresses that were processed (excludes rate-limited ones). */ + std::atomic<uint64_t> m_addr_processed{0}; /** Set of txids to reconsider once their parent transactions have been accepted **/ std::set<uint256> m_orphan_work_set GUARDED_BY(g_cs_orphans); @@ -241,9 +282,8 @@ struct Peer { /** Work queue of items requested by this peer **/ std::deque<CInv> m_getdata_requests GUARDED_BY(m_getdata_requests_mutex); - explicit Peer(NodeId id, bool addr_relay) + explicit Peer(NodeId id) : m_id(id) - , m_addr_known{addr_relay ? std::make_unique<CRollingBloomFilter>(5000, 0.001) : nullptr} {} }; @@ -252,8 +292,8 @@ using PeerRef = std::shared_ptr<Peer>; class PeerManagerImpl final : public PeerManager { public: - PeerManagerImpl(const CChainParams& chainparams, CConnman& connman, CAddrMan& addrman, - BanMan* banman, CScheduler& scheduler, ChainstateManager& chainman, + PeerManagerImpl(const CChainParams& chainparams, CConnman& connman, AddrMan& addrman, + BanMan* banman, ChainstateManager& chainman, CTxMemPool& pool, bool ignore_incoming_txs); /** Overridden from CValidationInterface. */ @@ -270,6 +310,7 @@ public: bool SendMessages(CNode* pto) override EXCLUSIVE_LOCKS_REQUIRED(pto->cs_sendProcessing); /** Implement PeerManager */ + void StartScheduledTasks(CScheduler& scheduler) override; void CheckForStaleTipAndEvictPeers() override; bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) const override; bool IgnoresIncomingTxs() override { return m_ignore_incoming_txs; } @@ -369,7 +410,7 @@ private: const CChainParams& m_chainparams; CConnman& m_connman; - CAddrMan& m_addrman; + AddrMan& m_addrman; /** Pointer to this node's banman. May be nullptr - check existence before dereferencing. */ BanMan* const m_banman; ChainstateManager& m_chainman; @@ -379,7 +420,8 @@ private: /** The height of the best chain */ std::atomic<int> m_best_height{-1}; - int64_t m_stale_tip_check_time; //!< Next time to check for stale tip + /** Next time to check for stale tip */ + int64_t m_stale_tip_check_time{0}; /** Whether this node is running in blocks only mode */ const bool m_ignore_incoming_txs; @@ -452,16 +494,26 @@ private: * * Memory used: 1.3 MB */ - std::unique_ptr<CRollingBloomFilter> recentRejects GUARDED_BY(cs_main); + CRollingBloomFilter m_recent_rejects GUARDED_BY(::cs_main){120'000, 0.000'001}; uint256 hashRecentRejectsChainTip GUARDED_BY(cs_main); /* * Filter for transactions that have been recently confirmed. * We use this to avoid requesting transactions that have already been * confirnmed. + * + * Blocks don't typically have more than 4000 transactions, so this should + * be at least six blocks (~1 hr) worth of transactions that we can store, + * inserting both a txid and wtxid for every observed transaction. + * If the number of transactions appearing in a block goes up, or if we are + * seeing getdata requests more than an hour after initial announcement, we + * can increase this number. + * The false positive rate of 1/1M should come out to less than 1 + * transaction per day that would be inadvertently ignored (which is the + * same probability that we have in the reject filter). */ Mutex m_recent_confirmed_transactions_mutex; - std::unique_ptr<CRollingBloomFilter> m_recent_confirmed_transactions GUARDED_BY(m_recent_confirmed_transactions_mutex); + CRollingBloomFilter m_recent_confirmed_transactions GUARDED_BY(m_recent_confirmed_transactions_mutex){48'000, 0.000'001}; /** Have we requested this block from a peer */ bool IsBlockRequested(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -595,6 +647,14 @@ private: * @param[in] vRecv The raw message received */ void ProcessGetCFCheckPt(CNode& peer, CDataStream& vRecv); + + /** Checks if address relay is permitted with peer. If needed, initializes + * the m_addr_known bloom filter and sets m_addr_relay_enabled to true. + * + * @return True if address relay is enabled with peer + * False if address relay is disallowed + */ + bool SetupAddressRelay(const CNode& node, Peer& peer); }; } // namespace @@ -715,11 +775,6 @@ static CNodeState *State(NodeId pnode) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return &it->second; } -static bool RelayAddrsWithPeer(const Peer& peer) -{ - return peer.m_addr_known != nullptr; -} - /** * Whether the peer supports the address. For example, a peer that does not * implement BIP155 cannot receive Tor v3 addresses because it requires @@ -830,6 +885,12 @@ bool PeerManagerImpl::BlockRequested(NodeId nodeid, const CBlockIndex& block, st void PeerManagerImpl::MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid) { AssertLockHeld(cs_main); + + // Never request high-bandwidth mode from peers if we're blocks-only. Our + // mempool will not contain the transactions necessary to reconstruct the + // compact block. + if (m_ignore_incoming_txs) return; + CNodeState* nodestate = State(nodeid); if (!nodestate || !nodestate->fSupportsDesiredCmpctVersion) { // Never ask from peers who can't provide witnesses. @@ -860,7 +921,7 @@ void PeerManagerImpl::MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid) } m_connman.ForNode(nodeid, [this](CNode* pfrom) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { AssertLockHeld(::cs_main); - uint64_t nCMPCTBLOCKVersion = (pfrom->GetLocalServices() & NODE_WITNESS) ? 2 : 1; + uint64_t nCMPCTBLOCKVersion = 2; if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { // As per BIP152, we only get 3 of our peers to announce // blocks using compact encodings. @@ -997,7 +1058,7 @@ void PeerManagerImpl::FindNextBlocksToDownload(NodeId nodeid, unsigned int count // We consider the chain that this peer is on invalid. return; } - if (!State(nodeid)->fHaveWitness && IsWitnessEnabled(pindex->pprev, consensusParams)) { + if (!State(nodeid)->fHaveWitness && DeploymentActiveAt(*pindex, consensusParams, Consensus::DEPLOYMENT_SEGWIT)) { // We wouldn't download this block or its descendants from this peer. return; } @@ -1033,25 +1094,25 @@ void PeerManagerImpl::PushNodeVersion(CNode& pnode, int64_t nTime) // Note that pnode->GetLocalServices() is a reflection of the local // services we were offering when the CNode object was created for this // peer. - ServiceFlags nLocalNodeServices = pnode.GetLocalServices(); + uint64_t my_services{pnode.GetLocalServices()}; uint64_t nonce = pnode.GetLocalNonce(); const int nNodeStartingHeight{m_best_height}; NodeId nodeid = pnode.GetId(); CAddress addr = pnode.addr; - CAddress addrYou = addr.IsRoutable() && !IsProxy(addr) && addr.IsAddrV1Compatible() ? - addr : - CAddress(CService(), addr.nServices); - CAddress addrMe = CAddress(CService(), nLocalNodeServices); + CService addr_you = addr.IsRoutable() && !IsProxy(addr) && addr.IsAddrV1Compatible() ? addr : CService(); + uint64_t your_services{addr.nServices}; const bool tx_relay = !m_ignore_incoming_txs && pnode.m_tx_relay != nullptr; - m_connman.PushMessage(&pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, (uint64_t)nLocalNodeServices, nTime, addrYou, addrMe, + m_connman.PushMessage(&pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, my_services, nTime, + your_services, addr_you, // Together the pre-version-31402 serialization of CAddress "addrYou" (without nTime) + my_services, CService(), // Together the pre-version-31402 serialization of CAddress "addrMe" (without nTime) nonce, strSubVersion, nNodeStartingHeight, tx_relay)); if (fLogIPs) { - LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, them=%s, txrelay=%d, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), addrYou.ToString(), tx_relay, nodeid); + LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, them=%s, txrelay=%d, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addr_you.ToString(), tx_relay, nodeid); } else { - LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, us=%s, txrelay=%d, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addrMe.ToString(), tx_relay, nodeid); + LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, txrelay=%d, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, tx_relay, nodeid); } } @@ -1100,9 +1161,7 @@ void PeerManagerImpl::InitializeNode(CNode *pnode) assert(m_txrequest.Count(nodeid) == 0); } { - // Addr relay is disabled for outbound block-relay-only peers to - // prevent adversaries from inferring these links from addr traffic. - PeerRef peer = std::make_shared<Peer>(nodeid, /* addr_relay = */ !pnode->IsBlockOnlyConn()); + PeerRef peer = std::make_shared<Peer>(nodeid); LOCK(m_peer_mutex); m_peer_map.emplace_hint(m_peer_map.end(), nodeid, std::move(peer)); } @@ -1177,6 +1236,7 @@ void PeerManagerImpl::FinalizeNode(const CNode& node) assert(m_outbound_peers_with_protect_from_disconnect == 0); assert(m_wtxid_relay_peers == 0); assert(m_txrequest.Size() == 0); + assert(m_orphanage.Size() == 0); } } // cs_main if (node.fSuccessfullyConnected && misbehavior == 0 && @@ -1238,13 +1298,16 @@ bool PeerManagerImpl::GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) c } stats.m_ping_wait = ping_wait; + stats.m_addr_processed = peer->m_addr_processed.load(); + stats.m_addr_rate_limited = peer->m_addr_rate_limited.load(); + stats.m_addr_relay_enabled = peer->m_addr_relay_enabled.load(); return true; } void PeerManagerImpl::AddToCompactExtraTransactions(const CTransactionRef& tx) { - size_t max_extra_txn = gArgs.GetArg("-blockreconstructionextratxn", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN); + size_t max_extra_txn = gArgs.GetIntArg("-blockreconstructionextratxn", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN); if (max_extra_txn <= 0) return; if (!vExtraTxnForCompact.size()) @@ -1261,14 +1324,20 @@ void PeerManagerImpl::Misbehaving(const NodeId pnode, const int howmuch, const s if (peer == nullptr) return; LOCK(peer->m_misbehavior_mutex); + const int score_before{peer->m_misbehavior_score}; peer->m_misbehavior_score += howmuch; + const int score_now{peer->m_misbehavior_score}; + const std::string message_prefixed = message.empty() ? "" : (": " + message); - if (peer->m_misbehavior_score >= DISCOURAGEMENT_THRESHOLD && peer->m_misbehavior_score - howmuch < DISCOURAGEMENT_THRESHOLD) { - LogPrint(BCLog::NET, "Misbehaving: peer=%d (%d -> %d) DISCOURAGE THRESHOLD EXCEEDED%s\n", pnode, peer->m_misbehavior_score - howmuch, peer->m_misbehavior_score, message_prefixed); + std::string warning; + + if (score_now >= DISCOURAGEMENT_THRESHOLD && score_before < DISCOURAGEMENT_THRESHOLD) { + warning = " DISCOURAGE THRESHOLD EXCEEDED"; peer->m_should_discourage = true; - } else { - LogPrint(BCLog::NET, "Misbehaving: peer=%d (%d -> %d)%s\n", pnode, peer->m_misbehavior_score - howmuch, peer->m_misbehavior_score, message_prefixed); } + + LogPrint(BCLog::NET, "Misbehaving: peer=%d (%d -> %d)%s%s\n", + pnode, score_before, score_now, warning, message_prefixed); } bool PeerManagerImpl::MaybePunishNodeForBlock(NodeId nodeid, const BlockValidationState& state, @@ -1357,15 +1426,15 @@ bool PeerManagerImpl::BlockRequestAllowed(const CBlockIndex* pindex) (GetBlockProofEquivalentTime(*pindexBestHeader, *pindex, *pindexBestHeader, m_chainparams.GetConsensus()) < STALE_RELAY_AGE_LIMIT); } -std::unique_ptr<PeerManager> PeerManager::make(const CChainParams& chainparams, CConnman& connman, CAddrMan& addrman, - BanMan* banman, CScheduler& scheduler, ChainstateManager& chainman, +std::unique_ptr<PeerManager> PeerManager::make(const CChainParams& chainparams, CConnman& connman, AddrMan& addrman, + BanMan* banman, ChainstateManager& chainman, CTxMemPool& pool, bool ignore_incoming_txs) { - return std::make_unique<PeerManagerImpl>(chainparams, connman, addrman, banman, scheduler, chainman, pool, ignore_incoming_txs); + return std::make_unique<PeerManagerImpl>(chainparams, connman, addrman, banman, chainman, pool, ignore_incoming_txs); } -PeerManagerImpl::PeerManagerImpl(const CChainParams& chainparams, CConnman& connman, CAddrMan& addrman, - BanMan* banman, CScheduler& scheduler, ChainstateManager& chainman, +PeerManagerImpl::PeerManagerImpl(const CChainParams& chainparams, CConnman& connman, AddrMan& addrman, + BanMan* banman, ChainstateManager& chainman, CTxMemPool& pool, bool ignore_incoming_txs) : m_chainparams(chainparams), m_connman(connman), @@ -1373,23 +1442,12 @@ PeerManagerImpl::PeerManagerImpl(const CChainParams& chainparams, CConnman& conn m_banman(banman), m_chainman(chainman), m_mempool(pool), - m_stale_tip_check_time(0), m_ignore_incoming_txs(ignore_incoming_txs) { - // Initialize global variables that cannot be constructed at startup. - recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); - - // Blocks don't typically have more than 4000 transactions, so this should - // be at least six blocks (~1 hr) worth of transactions that we can store, - // inserting both a txid and wtxid for every observed transaction. - // If the number of transactions appearing in a block goes up, or if we are - // seeing getdata requests more than an hour after initial announcement, we - // can increase this number. - // The false positive rate of 1/1M should come out to less than 1 - // transaction per day that would be inadvertently ignored (which is the - // same probability that we have in the reject filter). - m_recent_confirmed_transactions.reset(new CRollingBloomFilter(48000, 0.000001)); +} +void PeerManagerImpl::StartScheduledTasks(CScheduler& scheduler) +{ // Stale tip checking and peer eviction are on two different timers, but we // don't want them to get out of sync due to drift in the scheduler, so we // combine them in one function and schedule at the quicker (peer-eviction) @@ -1415,9 +1473,9 @@ void PeerManagerImpl::BlockConnected(const std::shared_ptr<const CBlock>& pblock { LOCK(m_recent_confirmed_transactions_mutex); for (const auto& ptx : pblock->vtx) { - m_recent_confirmed_transactions->insert(ptx->GetHash()); + m_recent_confirmed_transactions.insert(ptx->GetHash()); if (ptx->GetHash() != ptx->GetWitnessHash()) { - m_recent_confirmed_transactions->insert(ptx->GetWitnessHash()); + m_recent_confirmed_transactions.insert(ptx->GetWitnessHash()); } } } @@ -1441,7 +1499,7 @@ void PeerManagerImpl::BlockDisconnected(const std::shared_ptr<const CBlock> &blo // presumably the most common case of relaying a confirmed transaction // should be just after a new block containing it is found. LOCK(m_recent_confirmed_transactions_mutex); - m_recent_confirmed_transactions->reset(); + m_recent_confirmed_transactions.reset(); } // All of the following cache a recent block, and are protected by cs_most_recent_block @@ -1467,7 +1525,7 @@ void PeerManagerImpl::NewPoWValidBlock(const CBlockIndex *pindex, const std::sha return; nHighestFastAnnounce = pindex->nHeight; - bool fWitnessEnabled = IsWitnessEnabled(pindex->pprev, m_chainparams.GetConsensus()); + bool fWitnessEnabled = DeploymentActiveAt(*pindex, m_chainparams.GetConsensus(), Consensus::DEPLOYMENT_SEGWIT); uint256 hashBlock(pblock->GetHash()); { @@ -1581,14 +1639,13 @@ void PeerManagerImpl::BlockChecked(const CBlock& block, const BlockValidationSta bool PeerManagerImpl::AlreadyHaveTx(const GenTxid& gtxid) { - assert(recentRejects); if (m_chainman.ActiveChain().Tip()->GetBlockHash() != hashRecentRejectsChainTip) { // If the chain tip has changed previously rejected transactions // might be now valid, e.g. due to a nLockTime'd tx becoming valid, // or a double-spend. Reset the rejects filter and give those // txs a second chance. hashRecentRejectsChainTip = m_chainman.ActiveChain().Tip()->GetBlockHash(); - recentRejects->reset(); + m_recent_rejects.reset(); } const uint256& hash = gtxid.GetHash(); @@ -1597,10 +1654,10 @@ bool PeerManagerImpl::AlreadyHaveTx(const GenTxid& gtxid) { LOCK(m_recent_confirmed_transactions_mutex); - if (m_recent_confirmed_transactions->contains(hash)) return true; + if (m_recent_confirmed_transactions.contains(hash)) return true; } - return recentRejects->contains(hash) || m_mempool.exists(gtxid); + return m_recent_rejects.contains(hash) || m_mempool.exists(gtxid); } bool PeerManagerImpl::AlreadyHaveBlock(const uint256& block_hash) @@ -1662,7 +1719,7 @@ void PeerManagerImpl::RelayAddress(NodeId originator, LOCK(m_peer_mutex); for (auto& [id, peer] : m_peer_map) { - if (RelayAddrsWithPeer(*peer) && id != originator && IsAddrCompatible(*peer, addr)) { + if (peer->m_addr_relay_enabled && id != originator && IsAddrCompatible(*peer, addr)) { uint64_t hashKey = CSipHasher(hasher).Write(id).Finalize(); for (unsigned int i = 0; i < nRelayNodes; i++) { if (hashKey > best[i].first) { @@ -1709,7 +1766,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& } // release cs_main before calling ActivateBestChain if (need_activate_chain) { BlockValidationState state; - if (!m_chainman.ActiveChainstate().ActivateBestChain(state, m_chainparams, a_recent_block)) { + if (!m_chainman.ActiveChainstate().ActivateBestChain(state, a_recent_block)) { LogPrint(BCLog::NET, "failed to activate chain (%s)\n", state.ToString()); } } @@ -1954,7 +2011,7 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic static uint32_t GetFetchFlags(const CNode& pfrom) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { uint32_t nFetchFlags = 0; - if ((pfrom.GetLocalServices() & NODE_WITNESS) && State(pfrom.GetId())->fHaveWitness) { + if (State(pfrom.GetId())->fHaveWitness) { nFetchFlags |= MSG_WITNESS_FLAG; } return nFetchFlags; @@ -2082,7 +2139,7 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, const Peer& peer, while (pindexWalk && !m_chainman.ActiveChain().Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && !IsBlockRequested(pindexWalk->GetBlockHash()) && - (!IsWitnessEnabled(pindexWalk->pprev, m_chainparams.GetConsensus()) || State(pfrom.GetId())->fHaveWitness)) { + (!DeploymentActiveAt(*pindexWalk, m_chainparams.GetConsensus(), Consensus::DEPLOYMENT_SEGWIT) || State(pfrom.GetId())->fHaveWitness)) { // We don't have this block, and it's not yet in flight. vToFetch.push_back(pindexWalk); } @@ -2115,7 +2172,11 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, const Peer& peer, pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); } if (vGetData.size() > 0) { - if (nodestate->fSupportsDesiredCmpctVersion && vGetData.size() == 1 && mapBlocksInFlight.size() == 1 && pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { + if (!m_ignore_incoming_txs && + nodestate->fSupportsDesiredCmpctVersion && + vGetData.size() == 1 && + mapBlocksInFlight.size() == 1 && + pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { // In any case, we want to download using a compact block, not a regular one vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); } @@ -2219,8 +2280,7 @@ void PeerManagerImpl::ProcessOrphanTx(std::set<uint256>& orphan_work_set) // See also comments in https://github.com/bitcoin/bitcoin/pull/18044#discussion_r443419034 // for concerns around weakening security of unupgraded nodes // if we start doing this too early. - assert(recentRejects); - recentRejects->insert(porphanTx->GetWitnessHash()); + m_recent_rejects.insert(porphanTx->GetWitnessHash()); // If the transaction failed for TX_INPUTS_NOT_STANDARD, // then we know that the witness was irrelevant to the policy // failure, since this check depends only on the txid @@ -2232,7 +2292,7 @@ void PeerManagerImpl::ProcessOrphanTx(std::set<uint256>& orphan_work_set) if (state.GetResult() == TxValidationResult::TX_INPUTS_NOT_STANDARD && porphanTx->GetWitnessHash() != porphanTx->GetHash()) { // We only add the txid if it differs from the wtxid, to // avoid wasting entries in the rolling bloom filter. - recentRejects->insert(porphanTx->GetHash()); + m_recent_rejects.insert(porphanTx->GetHash()); } } m_orphanage.EraseTx(orphanHash); @@ -2438,21 +2498,20 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } int64_t nTime; - CAddress addrMe; - CAddress addrFrom; + CService addrMe; uint64_t nNonce = 1; - uint64_t nServiceInt; ServiceFlags nServices; int nVersion; std::string cleanSubVer; int starting_height = -1; bool fRelay = true; - vRecv >> nVersion >> nServiceInt >> nTime >> addrMe; + vRecv >> nVersion >> Using<CustomUintFormatter<8>>(nServices) >> nTime; if (nTime < 0) { nTime = 0; } - nServices = ServiceFlags(nServiceInt); + vRecv.ignore(8); // Ignore the addrMe service bits sent by the peer + vRecv >> addrMe; if (!pfrom.IsInboundConn()) { m_addrman.SetServices(pfrom.addr, nServices); @@ -2471,8 +2530,14 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } - if (!vRecv.empty()) - vRecv >> addrFrom >> nNonce; + if (!vRecv.empty()) { + // The version message includes information about the sending node which we don't use: + // - 8 bytes (service bits) + // - 16 bytes (ipv6 address) + // - 2 bytes (port) + vRecv.ignore(26); + vRecv >> nNonce; + } if (!vRecv.empty()) { std::string strSubVer; vRecv >> LIMITED_STRING(strSubVer, MAX_SUBVERSION_LENGTH); @@ -2553,7 +2618,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, UpdatePreferredDownload(pfrom, State(pfrom.GetId())); } - if (!pfrom.IsInboundConn() && !pfrom.IsBlockOnlyConn()) { + // Self advertisement & GETADDR logic + if (!pfrom.IsInboundConn() && SetupAddressRelay(pfrom, *peer)) { // For outbound peers, we try to relay our address (so that other // nodes can try to find us more quickly, as we have no guarantee // that an outbound peer is even aware of how to reach us) and do a @@ -2562,8 +2628,9 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // empty and no one will know who we are, so these mechanisms are // important to help us connect to the network. // - // We skip this for block-relay-only peers to avoid potentially leaking - // information about our block-relay-only connections via address relay. + // We skip this for block-relay-only peers. We want to avoid + // potentially leaking addr information and we do not want to + // indicate to the peer that we will participate in addr relay. if (fListen && !m_chainman.ActiveChainstate().IsInitialBlockDownload()) { CAddress addr = GetLocalAddress(&pfrom.addr, pfrom.GetLocalServices()); @@ -2582,6 +2649,9 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Get recent addresses m_connman.PushMessage(&pfrom, CNetMsgMaker(greatest_common_version).Make(NetMsgType::GETADDR)); peer->m_getaddr_sent = true; + // When requesting a getaddr, accept an additional MAX_ADDR_TO_SEND addresses in response + // (bypassing the MAX_ADDR_PROCESSING_TOKEN_BUCKET limit). + peer->m_addr_token_bucket += MAX_ADDR_TO_SEND; } if (!pfrom.IsInboundConn()) { @@ -2594,7 +2664,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // table is also potentially detrimental because new-table entries // are subject to eviction in the event of addrman collisions. We // mitigate the information-leak by never calling - // CAddrMan::Connected() on block-relay-only peers; see + // AddrMan::Connected() on block-relay-only peers; see // FinalizeNode(). // // This moves an address from New to Tried table in Addrman, @@ -2666,8 +2736,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // they may wish to request compact blocks from us bool fAnnounceUsingCMPCTBLOCK = false; uint64_t nCMPCTBLOCKVersion = 2; - if (pfrom.GetLocalServices() & NODE_WITNESS) - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); nCMPCTBLOCKVersion = 1; m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion)); } @@ -2685,7 +2754,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, bool fAnnounceUsingCMPCTBLOCK = false; uint64_t nCMPCTBLOCKVersion = 0; vRecv >> fAnnounceUsingCMPCTBLOCK >> nCMPCTBLOCKVersion; - if (nCMPCTBLOCKVersion == 1 || ((pfrom.GetLocalServices() & NODE_WITNESS) && nCMPCTBLOCKVersion == 2)) { + if (nCMPCTBLOCKVersion == 1 || nCMPCTBLOCKVersion == 2) { LOCK(cs_main); // fProvidesHeaderAndIDs is used to "lock in" version of compact blocks we send (fWantsCmpctWitness) if (!State(pfrom.GetId())->fProvidesHeaderAndIDs) { @@ -2699,10 +2768,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, pfrom.m_bip152_highbandwidth_from = fAnnounceUsingCMPCTBLOCK; } if (!State(pfrom.GetId())->fSupportsDesiredCmpctVersion) { - if (pfrom.GetLocalServices() & NODE_WITNESS) - State(pfrom.GetId())->fSupportsDesiredCmpctVersion = (nCMPCTBLOCKVersion == 2); - else - State(pfrom.GetId())->fSupportsDesiredCmpctVersion = (nCMPCTBLOCKVersion == 1); + State(pfrom.GetId())->fSupportsDesiredCmpctVersion = (nCMPCTBLOCKVersion == 2); } } return; @@ -2762,10 +2828,11 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, s >> vAddr; - if (!RelayAddrsWithPeer(*peer)) { + if (!SetupAddressRelay(pfrom, *peer)) { LogPrint(BCLog::NET, "ignoring %s message from %s peer=%d\n", msg_type, pfrom.ConnectionTypeAsString(), pfrom.GetId()); return; } + if (vAddr.size() > MAX_ADDR_TO_SEND) { Misbehaving(pfrom.GetId(), 20, strprintf("%s message size = %u", msg_type, vAddr.size())); @@ -2776,11 +2843,35 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, std::vector<CAddress> vAddrOk; int64_t nNow = GetAdjustedTime(); int64_t nSince = nNow - 10 * 60; + + // Update/increment addr rate limiting bucket. + const auto current_time = GetTime<std::chrono::microseconds>(); + if (peer->m_addr_token_bucket < MAX_ADDR_PROCESSING_TOKEN_BUCKET) { + // Don't increment bucket if it's already full + const auto time_diff = std::max(current_time - peer->m_addr_token_timestamp, 0us); + const double increment = CountSecondsDouble(time_diff) * MAX_ADDR_RATE_PER_SECOND; + peer->m_addr_token_bucket = std::min<double>(peer->m_addr_token_bucket + increment, MAX_ADDR_PROCESSING_TOKEN_BUCKET); + } + peer->m_addr_token_timestamp = current_time; + + const bool rate_limited = !pfrom.HasPermission(NetPermissionFlags::Addr); + uint64_t num_proc = 0; + uint64_t num_rate_limit = 0; + Shuffle(vAddr.begin(), vAddr.end(), FastRandomContext()); for (CAddress& addr : vAddr) { if (interruptMsgProc) return; + // Apply rate limiting. + if (peer->m_addr_token_bucket < 1.0) { + if (rate_limited) { + ++num_rate_limit; + continue; + } + } else { + peer->m_addr_token_bucket -= 1.0; + } // We only bother storing full nodes, though this may include // things which we would not make an outbound connection to, in // part because we may make feeler connections to them. @@ -2794,6 +2885,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Do not process banned/discouraged addresses beyond remembering we received them continue; } + ++num_proc; bool fReachable = IsReachable(addr); if (addr.nTime > nSince && !peer->m_getaddr_sent && vAddr.size() <= 10 && addr.IsRoutable()) { // Relay to a limited number of other nodes @@ -2803,9 +2895,16 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (fReachable) vAddrOk.push_back(addr); } + peer->m_addr_processed += num_proc; + peer->m_addr_rate_limited += num_rate_limit; + LogPrint(BCLog::NET, "Received addr: %u addresses (%u processed, %u rate-limited) from peer=%d\n", + vAddr.size(), num_proc, num_rate_limit, pfrom.GetId()); + m_addrman.Add(vAddrOk, pfrom.addr, 2 * 60 * 60); if (vAddr.size() < 1000) peer->m_getaddr_sent = false; - if (pfrom.IsAddrFetchConn()) { + + // AddrFetch: Require multiple addresses to avoid disconnecting on self-announcements + if (pfrom.IsAddrFetchConn() && vAddr.size() > 1) { LogPrint(BCLog::NET, "addrfetch connection completed peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; } @@ -2821,13 +2920,13 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } - // We won't accept tx inv's if we're in blocks-only mode, or this is a + // Reject tx INVs when the -blocksonly setting is enabled, or this is a // block-relay-only peer - bool fBlocksOnly = m_ignore_incoming_txs || (pfrom.m_tx_relay == nullptr); + bool reject_tx_invs{m_ignore_incoming_txs || (pfrom.m_tx_relay == nullptr)}; // Allow peers with relay permission to send data other than blocks in blocks only mode if (pfrom.HasPermission(NetPermissionFlags::Relay)) { - fBlocksOnly = false; + reject_tx_invs = false; } LOCK(cs_main); @@ -2866,7 +2965,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom.GetId()); pfrom.AddKnownTx(inv.hash); - if (fBlocksOnly) { + if (reject_tx_invs) { LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol, disconnecting peer=%d\n", inv.hash.ToString(), pfrom.GetId()); pfrom.fDisconnect = true; return; @@ -2935,7 +3034,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, a_recent_block = most_recent_block; } BlockValidationState state; - if (!m_chainman.ActiveChainstate().ActivateBestChain(state, m_chainparams, a_recent_block)) { + if (!m_chainman.ActiveChainstate().ActivateBestChain(state, a_recent_block)) { LogPrint(BCLog::NET, "failed to activate chain (%s)\n", state.ToString()); } } @@ -3199,7 +3298,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, std::sort(unique_parents.begin(), unique_parents.end()); unique_parents.erase(std::unique(unique_parents.begin(), unique_parents.end()), unique_parents.end()); for (const uint256& parent_txid : unique_parents) { - if (recentRejects->contains(parent_txid)) { + if (m_recent_rejects.contains(parent_txid)) { fRejectedParents = true; break; } @@ -3227,7 +3326,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, m_txrequest.ForgetTxHash(tx.GetWitnessHash()); // DoS prevention: do not allow m_orphanage to grow unbounded (see CVE-2012-3789) - unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, gArgs.GetArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS)); + unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, gArgs.GetIntArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS)); unsigned int nEvicted = m_orphanage.LimitOrphans(nMaxOrphanTx); if (nEvicted > 0) { LogPrint(BCLog::MEMPOOL, "orphanage overflow, removed %u tx\n", nEvicted); @@ -3240,8 +3339,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // regardless of what witness is provided, we will not accept // this, so we don't need to allow for redownload of this txid // from any of our non-wtxidrelay peers. - recentRejects->insert(tx.GetHash()); - recentRejects->insert(tx.GetWitnessHash()); + m_recent_rejects.insert(tx.GetHash()); + m_recent_rejects.insert(tx.GetWitnessHash()); m_txrequest.ForgetTxHash(tx.GetHash()); m_txrequest.ForgetTxHash(tx.GetWitnessHash()); } @@ -3260,8 +3359,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // See also comments in https://github.com/bitcoin/bitcoin/pull/18044#discussion_r443419034 // for concerns around weakening security of unupgraded nodes // if we start doing this too early. - assert(recentRejects); - recentRejects->insert(tx.GetWitnessHash()); + m_recent_rejects.insert(tx.GetWitnessHash()); m_txrequest.ForgetTxHash(tx.GetWitnessHash()); // If the transaction failed for TX_INPUTS_NOT_STANDARD, // then we know that the witness was irrelevant to the policy @@ -3272,7 +3370,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // transactions are later received (resulting in // parent-fetching by txid via the orphan-handling logic). if (state.GetResult() == TxValidationResult::TX_INPUTS_NOT_STANDARD && tx.GetWitnessHash() != tx.GetHash()) { - recentRejects->insert(tx.GetHash()); + m_recent_rejects.insert(tx.GetHash()); m_txrequest.ForgetTxHash(tx.GetHash()); } if (RecursiveDynamicUsage(*ptx) < 100000) { @@ -3281,21 +3379,21 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } } - // If a tx has been detected by recentRejects, we will have reached + // If a tx has been detected by m_recent_rejects, we will have reached // this point and the tx will have been ignored. Because we haven't run // the tx through AcceptToMemoryPool, we won't have computed a DoS // score for it or determined exactly why we consider it invalid. // // This means we won't penalize any peer subsequently relaying a DoSy // tx (even if we penalized the first peer who gave it to us) because - // we have to account for recentRejects showing false positives. In + // we have to account for m_recent_rejects showing false positives. In // other words, we shouldn't penalize a peer if we aren't *sure* they // submitted a DoSy tx. // - // Note that recentRejects doesn't just record DoSy or invalid + // Note that m_recent_rejects doesn't just record DoSy or invalid // transactions, but any tx not accepted by the mempool, which may be // due to node policy (vs. consensus). So we can't blanket penalize a - // peer simply for relaying a tx that our recentRejects has caught, + // peer simply for relaying a tx that our m_recent_rejects has caught, // regardless of false positives. if (state.IsInvalid()) { @@ -3397,7 +3495,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } - if (IsWitnessEnabled(pindex->pprev, m_chainparams.GetConsensus()) && !nodestate->fSupportsDesiredCmpctVersion) { + if (DeploymentActiveAt(*pindex, m_chainparams.GetConsensus(), Consensus::DEPLOYMENT_SEGWIT) && !nodestate->fSupportsDesiredCmpctVersion) { // Don't bother trying to process compact blocks from v1 peers // after segwit activates. return; @@ -3664,6 +3762,10 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } + // Since this must be an inbound connection, SetupAddressRelay will + // never fail. + Assume(SetupAddressRelay(pfrom, *peer)); + // Only send one GetAddr response per connection to reduce resource waste // and discourage addr stamping of INV announcements. if (peer->m_getaddr_recvd) { @@ -3993,6 +4095,15 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt } CNetMessage& msg(msgs.front()); + TRACE6(net, inbound_message, + pfrom->GetId(), + pfrom->m_addr_name.c_str(), + pfrom->ConnectionTypeAsString().c_str(), + msg.m_command.c_str(), + msg.m_recv.size(), + msg.m_recv.data() + ); + if (gArgs.GetBoolArg("-capturemessages", false)) { CaptureMessage(pfrom->addr, msg.m_command, MakeUCharSpan(msg.m_recv), /* incoming */ true); } @@ -4242,7 +4353,7 @@ void PeerManagerImpl::MaybeSendPing(CNode& node_to, Peer& peer, std::chrono::mic void PeerManagerImpl::MaybeSendAddr(CNode& node, Peer& peer, std::chrono::microseconds current_time) { // Nothing to do for non-address-relay peers - if (!RelayAddrsWithPeer(peer)) return; + if (!peer.m_addr_relay_enabled) return; LOCK(peer.m_addr_send_times_mutex); // Periodically advertise our local address to the peer. @@ -4316,7 +4427,7 @@ void PeerManagerImpl::MaybeSendFeefilter(CNode& pto, std::chrono::microseconds c // peers with the forcerelay permission should not filter txs to us if (pto.HasPermission(NetPermissionFlags::ForceRelay)) return; - CAmount currentFilter = m_mempool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); + CAmount currentFilter = m_mempool.GetMinFee(gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); static FeeFilterRounder g_filter_rounder{CFeeRate{DEFAULT_MIN_RELAY_TX_FEE}}; if (m_chainman.ActiveChainstate().IsInitialBlockDownload()) { @@ -4370,6 +4481,22 @@ public: }; } +bool PeerManagerImpl::SetupAddressRelay(const CNode& node, Peer& peer) +{ + // We don't participate in addr relay with outbound block-relay-only + // connections to prevent providing adversaries with the additional + // information of addr traffic to infer the link. + if (node.IsBlockOnlyConn()) return false; + + if (!peer.m_addr_relay_enabled.exchange(true)) { + // First addr message we have received from the peer, initialize + // m_addr_known + peer.m_addr_known = std::make_unique<CRollingBloomFilter>(5000, 0.001); + } + + return true; +} + bool PeerManagerImpl::SendMessages(CNode* pto) { PeerRef peer = GetPeerRef(pto->GetId()); @@ -4389,6 +4516,12 @@ bool PeerManagerImpl::SendMessages(CNode* pto) const auto current_time = GetTime<std::chrono::microseconds>(); + if (pto->IsAddrFetchConn() && current_time - std::chrono::seconds(pto->nTimeConnected) > 10 * AVG_ADDRESS_BROADCAST_INTERVAL) { + LogPrint(BCLog::NET, "addrfetch connection timeout; disconnecting peer=%d\n", pto->GetId()); + pto->fDisconnect = true; + return true; + } + MaybeSendPing(*pto, *peer, current_time); // MaybeSendPing may have marked peer for disconnection diff --git a/src/net_processing.h b/src/net_processing.h index d5801aadd3..27bc40687a 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -9,7 +9,7 @@ #include <net.h> #include <validationinterface.h> -class CAddrMan; +class AddrMan; class CChainParams; class CTxMemPool; class ChainstateManager; @@ -29,16 +29,22 @@ struct CNodeStateStats { int m_starting_height = -1; std::chrono::microseconds m_ping_wait; std::vector<int> vHeightInFlight; + uint64_t m_addr_processed = 0; + uint64_t m_addr_rate_limited = 0; + bool m_addr_relay_enabled{false}; }; class PeerManager : public CValidationInterface, public NetEventsInterface { public: - static std::unique_ptr<PeerManager> make(const CChainParams& chainparams, CConnman& connman, CAddrMan& addrman, - BanMan* banman, CScheduler& scheduler, ChainstateManager& chainman, + static std::unique_ptr<PeerManager> make(const CChainParams& chainparams, CConnman& connman, AddrMan& addrman, + BanMan* banman, ChainstateManager& chainman, CTxMemPool& pool, bool ignore_incoming_txs); virtual ~PeerManager() { } + /** Begin running background tasks, should only be called once */ + virtual void StartScheduledTasks(CScheduler& scheduler) = 0; + /** Get statistics from node state */ virtual bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) const = 0; diff --git a/src/net_types.cpp b/src/net_types.cpp new file mode 100644 index 0000000000..c8f57fe6c6 --- /dev/null +++ b/src/net_types.cpp @@ -0,0 +1,65 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <net_types.h> + +#include <netaddress.h> +#include <netbase.h> +#include <univalue.h> + +CBanEntry::CBanEntry(const UniValue& json) + : nVersion(json["version"].get_int()), nCreateTime(json["ban_created"].get_int64()), + nBanUntil(json["banned_until"].get_int64()) +{ +} + +UniValue CBanEntry::ToJson() const +{ + UniValue json(UniValue::VOBJ); + json.pushKV("version", nVersion); + json.pushKV("ban_created", nCreateTime); + json.pushKV("banned_until", nBanUntil); + return json; +} + +static const char* BANMAN_JSON_ADDR_KEY = "address"; + +/** + * Convert a `banmap_t` object to a JSON array. + * @param[in] bans Bans list to convert. + * @return a JSON array, similar to the one returned by the `listbanned` RPC. Suitable for + * passing to `BanMapFromJson()`. + */ +UniValue BanMapToJson(const banmap_t& bans) +{ + UniValue bans_json(UniValue::VARR); + for (const auto& it : bans) { + const auto& address = it.first; + const auto& ban_entry = it.second; + UniValue j = ban_entry.ToJson(); + j.pushKV(BANMAN_JSON_ADDR_KEY, address.ToString()); + bans_json.push_back(j); + } + return bans_json; +} + +/** + * Convert a JSON array to a `banmap_t` object. + * @param[in] bans_json JSON to convert, must be as returned by `BanMapToJson()`. + * @param[out] bans Bans list to create from the JSON. + * @throws std::runtime_error if the JSON does not have the expected fields or they contain + * unparsable values. + */ +void BanMapFromJson(const UniValue& bans_json, banmap_t& bans) +{ + for (const auto& ban_entry_json : bans_json.getValues()) { + CSubNet subnet; + const auto& subnet_str = ban_entry_json[BANMAN_JSON_ADDR_KEY].get_str(); + if (!LookupSubNet(subnet_str, subnet)) { + throw std::runtime_error( + strprintf("Cannot parse banned address or subnet: %s", subnet_str)); + } + bans.insert_or_assign(subnet, CBanEntry{ban_entry_json}); + } +} diff --git a/src/net_types.h b/src/net_types.h index d55a8cde6c..ffdc24c772 100644 --- a/src/net_types.h +++ b/src/net_types.h @@ -5,11 +5,56 @@ #ifndef BITCOIN_NET_TYPES_H #define BITCOIN_NET_TYPES_H +#include <cstdint> #include <map> -class CBanEntry; class CSubNet; +class UniValue; + +class CBanEntry +{ +public: + static constexpr int CURRENT_VERSION{1}; + int nVersion{CBanEntry::CURRENT_VERSION}; + int64_t nCreateTime{0}; + int64_t nBanUntil{0}; + + CBanEntry() {} + + explicit CBanEntry(int64_t nCreateTimeIn) + : nCreateTime{nCreateTimeIn} {} + + /** + * Create a ban entry from JSON. + * @param[in] json A JSON representation of a ban entry, as created by `ToJson()`. + * @throw std::runtime_error if the JSON does not have the expected fields. + */ + explicit CBanEntry(const UniValue& json); + + /** + * Generate a JSON representation of this ban entry. + * @return JSON suitable for passing to the `CBanEntry(const UniValue&)` constructor. + */ + UniValue ToJson() const; +}; using banmap_t = std::map<CSubNet, CBanEntry>; +/** + * Convert a `banmap_t` object to a JSON array. + * @param[in] bans Bans list to convert. + * @return a JSON array, similar to the one returned by the `listbanned` RPC. Suitable for + * passing to `BanMapFromJson()`. + */ +UniValue BanMapToJson(const banmap_t& bans); + +/** + * Convert a JSON array to a `banmap_t` object. + * @param[in] bans_json JSON to convert, must be as returned by `BanMapToJson()`. + * @param[out] bans Bans list to create from the JSON. + * @throws std::runtime_error if the JSON does not have the expected fields or they contain + * unparsable values. + */ +void BanMapFromJson(const UniValue& bans_json, banmap_t& bans); + #endif // BITCOIN_NET_TYPES_H diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 1ea3969978..f9fff5a6d5 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -165,7 +165,7 @@ void CNetAddr::SetLegacyIPv6(Span<const uint8_t> ipv6) } /** - * Create an "internal" address that represents a name or FQDN. CAddrMan uses + * Create an "internal" address that represents a name or FQDN. AddrMan uses * these fake addresses to keep track of which DNS seeds were used. * @returns Whether or not the operation was successful. * @see NET_INTERNAL, INTERNAL_IN_IPV6_PREFIX, CNetAddr::IsInternal(), CNetAddr::IsRFC4193() @@ -489,7 +489,7 @@ bool CNetAddr::IsValid() const */ bool CNetAddr::IsRoutable() const { - return IsValid() && !(IsRFC1918() || IsRFC2544() || IsRFC3927() || IsRFC4862() || IsRFC6598() || IsRFC5737() || (IsRFC4193() && !IsTor()) || IsRFC4843() || IsRFC7343() || IsLocal() || IsInternal()); + return IsValid() && !(IsRFC1918() || IsRFC2544() || IsRFC3927() || IsRFC4862() || IsRFC6598() || IsRFC5737() || IsRFC4193() || IsRFC4843() || IsRFC7343() || IsLocal() || IsInternal()); } /** @@ -1242,8 +1242,3 @@ bool operator<(const CSubNet& a, const CSubNet& b) { return (a.network < b.network || (a.network == b.network && memcmp(a.netmask, b.netmask, 16) < 0)); } - -bool SanityCheckASMap(const std::vector<bool>& asmap) -{ - return SanityCheckASMap(asmap, 128); // For IP address lookups, the input is 128 bits -} diff --git a/src/netaddress.h b/src/netaddress.h index dd47ab5749..66c8c48f08 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -42,8 +42,7 @@ static constexpr int ADDRV2_FORMAT = 0x20000000; * over all enum values and also `GetExtNetwork()` "extends" this enum by * introducing standalone constants starting from `NET_MAX`. */ -enum Network -{ +enum Network { /// Addresses from these networks are not publicly routable on the global Internet. NET_UNROUTABLE = 0, @@ -63,7 +62,7 @@ enum Network NET_CJDNS, /// A set of addresses that represent the hash of a string or FQDN. We use - /// them in CAddrMan to keep track of which DNS seeds were used. + /// them in AddrMan to keep track of which DNS seeds were used. NET_INTERNAL, /// Dummy value to indicate the number of NET_* constants. @@ -73,16 +72,14 @@ enum Network /// Prefix of an IPv6 address when it contains an embedded IPv4 address. /// Used when (un)serializing addresses in ADDRv1 format (pre-BIP155). static const std::array<uint8_t, 12> IPV4_IN_IPV6_PREFIX{ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF -}; + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF}; /// Prefix of an IPv6 address when it contains an embedded TORv2 address. /// Used when (un)serializing addresses in ADDRv1 format (pre-BIP155). /// Such dummy IPv6 addresses are guaranteed to not be publicly routable as they /// fall under RFC4193's fc00::/7 subnet allocated to unique-local addresses. static const std::array<uint8_t, 6> TORV2_IN_IPV6_PREFIX{ - 0xFD, 0x87, 0xD8, 0x7E, 0xEB, 0x43 -}; + 0xFD, 0x87, 0xD8, 0x7E, 0xEB, 0x43}; /// Prefix of an IPv6 address when it contains an embedded "internal" address. /// Used when (un)serializing addresses in ADDRv1 format (pre-BIP155). @@ -112,359 +109,362 @@ static constexpr size_t ADDR_CJDNS_SIZE = 16; /// Size of "internal" (NET_INTERNAL) address (in bytes). static constexpr size_t ADDR_INTERNAL_SIZE = 10; +/// SAM 3.1 and earlier do not support specifying ports and force the port to 0. +static constexpr uint16_t I2P_SAM31_PORT{0}; + /** * Network address. */ class CNetAddr { - protected: - /** - * Raw representation of the network address. - * In network byte order (big endian) for IPv4 and IPv6. - */ - prevector<ADDR_IPV6_SIZE, uint8_t> m_addr{ADDR_IPV6_SIZE, 0x0}; - - /** - * Network to which this address belongs. - */ - Network m_net{NET_IPV6}; - - /** - * Scope id if scoped/link-local IPV6 address. - * See https://tools.ietf.org/html/rfc4007 - */ - uint32_t m_scope_id{0}; - - public: - CNetAddr(); - explicit CNetAddr(const struct in_addr& ipv4Addr); - void SetIP(const CNetAddr& ip); - - /** - * Set from a legacy IPv6 address. - * Legacy IPv6 address may be a normal IPv6 address, or another address - * (e.g. IPv4) disguised as IPv6. This encoding is used in the legacy - * `addr` encoding. - */ - void SetLegacyIPv6(Span<const uint8_t> ipv6); - - bool SetInternal(const std::string& name); - - /** - * Parse a Tor or I2P address and set this object to it. - * @param[in] addr Address to parse, for example - * pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion or - * ukeu3k5oycgaauneqgtnvselmt4yemvoilkln7jpvamvfx7dnkdq.b32.i2p. - * @returns Whether the operation was successful. - * @see CNetAddr::IsTor(), CNetAddr::IsI2P() - */ - bool SetSpecial(const std::string& addr); - - bool IsBindAny() const; // INADDR_ANY equivalent - bool IsIPv4() const; // IPv4 mapped address (::FFFF:0:0/96, 0.0.0.0/0) - bool IsIPv6() const; // IPv6 address (not mapped IPv4, not Tor) - bool IsRFC1918() const; // IPv4 private networks (10.0.0.0/8, 192.168.0.0/16, 172.16.0.0/12) - bool IsRFC2544() const; // IPv4 inter-network communications (198.18.0.0/15) - bool IsRFC6598() const; // IPv4 ISP-level NAT (100.64.0.0/10) - bool IsRFC5737() const; // IPv4 documentation addresses (192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24) - bool IsRFC3849() const; // IPv6 documentation address (2001:0DB8::/32) - bool IsRFC3927() const; // IPv4 autoconfig (169.254.0.0/16) - bool IsRFC3964() const; // IPv6 6to4 tunnelling (2002::/16) - bool IsRFC4193() const; // IPv6 unique local (FC00::/7) - bool IsRFC4380() const; // IPv6 Teredo tunnelling (2001::/32) - bool IsRFC4843() const; // IPv6 ORCHID (deprecated) (2001:10::/28) - bool IsRFC7343() const; // IPv6 ORCHIDv2 (2001:20::/28) - bool IsRFC4862() const; // IPv6 autoconfig (FE80::/64) - bool IsRFC6052() const; // IPv6 well-known prefix for IPv4-embedded address (64:FF9B::/96) - bool IsRFC6145() const; // IPv6 IPv4-translated address (::FFFF:0:0:0/96) (actually defined in RFC2765) - bool IsHeNet() const; // IPv6 Hurricane Electric - https://he.net (2001:0470::/36) - bool IsTor() const; - bool IsI2P() const; - bool IsCJDNS() const; - bool IsLocal() const; - bool IsRoutable() const; - bool IsInternal() const; - bool IsValid() const; - - /** - * Check if the current object can be serialized in pre-ADDRv2/BIP155 format. - */ - bool IsAddrV1Compatible() const; - - enum Network GetNetwork() const; - std::string ToString() const; - std::string ToStringIP() const; - uint64_t GetHash() const; - bool GetInAddr(struct in_addr* pipv4Addr) const; - Network GetNetClass() const; - - //! For IPv4, mapped IPv4, SIIT translated IPv4, Teredo, 6to4 tunneled addresses, return the relevant IPv4 address as a uint32. - uint32_t GetLinkedIPv4() const; - //! Whether this address has a linked IPv4 address (see GetLinkedIPv4()). - bool HasLinkedIPv4() const; - - // The AS on the BGP path to the node we use to diversify - // peers in AddrMan bucketing based on the AS infrastructure. - // The ip->AS mapping depends on how asmap is constructed. - uint32_t GetMappedAS(const std::vector<bool> &asmap) const; - - std::vector<unsigned char> GetGroup(const std::vector<bool> &asmap) const; - std::vector<unsigned char> GetAddrBytes() const; - int GetReachabilityFrom(const CNetAddr *paddrPartner = nullptr) const; - - explicit CNetAddr(const struct in6_addr& pipv6Addr, const uint32_t scope = 0); - bool GetIn6Addr(struct in6_addr* pipv6Addr) const; - - friend bool operator==(const CNetAddr& a, const CNetAddr& b); - friend bool operator!=(const CNetAddr& a, const CNetAddr& b) { return !(a == b); } - friend bool operator<(const CNetAddr& a, const CNetAddr& b); - - /** - * Whether this address should be relayed to other peers even if we can't reach it ourselves. - */ - bool IsRelayable() const - { - return IsIPv4() || IsIPv6() || IsTor(); - } +protected: + /** + * Raw representation of the network address. + * In network byte order (big endian) for IPv4 and IPv6. + */ + prevector<ADDR_IPV6_SIZE, uint8_t> m_addr{ADDR_IPV6_SIZE, 0x0}; + + /** + * Network to which this address belongs. + */ + Network m_net{NET_IPV6}; + + /** + * Scope id if scoped/link-local IPV6 address. + * See https://tools.ietf.org/html/rfc4007 + */ + uint32_t m_scope_id{0}; - /** - * Serialize to a stream. - */ - template <typename Stream> - void Serialize(Stream& s) const - { - if (s.GetVersion() & ADDRV2_FORMAT) { - SerializeV2Stream(s); - } else { - SerializeV1Stream(s); - } - } +public: + CNetAddr(); + explicit CNetAddr(const struct in_addr& ipv4Addr); + void SetIP(const CNetAddr& ip); + + /** + * Set from a legacy IPv6 address. + * Legacy IPv6 address may be a normal IPv6 address, or another address + * (e.g. IPv4) disguised as IPv6. This encoding is used in the legacy + * `addr` encoding. + */ + void SetLegacyIPv6(Span<const uint8_t> ipv6); + + bool SetInternal(const std::string& name); + + /** + * Parse a Tor or I2P address and set this object to it. + * @param[in] addr Address to parse, for example + * pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion or + * ukeu3k5oycgaauneqgtnvselmt4yemvoilkln7jpvamvfx7dnkdq.b32.i2p. + * @returns Whether the operation was successful. + * @see CNetAddr::IsTor(), CNetAddr::IsI2P() + */ + bool SetSpecial(const std::string& addr); + + bool IsBindAny() const; // INADDR_ANY equivalent + bool IsIPv4() const; // IPv4 mapped address (::FFFF:0:0/96, 0.0.0.0/0) + bool IsIPv6() const; // IPv6 address (not mapped IPv4, not Tor) + bool IsRFC1918() const; // IPv4 private networks (10.0.0.0/8, 192.168.0.0/16, 172.16.0.0/12) + bool IsRFC2544() const; // IPv4 inter-network communications (198.18.0.0/15) + bool IsRFC6598() const; // IPv4 ISP-level NAT (100.64.0.0/10) + bool IsRFC5737() const; // IPv4 documentation addresses (192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24) + bool IsRFC3849() const; // IPv6 documentation address (2001:0DB8::/32) + bool IsRFC3927() const; // IPv4 autoconfig (169.254.0.0/16) + bool IsRFC3964() const; // IPv6 6to4 tunnelling (2002::/16) + bool IsRFC4193() const; // IPv6 unique local (FC00::/7) + bool IsRFC4380() const; // IPv6 Teredo tunnelling (2001::/32) + bool IsRFC4843() const; // IPv6 ORCHID (deprecated) (2001:10::/28) + bool IsRFC7343() const; // IPv6 ORCHIDv2 (2001:20::/28) + bool IsRFC4862() const; // IPv6 autoconfig (FE80::/64) + bool IsRFC6052() const; // IPv6 well-known prefix for IPv4-embedded address (64:FF9B::/96) + bool IsRFC6145() const; // IPv6 IPv4-translated address (::FFFF:0:0:0/96) (actually defined in RFC2765) + bool IsHeNet() const; // IPv6 Hurricane Electric - https://he.net (2001:0470::/36) + bool IsTor() const; + bool IsI2P() const; + bool IsCJDNS() const; + bool IsLocal() const; + bool IsRoutable() const; + bool IsInternal() const; + bool IsValid() const; + + /** + * Check if the current object can be serialized in pre-ADDRv2/BIP155 format. + */ + bool IsAddrV1Compatible() const; + + enum Network GetNetwork() const; + std::string ToString() const; + std::string ToStringIP() const; + uint64_t GetHash() const; + bool GetInAddr(struct in_addr* pipv4Addr) const; + Network GetNetClass() const; + + //! For IPv4, mapped IPv4, SIIT translated IPv4, Teredo, 6to4 tunneled addresses, return the relevant IPv4 address as a uint32. + uint32_t GetLinkedIPv4() const; + //! Whether this address has a linked IPv4 address (see GetLinkedIPv4()). + bool HasLinkedIPv4() const; + + // The AS on the BGP path to the node we use to diversify + // peers in AddrMan bucketing based on the AS infrastructure. + // The ip->AS mapping depends on how asmap is constructed. + uint32_t GetMappedAS(const std::vector<bool>& asmap) const; + + std::vector<unsigned char> GetGroup(const std::vector<bool>& asmap) const; + std::vector<unsigned char> GetAddrBytes() const; + int GetReachabilityFrom(const CNetAddr* paddrPartner = nullptr) const; + + explicit CNetAddr(const struct in6_addr& pipv6Addr, const uint32_t scope = 0); + bool GetIn6Addr(struct in6_addr* pipv6Addr) const; + + friend bool operator==(const CNetAddr& a, const CNetAddr& b); + friend bool operator!=(const CNetAddr& a, const CNetAddr& b) { return !(a == b); } + friend bool operator<(const CNetAddr& a, const CNetAddr& b); + + /** + * Whether this address should be relayed to other peers even if we can't reach it ourselves. + */ + bool IsRelayable() const + { + return IsIPv4() || IsIPv6() || IsTor() || IsI2P(); + } - /** - * Unserialize from a stream. - */ - template <typename Stream> - void Unserialize(Stream& s) - { - if (s.GetVersion() & ADDRV2_FORMAT) { - UnserializeV2Stream(s); - } else { - UnserializeV1Stream(s); - } + /** + * Serialize to a stream. + */ + template <typename Stream> + void Serialize(Stream& s) const + { + if (s.GetVersion() & ADDRV2_FORMAT) { + SerializeV2Stream(s); + } else { + SerializeV1Stream(s); } + } - friend class CNetAddrHash; - friend class CSubNet; - - private: - /** - * Parse a Tor address and set this object to it. - * @param[in] addr Address to parse, must be a valid C string, for example - * pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion. - * @returns Whether the operation was successful. - * @see CNetAddr::IsTor() - */ - bool SetTor(const std::string& addr); - - /** - * Parse an I2P address and set this object to it. - * @param[in] addr Address to parse, must be a valid C string, for example - * ukeu3k5oycgaauneqgtnvselmt4yemvoilkln7jpvamvfx7dnkdq.b32.i2p. - * @returns Whether the operation was successful. - * @see CNetAddr::IsI2P() - */ - bool SetI2P(const std::string& addr); - - /** - * BIP155 network ids recognized by this software. - */ - enum BIP155Network : uint8_t { - IPV4 = 1, - IPV6 = 2, - TORV2 = 3, - TORV3 = 4, - I2P = 5, - CJDNS = 6, - }; - - /** - * Size of CNetAddr when serialized as ADDRv1 (pre-BIP155) (in bytes). - */ - static constexpr size_t V1_SERIALIZATION_SIZE = ADDR_IPV6_SIZE; - - /** - * Maximum size of an address as defined in BIP155 (in bytes). - * This is only the size of the address, not the entire CNetAddr object - * when serialized. - */ - static constexpr size_t MAX_ADDRV2_SIZE = 512; - - /** - * Get the BIP155 network id of this address. - * Must not be called for IsInternal() objects. - * @returns BIP155 network id, except TORV2 which is no longer supported. - */ - BIP155Network GetBIP155Network() const; - - /** - * Set `m_net` from the provided BIP155 network id and size after validation. - * @retval true the network was recognized, is valid and `m_net` was set - * @retval false not recognised (from future?) and should be silently ignored - * @throws std::ios_base::failure if the network is one of the BIP155 founding - * networks (id 1..6) with wrong address size. - */ - bool SetNetFromBIP155Network(uint8_t possible_bip155_net, size_t address_size); - - /** - * Serialize in pre-ADDRv2/BIP155 format to an array. - */ - void SerializeV1Array(uint8_t (&arr)[V1_SERIALIZATION_SIZE]) const - { - size_t prefix_size; - - switch (m_net) { - case NET_IPV6: - assert(m_addr.size() == sizeof(arr)); - memcpy(arr, m_addr.data(), m_addr.size()); - return; - case NET_IPV4: - prefix_size = sizeof(IPV4_IN_IPV6_PREFIX); - assert(prefix_size + m_addr.size() == sizeof(arr)); - memcpy(arr, IPV4_IN_IPV6_PREFIX.data(), prefix_size); - memcpy(arr + prefix_size, m_addr.data(), m_addr.size()); - return; - case NET_INTERNAL: - prefix_size = sizeof(INTERNAL_IN_IPV6_PREFIX); - assert(prefix_size + m_addr.size() == sizeof(arr)); - memcpy(arr, INTERNAL_IN_IPV6_PREFIX.data(), prefix_size); - memcpy(arr + prefix_size, m_addr.data(), m_addr.size()); - return; - case NET_ONION: - case NET_I2P: - case NET_CJDNS: - break; - case NET_UNROUTABLE: - case NET_MAX: - assert(false); - } // no default case, so the compiler can warn about missing cases - - // Serialize ONION, I2P and CJDNS as all-zeros. - memset(arr, 0x0, V1_SERIALIZATION_SIZE); + /** + * Unserialize from a stream. + */ + template <typename Stream> + void Unserialize(Stream& s) + { + if (s.GetVersion() & ADDRV2_FORMAT) { + UnserializeV2Stream(s); + } else { + UnserializeV1Stream(s); } + } - /** - * Serialize in pre-ADDRv2/BIP155 format to a stream. - */ - template <typename Stream> - void SerializeV1Stream(Stream& s) const - { - uint8_t serialized[V1_SERIALIZATION_SIZE]; + friend class CNetAddrHash; + friend class CSubNet; - SerializeV1Array(serialized); +private: + /** + * Parse a Tor address and set this object to it. + * @param[in] addr Address to parse, must be a valid C string, for example + * pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion. + * @returns Whether the operation was successful. + * @see CNetAddr::IsTor() + */ + bool SetTor(const std::string& addr); + + /** + * Parse an I2P address and set this object to it. + * @param[in] addr Address to parse, must be a valid C string, for example + * ukeu3k5oycgaauneqgtnvselmt4yemvoilkln7jpvamvfx7dnkdq.b32.i2p. + * @returns Whether the operation was successful. + * @see CNetAddr::IsI2P() + */ + bool SetI2P(const std::string& addr); + + /** + * BIP155 network ids recognized by this software. + */ + enum BIP155Network : uint8_t { + IPV4 = 1, + IPV6 = 2, + TORV2 = 3, + TORV3 = 4, + I2P = 5, + CJDNS = 6, + }; + + /** + * Size of CNetAddr when serialized as ADDRv1 (pre-BIP155) (in bytes). + */ + static constexpr size_t V1_SERIALIZATION_SIZE = ADDR_IPV6_SIZE; + + /** + * Maximum size of an address as defined in BIP155 (in bytes). + * This is only the size of the address, not the entire CNetAddr object + * when serialized. + */ + static constexpr size_t MAX_ADDRV2_SIZE = 512; + + /** + * Get the BIP155 network id of this address. + * Must not be called for IsInternal() objects. + * @returns BIP155 network id, except TORV2 which is no longer supported. + */ + BIP155Network GetBIP155Network() const; + + /** + * Set `m_net` from the provided BIP155 network id and size after validation. + * @retval true the network was recognized, is valid and `m_net` was set + * @retval false not recognised (from future?) and should be silently ignored + * @throws std::ios_base::failure if the network is one of the BIP155 founding + * networks (id 1..6) with wrong address size. + */ + bool SetNetFromBIP155Network(uint8_t possible_bip155_net, size_t address_size); + + /** + * Serialize in pre-ADDRv2/BIP155 format to an array. + */ + void SerializeV1Array(uint8_t (&arr)[V1_SERIALIZATION_SIZE]) const + { + size_t prefix_size; + + switch (m_net) { + case NET_IPV6: + assert(m_addr.size() == sizeof(arr)); + memcpy(arr, m_addr.data(), m_addr.size()); + return; + case NET_IPV4: + prefix_size = sizeof(IPV4_IN_IPV6_PREFIX); + assert(prefix_size + m_addr.size() == sizeof(arr)); + memcpy(arr, IPV4_IN_IPV6_PREFIX.data(), prefix_size); + memcpy(arr + prefix_size, m_addr.data(), m_addr.size()); + return; + case NET_INTERNAL: + prefix_size = sizeof(INTERNAL_IN_IPV6_PREFIX); + assert(prefix_size + m_addr.size() == sizeof(arr)); + memcpy(arr, INTERNAL_IN_IPV6_PREFIX.data(), prefix_size); + memcpy(arr + prefix_size, m_addr.data(), m_addr.size()); + return; + case NET_ONION: + case NET_I2P: + case NET_CJDNS: + break; + case NET_UNROUTABLE: + case NET_MAX: + assert(false); + } // no default case, so the compiler can warn about missing cases + + // Serialize ONION, I2P and CJDNS as all-zeros. + memset(arr, 0x0, V1_SERIALIZATION_SIZE); + } - s << serialized; - } + /** + * Serialize in pre-ADDRv2/BIP155 format to a stream. + */ + template <typename Stream> + void SerializeV1Stream(Stream& s) const + { + uint8_t serialized[V1_SERIALIZATION_SIZE]; - /** - * Serialize as ADDRv2 / BIP155. - */ - template <typename Stream> - void SerializeV2Stream(Stream& s) const - { - if (IsInternal()) { - // Serialize NET_INTERNAL as embedded in IPv6. We need to - // serialize such addresses from addrman. - s << static_cast<uint8_t>(BIP155Network::IPV6); - s << COMPACTSIZE(ADDR_IPV6_SIZE); - SerializeV1Stream(s); - return; - } + SerializeV1Array(serialized); - s << static_cast<uint8_t>(GetBIP155Network()); - s << m_addr; - } + s << serialized; + } - /** - * Unserialize from a pre-ADDRv2/BIP155 format from an array. - */ - void UnserializeV1Array(uint8_t (&arr)[V1_SERIALIZATION_SIZE]) - { - // Use SetLegacyIPv6() so that m_net is set correctly. For example - // ::FFFF:0102:0304 should be set as m_net=NET_IPV4 (1.2.3.4). - SetLegacyIPv6(arr); + /** + * Serialize as ADDRv2 / BIP155. + */ + template <typename Stream> + void SerializeV2Stream(Stream& s) const + { + if (IsInternal()) { + // Serialize NET_INTERNAL as embedded in IPv6. We need to + // serialize such addresses from addrman. + s << static_cast<uint8_t>(BIP155Network::IPV6); + s << COMPACTSIZE(ADDR_IPV6_SIZE); + SerializeV1Stream(s); + return; } - /** - * Unserialize from a pre-ADDRv2/BIP155 format from a stream. - */ - template <typename Stream> - void UnserializeV1Stream(Stream& s) - { - uint8_t serialized[V1_SERIALIZATION_SIZE]; + s << static_cast<uint8_t>(GetBIP155Network()); + s << m_addr; + } + + /** + * Unserialize from a pre-ADDRv2/BIP155 format from an array. + */ + void UnserializeV1Array(uint8_t (&arr)[V1_SERIALIZATION_SIZE]) + { + // Use SetLegacyIPv6() so that m_net is set correctly. For example + // ::FFFF:0102:0304 should be set as m_net=NET_IPV4 (1.2.3.4). + SetLegacyIPv6(arr); + } + + /** + * Unserialize from a pre-ADDRv2/BIP155 format from a stream. + */ + template <typename Stream> + void UnserializeV1Stream(Stream& s) + { + uint8_t serialized[V1_SERIALIZATION_SIZE]; - s >> serialized; + s >> serialized; + + UnserializeV1Array(serialized); + } + + /** + * Unserialize from a ADDRv2 / BIP155 format. + */ + template <typename Stream> + void UnserializeV2Stream(Stream& s) + { + uint8_t bip155_net; + s >> bip155_net; - UnserializeV1Array(serialized); + size_t address_size; + s >> COMPACTSIZE(address_size); + + if (address_size > MAX_ADDRV2_SIZE) { + throw std::ios_base::failure(strprintf( + "Address too long: %u > %u", address_size, MAX_ADDRV2_SIZE)); } - /** - * Unserialize from a ADDRv2 / BIP155 format. - */ - template <typename Stream> - void UnserializeV2Stream(Stream& s) - { - uint8_t bip155_net; - s >> bip155_net; - - size_t address_size; - s >> COMPACTSIZE(address_size); - - if (address_size > MAX_ADDRV2_SIZE) { - throw std::ios_base::failure(strprintf( - "Address too long: %u > %u", address_size, MAX_ADDRV2_SIZE)); + m_scope_id = 0; + + if (SetNetFromBIP155Network(bip155_net, address_size)) { + m_addr.resize(address_size); + s >> MakeSpan(m_addr); + + if (m_net != NET_IPV6) { + return; + } + + // Do some special checks on IPv6 addresses. + + // Recognize NET_INTERNAL embedded in IPv6, such addresses are not + // gossiped but could be coming from addrman, when unserializing from + // disk. + if (HasPrefix(m_addr, INTERNAL_IN_IPV6_PREFIX)) { + m_net = NET_INTERNAL; + memmove(m_addr.data(), m_addr.data() + INTERNAL_IN_IPV6_PREFIX.size(), + ADDR_INTERNAL_SIZE); + m_addr.resize(ADDR_INTERNAL_SIZE); + return; } - m_scope_id = 0; - - if (SetNetFromBIP155Network(bip155_net, address_size)) { - m_addr.resize(address_size); - s >> MakeSpan(m_addr); - - if (m_net != NET_IPV6) { - return; - } - - // Do some special checks on IPv6 addresses. - - // Recognize NET_INTERNAL embedded in IPv6, such addresses are not - // gossiped but could be coming from addrman, when unserializing from - // disk. - if (HasPrefix(m_addr, INTERNAL_IN_IPV6_PREFIX)) { - m_net = NET_INTERNAL; - memmove(m_addr.data(), m_addr.data() + INTERNAL_IN_IPV6_PREFIX.size(), - ADDR_INTERNAL_SIZE); - m_addr.resize(ADDR_INTERNAL_SIZE); - return; - } - - if (!HasPrefix(m_addr, IPV4_IN_IPV6_PREFIX) && - !HasPrefix(m_addr, TORV2_IN_IPV6_PREFIX)) { - return; - } - - // IPv4 and TORv2 are not supposed to be embedded in IPv6 (like in V1 - // encoding). Unserialize as !IsValid(), thus ignoring them. - } else { - // If we receive an unknown BIP155 network id (from the future?) then - // ignore the address - unserialize as !IsValid(). - s.ignore(address_size); + if (!HasPrefix(m_addr, IPV4_IN_IPV6_PREFIX) && + !HasPrefix(m_addr, TORV2_IN_IPV6_PREFIX)) { + return; } - // Mimic a default-constructed CNetAddr object which is !IsValid() and thus - // will not be gossiped, but continue reading next addresses from the stream. - m_net = NET_IPV6; - m_addr.assign(ADDR_IPV6_SIZE, 0x0); + // IPv4 and TORv2 are not supposed to be embedded in IPv6 (like in V1 + // encoding). Unserialize as !IsValid(), thus ignoring them. + } else { + // If we receive an unknown BIP155 network id (from the future?) then + // ignore the address - unserialize as !IsValid(). + s.ignore(address_size); } + + // Mimic a default-constructed CNetAddr object which is !IsValid() and thus + // will not be gossiped, but continue reading next addresses from the stream. + m_net = NET_IPV6; + m_addr.assign(ADDR_IPV6_SIZE, 0x0); + } }; class CNetAddrHash @@ -485,106 +485,86 @@ private: class CSubNet { - protected: - /// Network (base) address - CNetAddr network; - /// Netmask, in network byte order - uint8_t netmask[16]; - /// Is this value valid? (only used to signal parse errors) - bool valid; - - bool SanityCheck() const; - - public: - /** - * Construct an invalid subnet (empty, `Match()` always returns false). - */ - CSubNet(); - - /** - * Construct from a given network start and number of bits (CIDR mask). - * @param[in] addr Network start. Must be IPv4 or IPv6, otherwise an invalid subnet is - * created. - * @param[in] mask CIDR mask, must be in [0, 32] for IPv4 addresses and in [0, 128] for - * IPv6 addresses. Otherwise an invalid subnet is created. - */ - CSubNet(const CNetAddr& addr, uint8_t mask); - - /** - * Construct from a given network start and mask. - * @param[in] addr Network start. Must be IPv4 or IPv6, otherwise an invalid subnet is - * created. - * @param[in] mask Network mask, must be of the same type as `addr` and not contain 0-bits - * followed by 1-bits. Otherwise an invalid subnet is created. - */ - CSubNet(const CNetAddr& addr, const CNetAddr& mask); - - /** - * Construct a single-host subnet. - * @param[in] addr The sole address to be contained in the subnet, can also be non-IPv[46]. - */ - explicit CSubNet(const CNetAddr& addr); - - bool Match(const CNetAddr &addr) const; - - std::string ToString() const; - bool IsValid() const; - - friend bool operator==(const CSubNet& a, const CSubNet& b); - friend bool operator!=(const CSubNet& a, const CSubNet& b) { return !(a == b); } - friend bool operator<(const CSubNet& a, const CSubNet& b); - - SERIALIZE_METHODS(CSubNet, obj) - { - READWRITE(obj.network); - if (obj.network.IsIPv4()) { - // Before commit 102867c587f5f7954232fb8ed8e85cda78bb4d32, CSubNet used the last 4 bytes of netmask - // to store the relevant bytes for an IPv4 mask. For compatibility reasons, keep doing so in - // serialized form. - unsigned char dummy[12] = {0}; - READWRITE(dummy); - READWRITE(MakeSpan(obj.netmask).first(4)); - } else { - READWRITE(obj.netmask); - } - READWRITE(obj.valid); - // Mark invalid if the result doesn't pass sanity checking. - SER_READ(obj, if (obj.valid) obj.valid = obj.SanityCheck()); - } +protected: + /// Network (base) address + CNetAddr network; + /// Netmask, in network byte order + uint8_t netmask[16]; + /// Is this value valid? (only used to signal parse errors) + bool valid; + + bool SanityCheck() const; + +public: + /** + * Construct an invalid subnet (empty, `Match()` always returns false). + */ + CSubNet(); + + /** + * Construct from a given network start and number of bits (CIDR mask). + * @param[in] addr Network start. Must be IPv4 or IPv6, otherwise an invalid subnet is + * created. + * @param[in] mask CIDR mask, must be in [0, 32] for IPv4 addresses and in [0, 128] for + * IPv6 addresses. Otherwise an invalid subnet is created. + */ + CSubNet(const CNetAddr& addr, uint8_t mask); + + /** + * Construct from a given network start and mask. + * @param[in] addr Network start. Must be IPv4 or IPv6, otherwise an invalid subnet is + * created. + * @param[in] mask Network mask, must be of the same type as `addr` and not contain 0-bits + * followed by 1-bits. Otherwise an invalid subnet is created. + */ + CSubNet(const CNetAddr& addr, const CNetAddr& mask); + + /** + * Construct a single-host subnet. + * @param[in] addr The sole address to be contained in the subnet, can also be non-IPv[46]. + */ + explicit CSubNet(const CNetAddr& addr); + + bool Match(const CNetAddr& addr) const; + + std::string ToString() const; + bool IsValid() const; + + friend bool operator==(const CSubNet& a, const CSubNet& b); + friend bool operator!=(const CSubNet& a, const CSubNet& b) { return !(a == b); } + friend bool operator<(const CSubNet& a, const CSubNet& b); }; /** A combination of a network address (CNetAddr) and a (TCP) port */ class CService : public CNetAddr { - protected: - uint16_t port; // host order - - public: - CService(); - CService(const CNetAddr& ip, uint16_t port); - CService(const struct in_addr& ipv4Addr, uint16_t port); - explicit CService(const struct sockaddr_in& addr); - uint16_t GetPort() const; - bool GetSockAddr(struct sockaddr* paddr, socklen_t *addrlen) const; - bool SetSockAddr(const struct sockaddr* paddr); - friend bool operator==(const CService& a, const CService& b); - friend bool operator!=(const CService& a, const CService& b) { return !(a == b); } - friend bool operator<(const CService& a, const CService& b); - std::vector<unsigned char> GetKey() const; - std::string ToString() const; - std::string ToStringPort() const; - std::string ToStringIPPort() const; - - CService(const struct in6_addr& ipv6Addr, uint16_t port); - explicit CService(const struct sockaddr_in6& addr); - - SERIALIZE_METHODS(CService, obj) - { - READWRITEAS(CNetAddr, obj); - READWRITE(Using<BigEndianFormatter<2>>(obj.port)); - } -}; +protected: + uint16_t port; // host order -bool SanityCheckASMap(const std::vector<bool>& asmap); +public: + CService(); + CService(const CNetAddr& ip, uint16_t port); + CService(const struct in_addr& ipv4Addr, uint16_t port); + explicit CService(const struct sockaddr_in& addr); + uint16_t GetPort() const; + bool GetSockAddr(struct sockaddr* paddr, socklen_t* addrlen) const; + bool SetSockAddr(const struct sockaddr* paddr); + friend bool operator==(const CService& a, const CService& b); + friend bool operator!=(const CService& a, const CService& b) { return !(a == b); } + friend bool operator<(const CService& a, const CService& b); + std::vector<unsigned char> GetKey() const; + std::string ToString() const; + std::string ToStringPort() const; + std::string ToStringIPPort() const; + + CService(const struct in6_addr& ipv6Addr, uint16_t port); + explicit CService(const struct sockaddr_in6& addr); + + SERIALIZE_METHODS(CService, obj) + { + READWRITEAS(CNetAddr, obj); + READWRITE(Using<BigEndianFormatter<2>>(obj.port)); + } +}; #endif // BITCOIN_NETADDRESS_H diff --git a/src/netbase.cpp b/src/netbase.cpp index 2980bdf459..64d17189a6 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -23,8 +23,6 @@ #ifndef WIN32 #include <fcntl.h> -#else -#include <codecvt> #endif #ifdef USE_POLL diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 013d61282b..0b7df9bd9a 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -16,6 +16,7 @@ #include <signet.h> #include <streams.h> #include <undo.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <validation.h> @@ -85,7 +86,7 @@ void CleanupBlockRevFiles() // start removing block files. int nContigCounter = 0; for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) { - if (atoi(item.first) == nContigCounter) { + if (LocaleIndependentAtoi<int>(item.first) == nContigCounter) { nContigCounter++; continue; } @@ -394,18 +395,14 @@ bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::P bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams) { - FlatFilePos blockPos; - { - LOCK(cs_main); - blockPos = pindex->GetBlockPos(); - } + const FlatFilePos block_pos{WITH_LOCK(cs_main, return pindex->GetBlockPos())}; - if (!ReadBlockFromDisk(block, blockPos, consensusParams)) { + if (!ReadBlockFromDisk(block, block_pos, consensusParams)) { return false; } if (block.GetHash() != pindex->GetBlockHash()) { return error("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() doesn't match index for %s at %s", - pindex->ToString(), pindex->GetBlockPos().ToString()); + pindex->ToString(), block_pos.ToString()); } return true; } @@ -493,7 +490,7 @@ struct CImportingNow { void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args) { - const CChainParams& chainparams = Params(); + SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION_LOAD_BLOCKS); ScheduleBatchPriority(); { @@ -512,18 +509,18 @@ void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFile break; // This error is logged in OpenBlockFile } LogPrintf("Reindexing block file blk%05u.dat...\n", (unsigned int)nFile); - chainman.ActiveChainstate().LoadExternalBlockFile(chainparams, file, &pos); + chainman.ActiveChainstate().LoadExternalBlockFile(file, &pos); if (ShutdownRequested()) { LogPrintf("Shutdown requested. Exit %s\n", __func__); return; } nFile++; } - pblocktree->WriteReindexing(false); + WITH_LOCK(::cs_main, chainman.m_blockman.m_block_tree_db->WriteReindexing(false)); fReindex = false; LogPrintf("Reindexing finished\n"); // To avoid ending up in a situation without genesis block, re-try initializing (no-op if reindexing worked): - chainman.ActiveChainstate().LoadGenesisBlock(chainparams); + chainman.ActiveChainstate().LoadGenesisBlock(); } // -loadblock= @@ -531,7 +528,7 @@ void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFile FILE* file = fsbridge::fopen(path, "rb"); if (file) { LogPrintf("Importing blocks file %s...\n", path.string()); - chainman.ActiveChainstate().LoadExternalBlockFile(chainparams, file); + chainman.ActiveChainstate().LoadExternalBlockFile(file); if (ShutdownRequested()) { LogPrintf("Shutdown requested. Exit %s\n", __func__); return; @@ -548,7 +545,7 @@ void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFile // the relevant pointers before the ABC call. for (CChainState* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) { BlockValidationState state; - if (!chainstate->ActivateBestChain(state, chainparams, nullptr)) { + if (!chainstate->ActivateBestChain(state, nullptr)) { LogPrintf("Failed to connect best block (%s)\n", state.ToString()); StartShutdown(); return; diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 8be256edc9..9e9503ff5d 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -6,9 +6,9 @@ #ifndef BITCOIN_NODE_COINSTATS_H #define BITCOIN_NODE_COINSTATS_H -#include <amount.h> #include <chain.h> #include <coins.h> +#include <consensus/amount.h> #include <streams.h> #include <uint256.h> @@ -45,15 +45,25 @@ struct CCoinsStats bool index_used{false}; // Following values are only available from coinstats index + + //! Total cumulative amount of block subsidies up to and including this block CAmount total_subsidy{0}; - CAmount block_unspendable_amount{0}; - CAmount block_prevout_spent_amount{0}; - CAmount block_new_outputs_ex_coinbase_amount{0}; - CAmount block_coinbase_amount{0}; - CAmount unspendables_genesis_block{0}; - CAmount unspendables_bip30{0}; - CAmount unspendables_scripts{0}; - CAmount unspendables_unclaimed_rewards{0}; + //! Total cumulative amount of unspendable coins up to and including this block + CAmount total_unspendable_amount{0}; + //! Total cumulative amount of prevouts spent up to and including this block + CAmount total_prevout_spent_amount{0}; + //! Total cumulative amount of outputs created up to and including this block + CAmount total_new_outputs_ex_coinbase_amount{0}; + //! Total cumulative amount of coinbase outputs up to and including this block + CAmount total_coinbase_amount{0}; + //! The unspendable coinbase amount from the genesis block + CAmount total_unspendables_genesis_block{0}; + //! The two unspendable coinbase outputs total amount caused by BIP30 + CAmount total_unspendables_bip30{0}; + //! Total cumulative amount of outputs sent to unspendable scripts (OP_RETURN for example) up to and including this block + CAmount total_unspendables_scripts{0}; + //! Total cumulative amount of coins lost due to unclaimed miner rewards up to and including this block + CAmount total_unspendables_unclaimed_rewards{0}; CCoinsStats(CoinStatsHashType hash_type) : m_hash_type(hash_type) {} }; diff --git a/src/node/context.h b/src/node/context.h index 135f9ea1c6..26873345b4 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -12,7 +12,7 @@ class ArgsManager; class BanMan; -class CAddrMan; +class AddrMan; class CBlockPolicyEstimator; class CConnman; class CScheduler; @@ -39,7 +39,7 @@ class WalletClient; struct NodeContext { //! Init interface for initializing current process and connecting to other processes. interfaces::Init* init{nullptr}; - std::unique_ptr<CAddrMan> addrman; + std::unique_ptr<AddrMan> addrman; std::unique_ptr<CConnman> connman; std::unique_ptr<CTxMemPool> mempool; std::unique_ptr<CBlockPolicyEstimator> fee_estimator; diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 807b0143a6..5b6d8416a7 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -6,6 +6,7 @@ #include <banman.h> #include <chain.h> #include <chainparams.h> +#include <deploymentstatus.h> #include <external_signer.h> #include <init.h> #include <interfaces/chain.h> @@ -71,7 +72,7 @@ class NodeImpl : public Node private: ChainstateManager& chainman() { return *Assert(m_context->chainman); } public: - explicit NodeImpl(NodeContext* context) { setContext(context); } + explicit NodeImpl(NodeContext& context) { setContext(&context); } void initLogging() override { InitLogging(*Assert(m_context->args)); } void initParameterInteraction() override { InitParameterInteraction(*Assert(m_context->args)); } bilingual_str getWarnings() override { return GetWarnings(true); } @@ -333,6 +334,7 @@ bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<Rec REVERSE_LOCK(lock); if (!ReadBlockFromDisk(*block.m_data, index, Params().GetConsensus())) block.m_data->SetNull(); } + block.found = true; return true; } @@ -573,16 +575,16 @@ public: // that Chain clients do not need to know about. return TransactionError::OK == err; } - void getTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants) override + void getTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants, size_t* ancestorsize, CAmount* ancestorfees) override { ancestors = descendants = 0; if (!m_node.mempool) return; - m_node.mempool->GetTransactionAncestry(txid, ancestors, descendants); + m_node.mempool->GetTransactionAncestry(txid, ancestors, descendants, ancestorsize, ancestorfees); } void getPackageLimits(unsigned int& limit_ancestor_count, unsigned int& limit_descendant_count) override { - limit_ancestor_count = gArgs.GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT); - limit_descendant_count = gArgs.GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT); + limit_ancestor_count = gArgs.GetIntArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT); + limit_descendant_count = gArgs.GetIntArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT); } bool checkChainLimits(const CTransactionRef& tx) override { @@ -590,10 +592,10 @@ public: LockPoints lp; CTxMemPoolEntry entry(tx, 0, 0, 0, false, 0, lp); CTxMemPool::setEntries ancestors; - auto limit_ancestor_count = gArgs.GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT); - auto limit_ancestor_size = gArgs.GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT) * 1000; - auto limit_descendant_count = gArgs.GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT); - auto limit_descendant_size = gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000; + auto limit_ancestor_count = gArgs.GetIntArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT); + auto limit_ancestor_size = gArgs.GetIntArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT) * 1000; + auto limit_descendant_count = gArgs.GetIntArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT); + auto limit_descendant_size = gArgs.GetIntArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000; std::string unused_error_string; LOCK(m_node.mempool->cs); return m_node.mempool->CalculateMemPoolAncestors( @@ -613,7 +615,7 @@ public: CFeeRate mempoolMinFee() override { if (!m_node.mempool) return {}; - return m_node.mempool->GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + return m_node.mempool->GetMinFee(gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); } CFeeRate relayMinFee() override { return ::minRelayTxFee; } CFeeRate relayIncrementalFee() override { return ::incrementalRelayFee; } @@ -659,6 +661,14 @@ public: RPCRunLater(name, std::move(fn), seconds); } int rpcSerializationFlags() override { return RPCSerializationFlags(); } + util::SettingsValue getSetting(const std::string& name) override + { + return gArgs.GetSetting(name); + } + std::vector<util::SettingsValue> getSettingsList(const std::string& name) override + { + return gArgs.GetSettingsList(name); + } util::SettingsValue getRwSetting(const std::string& name) override { util::SettingsValue result; @@ -669,7 +679,7 @@ public: }); return result; } - bool updateRwSetting(const std::string& name, const util::SettingsValue& value) override + bool updateRwSetting(const std::string& name, const util::SettingsValue& value, bool write) override { gArgs.LockSettings([&](util::Settings& settings) { if (value.isNull()) { @@ -678,7 +688,7 @@ public: settings.rw_settings[name] = value; } }); - return gArgs.WriteSettingsFile(); + return !write || gArgs.WriteSettingsFile(); } void requestMempoolTransactions(Notifications& notifications) override { @@ -692,7 +702,7 @@ public: { LOCK(::cs_main); const CBlockIndex* tip = Assert(m_node.chainman)->ActiveChain().Tip(); - return VersionBitsState(tip, Params().GetConsensus(), Consensus::DEPLOYMENT_TAPROOT, versionbitscache) == ThresholdState::ACTIVE; + return DeploymentActiveAfter(tip, Params().GetConsensus(), Consensus::DEPLOYMENT_TAPROOT); } NodeContext& m_node; }; @@ -700,6 +710,6 @@ public: } // namespace node namespace interfaces { -std::unique_ptr<Node> MakeNode(NodeContext* context) { return std::make_unique<node::NodeImpl>(context); } +std::unique_ptr<Node> MakeNode(NodeContext& context) { return std::make_unique<node::NodeImpl>(context); } std::unique_ptr<Chain> MakeChain(NodeContext& context) { return std::make_unique<node::ChainImpl>(context); } } // namespace interfaces diff --git a/src/node/psbt.cpp b/src/node/psbt.cpp index b013b6d579..9ad65d15d2 100644 --- a/src/node/psbt.cpp +++ b/src/node/psbt.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> #include <coins.h> +#include <consensus/amount.h> #include <consensus/tx_verify.h> #include <node/psbt.h> #include <policy/policy.h> diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index f21b390915..2a7bcc057f 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -4,9 +4,12 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/validation.h> +#include <index/txindex.h> #include <net.h> #include <net_processing.h> +#include <node/blockstorage.h> #include <node/context.h> +#include <txmempool.h> #include <validation.h> #include <validationinterface.h> #include <node/transaction.h> @@ -28,65 +31,83 @@ static TransactionError HandleATMPError(const TxValidationState& state, std::str TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef tx, std::string& err_string, const CAmount& max_tx_fee, bool relay, bool wait_callback) { - // BroadcastTransaction can be called by either sendrawtransaction RPC or wallet RPCs. - // node.peerman is assigned both before chain clients and before RPC server is accepting calls, - // and reset after chain clients and RPC sever are stopped. node.peerman should never be null here. - assert(node.peerman); + // BroadcastTransaction can be called by either sendrawtransaction RPC or the wallet. + // chainman, mempool and peerman are initialized before the RPC server and wallet are started + // and reset after the RPC sever and wallet are stopped. + assert(node.chainman); assert(node.mempool); + assert(node.peerman); + std::promise<void> promise; - uint256 hashTx = tx->GetHash(); + uint256 txid = tx->GetHash(); + uint256 wtxid = tx->GetWitnessHash(); bool callback_set = false; - { // cs_main scope - assert(node.chainman); - LOCK(cs_main); - // If the transaction is already confirmed in the chain, don't do anything - // and return early. - CCoinsViewCache &view = node.chainman->ActiveChainstate().CoinsTip(); - for (size_t o = 0; o < tx->vout.size(); o++) { - const Coin& existingCoin = view.AccessCoin(COutPoint(hashTx, o)); - // IsSpent doesn't mean the coin is spent, it means the output doesn't exist. - // So if the output does exist, then this transaction exists in the chain. - if (!existingCoin.IsSpent()) return TransactionError::ALREADY_IN_CHAIN; - } - if (!node.mempool->exists(hashTx)) { - // Transaction is not already in the mempool. - if (max_tx_fee > 0) { - // First, call ATMP with test_accept and check the fee. If ATMP - // fails here, return error immediately. + { + LOCK(cs_main); + + // If the transaction is already confirmed in the chain, don't do anything + // and return early. + CCoinsViewCache &view = node.chainman->ActiveChainstate().CoinsTip(); + for (size_t o = 0; o < tx->vout.size(); o++) { + const Coin& existingCoin = view.AccessCoin(COutPoint(txid, o)); + // IsSpent doesn't mean the coin is spent, it means the output doesn't exist. + // So if the output does exist, then this transaction exists in the chain. + if (!existingCoin.IsSpent()) return TransactionError::ALREADY_IN_CHAIN; + } + + if (auto mempool_tx = node.mempool->get(txid); mempool_tx) { + // There's already a transaction in the mempool with this txid. Don't + // try to submit this transaction to the mempool (since it'll be + // rejected as a TX_CONFLICT), but do attempt to reannounce the mempool + // transaction if relay=true. + // + // The mempool transaction may have the same or different witness (and + // wtxid) as this transaction. Use the mempool's wtxid for reannouncement. + wtxid = mempool_tx->GetWitnessHash(); + } else { + // Transaction is not already in the mempool. + if (max_tx_fee > 0) { + // First, call ATMP with test_accept and check the fee. If ATMP + // fails here, return error immediately. + const MempoolAcceptResult result = AcceptToMemoryPool(node.chainman->ActiveChainstate(), *node.mempool, tx, false /* bypass_limits */, + true /* test_accept */); + if (result.m_result_type != MempoolAcceptResult::ResultType::VALID) { + return HandleATMPError(result.m_state, err_string); + } else if (result.m_base_fees.value() > max_tx_fee) { + return TransactionError::MAX_FEE_EXCEEDED; + } + } + // Try to submit the transaction to the mempool. const MempoolAcceptResult result = AcceptToMemoryPool(node.chainman->ActiveChainstate(), *node.mempool, tx, false /* bypass_limits */, - true /* test_accept */); + false /* test_accept */); if (result.m_result_type != MempoolAcceptResult::ResultType::VALID) { return HandleATMPError(result.m_state, err_string); - } else if (result.m_base_fees.value() > max_tx_fee) { - return TransactionError::MAX_FEE_EXCEEDED; } - } - // Try to submit the transaction to the mempool. - const MempoolAcceptResult result = AcceptToMemoryPool(node.chainman->ActiveChainstate(), *node.mempool, tx, false /* bypass_limits */, - false /* test_accept */); - if (result.m_result_type != MempoolAcceptResult::ResultType::VALID) { - return HandleATMPError(result.m_state, err_string); - } - // Transaction was accepted to the mempool. + // Transaction was accepted to the mempool. - if (wait_callback) { - // For transactions broadcast from outside the wallet, make sure - // that the wallet has been notified of the transaction before - // continuing. - // - // This prevents a race where a user might call sendrawtransaction - // with a transaction to/from their wallet, immediately call some - // wallet RPC, and get a stale result because callbacks have not - // yet been processed. - CallFunctionInValidationInterfaceQueue([&promise] { - promise.set_value(); - }); - callback_set = true; - } - } + if (relay) { + // the mempool tracks locally submitted transactions to make a + // best-effort of initial broadcast + node.mempool->AddUnbroadcastTx(txid); + } + if (wait_callback) { + // For transactions broadcast from outside the wallet, make sure + // that the wallet has been notified of the transaction before + // continuing. + // + // This prevents a race where a user might call sendrawtransaction + // with a transaction to/from their wallet, immediately call some + // wallet RPC, and get a stale result because callbacks have not + // yet been processed. + CallFunctionInValidationInterfaceQueue([&promise] { + promise.set_value(); + }); + callback_set = true; + } + } } // cs_main if (callback_set) { @@ -96,11 +117,41 @@ TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef t } if (relay) { - // the mempool tracks locally submitted transactions to make a - // best-effort of initial broadcast - node.mempool->AddUnbroadcastTx(hashTx); - node.peerman->RelayTransaction(hashTx, tx->GetWitnessHash()); + node.peerman->RelayTransaction(txid, wtxid); } return TransactionError::OK; } + +CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock) +{ + if (mempool && !block_index) { + CTransactionRef ptx = mempool->get(hash); + if (ptx) return ptx; + } + if (g_txindex) { + CTransactionRef tx; + uint256 block_hash; + if (g_txindex->FindTx(hash, block_hash, tx)) { + if (!block_index || block_index->GetBlockHash() == block_hash) { + // Don't return the transaction if the provided block hash doesn't match. + // The case where a transaction appears in multiple blocks (e.g. reorgs or + // BIP30) is handled by the block lookup below. + hashBlock = block_hash; + return tx; + } + } + } + if (block_index) { + CBlock block; + if (ReadBlockFromDisk(block, block_index, consensusParams)) { + for (const auto& tx : block.vtx) { + if (tx->GetHash() == hash) { + hashBlock = block_index->GetBlockHash(); + return tx; + } + } + } + } + return nullptr; +} diff --git a/src/node/transaction.h b/src/node/transaction.h index 0c016ff04e..aed519cf7f 100644 --- a/src/node/transaction.h +++ b/src/node/transaction.h @@ -10,7 +10,12 @@ #include <primitives/transaction.h> #include <util/error.h> +class CBlockIndex; +class CTxMemPool; struct NodeContext; +namespace Consensus { +struct Params; +} /** Maximum fee rate for sendrawtransaction and testmempoolaccept RPC calls. * Also used by the GUI when broadcasting a completed PSBT. @@ -38,4 +43,19 @@ static const CFeeRate DEFAULT_MAX_RAW_TX_FEE_RATE{COIN / 10}; */ [[nodiscard]] TransactionError BroadcastTransaction(NodeContext& node, CTransactionRef tx, std::string& err_string, const CAmount& max_tx_fee, bool relay, bool wait_callback); +/** + * Return transaction with a given hash. + * If mempool is provided and block_index is not provided, check it first for the tx. + * If -txindex is available, check it next for the tx. + * Finally, if block_index is provided, check for tx by reading entire block from disk. + * + * @param[in] block_index The block to read from disk, or nullptr + * @param[in] mempool If provided, check mempool for tx + * @param[in] hash The txid + * @param[in] consensusParams The params + * @param[out] hashBlock The block hash, if the tx was found via -txindex or block_index + * @returns The tx if found, otherwise nullptr + */ +CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock); + #endif // BITCOIN_NODE_TRANSACTION_H diff --git a/src/outputtype.cpp b/src/outputtype.cpp index 8ede7b9974..b5f1df9792 100644 --- a/src/outputtype.cpp +++ b/src/outputtype.cpp @@ -13,6 +13,7 @@ #include <util/vector.h> #include <assert.h> +#include <optional> #include <string> static const std::string OUTPUT_TYPE_STRING_LEGACY = "legacy"; @@ -20,22 +21,18 @@ static const std::string OUTPUT_TYPE_STRING_P2SH_SEGWIT = "p2sh-segwit"; static const std::string OUTPUT_TYPE_STRING_BECH32 = "bech32"; static const std::string OUTPUT_TYPE_STRING_BECH32M = "bech32m"; -bool ParseOutputType(const std::string& type, OutputType& output_type) +std::optional<OutputType> ParseOutputType(const std::string& type) { if (type == OUTPUT_TYPE_STRING_LEGACY) { - output_type = OutputType::LEGACY; - return true; + return OutputType::LEGACY; } else if (type == OUTPUT_TYPE_STRING_P2SH_SEGWIT) { - output_type = OutputType::P2SH_SEGWIT; - return true; + return OutputType::P2SH_SEGWIT; } else if (type == OUTPUT_TYPE_STRING_BECH32) { - output_type = OutputType::BECH32; - return true; + return OutputType::BECH32; } else if (type == OUTPUT_TYPE_STRING_BECH32M) { - output_type = OutputType::BECH32M; - return true; + return OutputType::BECH32M; } - return false; + return std::nullopt; } const std::string& FormatOutputType(OutputType type) diff --git a/src/outputtype.h b/src/outputtype.h index 2b83235cd0..0de7689125 100644 --- a/src/outputtype.h +++ b/src/outputtype.h @@ -11,6 +11,7 @@ #include <script/standard.h> #include <array> +#include <optional> #include <string> #include <vector> @@ -28,7 +29,7 @@ static constexpr auto OUTPUT_TYPES = std::array{ OutputType::BECH32M, }; -[[nodiscard]] bool ParseOutputType(const std::string& str, OutputType& output_type); +std::optional<OutputType> ParseOutputType(const std::string& str); const std::string& FormatOutputType(OutputType type); /** diff --git a/src/policy/feerate.h b/src/policy/feerate.h index d296d32774..b16f3f8251 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -6,7 +6,7 @@ #ifndef BITCOIN_POLICY_FEERATE_H #define BITCOIN_POLICY_FEERATE_H -#include <amount.h> +#include <consensus/amount.h> #include <serialize.h> #include <string> diff --git a/src/policy/fees.h b/src/policy/fees.h index c444d71a31..27f9120c64 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_POLICY_FEES_H #define BITCOIN_POLICY_FEES_H -#include <amount.h> +#include <consensus/amount.h> #include <policy/feerate.h> #include <uint256.h> #include <random.h> diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index 8125b41c41..7ac2e22006 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -3,13 +3,17 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <policy/rbf.h> + +#include <policy/settings.h> +#include <tinyformat.h> +#include <util/moneystr.h> #include <util/rbf.h> RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) { AssertLockHeld(pool.cs); - CTxMemPool::setEntries setAncestors; + CTxMemPool::setEntries ancestors; // First check the transaction itself. if (SignalsOptInRBF(tx)) { @@ -27,9 +31,9 @@ RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) uint64_t noLimit = std::numeric_limits<uint64_t>::max(); std::string dummy; CTxMemPoolEntry entry = *pool.mapTx.find(tx.GetHash()); - pool.CalculateMemPoolAncestors(entry, setAncestors, noLimit, noLimit, noLimit, noLimit, dummy, false); + pool.CalculateMemPoolAncestors(entry, ancestors, noLimit, noLimit, noLimit, noLimit, dummy, false); - for (CTxMemPool::txiter it : setAncestors) { + for (CTxMemPool::txiter it : ancestors) { if (SignalsOptInRBF(it->GetTx())) { return RBFTransactionState::REPLACEABLE_BIP125; } @@ -42,3 +46,131 @@ RBFTransactionState IsRBFOptInEmptyMempool(const CTransaction& tx) // If we don't have a local mempool we can only check the transaction itself. return SignalsOptInRBF(tx) ? RBFTransactionState::REPLACEABLE_BIP125 : RBFTransactionState::UNKNOWN; } + +std::optional<std::string> GetEntriesForConflicts(const CTransaction& tx, + CTxMemPool& pool, + const CTxMemPool::setEntries& iters_conflicting, + CTxMemPool::setEntries& all_conflicts) +{ + AssertLockHeld(pool.cs); + const uint256 txid = tx.GetHash(); + uint64_t nConflictingCount = 0; + for (const auto& mi : iters_conflicting) { + nConflictingCount += mi->GetCountWithDescendants(); + // BIP125 Rule #5: don't consider replacing more than MAX_BIP125_REPLACEMENT_CANDIDATES + // entries from the mempool. This potentially overestimates the number of actual + // descendants (i.e. if multiple conflicts share a descendant, it will be counted multiple + // times), but we just want to be conservative to avoid doing too much work. + if (nConflictingCount > MAX_BIP125_REPLACEMENT_CANDIDATES) { + return strprintf("rejecting replacement %s; too many potential replacements (%d > %d)\n", + txid.ToString(), + nConflictingCount, + MAX_BIP125_REPLACEMENT_CANDIDATES); + } + } + // Calculate the set of all transactions that would have to be evicted. + for (CTxMemPool::txiter it : iters_conflicting) { + pool.CalculateDescendants(it, all_conflicts); + } + return std::nullopt; +} + +std::optional<std::string> HasNoNewUnconfirmed(const CTransaction& tx, + const CTxMemPool& pool, + const CTxMemPool::setEntries& iters_conflicting) +{ + AssertLockHeld(pool.cs); + std::set<uint256> parents_of_conflicts; + for (const auto& mi : iters_conflicting) { + for (const CTxIn& txin : mi->GetTx().vin) { + parents_of_conflicts.insert(txin.prevout.hash); + } + } + + for (unsigned int j = 0; j < tx.vin.size(); j++) { + // BIP125 Rule #2: We don't want to accept replacements that require low feerate junk to be + // mined first. Ideally we'd keep track of the ancestor feerates and make the decision + // based on that, but for now requiring all new inputs to be confirmed works. + // + // Note that if you relax this to make RBF a little more useful, this may break the + // CalculateMempoolAncestors RBF relaxation which subtracts the conflict count/size from the + // descendant limit. + if (!parents_of_conflicts.count(tx.vin[j].prevout.hash)) { + // Rather than check the UTXO set - potentially expensive - it's cheaper to just check + // if the new input refers to a tx that's in the mempool. + if (pool.exists(tx.vin[j].prevout.hash)) { + return strprintf("replacement %s adds unconfirmed input, idx %d", + tx.GetHash().ToString(), j); + } + } + } + return std::nullopt; +} + +std::optional<std::string> EntriesAndTxidsDisjoint(const CTxMemPool::setEntries& ancestors, + const std::set<uint256>& direct_conflicts, + const uint256& txid) +{ + for (CTxMemPool::txiter ancestorIt : ancestors) { + const uint256& hashAncestor = ancestorIt->GetTx().GetHash(); + if (direct_conflicts.count(hashAncestor)) { + return strprintf("%s spends conflicting transaction %s", + txid.ToString(), + hashAncestor.ToString()); + } + } + return std::nullopt; +} + +std::optional<std::string> PaysMoreThanConflicts(const CTxMemPool::setEntries& iters_conflicting, + CFeeRate replacement_feerate, + const uint256& txid) +{ + for (const auto& mi : iters_conflicting) { + // Don't allow the replacement to reduce the feerate of the mempool. + // + // We usually don't want to accept replacements with lower feerates than what they replaced + // as that would lower the feerate of the next block. Requiring that the feerate always be + // increased is also an easy-to-reason about way to prevent DoS attacks via replacements. + // + // We only consider the feerates of transactions being directly replaced, not their indirect + // descendants. While that does mean high feerate children are ignored when deciding whether + // or not to replace, we do require the replacement to pay more overall fees too, mitigating + // most cases. + CFeeRate original_feerate(mi->GetModifiedFee(), mi->GetTxSize()); + if (replacement_feerate <= original_feerate) { + return strprintf("rejecting replacement %s; new feerate %s <= old feerate %s", + txid.ToString(), + replacement_feerate.ToString(), + original_feerate.ToString()); + } + } + return std::nullopt; +} + +std::optional<std::string> PaysForRBF(CAmount original_fees, + CAmount replacement_fees, + size_t replacement_vsize, + CFeeRate relay_fee, + const uint256& txid) +{ + // BIP125 Rule #3: The replacement fees must be greater than or equal to fees of the + // transactions it replaces, otherwise the bandwidth used by those conflicting transactions + // would not be paid for. + if (replacement_fees < original_fees) { + return strprintf("rejecting replacement %s, less fees than conflicting txs; %s < %s", + txid.ToString(), FormatMoney(replacement_fees), FormatMoney(original_fees)); + } + + // BIP125 Rule #4: The new transaction must pay for its own bandwidth. Otherwise, we have a DoS + // vector where attackers can cause a transaction to be replaced (and relayed) repeatedly by + // increasing the fee by tiny amounts. + CAmount additional_fees = replacement_fees - original_fees; + if (additional_fees < relay_fee.GetFee(replacement_vsize)) { + return strprintf("rejecting replacement %s, not enough additional fees to relay; %s < %s", + txid.ToString(), + FormatMoney(additional_fees), + FormatMoney(relay_fee.GetFee(replacement_vsize))); + } + return std::nullopt; +} diff --git a/src/policy/rbf.h b/src/policy/rbf.h index e078070c1c..be8c2e5b8b 100644 --- a/src/policy/rbf.h +++ b/src/policy/rbf.h @@ -5,7 +5,16 @@ #ifndef BITCOIN_POLICY_RBF_H #define BITCOIN_POLICY_RBF_H +#include <primitives/transaction.h> #include <txmempool.h> +#include <uint256.h> + +#include <optional> +#include <string> + +/** Maximum number of transactions that can be replaced by BIP125 RBF (Rule #5). This includes all + * mempool conflicts and their descendants. */ +static constexpr uint32_t MAX_BIP125_REPLACEMENT_CANDIDATES{100}; /** The rbf state of unconfirmed transactions */ enum class RBFTransactionState { @@ -31,4 +40,63 @@ enum class RBFTransactionState { RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(pool.cs); RBFTransactionState IsRBFOptInEmptyMempool(const CTransaction& tx); +/** Get all descendants of iters_conflicting. Also enforce BIP125 Rule #5, "The number of original + * transactions to be replaced and their descendant transactions which will be evicted from the + * mempool must not exceed a total of 100 transactions." Quit as early as possible. There cannot be + * more than MAX_BIP125_REPLACEMENT_CANDIDATES potential entries. + * @param[in] iters_conflicting The set of iterators to mempool entries. + * @param[out] all_conflicts Populated with all the mempool entries that would be replaced, + * which includes descendants of iters_conflicting. Not cleared at + * the start; any existing mempool entries will remain in the set. + * @returns an error message if Rule #5 is broken, otherwise a std::nullopt. + */ +std::optional<std::string> GetEntriesForConflicts(const CTransaction& tx, CTxMemPool& pool, + const CTxMemPool::setEntries& iters_conflicting, + CTxMemPool::setEntries& all_conflicts) + EXCLUSIVE_LOCKS_REQUIRED(pool.cs); + +/** BIP125 Rule #2: "The replacement transaction may only include an unconfirmed input if that input + * was included in one of the original transactions." + * @returns error message if Rule #2 is broken, otherwise std::nullopt. */ +std::optional<std::string> HasNoNewUnconfirmed(const CTransaction& tx, const CTxMemPool& pool, + const CTxMemPool::setEntries& iters_conflicting) + EXCLUSIVE_LOCKS_REQUIRED(pool.cs); + +/** Check the intersection between two sets of transactions (a set of mempool entries and a set of + * txids) to make sure they are disjoint. + * @param[in] ancestors Set of mempool entries corresponding to ancestors of the + * replacement transactions. + * @param[in] direct_conflicts Set of txids corresponding to the mempool conflicts + * (candidates to be replaced). + * @param[in] txid Transaction ID, included in the error message if violation occurs. + * @returns error message if the sets intersect, std::nullopt if they are disjoint. + */ +std::optional<std::string> EntriesAndTxidsDisjoint(const CTxMemPool::setEntries& ancestors, + const std::set<uint256>& direct_conflicts, + const uint256& txid); + +/** Check that the feerate of the replacement transaction(s) is higher than the feerate of each + * of the transactions in iters_conflicting. + * @param[in] iters_conflicting The set of mempool entries. + * @returns error message if fees insufficient, otherwise std::nullopt. + */ +std::optional<std::string> PaysMoreThanConflicts(const CTxMemPool::setEntries& iters_conflicting, + CFeeRate replacement_feerate, const uint256& txid); + +/** Enforce BIP125 Rule #3 "The replacement transaction pays an absolute fee of at least the sum + * paid by the original transactions." Enforce BIP125 Rule #4 "The replacement transaction must also + * pay for its own bandwidth at or above the rate set by the node's minimum relay fee setting." + * @param[in] original_fees Total modified fees of original transaction(s). + * @param[in] replacement_fees Total modified fees of replacement transaction(s). + * @param[in] replacement_vsize Total virtual size of replacement transaction(s). + * @param[in] relay_fee The node's minimum feerate for transaction relay. + * @param[in] txid Transaction ID, included in the error message if violation occurs. + * @returns error string if fees are insufficient, otherwise std::nullopt. + */ +std::optional<std::string> PaysForRBF(CAmount original_fees, + CAmount replacement_fees, + size_t replacement_vsize, + CFeeRate relay_fee, + const uint256& txid); + #endif // BITCOIN_POLICY_RBF_H diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index 245206b906..a871912225 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -5,6 +5,7 @@ #include <primitives/transaction.h> +#include <consensus/amount.h> #include <hash.h> #include <tinyformat.h> #include <util/strencodings.h> diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 6bf36ee854..46db39f8db 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -7,7 +7,7 @@ #define BITCOIN_PRIMITIVES_TRANSACTION_H #include <stdint.h> -#include <amount.h> +#include <consensus/amount.h> #include <script/script.h> #include <serialize.h> #include <uint256.h> diff --git a/src/protocol.h b/src/protocol.h index f9248899dc..2149e45993 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -396,7 +396,6 @@ public: // ambiguous what that would mean. Make sure no code relying on that is introduced: assert(!(s.GetType() & SER_GETHASH)); bool use_v2; - bool store_time; if (s.GetType() & SER_DISK) { // In the disk serialization format, the encoding (v1 or v2) is determined by a flag version // that's part of the serialization itself. ADDRV2_FORMAT in the stream version only determines @@ -413,24 +412,16 @@ public: } else { throw std::ios_base::failure("Unsupported CAddress disk format version"); } - store_time = true; } else { // In the network serialization format, the encoding (v1 or v2) is determined directly by // the value of ADDRV2_FORMAT in the stream version, as no explicitly encoded version // exists in the stream. assert(s.GetType() & SER_NETWORK); use_v2 = s.GetVersion() & ADDRV2_FORMAT; - // The only time we serialize a CAddress object without nTime is in - // the initial VERSION messages which contain two CAddress records. - // At that point, the serialization version is INIT_PROTO_VERSION. - // After the version handshake, serialization version is >= - // MIN_PEER_PROTO_VERSION and all ADDR messages are serialized with - // nTime. - store_time = s.GetVersion() != INIT_PROTO_VERSION; } SER_READ(obj, obj.nTime = TIME_INIT); - if (store_time) READWRITE(obj.nTime); + READWRITE(obj.nTime); // nServices is serialized as CompactSize in V2; as uint64_t in V1. if (use_v2) { uint64_t services_tmp; @@ -445,7 +436,7 @@ public: SerReadWriteMany(os, ser_action, ReadWriteAsHelper<CService>(obj)); } - //! Always included in serialization, except in the network format on INIT_PROTO_VERSION. + //! Always included in serialization. uint32_t nTime{TIME_INIT}; //! Serialized as uint64_t in V1, and as CompactSize in V2. ServiceFlags nServices{NODE_NONE}; diff --git a/src/pubkey.cpp b/src/pubkey.cpp index 175a39b805..956ff2b34a 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -180,6 +180,23 @@ XOnlyPubKey::XOnlyPubKey(Span<const unsigned char> bytes) std::copy(bytes.begin(), bytes.end(), m_keydata.begin()); } +std::vector<CKeyID> XOnlyPubKey::GetKeyIDs() const +{ + std::vector<CKeyID> out; + // For now, use the old full pubkey-based key derivation logic. As it is indexed by + // Hash160(full pubkey), we need to return both a version prefixed with 0x02, and one + // with 0x03. + unsigned char b[33] = {0x02}; + std::copy(m_keydata.begin(), m_keydata.end(), b + 1); + CPubKey fullpubkey; + fullpubkey.Set(b, b + 33); + out.push_back(fullpubkey.GetID()); + b[0] = 0x03; + fullpubkey.Set(b, b + 33); + out.push_back(fullpubkey.GetID()); + return out; +} + bool XOnlyPubKey::IsFullyValid() const { secp256k1_xonly_pubkey pubkey; @@ -191,7 +208,7 @@ bool XOnlyPubKey::VerifySchnorr(const uint256& msg, Span<const unsigned char> si assert(sigbytes.size() == 64); secp256k1_xonly_pubkey pubkey; if (!secp256k1_xonly_pubkey_parse(secp256k1_context_verify, &pubkey, m_keydata.data())) return false; - return secp256k1_schnorrsig_verify(secp256k1_context_verify, sigbytes.data(), msg.begin(), &pubkey); + return secp256k1_schnorrsig_verify(secp256k1_context_verify, sigbytes.data(), msg.begin(), 32, &pubkey); } static const CHashWriter HASHER_TAPTWEAK = TaggedHash("TapTweak"); @@ -320,8 +337,7 @@ bool CPubKey::Derive(CPubKey& pubkeyChild, ChainCode &ccChild, unsigned int nChi void CExtPubKey::Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const { code[0] = nDepth; memcpy(code+1, vchFingerprint, 4); - code[5] = (nChild >> 24) & 0xFF; code[6] = (nChild >> 16) & 0xFF; - code[7] = (nChild >> 8) & 0xFF; code[8] = (nChild >> 0) & 0xFF; + WriteBE32(code+5, nChild); memcpy(code+9, chaincode.begin(), 32); assert(pubkey.size() == CPubKey::COMPRESSED_SIZE); memcpy(code+41, pubkey.begin(), CPubKey::COMPRESSED_SIZE); @@ -330,9 +346,10 @@ void CExtPubKey::Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const { void CExtPubKey::Decode(const unsigned char code[BIP32_EXTKEY_SIZE]) { nDepth = code[0]; memcpy(vchFingerprint, code+1, 4); - nChild = (code[5] << 24) | (code[6] << 16) | (code[7] << 8) | code[8]; + nChild = ReadBE32(code+5); memcpy(chaincode.begin(), code+9, 32); pubkey.Set(code+41, code+BIP32_EXTKEY_SIZE); + if ((nDepth == 0 && (nChild != 0 || ReadLE32(vchFingerprint) != 0)) || !pubkey.IsFullyValid()) pubkey = CPubKey(); } bool CExtPubKey::Derive(CExtPubKey &out, unsigned int _nChild) const { diff --git a/src/pubkey.h b/src/pubkey.h index eec34a89c2..861a2cf500 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -267,6 +267,11 @@ public: /** Construct a Taproot tweaked output point with this point as internal key. */ std::optional<std::pair<XOnlyPubKey, bool>> CreateTapTweak(const uint256* merkle_root) const; + /** Returns a list of CKeyIDs for the CPubKeys that could have been used to create this XOnlyPubKey. + * This is needed for key lookups since keys are indexed by CKeyID. + */ + std::vector<CKeyID> GetKeyIDs() const; + const unsigned char& operator[](int pos) const { return *(m_keydata.begin() + pos); } const unsigned char* data() const { return m_keydata.begin(); } static constexpr size_t size() { return decltype(m_keydata)::size(); } diff --git a/src/qt/addressbookpage.cpp b/src/qt/addressbookpage.cpp index c31f0aceea..a617bb4451 100644 --- a/src/qt/addressbookpage.cpp +++ b/src/qt/addressbookpage.cpp @@ -182,14 +182,14 @@ void AddressBookPage::onEditAction() if(indexes.isEmpty()) return; - EditAddressDialog dlg( + auto dlg = new EditAddressDialog( tab == SendingTab ? EditAddressDialog::EditSendingAddress : EditAddressDialog::EditReceivingAddress, this); - dlg.setModel(model); + dlg->setModel(model); QModelIndex origIndex = proxyModel->mapToSource(indexes.at(0)); - dlg.loadRow(origIndex.row()); - dlg.exec(); + dlg->loadRow(origIndex.row()); + GUIUtil::ShowModalDialogAndDeleteOnClose(dlg); } void AddressBookPage::on_newAddress_clicked() @@ -282,7 +282,7 @@ void AddressBookPage::on_exportButton_clicked() QString filename = GUIUtil::getSaveFileName(this, tr("Export Address List"), QString(), /*: Expanded name of the CSV file format. - See https://en.wikipedia.org/wiki/Comma-separated_values */ + See: https://en.wikipedia.org/wiki/Comma-separated_values. */ tr("Comma separated file") + QLatin1String(" (*.csv)"), nullptr); if (filename.isNull()) diff --git a/src/qt/bantablemodel.h b/src/qt/bantablemodel.h index 57f559fc14..4b5b38e43f 100644 --- a/src/qt/bantablemodel.h +++ b/src/qt/bantablemodel.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_QT_BANTABLEMODEL_H #define BITCOIN_QT_BANTABLEMODEL_H +#include <addrdb.h> #include <net.h> #include <memory> diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 442c813a5a..7de56a648a 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -7,12 +7,20 @@ #endif #include <qt/bitcoin.h> -#include <qt/bitcoingui.h> #include <chainparams.h> +#include <init.h> +#include <interfaces/handler.h> +#include <interfaces/init.h> +#include <interfaces/node.h> +#include <node/context.h> +#include <node/ui_interface.h> +#include <noui.h> +#include <qt/bitcoingui.h> #include <qt/clientmodel.h> #include <qt/guiconstants.h> #include <qt/guiutil.h> +#include <qt/initexecutor.h> #include <qt/intro.h> #include <qt/networkstyle.h> #include <qt/optionsmodel.h> @@ -20,6 +28,12 @@ #include <qt/splashscreen.h> #include <qt/utilitydialog.h> #include <qt/winshutdownmonitor.h> +#include <uint256.h> +#include <util/string.h> +#include <util/system.h> +#include <util/threadnames.h> +#include <util/translation.h> +#include <validation.h> #ifdef ENABLE_WALLET #include <qt/paymentserver.h> @@ -27,24 +41,11 @@ #include <qt/walletmodel.h> #endif // ENABLE_WALLET -#include <init.h> -#include <interfaces/handler.h> -#include <interfaces/node.h> -#include <node/context.h> -#include <node/ui_interface.h> -#include <noui.h> -#include <uint256.h> -#include <util/system.h> -#include <util/threadnames.h> -#include <util/translation.h> -#include <validation.h> - #include <boost/signals2/connection.hpp> #include <memory> #include <QApplication> #include <QDebug> -#include <QFontDatabase> #include <QLatin1String> #include <QLibraryInfo> #include <QLocale> @@ -53,6 +54,7 @@ #include <QThread> #include <QTimer> #include <QTranslator> +#include <QWindow> #if defined(QT_STATICPLUGIN) #include <QtPlugin> @@ -144,56 +146,61 @@ static void initTranslations(QTranslator &qtTranslatorBase, QTranslator &qtTrans QApplication::installTranslator(&translator); } -/* qDebug() message handler --> debug.log */ -void DebugMessageHandler(QtMsgType type, const QMessageLogContext& context, const QString &msg) +static bool InitSettings() { - Q_UNUSED(context); - if (type == QtDebugMsg) { - LogPrint(BCLog::QT, "GUI: %s\n", msg.toStdString()); - } else { - LogPrintf("GUI: %s\n", msg.toStdString()); + if (!gArgs.GetSettingsPath()) { + return true; // Do nothing if settings file disabled. } -} -BitcoinCore::BitcoinCore(interfaces::Node& node) : - QObject(), m_node(node) -{ -} - -void BitcoinCore::handleRunawayException(const std::exception *e) -{ - PrintExceptionContinue(e, "Runaway exception"); - Q_EMIT runawayException(QString::fromStdString(m_node.getWarnings().translated)); -} + std::vector<std::string> errors; + if (!gArgs.ReadSettingsFile(&errors)) { + bilingual_str error = _("Settings file could not be read"); + InitError(Untranslated(strprintf("%s:\n%s\n", error.original, MakeUnorderedList(errors)))); + + QMessageBox messagebox(QMessageBox::Critical, PACKAGE_NAME, QString::fromStdString(strprintf("%s.", error.translated)), QMessageBox::Reset | QMessageBox::Abort); + /*: Explanatory text shown on startup when the settings file cannot be read. + Prompts user to make a choice between resetting or aborting. */ + messagebox.setInformativeText(QObject::tr("Do you want to reset settings to default values, or to abort without making changes?")); + messagebox.setDetailedText(QString::fromStdString(MakeUnorderedList(errors))); + messagebox.setTextFormat(Qt::PlainText); + messagebox.setDefaultButton(QMessageBox::Reset); + switch (messagebox.exec()) { + case QMessageBox::Reset: + break; + case QMessageBox::Abort: + return false; + default: + assert(false); + } + } -void BitcoinCore::initialize() -{ - try - { - util::ThreadRename("qt-init"); - qDebug() << __func__ << ": Running initialization in thread"; - interfaces::BlockAndHeaderTipInfo tip_info; - bool rv = m_node.appInitMain(&tip_info); - Q_EMIT initializeResult(rv, tip_info); - } catch (const std::exception& e) { - handleRunawayException(&e); - } catch (...) { - handleRunawayException(nullptr); + errors.clear(); + if (!gArgs.WriteSettingsFile(&errors)) { + bilingual_str error = _("Settings file could not be written"); + InitError(Untranslated(strprintf("%s:\n%s\n", error.original, MakeUnorderedList(errors)))); + + QMessageBox messagebox(QMessageBox::Critical, PACKAGE_NAME, QString::fromStdString(strprintf("%s.", error.translated)), QMessageBox::Ok); + /*: Explanatory text shown on startup when the settings file could not be written. + Prompts user to check that we have the ability to write to the file. + Explains that the user has the option of running without a settings file.*/ + messagebox.setInformativeText(QObject::tr("A fatal error occurred. Check that settings file is writable, or try running with -nosettings.")); + messagebox.setDetailedText(QString::fromStdString(MakeUnorderedList(errors))); + messagebox.setTextFormat(Qt::PlainText); + messagebox.setDefaultButton(QMessageBox::Ok); + messagebox.exec(); + return false; } + return true; } -void BitcoinCore::shutdown() +/* qDebug() message handler --> debug.log */ +void DebugMessageHandler(QtMsgType type, const QMessageLogContext& context, const QString &msg) { - try - { - qDebug() << __func__ << ": Running Shutdown in thread"; - m_node.appShutdown(); - qDebug() << __func__ << ": Shutdown finished"; - Q_EMIT shutdownResult(); - } catch (const std::exception& e) { - handleRunawayException(&e); - } catch (...) { - handleRunawayException(nullptr); + Q_UNUSED(context); + if (type == QtDebugMsg) { + LogPrint(BCLog::QT, "GUI: %s\n", msg.toStdString()); + } else { + LogPrintf("GUI: %s\n", msg.toStdString()); } } @@ -202,7 +209,6 @@ static const char* qt_argv = "bitcoin-qt"; BitcoinApplication::BitcoinApplication(): QApplication(qt_argc, const_cast<char **>(&qt_argv)), - coreThread(nullptr), optionsModel(nullptr), clientModel(nullptr), window(nullptr), @@ -230,13 +236,7 @@ void BitcoinApplication::setupPlatformStyle() BitcoinApplication::~BitcoinApplication() { - if(coreThread) - { - qDebug() << __func__ << ": Stopping thread"; - coreThread->quit(); - coreThread->wait(); - qDebug() << __func__ << ": Stopped thread"; - } + m_executor.reset(); delete window; window = nullptr; @@ -259,6 +259,7 @@ void BitcoinApplication::createOptionsModel(bool resetSettings) void BitcoinApplication::createWindow(const NetworkStyle *networkStyle) { window = new BitcoinGUI(node(), platformStyle, networkStyle, nullptr); + connect(window, &BitcoinGUI::quitRequested, this, &BitcoinApplication::requestShutdown); pollShutdownTimer = new QTimer(window); connect(pollShutdownTimer, &QTimer::timeout, window, &BitcoinGUI::detectShutdown); @@ -276,10 +277,10 @@ void BitcoinApplication::createSplashScreen(const NetworkStyle *networkStyle) connect(this, &BitcoinApplication::requestedShutdown, m_splash, &QWidget::close); } -void BitcoinApplication::setNode(interfaces::Node& node) +void BitcoinApplication::createNode(interfaces::Init& init) { assert(!m_node); - m_node = &node; + m_node = init.makeNode(); if (optionsModel) optionsModel->setNode(*m_node); if (m_splash) m_splash->setNode(*m_node); } @@ -291,22 +292,15 @@ bool BitcoinApplication::baseInitialize() void BitcoinApplication::startThread() { - if(coreThread) - return; - coreThread = new QThread(this); - BitcoinCore *executor = new BitcoinCore(node()); - executor->moveToThread(coreThread); + assert(!m_executor); + m_executor.emplace(node()); /* communication to and from thread */ - connect(executor, &BitcoinCore::initializeResult, this, &BitcoinApplication::initializeResult); - connect(executor, &BitcoinCore::shutdownResult, this, &BitcoinApplication::shutdownResult); - connect(executor, &BitcoinCore::runawayException, this, &BitcoinApplication::handleRunawayException); - connect(this, &BitcoinApplication::requestedInitialize, executor, &BitcoinCore::initialize); - connect(this, &BitcoinApplication::requestedShutdown, executor, &BitcoinCore::shutdown); - /* make sure executor object is deleted in its own thread */ - connect(coreThread, &QThread::finished, executor, &QObject::deleteLater); - - coreThread->start(); + connect(&m_executor.value(), &InitExecutor::initializeResult, this, &BitcoinApplication::initializeResult); + connect(&m_executor.value(), &InitExecutor::shutdownResult, this, &QCoreApplication::quit); + connect(&m_executor.value(), &InitExecutor::runawayException, this, &BitcoinApplication::handleRunawayException); + connect(this, &BitcoinApplication::requestedInitialize, &m_executor.value(), &InitExecutor::initialize); + connect(this, &BitcoinApplication::requestedShutdown, &m_executor.value(), &InitExecutor::shutdown); } void BitcoinApplication::parameterSetup() @@ -333,14 +327,17 @@ void BitcoinApplication::requestInitialize() void BitcoinApplication::requestShutdown() { + for (const auto w : QGuiApplication::topLevelWindows()) { + w->hide(); + } + // Show a simple window indicating shutdown status // Do this first as some of the steps may take some time below, // for example the RPC console may still be executing a command. shutdownWindow.reset(ShutdownWindow::showShutdownWindow(window)); qDebug() << __func__ << ": Requesting shutdown"; - startThread(); - window->hide(); + // Must disconnect node signals otherwise current thread can deadlock since // no event loop is running. window->unsubscribeFromCoreSignals(); @@ -352,6 +349,17 @@ void BitcoinApplication::requestShutdown() window->setClientModel(nullptr); pollShutdownTimer->stop(); +#ifdef ENABLE_WALLET + // Delete wallet controller here manually, instead of relying on Qt object + // tracking (https://doc.qt.io/qt-5/objecttrees.html). This makes sure + // walletmodel m_handle_* notification handlers are deleted before wallets + // are unloaded, which can simplify wallet implementations. It also avoids + // these notifications having to be handled while GUI objects are being + // destroyed, making GUI code less fragile as well. + delete m_wallet_controller; + m_wallet_controller = nullptr; +#endif // ENABLE_WALLET + delete clientModel; clientModel = nullptr; @@ -406,15 +414,10 @@ void BitcoinApplication::initializeResult(bool success, interfaces::BlockAndHead pollShutdownTimer->start(200); } else { Q_EMIT splashFinished(); // Make sure splash screen doesn't stick around during shutdown - quit(); // Exit first main loop invocation + requestShutdown(); } } -void BitcoinApplication::shutdownResult() -{ - quit(); // Exit second main loop invocation after shutdown finished -} - void BitcoinApplication::handleRunawayException(const QString &message) { QMessageBox::critical( @@ -458,11 +461,13 @@ int GuiMain(int argc, char* argv[]) util::WinCmdLineArgs winArgs; std::tie(argc, argv) = winArgs.get(); #endif - SetupEnvironment(); - util::ThreadSetInternalName("main"); NodeContext node_context; - std::unique_ptr<interfaces::Node> node = interfaces::MakeNode(&node_context); + int unused_exit_status; + std::unique_ptr<interfaces::Init> init = interfaces::MakeNodeInit(node_context, argc, argv, unused_exit_status); + + SetupEnvironment(); + util::ThreadSetInternalName("main"); // Subscribe to global signals from core boost::signals2::scoped_connection handler_message_box = ::uiInterface.ThreadSafeMessageBox_connect(noui_ThreadSafeMessageBox); @@ -486,11 +491,10 @@ int GuiMain(int argc, char* argv[]) #endif BitcoinApplication app; - QFontDatabase::addApplicationFont(":/fonts/monospace"); + GUIUtil::LoadFont(QStringLiteral(":/fonts/monospace")); /// 2. Parse command-line options. We do this after qt in order to show an error if there are problems parsing these // Command-line options take precedence: - node_context.args = &gArgs; SetupServerArgs(gArgs); SetupUIArgs(gArgs); std::string error; @@ -569,9 +573,8 @@ int GuiMain(int argc, char* argv[]) // Parse URIs on command line -- this can affect Params() PaymentServer::ipcParseCommandLine(argc, argv); #endif - if (!gArgs.InitSettings(error)) { - InitError(Untranslated(error)); - QMessageBox::critical(nullptr, PACKAGE_NAME, QObject::tr("Error initializing settings: %1").arg(QString::fromStdString(error))); + + if (!InitSettings()) { return EXIT_FAILURE; } @@ -622,7 +625,7 @@ int GuiMain(int argc, char* argv[]) if (gArgs.GetBoolArg("-splash", DEFAULT_SPLASHSCREEN) && !gArgs.GetBoolArg("-min", false)) app.createSplashScreen(networkStyle.data()); - app.setNode(*node); + app.createNode(*init); int rv = EXIT_SUCCESS; try @@ -637,8 +640,6 @@ int GuiMain(int argc, char* argv[]) WinShutdownMonitor::registerShutdownBlockReason(QObject::tr("%1 didn't yet exit safely…").arg(PACKAGE_NAME), (HWND)app.getMainWinId()); #endif app.exec(); - app.requestShutdown(); - app.exec(); rv = app.getReturnValue(); } else { // A dialog with detailed error will have been shown by InitError() diff --git a/src/qt/bitcoin.h b/src/qt/bitcoin.h index f9fab0534b..5678ca90d2 100644 --- a/src/qt/bitcoin.h +++ b/src/qt/bitcoin.h @@ -9,11 +9,14 @@ #include <config/bitcoin-config.h> #endif -#include <QApplication> +#include <interfaces/node.h> +#include <qt/initexecutor.h> + #include <assert.h> #include <memory> +#include <optional> -#include <interfaces/node.h> +#include <QApplication> class BitcoinGUI; class ClientModel; @@ -24,33 +27,11 @@ class PlatformStyle; class SplashScreen; class WalletController; class WalletModel; +namespace interfaces { +class Init; +} // namespace interfaces -/** Class encapsulating Bitcoin Core startup and shutdown. - * Allows running startup and shutdown in a different thread from the UI thread. - */ -class BitcoinCore: public QObject -{ - Q_OBJECT -public: - explicit BitcoinCore(interfaces::Node& node); - -public Q_SLOTS: - void initialize(); - void shutdown(); - -Q_SIGNALS: - void initializeResult(bool success, interfaces::BlockAndHeaderTipInfo tip_info); - void shutdownResult(); - void runawayException(const QString &message); - -private: - /// Pass fatal exception message to UI thread - void handleRunawayException(const std::exception *e); - - interfaces::Node& m_node; -}; - /** Main Bitcoin application object */ class BitcoinApplication: public QApplication { @@ -73,13 +54,13 @@ public: void createWindow(const NetworkStyle *networkStyle); /// Create splash screen void createSplashScreen(const NetworkStyle *networkStyle); + /// Create or spawn node + void createNode(interfaces::Init& init); /// Basic initialization, before starting initialization/shutdown thread. Return true on success. bool baseInitialize(); /// Request core initialization void requestInitialize(); - /// Request core shutdown - void requestShutdown(); /// Get process return value int getReturnValue() const { return returnValue; } @@ -91,11 +72,11 @@ public: void setupPlatformStyle(); interfaces::Node& node() const { assert(m_node); return *m_node; } - void setNode(interfaces::Node& node); public Q_SLOTS: void initializeResult(bool success, interfaces::BlockAndHeaderTipInfo tip_info); - void shutdownResult(); + /// Request core shutdown + void requestShutdown(); /// Handle runaway exceptions. Shows a message box with the problem and quits the program. void handleRunawayException(const QString &message); @@ -112,7 +93,7 @@ Q_SIGNALS: void windowShown(BitcoinGUI* window); private: - QThread *coreThread; + std::optional<InitExecutor> m_executor; OptionsModel *optionsModel; ClientModel *clientModel; BitcoinGUI *window; @@ -125,7 +106,7 @@ private: const PlatformStyle *platformStyle; std::unique_ptr<QWidget> shutdownWindow; SplashScreen* m_splash = nullptr; - interfaces::Node* m_node = nullptr; + std::unique_ptr<interfaces::Node> m_node; void startThread(); }; diff --git a/src/qt/bitcoinamountfield.h b/src/qt/bitcoinamountfield.h index c60d9a2c90..4855ada513 100644 --- a/src/qt/bitcoinamountfield.h +++ b/src/qt/bitcoinamountfield.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_BITCOINAMOUNTFIELD_H #define BITCOIN_QT_BITCOINAMOUNTFIELD_H -#include <amount.h> +#include <consensus/amount.h> #include <QWidget> diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 3d632ec702..610637360b 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -107,9 +107,12 @@ BitcoinGUI::BitcoinGUI(interfaces::Node& node, const PlatformStyle *_platformSty walletFrame = new WalletFrame(_platformStyle, this); connect(walletFrame, &WalletFrame::createWalletButtonClicked, [this] { auto activity = new CreateWalletActivity(getWalletController(), this); - connect(activity, &CreateWalletActivity::finished, activity, &QObject::deleteLater); activity->create(); }); + connect(walletFrame, &WalletFrame::message, [this](const QString& title, const QString& message, unsigned int style) { + this->message(title, message, style); + }); + connect(walletFrame, &WalletFrame::currentWalletSet, [this] { updateWalletStatus(); }); setCentralWidget(walletFrame); } else #endif // ENABLE_WALLET @@ -167,7 +170,9 @@ BitcoinGUI::BitcoinGUI(interfaces::Node& node, const PlatformStyle *_platformSty frameBlocksLayout->addWidget(unitDisplayControl); frameBlocksLayout->addStretch(); frameBlocksLayout->addWidget(labelWalletEncryptionIcon); + labelWalletEncryptionIcon->hide(); frameBlocksLayout->addWidget(labelWalletHDStatusIcon); + labelWalletHDStatusIcon->hide(); } frameBlocksLayout->addWidget(labelProxyIcon); frameBlocksLayout->addStretch(); @@ -326,7 +331,7 @@ void BitcoinGUI::createActions() verifyMessageAction->setStatusTip(tr("Verify messages to ensure they were signed with specified Bitcoin addresses")); m_load_psbt_action = new QAction(tr("&Load PSBT from file…"), this); m_load_psbt_action->setStatusTip(tr("Load Partially Signed Bitcoin Transaction")); - m_load_psbt_clipboard_action = new QAction(tr("Load PSBT from clipboard…"), this); + m_load_psbt_clipboard_action = new QAction(tr("Load PSBT from &clipboard…"), this); m_load_psbt_clipboard_action->setStatusTip(tr("Load Partially Signed Bitcoin Transaction from clipboard")); openRPCConsoleAction = new QAction(tr("Node window"), this); @@ -367,7 +372,7 @@ void BitcoinGUI::createActions() m_mask_values_action->setStatusTip(tr("Mask the values in the Overview tab")); m_mask_values_action->setCheckable(true); - connect(quitAction, &QAction::triggered, qApp, QApplication::quit); + connect(quitAction, &QAction::triggered, this, &BitcoinGUI::quitRequested); connect(aboutAction, &QAction::triggered, this, &BitcoinGUI::aboutClicked); connect(aboutQtAction, &QAction::triggered, qApp, QApplication::aboutQt); connect(optionsAction, &QAction::triggered, this, &BitcoinGUI::optionsClicked); @@ -412,7 +417,6 @@ void BitcoinGUI::createActions() connect(action, &QAction::triggered, [this, path] { auto activity = new OpenWalletActivity(m_wallet_controller, this); connect(activity, &OpenWalletActivity::opened, this, &BitcoinGUI::setCurrentWallet); - connect(activity, &OpenWalletActivity::finished, activity, &QObject::deleteLater); activity->open(path); }); } @@ -427,7 +431,6 @@ void BitcoinGUI::createActions() connect(m_create_wallet_action, &QAction::triggered, [this] { auto activity = new CreateWalletActivity(m_wallet_controller, this); connect(activity, &CreateWalletActivity::created, this, &BitcoinGUI::setCurrentWallet); - connect(activity, &CreateWalletActivity::finished, activity, &QObject::deleteLater); activity->create(); }); connect(m_close_all_wallets_action, &QAction::triggered, [this] { @@ -483,7 +486,7 @@ void BitcoinGUI::createMenuBar() QMenu* window_menu = appMenuBar->addMenu(tr("&Window")); - QAction* minimize_action = window_menu->addAction(tr("Minimize")); + QAction* minimize_action = window_menu->addAction(tr("&Minimize")); minimize_action->setShortcut(QKeySequence(Qt::CTRL + Qt::Key_M)); connect(minimize_action, &QAction::triggered, [] { QApplication::activeWindow()->showMinimized(); @@ -591,8 +594,8 @@ void BitcoinGUI::setClientModel(ClientModel *_clientModel, interfaces::BlockAndH connect(_clientModel, &ClientModel::numConnectionsChanged, this, &BitcoinGUI::setNumConnections); connect(_clientModel, &ClientModel::networkActiveChanged, this, &BitcoinGUI::setNetworkActive); - modalOverlay->setKnownBestHeight(tip_info->header_height, QDateTime::fromTime_t(tip_info->header_time)); - setNumBlocks(tip_info->block_height, QDateTime::fromTime_t(tip_info->block_time), tip_info->verification_progress, false, SynchronizationState::INIT_DOWNLOAD); + modalOverlay->setKnownBestHeight(tip_info->header_height, QDateTime::fromSecsSinceEpoch(tip_info->header_time)); + setNumBlocks(tip_info->block_height, QDateTime::fromSecsSinceEpoch(tip_info->block_time), tip_info->verification_progress, false, SynchronizationState::INIT_DOWNLOAD); connect(_clientModel, &ClientModel::numBlocksChanged, this, &BitcoinGUI::setNumBlocks); // Receive and report messages from client model @@ -658,9 +661,8 @@ void BitcoinGUI::setWalletController(WalletController* wallet_controller) GUIUtil::ExceptionSafeConnect(wallet_controller, &WalletController::walletAdded, this, &BitcoinGUI::addWallet); connect(wallet_controller, &WalletController::walletRemoved, this, &BitcoinGUI::removeWallet); - for (WalletModel* wallet_model : m_wallet_controller->getOpenWallets()) { - addWallet(wallet_model); - } + auto activity = new LoadWalletsActivity(m_wallet_controller, this); + activity->load(); } WalletController* BitcoinGUI::getWalletController() @@ -672,8 +674,8 @@ void BitcoinGUI::addWallet(WalletModel* walletModel) { if (!walletFrame) return; - WalletView* wallet_view = new WalletView(platformStyle, walletFrame); - if (!walletFrame->addWallet(walletModel, wallet_view)) return; + WalletView* wallet_view = new WalletView(walletModel, platformStyle, walletFrame); + if (!walletFrame->addView(wallet_view)) return; rpcConsole->addWallet(walletModel); if (m_wallet_selector->count() == 0) { @@ -682,8 +684,6 @@ void BitcoinGUI::addWallet(WalletModel* walletModel) m_wallet_selector_label_action->setVisible(true); m_wallet_selector_action->setVisible(true); } - const QString display_name = walletModel->getDisplayName(); - m_wallet_selector->addItem(display_name, QVariant::fromValue(walletModel)); connect(wallet_view, &WalletView::outOfSyncWarningClicked, this, &BitcoinGUI::showModalOverlay); connect(wallet_view, &WalletView::transactionClicked, this, &BitcoinGUI::gotoHistoryPage); @@ -693,9 +693,10 @@ void BitcoinGUI::addWallet(WalletModel* walletModel) }); connect(wallet_view, &WalletView::encryptionStatusChanged, this, &BitcoinGUI::updateWalletStatus); connect(wallet_view, &WalletView::incomingTransaction, this, &BitcoinGUI::incomingTransaction); - connect(wallet_view, &WalletView::hdEnabledStatusChanged, this, &BitcoinGUI::updateWalletStatus); connect(this, &BitcoinGUI::setPrivacy, wallet_view, &WalletView::setPrivacy); wallet_view->setPrivacy(isPrivacyModeActivated()); + const QString display_name = walletModel->getDisplayName(); + m_wallet_selector->addItem(display_name, QVariant::fromValue(walletModel)); } void BitcoinGUI::removeWallet(WalletModel* walletModel) @@ -845,8 +846,8 @@ void BitcoinGUI::aboutClicked() if(!clientModel) return; - HelpMessageDialog dlg(this, true); - dlg.exec(); + auto dlg = new HelpMessageDialog(this, /* about */ true); + GUIUtil::ShowModalDialogAndDeleteOnClose(dlg); } void BitcoinGUI::showDebugWindow() @@ -987,10 +988,11 @@ void BitcoinGUI::openOptionsDialogWithTab(OptionsDialog::Tab tab) if (!clientModel || !clientModel->getOptionsModel()) return; - OptionsDialog dlg(this, enableWallet); - dlg.setCurrentTab(tab); - dlg.setModel(clientModel->getOptionsModel()); - dlg.exec(); + auto dlg = new OptionsDialog(this, enableWallet); + connect(dlg, &OptionsDialog::quitOnReset, this, &BitcoinGUI::quitRequested); + dlg->setCurrentTab(tab); + dlg->setModel(clientModel->getOptionsModel()); + GUIUtil::ShowModalDialogAndDeleteOnClose(dlg); } void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool header, SynchronizationState sync_state) @@ -1173,14 +1175,12 @@ void BitcoinGUI::message(const QString& title, QString message, unsigned int sty void BitcoinGUI::changeEvent(QEvent *e) { -#ifdef Q_OS_MACOS if (e->type() == QEvent::PaletteChange) { overviewAction->setIcon(platformStyle->SingleColorIcon(QStringLiteral(":/icons/overview"))); sendCoinsAction->setIcon(platformStyle->SingleColorIcon(QStringLiteral(":/icons/send"))); receiveCoinsAction->setIcon(platformStyle->SingleColorIcon(QStringLiteral(":/icons/receiving_addresses"))); historyAction->setIcon(platformStyle->SingleColorIcon(QStringLiteral(":/icons/history"))); } -#endif QMainWindow::changeEvent(e); @@ -1215,7 +1215,7 @@ void BitcoinGUI::closeEvent(QCloseEvent *event) // close rpcConsole in case it was open to make some space for the shutdown window rpcConsole->close(); - QApplication::quit(); + Q_EMIT quitRequested(); } else { @@ -1339,9 +1339,8 @@ void BitcoinGUI::setEncryptionStatus(int status) void BitcoinGUI::updateWalletStatus() { - if (!walletFrame) { - return; - } + assert(walletFrame); + WalletView * const walletView = walletFrame->currentWalletView(); if (!walletView) { return; @@ -1410,7 +1409,7 @@ void BitcoinGUI::detectShutdown() { if(rpcConsole) rpcConsole->hide(); - qApp->quit(); + Q_EMIT quitRequested(); } } @@ -1511,14 +1510,14 @@ void UnitDisplayStatusBarControl::mousePressEvent(QMouseEvent *event) void UnitDisplayStatusBarControl::changeEvent(QEvent* e) { -#ifdef Q_OS_MACOS if (e->type() == QEvent::PaletteChange) { QString style = QString("QLabel { color : %1 }").arg(m_platform_style->SingleColor().name()); if (style != styleSheet()) { setStyleSheet(style); } } -#endif + + QLabel::changeEvent(e); } /** Creates context menu, its actions, and wires up all the relevant signals for mouse events. */ diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index c83cd446a0..fa7ae4b87d 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -12,7 +12,7 @@ #include <qt/guiutil.h> #include <qt/optionsdialog.h> -#include <amount.h> +#include <consensus/amount.h> #include <QLabel> #include <QMainWindow> @@ -214,6 +214,7 @@ private: void openOptionsDialogWithTab(OptionsDialog::Tab tab); Q_SIGNALS: + void quitRequested(); /** Signal raised when a URI was entered or dragged to the GUI */ void receivedURI(const QString &uri); /** Signal raised when RPC console shown */ diff --git a/src/qt/bitcoinstrings.cpp b/src/qt/bitcoinstrings.cpp index bff253e58a..d2d4079ea9 100644 --- a/src/qt/bitcoinstrings.cpp +++ b/src/qt/bitcoinstrings.cpp @@ -42,6 +42,9 @@ QT_TRANSLATE_NOOP("bitcoin-core", "" "Error: Dumpfile version is not supported. This version of bitcoin-wallet " "only supports version 1 dumpfiles. Got dumpfile with version %s"), QT_TRANSLATE_NOOP("bitcoin-core", "" +"Error: Legacy wallets only support the \"legacy\", \"p2sh-segwit\", and " +"\"bech32\" address types"), +QT_TRANSLATE_NOOP("bitcoin-core", "" "Error: Listening for incoming connections failed (listen returned error %s)"), QT_TRANSLATE_NOOP("bitcoin-core", "" "Fee estimation failed. Fallbackfee is disabled. Wait a few blocks or enable -" @@ -103,9 +106,6 @@ QT_TRANSLATE_NOOP("bitcoin-core", "" "Total length of network version string (%i) exceeds maximum length (%i). " "Reduce the number or size of uacomments."), QT_TRANSLATE_NOOP("bitcoin-core", "" -"Transaction needs a change address, but we can't generate it. Please call " -"keypoolrefill first."), -QT_TRANSLATE_NOOP("bitcoin-core", "" "Unable to replay blocks. You will need to rebuild the database using -" "reindex-chainstate."), QT_TRANSLATE_NOOP("bitcoin-core", "" @@ -160,6 +160,7 @@ QT_TRANSLATE_NOOP("bitcoin-core", "Error: Got key that was not hex: %s"), QT_TRANSLATE_NOOP("bitcoin-core", "Error: Got value that was not hex: %s"), QT_TRANSLATE_NOOP("bitcoin-core", "Error: Keypool ran out, please call keypoolrefill first"), QT_TRANSLATE_NOOP("bitcoin-core", "Error: Missing checksum"), +QT_TRANSLATE_NOOP("bitcoin-core", "Error: No %s addresses available."), QT_TRANSLATE_NOOP("bitcoin-core", "Error: Unable to parse version %u as a uint32_t"), QT_TRANSLATE_NOOP("bitcoin-core", "Error: Unable to write record to new wallet"), QT_TRANSLATE_NOOP("bitcoin-core", "Failed to listen on any port. Use -listen=0 if you want this."), @@ -216,6 +217,7 @@ QT_TRANSLATE_NOOP("bitcoin-core", "Transaction amount too small"), QT_TRANSLATE_NOOP("bitcoin-core", "Transaction amounts must not be negative"), QT_TRANSLATE_NOOP("bitcoin-core", "Transaction has too long of a mempool chain"), QT_TRANSLATE_NOOP("bitcoin-core", "Transaction must have at least one recipient"), +QT_TRANSLATE_NOOP("bitcoin-core", "Transaction needs a change address, but we can't generate it. %s"), QT_TRANSLATE_NOOP("bitcoin-core", "Transaction too large"), QT_TRANSLATE_NOOP("bitcoin-core", "Unable to bind to %s on this computer (bind returned error %s)"), QT_TRANSLATE_NOOP("bitcoin-core", "Unable to bind to %s on this computer. %s is probably already running."), @@ -228,6 +230,7 @@ QT_TRANSLATE_NOOP("bitcoin-core", "Unknown -blockfilterindex value %s."), QT_TRANSLATE_NOOP("bitcoin-core", "Unknown address type '%s'"), QT_TRANSLATE_NOOP("bitcoin-core", "Unknown change type '%s'"), QT_TRANSLATE_NOOP("bitcoin-core", "Unknown network specified in -onlynet: '%s'"), +QT_TRANSLATE_NOOP("bitcoin-core", "Unknown new rules activated (versionbit %i)"), QT_TRANSLATE_NOOP("bitcoin-core", "Unsupported logging category %s=%s."), QT_TRANSLATE_NOOP("bitcoin-core", "Upgrading UTXO database"), QT_TRANSLATE_NOOP("bitcoin-core", "Upgrading txindex database"), @@ -235,5 +238,4 @@ QT_TRANSLATE_NOOP("bitcoin-core", "User Agent comment (%s) contains unsafe chara QT_TRANSLATE_NOOP("bitcoin-core", "Verifying blocks…"), QT_TRANSLATE_NOOP("bitcoin-core", "Verifying wallet(s)…"), QT_TRANSLATE_NOOP("bitcoin-core", "Wallet needed to be rewritten: restart %s to complete"), -QT_TRANSLATE_NOOP("bitcoin-core", "Warning: unknown new rules activated (versionbit %i)"), }; diff --git a/src/qt/bitcoinunits.cpp b/src/qt/bitcoinunits.cpp index 9660ba99f7..66d5eea7ac 100644 --- a/src/qt/bitcoinunits.cpp +++ b/src/qt/bitcoinunits.cpp @@ -4,6 +4,8 @@ #include <qt/bitcoinunits.h> +#include <consensus/amount.h> + #include <QStringList> #include <cassert> diff --git a/src/qt/bitcoinunits.h b/src/qt/bitcoinunits.h index e22ba0a938..e78a347bb1 100644 --- a/src/qt/bitcoinunits.h +++ b/src/qt/bitcoinunits.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_BITCOINUNITS_H #define BITCOIN_QT_BITCOINUNITS_H -#include <amount.h> +#include <consensus/amount.h> #include <QAbstractListModel> #include <QString> diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index bb2073b9fe..c86cb16af6 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -216,7 +216,7 @@ bool ClientModel::isReleaseVersion() const QString ClientModel::formatClientStartupTime() const { - return QDateTime::fromTime_t(GetStartupTime()).toString(); + return QDateTime::fromSecsSinceEpoch(GetStartupTime()).toString(); } QString ClientModel::dataDir() const @@ -294,7 +294,7 @@ static void BlockTipChanged(ClientModel* clientmodel, SynchronizationState sync_ bool invoked = QMetaObject::invokeMethod(clientmodel, "numBlocksChanged", Qt::QueuedConnection, Q_ARG(int, tip.block_height), - Q_ARG(QDateTime, QDateTime::fromTime_t(tip.block_time)), + Q_ARG(QDateTime, QDateTime::fromSecsSinceEpoch(tip.block_time)), Q_ARG(double, verificationProgress), Q_ARG(bool, fHeader), Q_ARG(SynchronizationState, sync_state)); diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 2360fa9b37..e93fedad28 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -55,7 +55,7 @@ CoinControlDialog::CoinControlDialog(CCoinControl& coin_control, WalletModel* _m contextMenu->addAction(tr("&Copy address"), this, &CoinControlDialog::copyAddress); contextMenu->addAction(tr("Copy &label"), this, &CoinControlDialog::copyLabel); contextMenu->addAction(tr("Copy &amount"), this, &CoinControlDialog::copyAmount); - copyTransactionHashAction = contextMenu->addAction(tr("Copy transaction &ID"), this, &CoinControlDialog::copyTransactionHash); + m_copy_transaction_outpoint_action = contextMenu->addAction(tr("Copy transaction &ID and output index"), this, &CoinControlDialog::copyTransactionOutpoint); contextMenu->addSeparator(); lockAction = contextMenu->addAction(tr("L&ock unspent"), this, &CoinControlDialog::lockCoin); unlockAction = contextMenu->addAction(tr("&Unlock unspent"), this, &CoinControlDialog::unlockCoin); @@ -180,7 +180,7 @@ void CoinControlDialog::showMenu(const QPoint &point) // disable some items (like Copy Transaction ID, lock, unlock) for tree roots in context menu if (item->data(COLUMN_ADDRESS, TxHashRole).toString().length() == 64) // transaction hash is 64 characters (this means it is a child node, so it is not a parent node in tree mode) { - copyTransactionHashAction->setEnabled(true); + m_copy_transaction_outpoint_action->setEnabled(true); if (model->wallet().isLockedCoin(COutPoint(uint256S(item->data(COLUMN_ADDRESS, TxHashRole).toString().toStdString()), item->data(COLUMN_ADDRESS, VOutRole).toUInt()))) { lockAction->setEnabled(false); @@ -194,7 +194,7 @@ void CoinControlDialog::showMenu(const QPoint &point) } else // this means click on parent node in tree mode -> disable all { - copyTransactionHashAction->setEnabled(false); + m_copy_transaction_outpoint_action->setEnabled(false); lockAction->setEnabled(false); unlockAction->setEnabled(false); } @@ -228,10 +228,14 @@ void CoinControlDialog::copyAddress() GUIUtil::setClipboard(contextMenuItem->text(COLUMN_ADDRESS)); } -// context menu action: copy transaction id -void CoinControlDialog::copyTransactionHash() +// context menu action: copy transaction id and vout index +void CoinControlDialog::copyTransactionOutpoint() { - GUIUtil::setClipboard(contextMenuItem->data(COLUMN_ADDRESS, TxHashRole).toString()); + const QString address = contextMenuItem->data(COLUMN_ADDRESS, TxHashRole).toString(); + const QString vout = contextMenuItem->data(COLUMN_ADDRESS, VOutRole).toString(); + const QString outpoint = QString("%1:%2").arg(address).arg(vout); + + GUIUtil::setClipboard(outpoint); } // context menu action: lock coin @@ -241,7 +245,7 @@ void CoinControlDialog::lockCoin() contextMenuItem->setCheckState(COLUMN_CHECKBOX, Qt::Unchecked); COutPoint outpt(uint256S(contextMenuItem->data(COLUMN_ADDRESS, TxHashRole).toString().toStdString()), contextMenuItem->data(COLUMN_ADDRESS, VOutRole).toUInt()); - model->wallet().lockCoin(outpt); + model->wallet().lockCoin(outpt, /* write_to_db = */ true); contextMenuItem->setDisabled(true); contextMenuItem->setIcon(COLUMN_CHECKBOX, platformStyle->SingleColorIcon(":/icons/lock_closed")); updateLabelLocked(); @@ -564,11 +568,11 @@ void CoinControlDialog::updateLabels(CCoinControl& m_coin_control, WalletModel * void CoinControlDialog::changeEvent(QEvent* e) { -#ifdef Q_OS_MACOS if (e->type() == QEvent::PaletteChange) { updateView(); } -#endif + + QDialog::changeEvent(e); } void CoinControlDialog::updateView() diff --git a/src/qt/coincontroldialog.h b/src/qt/coincontroldialog.h index 3a03341c9e..ec2619d115 100644 --- a/src/qt/coincontroldialog.h +++ b/src/qt/coincontroldialog.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_COINCONTROLDIALOG_H #define BITCOIN_QT_COINCONTROLDIALOG_H -#include <amount.h> +#include <consensus/amount.h> #include <QAbstractButton> #include <QAction> @@ -63,7 +63,7 @@ private: QMenu *contextMenu; QTreeWidgetItem *contextMenuItem; - QAction *copyTransactionHashAction; + QAction* m_copy_transaction_outpoint_action; QAction *lockAction; QAction *unlockAction; @@ -95,7 +95,7 @@ private Q_SLOTS: void copyAmount(); void copyLabel(); void copyAddress(); - void copyTransactionHash(); + void copyTransactionOutpoint(); void lockCoin(); void unlockCoin(); void clipboardQuantity(); diff --git a/src/qt/createwalletdialog.cpp b/src/qt/createwalletdialog.cpp index dc24bbc6a6..f9a61c3e60 100644 --- a/src/qt/createwalletdialog.cpp +++ b/src/qt/createwalletdialog.cpp @@ -32,7 +32,7 @@ CreateWalletDialog::CreateWalletDialog(QWidget* parent) : // set to true, enable it when isEncryptWalletChecked is false. ui->disable_privkeys_checkbox->setEnabled(!checked); #ifdef ENABLE_EXTERNAL_SIGNER - ui->external_signer_checkbox->setEnabled(!checked); + ui->external_signer_checkbox->setEnabled(m_has_signers && !checked); #endif // When the disable_privkeys_checkbox is disabled, uncheck it. if (!ui->disable_privkeys_checkbox->isEnabled()) { @@ -115,7 +115,8 @@ CreateWalletDialog::~CreateWalletDialog() void CreateWalletDialog::setSigners(const std::vector<ExternalSigner>& signers) { - if (!signers.empty()) { + m_has_signers = !signers.empty(); + if (m_has_signers) { ui->external_signer_checkbox->setEnabled(true); ui->external_signer_checkbox->setChecked(true); ui->encrypt_wallet_checkbox->setEnabled(false); diff --git a/src/qt/createwalletdialog.h b/src/qt/createwalletdialog.h index 25ddf97585..fc13cc44eb 100644 --- a/src/qt/createwalletdialog.h +++ b/src/qt/createwalletdialog.h @@ -35,6 +35,7 @@ public: private: Ui::CreateWalletDialog *ui; + bool m_has_signers = false; }; #endif // BITCOIN_QT_CREATEWALLETDIALOG_H diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index bd72328c02..1c22124616 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -33,7 +33,7 @@ <string>Automatically start %1 after logging in to the system.</string> </property> <property name="text"> - <string>&Start %1 on system login</string> + <string>Start %1 on system &login</string> </property> </widget> </item> @@ -51,20 +51,20 @@ </spacer> </item> <item> - <layout class="QHBoxLayout" name="horizontalLayout_Main_Prune"> - <item> - <widget class="QCheckBox" name="prune"> - <property name="toolTip"> - <string>Enabling pruning significantly reduces the disk space required to store transactions. All blocks are still fully validated. Reverting this setting requires re-downloading the entire blockchain.</string> - </property> - <property name="text"> - <string>Prune &block storage to</string> - </property> - </widget> - </item> - <item> - <widget class="QSpinBox" name="pruneSize"/> - </item> + <layout class="QHBoxLayout" name="horizontalLayout_Main_Prune"> + <item> + <widget class="QCheckBox" name="prune"> + <property name="toolTip"> + <string>Enabling pruning significantly reduces the disk space required to store transactions. All blocks are still fully validated. Reverting this setting requires re-downloading the entire blockchain.</string> + </property> + <property name="text"> + <string>Prune &block storage to</string> + </property> + </widget> + </item> + <item> + <widget class="QSpinBox" name="pruneSize"/> + </item> <item> <widget class="QLabel" name="pruneSizeUnitLabel"> <property name="text"> @@ -104,6 +104,9 @@ <layout class="QHBoxLayout" name="horizontalLayout_2_Main"> <item> <widget class="QLabel" name="databaseCacheLabel"> + <property name="toolTip"> + <string extracomment="Tooltip text for Options window setting that sets the size of the database cache. Explains the corresponding effects of increasing/decreasing this value.">Maximum database cache size. A larger cache can contribute to faster sync, after which the benefit is less pronounced for most use cases. Lowering the cache size will reduce memory usage. Unused mempool memory is shared for this cache.</string> + </property> <property name="text"> <string>Size of &database cache</string> </property> @@ -147,6 +150,9 @@ <layout class="QHBoxLayout" name="horizontalLayout_Main_VerifyLabel"> <item> <widget class="QLabel" name="threadsScriptVerifLabel"> + <property name="toolTip"> + <string extracomment="Tooltip text for Options window setting that sets the number of script verification threads. Explains that negative values mean to leave these many cores free to the system.">Set the number of script verification threads. Negative values correspond to the number of cores you want to leave free to the system.</string> + </property> <property name="text"> <string>Number of script &verification threads</string> </property> @@ -173,7 +179,7 @@ <property name="sizeHint" stdset="0"> <size> <width>40</width> - <height>20</height> + <height>40</height> </size> </property> </spacer> @@ -181,6 +187,16 @@ </layout> </item> <item> + <widget class="QCheckBox" name="enableServer"> + <property name="toolTip"> + <string extracomment="Tooltip text for Options window setting that enables the RPC server.">This allows you or a third party tool to communicate with the node through command-line and JSON-RPC commands.</string> + </property> + <property name="text"> + <string extracomment="An Options window setting to enable the RPC server.">Enable RPC &server</string> + </property> + </widget> + </item> + <item> <spacer name="verticalSpacer_Main"> <property name="orientation"> <enum>Qt::Vertical</enum> @@ -201,6 +217,16 @@ </attribute> <layout class="QVBoxLayout" name="verticalLayout_Wallet"> <item> + <widget class="QCheckBox" name="subFeeFromAmount"> + <property name="toolTip"> + <string extracomment="Tooltip text for Options window setting that sets subtracting the fee from a sending amount as default.">Whether to set subtract fee from amount as default or not.</string> + </property> + <property name="text"> + <string extracomment="An Options window setting to set subtracting the fee from a sending amount as default.">Subtract &fee from amount by default</string> + </property> + </widget> + </item> + <item> <widget class="QGroupBox" name="groupBox"> <property name="title"> <string>Expert</string> @@ -235,27 +261,27 @@ <string>External Signer (e.g. hardware wallet)</string> </property> <layout class="QVBoxLayout" name="verticalLayoutHww"> - <item> - <layout class="QHBoxLayout" name="horizontalLayoutHww"> - <item> - <widget class="QLabel" name="externalSignerPathLabel"> - <property name="text"> - <string>&External signer script path</string> - </property> - <property name="buddy"> - <cstring>externalSignerPath</cstring> - </property> - </widget> - </item> - <item> - <widget class="QLineEdit" name="externalSignerPath"> - <property name="toolTip"> - <string>Full path to a Bitcoin Core compatible script (e.g. C:\Downloads\hwi.exe or /Users/you/Downloads/hwi.py). Beware: malware can steal your coins!</string> - </property> - </widget> - </item> - </layout> - </item> + <item> + <layout class="QHBoxLayout" name="horizontalLayoutHww"> + <item> + <widget class="QLabel" name="externalSignerPathLabel"> + <property name="text"> + <string>&External signer script path</string> + </property> + <property name="buddy"> + <cstring>externalSignerPath</cstring> + </property> + </widget> + </item> + <item> + <widget class="QLineEdit" name="externalSignerPath"> + <property name="toolTip"> + <string>Full path to a Bitcoin Core compatible script (e.g. C:\Downloads\hwi.exe or /Users/you/Downloads/hwi.py). Beware: malware can steal your coins!</string> + </property> + </widget> + </item> + </layout> + </item> </layout> </widget> </item> @@ -713,10 +739,10 @@ <item> <widget class="QLabel" name="thirdPartyTxUrlsLabel"> <property name="toolTip"> - <string>Third party URLs (e.g. a block explorer) that appear in the transactions tab as context menu items. %s in the URL is replaced by transaction hash. Multiple URLs are separated by vertical bar |.</string> + <string>Third-party URLs (e.g. a block explorer) that appear in the transactions tab as context menu items. %s in the URL is replaced by transaction hash. Multiple URLs are separated by vertical bar |.</string> </property> <property name="text"> - <string>&Third party transaction URLs</string> + <string>&Third-party transaction URLs</string> </property> <property name="buddy"> <cstring>thirdPartyTxUrls</cstring> @@ -726,7 +752,7 @@ <item> <widget class="QLineEdit" name="thirdPartyTxUrls"> <property name="toolTip"> - <string>Third party URLs (e.g. a block explorer) that appear in the transactions tab as context menu items. %s in the URL is replaced by transaction hash. Multiple URLs are separated by vertical bar |.</string> + <string>Third-party URLs (e.g. a block explorer) that appear in the transactions tab as context menu items. %s in the URL is replaced by transaction hash. Multiple URLs are separated by vertical bar |.</string> </property> <property name="placeholderText"> <string notr="true">https://example.com/tx/%s</string> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 393dca8ccd..12d3a48d01 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -36,6 +36,7 @@ #include <QClipboard> #include <QDateTime> #include <QDesktopServices> +#include <QDialog> #include <QDoubleValidator> #include <QFileDialog> #include <QFont> @@ -81,7 +82,7 @@ QString dateTimeStr(const QDateTime &date) QString dateTimeStr(qint64 nTime) { - return dateTimeStr(QDateTime::fromTime_t((qint32)nTime)); + return dateTimeStr(QDateTime::fromSecsSinceEpoch(nTime)); } QFont fixedPitchFont(bool use_embedded_font) @@ -271,6 +272,12 @@ bool hasEntryData(const QAbstractItemView *view, int column, int role) return !selection.at(0).data(role).toString().isEmpty(); } +void LoadFont(const QString& file_name) +{ + const int id = QFontDatabase::addApplicationFont(file_name); + assert(id != -1); +} + QString getDefaultDataDirectory() { return boostPathToQString(GetDefaultDataDir()); @@ -673,14 +680,26 @@ QString ConnectionTypeToQString(ConnectionType conn_type, bool prepend_direction { QString prefix; if (prepend_direction) { - prefix = (conn_type == ConnectionType::INBOUND) ? QObject::tr("Inbound") : QObject::tr("Outbound") + " "; + prefix = (conn_type == ConnectionType::INBOUND) ? + /*: An inbound connection from a peer. An inbound connection + is a connection initiated by a peer. */ + QObject::tr("Inbound") : + /*: An outbound connection to a peer. An outbound connection + is a connection initiated by us. */ + QObject::tr("Outbound") + " "; } switch (conn_type) { case ConnectionType::INBOUND: return prefix; + //: Peer connection type that relays all network information. case ConnectionType::OUTBOUND_FULL_RELAY: return prefix + QObject::tr("Full Relay"); + /*: Peer connection type that relays network information about + blocks and not transactions or addresses. */ case ConnectionType::BLOCK_RELAY: return prefix + QObject::tr("Block Relay"); + //: Peer connection type established manually through one of several methods. case ConnectionType::MANUAL: return prefix + QObject::tr("Manual"); + //: Short-lived peer connection type that tests the aliveness of known addresses. case ConnectionType::FEELER: return prefix + QObject::tr("Feeler"); + //: Short-lived peer connection type that solicits known addresses from a peer. case ConnectionType::ADDR_FETCH: return prefix + QObject::tr("Address Fetch"); } // no default case, so the compiler can warn about missing cases assert(false); @@ -809,11 +828,10 @@ void ThemedLabel::setThemedPixmap(const QString& image_filename, int width, int void ThemedLabel::changeEvent(QEvent* e) { -#ifdef Q_OS_MACOS if (e->type() == QEvent::PaletteChange) { updateThemedPixmap(); } -#endif + QLabel::changeEvent(e); } @@ -959,4 +977,11 @@ void PrintSlotException( PrintExceptionContinue(exception, description.c_str()); } +void ShowModalDialogAndDeleteOnClose(QDialog* dialog) +{ + dialog->setAttribute(Qt::WA_DeleteOnClose); + dialog->setWindowModality(Qt::ApplicationModal); + dialog->show(); +} + } // namespace GUIUtil diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index 06a3b63668..211f3f506d 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_GUIUTIL_H #define BITCOIN_QT_GUIUTIL_H -#include <amount.h> +#include <consensus/amount.h> #include <fs.h> #include <net.h> #include <netaddress.h> @@ -41,6 +41,7 @@ class QAbstractButton; class QAbstractItemView; class QAction; class QDateTime; +class QDialog; class QFont; class QKeySequence; class QLineEdit; @@ -113,6 +114,11 @@ namespace GUIUtil void setClipboard(const QString& str); /** + * Loads the font from the file specified by file_name, aborts if it fails. + */ + void LoadFont(const QString& file_name); + + /** * Determine default data directory for operating system. */ QString getDefaultDataDirectory(); @@ -417,6 +423,11 @@ namespace GUIUtil type); } + /** + * Shows a QDialog instance asynchronously, and deletes it on close. + */ + void ShowModalDialogAndDeleteOnClose(QDialog* dialog); + } // namespace GUIUtil #endif // BITCOIN_QT_GUIUTIL_H diff --git a/src/qt/initexecutor.cpp b/src/qt/initexecutor.cpp new file mode 100644 index 0000000000..24ae7ba73d --- /dev/null +++ b/src/qt/initexecutor.cpp @@ -0,0 +1,71 @@ +// Copyright (c) 2014-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <qt/initexecutor.h> + +#include <interfaces/node.h> +#include <qt/guiutil.h> +#include <util/system.h> +#include <util/threadnames.h> + +#include <exception> + +#include <QDebug> +#include <QObject> +#include <QString> +#include <QThread> + +InitExecutor::InitExecutor(interfaces::Node& node) + : QObject(), m_node(node) +{ + m_context.moveToThread(&m_thread); + m_thread.start(); +} + +InitExecutor::~InitExecutor() +{ + qDebug() << __func__ << ": Stopping thread"; + m_thread.quit(); + m_thread.wait(); + qDebug() << __func__ << ": Stopped thread"; +} + +void InitExecutor::handleRunawayException(const std::exception* e) +{ + PrintExceptionContinue(e, "Runaway exception"); + Q_EMIT runawayException(QString::fromStdString(m_node.getWarnings().translated)); +} + +void InitExecutor::initialize() +{ + GUIUtil::ObjectInvoke(&m_context, [this] { + try { + util::ThreadRename("qt-init"); + qDebug() << "Running initialization in thread"; + interfaces::BlockAndHeaderTipInfo tip_info; + bool rv = m_node.appInitMain(&tip_info); + Q_EMIT initializeResult(rv, tip_info); + } catch (const std::exception& e) { + handleRunawayException(&e); + } catch (...) { + handleRunawayException(nullptr); + } + }); +} + +void InitExecutor::shutdown() +{ + GUIUtil::ObjectInvoke(&m_context, [this] { + try { + qDebug() << "Running Shutdown in thread"; + m_node.appShutdown(); + qDebug() << "Shutdown finished"; + Q_EMIT shutdownResult(); + } catch (const std::exception& e) { + handleRunawayException(&e); + } catch (...) { + handleRunawayException(nullptr); + } + }); +} diff --git a/src/qt/initexecutor.h b/src/qt/initexecutor.h new file mode 100644 index 0000000000..410c44fa2d --- /dev/null +++ b/src/qt/initexecutor.h @@ -0,0 +1,47 @@ +// Copyright (c) 2014-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_QT_INITEXECUTOR_H +#define BITCOIN_QT_INITEXECUTOR_H + +#include <interfaces/node.h> + +#include <exception> + +#include <QObject> +#include <QThread> + +QT_BEGIN_NAMESPACE +class QString; +QT_END_NAMESPACE + +/** Class encapsulating Bitcoin Core startup and shutdown. + * Allows running startup and shutdown in a different thread from the UI thread. + */ +class InitExecutor : public QObject +{ + Q_OBJECT +public: + explicit InitExecutor(interfaces::Node& node); + ~InitExecutor(); + +public Q_SLOTS: + void initialize(); + void shutdown(); + +Q_SIGNALS: + void initializeResult(bool success, interfaces::BlockAndHeaderTipInfo tip_info); + void shutdownResult(); + void runawayException(const QString& message); + +private: + /// Pass fatal exception message to UI thread + void handleRunawayException(const std::exception* e); + + interfaces::Node& m_node; + QObject m_context; + QThread m_thread; +}; + +#endif // BITCOIN_QT_INITEXECUTOR_H diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index a698a96857..4c78fba752 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -113,7 +113,7 @@ namespace { //! Return pruning size that will be used if automatic pruning is enabled. int GetPruneTargetGB() { - int64_t prune_target_mib = gArgs.GetArg("-prune", 0); + int64_t prune_target_mib = gArgs.GetIntArg("-prune", 0); // >1 means automatic pruning is enabled by config, 1 means manual pruning, 0 means no pruning. return prune_target_mib > 1 ? PruneMiBtoGB(prune_target_mib) : DEFAULT_PRUNE_TARGET_GB; } @@ -142,7 +142,7 @@ Intro::Intro(QWidget *parent, int64_t blockchain_size_gb, int64_t chain_state_si const int min_prune_target_GB = std::ceil(MIN_DISK_SPACE_FOR_BLOCK_FILES / 1e9); ui->pruneGB->setRange(min_prune_target_GB, std::numeric_limits<int>::max()); - if (gArgs.GetArg("-prune", 0) > 1) { // -prune=1 means enabled, above that it's a size in MiB + if (gArgs.GetIntArg("-prune", 0) > 1) { // -prune=1 means enabled, above that it's a size in MiB ui->prune->setChecked(true); ui->prune->setEnabled(false); } diff --git a/src/qt/locale/bitcoin_en.ts b/src/qt/locale/bitcoin_en.ts index b146489ba8..47c002498a 100644 --- a/src/qt/locale/bitcoin_en.ts +++ b/src/qt/locale/bitcoin_en.ts @@ -55,11 +55,12 @@ </message> <message> <location line="-30"/> + <location filename="../addressbookpage.cpp" line="+122"/> <source>&Delete</source> <translation>&Delete</translation> </message> <message> - <location filename="../addressbookpage.cpp" line="+84"/> + <location filename="../addressbookpage.cpp" line="-38"/> <source>Choose the address to send coins to</source> <translation type="unfinished"></translation> </message> @@ -96,26 +97,21 @@ Signing is only possible with addresses of the type 'legacy'.</source> </message> <message> <location line="+8"/> - <source>Copy Address</source> + <source>&Copy Address</source> <translation type="unfinished"></translation> </message> <message> <location line="+1"/> - <source>Copy Label</source> + <source>Copy &Label</source> <translation type="unfinished"></translation> </message> <message> <location line="+1"/> - <source>Edit</source> + <source>&Edit</source> <translation type="unfinished"></translation> </message> <message> - <location line="+3"/> - <source>Delete</source> - <translation type="unfinished"></translation> - </message> - <message> - <location line="+161"/> + <location line="+164"/> <source>Export Address List</source> <translation type="unfinished"></translation> </message> @@ -317,7 +313,7 @@ Signing is only possible with addresses of the type 'legacy'.</source> <context> <name>BitcoinApplication</name> <message> - <location filename="../bitcoin.cpp" line="+420"/> + <location filename="../bitcoin.cpp" line="+421"/> <source>Runaway exception</source> <translation type="unfinished"></translation> </message> @@ -405,18 +401,18 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation type="unfinished"></translation> </message> <message> - <location line="+358"/> + <location line="+373"/> <source>Network activity disabled.</source> <extracomment>A substring of the tooltip.</extracomment> <translation type="unfinished"></translation> </message> <message> - <location line="+426"/> + <location line="+424"/> <source>Proxy is <b>enabled</b>: %1</source> <translation type="unfinished"></translation> </message> <message> - <location line="-1096"/> + <location line="-1109"/> <source>Send coins to a Bitcoin address</source> <translation>Send coins to a Bitcoin address</translation> </message> @@ -546,7 +542,7 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation>Tabs toolbar</translation> </message> <message> - <location line="+422"/> + <location line="+437"/> <source>Syncing Headers (%1%)…</source> <translation type="unfinished"></translation> </message> @@ -576,7 +572,7 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation type="unfinished"></translation> </message> <message> - <location line="-766"/> + <location line="-781"/> <source>Request payments (generates QR codes and bitcoin: URIs)</source> <translation type="unfinished"></translation> </message> @@ -596,7 +592,7 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation type="unfinished"></translation> </message> <message numerus="yes"> - <location line="+678"/> + <location line="+693"/> <source>Processed %n block(s) of transaction history.</source> <translation> <numerusform>Processed %n block of transaction history.</numerusform> @@ -644,7 +640,7 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation>Up to date</translation> </message> <message> - <location line="-715"/> + <location line="-730"/> <source>Load Partially Signed Bitcoin Transaction</source> <translation type="unfinished"></translation> </message> @@ -744,7 +740,7 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation type="unfinished"></translation> </message> <message> - <location line="+248"/> + <location line="+263"/> <source>%1 client</source> <translation type="unfinished"></translation> </message> @@ -753,8 +749,8 @@ Signing is only possible with addresses of the type 'legacy'.</source> <source>%n active connection(s) to Bitcoin network.</source> <extracomment>A substring of the tooltip.</extracomment> <translation type="unfinished"> - <numerusform></numerusform> - <numerusform></numerusform> + <numerusform>%n active connection to Bitcoin network.</numerusform> + <numerusform>%n active connections to Bitcoin network.</numerusform> </translation> </message> <message> @@ -792,7 +788,7 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation type="unfinished"></translation> </message> <message> - <location line="+110"/> + <location line="+108"/> <source>Date: %1 </source> <translation type="unfinished"></translation> @@ -956,34 +952,38 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation type="unfinished">Confirmed</translation> </message> <message> - <location filename="../coincontroldialog.cpp" line="+55"/> - <source>Copy address</source> + <location filename="../coincontroldialog.cpp" line="+66"/> + <source>Copy amount</source> + <translation type="unfinished"></translation> + </message> + <message> + <location line="-11"/> + <source>&Copy address</source> <translation type="unfinished"></translation> </message> <message> <location line="+1"/> - <source>Copy label</source> + <source>Copy &label</source> <translation type="unfinished"></translation> </message> <message> <location line="+1"/> - <location line="+9"/> - <source>Copy amount</source> + <source>Copy &amount</source> <translation type="unfinished"></translation> </message> <message> - <location line="-8"/> - <source>Copy transaction ID</source> + <location line="+1"/> + <source>Copy transaction &ID</source> <translation type="unfinished"></translation> </message> <message> <location line="+2"/> - <source>Lock unspent</source> + <source>L&ock unspent</source> <translation type="unfinished"></translation> </message> <message> <location line="+1"/> - <source>Unlock unspent</source> + <source>&Unlock unspent</source> <translation type="unfinished"></translation> </message> <message> @@ -1061,12 +1061,12 @@ Signing is only possible with addresses of the type 'legacy'.</source> <context> <name>CreateWalletActivity</name> <message> - <location filename="../walletcontroller.cpp" line="+253"/> + <location filename="../walletcontroller.cpp" line="+254"/> <source>Creating Wallet <b>%1</b>…</source> <translation type="unfinished"></translation> </message> <message> - <location line="+28"/> + <location line="+31"/> <source>Create wallet failed</source> <translation type="unfinished"></translation> </message> @@ -1075,6 +1075,11 @@ Signing is only possible with addresses of the type 'legacy'.</source> <source>Create wallet warning</source> <translation type="unfinished"></translation> </message> + <message> + <location line="+16"/> + <source>Can't list signers</source> + <translation type="unfinished"></translation> + </message> </context> <context> <name>CreateWalletDialog</name> @@ -1139,15 +1144,31 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation type="unfinished"></translation> </message> <message> - <location filename="../createwalletdialog.cpp" line="+21"/> + <location line="+7"/> + <source>Use an external signing device such as a hardware wallet. Configure the external signer script in wallet preferences first.</source> + <translation type="unfinished"></translation> + </message> + <message> + <location line="+3"/> + <source>External signer</source> + <translation type="unfinished"></translation> + </message> + <message> + <location filename="../createwalletdialog.cpp" line="+22"/> <source>Create</source> <translation type="unfinished"></translation> </message> <message> - <location line="+42"/> + <location line="+68"/> <source>Compiled without sqlite support (required for descriptor wallets)</source> <translation type="unfinished"></translation> </message> + <message> + <location line="+14"/> + <source>Compiled without external signing support (required for external signing)</source> + <extracomment>"External signing" means using devices such as hardware wallets.</extracomment> + <translation type="unfinished"></translation> + </message> </context> <context> <name>EditAddressDialog</name> @@ -1355,8 +1376,8 @@ Signing is only possible with addresses of the type 'legacy'.</source> <source>(sufficient to restore backups %n day(s) old)</source> <extracomment>Explanatory text on the capability of the current prune target.</extracomment> <translation type="unfinished"> - <numerusform></numerusform> - <numerusform></numerusform> + <numerusform>(sufficient to restore backups %n day old)</numerusform> + <numerusform>(sufficient to restore backups %n days old)</numerusform> </translation> </message> <message> @@ -1472,7 +1493,7 @@ Signing is only possible with addresses of the type 'legacy'.</source> <context> <name>OpenWalletActivity</name> <message> - <location filename="../walletcontroller.cpp" line="+39"/> + <location filename="../walletcontroller.cpp" line="+32"/> <source>Open wallet failed</source> <translation type="unfinished"></translation> </message> @@ -1530,7 +1551,7 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation type="unfinished"></translation> </message> <message> - <location line="+171"/> + <location line="+201"/> <location line="+187"/> <source>IP address of the proxy (e.g. IPv4: 127.0.0.1 / IPv6: ::1)</source> <translation type="unfinished"></translation> @@ -1579,7 +1600,7 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation>&Network</translation> </message> <message> - <location line="-188"/> + <location line="-218"/> <source>Prune &block storage to</source> <translation type="unfinished"></translation> </message> @@ -1629,7 +1650,22 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation type="unfinished"></translation> </message> <message> - <location line="+30"/> + <location line="+10"/> + <source>External Signer (e.g. hardware wallet)</source> + <translation type="unfinished"></translation> + </message> + <message> + <location line="+8"/> + <source>&External signer script path</source> + <translation type="unfinished"></translation> + </message> + <message> + <location line="+10"/> + <source>Full path to a Bitcoin Core compatible script (e.g. C:\Downloads\hwi.exe or /Users/you/Downloads/hwi.py). Beware: malware can steal your coins!</source> + <translation type="unfinished"></translation> + </message> + <message> + <location line="+32"/> <source>Automatically open the Bitcoin client port on the router. This only works when your router supports UPnP and it is enabled.</source> <translation>Automatically open the Bitcoin client port on the router. This only works when your router supports UPnP and it is enabled.</translation> </message> @@ -1762,12 +1798,12 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation>Choose the default subdivision unit to show in the interface and when sending coins.</translation> </message> <message> - <location line="-463"/> + <location line="-493"/> <source>Whether to show coin control features or not.</source> <translation type="unfinished"></translation> </message> <message> - <location line="+260"/> + <location line="+290"/> <source>Connect to the Bitcoin network through a separate SOCKS5 proxy for Tor onion services.</source> <translation type="unfinished"></translation> </message> @@ -1812,7 +1848,13 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation>&Cancel</translation> </message> <message> - <location filename="../optionsdialog.cpp" line="+104"/> + <location filename="../optionsdialog.cpp" line="+97"/> + <source>Compiled without external signing support (required for external signing)</source> + <extracomment>"External signing" means using devices such as hardware wallets.</extracomment> + <translation type="unfinished"></translation> + </message> + <message> + <location line="+12"/> <source>default</source> <translation>default</translation> </message> @@ -1822,7 +1864,7 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation type="unfinished"></translation> </message> <message> - <location line="+91"/> + <location line="+93"/> <source>Confirm options reset</source> <translation>Confirm options reset</translation> </message> @@ -1962,7 +2004,7 @@ Signing is only possible with addresses of the type 'legacy'.</source> <translation type="unfinished"></translation> </message> <message> - <location filename="../overviewpage.cpp" line="+193"/> + <location filename="../overviewpage.cpp" line="+188"/> <source>Privacy mode activated for the Overview tab. To unmask the values, uncheck Settings->Mask values.</source> <translation type="unfinished"></translation> </message> @@ -2168,7 +2210,7 @@ If you are receiving this error you should request the merchant provide a BIP21 <context> <name>PeerTableModel</name> <message> - <location filename="../peertablemodel.h" line="+101"/> + <location filename="../peertablemodel.h" line="+107"/> <source>User Agent</source> <extracomment>Title of Peers Table column which contains the peer's User Agent string.</extracomment> <translation type="unfinished"></translation> @@ -2384,7 +2426,7 @@ If you are receiving this error you should request the merchant provide a BIP21 <translation type="unfinished"></translation> </message> <message> - <location filename="../bitcoin.cpp" line="+111"/> + <location filename="../bitcoin.cpp" line="+112"/> <source>Error: Specified data directory "%1" does not exist.</source> <translation type="unfinished"></translation> </message> @@ -2418,12 +2460,12 @@ If you are receiving this error you should request the merchant provide a BIP21 <name>QRImageWidget</name> <message> <location filename="../qrimagewidget.cpp" line="+30"/> - <source>Save Image…</source> + <source>&Save Image…</source> <translation type="unfinished"></translation> </message> <message> <location line="+1"/> - <source>Copy Image</source> + <source>&Copy Image</source> <translation type="unfinished"></translation> </message> <message> @@ -2467,7 +2509,7 @@ If you are receiving this error you should request the merchant provide a BIP21 <location line="+23"/> <location line="+36"/> <location line="+23"/> - <location line="+722"/> + <location line="+692"/> <location line="+26"/> <location line="+26"/> <location line="+23"/> @@ -2490,12 +2532,12 @@ If you are receiving this error you should request the merchant provide a BIP21 <location line="+23"/> <location line="+23"/> <location line="+26"/> - <location filename="../rpcconsole.h" line="+138"/> + <location filename="../rpcconsole.h" line="+139"/> <source>N/A</source> <translation>N/A</translation> </message> <message> - <location line="-1549"/> + <location line="-1519"/> <source>Client version</source> <translation>Client version</translation> </message> @@ -2536,12 +2578,12 @@ If you are receiving this error you should request the merchant provide a BIP21 </message> <message> <location line="+29"/> - <location line="+922"/> + <location line="+892"/> <source>Network</source> <translation>Network</translation> </message> <message> - <location line="-915"/> + <location line="-885"/> <source>Name</source> <translation type="unfinished"></translation> </message> @@ -2581,7 +2623,7 @@ If you are receiving this error you should request the merchant provide a BIP21 <translation type="unfinished"></translation> </message> <message> - <location line="+241"/> + <location line="+211"/> <source>&Reset</source> <translation type="unfinished"></translation> </message> @@ -2609,7 +2651,7 @@ If you are receiving this error you should request the merchant provide a BIP21 </message> <message> <location line="+68"/> - <location filename="../rpcconsole.cpp" line="+1091"/> + <location filename="../rpcconsole.cpp" line="+1124"/> <source>Select a peer to view detailed information.</source> <translation type="unfinished"></translation> </message> @@ -2644,13 +2686,13 @@ If you are receiving this error you should request the merchant provide a BIP21 <translation type="unfinished"></translation> </message> <message> - <location line="-1516"/> - <location line="+1081"/> + <location line="-1486"/> + <location line="+1051"/> <source>User Agent</source> <translation type="unfinished"></translation> </message> <message> - <location line="-1155"/> + <location line="-1125"/> <source>Node window</source> <translation type="unfinished"></translation> </message> @@ -2665,17 +2707,17 @@ If you are receiving this error you should request the merchant provide a BIP21 <translation type="unfinished"></translation> </message> <message> - <location line="+84"/> + <location line="+78"/> <source>Decrease font size</source> <translation type="unfinished"></translation> </message> <message> - <location line="+32"/> + <location line="+20"/> <source>Increase font size</source> <translation type="unfinished"></translation> </message> <message> - <location line="+558"/> + <location line="+546"/> <source>Permissions</source> <translation type="unfinished"></translation> </message> @@ -2780,7 +2822,7 @@ If you are receiving this error you should request the merchant provide a BIP21 <translation type="unfinished"></translation> </message> <message> - <location line="-1288"/> + <location line="-1258"/> <source>Last block time</source> <translation>Last block time</translation> </message> @@ -2795,7 +2837,7 @@ If you are receiving this error you should request the merchant provide a BIP21 <translation>&Console</translation> </message> <message> - <location line="+217"/> + <location line="+187"/> <source>&Network Traffic</source> <translation type="unfinished"></translation> </message> @@ -2815,12 +2857,12 @@ If you are receiving this error you should request the merchant provide a BIP21 <translation type="unfinished"></translation> </message> <message> - <location filename="../forms/debugwindow.ui" line="-321"/> + <location filename="../forms/debugwindow.ui" line="-291"/> <source>Debug log file</source> <translation>Debug log file</translation> </message> <message> - <location line="+155"/> + <location line="+125"/> <source>Clear console</source> <translation>Clear console</translation> </message> @@ -2850,12 +2892,12 @@ If you are receiving this error you should request the merchant provide a BIP21 <translation type="unfinished"></translation> </message> <message> - <location line="+38"/> + <location line="+40"/> <source>Never</source> <translation type="unfinished"></translation> </message> <message> - <location filename="../rpcconsole.cpp" line="-417"/> + <location filename="../rpcconsole.cpp" line="-429"/> <source>Inbound: initiated by peer</source> <translation type="unfinished"></translation> </message> @@ -2924,52 +2966,52 @@ If you are receiving this error you should request the merchant provide a BIP21 <translation type="unfinished"></translation> </message> <message> - <location line="+385"/> - <source>Network activity disabled</source> + <location line="+150"/> + <source>&Disconnect</source> <translation type="unfinished"></translation> </message> <message> - <location line="+77"/> - <source>Executing command without any wallet</source> + <location line="+1"/> + <source>1 &hour</source> <translation type="unfinished"></translation> </message> <message> - <location line="-2"/> - <source>Executing command using "%1" wallet</source> + <location line="+1"/> + <source>1 d&ay</source> <translation type="unfinished"></translation> </message> <message> - <location line="-319"/> - <source>Disconnect</source> + <location line="+1"/> + <source>1 &week</source> <translation type="unfinished"></translation> </message> <message> <location line="+1"/> - <source>1 hour</source> + <source>1 &year</source> <translation type="unfinished"></translation> </message> <message> - <location line="+1"/> - <source>1 day</source> + <location line="+22"/> + <source>&Unban</source> <translation type="unfinished"></translation> </message> <message> - <location line="+1"/> - <source>1 week</source> + <location line="+221"/> + <source>Network activity disabled</source> <translation type="unfinished"></translation> </message> <message> - <location line="+1"/> - <source>1 year</source> + <location line="+77"/> + <source>Executing command without any wallet</source> <translation type="unfinished"></translation> </message> <message> - <location line="+19"/> - <source>Unban</source> + <location line="-2"/> + <source>Executing command using "%1" wallet</source> <translation type="unfinished"></translation> </message> <message> - <location line="+150"/> + <location line="-146"/> <source>Welcome to the %1 RPC console. Use up and down arrows to navigate history, and %2 to clear screen. Use %3 and %4 to increase or decrease the font size. @@ -2997,7 +3039,7 @@ For more information on using this console, type %6. <translation type="unfinished"></translation> </message> <message> - <location filename="../rpcconsole.h" line="-38"/> + <location filename="../rpcconsole.h" line="-40"/> <source>Unknown</source> <translation type="unfinished"></translation> </message> @@ -3102,27 +3144,27 @@ For more information on using this console, type %6. </message> <message> <location filename="../receivecoinsdialog.cpp" line="+47"/> - <source>Copy URI</source> + <source>Copy &URI</source> <translation type="unfinished"></translation> </message> <message> <location line="+1"/> - <source>Copy address</source> + <source>&Copy address</source> <translation type="unfinished"></translation> </message> <message> <location line="+1"/> - <source>Copy label</source> + <source>Copy &label</source> <translation type="unfinished"></translation> </message> <message> <location line="+1"/> - <source>Copy message</source> + <source>Copy &message</source> <translation type="unfinished"></translation> </message> <message> <location line="+1"/> - <source>Copy amount</source> + <source>Copy &amount</source> <translation type="unfinished"></translation> </message> <message> @@ -3180,6 +3222,16 @@ For more information on using this console, type %6. </message> <message> <location line="+10"/> + <source>&Verify</source> + <translation type="unfinished"></translation> + </message> + <message> + <location line="+3"/> + <source>Verify this address on e.g. a hardware wallet screen</source> + <translation type="unfinished"></translation> + </message> + <message> + <location line="+10"/> <source>&Save Image…</source> <translation type="unfinished"></translation> </message> @@ -3189,7 +3241,7 @@ For more information on using this console, type %6. <translation type="unfinished"></translation> </message> <message> - <location filename="../forms/receiverequestdialog.ui" line="-221"/> + <location filename="../forms/receiverequestdialog.ui" line="-234"/> <source>Payment information</source> <translation type="unfinished"></translation> </message> @@ -3197,7 +3249,7 @@ For more information on using this console, type %6. <context> <name>RecentRequestsTableModel</name> <message> - <location filename="../recentrequeststablemodel.cpp" line="+30"/> + <location filename="../recentrequeststablemodel.cpp" line="+32"/> <source>Date</source> <translation type="unfinished">Date</translation> </message> @@ -3236,7 +3288,7 @@ For more information on using this console, type %6. <name>SendCoinsDialog</name> <message> <location filename="../forms/sendcoinsdialog.ui" line="+14"/> - <location filename="../sendcoinsdialog.cpp" line="+674"/> + <location filename="../sendcoinsdialog.cpp" line="+738"/> <source>Send Coins</source> <translation>Send Coins</translation> </message> @@ -3423,7 +3475,7 @@ Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 <translation>S&end</translation> </message> <message> - <location filename="../sendcoinsdialog.cpp" line="-582"/> + <location filename="../sendcoinsdialog.cpp" line="-646"/> <source>Copy quantity</source> <translation type="unfinished"></translation> </message> @@ -3463,7 +3515,24 @@ Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 <translation type="unfinished"></translation> </message> <message> - <location line="+29"/> + <location line="+30"/> + <source>Sign on device</source> + <extracomment>"device" usually means a hardware wallet</extracomment> + <translation type="unfinished"></translation> + </message> + <message> + <location line="+3"/> + <source>Connect your hardware wallet first.</source> + <translation type="unfinished"></translation> + </message> + <message> + <location line="+4"/> + <source>Set external signer script path in Options -> Wallet</source> + <extracomment>"External signer" means using devices such as hardware wallets.</extracomment> + <translation type="unfinished"></translation> + </message> + <message> + <location line="+3"/> <source>Cr&eate Unsigned</source> <translation type="unfinished"></translation> </message> @@ -3508,7 +3577,29 @@ Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 <translation type="unfinished"></translation> </message> <message> - <location line="+44"/> + <location line="+0"/> + <source>Sign and send</source> + <translation type="unfinished"></translation> + </message> + <message> + <location line="+25"/> + <source>Sign failed</source> + <translation type="unfinished"></translation> + </message> + <message> + <location line="+6"/> + <source>External signer not found</source> + <extracomment>"External signer" means using devices such as hardware wallets.</extracomment> + <translation type="unfinished"></translation> + </message> + <message> + <location line="+6"/> + <source>External signer failure</source> + <extracomment>"External signer" means using devices such as hardware wallets.</extracomment> + <translation type="unfinished"></translation> + </message> + <message> + <location line="+58"/> <source>Save Transaction Data</source> <translation type="unfinished"></translation> </message> @@ -3524,7 +3615,12 @@ Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 <translation type="unfinished"></translation> </message> <message> - <location line="-76"/> + <location line="+175"/> + <source>External balance:</source> + <translation type="unfinished"></translation> + </message> + <message> + <location line="-302"/> <source>or</source> <translation type="unfinished"></translation> </message> @@ -3569,12 +3665,7 @@ Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 <translation type="unfinished"></translation> </message> <message> - <location line="+1"/> - <source>Send</source> - <translation type="unfinished"></translation> - </message> - <message> - <location line="+229"/> + <location line="+283"/> <source>Watch-only balance:</source> <translation type="unfinished"></translation> </message> @@ -4434,62 +4525,62 @@ Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 <translation type="unfinished"></translation> </message> <message> - <location line="+75"/> - <source>Abandon transaction</source> + <location line="-26"/> + <source>Range…</source> <translation type="unfinished"></translation> </message> <message> - <location line="-3"/> - <source>Increase transaction fee</source> + <location line="+90"/> + <source>&Copy address</source> <translation type="unfinished"></translation> </message> <message> - <location line="-8"/> - <source>Copy address</source> + <location line="+1"/> + <source>Copy &label</source> <translation type="unfinished"></translation> </message> <message> <location line="+1"/> - <source>Copy label</source> + <source>Copy &amount</source> <translation type="unfinished"></translation> </message> <message> <location line="+1"/> - <source>Copy amount</source> + <source>Copy transaction &ID</source> <translation type="unfinished"></translation> </message> <message> <location line="+1"/> - <source>Copy transaction ID</source> + <source>Copy &raw transaction</source> <translation type="unfinished"></translation> </message> <message> <location line="+1"/> - <source>Copy raw transaction</source> + <source>Copy full transaction &details</source> <translation type="unfinished"></translation> </message> <message> <location line="+1"/> - <source>Copy full transaction details</source> + <source>&Show transaction details</source> <translation type="unfinished"></translation> </message> <message> - <location line="+7"/> - <source>Edit address label</source> + <location line="+2"/> + <source>Increase transaction &fee</source> <translation type="unfinished"></translation> </message> <message> - <location line="-6"/> - <source>Show transaction details</source> + <location line="+3"/> + <source>A&bandon transaction</source> <translation type="unfinished"></translation> </message> <message> - <location line="-96"/> - <source>Range…</source> + <location line="+1"/> + <source>&Edit address label</source> <translation type="unfinished"></translation> </message> <message> - <location line="+276"/> + <location line="+174"/> <source>Export Transaction History</source> <translation type="unfinished"></translation> </message> @@ -4576,7 +4667,7 @@ Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 <context> <name>WalletController</name> <message> - <location filename="../walletcontroller.cpp" line="-250"/> + <location filename="../walletcontroller.cpp" line="-262"/> <source>Close wallet</source> <translation type="unfinished"></translation> </message> @@ -4604,7 +4695,7 @@ Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 <context> <name>WalletFrame</name> <message> - <location filename="../walletframe.cpp" line="+39"/> + <location filename="../walletframe.cpp" line="+35"/> <source>No wallet has been loaded. Go to File > Open Wallet to load a wallet. - OR -</source> @@ -4624,7 +4715,7 @@ Go to File > Open Wallet to load a wallet. <translation type="unfinished">Send Coins</translation> </message> <message> - <location line="+279"/> + <location line="+260"/> <location line="+52"/> <location line="+13"/> <location line="+5"/> @@ -4692,7 +4783,12 @@ Go to File > Open Wallet to load a wallet. <translation type="unfinished"></translation> </message> <message> - <location line="+20"/> + <location line="+14"/> + <source>Can't display address</source> + <translation type="unfinished"></translation> + </message> + <message> + <location line="+18"/> <source>default wallet</source> <translation type="unfinished"></translation> </message> @@ -4842,6 +4938,11 @@ Go to File > Open Wallet to load a wallet. </message> <message> <location line="+3"/> + <source>Error: Legacy wallets only support the "legacy", "p2sh-segwit", and "bech32" address types</source> + <translation type="unfinished"></translation> + </message> + <message> + <location line="+3"/> <source>Error: Listening for incoming connections failed (listen returned error %s)</source> <translation type="unfinished"></translation> </message> @@ -4947,11 +5048,6 @@ Go to File > Open Wallet to load a wallet. </message> <message> <location line="+3"/> - <source>Transaction needs a change address, but we can't generate it. Please call keypoolrefill first.</source> - <translation type="unfinished"></translation> - </message> - <message> - <location line="+3"/> <source>Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate.</source> <translation type="unfinished"></translation> </message> @@ -5162,6 +5258,11 @@ Go to File > Open Wallet to load a wallet. </message> <message> <location line="+1"/> + <source>Error: No %s addresses available.</source> + <translation type="unfinished"></translation> + </message> + <message> + <location line="+1"/> <source>Error: Unable to parse version %u as a uint32_t</source> <translation type="unfinished"></translation> </message> @@ -5442,6 +5543,11 @@ Go to File > Open Wallet to load a wallet. </message> <message> <location line="+1"/> + <source>Transaction needs a change address, but we can't generate it. %s</source> + <translation type="unfinished"></translation> + </message> + <message> + <location line="+1"/> <source>Transaction too large</source> <translation type="unfinished"></translation> </message> @@ -5502,6 +5608,11 @@ Go to File > Open Wallet to load a wallet. </message> <message> <location line="+1"/> + <source>Unknown new rules activated (versionbit %i)</source> + <translation type="unfinished"></translation> + </message> + <message> + <location line="+1"/> <source>Unsupported logging category %s=%s.</source> <translation type="unfinished"></translation> </message> @@ -5535,10 +5646,5 @@ Go to File > Open Wallet to load a wallet. <source>Wallet needed to be rewritten: restart %s to complete</source> <translation type="unfinished"></translation> </message> - <message> - <location line="+1"/> - <source>Warning: unknown new rules activated (versionbit %i)</source> - <translation type="unfinished"></translation> - </message> </context> </TS> diff --git a/src/qt/locale/bitcoin_en.xlf b/src/qt/locale/bitcoin_en.xlf index ffe7812738..caefddc663 100644 --- a/src/qt/locale/bitcoin_en.xlf +++ b/src/qt/locale/bitcoin_en.xlf @@ -56,6 +56,7 @@ <source xml:space="preserve">&Delete</source> <target xml:space="preserve">&Delete</target> <context-group purpose="location"><context context-type="linenumber">101</context></context-group> + <context-group purpose="location"><context context-type="sourcefile">../addressbookpage.cpp</context><context context-type="linenumber">122</context></context-group> </trans-unit> </group> </body></file> @@ -98,43 +99,38 @@ Signing is only possible with addresses of the type 'legacy'.</source> <context-group purpose="location"><context context-type="linenumber">109</context></context-group> </trans-unit> <trans-unit id="_msg19"> - <source xml:space="preserve">Copy Address</source> + <source xml:space="preserve">&Copy Address</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">117</context></context-group> </trans-unit> <trans-unit id="_msg20"> - <source xml:space="preserve">Copy Label</source> + <source xml:space="preserve">Copy &Label</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">118</context></context-group> </trans-unit> <trans-unit id="_msg21"> - <source xml:space="preserve">Edit</source> + <source xml:space="preserve">&Edit</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">119</context></context-group> </trans-unit> <trans-unit id="_msg22"> - <source xml:space="preserve">Delete</source> - <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">122</context></context-group> - </trans-unit> - <trans-unit id="_msg23"> <source xml:space="preserve">Export Address List</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">283</context></context-group> </trans-unit> - <trans-unit id="_msg24"> + <trans-unit id="_msg23"> <source xml:space="preserve">Comma separated file</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">286</context></context-group> <note annotates="source" from="developer">Expanded name of the CSV file format. See https://en.wikipedia.org/wiki/Comma-separated_values</note> </trans-unit> - <trans-unit id="_msg25"> + <trans-unit id="_msg24"> <source xml:space="preserve">There was an error trying to save the address list to %1. Please try again.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">302</context></context-group> <note annotates="source" from="developer">An error message. %1 is a stand-in argument for the name of the file we attempted to save to.</note> </trans-unit> - <trans-unit id="_msg26"> + <trans-unit id="_msg25"> <source xml:space="preserve">Exporting Failed</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">299</context></context-group> @@ -143,17 +139,17 @@ Signing is only possible with addresses of the type 'legacy'.</source> </body></file> <file original="../addresstablemodel.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="AddressTableModel"> - <trans-unit id="_msg27"> + <trans-unit id="_msg26"> <source xml:space="preserve">Label</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">168</context></context-group> </trans-unit> - <trans-unit id="_msg28"> + <trans-unit id="_msg27"> <source xml:space="preserve">Address</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">168</context></context-group> </trans-unit> - <trans-unit id="_msg29"> + <trans-unit id="_msg28"> <source xml:space="preserve">(no label)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">206</context></context-group> @@ -162,27 +158,27 @@ Signing is only possible with addresses of the type 'legacy'.</source> </body></file> <file original="../forms/askpassphrasedialog.ui" datatype="x-trolltech-designer-ui" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="AskPassphraseDialog"> - <trans-unit id="_msg30" approved="yes"> + <trans-unit id="_msg29" approved="yes"> <source xml:space="preserve">Passphrase Dialog</source> <target xml:space="preserve">Passphrase Dialog</target> <context-group purpose="location"><context context-type="linenumber">26</context></context-group> </trans-unit> - <trans-unit id="_msg31" approved="yes"> + <trans-unit id="_msg30" approved="yes"> <source xml:space="preserve">Enter passphrase</source> <target xml:space="preserve">Enter passphrase</target> <context-group purpose="location"><context context-type="linenumber">56</context></context-group> </trans-unit> - <trans-unit id="_msg32" approved="yes"> + <trans-unit id="_msg31" approved="yes"> <source xml:space="preserve">New passphrase</source> <target xml:space="preserve">New passphrase</target> <context-group purpose="location"><context context-type="linenumber">70</context></context-group> </trans-unit> - <trans-unit id="_msg33" approved="yes"> + <trans-unit id="_msg32" approved="yes"> <source xml:space="preserve">Repeat new passphrase</source> <target xml:space="preserve">Repeat new passphrase</target> <context-group purpose="location"><context context-type="linenumber">84</context></context-group> </trans-unit> - <trans-unit id="_msg34"> + <trans-unit id="_msg33"> <source xml:space="preserve">Show passphrase</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">98</context></context-group> @@ -191,83 +187,83 @@ Signing is only possible with addresses of the type 'legacy'.</source> </body></file> <file original="../askpassphrasedialog.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="AskPassphraseDialog"> - <trans-unit id="_msg35"> + <trans-unit id="_msg34"> <source xml:space="preserve">Encrypt wallet</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">51</context></context-group> </trans-unit> - <trans-unit id="_msg36"> + <trans-unit id="_msg35"> <source xml:space="preserve">This operation needs your wallet passphrase to unlock the wallet.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">54</context></context-group> </trans-unit> - <trans-unit id="_msg37"> + <trans-unit id="_msg36"> <source xml:space="preserve">Unlock wallet</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">59</context></context-group> </trans-unit> - <trans-unit id="_msg38"> + <trans-unit id="_msg37"> <source xml:space="preserve">Change passphrase</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">62</context></context-group> </trans-unit> - <trans-unit id="_msg39"> + <trans-unit id="_msg38"> <source xml:space="preserve">Confirm wallet encryption</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">110</context></context-group> </trans-unit> - <trans-unit id="_msg40"> + <trans-unit id="_msg39"> <source xml:space="preserve">Warning: If you encrypt your wallet and lose your passphrase, you will <b>LOSE ALL OF YOUR BITCOINS</b>!</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">111</context></context-group> </trans-unit> - <trans-unit id="_msg41"> + <trans-unit id="_msg40"> <source xml:space="preserve">Are you sure you wish to encrypt your wallet?</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">111</context></context-group> </trans-unit> - <trans-unit id="_msg42"> + <trans-unit id="_msg41"> <source xml:space="preserve">Wallet encrypted</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">129</context></context-group> <context-group purpose="location"><context context-type="linenumber">173</context></context-group> </trans-unit> - <trans-unit id="_msg43"> + <trans-unit id="_msg42"> <source xml:space="preserve">Enter the new passphrase for the wallet.<br/>Please use a passphrase of <b>ten or more random characters</b>, or <b>eight or more words</b>.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">48</context></context-group> </trans-unit> - <trans-unit id="_msg44"> + <trans-unit id="_msg43"> <source xml:space="preserve">Enter the old passphrase and new passphrase for the wallet.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">63</context></context-group> </trans-unit> - <trans-unit id="_msg45"> + <trans-unit id="_msg44"> <source xml:space="preserve">Remember that encrypting your wallet cannot fully protect your bitcoins from being stolen by malware infecting your computer.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">118</context></context-group> </trans-unit> - <trans-unit id="_msg46"> + <trans-unit id="_msg45"> <source xml:space="preserve">Wallet to be encrypted</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">122</context></context-group> </trans-unit> - <trans-unit id="_msg47"> + <trans-unit id="_msg46"> <source xml:space="preserve">Your wallet is about to be encrypted. </source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">124</context></context-group> </trans-unit> - <trans-unit id="_msg48"> + <trans-unit id="_msg47"> <source xml:space="preserve">Your wallet is now encrypted. </source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">131</context></context-group> </trans-unit> - <trans-unit id="_msg49"> + <trans-unit id="_msg48"> <source xml:space="preserve">IMPORTANT: Any previous backups you have made of your wallet file should be replaced with the newly generated, encrypted wallet file. For security reasons, previous backups of the unencrypted wallet file will become useless as soon as you start using the new, encrypted wallet.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">133</context></context-group> </trans-unit> - <trans-unit id="_msg50"> + <trans-unit id="_msg49"> <source xml:space="preserve">Wallet encryption failed</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">139</context></context-group> @@ -275,35 +271,35 @@ Signing is only possible with addresses of the type 'legacy'.</source> <context-group purpose="location"><context context-type="linenumber">179</context></context-group> <context-group purpose="location"><context context-type="linenumber">185</context></context-group> </trans-unit> - <trans-unit id="_msg51"> + <trans-unit id="_msg50"> <source xml:space="preserve">Wallet encryption failed due to an internal error. Your wallet was not encrypted.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">140</context></context-group> </trans-unit> - <trans-unit id="_msg52"> + <trans-unit id="_msg51"> <source xml:space="preserve">The supplied passphrases do not match.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">148</context></context-group> <context-group purpose="location"><context context-type="linenumber">186</context></context-group> </trans-unit> - <trans-unit id="_msg53"> + <trans-unit id="_msg52"> <source xml:space="preserve">Wallet unlock failed</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">159</context></context-group> <context-group purpose="location"><context context-type="linenumber">165</context></context-group> </trans-unit> - <trans-unit id="_msg54"> + <trans-unit id="_msg53"> <source xml:space="preserve">The passphrase entered for the wallet decryption was incorrect.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">160</context></context-group> <context-group purpose="location"><context context-type="linenumber">180</context></context-group> </trans-unit> - <trans-unit id="_msg55"> + <trans-unit id="_msg54"> <source xml:space="preserve">Wallet passphrase was successfully changed.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">174</context></context-group> </trans-unit> - <trans-unit id="_msg56"> + <trans-unit id="_msg55"> <source xml:space="preserve">Warning: The Caps Lock key is on!</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">220</context></context-group> @@ -313,12 +309,12 @@ Signing is only possible with addresses of the type 'legacy'.</source> </body></file> <file original="../bantablemodel.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="BanTableModel"> - <trans-unit id="_msg57"> + <trans-unit id="_msg56"> <source xml:space="preserve">IP/Netmask</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">85</context></context-group> </trans-unit> - <trans-unit id="_msg58"> + <trans-unit id="_msg57"> <source xml:space="preserve">Banned Until</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">85</context></context-group> @@ -327,683 +323,683 @@ Signing is only possible with addresses of the type 'legacy'.</source> </body></file> <file original="../bitcoin.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="BitcoinApplication"> - <trans-unit id="_msg59"> + <trans-unit id="_msg58"> <source xml:space="preserve">Runaway exception</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">420</context></context-group> + <context-group purpose="location"><context context-type="linenumber">421</context></context-group> </trans-unit> - <trans-unit id="_msg60"> + <trans-unit id="_msg59"> <source xml:space="preserve">A fatal error occurred. %1 can no longer continue safely and will quit.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">421</context></context-group> + <context-group purpose="location"><context context-type="linenumber">422</context></context-group> </trans-unit> - <trans-unit id="_msg61"> + <trans-unit id="_msg60"> <source xml:space="preserve">Internal error</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">430</context></context-group> + <context-group purpose="location"><context context-type="linenumber">431</context></context-group> </trans-unit> - <trans-unit id="_msg62"> + <trans-unit id="_msg61"> <source xml:space="preserve">An internal error occurred. %1 will attempt to continue safely. This is an unexpected bug which can be reported as described below.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">431</context></context-group> + <context-group purpose="location"><context context-type="linenumber">432</context></context-group> </trans-unit> </group> <group restype="x-trolltech-linguist-context" resname="QObject"> - <trans-unit id="_msg63"> + <trans-unit id="_msg62"> <source xml:space="preserve">Error: Specified data directory "%1" does not exist.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">542</context></context-group> + <context-group purpose="location"><context context-type="linenumber">544</context></context-group> </trans-unit> - <trans-unit id="_msg64"> + <trans-unit id="_msg63"> <source xml:space="preserve">Error: Cannot parse configuration file: %1.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">548</context></context-group> + <context-group purpose="location"><context context-type="linenumber">550</context></context-group> </trans-unit> - <trans-unit id="_msg65"> + <trans-unit id="_msg64"> <source xml:space="preserve">Error: %1</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">563</context></context-group> + <context-group purpose="location"><context context-type="linenumber">565</context></context-group> </trans-unit> - <trans-unit id="_msg66"> + <trans-unit id="_msg65"> <source xml:space="preserve">Error initializing settings: %1</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">572</context></context-group> + <context-group purpose="location"><context context-type="linenumber">574</context></context-group> </trans-unit> - <trans-unit id="_msg67"> + <trans-unit id="_msg66"> <source xml:space="preserve">%1 didn't yet exit safely…</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">635</context></context-group> + <context-group purpose="location"><context context-type="linenumber">637</context></context-group> </trans-unit> </group> </body></file> <file original="../bitcoingui.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="BitcoinGUI"> - <trans-unit id="_msg68" approved="yes"> + <trans-unit id="_msg67" approved="yes"> <source xml:space="preserve">&Overview</source> <target xml:space="preserve">&Overview</target> <context-group purpose="location"><context context-type="linenumber">245</context></context-group> </trans-unit> - <trans-unit id="_msg69" approved="yes"> + <trans-unit id="_msg68" approved="yes"> <source xml:space="preserve">Show general overview of wallet</source> <target xml:space="preserve">Show general overview of wallet</target> <context-group purpose="location"><context context-type="linenumber">246</context></context-group> </trans-unit> - <trans-unit id="_msg70" approved="yes"> + <trans-unit id="_msg69" approved="yes"> <source xml:space="preserve">&Transactions</source> <target xml:space="preserve">&Transactions</target> <context-group purpose="location"><context context-type="linenumber">274</context></context-group> </trans-unit> - <trans-unit id="_msg71" approved="yes"> + <trans-unit id="_msg70" approved="yes"> <source xml:space="preserve">Browse transaction history</source> <target xml:space="preserve">Browse transaction history</target> <context-group purpose="location"><context context-type="linenumber">275</context></context-group> </trans-unit> - <trans-unit id="_msg72" approved="yes"> + <trans-unit id="_msg71" approved="yes"> <source xml:space="preserve">E&xit</source> <target xml:space="preserve">E&xit</target> <context-group purpose="location"><context context-type="linenumber">298</context></context-group> </trans-unit> - <trans-unit id="_msg73" approved="yes"> + <trans-unit id="_msg72" approved="yes"> <source xml:space="preserve">Quit application</source> <target xml:space="preserve">Quit application</target> <context-group purpose="location"><context context-type="linenumber">299</context></context-group> </trans-unit> - <trans-unit id="_msg74"> + <trans-unit id="_msg73"> <source xml:space="preserve">&About %1</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">302</context></context-group> </trans-unit> - <trans-unit id="_msg75"> + <trans-unit id="_msg74"> <source xml:space="preserve">Show information about %1</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">303</context></context-group> </trans-unit> - <trans-unit id="_msg76" approved="yes"> + <trans-unit id="_msg75" approved="yes"> <source xml:space="preserve">About &Qt</source> <target xml:space="preserve">About &Qt</target> <context-group purpose="location"><context context-type="linenumber">306</context></context-group> </trans-unit> - <trans-unit id="_msg77" approved="yes"> + <trans-unit id="_msg76" approved="yes"> <source xml:space="preserve">Show information about Qt</source> <target xml:space="preserve">Show information about Qt</target> <context-group purpose="location"><context context-type="linenumber">307</context></context-group> </trans-unit> - <trans-unit id="_msg78"> + <trans-unit id="_msg77"> <source xml:space="preserve">Modify configuration options for %1</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">310</context></context-group> </trans-unit> - <trans-unit id="_msg79"> + <trans-unit id="_msg78"> <source xml:space="preserve">Create a new wallet</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">356</context></context-group> </trans-unit> - <trans-unit id="_msg80"> + <trans-unit id="_msg79"> <source xml:space="preserve">Wallet:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">565</context></context-group> </trans-unit> - <trans-unit id="_msg81"> + <trans-unit id="_msg80"> <source xml:space="preserve">Network activity disabled.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">923</context></context-group> + <context-group purpose="location"><context context-type="linenumber">938</context></context-group> <note annotates="source" from="developer">A substring of the tooltip.</note> </trans-unit> - <trans-unit id="_msg82"> + <trans-unit id="_msg81"> <source xml:space="preserve">Proxy is <b>enabled</b>: %1</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1349</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1362</context></context-group> </trans-unit> - <trans-unit id="_msg83" approved="yes"> + <trans-unit id="_msg82" approved="yes"> <source xml:space="preserve">Send coins to a Bitcoin address</source> <target xml:space="preserve">Send coins to a Bitcoin address</target> <context-group purpose="location"><context context-type="linenumber">253</context></context-group> </trans-unit> - <trans-unit id="_msg84" approved="yes"> + <trans-unit id="_msg83" approved="yes"> <source xml:space="preserve">Backup wallet to another location</source> <target xml:space="preserve">Backup wallet to another location</target> <context-group purpose="location"><context context-type="linenumber">320</context></context-group> </trans-unit> - <trans-unit id="_msg85" approved="yes"> + <trans-unit id="_msg84" approved="yes"> <source xml:space="preserve">Change the passphrase used for wallet encryption</source> <target xml:space="preserve">Change the passphrase used for wallet encryption</target> <context-group purpose="location"><context context-type="linenumber">322</context></context-group> </trans-unit> - <trans-unit id="_msg86" approved="yes"> + <trans-unit id="_msg85" approved="yes"> <source xml:space="preserve">&Send</source> <target xml:space="preserve">&Send</target> <context-group purpose="location"><context context-type="linenumber">252</context></context-group> </trans-unit> - <trans-unit id="_msg87" approved="yes"> + <trans-unit id="_msg86" approved="yes"> <source xml:space="preserve">&Receive</source> <target xml:space="preserve">&Receive</target> <context-group purpose="location"><context context-type="linenumber">263</context></context-group> </trans-unit> - <trans-unit id="_msg88"> + <trans-unit id="_msg87"> <source xml:space="preserve">&Options…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">309</context></context-group> </trans-unit> - <trans-unit id="_msg89" approved="yes"> + <trans-unit id="_msg88" approved="yes"> <source xml:space="preserve">&Show / Hide</source> <target xml:space="preserve">&Show / Hide</target> <context-group purpose="location"><context context-type="linenumber">313</context></context-group> </trans-unit> - <trans-unit id="_msg90" approved="yes"> + <trans-unit id="_msg89" approved="yes"> <source xml:space="preserve">Show or hide the main Window</source> <target xml:space="preserve">Show or hide the main Window</target> <context-group purpose="location"><context context-type="linenumber">314</context></context-group> </trans-unit> - <trans-unit id="_msg91"> + <trans-unit id="_msg90"> <source xml:space="preserve">&Encrypt Wallet…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">316</context></context-group> </trans-unit> - <trans-unit id="_msg92" approved="yes"> + <trans-unit id="_msg91" approved="yes"> <source xml:space="preserve">Encrypt the private keys that belong to your wallet</source> <target xml:space="preserve">Encrypt the private keys that belong to your wallet</target> <context-group purpose="location"><context context-type="linenumber">317</context></context-group> </trans-unit> - <trans-unit id="_msg93"> + <trans-unit id="_msg92"> <source xml:space="preserve">&Backup Wallet…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">319</context></context-group> </trans-unit> - <trans-unit id="_msg94"> + <trans-unit id="_msg93"> <source xml:space="preserve">&Change Passphrase…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">321</context></context-group> </trans-unit> - <trans-unit id="_msg95"> + <trans-unit id="_msg94"> <source xml:space="preserve">Sign &message…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">323</context></context-group> </trans-unit> - <trans-unit id="_msg96" approved="yes"> + <trans-unit id="_msg95" approved="yes"> <source xml:space="preserve">Sign messages with your Bitcoin addresses to prove you own them</source> <target xml:space="preserve">Sign messages with your Bitcoin addresses to prove you own them</target> <context-group purpose="location"><context context-type="linenumber">324</context></context-group> </trans-unit> - <trans-unit id="_msg97"> + <trans-unit id="_msg96"> <source xml:space="preserve">&Verify message…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">325</context></context-group> </trans-unit> - <trans-unit id="_msg98" approved="yes"> + <trans-unit id="_msg97" approved="yes"> <source xml:space="preserve">Verify messages to ensure they were signed with specified Bitcoin addresses</source> <target xml:space="preserve">Verify messages to ensure they were signed with specified Bitcoin addresses</target> <context-group purpose="location"><context context-type="linenumber">326</context></context-group> </trans-unit> - <trans-unit id="_msg99"> + <trans-unit id="_msg98"> <source xml:space="preserve">&Load PSBT from file…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">327</context></context-group> </trans-unit> - <trans-unit id="_msg100"> + <trans-unit id="_msg99"> <source xml:space="preserve">Load PSBT from clipboard…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">329</context></context-group> </trans-unit> - <trans-unit id="_msg101"> + <trans-unit id="_msg100"> <source xml:space="preserve">Open &URI…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">343</context></context-group> </trans-unit> - <trans-unit id="_msg102"> + <trans-unit id="_msg101"> <source xml:space="preserve">Close Wallet…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">351</context></context-group> </trans-unit> - <trans-unit id="_msg103"> + <trans-unit id="_msg102"> <source xml:space="preserve">Create Wallet…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">354</context></context-group> </trans-unit> - <trans-unit id="_msg104"> + <trans-unit id="_msg103"> <source xml:space="preserve">Close All Wallets…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">358</context></context-group> </trans-unit> - <trans-unit id="_msg105" approved="yes"> + <trans-unit id="_msg104" approved="yes"> <source xml:space="preserve">&File</source> <target xml:space="preserve">&File</target> <context-group purpose="location"><context context-type="linenumber">455</context></context-group> </trans-unit> - <trans-unit id="_msg106" approved="yes"> + <trans-unit id="_msg105" approved="yes"> <source xml:space="preserve">&Settings</source> <target xml:space="preserve">&Settings</target> <context-group purpose="location"><context context-type="linenumber">473</context></context-group> </trans-unit> - <trans-unit id="_msg107" approved="yes"> + <trans-unit id="_msg106" approved="yes"> <source xml:space="preserve">&Help</source> <target xml:space="preserve">&Help</target> <context-group purpose="location"><context context-type="linenumber">534</context></context-group> </trans-unit> - <trans-unit id="_msg108" approved="yes"> + <trans-unit id="_msg107" approved="yes"> <source xml:space="preserve">Tabs toolbar</source> <target xml:space="preserve">Tabs toolbar</target> <context-group purpose="location"><context context-type="linenumber">545</context></context-group> </trans-unit> - <trans-unit id="_msg109"> + <trans-unit id="_msg108"> <source xml:space="preserve">Syncing Headers (%1%)…</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">967</context></context-group> + <context-group purpose="location"><context context-type="linenumber">982</context></context-group> </trans-unit> - <trans-unit id="_msg110"> + <trans-unit id="_msg109"> <source xml:space="preserve">Synchronizing with network…</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1013</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1028</context></context-group> </trans-unit> - <trans-unit id="_msg111"> + <trans-unit id="_msg110"> <source xml:space="preserve">Indexing blocks on disk…</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1018</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1033</context></context-group> </trans-unit> - <trans-unit id="_msg112"> + <trans-unit id="_msg111"> <source xml:space="preserve">Processing blocks on disk…</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1020</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1035</context></context-group> </trans-unit> - <trans-unit id="_msg113"> + <trans-unit id="_msg112"> <source xml:space="preserve">Reindexing blocks on disk…</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1024</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1039</context></context-group> </trans-unit> - <trans-unit id="_msg114"> + <trans-unit id="_msg113"> <source xml:space="preserve">Connecting to peers…</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1030</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1045</context></context-group> </trans-unit> - <trans-unit id="_msg115"> + <trans-unit id="_msg114"> <source xml:space="preserve">Request payments (generates QR codes and bitcoin: URIs)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">264</context></context-group> </trans-unit> - <trans-unit id="_msg116"> + <trans-unit id="_msg115"> <source xml:space="preserve">Show the list of used sending addresses and labels</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">339</context></context-group> </trans-unit> - <trans-unit id="_msg117"> + <trans-unit id="_msg116"> <source xml:space="preserve">Show the list of used receiving addresses and labels</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">341</context></context-group> </trans-unit> - <trans-unit id="_msg118"> + <trans-unit id="_msg117"> <source xml:space="preserve">&Command-line options</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">361</context></context-group> </trans-unit> <group restype="x-gettext-plurals"> - <context-group purpose="location"><context context-type="linenumber">1039</context></context-group> - <trans-unit id="_msg119[0]" approved="yes"> + <context-group purpose="location"><context context-type="linenumber">1054</context></context-group> + <trans-unit id="_msg118[0]" approved="yes"> <source xml:space="preserve">Processed %n block(s) of transaction history.</source> <target xml:space="preserve">Processed %n block of transaction history.</target> </trans-unit> - <trans-unit id="_msg119[1]" approved="yes"> + <trans-unit id="_msg118[1]" approved="yes"> <source xml:space="preserve">Processed %n block(s) of transaction history.</source> <target xml:space="preserve">Processed %n blocks of transaction history.</target> </trans-unit> </group> - <trans-unit id="_msg120" approved="yes"> + <trans-unit id="_msg119" approved="yes"> <source xml:space="preserve">%1 behind</source> <target xml:space="preserve">%1 behind</target> - <context-group purpose="location"><context context-type="linenumber">1062</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1077</context></context-group> </trans-unit> - <trans-unit id="_msg121"> + <trans-unit id="_msg120"> <source xml:space="preserve">Catching up…</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1067</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1082</context></context-group> </trans-unit> - <trans-unit id="_msg122" approved="yes"> + <trans-unit id="_msg121" approved="yes"> <source xml:space="preserve">Last received block was generated %1 ago.</source> <target xml:space="preserve">Last received block was generated %1 ago.</target> - <context-group purpose="location"><context context-type="linenumber">1086</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1101</context></context-group> </trans-unit> - <trans-unit id="_msg123" approved="yes"> + <trans-unit id="_msg122" approved="yes"> <source xml:space="preserve">Transactions after this will not yet be visible.</source> <target xml:space="preserve">Transactions after this will not yet be visible.</target> - <context-group purpose="location"><context context-type="linenumber">1088</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1103</context></context-group> </trans-unit> - <trans-unit id="_msg124" approved="yes"> + <trans-unit id="_msg123" approved="yes"> <source xml:space="preserve">Error</source> <target xml:space="preserve">Error</target> - <context-group purpose="location"><context context-type="linenumber">1113</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1128</context></context-group> </trans-unit> - <trans-unit id="_msg125" approved="yes"> + <trans-unit id="_msg124" approved="yes"> <source xml:space="preserve">Warning</source> <target xml:space="preserve">Warning</target> - <context-group purpose="location"><context context-type="linenumber">1117</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1132</context></context-group> </trans-unit> - <trans-unit id="_msg126" approved="yes"> + <trans-unit id="_msg125" approved="yes"> <source xml:space="preserve">Information</source> <target xml:space="preserve">Information</target> - <context-group purpose="location"><context context-type="linenumber">1121</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1136</context></context-group> </trans-unit> - <trans-unit id="_msg127" approved="yes"> + <trans-unit id="_msg126" approved="yes"> <source xml:space="preserve">Up to date</source> <target xml:space="preserve">Up to date</target> - <context-group purpose="location"><context context-type="linenumber">1043</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1058</context></context-group> </trans-unit> - <trans-unit id="_msg128"> + <trans-unit id="_msg127"> <source xml:space="preserve">Load Partially Signed Bitcoin Transaction</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">328</context></context-group> </trans-unit> - <trans-unit id="_msg129"> + <trans-unit id="_msg128"> <source xml:space="preserve">Load Partially Signed Bitcoin Transaction from clipboard</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">330</context></context-group> </trans-unit> - <trans-unit id="_msg130"> + <trans-unit id="_msg129"> <source xml:space="preserve">Node window</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">332</context></context-group> </trans-unit> - <trans-unit id="_msg131"> + <trans-unit id="_msg130"> <source xml:space="preserve">Open node debugging and diagnostic console</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">333</context></context-group> </trans-unit> - <trans-unit id="_msg132"> + <trans-unit id="_msg131"> <source xml:space="preserve">&Sending addresses</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">338</context></context-group> </trans-unit> - <trans-unit id="_msg133"> + <trans-unit id="_msg132"> <source xml:space="preserve">&Receiving addresses</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">340</context></context-group> </trans-unit> - <trans-unit id="_msg134"> + <trans-unit id="_msg133"> <source xml:space="preserve">Open a bitcoin: URI</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">344</context></context-group> </trans-unit> - <trans-unit id="_msg135"> + <trans-unit id="_msg134"> <source xml:space="preserve">Open Wallet</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">346</context></context-group> </trans-unit> - <trans-unit id="_msg136"> + <trans-unit id="_msg135"> <source xml:space="preserve">Open a wallet</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">348</context></context-group> </trans-unit> - <trans-unit id="_msg137"> + <trans-unit id="_msg136"> <source xml:space="preserve">Close wallet</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">352</context></context-group> </trans-unit> - <trans-unit id="_msg138"> + <trans-unit id="_msg137"> <source xml:space="preserve">Close all wallets</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">359</context></context-group> </trans-unit> - <trans-unit id="_msg139"> + <trans-unit id="_msg138"> <source xml:space="preserve">Show the %1 help message to get a list with possible Bitcoin command-line options</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">363</context></context-group> </trans-unit> - <trans-unit id="_msg140"> + <trans-unit id="_msg139"> <source xml:space="preserve">&Mask values</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">365</context></context-group> </trans-unit> - <trans-unit id="_msg141"> + <trans-unit id="_msg140"> <source xml:space="preserve">Mask the values in the Overview tab</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">367</context></context-group> </trans-unit> - <trans-unit id="_msg142"> + <trans-unit id="_msg141"> <source xml:space="preserve">default wallet</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">399</context></context-group> </trans-unit> - <trans-unit id="_msg143"> + <trans-unit id="_msg142"> <source xml:space="preserve">No wallets available</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">420</context></context-group> </trans-unit> - <trans-unit id="_msg144"> + <trans-unit id="_msg143"> <source xml:space="preserve">&Window</source> <target xml:space="preserve" state="needs-review-translation">&Window</target> <context-group purpose="location"><context context-type="linenumber">484</context></context-group> </trans-unit> - <trans-unit id="_msg145"> + <trans-unit id="_msg144"> <source xml:space="preserve">Minimize</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">486</context></context-group> </trans-unit> - <trans-unit id="_msg146"> + <trans-unit id="_msg145"> <source xml:space="preserve">Zoom</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">496</context></context-group> </trans-unit> - <trans-unit id="_msg147"> + <trans-unit id="_msg146"> <source xml:space="preserve">Main Window</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">514</context></context-group> </trans-unit> - <trans-unit id="_msg148"> + <trans-unit id="_msg147"> <source xml:space="preserve">%1 client</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">762</context></context-group> + <context-group purpose="location"><context context-type="linenumber">777</context></context-group> </trans-unit> <group restype="x-gettext-plurals"> - <context-group purpose="location"><context context-type="linenumber">920</context></context-group> + <context-group purpose="location"><context context-type="linenumber">935</context></context-group> <note annotates="source" from="developer">A substring of the tooltip.</note> - <trans-unit id="_msg149[0]"> + <trans-unit id="_msg148[0]"> <source xml:space="preserve">%n active connection(s) to Bitcoin network.</source> <target xml:space="preserve"></target> </trans-unit> - <trans-unit id="_msg149[1]"> + <trans-unit id="_msg148[1]"> <source xml:space="preserve">%n active connection(s) to Bitcoin network.</source> <target xml:space="preserve"></target> </trans-unit> </group> - <trans-unit id="_msg150"> + <trans-unit id="_msg149"> <source xml:space="preserve">Click for more actions.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">930</context></context-group> + <context-group purpose="location"><context context-type="linenumber">945</context></context-group> <note annotates="source" from="developer">A substring of the tooltip. "More actions" are available via the context menu.</note> </trans-unit> - <trans-unit id="_msg151"> + <trans-unit id="_msg150"> <source xml:space="preserve">Show Peers tab</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">947</context></context-group> + <context-group purpose="location"><context context-type="linenumber">962</context></context-group> <note annotates="source" from="developer">A context menu item. The "Peers tab" is an element of the "Node window".</note> </trans-unit> - <trans-unit id="_msg152"> + <trans-unit id="_msg151"> <source xml:space="preserve">Disable network activity</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">955</context></context-group> + <context-group purpose="location"><context context-type="linenumber">970</context></context-group> <note annotates="source" from="developer">A context menu item.</note> </trans-unit> - <trans-unit id="_msg153"> + <trans-unit id="_msg152"> <source xml:space="preserve">Enable network activity</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">957</context></context-group> + <context-group purpose="location"><context context-type="linenumber">972</context></context-group> <note annotates="source" from="developer">A context menu item. The network activity was disabled previously.</note> </trans-unit> - <trans-unit id="_msg154"> + <trans-unit id="_msg153"> <source xml:space="preserve">Error: %1</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1114</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1129</context></context-group> </trans-unit> - <trans-unit id="_msg155"> + <trans-unit id="_msg154"> <source xml:space="preserve">Warning: %1</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1118</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1133</context></context-group> </trans-unit> - <trans-unit id="_msg156"> + <trans-unit id="_msg155"> <source xml:space="preserve">Date: %1 </source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1228</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1241</context></context-group> </trans-unit> - <trans-unit id="_msg157"> + <trans-unit id="_msg156"> <source xml:space="preserve">Amount: %1 </source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1229</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1242</context></context-group> </trans-unit> - <trans-unit id="_msg158"> + <trans-unit id="_msg157"> <source xml:space="preserve">Wallet: %1 </source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1231</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1244</context></context-group> </trans-unit> - <trans-unit id="_msg159"> + <trans-unit id="_msg158"> <source xml:space="preserve">Type: %1 </source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1233</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1246</context></context-group> </trans-unit> - <trans-unit id="_msg160"> + <trans-unit id="_msg159"> <source xml:space="preserve">Label: %1 </source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1235</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1248</context></context-group> </trans-unit> - <trans-unit id="_msg161"> + <trans-unit id="_msg160"> <source xml:space="preserve">Address: %1 </source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1237</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1250</context></context-group> </trans-unit> - <trans-unit id="_msg162" approved="yes"> + <trans-unit id="_msg161" approved="yes"> <source xml:space="preserve">Sent transaction</source> <target xml:space="preserve">Sent transaction</target> - <context-group purpose="location"><context context-type="linenumber">1238</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1251</context></context-group> </trans-unit> - <trans-unit id="_msg163" approved="yes"> + <trans-unit id="_msg162" approved="yes"> <source xml:space="preserve">Incoming transaction</source> <target xml:space="preserve">Incoming transaction</target> - <context-group purpose="location"><context context-type="linenumber">1238</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1251</context></context-group> </trans-unit> - <trans-unit id="_msg164"> + <trans-unit id="_msg163"> <source xml:space="preserve">HD key generation is <b>enabled</b></source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1290</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1303</context></context-group> </trans-unit> - <trans-unit id="_msg165"> + <trans-unit id="_msg164"> <source xml:space="preserve">HD key generation is <b>disabled</b></source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1290</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1303</context></context-group> </trans-unit> - <trans-unit id="_msg166"> + <trans-unit id="_msg165"> <source xml:space="preserve">Private key <b>disabled</b></source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1290</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1303</context></context-group> </trans-unit> - <trans-unit id="_msg167" approved="yes"> + <trans-unit id="_msg166" approved="yes"> <source xml:space="preserve">Wallet is <b>encrypted</b> and currently <b>unlocked</b></source> <target xml:space="preserve">Wallet is <b>encrypted</b> and currently <b>unlocked</b></target> - <context-group purpose="location"><context context-type="linenumber">1309</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1322</context></context-group> </trans-unit> - <trans-unit id="_msg168" approved="yes"> + <trans-unit id="_msg167" approved="yes"> <source xml:space="preserve">Wallet is <b>encrypted</b> and currently <b>locked</b></source> <target xml:space="preserve">Wallet is <b>encrypted</b> and currently <b>locked</b></target> - <context-group purpose="location"><context context-type="linenumber">1317</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1330</context></context-group> </trans-unit> - <trans-unit id="_msg169"> + <trans-unit id="_msg168"> <source xml:space="preserve">Original message:</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1437</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1450</context></context-group> </trans-unit> </group> <group restype="x-trolltech-linguist-context" resname="UnitDisplayStatusBarControl"> - <trans-unit id="_msg170"> + <trans-unit id="_msg169"> <source xml:space="preserve">Unit to show amounts in. Click to select another unit.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1478</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1491</context></context-group> </trans-unit> </group> </body></file> <file original="../forms/coincontroldialog.ui" datatype="x-trolltech-designer-ui" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="CoinControlDialog"> - <trans-unit id="_msg171"> + <trans-unit id="_msg170"> <source xml:space="preserve">Coin Selection</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">14</context></context-group> </trans-unit> - <trans-unit id="_msg172"> + <trans-unit id="_msg171"> <source xml:space="preserve">Quantity:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">48</context></context-group> </trans-unit> - <trans-unit id="_msg173"> + <trans-unit id="_msg172"> <source xml:space="preserve">Bytes:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">77</context></context-group> </trans-unit> - <trans-unit id="_msg174"> + <trans-unit id="_msg173"> <source xml:space="preserve">Amount:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">122</context></context-group> </trans-unit> - <trans-unit id="_msg175"> + <trans-unit id="_msg174"> <source xml:space="preserve">Fee:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">202</context></context-group> </trans-unit> - <trans-unit id="_msg176"> + <trans-unit id="_msg175"> <source xml:space="preserve">Dust:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">154</context></context-group> </trans-unit> - <trans-unit id="_msg177"> + <trans-unit id="_msg176"> <source xml:space="preserve">After Fee:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">247</context></context-group> </trans-unit> - <trans-unit id="_msg178"> + <trans-unit id="_msg177"> <source xml:space="preserve">Change:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">279</context></context-group> </trans-unit> - <trans-unit id="_msg179"> + <trans-unit id="_msg178"> <source xml:space="preserve">(un)select all</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">335</context></context-group> </trans-unit> - <trans-unit id="_msg180"> + <trans-unit id="_msg179"> <source xml:space="preserve">Tree mode</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">351</context></context-group> </trans-unit> - <trans-unit id="_msg181"> + <trans-unit id="_msg180"> <source xml:space="preserve">List mode</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">364</context></context-group> </trans-unit> - <trans-unit id="_msg182"> + <trans-unit id="_msg181"> <source xml:space="preserve">Amount</source> <target xml:space="preserve" state="needs-review-translation">Amount</target> <context-group purpose="location"><context context-type="linenumber">420</context></context-group> </trans-unit> - <trans-unit id="_msg183"> + <trans-unit id="_msg182"> <source xml:space="preserve">Received with label</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">425</context></context-group> </trans-unit> - <trans-unit id="_msg184"> + <trans-unit id="_msg183"> <source xml:space="preserve">Received with address</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">430</context></context-group> </trans-unit> - <trans-unit id="_msg185"> + <trans-unit id="_msg184"> <source xml:space="preserve">Date</source> <target xml:space="preserve" state="needs-review-translation">Date</target> <context-group purpose="location"><context context-type="linenumber">435</context></context-group> </trans-unit> - <trans-unit id="_msg186"> + <trans-unit id="_msg185"> <source xml:space="preserve">Confirmations</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">440</context></context-group> </trans-unit> - <trans-unit id="_msg187"> + <trans-unit id="_msg186"> <source xml:space="preserve">Confirmed</source> <target xml:space="preserve" state="needs-review-translation">Confirmed</target> <context-group purpose="location"><context context-type="linenumber">443</context></context-group> @@ -1012,34 +1008,38 @@ Signing is only possible with addresses of the type 'legacy'.</source> </body></file> <file original="../coincontroldialog.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="CoinControlDialog"> + <trans-unit id="_msg187"> + <source xml:space="preserve">Copy amount</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">66</context></context-group> + </trans-unit> <trans-unit id="_msg188"> - <source xml:space="preserve">Copy address</source> + <source xml:space="preserve">&Copy address</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">55</context></context-group> </trans-unit> <trans-unit id="_msg189"> - <source xml:space="preserve">Copy label</source> + <source xml:space="preserve">Copy &label</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">56</context></context-group> </trans-unit> <trans-unit id="_msg190"> - <source xml:space="preserve">Copy amount</source> + <source xml:space="preserve">Copy &amount</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">57</context></context-group> - <context-group purpose="location"><context context-type="linenumber">66</context></context-group> </trans-unit> <trans-unit id="_msg191"> - <source xml:space="preserve">Copy transaction ID</source> + <source xml:space="preserve">Copy transaction &ID</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">58</context></context-group> </trans-unit> <trans-unit id="_msg192"> - <source xml:space="preserve">Lock unspent</source> + <source xml:space="preserve">L&ock unspent</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">60</context></context-group> </trans-unit> <trans-unit id="_msg193"> - <source xml:space="preserve">Unlock unspent</source> + <source xml:space="preserve">&Unlock unspent</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">61</context></context-group> </trans-unit> @@ -1121,170 +1121,191 @@ Signing is only possible with addresses of the type 'legacy'.</source> <trans-unit id="_msg208"> <source xml:space="preserve">Creating Wallet <b>%1</b>…</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">253</context></context-group> + <context-group purpose="location"><context context-type="linenumber">254</context></context-group> </trans-unit> <trans-unit id="_msg209"> <source xml:space="preserve">Create wallet failed</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">281</context></context-group> + <context-group purpose="location"><context context-type="linenumber">285</context></context-group> </trans-unit> <trans-unit id="_msg210"> <source xml:space="preserve">Create wallet warning</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">283</context></context-group> + <context-group purpose="location"><context context-type="linenumber">287</context></context-group> + </trans-unit> + <trans-unit id="_msg211"> + <source xml:space="preserve">Can't list signers</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">303</context></context-group> </trans-unit> </group> <group restype="x-trolltech-linguist-context" resname="OpenWalletActivity"> - <trans-unit id="_msg211"> + <trans-unit id="_msg212"> <source xml:space="preserve">Open wallet failed</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">322</context></context-group> + <context-group purpose="location"><context context-type="linenumber">335</context></context-group> </trans-unit> - <trans-unit id="_msg212"> + <trans-unit id="_msg213"> <source xml:space="preserve">Open wallet warning</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">324</context></context-group> + <context-group purpose="location"><context context-type="linenumber">337</context></context-group> </trans-unit> - <trans-unit id="_msg213"> + <trans-unit id="_msg214"> <source xml:space="preserve">default wallet</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">334</context></context-group> + <context-group purpose="location"><context context-type="linenumber">347</context></context-group> </trans-unit> - <trans-unit id="_msg214"> + <trans-unit id="_msg215"> <source xml:space="preserve">Opening Wallet <b>%1</b>…</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">336</context></context-group> + <context-group purpose="location"><context context-type="linenumber">349</context></context-group> </trans-unit> </group> <group restype="x-trolltech-linguist-context" resname="WalletController"> - <trans-unit id="_msg215"> - <source xml:space="preserve">Close wallet</source> - <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">86</context></context-group> - </trans-unit> <trans-unit id="_msg216"> - <source xml:space="preserve">Are you sure you wish to close the wallet <i>%1</i>?</source> + <source xml:space="preserve">Close wallet</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">87</context></context-group> </trans-unit> <trans-unit id="_msg217"> - <source xml:space="preserve">Closing the wallet for too long can result in having to resync the entire chain if pruning is enabled.</source> + <source xml:space="preserve">Are you sure you wish to close the wallet <i>%1</i>?</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">88</context></context-group> </trans-unit> <trans-unit id="_msg218"> - <source xml:space="preserve">Close all wallets</source> + <source xml:space="preserve">Closing the wallet for too long can result in having to resync the entire chain if pruning is enabled.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">101</context></context-group> + <context-group purpose="location"><context context-type="linenumber">89</context></context-group> </trans-unit> <trans-unit id="_msg219"> - <source xml:space="preserve">Are you sure you wish to close all wallets?</source> + <source xml:space="preserve">Close all wallets</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">102</context></context-group> </trans-unit> + <trans-unit id="_msg220"> + <source xml:space="preserve">Are you sure you wish to close all wallets?</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">103</context></context-group> + </trans-unit> </group> </body></file> <file original="../forms/createwalletdialog.ui" datatype="x-trolltech-designer-ui" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="CreateWalletDialog"> - <trans-unit id="_msg220"> + <trans-unit id="_msg221"> <source xml:space="preserve">Create Wallet</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">14</context></context-group> </trans-unit> - <trans-unit id="_msg221"> + <trans-unit id="_msg222"> <source xml:space="preserve">Wallet Name</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">25</context></context-group> </trans-unit> - <trans-unit id="_msg222"> + <trans-unit id="_msg223"> <source xml:space="preserve">Wallet</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">38</context></context-group> </trans-unit> - <trans-unit id="_msg223"> + <trans-unit id="_msg224"> <source xml:space="preserve">Encrypt the wallet. The wallet will be encrypted with a passphrase of your choice.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">47</context></context-group> </trans-unit> - <trans-unit id="_msg224"> + <trans-unit id="_msg225"> <source xml:space="preserve">Encrypt Wallet</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">50</context></context-group> </trans-unit> - <trans-unit id="_msg225"> + <trans-unit id="_msg226"> <source xml:space="preserve">Advanced Options</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">76</context></context-group> </trans-unit> - <trans-unit id="_msg226"> + <trans-unit id="_msg227"> <source xml:space="preserve">Disable private keys for this wallet. Wallets with private keys disabled will have no private keys and cannot have an HD seed or imported private keys. This is ideal for watch-only wallets.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">85</context></context-group> </trans-unit> - <trans-unit id="_msg227"> + <trans-unit id="_msg228"> <source xml:space="preserve">Disable Private Keys</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">88</context></context-group> </trans-unit> - <trans-unit id="_msg228"> + <trans-unit id="_msg229"> <source xml:space="preserve">Make a blank wallet. Blank wallets do not initially have private keys or scripts. Private keys and addresses can be imported, or an HD seed can be set, at a later time.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">95</context></context-group> </trans-unit> - <trans-unit id="_msg229"> + <trans-unit id="_msg230"> <source xml:space="preserve">Make Blank Wallet</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">98</context></context-group> </trans-unit> - <trans-unit id="_msg230"> + <trans-unit id="_msg231"> <source xml:space="preserve">Use descriptors for scriptPubKey management</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">105</context></context-group> </trans-unit> - <trans-unit id="_msg231"> + <trans-unit id="_msg232"> <source xml:space="preserve">Descriptor Wallet</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">108</context></context-group> </trans-unit> + <trans-unit id="_msg233"> + <source xml:space="preserve">Use an external signing device such as a hardware wallet. Configure the external signer script in wallet preferences first.</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">115</context></context-group> + </trans-unit> + <trans-unit id="_msg234"> + <source xml:space="preserve">External signer</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">118</context></context-group> + </trans-unit> </group> </body></file> <file original="../createwalletdialog.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="CreateWalletDialog"> - <trans-unit id="_msg232"> + <trans-unit id="_msg235"> <source xml:space="preserve">Create</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">21</context></context-group> + <context-group purpose="location"><context context-type="linenumber">22</context></context-group> </trans-unit> - <trans-unit id="_msg233"> + <trans-unit id="_msg236"> <source xml:space="preserve">Compiled without sqlite support (required for descriptor wallets)</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">63</context></context-group> + <context-group purpose="location"><context context-type="linenumber">90</context></context-group> + </trans-unit> + <trans-unit id="_msg237"> + <source xml:space="preserve">Compiled without external signing support (required for external signing)</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">104</context></context-group> + <note annotates="source" from="developer">"External signing" means using devices such as hardware wallets.</note> </trans-unit> </group> </body></file> <file original="../forms/editaddressdialog.ui" datatype="x-trolltech-designer-ui" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="EditAddressDialog"> - <trans-unit id="_msg234" approved="yes"> + <trans-unit id="_msg238" approved="yes"> <source xml:space="preserve">Edit Address</source> <target xml:space="preserve">Edit Address</target> <context-group purpose="location"><context context-type="linenumber">14</context></context-group> </trans-unit> - <trans-unit id="_msg235" approved="yes"> + <trans-unit id="_msg239" approved="yes"> <source xml:space="preserve">&Label</source> <target xml:space="preserve">&Label</target> <context-group purpose="location"><context context-type="linenumber">25</context></context-group> </trans-unit> - <trans-unit id="_msg236"> + <trans-unit id="_msg240"> <source xml:space="preserve">The label associated with this address list entry</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">35</context></context-group> </trans-unit> - <trans-unit id="_msg237"> + <trans-unit id="_msg241"> <source xml:space="preserve">The address associated with this address list entry. This can only be modified for sending addresses.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">52</context></context-group> </trans-unit> - <trans-unit id="_msg238" approved="yes"> + <trans-unit id="_msg242" approved="yes"> <source xml:space="preserve">&Address</source> <target xml:space="preserve">&Address</target> <context-group purpose="location"><context context-type="linenumber">42</context></context-group> @@ -1293,42 +1314,42 @@ Signing is only possible with addresses of the type 'legacy'.</source> </body></file> <file original="../editaddressdialog.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="EditAddressDialog"> - <trans-unit id="_msg239"> + <trans-unit id="_msg243"> <source xml:space="preserve">New sending address</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">29</context></context-group> </trans-unit> - <trans-unit id="_msg240"> + <trans-unit id="_msg244"> <source xml:space="preserve">Edit receiving address</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">32</context></context-group> </trans-unit> - <trans-unit id="_msg241"> + <trans-unit id="_msg245"> <source xml:space="preserve">Edit sending address</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">36</context></context-group> </trans-unit> - <trans-unit id="_msg242"> + <trans-unit id="_msg246"> <source xml:space="preserve">The entered address "%1" is not a valid Bitcoin address.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">113</context></context-group> </trans-unit> - <trans-unit id="_msg243"> + <trans-unit id="_msg247"> <source xml:space="preserve">Address "%1" already exists as a receiving address with label "%2" and so cannot be added as a sending address.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">146</context></context-group> </trans-unit> - <trans-unit id="_msg244"> + <trans-unit id="_msg248"> <source xml:space="preserve">The entered address "%1" is already in the address book with label "%2".</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">151</context></context-group> </trans-unit> - <trans-unit id="_msg245"> + <trans-unit id="_msg249"> <source xml:space="preserve">Could not unlock wallet.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">123</context></context-group> </trans-unit> - <trans-unit id="_msg246"> + <trans-unit id="_msg250"> <source xml:space="preserve">New key generation failed.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">128</context></context-group> @@ -1337,59 +1358,59 @@ Signing is only possible with addresses of the type 'legacy'.</source> </body></file> <file original="../intro.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="FreespaceChecker"> - <trans-unit id="_msg247" approved="yes"> + <trans-unit id="_msg251" approved="yes"> <source xml:space="preserve">A new data directory will be created.</source> <target xml:space="preserve">A new data directory will be created.</target> <context-group purpose="location"><context context-type="linenumber">73</context></context-group> </trans-unit> - <trans-unit id="_msg248" approved="yes"> + <trans-unit id="_msg252" approved="yes"> <source xml:space="preserve">name</source> <target xml:space="preserve">name</target> <context-group purpose="location"><context context-type="linenumber">95</context></context-group> </trans-unit> - <trans-unit id="_msg249" approved="yes"> + <trans-unit id="_msg253" approved="yes"> <source xml:space="preserve">Directory already exists. Add %1 if you intend to create a new directory here.</source> <target xml:space="preserve">Directory already exists. Add %1 if you intend to create a new directory here.</target> <context-group purpose="location"><context context-type="linenumber">97</context></context-group> </trans-unit> - <trans-unit id="_msg250" approved="yes"> + <trans-unit id="_msg254" approved="yes"> <source xml:space="preserve">Path already exists, and is not a directory.</source> <target xml:space="preserve">Path already exists, and is not a directory.</target> <context-group purpose="location"><context context-type="linenumber">100</context></context-group> </trans-unit> - <trans-unit id="_msg251" approved="yes"> + <trans-unit id="_msg255" approved="yes"> <source xml:space="preserve">Cannot create data directory here.</source> <target xml:space="preserve">Cannot create data directory here.</target> <context-group purpose="location"><context context-type="linenumber">107</context></context-group> </trans-unit> </group> <group restype="x-trolltech-linguist-context" resname="Intro"> - <trans-unit id="_msg252"> + <trans-unit id="_msg256"> <source xml:space="preserve">Bitcoin</source> <target xml:space="preserve" state="needs-review-translation">Bitcoin</target> <context-group purpose="location"><context context-type="linenumber">139</context></context-group> </trans-unit> - <trans-unit id="_msg253"> + <trans-unit id="_msg257"> <source xml:space="preserve">%1 GB of free space available</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">301</context></context-group> </trans-unit> - <trans-unit id="_msg254"> + <trans-unit id="_msg258"> <source xml:space="preserve">(of %1 GB needed)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">303</context></context-group> </trans-unit> - <trans-unit id="_msg255"> + <trans-unit id="_msg259"> <source xml:space="preserve">(%1 GB needed for full chain)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">306</context></context-group> </trans-unit> - <trans-unit id="_msg256"> + <trans-unit id="_msg260"> <source xml:space="preserve">At least %1 GB of data will be stored in this directory, and it will grow over time.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">378</context></context-group> </trans-unit> - <trans-unit id="_msg257"> + <trans-unit id="_msg261"> <source xml:space="preserve">Approximately %1 GB of data will be stored in this directory.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">381</context></context-group> @@ -1397,31 +1418,31 @@ Signing is only possible with addresses of the type 'legacy'.</source> <group restype="x-gettext-plurals"> <context-group purpose="location"><context context-type="linenumber">390</context></context-group> <note annotates="source" from="developer">Explanatory text on the capability of the current prune target.</note> - <trans-unit id="_msg258[0]"> + <trans-unit id="_msg262[0]"> <source xml:space="preserve">(sufficient to restore backups %n day(s) old)</source> <target xml:space="preserve"></target> </trans-unit> - <trans-unit id="_msg258[1]"> + <trans-unit id="_msg262[1]"> <source xml:space="preserve">(sufficient to restore backups %n day(s) old)</source> <target xml:space="preserve"></target> </trans-unit> </group> - <trans-unit id="_msg259"> + <trans-unit id="_msg263"> <source xml:space="preserve">%1 will download and store a copy of the Bitcoin block chain.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">392</context></context-group> </trans-unit> - <trans-unit id="_msg260"> + <trans-unit id="_msg264"> <source xml:space="preserve">The wallet will also be stored in this directory.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">394</context></context-group> </trans-unit> - <trans-unit id="_msg261"> + <trans-unit id="_msg265"> <source xml:space="preserve">Error: Specified data directory "%1" cannot be created.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">250</context></context-group> </trans-unit> - <trans-unit id="_msg262" approved="yes"> + <trans-unit id="_msg266" approved="yes"> <source xml:space="preserve">Error</source> <target xml:space="preserve">Error</target> <context-group purpose="location"><context context-type="linenumber">280</context></context-group> @@ -1430,29 +1451,29 @@ Signing is only possible with addresses of the type 'legacy'.</source> </body></file> <file original="../utilitydialog.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="HelpMessageDialog"> - <trans-unit id="_msg263"> + <trans-unit id="_msg267"> <source xml:space="preserve">version</source> <target xml:space="preserve" state="needs-review-translation">version</target> <context-group purpose="location"><context context-type="linenumber">37</context></context-group> </trans-unit> - <trans-unit id="_msg264"> + <trans-unit id="_msg268"> <source xml:space="preserve">About %1</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">41</context></context-group> </trans-unit> - <trans-unit id="_msg265"> + <trans-unit id="_msg269"> <source xml:space="preserve">Command-line options</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">60</context></context-group> </trans-unit> </group> <group restype="x-trolltech-linguist-context" resname="ShutdownWindow"> - <trans-unit id="_msg266"> + <trans-unit id="_msg270"> <source xml:space="preserve">%1 is shutting down…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">145</context></context-group> </trans-unit> - <trans-unit id="_msg267"> + <trans-unit id="_msg271"> <source xml:space="preserve">Do not shut down the computer until this window disappears.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">146</context></context-group> @@ -1461,57 +1482,57 @@ Signing is only possible with addresses of the type 'legacy'.</source> </body></file> <file original="../forms/intro.ui" datatype="x-trolltech-designer-ui" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="Intro"> - <trans-unit id="_msg268" approved="yes"> + <trans-unit id="_msg272" approved="yes"> <source xml:space="preserve">Welcome</source> <target xml:space="preserve">Welcome</target> <context-group purpose="location"><context context-type="linenumber">14</context></context-group> </trans-unit> - <trans-unit id="_msg269"> + <trans-unit id="_msg273"> <source xml:space="preserve">Welcome to %1.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">23</context></context-group> </trans-unit> - <trans-unit id="_msg270"> + <trans-unit id="_msg274"> <source xml:space="preserve">As this is the first time the program is launched, you can choose where %1 will store its data.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">49</context></context-group> </trans-unit> - <trans-unit id="_msg271"> + <trans-unit id="_msg275"> <source xml:space="preserve">When you click OK, %1 will begin to download and process the full %4 block chain (%2GB) starting with the earliest transactions in %3 when %4 initially launched.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">206</context></context-group> </trans-unit> - <trans-unit id="_msg272"> + <trans-unit id="_msg276"> <source xml:space="preserve">Limit block chain storage to</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">238</context></context-group> </trans-unit> - <trans-unit id="_msg273"> + <trans-unit id="_msg277"> <source xml:space="preserve">Reverting this setting requires re-downloading the entire blockchain. It is faster to download the full chain first and prune it later. Disables some advanced features.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">241</context></context-group> </trans-unit> - <trans-unit id="_msg274"> + <trans-unit id="_msg278"> <source xml:space="preserve"> GB</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">248</context></context-group> </trans-unit> - <trans-unit id="_msg275"> + <trans-unit id="_msg279"> <source xml:space="preserve">This initial synchronisation is very demanding, and may expose hardware problems with your computer that had previously gone unnoticed. Each time you run %1, it will continue downloading where it left off.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">216</context></context-group> </trans-unit> - <trans-unit id="_msg276"> + <trans-unit id="_msg280"> <source xml:space="preserve">If you have chosen to limit block chain storage (pruning), the historical data must still be downloaded and processed, but will be deleted afterward to keep your disk usage low.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">226</context></context-group> </trans-unit> - <trans-unit id="_msg277" approved="yes"> + <trans-unit id="_msg281" approved="yes"> <source xml:space="preserve">Use the default data directory</source> <target xml:space="preserve">Use the default data directory</target> <context-group purpose="location"><context context-type="linenumber">66</context></context-group> </trans-unit> - <trans-unit id="_msg278" approved="yes"> + <trans-unit id="_msg282" approved="yes"> <source xml:space="preserve">Use a custom data directory:</source> <target xml:space="preserve">Use a custom data directory:</target> <context-group purpose="location"><context context-type="linenumber">73</context></context-group> @@ -1520,65 +1541,65 @@ Signing is only possible with addresses of the type 'legacy'.</source> </body></file> <file original="../forms/modaloverlay.ui" datatype="x-trolltech-designer-ui" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="ModalOverlay"> - <trans-unit id="_msg279"> + <trans-unit id="_msg283"> <source xml:space="preserve">Form</source> <target xml:space="preserve" state="needs-review-translation">Form</target> <context-group purpose="location"><context context-type="linenumber">14</context></context-group> </trans-unit> - <trans-unit id="_msg280"> + <trans-unit id="_msg284"> <source xml:space="preserve">Recent transactions may not yet be visible, and therefore your wallet's balance might be incorrect. This information will be correct once your wallet has finished synchronizing with the bitcoin network, as detailed below.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">133</context></context-group> </trans-unit> - <trans-unit id="_msg281"> + <trans-unit id="_msg285"> <source xml:space="preserve">Attempting to spend bitcoins that are affected by not-yet-displayed transactions will not be accepted by the network.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">152</context></context-group> </trans-unit> - <trans-unit id="_msg282"> + <trans-unit id="_msg286"> <source xml:space="preserve">Number of blocks left</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">215</context></context-group> </trans-unit> - <trans-unit id="_msg283"> + <trans-unit id="_msg287"> <source xml:space="preserve">Unknown…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">222</context></context-group> <context-group purpose="location"><context context-type="linenumber">248</context></context-group> <context-group purpose="location"><context context-type="sourcefile">../modaloverlay.cpp</context><context context-type="linenumber">152</context></context-group> </trans-unit> - <trans-unit id="_msg284"> + <trans-unit id="_msg288"> <source xml:space="preserve">calculating…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">292</context></context-group> <context-group purpose="location"><context context-type="linenumber">312</context></context-group> </trans-unit> - <trans-unit id="_msg285"> + <trans-unit id="_msg289"> <source xml:space="preserve">Last block time</source> <target xml:space="preserve" state="needs-review-translation">Last block time</target> <context-group purpose="location"><context context-type="linenumber">235</context></context-group> </trans-unit> - <trans-unit id="_msg286"> + <trans-unit id="_msg290"> <source xml:space="preserve">Progress</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">261</context></context-group> </trans-unit> - <trans-unit id="_msg287"> + <trans-unit id="_msg291"> <source xml:space="preserve">Progress increase per hour</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">285</context></context-group> </trans-unit> - <trans-unit id="_msg288"> + <trans-unit id="_msg292"> <source xml:space="preserve">Estimated time left until synced</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">305</context></context-group> </trans-unit> - <trans-unit id="_msg289"> + <trans-unit id="_msg293"> <source xml:space="preserve">Hide</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">342</context></context-group> </trans-unit> - <trans-unit id="_msg290"> + <trans-unit id="_msg294"> <source xml:space="preserve">Esc</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">345</context></context-group> @@ -1587,19 +1608,19 @@ Signing is only possible with addresses of the type 'legacy'.</source> </body></file> <file original="../modaloverlay.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="ModalOverlay"> - <trans-unit id="_msg291"> + <trans-unit id="_msg295"> <source xml:space="preserve">%1 is currently syncing. It will download headers and blocks from peers and validate them until reaching the tip of the block chain.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">34</context></context-group> </trans-unit> - <trans-unit id="_msg292"> + <trans-unit id="_msg296"> <source xml:space="preserve">Unknown. Syncing Headers (%1, %2%)…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">158</context></context-group> </trans-unit> </group> <group restype="x-trolltech-linguist-context" resname="QObject"> - <trans-unit id="_msg293"> + <trans-unit id="_msg297"> <source xml:space="preserve">unknown</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">123</context></context-group> @@ -1608,12 +1629,12 @@ Signing is only possible with addresses of the type 'legacy'.</source> </body></file> <file original="../forms/openuridialog.ui" datatype="x-trolltech-designer-ui" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="OpenURIDialog"> - <trans-unit id="_msg294"> + <trans-unit id="_msg298"> <source xml:space="preserve">Open bitcoin URI</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">14</context></context-group> </trans-unit> - <trans-unit id="_msg295"> + <trans-unit id="_msg299"> <source xml:space="preserve">URI:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">22</context></context-group> @@ -1622,474 +1643,495 @@ Signing is only possible with addresses of the type 'legacy'.</source> </body></file> <file original="../forms/optionsdialog.ui" datatype="x-trolltech-designer-ui" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="OptionsDialog"> - <trans-unit id="_msg296" approved="yes"> + <trans-unit id="_msg300" approved="yes"> <source xml:space="preserve">Options</source> <target xml:space="preserve">Options</target> <context-group purpose="location"><context context-type="linenumber">14</context></context-group> </trans-unit> - <trans-unit id="_msg297" approved="yes"> + <trans-unit id="_msg301" approved="yes"> <source xml:space="preserve">&Main</source> <target xml:space="preserve">&Main</target> <context-group purpose="location"><context context-type="linenumber">27</context></context-group> </trans-unit> - <trans-unit id="_msg298"> + <trans-unit id="_msg302"> <source xml:space="preserve">Automatically start %1 after logging in to the system.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">33</context></context-group> </trans-unit> - <trans-unit id="_msg299"> + <trans-unit id="_msg303"> <source xml:space="preserve">&Start %1 on system login</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">36</context></context-group> </trans-unit> - <trans-unit id="_msg300"> + <trans-unit id="_msg304"> <source xml:space="preserve">Enabling pruning significantly reduces the disk space required to store transactions. All blocks are still fully validated. Reverting this setting requires re-downloading the entire blockchain.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">58</context></context-group> </trans-unit> - <trans-unit id="_msg301"> + <trans-unit id="_msg305"> <source xml:space="preserve">Size of &database cache</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">108</context></context-group> </trans-unit> - <trans-unit id="_msg302"> + <trans-unit id="_msg306"> <source xml:space="preserve">Number of script &verification threads</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">151</context></context-group> </trans-unit> - <trans-unit id="_msg303"> + <trans-unit id="_msg307"> <source xml:space="preserve">IP address of the proxy (e.g. IPv4: 127.0.0.1 / IPv6: ::1)</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">322</context></context-group> - <context-group purpose="location"><context context-type="linenumber">509</context></context-group> + <context-group purpose="location"><context context-type="linenumber">352</context></context-group> + <context-group purpose="location"><context context-type="linenumber">539</context></context-group> </trans-unit> - <trans-unit id="_msg304"> + <trans-unit id="_msg308"> <source xml:space="preserve">Shows if the supplied default SOCKS5 proxy is used to reach peers via this network type.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">391</context></context-group> - <context-group purpose="location"><context context-type="linenumber">414</context></context-group> - <context-group purpose="location"><context context-type="linenumber">437</context></context-group> + <context-group purpose="location"><context context-type="linenumber">421</context></context-group> + <context-group purpose="location"><context context-type="linenumber">444</context></context-group> + <context-group purpose="location"><context context-type="linenumber">467</context></context-group> </trans-unit> - <trans-unit id="_msg305"> + <trans-unit id="_msg309"> <source xml:space="preserve">Minimize instead of exit the application when the window is closed. When this option is enabled, the application will be closed only after selecting Exit in the menu.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">606</context></context-group> + <context-group purpose="location"><context context-type="linenumber">636</context></context-group> </trans-unit> - <trans-unit id="_msg306"> + <trans-unit id="_msg310"> <source xml:space="preserve">Third party URLs (e.g. a block explorer) that appear in the transactions tab as context menu items. %s in the URL is replaced by transaction hash. Multiple URLs are separated by vertical bar |.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">686</context></context-group> - <context-group purpose="location"><context context-type="linenumber">699</context></context-group> + <context-group purpose="location"><context context-type="linenumber">716</context></context-group> + <context-group purpose="location"><context context-type="linenumber">729</context></context-group> </trans-unit> - <trans-unit id="_msg307"> + <trans-unit id="_msg311"> <source xml:space="preserve">Open the %1 configuration file from the working directory.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">878</context></context-group> + <context-group purpose="location"><context context-type="linenumber">908</context></context-group> </trans-unit> - <trans-unit id="_msg308"> + <trans-unit id="_msg312"> <source xml:space="preserve">Open Configuration File</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">881</context></context-group> + <context-group purpose="location"><context context-type="linenumber">911</context></context-group> </trans-unit> - <trans-unit id="_msg309" approved="yes"> + <trans-unit id="_msg313" approved="yes"> <source xml:space="preserve">Reset all client options to default.</source> <target xml:space="preserve">Reset all client options to default.</target> - <context-group purpose="location"><context context-type="linenumber">891</context></context-group> + <context-group purpose="location"><context context-type="linenumber">921</context></context-group> </trans-unit> - <trans-unit id="_msg310" approved="yes"> + <trans-unit id="_msg314" approved="yes"> <source xml:space="preserve">&Reset Options</source> <target xml:space="preserve">&Reset Options</target> - <context-group purpose="location"><context context-type="linenumber">894</context></context-group> + <context-group purpose="location"><context context-type="linenumber">924</context></context-group> </trans-unit> - <trans-unit id="_msg311" approved="yes"> + <trans-unit id="_msg315" approved="yes"> <source xml:space="preserve">&Network</source> <target xml:space="preserve">&Network</target> - <context-group purpose="location"><context context-type="linenumber">249</context></context-group> + <context-group purpose="location"><context context-type="linenumber">279</context></context-group> </trans-unit> - <trans-unit id="_msg312"> + <trans-unit id="_msg316"> <source xml:space="preserve">Prune &block storage to</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">61</context></context-group> </trans-unit> - <trans-unit id="_msg313"> + <trans-unit id="_msg317"> <source xml:space="preserve">GB</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">71</context></context-group> </trans-unit> - <trans-unit id="_msg314"> + <trans-unit id="_msg318"> <source xml:space="preserve">Reverting this setting requires re-downloading the entire blockchain.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">96</context></context-group> </trans-unit> - <trans-unit id="_msg315"> + <trans-unit id="_msg319"> <source xml:space="preserve">MiB</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">124</context></context-group> </trans-unit> - <trans-unit id="_msg316"> + <trans-unit id="_msg320"> <source xml:space="preserve">(0 = auto, <0 = leave that many cores free)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">164</context></context-group> </trans-unit> - <trans-unit id="_msg317"> + <trans-unit id="_msg321"> <source xml:space="preserve">W&allet</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">200</context></context-group> </trans-unit> - <trans-unit id="_msg318"> + <trans-unit id="_msg322"> <source xml:space="preserve">Expert</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">206</context></context-group> </trans-unit> - <trans-unit id="_msg319"> + <trans-unit id="_msg323"> <source xml:space="preserve">Enable coin &control features</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">215</context></context-group> </trans-unit> - <trans-unit id="_msg320"> + <trans-unit id="_msg324"> <source xml:space="preserve">If you disable the spending of unconfirmed change, the change from a transaction cannot be used until that transaction has at least one confirmation. This also affects how your balance is computed.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">222</context></context-group> </trans-unit> - <trans-unit id="_msg321"> + <trans-unit id="_msg325"> <source xml:space="preserve">&Spend unconfirmed change</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">225</context></context-group> </trans-unit> - <trans-unit id="_msg322" approved="yes"> + <trans-unit id="_msg326"> + <source xml:space="preserve">External Signer (e.g. hardware wallet)</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">235</context></context-group> + </trans-unit> + <trans-unit id="_msg327"> + <source xml:space="preserve">&External signer script path</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">243</context></context-group> + </trans-unit> + <trans-unit id="_msg328"> + <source xml:space="preserve">Full path to a Bitcoin Core compatible script (e.g. C:\Downloads\hwi.exe or /Users/you/Downloads/hwi.py). Beware: malware can steal your coins!</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">253</context></context-group> + </trans-unit> + <trans-unit id="_msg329" approved="yes"> <source xml:space="preserve">Automatically open the Bitcoin client port on the router. This only works when your router supports UPnP and it is enabled.</source> <target xml:space="preserve">Automatically open the Bitcoin client port on the router. This only works when your router supports UPnP and it is enabled.</target> - <context-group purpose="location"><context context-type="linenumber">255</context></context-group> + <context-group purpose="location"><context context-type="linenumber">285</context></context-group> </trans-unit> - <trans-unit id="_msg323" approved="yes"> + <trans-unit id="_msg330" approved="yes"> <source xml:space="preserve">Map port using &UPnP</source> <target xml:space="preserve">Map port using &UPnP</target> - <context-group purpose="location"><context context-type="linenumber">258</context></context-group> + <context-group purpose="location"><context context-type="linenumber">288</context></context-group> </trans-unit> - <trans-unit id="_msg324"> + <trans-unit id="_msg331"> <source xml:space="preserve">Automatically open the Bitcoin client port on the router. This only works when your router supports NAT-PMP and it is enabled. The external port could be random.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">265</context></context-group> + <context-group purpose="location"><context context-type="linenumber">295</context></context-group> </trans-unit> - <trans-unit id="_msg325"> + <trans-unit id="_msg332"> <source xml:space="preserve">Map port using NA&T-PMP</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">268</context></context-group> + <context-group purpose="location"><context context-type="linenumber">298</context></context-group> </trans-unit> - <trans-unit id="_msg326"> + <trans-unit id="_msg333"> <source xml:space="preserve">Accept connections from outside.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">275</context></context-group> + <context-group purpose="location"><context context-type="linenumber">305</context></context-group> </trans-unit> - <trans-unit id="_msg327"> + <trans-unit id="_msg334"> <source xml:space="preserve">Allow incomin&g connections</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">278</context></context-group> + <context-group purpose="location"><context context-type="linenumber">308</context></context-group> </trans-unit> - <trans-unit id="_msg328"> + <trans-unit id="_msg335"> <source xml:space="preserve">Connect to the Bitcoin network through a SOCKS5 proxy.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">285</context></context-group> + <context-group purpose="location"><context context-type="linenumber">315</context></context-group> </trans-unit> - <trans-unit id="_msg329"> + <trans-unit id="_msg336"> <source xml:space="preserve">&Connect through SOCKS5 proxy (default proxy):</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">288</context></context-group> + <context-group purpose="location"><context context-type="linenumber">318</context></context-group> </trans-unit> - <trans-unit id="_msg330" approved="yes"> + <trans-unit id="_msg337" approved="yes"> <source xml:space="preserve">Proxy &IP:</source> <target xml:space="preserve">Proxy &IP:</target> - <context-group purpose="location"><context context-type="linenumber">297</context></context-group> - <context-group purpose="location"><context context-type="linenumber">484</context></context-group> + <context-group purpose="location"><context context-type="linenumber">327</context></context-group> + <context-group purpose="location"><context context-type="linenumber">514</context></context-group> </trans-unit> - <trans-unit id="_msg331" approved="yes"> + <trans-unit id="_msg338" approved="yes"> <source xml:space="preserve">&Port:</source> <target xml:space="preserve">&Port:</target> - <context-group purpose="location"><context context-type="linenumber">329</context></context-group> - <context-group purpose="location"><context context-type="linenumber">516</context></context-group> + <context-group purpose="location"><context context-type="linenumber">359</context></context-group> + <context-group purpose="location"><context context-type="linenumber">546</context></context-group> </trans-unit> - <trans-unit id="_msg332" approved="yes"> + <trans-unit id="_msg339" approved="yes"> <source xml:space="preserve">Port of the proxy (e.g. 9050)</source> <target xml:space="preserve">Port of the proxy (e.g. 9050)</target> - <context-group purpose="location"><context context-type="linenumber">354</context></context-group> - <context-group purpose="location"><context context-type="linenumber">541</context></context-group> + <context-group purpose="location"><context context-type="linenumber">384</context></context-group> + <context-group purpose="location"><context context-type="linenumber">571</context></context-group> </trans-unit> - <trans-unit id="_msg333"> + <trans-unit id="_msg340"> <source xml:space="preserve">Used for reaching peers via:</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">378</context></context-group> + <context-group purpose="location"><context context-type="linenumber">408</context></context-group> </trans-unit> - <trans-unit id="_msg334"> + <trans-unit id="_msg341"> <source xml:space="preserve">IPv4</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">401</context></context-group> + <context-group purpose="location"><context context-type="linenumber">431</context></context-group> </trans-unit> - <trans-unit id="_msg335"> + <trans-unit id="_msg342"> <source xml:space="preserve">IPv6</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">424</context></context-group> + <context-group purpose="location"><context context-type="linenumber">454</context></context-group> </trans-unit> - <trans-unit id="_msg336"> + <trans-unit id="_msg343"> <source xml:space="preserve">Tor</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">447</context></context-group> + <context-group purpose="location"><context context-type="linenumber">477</context></context-group> </trans-unit> - <trans-unit id="_msg337" approved="yes"> + <trans-unit id="_msg344" approved="yes"> <source xml:space="preserve">&Window</source> <target xml:space="preserve">&Window</target> - <context-group purpose="location"><context context-type="linenumber">577</context></context-group> + <context-group purpose="location"><context context-type="linenumber">607</context></context-group> </trans-unit> - <trans-unit id="_msg338"> + <trans-unit id="_msg345"> <source xml:space="preserve">Show the icon in the system tray.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">583</context></context-group> + <context-group purpose="location"><context context-type="linenumber">613</context></context-group> </trans-unit> - <trans-unit id="_msg339"> + <trans-unit id="_msg346"> <source xml:space="preserve">&Show tray icon</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">586</context></context-group> + <context-group purpose="location"><context context-type="linenumber">616</context></context-group> </trans-unit> - <trans-unit id="_msg340" approved="yes"> + <trans-unit id="_msg347" approved="yes"> <source xml:space="preserve">Show only a tray icon after minimizing the window.</source> <target xml:space="preserve">Show only a tray icon after minimizing the window.</target> - <context-group purpose="location"><context context-type="linenumber">596</context></context-group> + <context-group purpose="location"><context context-type="linenumber">626</context></context-group> </trans-unit> - <trans-unit id="_msg341" approved="yes"> + <trans-unit id="_msg348" approved="yes"> <source xml:space="preserve">&Minimize to the tray instead of the taskbar</source> <target xml:space="preserve">&Minimize to the tray instead of the taskbar</target> - <context-group purpose="location"><context context-type="linenumber">599</context></context-group> + <context-group purpose="location"><context context-type="linenumber">629</context></context-group> </trans-unit> - <trans-unit id="_msg342" approved="yes"> + <trans-unit id="_msg349" approved="yes"> <source xml:space="preserve">M&inimize on close</source> <target xml:space="preserve">M&inimize on close</target> - <context-group purpose="location"><context context-type="linenumber">609</context></context-group> + <context-group purpose="location"><context context-type="linenumber">639</context></context-group> </trans-unit> - <trans-unit id="_msg343" approved="yes"> + <trans-unit id="_msg350" approved="yes"> <source xml:space="preserve">&Display</source> <target xml:space="preserve">&Display</target> - <context-group purpose="location"><context context-type="linenumber">630</context></context-group> + <context-group purpose="location"><context context-type="linenumber">660</context></context-group> </trans-unit> - <trans-unit id="_msg344" approved="yes"> + <trans-unit id="_msg351" approved="yes"> <source xml:space="preserve">User Interface &language:</source> <target xml:space="preserve">User Interface &language:</target> - <context-group purpose="location"><context context-type="linenumber">638</context></context-group> + <context-group purpose="location"><context context-type="linenumber">668</context></context-group> </trans-unit> - <trans-unit id="_msg345"> + <trans-unit id="_msg352"> <source xml:space="preserve">The user interface language can be set here. This setting will take effect after restarting %1.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">651</context></context-group> + <context-group purpose="location"><context context-type="linenumber">681</context></context-group> </trans-unit> - <trans-unit id="_msg346" approved="yes"> + <trans-unit id="_msg353" approved="yes"> <source xml:space="preserve">&Unit to show amounts in:</source> <target xml:space="preserve">&Unit to show amounts in:</target> - <context-group purpose="location"><context context-type="linenumber">662</context></context-group> + <context-group purpose="location"><context context-type="linenumber">692</context></context-group> </trans-unit> - <trans-unit id="_msg347" approved="yes"> + <trans-unit id="_msg354" approved="yes"> <source xml:space="preserve">Choose the default subdivision unit to show in the interface and when sending coins.</source> <target xml:space="preserve">Choose the default subdivision unit to show in the interface and when sending coins.</target> - <context-group purpose="location"><context context-type="linenumber">675</context></context-group> + <context-group purpose="location"><context context-type="linenumber">705</context></context-group> </trans-unit> - <trans-unit id="_msg348"> + <trans-unit id="_msg355"> <source xml:space="preserve">Whether to show coin control features or not.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">212</context></context-group> </trans-unit> - <trans-unit id="_msg349"> + <trans-unit id="_msg356"> <source xml:space="preserve">Connect to the Bitcoin network through a separate SOCKS5 proxy for Tor onion services.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">472</context></context-group> + <context-group purpose="location"><context context-type="linenumber">502</context></context-group> </trans-unit> - <trans-unit id="_msg350"> + <trans-unit id="_msg357"> <source xml:space="preserve">Use separate SOCKS&5 proxy to reach peers via Tor onion services:</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">475</context></context-group> + <context-group purpose="location"><context context-type="linenumber">505</context></context-group> </trans-unit> - <trans-unit id="_msg351"> + <trans-unit id="_msg358"> <source xml:space="preserve">&Third party transaction URLs</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">689</context></context-group> + <context-group purpose="location"><context context-type="linenumber">719</context></context-group> </trans-unit> - <trans-unit id="_msg352"> + <trans-unit id="_msg359"> <source xml:space="preserve">Monospaced font in the Overview tab:</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">711</context></context-group> + <context-group purpose="location"><context context-type="linenumber">741</context></context-group> </trans-unit> - <trans-unit id="_msg353"> + <trans-unit id="_msg360"> <source xml:space="preserve">embedded "%1"</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">719</context></context-group> + <context-group purpose="location"><context context-type="linenumber">749</context></context-group> </trans-unit> - <trans-unit id="_msg354"> + <trans-unit id="_msg361"> <source xml:space="preserve">closest matching "%1"</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">768</context></context-group> + <context-group purpose="location"><context context-type="linenumber">798</context></context-group> </trans-unit> - <trans-unit id="_msg355"> + <trans-unit id="_msg362"> <source xml:space="preserve">Options set in this dialog are overridden by the command line or in the configuration file:</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">833</context></context-group> + <context-group purpose="location"><context context-type="linenumber">863</context></context-group> </trans-unit> - <trans-unit id="_msg356" approved="yes"> + <trans-unit id="_msg363" approved="yes"> <source xml:space="preserve">&OK</source> <target xml:space="preserve">&OK</target> - <context-group purpose="location"><context context-type="linenumber">974</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1004</context></context-group> </trans-unit> - <trans-unit id="_msg357" approved="yes"> + <trans-unit id="_msg364" approved="yes"> <source xml:space="preserve">&Cancel</source> <target xml:space="preserve">&Cancel</target> - <context-group purpose="location"><context context-type="linenumber">987</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1017</context></context-group> </trans-unit> </group> </body></file> <file original="../optionsdialog.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="OptionsDialog"> - <trans-unit id="_msg358" approved="yes"> + <trans-unit id="_msg365"> + <source xml:space="preserve">Compiled without external signing support (required for external signing)</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">97</context></context-group> + <note annotates="source" from="developer">"External signing" means using devices such as hardware wallets.</note> + </trans-unit> + <trans-unit id="_msg366" approved="yes"> <source xml:space="preserve">default</source> <target xml:space="preserve">default</target> - <context-group purpose="location"><context context-type="linenumber">104</context></context-group> + <context-group purpose="location"><context context-type="linenumber">109</context></context-group> </trans-unit> - <trans-unit id="_msg359"> + <trans-unit id="_msg367"> <source xml:space="preserve">none</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">185</context></context-group> + <context-group purpose="location"><context context-type="linenumber">190</context></context-group> </trans-unit> - <trans-unit id="_msg360" approved="yes"> + <trans-unit id="_msg368" approved="yes"> <source xml:space="preserve">Confirm options reset</source> <target xml:space="preserve">Confirm options reset</target> - <context-group purpose="location"><context context-type="linenumber">276</context></context-group> + <context-group purpose="location"><context context-type="linenumber">283</context></context-group> </trans-unit> - <trans-unit id="_msg361"> + <trans-unit id="_msg369"> <source xml:space="preserve">Client restart required to activate changes.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">277</context></context-group> - <context-group purpose="location"><context context-type="linenumber">334</context></context-group> + <context-group purpose="location"><context context-type="linenumber">284</context></context-group> + <context-group purpose="location"><context context-type="linenumber">341</context></context-group> </trans-unit> - <trans-unit id="_msg362"> + <trans-unit id="_msg370"> <source xml:space="preserve">Client will be shut down. Do you want to proceed?</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">277</context></context-group> + <context-group purpose="location"><context context-type="linenumber">284</context></context-group> </trans-unit> - <trans-unit id="_msg363"> + <trans-unit id="_msg371"> <source xml:space="preserve">Configuration options</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">292</context></context-group> + <context-group purpose="location"><context context-type="linenumber">299</context></context-group> </trans-unit> - <trans-unit id="_msg364"> + <trans-unit id="_msg372"> <source xml:space="preserve">The configuration file is used to specify advanced user options which override GUI settings. Additionally, any command-line options will override this configuration file.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">293</context></context-group> + <context-group purpose="location"><context context-type="linenumber">300</context></context-group> </trans-unit> - <trans-unit id="_msg365"> + <trans-unit id="_msg373"> <source xml:space="preserve">Error</source> <target xml:space="preserve" state="needs-review-translation">Error</target> - <context-group purpose="location"><context context-type="linenumber">298</context></context-group> + <context-group purpose="location"><context context-type="linenumber">305</context></context-group> </trans-unit> - <trans-unit id="_msg366"> + <trans-unit id="_msg374"> <source xml:space="preserve">The configuration file could not be opened.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">298</context></context-group> + <context-group purpose="location"><context context-type="linenumber">305</context></context-group> </trans-unit> - <trans-unit id="_msg367"> + <trans-unit id="_msg375"> <source xml:space="preserve">This change would require a client restart.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">338</context></context-group> + <context-group purpose="location"><context context-type="linenumber">345</context></context-group> </trans-unit> - <trans-unit id="_msg368" approved="yes"> + <trans-unit id="_msg376" approved="yes"> <source xml:space="preserve">The supplied proxy address is invalid.</source> <target xml:space="preserve">The supplied proxy address is invalid.</target> - <context-group purpose="location"><context context-type="linenumber">366</context></context-group> + <context-group purpose="location"><context context-type="linenumber">373</context></context-group> </trans-unit> </group> </body></file> <file original="../forms/overviewpage.ui" datatype="x-trolltech-designer-ui" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="OverviewPage"> - <trans-unit id="_msg369" approved="yes"> + <trans-unit id="_msg377" approved="yes"> <source xml:space="preserve">Form</source> <target xml:space="preserve">Form</target> <context-group purpose="location"><context context-type="linenumber">14</context></context-group> </trans-unit> - <trans-unit id="_msg370" approved="yes"> + <trans-unit id="_msg378" approved="yes"> <source xml:space="preserve">The displayed information may be out of date. Your wallet automatically synchronizes with the Bitcoin network after a connection is established, but this process has not completed yet.</source> <target xml:space="preserve">The displayed information may be out of date. Your wallet automatically synchronizes with the Bitcoin network after a connection is established, but this process has not completed yet.</target> <context-group purpose="location"><context context-type="linenumber">76</context></context-group> <context-group purpose="location"><context context-type="linenumber">411</context></context-group> </trans-unit> - <trans-unit id="_msg371"> + <trans-unit id="_msg379"> <source xml:space="preserve">Watch-only:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">284</context></context-group> </trans-unit> - <trans-unit id="_msg372"> + <trans-unit id="_msg380"> <source xml:space="preserve">Available:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">294</context></context-group> </trans-unit> - <trans-unit id="_msg373" approved="yes"> + <trans-unit id="_msg381" approved="yes"> <source xml:space="preserve">Your current spendable balance</source> <target xml:space="preserve">Your current spendable balance</target> <context-group purpose="location"><context context-type="linenumber">304</context></context-group> </trans-unit> - <trans-unit id="_msg374"> + <trans-unit id="_msg382"> <source xml:space="preserve">Pending:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">339</context></context-group> </trans-unit> - <trans-unit id="_msg375" approved="yes"> + <trans-unit id="_msg383" approved="yes"> <source xml:space="preserve">Total of transactions that have yet to be confirmed, and do not yet count toward the spendable balance</source> <target xml:space="preserve">Total of transactions that have yet to be confirmed, and do not yet count toward the spendable balance</target> <context-group purpose="location"><context context-type="linenumber">139</context></context-group> </trans-unit> - <trans-unit id="_msg376" approved="yes"> + <trans-unit id="_msg384" approved="yes"> <source xml:space="preserve">Immature:</source> <target xml:space="preserve">Immature:</target> <context-group purpose="location"><context context-type="linenumber">239</context></context-group> </trans-unit> - <trans-unit id="_msg377" approved="yes"> + <trans-unit id="_msg385" approved="yes"> <source xml:space="preserve">Mined balance that has not yet matured</source> <target xml:space="preserve">Mined balance that has not yet matured</target> <context-group purpose="location"><context context-type="linenumber">210</context></context-group> </trans-unit> - <trans-unit id="_msg378"> + <trans-unit id="_msg386"> <source xml:space="preserve">Balances</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">60</context></context-group> </trans-unit> - <trans-unit id="_msg379" approved="yes"> + <trans-unit id="_msg387" approved="yes"> <source xml:space="preserve">Total:</source> <target xml:space="preserve">Total:</target> <context-group purpose="location"><context context-type="linenumber">200</context></context-group> </trans-unit> - <trans-unit id="_msg380" approved="yes"> + <trans-unit id="_msg388" approved="yes"> <source xml:space="preserve">Your current total balance</source> <target xml:space="preserve">Your current total balance</target> <context-group purpose="location"><context context-type="linenumber">249</context></context-group> </trans-unit> - <trans-unit id="_msg381"> + <trans-unit id="_msg389"> <source xml:space="preserve">Your current balance in watch-only addresses</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">323</context></context-group> </trans-unit> - <trans-unit id="_msg382"> + <trans-unit id="_msg390"> <source xml:space="preserve">Spendable:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">346</context></context-group> </trans-unit> - <trans-unit id="_msg383"> + <trans-unit id="_msg391"> <source xml:space="preserve">Recent transactions</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">395</context></context-group> </trans-unit> - <trans-unit id="_msg384"> + <trans-unit id="_msg392"> <source xml:space="preserve">Unconfirmed transactions to watch-only addresses</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">120</context></context-group> </trans-unit> - <trans-unit id="_msg385"> + <trans-unit id="_msg393"> <source xml:space="preserve">Mined balance in watch-only addresses that has not yet matured</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">158</context></context-group> </trans-unit> - <trans-unit id="_msg386"> + <trans-unit id="_msg394"> <source xml:space="preserve">Current total balance in watch-only addresses</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">268</context></context-group> @@ -2098,41 +2140,41 @@ Signing is only possible with addresses of the type 'legacy'.</source> </body></file> <file original="../overviewpage.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="OverviewPage"> - <trans-unit id="_msg387"> + <trans-unit id="_msg395"> <source xml:space="preserve">Privacy mode activated for the Overview tab. To unmask the values, uncheck Settings->Mask values.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">193</context></context-group> + <context-group purpose="location"><context context-type="linenumber">188</context></context-group> </trans-unit> </group> </body></file> <file original="../forms/psbtoperationsdialog.ui" datatype="x-trolltech-designer-ui" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="PSBTOperationsDialog"> - <trans-unit id="_msg388"> + <trans-unit id="_msg396"> <source xml:space="preserve">Dialog</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">14</context></context-group> </trans-unit> - <trans-unit id="_msg389"> + <trans-unit id="_msg397"> <source xml:space="preserve">Sign Tx</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">86</context></context-group> </trans-unit> - <trans-unit id="_msg390"> + <trans-unit id="_msg398"> <source xml:space="preserve">Broadcast Tx</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">102</context></context-group> </trans-unit> - <trans-unit id="_msg391"> + <trans-unit id="_msg399"> <source xml:space="preserve">Copy to Clipboard</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">122</context></context-group> </trans-unit> - <trans-unit id="_msg392"> + <trans-unit id="_msg400"> <source xml:space="preserve">Save…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">129</context></context-group> </trans-unit> - <trans-unit id="_msg393"> + <trans-unit id="_msg401"> <source xml:space="preserve">Close</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">136</context></context-group> @@ -2141,123 +2183,123 @@ Signing is only possible with addresses of the type 'legacy'.</source> </body></file> <file original="../psbtoperationsdialog.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="PSBTOperationsDialog"> - <trans-unit id="_msg394"> + <trans-unit id="_msg402"> <source xml:space="preserve">Failed to load transaction: %1</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">55</context></context-group> </trans-unit> - <trans-unit id="_msg395"> + <trans-unit id="_msg403"> <source xml:space="preserve">Failed to sign transaction: %1</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">73</context></context-group> </trans-unit> - <trans-unit id="_msg396"> + <trans-unit id="_msg404"> <source xml:space="preserve">Could not sign any more inputs.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">81</context></context-group> </trans-unit> - <trans-unit id="_msg397"> + <trans-unit id="_msg405"> <source xml:space="preserve">Signed %1 inputs, but more signatures are still required.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">83</context></context-group> </trans-unit> - <trans-unit id="_msg398"> + <trans-unit id="_msg406"> <source xml:space="preserve">Signed transaction successfully. Transaction is ready to broadcast.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">86</context></context-group> </trans-unit> - <trans-unit id="_msg399"> + <trans-unit id="_msg407"> <source xml:space="preserve">Unknown error processing transaction.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">98</context></context-group> </trans-unit> - <trans-unit id="_msg400"> + <trans-unit id="_msg408"> <source xml:space="preserve">Transaction broadcast successfully! Transaction ID: %1</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">108</context></context-group> </trans-unit> - <trans-unit id="_msg401"> + <trans-unit id="_msg409"> <source xml:space="preserve">Transaction broadcast failed: %1</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">111</context></context-group> </trans-unit> - <trans-unit id="_msg402"> + <trans-unit id="_msg410"> <source xml:space="preserve">PSBT copied to clipboard.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">120</context></context-group> </trans-unit> - <trans-unit id="_msg403"> + <trans-unit id="_msg411"> <source xml:space="preserve">Save Transaction Data</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">143</context></context-group> </trans-unit> - <trans-unit id="_msg404"> + <trans-unit id="_msg412"> <source xml:space="preserve">Partially Signed Transaction (Binary)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">145</context></context-group> <note annotates="source" from="developer">Expanded name of the binary PSBT file format. See: BIP 174.</note> </trans-unit> - <trans-unit id="_msg405"> + <trans-unit id="_msg413"> <source xml:space="preserve">PSBT saved to disk.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">152</context></context-group> </trans-unit> - <trans-unit id="_msg406"> + <trans-unit id="_msg414"> <source xml:space="preserve"> * Sends %1 to %2</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">168</context></context-group> </trans-unit> - <trans-unit id="_msg407"> + <trans-unit id="_msg415"> <source xml:space="preserve">Unable to calculate transaction fee or total transaction amount.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">178</context></context-group> </trans-unit> - <trans-unit id="_msg408"> + <trans-unit id="_msg416"> <source xml:space="preserve">Pays transaction fee: </source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">180</context></context-group> </trans-unit> - <trans-unit id="_msg409"> + <trans-unit id="_msg417"> <source xml:space="preserve">Total Amount</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">192</context></context-group> </trans-unit> - <trans-unit id="_msg410"> + <trans-unit id="_msg418"> <source xml:space="preserve">or</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">195</context></context-group> </trans-unit> - <trans-unit id="_msg411"> + <trans-unit id="_msg419"> <source xml:space="preserve">Transaction has %1 unsigned inputs.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">201</context></context-group> </trans-unit> - <trans-unit id="_msg412"> + <trans-unit id="_msg420"> <source xml:space="preserve">Transaction is missing some information about inputs.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">243</context></context-group> </trans-unit> - <trans-unit id="_msg413"> + <trans-unit id="_msg421"> <source xml:space="preserve">Transaction still needs signature(s).</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">247</context></context-group> </trans-unit> - <trans-unit id="_msg414"> + <trans-unit id="_msg422"> <source xml:space="preserve">(But this wallet cannot sign transactions.)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">250</context></context-group> </trans-unit> - <trans-unit id="_msg415"> + <trans-unit id="_msg423"> <source xml:space="preserve">(But this wallet does not have the right keys.)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">253</context></context-group> </trans-unit> - <trans-unit id="_msg416"> + <trans-unit id="_msg424"> <source xml:space="preserve">Transaction is fully signed and ready for broadcast.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">261</context></context-group> </trans-unit> - <trans-unit id="_msg417"> + <trans-unit id="_msg425"> <source xml:space="preserve">Transaction status is unknown.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">265</context></context-group> @@ -2266,17 +2308,17 @@ Signing is only possible with addresses of the type 'legacy'.</source> </body></file> <file original="../paymentserver.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="PaymentServer"> - <trans-unit id="_msg418"> + <trans-unit id="_msg426"> <source xml:space="preserve">Payment request error</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">173</context></context-group> </trans-unit> - <trans-unit id="_msg419"> + <trans-unit id="_msg427"> <source xml:space="preserve">Cannot start bitcoin: click-to-pay handler</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">174</context></context-group> </trans-unit> - <trans-unit id="_msg420"> + <trans-unit id="_msg428"> <source xml:space="preserve">URI handling</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">224</context></context-group> @@ -2284,12 +2326,12 @@ Signing is only possible with addresses of the type 'legacy'.</source> <context-group purpose="location"><context context-type="linenumber">246</context></context-group> <context-group purpose="location"><context context-type="linenumber">253</context></context-group> </trans-unit> - <trans-unit id="_msg421"> + <trans-unit id="_msg429"> <source xml:space="preserve">'bitcoin://' is not a valid URI. Use 'bitcoin:' instead.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">224</context></context-group> </trans-unit> - <trans-unit id="_msg422"> + <trans-unit id="_msg430"> <source xml:space="preserve">Cannot process payment request because BIP70 is not supported. Due to widespread security flaws in BIP70 it's strongly recommended that any merchant instructions to switch wallets be ignored. If you are receiving this error you should request the merchant provide a BIP21 compatible URI.</source> @@ -2297,12 +2339,12 @@ If you are receiving this error you should request the merchant provide a BIP21 <context-group purpose="location"><context context-type="linenumber">241</context></context-group> <context-group purpose="location"><context context-type="linenumber">264</context></context-group> </trans-unit> - <trans-unit id="_msg423"> + <trans-unit id="_msg431"> <source xml:space="preserve">URI cannot be parsed! This can be caused by an invalid Bitcoin address or malformed URI parameters.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">254</context></context-group> </trans-unit> - <trans-unit id="_msg424"> + <trans-unit id="_msg432"> <source xml:space="preserve">Payment request file handling</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">263</context></context-group> @@ -2311,59 +2353,59 @@ If you are receiving this error you should request the merchant provide a BIP21 </body></file> <file original="../peertablemodel.h" datatype="c" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="PeerTableModel"> - <trans-unit id="_msg425"> + <trans-unit id="_msg433"> <source xml:space="preserve">User Agent</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">101</context></context-group> + <context-group purpose="location"><context context-type="linenumber">107</context></context-group> <note annotates="source" from="developer">Title of Peers Table column which contains the peer's User Agent string.</note> </trans-unit> - <trans-unit id="_msg426"> + <trans-unit id="_msg434"> <source xml:space="preserve">Ping</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">92</context></context-group> + <context-group purpose="location"><context context-type="linenumber">98</context></context-group> <note annotates="source" from="developer">Title of Peers Table column which indicates the current latency of the connection with the peer.</note> </trans-unit> - <trans-unit id="_msg427"> + <trans-unit id="_msg435"> <source xml:space="preserve">Peer</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">80</context></context-group> + <context-group purpose="location"><context context-type="linenumber">86</context></context-group> <note annotates="source" from="developer">Title of Peers Table column which contains a unique number used to identify a connection.</note> </trans-unit> - <trans-unit id="_msg428"> + <trans-unit id="_msg436"> <source xml:space="preserve">Sent</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">95</context></context-group> + <context-group purpose="location"><context context-type="linenumber">101</context></context-group> <note annotates="source" from="developer">Title of Peers Table column which indicates the total amount of network information we have sent to the peer.</note> </trans-unit> - <trans-unit id="_msg429"> + <trans-unit id="_msg437"> <source xml:space="preserve">Received</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">98</context></context-group> + <context-group purpose="location"><context context-type="linenumber">104</context></context-group> <note annotates="source" from="developer">Title of Peers Table column which indicates the total amount of network information we have received from the peer.</note> </trans-unit> - <trans-unit id="_msg430"> + <trans-unit id="_msg438"> <source xml:space="preserve">Address</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">83</context></context-group> + <context-group purpose="location"><context context-type="linenumber">89</context></context-group> <note annotates="source" from="developer">Title of Peers Table column which contains the IP/Onion/I2P address of the connected peer.</note> </trans-unit> - <trans-unit id="_msg431"> + <trans-unit id="_msg439"> <source xml:space="preserve">Type</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">86</context></context-group> + <context-group purpose="location"><context context-type="linenumber">92</context></context-group> <note annotates="source" from="developer">Title of Peers Table column which describes the type of peer connection. The "type" describes why the connection exists.</note> </trans-unit> - <trans-unit id="_msg432"> + <trans-unit id="_msg440"> <source xml:space="preserve">Network</source> <target xml:space="preserve" state="needs-review-translation">Network</target> - <context-group purpose="location"><context context-type="linenumber">89</context></context-group> + <context-group purpose="location"><context context-type="linenumber">95</context></context-group> <note annotates="source" from="developer">Title of Peers Table column which states the network the peer connected through.</note> </trans-unit> </group> </body></file> <file original="../bitcoinunits.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="QObject"> - <trans-unit id="_msg433"> + <trans-unit id="_msg441"> <source xml:space="preserve">Amount</source> <target xml:space="preserve" state="needs-review-translation">Amount</target> <context-group purpose="location"><context context-type="linenumber">213</context></context-group> @@ -2372,132 +2414,132 @@ If you are receiving this error you should request the merchant provide a BIP21 </body></file> <file original="../guiutil.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="QObject"> - <trans-unit id="_msg434"> + <trans-unit id="_msg442"> <source xml:space="preserve">Enter a Bitcoin address (e.g. %1)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">120</context></context-group> </trans-unit> - <trans-unit id="_msg435"> + <trans-unit id="_msg443"> <source xml:space="preserve">Unroutable</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">660</context></context-group> </trans-unit> - <trans-unit id="_msg436"> + <trans-unit id="_msg444"> <source xml:space="preserve">Internal</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">666</context></context-group> </trans-unit> - <trans-unit id="_msg437"> + <trans-unit id="_msg445"> <source xml:space="preserve">Inbound</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">676</context></context-group> </trans-unit> - <trans-unit id="_msg438"> + <trans-unit id="_msg446"> <source xml:space="preserve">Outbound</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">676</context></context-group> </trans-unit> - <trans-unit id="_msg439"> + <trans-unit id="_msg447"> <source xml:space="preserve">Full Relay</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">680</context></context-group> </trans-unit> - <trans-unit id="_msg440"> + <trans-unit id="_msg448"> <source xml:space="preserve">Block Relay</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">681</context></context-group> </trans-unit> - <trans-unit id="_msg441"> + <trans-unit id="_msg449"> <source xml:space="preserve">Manual</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">682</context></context-group> </trans-unit> - <trans-unit id="_msg442"> + <trans-unit id="_msg450"> <source xml:space="preserve">Feeler</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">683</context></context-group> </trans-unit> - <trans-unit id="_msg443"> + <trans-unit id="_msg451"> <source xml:space="preserve">Address Fetch</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">684</context></context-group> </trans-unit> - <trans-unit id="_msg444"> + <trans-unit id="_msg452"> <source xml:space="preserve">%1 d</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">698</context></context-group> </trans-unit> - <trans-unit id="_msg445"> + <trans-unit id="_msg453"> <source xml:space="preserve">%1 h</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">700</context></context-group> </trans-unit> - <trans-unit id="_msg446"> + <trans-unit id="_msg454"> <source xml:space="preserve">%1 m</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">702</context></context-group> </trans-unit> - <trans-unit id="_msg447"> + <trans-unit id="_msg455"> <source xml:space="preserve">%1 s</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">704</context></context-group> <context-group purpose="location"><context context-type="linenumber">732</context></context-group> </trans-unit> - <trans-unit id="_msg448"> + <trans-unit id="_msg456"> <source xml:space="preserve">None</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">720</context></context-group> </trans-unit> - <trans-unit id="_msg449"> + <trans-unit id="_msg457"> <source xml:space="preserve">N/A</source> <target xml:space="preserve" state="needs-review-translation">N/A</target> <context-group purpose="location"><context context-type="linenumber">726</context></context-group> </trans-unit> - <trans-unit id="_msg450"> + <trans-unit id="_msg458"> <source xml:space="preserve">%1 ms</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">727</context></context-group> </trans-unit> <group restype="x-gettext-plurals"> <context-group purpose="location"><context context-type="linenumber">745</context></context-group> - <trans-unit id="_msg451[0]" approved="yes"> + <trans-unit id="_msg459[0]" approved="yes"> <source xml:space="preserve">%n second(s)</source> <target xml:space="preserve">%n second</target> </trans-unit> - <trans-unit id="_msg451[1]" approved="yes"> + <trans-unit id="_msg459[1]" approved="yes"> <source xml:space="preserve">%n second(s)</source> <target xml:space="preserve">%n seconds</target> </trans-unit> </group> <group restype="x-gettext-plurals"> <context-group purpose="location"><context context-type="linenumber">749</context></context-group> - <trans-unit id="_msg452[0]" approved="yes"> + <trans-unit id="_msg460[0]" approved="yes"> <source xml:space="preserve">%n minute(s)</source> <target xml:space="preserve">%n minute</target> </trans-unit> - <trans-unit id="_msg452[1]" approved="yes"> + <trans-unit id="_msg460[1]" approved="yes"> <source xml:space="preserve">%n minute(s)</source> <target xml:space="preserve">%n minutes</target> </trans-unit> </group> <group restype="x-gettext-plurals"> <context-group purpose="location"><context context-type="linenumber">753</context></context-group> - <trans-unit id="_msg453[0]"> + <trans-unit id="_msg461[0]"> <source xml:space="preserve">%n hour(s)</source> <target xml:space="preserve" state="needs-review-translation">%n hour</target> </trans-unit> - <trans-unit id="_msg453[1]"> + <trans-unit id="_msg461[1]"> <source xml:space="preserve">%n hour(s)</source> <target xml:space="preserve" state="needs-review-translation">%n hours</target> </trans-unit> </group> <group restype="x-gettext-plurals"> <context-group purpose="location"><context context-type="linenumber">757</context></context-group> - <trans-unit id="_msg454[0]"> + <trans-unit id="_msg462[0]"> <source xml:space="preserve">%n day(s)</source> <target xml:space="preserve" state="needs-review-translation">%n day</target> </trans-unit> - <trans-unit id="_msg454[1]"> + <trans-unit id="_msg462[1]"> <source xml:space="preserve">%n day(s)</source> <target xml:space="preserve" state="needs-review-translation">%n days</target> </trans-unit> @@ -2505,47 +2547,47 @@ If you are receiving this error you should request the merchant provide a BIP21 <group restype="x-gettext-plurals"> <context-group purpose="location"><context context-type="linenumber">761</context></context-group> <context-group purpose="location"><context context-type="linenumber">767</context></context-group> - <trans-unit id="_msg455[0]"> + <trans-unit id="_msg463[0]"> <source xml:space="preserve">%n week(s)</source> <target xml:space="preserve" state="needs-review-translation">%n week</target> </trans-unit> - <trans-unit id="_msg455[1]"> + <trans-unit id="_msg463[1]"> <source xml:space="preserve">%n week(s)</source> <target xml:space="preserve" state="needs-review-translation">%n weeks</target> </trans-unit> </group> - <trans-unit id="_msg456"> + <trans-unit id="_msg464"> <source xml:space="preserve">%1 and %2</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">767</context></context-group> </trans-unit> <group restype="x-gettext-plurals"> <context-group purpose="location"><context context-type="linenumber">767</context></context-group> - <trans-unit id="_msg457[0]"> + <trans-unit id="_msg465[0]"> <source xml:space="preserve">%n year(s)</source> <target xml:space="preserve" state="needs-review-translation">%n year</target> </trans-unit> - <trans-unit id="_msg457[1]"> + <trans-unit id="_msg465[1]"> <source xml:space="preserve">%n year(s)</source> <target xml:space="preserve" state="needs-review-translation">%n years</target> </trans-unit> </group> - <trans-unit id="_msg458"> + <trans-unit id="_msg466"> <source xml:space="preserve">%1 B</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">775</context></context-group> </trans-unit> - <trans-unit id="_msg459"> + <trans-unit id="_msg467"> <source xml:space="preserve">%1 kB</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">777</context></context-group> </trans-unit> - <trans-unit id="_msg460"> + <trans-unit id="_msg468"> <source xml:space="preserve">%1 MB</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">779</context></context-group> </trans-unit> - <trans-unit id="_msg461"> + <trans-unit id="_msg469"> <source xml:space="preserve">%1 GB</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">781</context></context-group> @@ -2554,37 +2596,37 @@ If you are receiving this error you should request the merchant provide a BIP21 </body></file> <file original="../qrimagewidget.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="QRImageWidget"> - <trans-unit id="_msg462"> - <source xml:space="preserve">Save Image…</source> + <trans-unit id="_msg470"> + <source xml:space="preserve">&Save Image…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">30</context></context-group> </trans-unit> - <trans-unit id="_msg463"> - <source xml:space="preserve">Copy Image</source> + <trans-unit id="_msg471"> + <source xml:space="preserve">&Copy Image</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">31</context></context-group> </trans-unit> - <trans-unit id="_msg464"> + <trans-unit id="_msg472"> <source xml:space="preserve">Resulting URI too long, try to reduce the text for label / message.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">42</context></context-group> </trans-unit> - <trans-unit id="_msg465"> + <trans-unit id="_msg473"> <source xml:space="preserve">Error encoding URI into QR Code.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">49</context></context-group> </trans-unit> - <trans-unit id="_msg466"> + <trans-unit id="_msg474"> <source xml:space="preserve">QR code support not available.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">90</context></context-group> </trans-unit> - <trans-unit id="_msg467"> + <trans-unit id="_msg475"> <source xml:space="preserve">Save QR Code</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">120</context></context-group> </trans-unit> - <trans-unit id="_msg468"> + <trans-unit id="_msg476"> <source xml:space="preserve">PNG Image</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">123</context></context-group> @@ -2594,7 +2636,7 @@ If you are receiving this error you should request the merchant provide a BIP21 </body></file> <file original="../forms/debugwindow.ui" datatype="x-trolltech-designer-ui" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="RPCConsole"> - <trans-unit id="_msg469" approved="yes"> + <trans-unit id="_msg477" approved="yes"> <source xml:space="preserve">N/A</source> <target xml:space="preserve">N/A</target> <context-group purpose="location"><context context-type="linenumber">75</context></context-group> @@ -2608,480 +2650,480 @@ If you are receiving this error you should request the merchant provide a BIP21 <context-group purpose="location"><context context-type="linenumber">300</context></context-group> <context-group purpose="location"><context context-type="linenumber">336</context></context-group> <context-group purpose="location"><context context-type="linenumber">359</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1081</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1107</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1133</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1156</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1179</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1202</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1231</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1257</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1280</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1303</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1326</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1349</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1375</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1401</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1424</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1447</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1470</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1493</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1516</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1542</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1565</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1588</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1614</context></context-group> - <context-group purpose="location"><context context-type="sourcefile">../rpcconsole.h</context><context context-type="linenumber">138</context></context-group> - </trans-unit> - <trans-unit id="_msg470" approved="yes"> + <context-group purpose="location"><context context-type="linenumber">1051</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1077</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1103</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1126</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1149</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1172</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1201</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1227</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1250</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1273</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1296</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1319</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1345</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1371</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1394</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1417</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1440</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1463</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1486</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1512</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1535</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1558</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1584</context></context-group> + <context-group purpose="location"><context context-type="sourcefile">../rpcconsole.h</context><context context-type="linenumber">139</context></context-group> + </trans-unit> + <trans-unit id="_msg478" approved="yes"> <source xml:space="preserve">Client version</source> <target xml:space="preserve">Client version</target> <context-group purpose="location"><context context-type="linenumber">65</context></context-group> </trans-unit> - <trans-unit id="_msg471" approved="yes"> + <trans-unit id="_msg479" approved="yes"> <source xml:space="preserve">&Information</source> <target xml:space="preserve">&Information</target> <context-group purpose="location"><context context-type="linenumber">43</context></context-group> </trans-unit> - <trans-unit id="_msg472"> + <trans-unit id="_msg480"> <source xml:space="preserve">General</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">58</context></context-group> </trans-unit> - <trans-unit id="_msg473"> + <trans-unit id="_msg481"> <source xml:space="preserve">Datadir</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">114</context></context-group> </trans-unit> - <trans-unit id="_msg474"> + <trans-unit id="_msg482"> <source xml:space="preserve">To specify a non-default location of the data directory use the '%1' option.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">124</context></context-group> </trans-unit> - <trans-unit id="_msg475"> + <trans-unit id="_msg483"> <source xml:space="preserve">Blocksdir</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">143</context></context-group> </trans-unit> - <trans-unit id="_msg476"> + <trans-unit id="_msg484"> <source xml:space="preserve">To specify a non-default location of the blocks directory use the '%1' option.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">153</context></context-group> </trans-unit> - <trans-unit id="_msg477" approved="yes"> + <trans-unit id="_msg485" approved="yes"> <source xml:space="preserve">Startup time</source> <target xml:space="preserve">Startup time</target> <context-group purpose="location"><context context-type="linenumber">172</context></context-group> </trans-unit> - <trans-unit id="_msg478" approved="yes"> + <trans-unit id="_msg486" approved="yes"> <source xml:space="preserve">Network</source> <target xml:space="preserve">Network</target> <context-group purpose="location"><context context-type="linenumber">201</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1123</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1093</context></context-group> </trans-unit> - <trans-unit id="_msg479"> + <trans-unit id="_msg487"> <source xml:space="preserve">Name</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">208</context></context-group> </trans-unit> - <trans-unit id="_msg480" approved="yes"> + <trans-unit id="_msg488" approved="yes"> <source xml:space="preserve">Number of connections</source> <target xml:space="preserve">Number of connections</target> <context-group purpose="location"><context context-type="linenumber">231</context></context-group> </trans-unit> - <trans-unit id="_msg481" approved="yes"> + <trans-unit id="_msg489" approved="yes"> <source xml:space="preserve">Block chain</source> <target xml:space="preserve">Block chain</target> <context-group purpose="location"><context context-type="linenumber">260</context></context-group> </trans-unit> - <trans-unit id="_msg482"> + <trans-unit id="_msg490"> <source xml:space="preserve">Memory Pool</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">319</context></context-group> </trans-unit> - <trans-unit id="_msg483"> + <trans-unit id="_msg491"> <source xml:space="preserve">Current number of transactions</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">326</context></context-group> </trans-unit> - <trans-unit id="_msg484"> + <trans-unit id="_msg492"> <source xml:space="preserve">Memory usage</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">349</context></context-group> </trans-unit> - <trans-unit id="_msg485"> + <trans-unit id="_msg493"> <source xml:space="preserve">Wallet: </source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">443</context></context-group> </trans-unit> - <trans-unit id="_msg486"> + <trans-unit id="_msg494"> <source xml:space="preserve">(none)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">454</context></context-group> </trans-unit> - <trans-unit id="_msg487"> + <trans-unit id="_msg495"> <source xml:space="preserve">&Reset</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">695</context></context-group> + <context-group purpose="location"><context context-type="linenumber">665</context></context-group> </trans-unit> - <trans-unit id="_msg488"> + <trans-unit id="_msg496"> <source xml:space="preserve">Received</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">775</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1483</context></context-group> + <context-group purpose="location"><context context-type="linenumber">745</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1453</context></context-group> </trans-unit> - <trans-unit id="_msg489"> + <trans-unit id="_msg497"> <source xml:space="preserve">Sent</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">855</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1460</context></context-group> + <context-group purpose="location"><context context-type="linenumber">825</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1430</context></context-group> </trans-unit> - <trans-unit id="_msg490"> + <trans-unit id="_msg498"> <source xml:space="preserve">&Peers</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">896</context></context-group> + <context-group purpose="location"><context context-type="linenumber">866</context></context-group> </trans-unit> - <trans-unit id="_msg491"> + <trans-unit id="_msg499"> <source xml:space="preserve">Banned peers</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">972</context></context-group> + <context-group purpose="location"><context context-type="linenumber">942</context></context-group> </trans-unit> - <trans-unit id="_msg492"> + <trans-unit id="_msg500"> <source xml:space="preserve">Select a peer to view detailed information.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1040</context></context-group> - <context-group purpose="location"><context context-type="sourcefile">../rpcconsole.cpp</context><context context-type="linenumber">1091</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1010</context></context-group> + <context-group purpose="location"><context context-type="sourcefile">../rpcconsole.cpp</context><context context-type="linenumber">1124</context></context-group> </trans-unit> - <trans-unit id="_msg493"> + <trans-unit id="_msg501"> <source xml:space="preserve">Version</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1146</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1116</context></context-group> </trans-unit> - <trans-unit id="_msg494"> + <trans-unit id="_msg502"> <source xml:space="preserve">Starting Block</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1270</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1240</context></context-group> </trans-unit> - <trans-unit id="_msg495"> + <trans-unit id="_msg503"> <source xml:space="preserve">Synced Headers</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1293</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1263</context></context-group> </trans-unit> - <trans-unit id="_msg496"> + <trans-unit id="_msg504"> <source xml:space="preserve">Synced Blocks</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1316</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1286</context></context-group> </trans-unit> - <trans-unit id="_msg497"> + <trans-unit id="_msg505"> <source xml:space="preserve">The mapped Autonomous System used for diversifying peer selection.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1601</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1571</context></context-group> </trans-unit> - <trans-unit id="_msg498"> + <trans-unit id="_msg506"> <source xml:space="preserve">Mapped AS</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1604</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1574</context></context-group> </trans-unit> - <trans-unit id="_msg499"> + <trans-unit id="_msg507"> <source xml:space="preserve">User Agent</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">88</context></context-group> - <context-group purpose="location"><context context-type="linenumber">1169</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1139</context></context-group> </trans-unit> - <trans-unit id="_msg500"> + <trans-unit id="_msg508"> <source xml:space="preserve">Node window</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">14</context></context-group> </trans-unit> - <trans-unit id="_msg501"> + <trans-unit id="_msg509"> <source xml:space="preserve">Current block height</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">267</context></context-group> </trans-unit> - <trans-unit id="_msg502"> + <trans-unit id="_msg510"> <source xml:space="preserve">Open the %1 debug log file from the current data directory. This can take a few seconds for large log files.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">397</context></context-group> </trans-unit> - <trans-unit id="_msg503"> + <trans-unit id="_msg511"> <source xml:space="preserve">Decrease font size</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">481</context></context-group> + <context-group purpose="location"><context context-type="linenumber">475</context></context-group> </trans-unit> - <trans-unit id="_msg504"> + <trans-unit id="_msg512"> <source xml:space="preserve">Increase font size</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">513</context></context-group> + <context-group purpose="location"><context context-type="linenumber">495</context></context-group> </trans-unit> - <trans-unit id="_msg505"> + <trans-unit id="_msg513"> <source xml:space="preserve">Permissions</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1071</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1041</context></context-group> </trans-unit> - <trans-unit id="_msg506"> + <trans-unit id="_msg514"> <source xml:space="preserve">The direction and type of peer connection: %1</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1094</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1064</context></context-group> </trans-unit> - <trans-unit id="_msg507"> + <trans-unit id="_msg515"> <source xml:space="preserve">Direction/Type</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1097</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1067</context></context-group> </trans-unit> - <trans-unit id="_msg508"> + <trans-unit id="_msg516"> <source xml:space="preserve">The network protocol this peer is connected through: IPv4, IPv6, Onion, I2P, or CJDNS.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1120</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1090</context></context-group> </trans-unit> - <trans-unit id="_msg509"> + <trans-unit id="_msg517"> <source xml:space="preserve">Services</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1192</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1162</context></context-group> </trans-unit> - <trans-unit id="_msg510"> + <trans-unit id="_msg518"> <source xml:space="preserve">Whether the peer requested us to relay transactions.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1218</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1188</context></context-group> </trans-unit> - <trans-unit id="_msg511"> + <trans-unit id="_msg519"> <source xml:space="preserve">Wants Tx Relay</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1221</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1191</context></context-group> </trans-unit> - <trans-unit id="_msg512"> + <trans-unit id="_msg520"> <source xml:space="preserve">High bandwidth BIP152 compact block relay: %1</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1244</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1214</context></context-group> </trans-unit> - <trans-unit id="_msg513"> + <trans-unit id="_msg521"> <source xml:space="preserve">High Bandwidth</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1247</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1217</context></context-group> </trans-unit> - <trans-unit id="_msg514"> + <trans-unit id="_msg522"> <source xml:space="preserve">Connection Time</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1339</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1309</context></context-group> </trans-unit> - <trans-unit id="_msg515"> + <trans-unit id="_msg523"> <source xml:space="preserve">Elapsed time since a novel block passing initial validity checks was received from this peer.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1362</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1332</context></context-group> </trans-unit> - <trans-unit id="_msg516"> + <trans-unit id="_msg524"> <source xml:space="preserve">Last Block</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1365</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1335</context></context-group> </trans-unit> - <trans-unit id="_msg517"> + <trans-unit id="_msg525"> <source xml:space="preserve">Elapsed time since a novel transaction accepted into our mempool was received from this peer.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1388</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1358</context></context-group> </trans-unit> - <trans-unit id="_msg518"> + <trans-unit id="_msg526"> <source xml:space="preserve">Last Tx</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1391</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1361</context></context-group> </trans-unit> - <trans-unit id="_msg519"> + <trans-unit id="_msg527"> <source xml:space="preserve">Last Send</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1414</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1384</context></context-group> </trans-unit> - <trans-unit id="_msg520"> + <trans-unit id="_msg528"> <source xml:space="preserve">Last Receive</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1437</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1407</context></context-group> </trans-unit> - <trans-unit id="_msg521"> + <trans-unit id="_msg529"> <source xml:space="preserve">Ping Time</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1506</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1476</context></context-group> </trans-unit> - <trans-unit id="_msg522"> + <trans-unit id="_msg530"> <source xml:space="preserve">The duration of a currently outstanding ping.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1529</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1499</context></context-group> </trans-unit> - <trans-unit id="_msg523"> + <trans-unit id="_msg531"> <source xml:space="preserve">Ping Wait</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1532</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1502</context></context-group> </trans-unit> - <trans-unit id="_msg524"> + <trans-unit id="_msg532"> <source xml:space="preserve">Min Ping</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1555</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1525</context></context-group> </trans-unit> - <trans-unit id="_msg525"> + <trans-unit id="_msg533"> <source xml:space="preserve">Time Offset</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1578</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1548</context></context-group> </trans-unit> - <trans-unit id="_msg526" approved="yes"> + <trans-unit id="_msg534" approved="yes"> <source xml:space="preserve">Last block time</source> <target xml:space="preserve">Last block time</target> <context-group purpose="location"><context context-type="linenumber">290</context></context-group> </trans-unit> - <trans-unit id="_msg527" approved="yes"> + <trans-unit id="_msg535" approved="yes"> <source xml:space="preserve">&Open</source> <target xml:space="preserve">&Open</target> <context-group purpose="location"><context context-type="linenumber">400</context></context-group> </trans-unit> - <trans-unit id="_msg528" approved="yes"> + <trans-unit id="_msg536" approved="yes"> <source xml:space="preserve">&Console</source> <target xml:space="preserve">&Console</target> <context-group purpose="location"><context context-type="linenumber">426</context></context-group> </trans-unit> - <trans-unit id="_msg529"> + <trans-unit id="_msg537"> <source xml:space="preserve">&Network Traffic</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">643</context></context-group> + <context-group purpose="location"><context context-type="linenumber">613</context></context-group> </trans-unit> - <trans-unit id="_msg530"> + <trans-unit id="_msg538"> <source xml:space="preserve">Totals</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">711</context></context-group> + <context-group purpose="location"><context context-type="linenumber">681</context></context-group> </trans-unit> - <trans-unit id="_msg531" approved="yes"> + <trans-unit id="_msg539" approved="yes"> <source xml:space="preserve">Debug log file</source> <target xml:space="preserve">Debug log file</target> <context-group purpose="location"><context context-type="linenumber">390</context></context-group> </trans-unit> - <trans-unit id="_msg532" approved="yes"> + <trans-unit id="_msg540" approved="yes"> <source xml:space="preserve">Clear console</source> <target xml:space="preserve">Clear console</target> - <context-group purpose="location"><context context-type="linenumber">545</context></context-group> + <context-group purpose="location"><context context-type="linenumber">515</context></context-group> </trans-unit> </group> </body></file> <file original="../rpcconsole.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="RPCConsole"> - <trans-unit id="_msg533"> + <trans-unit id="_msg541"> <source xml:space="preserve">In:</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">890</context></context-group> + <context-group purpose="location"><context context-type="linenumber">923</context></context-group> </trans-unit> - <trans-unit id="_msg534"> + <trans-unit id="_msg542"> <source xml:space="preserve">Out:</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">891</context></context-group> + <context-group purpose="location"><context context-type="linenumber">924</context></context-group> </trans-unit> - <trans-unit id="_msg535"> + <trans-unit id="_msg543"> <source xml:space="preserve">Inbound: initiated by peer</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">474</context></context-group> + <context-group purpose="location"><context context-type="linenumber">495</context></context-group> </trans-unit> - <trans-unit id="_msg536"> + <trans-unit id="_msg544"> <source xml:space="preserve">Outbound Full Relay: default</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">475</context></context-group> + <context-group purpose="location"><context context-type="linenumber">496</context></context-group> </trans-unit> - <trans-unit id="_msg537"> + <trans-unit id="_msg545"> <source xml:space="preserve">Outbound Block Relay: does not relay transactions or addresses</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">476</context></context-group> + <context-group purpose="location"><context context-type="linenumber">497</context></context-group> </trans-unit> - <trans-unit id="_msg538"> + <trans-unit id="_msg546"> <source xml:space="preserve">Outbound Manual: added using RPC %1 or %2/%3 configuration options</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">477</context></context-group> + <context-group purpose="location"><context context-type="linenumber">498</context></context-group> </trans-unit> - <trans-unit id="_msg539"> + <trans-unit id="_msg547"> <source xml:space="preserve">Outbound Feeler: short-lived, for testing addresses</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">481</context></context-group> + <context-group purpose="location"><context context-type="linenumber">502</context></context-group> </trans-unit> - <trans-unit id="_msg540"> + <trans-unit id="_msg548"> <source xml:space="preserve">Outbound Address Fetch: short-lived, for soliciting addresses</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">482</context></context-group> + <context-group purpose="location"><context context-type="linenumber">503</context></context-group> </trans-unit> - <trans-unit id="_msg541"> + <trans-unit id="_msg549"> <source xml:space="preserve">we selected the peer for high bandwidth relay</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">486</context></context-group> + <context-group purpose="location"><context context-type="linenumber">507</context></context-group> </trans-unit> - <trans-unit id="_msg542"> + <trans-unit id="_msg550"> <source xml:space="preserve">the peer selected us for high bandwidth relay</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">487</context></context-group> + <context-group purpose="location"><context context-type="linenumber">508</context></context-group> </trans-unit> - <trans-unit id="_msg543"> + <trans-unit id="_msg551"> <source xml:space="preserve">no high bandwidth relay selected</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">488</context></context-group> + <context-group purpose="location"><context context-type="linenumber">509</context></context-group> </trans-unit> - <trans-unit id="_msg544"> + <trans-unit id="_msg552"> <source xml:space="preserve">Ctrl++</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">501</context></context-group> + <context-group purpose="location"><context context-type="linenumber">522</context></context-group> <note annotates="source" from="developer">Main shortcut to increase the RPC console font size.</note> </trans-unit> - <trans-unit id="_msg545"> + <trans-unit id="_msg553"> <source xml:space="preserve">Ctrl+=</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">503</context></context-group> + <context-group purpose="location"><context context-type="linenumber">524</context></context-group> <note annotates="source" from="developer">Secondary shortcut to increase the RPC console font size.</note> </trans-unit> - <trans-unit id="_msg546"> + <trans-unit id="_msg554"> <source xml:space="preserve">Ctrl+-</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">507</context></context-group> + <context-group purpose="location"><context context-type="linenumber">528</context></context-group> <note annotates="source" from="developer">Main shortcut to decrease the RPC console font size.</note> </trans-unit> - <trans-unit id="_msg547"> + <trans-unit id="_msg555"> <source xml:space="preserve">Ctrl+_</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">509</context></context-group> + <context-group purpose="location"><context context-type="linenumber">530</context></context-group> <note annotates="source" from="developer">Secondary shortcut to decrease the RPC console font size.</note> </trans-unit> - <trans-unit id="_msg548"> - <source xml:space="preserve">Network activity disabled</source> + <trans-unit id="_msg556"> + <source xml:space="preserve">&Disconnect</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">894</context></context-group> + <context-group purpose="location"><context context-type="linenumber">680</context></context-group> </trans-unit> - <trans-unit id="_msg549"> - <source xml:space="preserve">Executing command without any wallet</source> + <trans-unit id="_msg557"> + <source xml:space="preserve">1 &hour</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">971</context></context-group> + <context-group purpose="location"><context context-type="linenumber">681</context></context-group> </trans-unit> - <trans-unit id="_msg550"> - <source xml:space="preserve">Executing command using "%1" wallet</source> + <trans-unit id="_msg558"> + <source xml:space="preserve">1 d&ay</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">969</context></context-group> + <context-group purpose="location"><context context-type="linenumber">682</context></context-group> </trans-unit> - <trans-unit id="_msg551"> - <source xml:space="preserve">Disconnect</source> + <trans-unit id="_msg559"> + <source xml:space="preserve">1 &week</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">650</context></context-group> + <context-group purpose="location"><context context-type="linenumber">683</context></context-group> </trans-unit> - <trans-unit id="_msg552"> - <source xml:space="preserve">1 hour</source> + <trans-unit id="_msg560"> + <source xml:space="preserve">1 &year</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">651</context></context-group> + <context-group purpose="location"><context context-type="linenumber">684</context></context-group> </trans-unit> - <trans-unit id="_msg553"> - <source xml:space="preserve">1 day</source> + <trans-unit id="_msg561"> + <source xml:space="preserve">&Unban</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">652</context></context-group> + <context-group purpose="location"><context context-type="linenumber">706</context></context-group> </trans-unit> - <trans-unit id="_msg554"> - <source xml:space="preserve">1 week</source> + <trans-unit id="_msg562"> + <source xml:space="preserve">Network activity disabled</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">653</context></context-group> + <context-group purpose="location"><context context-type="linenumber">927</context></context-group> </trans-unit> - <trans-unit id="_msg555"> - <source xml:space="preserve">1 year</source> + <trans-unit id="_msg563"> + <source xml:space="preserve">Executing command without any wallet</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">654</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1004</context></context-group> </trans-unit> - <trans-unit id="_msg556"> - <source xml:space="preserve">Unban</source> + <trans-unit id="_msg564"> + <source xml:space="preserve">Executing command using "%1" wallet</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">673</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1002</context></context-group> </trans-unit> - <trans-unit id="_msg557"> + <trans-unit id="_msg565"> <source xml:space="preserve">Welcome to the %1 RPC console. Use up and down arrows to navigate history, and %2 to clear screen. Use %3 and %4 to increase or decrease the font size. @@ -3090,160 +3132,160 @@ For more information on using this console, type %6. %7WARNING: Scammers have been active, telling users to type commands here, stealing their wallet contents. Do not use this console without fully understanding the ramifications of a command.%8</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">823</context></context-group> + <context-group purpose="location"><context context-type="linenumber">856</context></context-group> <note annotates="source" from="developer">RPC console welcome message. Placeholders %7 and %8 are style tags for the warning content, and they are not space separated from the rest of the text intentionally.</note> </trans-unit> - <trans-unit id="_msg558"> + <trans-unit id="_msg566"> <source xml:space="preserve">Executing…</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">979</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1012</context></context-group> <note annotates="source" from="developer">A console message indicating an entered command is currently being executed.</note> </trans-unit> - <trans-unit id="_msg559"> + <trans-unit id="_msg567"> <source xml:space="preserve">(peer: %1)</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1097</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1130</context></context-group> </trans-unit> - <trans-unit id="_msg560"> + <trans-unit id="_msg568"> <source xml:space="preserve">via %1</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">1099</context></context-group> + <context-group purpose="location"><context context-type="linenumber">1132</context></context-group> </trans-unit> </group> </body></file> <file original="../rpcconsole.h" datatype="c" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="RPCConsole"> - <trans-unit id="_msg561"> + <trans-unit id="_msg569"> <source xml:space="preserve">Yes</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">137</context></context-group> + <context-group purpose="location"><context context-type="linenumber">138</context></context-group> </trans-unit> - <trans-unit id="_msg562"> + <trans-unit id="_msg570"> <source xml:space="preserve">No</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">137</context></context-group> + <context-group purpose="location"><context context-type="linenumber">138</context></context-group> </trans-unit> - <trans-unit id="_msg563"> + <trans-unit id="_msg571"> <source xml:space="preserve">To</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">137</context></context-group> + <context-group purpose="location"><context context-type="linenumber">138</context></context-group> </trans-unit> - <trans-unit id="_msg564"> + <trans-unit id="_msg572"> <source xml:space="preserve">From</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">137</context></context-group> + <context-group purpose="location"><context context-type="linenumber">138</context></context-group> </trans-unit> - <trans-unit id="_msg565"> + <trans-unit id="_msg573"> <source xml:space="preserve">Ban for</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">138</context></context-group> + <context-group purpose="location"><context context-type="linenumber">139</context></context-group> </trans-unit> - <trans-unit id="_msg566"> + <trans-unit id="_msg574"> <source xml:space="preserve">Never</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">176</context></context-group> + <context-group purpose="location"><context context-type="linenumber">179</context></context-group> </trans-unit> - <trans-unit id="_msg567"> + <trans-unit id="_msg575"> <source xml:space="preserve">Unknown</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">138</context></context-group> + <context-group purpose="location"><context context-type="linenumber">139</context></context-group> </trans-unit> </group> </body></file> <file original="../forms/receivecoinsdialog.ui" datatype="x-trolltech-designer-ui" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="ReceiveCoinsDialog"> - <trans-unit id="_msg568"> + <trans-unit id="_msg576"> <source xml:space="preserve">&Amount:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">37</context></context-group> </trans-unit> - <trans-unit id="_msg569"> + <trans-unit id="_msg577"> <source xml:space="preserve">&Label:</source> <target xml:space="preserve" state="needs-review-translation">&Label:</target> <context-group purpose="location"><context context-type="linenumber">83</context></context-group> </trans-unit> - <trans-unit id="_msg570"> + <trans-unit id="_msg578"> <source xml:space="preserve">&Message:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">53</context></context-group> </trans-unit> - <trans-unit id="_msg571"> + <trans-unit id="_msg579"> <source xml:space="preserve">An optional message to attach to the payment request, which will be displayed when the request is opened. Note: The message will not be sent with the payment over the Bitcoin network.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">50</context></context-group> </trans-unit> - <trans-unit id="_msg572"> + <trans-unit id="_msg580"> <source xml:space="preserve">An optional label to associate with the new receiving address.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">80</context></context-group> </trans-unit> - <trans-unit id="_msg573"> + <trans-unit id="_msg581"> <source xml:space="preserve">Use this form to request payments. All fields are <b>optional</b>.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">73</context></context-group> </trans-unit> - <trans-unit id="_msg574"> + <trans-unit id="_msg582"> <source xml:space="preserve">An optional amount to request. Leave this empty or zero to not request a specific amount.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">34</context></context-group> <context-group purpose="location"><context context-type="linenumber">193</context></context-group> </trans-unit> - <trans-unit id="_msg575"> + <trans-unit id="_msg583"> <source xml:space="preserve">An optional label to associate with the new receiving address (used by you to identify an invoice). It is also attached to the payment request.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">66</context></context-group> </trans-unit> - <trans-unit id="_msg576"> + <trans-unit id="_msg584"> <source xml:space="preserve">An optional message that is attached to the payment request and may be displayed to the sender.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">96</context></context-group> </trans-unit> - <trans-unit id="_msg577"> + <trans-unit id="_msg585"> <source xml:space="preserve">&Create new receiving address</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">111</context></context-group> </trans-unit> - <trans-unit id="_msg578"> + <trans-unit id="_msg586"> <source xml:space="preserve">Clear all fields of the form.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">134</context></context-group> </trans-unit> - <trans-unit id="_msg579"> + <trans-unit id="_msg587"> <source xml:space="preserve">Clear</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">137</context></context-group> </trans-unit> - <trans-unit id="_msg580"> + <trans-unit id="_msg588"> <source xml:space="preserve">Native segwit addresses (aka Bech32 or BIP-173) reduce your transaction fees later on and offer better protection against typos, but old wallets don't support them. When unchecked, an address compatible with older wallets will be created instead.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">215</context></context-group> </trans-unit> - <trans-unit id="_msg581"> + <trans-unit id="_msg589"> <source xml:space="preserve">Generate native segwit (Bech32) address</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">218</context></context-group> </trans-unit> - <trans-unit id="_msg582"> + <trans-unit id="_msg590"> <source xml:space="preserve">Requested payments history</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">279</context></context-group> </trans-unit> - <trans-unit id="_msg583"> + <trans-unit id="_msg591"> <source xml:space="preserve">Show the selected request (does the same as double clicking an entry)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">304</context></context-group> </trans-unit> - <trans-unit id="_msg584"> + <trans-unit id="_msg592"> <source xml:space="preserve">Show</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">307</context></context-group> </trans-unit> - <trans-unit id="_msg585"> + <trans-unit id="_msg593"> <source xml:space="preserve">Remove the selected entries from the list</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">324</context></context-group> </trans-unit> - <trans-unit id="_msg586"> + <trans-unit id="_msg594"> <source xml:space="preserve">Remove</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">327</context></context-group> @@ -3252,37 +3294,37 @@ For more information on using this console, type %6. </body></file> <file original="../receivecoinsdialog.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="ReceiveCoinsDialog"> - <trans-unit id="_msg587"> - <source xml:space="preserve">Copy URI</source> + <trans-unit id="_msg595"> + <source xml:space="preserve">Copy &URI</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">47</context></context-group> </trans-unit> - <trans-unit id="_msg588"> - <source xml:space="preserve">Copy address</source> + <trans-unit id="_msg596"> + <source xml:space="preserve">&Copy address</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">48</context></context-group> </trans-unit> - <trans-unit id="_msg589"> - <source xml:space="preserve">Copy label</source> + <trans-unit id="_msg597"> + <source xml:space="preserve">Copy &label</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">49</context></context-group> </trans-unit> - <trans-unit id="_msg590"> - <source xml:space="preserve">Copy message</source> + <trans-unit id="_msg598"> + <source xml:space="preserve">Copy &message</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">50</context></context-group> </trans-unit> - <trans-unit id="_msg591"> - <source xml:space="preserve">Copy amount</source> + <trans-unit id="_msg599"> + <source xml:space="preserve">Copy &amount</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">51</context></context-group> </trans-unit> - <trans-unit id="_msg592"> + <trans-unit id="_msg600"> <source xml:space="preserve">Could not unlock wallet.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">176</context></context-group> </trans-unit> - <trans-unit id="_msg593"> + <trans-unit id="_msg601"> <source xml:space="preserve">Could not generate new %1 address</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">181</context></context-group> @@ -3291,52 +3333,62 @@ For more information on using this console, type %6. </body></file> <file original="../forms/receiverequestdialog.ui" datatype="x-trolltech-designer-ui" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="ReceiveRequestDialog"> - <trans-unit id="_msg594"> + <trans-unit id="_msg602"> <source xml:space="preserve">Request payment to …</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">14</context></context-group> </trans-unit> - <trans-unit id="_msg595"> + <trans-unit id="_msg603"> <source xml:space="preserve">Address:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">90</context></context-group> </trans-unit> - <trans-unit id="_msg596"> + <trans-unit id="_msg604"> <source xml:space="preserve">Amount:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">119</context></context-group> </trans-unit> - <trans-unit id="_msg597"> + <trans-unit id="_msg605"> <source xml:space="preserve">Label:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">148</context></context-group> </trans-unit> - <trans-unit id="_msg598"> + <trans-unit id="_msg606"> <source xml:space="preserve">Message:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">180</context></context-group> </trans-unit> - <trans-unit id="_msg599"> + <trans-unit id="_msg607"> <source xml:space="preserve">Wallet:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">212</context></context-group> </trans-unit> - <trans-unit id="_msg600"> + <trans-unit id="_msg608"> <source xml:space="preserve">Copy &URI</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">240</context></context-group> </trans-unit> - <trans-unit id="_msg601"> + <trans-unit id="_msg609"> <source xml:space="preserve">Copy &Address</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">250</context></context-group> </trans-unit> - <trans-unit id="_msg602"> - <source xml:space="preserve">&Save Image…</source> + <trans-unit id="_msg610"> + <source xml:space="preserve">&Verify</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">260</context></context-group> </trans-unit> - <trans-unit id="_msg603"> + <trans-unit id="_msg611"> + <source xml:space="preserve">Verify this address on e.g. a hardware wallet screen</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">263</context></context-group> + </trans-unit> + <trans-unit id="_msg612"> + <source xml:space="preserve">&Save Image…</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">273</context></context-group> + </trans-unit> + <trans-unit id="_msg613"> <source xml:space="preserve">Payment information</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">39</context></context-group> @@ -3345,7 +3397,7 @@ For more information on using this console, type %6. </body></file> <file original="../receiverequestdialog.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="ReceiveRequestDialog"> - <trans-unit id="_msg604"> + <trans-unit id="_msg614"> <source xml:space="preserve">Request payment to %1</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">49</context></context-group> @@ -3354,229 +3406,229 @@ For more information on using this console, type %6. </body></file> <file original="../recentrequeststablemodel.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="RecentRequestsTableModel"> - <trans-unit id="_msg605"> + <trans-unit id="_msg615"> <source xml:space="preserve">Date</source> <target xml:space="preserve" state="needs-review-translation">Date</target> - <context-group purpose="location"><context context-type="linenumber">30</context></context-group> + <context-group purpose="location"><context context-type="linenumber">32</context></context-group> </trans-unit> - <trans-unit id="_msg606"> + <trans-unit id="_msg616"> <source xml:space="preserve">Label</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">30</context></context-group> + <context-group purpose="location"><context context-type="linenumber">32</context></context-group> </trans-unit> - <trans-unit id="_msg607"> + <trans-unit id="_msg617"> <source xml:space="preserve">Message</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">30</context></context-group> + <context-group purpose="location"><context context-type="linenumber">32</context></context-group> </trans-unit> - <trans-unit id="_msg608"> + <trans-unit id="_msg618"> <source xml:space="preserve">(no label)</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">71</context></context-group> + <context-group purpose="location"><context context-type="linenumber">73</context></context-group> </trans-unit> - <trans-unit id="_msg609"> + <trans-unit id="_msg619"> <source xml:space="preserve">(no message)</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">80</context></context-group> + <context-group purpose="location"><context context-type="linenumber">82</context></context-group> </trans-unit> - <trans-unit id="_msg610"> + <trans-unit id="_msg620"> <source xml:space="preserve">(no amount requested)</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">88</context></context-group> + <context-group purpose="location"><context context-type="linenumber">90</context></context-group> </trans-unit> - <trans-unit id="_msg611"> + <trans-unit id="_msg621"> <source xml:space="preserve">Requested</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">131</context></context-group> + <context-group purpose="location"><context context-type="linenumber">133</context></context-group> </trans-unit> </group> </body></file> <file original="../forms/sendcoinsdialog.ui" datatype="x-trolltech-designer-ui" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="SendCoinsDialog"> - <trans-unit id="_msg612" approved="yes"> + <trans-unit id="_msg622" approved="yes"> <source xml:space="preserve">Send Coins</source> <target xml:space="preserve">Send Coins</target> <context-group purpose="location"><context context-type="linenumber">14</context></context-group> - <context-group purpose="location"><context context-type="sourcefile">../sendcoinsdialog.cpp</context><context context-type="linenumber">674</context></context-group> + <context-group purpose="location"><context context-type="sourcefile">../sendcoinsdialog.cpp</context><context context-type="linenumber">738</context></context-group> </trans-unit> - <trans-unit id="_msg613"> + <trans-unit id="_msg623"> <source xml:space="preserve">Coin Control Features</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">90</context></context-group> </trans-unit> - <trans-unit id="_msg614"> + <trans-unit id="_msg624"> <source xml:space="preserve">automatically selected</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">120</context></context-group> </trans-unit> - <trans-unit id="_msg615"> + <trans-unit id="_msg625"> <source xml:space="preserve">Insufficient funds!</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">139</context></context-group> </trans-unit> - <trans-unit id="_msg616"> + <trans-unit id="_msg626"> <source xml:space="preserve">Quantity:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">228</context></context-group> </trans-unit> - <trans-unit id="_msg617"> + <trans-unit id="_msg627"> <source xml:space="preserve">Bytes:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">263</context></context-group> </trans-unit> - <trans-unit id="_msg618"> + <trans-unit id="_msg628"> <source xml:space="preserve">Amount:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">311</context></context-group> </trans-unit> - <trans-unit id="_msg619"> + <trans-unit id="_msg629"> <source xml:space="preserve">Fee:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">391</context></context-group> </trans-unit> - <trans-unit id="_msg620"> + <trans-unit id="_msg630"> <source xml:space="preserve">After Fee:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">442</context></context-group> </trans-unit> - <trans-unit id="_msg621"> + <trans-unit id="_msg631"> <source xml:space="preserve">Change:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">474</context></context-group> </trans-unit> - <trans-unit id="_msg622"> + <trans-unit id="_msg632"> <source xml:space="preserve">If this is activated, but the change address is empty or invalid, change will be sent to a newly generated address.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">518</context></context-group> </trans-unit> - <trans-unit id="_msg623"> + <trans-unit id="_msg633"> <source xml:space="preserve">Custom change address</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">521</context></context-group> </trans-unit> - <trans-unit id="_msg624"> + <trans-unit id="_msg634"> <source xml:space="preserve">Transaction Fee:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">727</context></context-group> </trans-unit> - <trans-unit id="_msg625"> + <trans-unit id="_msg635"> <source xml:space="preserve">Using the fallbackfee can result in sending a transaction that will take several hours or days (or never) to confirm. Consider choosing your fee manually or wait until you have validated the complete chain.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">765</context></context-group> </trans-unit> - <trans-unit id="_msg626"> + <trans-unit id="_msg636"> <source xml:space="preserve">Warning: Fee estimation is currently not possible.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">774</context></context-group> </trans-unit> - <trans-unit id="_msg627"> + <trans-unit id="_msg637"> <source xml:space="preserve">per kilobyte</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">856</context></context-group> </trans-unit> - <trans-unit id="_msg628"> + <trans-unit id="_msg638"> <source xml:space="preserve">Hide</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">803</context></context-group> </trans-unit> - <trans-unit id="_msg629"> + <trans-unit id="_msg639"> <source xml:space="preserve">Recommended:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">915</context></context-group> </trans-unit> - <trans-unit id="_msg630"> + <trans-unit id="_msg640"> <source xml:space="preserve">Custom:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">945</context></context-group> </trans-unit> - <trans-unit id="_msg631" approved="yes"> + <trans-unit id="_msg641" approved="yes"> <source xml:space="preserve">Send to multiple recipients at once</source> <target xml:space="preserve">Send to multiple recipients at once</target> <context-group purpose="location"><context context-type="linenumber">1160</context></context-group> </trans-unit> - <trans-unit id="_msg632" approved="yes"> + <trans-unit id="_msg642" approved="yes"> <source xml:space="preserve">Add &Recipient</source> <target xml:space="preserve">Add &Recipient</target> <context-group purpose="location"><context context-type="linenumber">1163</context></context-group> </trans-unit> - <trans-unit id="_msg633"> + <trans-unit id="_msg643"> <source xml:space="preserve">Clear all fields of the form.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">1143</context></context-group> </trans-unit> - <trans-unit id="_msg634"> + <trans-unit id="_msg644"> <source xml:space="preserve">Inputs…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">110</context></context-group> </trans-unit> - <trans-unit id="_msg635"> + <trans-unit id="_msg645"> <source xml:space="preserve">Dust:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">343</context></context-group> </trans-unit> - <trans-unit id="_msg636"> + <trans-unit id="_msg646"> <source xml:space="preserve">Choose…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">741</context></context-group> </trans-unit> - <trans-unit id="_msg637"> + <trans-unit id="_msg647"> <source xml:space="preserve">Hide transaction fee settings</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">800</context></context-group> </trans-unit> - <trans-unit id="_msg638"> + <trans-unit id="_msg648"> <source xml:space="preserve">Specify a custom fee per kB (1,000 bytes) of the transaction's virtual size. Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 satoshis per kvB" for a transaction size of 500 virtual bytes (half of 1 kvB) would ultimately yield a fee of only 50 satoshis.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">851</context></context-group> </trans-unit> - <trans-unit id="_msg639"> + <trans-unit id="_msg649"> <source xml:space="preserve">When there is less transaction volume than space in the blocks, miners as well as relaying nodes may enforce a minimum fee. Paying only this minimum fee is just fine, but be aware that this can result in a never confirming transaction once there is more demand for bitcoin transactions than the network can process.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">886</context></context-group> </trans-unit> - <trans-unit id="_msg640"> + <trans-unit id="_msg650"> <source xml:space="preserve">A too low fee might result in a never confirming transaction (read the tooltip)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">889</context></context-group> </trans-unit> - <trans-unit id="_msg641"> + <trans-unit id="_msg651"> <source xml:space="preserve">(Smart fee not initialized yet. This usually takes a few blocks…)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">994</context></context-group> </trans-unit> - <trans-unit id="_msg642"> + <trans-unit id="_msg652"> <source xml:space="preserve">Confirmation time target:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">1020</context></context-group> </trans-unit> - <trans-unit id="_msg643"> + <trans-unit id="_msg653"> <source xml:space="preserve">Enable Replace-By-Fee</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">1078</context></context-group> </trans-unit> - <trans-unit id="_msg644"> + <trans-unit id="_msg654"> <source xml:space="preserve">With Replace-By-Fee (BIP-125) you can increase a transaction's fee after it is sent. Without this, a higher fee may be recommended to compensate for increased transaction delay risk.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">1081</context></context-group> </trans-unit> - <trans-unit id="_msg645" approved="yes"> + <trans-unit id="_msg655" approved="yes"> <source xml:space="preserve">Clear &All</source> <target xml:space="preserve">Clear &All</target> <context-group purpose="location"><context context-type="linenumber">1146</context></context-group> </trans-unit> - <trans-unit id="_msg646" approved="yes"> + <trans-unit id="_msg656" approved="yes"> <source xml:space="preserve">Balance:</source> <target xml:space="preserve">Balance:</target> <context-group purpose="location"><context context-type="linenumber">1201</context></context-group> </trans-unit> - <trans-unit id="_msg647" approved="yes"> + <trans-unit id="_msg657" approved="yes"> <source xml:space="preserve">Confirm the send action</source> <target xml:space="preserve">Confirm the send action</target> <context-group purpose="location"><context context-type="linenumber">1117</context></context-group> </trans-unit> - <trans-unit id="_msg648" approved="yes"> + <trans-unit id="_msg658" approved="yes"> <source xml:space="preserve">S&end</source> <target xml:space="preserve">S&end</target> <context-group purpose="location"><context context-type="linenumber">1120</context></context-group> @@ -3585,344 +3637,383 @@ Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 </body></file> <file original="../sendcoinsdialog.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="SendCoinsDialog"> - <trans-unit id="_msg649"> + <trans-unit id="_msg659"> <source xml:space="preserve">Copy quantity</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">92</context></context-group> </trans-unit> - <trans-unit id="_msg650"> + <trans-unit id="_msg660"> <source xml:space="preserve">Copy amount</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">93</context></context-group> </trans-unit> - <trans-unit id="_msg651"> + <trans-unit id="_msg661"> <source xml:space="preserve">Copy fee</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">94</context></context-group> </trans-unit> - <trans-unit id="_msg652"> + <trans-unit id="_msg662"> <source xml:space="preserve">Copy after fee</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">95</context></context-group> </trans-unit> - <trans-unit id="_msg653"> + <trans-unit id="_msg663"> <source xml:space="preserve">Copy bytes</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">96</context></context-group> </trans-unit> - <trans-unit id="_msg654"> + <trans-unit id="_msg664"> <source xml:space="preserve">Copy dust</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">97</context></context-group> </trans-unit> - <trans-unit id="_msg655"> + <trans-unit id="_msg665"> <source xml:space="preserve">Copy change</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">98</context></context-group> </trans-unit> - <trans-unit id="_msg656"> + <trans-unit id="_msg666"> <source xml:space="preserve">%1 (%2 blocks)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">174</context></context-group> </trans-unit> - <trans-unit id="_msg657"> + <trans-unit id="_msg667"> + <source xml:space="preserve">Sign on device</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">204</context></context-group> + <note annotates="source" from="developer">"device" usually means a hardware wallet</note> + </trans-unit> + <trans-unit id="_msg668"> + <source xml:space="preserve">Connect your hardware wallet first.</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">207</context></context-group> + </trans-unit> + <trans-unit id="_msg669"> + <source xml:space="preserve">Set external signer script path in Options -> Wallet</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">211</context></context-group> + <note annotates="source" from="developer">"External signer" means using devices such as hardware wallets.</note> + </trans-unit> + <trans-unit id="_msg670"> <source xml:space="preserve">Cr&eate Unsigned</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">203</context></context-group> + <context-group purpose="location"><context context-type="linenumber">214</context></context-group> </trans-unit> - <trans-unit id="_msg658"> + <trans-unit id="_msg671"> <source xml:space="preserve">Creates a Partially Signed Bitcoin Transaction (PSBT) for use with e.g. an offline %1 wallet, or a PSBT-compatible hardware wallet.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">204</context></context-group> + <context-group purpose="location"><context context-type="linenumber">215</context></context-group> </trans-unit> - <trans-unit id="_msg659"> + <trans-unit id="_msg672"> <source xml:space="preserve"> from wallet '%1'</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">294</context></context-group> + <context-group purpose="location"><context context-type="linenumber">305</context></context-group> </trans-unit> - <trans-unit id="_msg660"> + <trans-unit id="_msg673"> <source xml:space="preserve">%1 to '%2'</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">305</context></context-group> + <context-group purpose="location"><context context-type="linenumber">316</context></context-group> </trans-unit> - <trans-unit id="_msg661"> + <trans-unit id="_msg674"> <source xml:space="preserve">%1 to %2</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">310</context></context-group> + <context-group purpose="location"><context context-type="linenumber">321</context></context-group> </trans-unit> - <trans-unit id="_msg662"> + <trans-unit id="_msg675"> <source xml:space="preserve">Do you want to draft this transaction?</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">317</context></context-group> + <context-group purpose="location"><context context-type="linenumber">328</context></context-group> </trans-unit> - <trans-unit id="_msg663"> + <trans-unit id="_msg676"> <source xml:space="preserve">Are you sure you want to send?</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">319</context></context-group> + <context-group purpose="location"><context context-type="linenumber">330</context></context-group> </trans-unit> - <trans-unit id="_msg664"> + <trans-unit id="_msg677"> <source xml:space="preserve">To review recipient list click "Show Details…"</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">371</context></context-group> + <context-group purpose="location"><context context-type="linenumber">382</context></context-group> </trans-unit> - <trans-unit id="_msg665"> + <trans-unit id="_msg678"> <source xml:space="preserve">Create Unsigned</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">390</context></context-group> + <context-group purpose="location"><context context-type="linenumber">401</context></context-group> </trans-unit> - <trans-unit id="_msg666"> + <trans-unit id="_msg679"> + <source xml:space="preserve">Sign and send</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">401</context></context-group> + </trans-unit> + <trans-unit id="_msg680"> + <source xml:space="preserve">Sign failed</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">426</context></context-group> + </trans-unit> + <trans-unit id="_msg681"> + <source xml:space="preserve">External signer not found</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">432</context></context-group> + <note annotates="source" from="developer">"External signer" means using devices such as hardware wallets.</note> + </trans-unit> + <trans-unit id="_msg682"> + <source xml:space="preserve">External signer failure</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">438</context></context-group> + <note annotates="source" from="developer">"External signer" means using devices such as hardware wallets.</note> + </trans-unit> + <trans-unit id="_msg683"> <source xml:space="preserve">Save Transaction Data</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">434</context></context-group> + <context-group purpose="location"><context context-type="linenumber">496</context></context-group> </trans-unit> - <trans-unit id="_msg667"> + <trans-unit id="_msg684"> <source xml:space="preserve">Partially Signed Transaction (Binary)</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">436</context></context-group> + <context-group purpose="location"><context context-type="linenumber">498</context></context-group> <note annotates="source" from="developer">Expanded name of the binary PSBT file format. See: BIP 174.</note> </trans-unit> - <trans-unit id="_msg668"> + <trans-unit id="_msg685"> <source xml:space="preserve">PSBT saved</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">443</context></context-group> + <context-group purpose="location"><context context-type="linenumber">505</context></context-group> </trans-unit> - <trans-unit id="_msg669"> + <trans-unit id="_msg686"> + <source xml:space="preserve">External balance:</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">680</context></context-group> + </trans-unit> + <trans-unit id="_msg687"> <source xml:space="preserve">or</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">367</context></context-group> + <context-group purpose="location"><context context-type="linenumber">378</context></context-group> </trans-unit> - <trans-unit id="_msg670"> + <trans-unit id="_msg688"> <source xml:space="preserve">You can increase the fee later (signals Replace-By-Fee, BIP-125).</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">348</context></context-group> + <context-group purpose="location"><context context-type="linenumber">359</context></context-group> </trans-unit> - <trans-unit id="_msg671"> + <trans-unit id="_msg689"> <source xml:space="preserve">Please, review your transaction proposal. This will produce a Partially Signed Bitcoin Transaction (PSBT) which you can save or copy and then sign with e.g. an offline %1 wallet, or a PSBT-compatible hardware wallet.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">324</context></context-group> + <context-group purpose="location"><context context-type="linenumber">335</context></context-group> </trans-unit> - <trans-unit id="_msg672"> + <trans-unit id="_msg690"> <source xml:space="preserve">Please, review your transaction.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">326</context></context-group> + <context-group purpose="location"><context context-type="linenumber">337</context></context-group> </trans-unit> - <trans-unit id="_msg673"> + <trans-unit id="_msg691"> <source xml:space="preserve">Transaction fee</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">334</context></context-group> + <context-group purpose="location"><context context-type="linenumber">345</context></context-group> </trans-unit> - <trans-unit id="_msg674"> + <trans-unit id="_msg692"> <source xml:space="preserve">Not signalling Replace-By-Fee, BIP-125.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">350</context></context-group> + <context-group purpose="location"><context context-type="linenumber">361</context></context-group> </trans-unit> - <trans-unit id="_msg675"> + <trans-unit id="_msg693"> <source xml:space="preserve">Total Amount</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">364</context></context-group> + <context-group purpose="location"><context context-type="linenumber">375</context></context-group> </trans-unit> - <trans-unit id="_msg676"> + <trans-unit id="_msg694"> <source xml:space="preserve">Confirm send coins</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">389</context></context-group> + <context-group purpose="location"><context context-type="linenumber">400</context></context-group> </trans-unit> - <trans-unit id="_msg677"> + <trans-unit id="_msg695"> <source xml:space="preserve">Confirm transaction proposal</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">389</context></context-group> - </trans-unit> - <trans-unit id="_msg678"> - <source xml:space="preserve">Send</source> - <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">390</context></context-group> + <context-group purpose="location"><context context-type="linenumber">400</context></context-group> </trans-unit> - <trans-unit id="_msg679"> + <trans-unit id="_msg696"> <source xml:space="preserve">Watch-only balance:</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">619</context></context-group> + <context-group purpose="location"><context context-type="linenumber">683</context></context-group> </trans-unit> - <trans-unit id="_msg680"> + <trans-unit id="_msg697"> <source xml:space="preserve">The recipient address is not valid. Please recheck.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">643</context></context-group> + <context-group purpose="location"><context context-type="linenumber">707</context></context-group> </trans-unit> - <trans-unit id="_msg681"> + <trans-unit id="_msg698"> <source xml:space="preserve">The amount to pay must be larger than 0.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">646</context></context-group> + <context-group purpose="location"><context context-type="linenumber">710</context></context-group> </trans-unit> - <trans-unit id="_msg682"> + <trans-unit id="_msg699"> <source xml:space="preserve">The amount exceeds your balance.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">649</context></context-group> + <context-group purpose="location"><context context-type="linenumber">713</context></context-group> </trans-unit> - <trans-unit id="_msg683"> + <trans-unit id="_msg700"> <source xml:space="preserve">The total exceeds your balance when the %1 transaction fee is included.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">652</context></context-group> + <context-group purpose="location"><context context-type="linenumber">716</context></context-group> </trans-unit> - <trans-unit id="_msg684"> + <trans-unit id="_msg701"> <source xml:space="preserve">Duplicate address found: addresses should only be used once each.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">655</context></context-group> + <context-group purpose="location"><context context-type="linenumber">719</context></context-group> </trans-unit> - <trans-unit id="_msg685"> + <trans-unit id="_msg702"> <source xml:space="preserve">Transaction creation failed!</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">658</context></context-group> + <context-group purpose="location"><context context-type="linenumber">722</context></context-group> </trans-unit> - <trans-unit id="_msg686"> + <trans-unit id="_msg703"> <source xml:space="preserve">A fee higher than %1 is considered an absurdly high fee.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">662</context></context-group> + <context-group purpose="location"><context context-type="linenumber">726</context></context-group> </trans-unit> - <trans-unit id="_msg687"> + <trans-unit id="_msg704"> <source xml:space="preserve">Payment request expired.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">665</context></context-group> + <context-group purpose="location"><context context-type="linenumber">729</context></context-group> </trans-unit> <group restype="x-gettext-plurals"> - <context-group purpose="location"><context context-type="linenumber">789</context></context-group> - <trans-unit id="_msg688[0]" approved="yes"> + <context-group purpose="location"><context context-type="linenumber">853</context></context-group> + <trans-unit id="_msg705[0]" approved="yes"> <source xml:space="preserve">Estimated to begin confirmation within %n block(s).</source> <target xml:space="preserve">Estimated to begin confirmation within %n block.</target> </trans-unit> - <trans-unit id="_msg688[1]" approved="yes"> + <trans-unit id="_msg705[1]" approved="yes"> <source xml:space="preserve">Estimated to begin confirmation within %n block(s).</source> <target xml:space="preserve">Estimated to begin confirmation within %n blocks.</target> </trans-unit> </group> - <trans-unit id="_msg689"> + <trans-unit id="_msg706"> <source xml:space="preserve">Warning: Invalid Bitcoin address</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">890</context></context-group> + <context-group purpose="location"><context context-type="linenumber">954</context></context-group> </trans-unit> - <trans-unit id="_msg690"> + <trans-unit id="_msg707"> <source xml:space="preserve">Warning: Unknown change address</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">895</context></context-group> + <context-group purpose="location"><context context-type="linenumber">959</context></context-group> </trans-unit> - <trans-unit id="_msg691"> + <trans-unit id="_msg708"> <source xml:space="preserve">Confirm custom change address</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">898</context></context-group> + <context-group purpose="location"><context context-type="linenumber">962</context></context-group> </trans-unit> - <trans-unit id="_msg692"> + <trans-unit id="_msg709"> <source xml:space="preserve">The address you selected for change is not part of this wallet. Any or all funds in your wallet may be sent to this address. Are you sure?</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">898</context></context-group> + <context-group purpose="location"><context context-type="linenumber">962</context></context-group> </trans-unit> - <trans-unit id="_msg693"> + <trans-unit id="_msg710"> <source xml:space="preserve">(no label)</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">919</context></context-group> + <context-group purpose="location"><context context-type="linenumber">983</context></context-group> </trans-unit> </group> </body></file> <file original="../forms/sendcoinsentry.ui" datatype="x-trolltech-designer-ui" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="SendCoinsEntry"> - <trans-unit id="_msg694" approved="yes"> + <trans-unit id="_msg711" approved="yes"> <source xml:space="preserve">A&mount:</source> <target xml:space="preserve">A&mount:</target> <context-group purpose="location"><context context-type="linenumber">155</context></context-group> <context-group purpose="location"><context context-type="linenumber">705</context></context-group> <context-group purpose="location"><context context-type="linenumber">1238</context></context-group> </trans-unit> - <trans-unit id="_msg695" approved="yes"> + <trans-unit id="_msg712" approved="yes"> <source xml:space="preserve">Pay &To:</source> <target xml:space="preserve">Pay &To:</target> <context-group purpose="location"><context context-type="linenumber">39</context></context-group> </trans-unit> - <trans-unit id="_msg696" approved="yes"> + <trans-unit id="_msg713" approved="yes"> <source xml:space="preserve">&Label:</source> <target xml:space="preserve">&Label:</target> <context-group purpose="location"><context context-type="linenumber">132</context></context-group> </trans-unit> - <trans-unit id="_msg697"> + <trans-unit id="_msg714"> <source xml:space="preserve">Choose previously used address</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">64</context></context-group> </trans-unit> - <trans-unit id="_msg698"> + <trans-unit id="_msg715"> <source xml:space="preserve">The Bitcoin address to send the payment to</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">57</context></context-group> </trans-unit> - <trans-unit id="_msg699" approved="yes"> + <trans-unit id="_msg716" approved="yes"> <source xml:space="preserve">Alt+A</source> <target xml:space="preserve">Alt+A</target> <context-group purpose="location"><context context-type="linenumber">80</context></context-group> </trans-unit> - <trans-unit id="_msg700" approved="yes"> + <trans-unit id="_msg717" approved="yes"> <source xml:space="preserve">Paste address from clipboard</source> <target xml:space="preserve">Paste address from clipboard</target> <context-group purpose="location"><context context-type="linenumber">87</context></context-group> </trans-unit> - <trans-unit id="_msg701" approved="yes"> + <trans-unit id="_msg718" approved="yes"> <source xml:space="preserve">Alt+P</source> <target xml:space="preserve">Alt+P</target> <context-group purpose="location"><context context-type="linenumber">103</context></context-group> </trans-unit> - <trans-unit id="_msg702"> + <trans-unit id="_msg719"> <source xml:space="preserve">Remove this entry</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">110</context></context-group> <context-group purpose="location"><context context-type="linenumber">672</context></context-group> <context-group purpose="location"><context context-type="linenumber">1205</context></context-group> </trans-unit> - <trans-unit id="_msg703"> + <trans-unit id="_msg720"> <source xml:space="preserve">The amount to send in the selected unit</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">170</context></context-group> </trans-unit> - <trans-unit id="_msg704"> + <trans-unit id="_msg721"> <source xml:space="preserve">The fee will be deducted from the amount being sent. The recipient will receive less bitcoins than you enter in the amount field. If multiple recipients are selected, the fee is split equally.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">177</context></context-group> </trans-unit> - <trans-unit id="_msg705"> + <trans-unit id="_msg722"> <source xml:space="preserve">S&ubtract fee from amount</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">180</context></context-group> </trans-unit> - <trans-unit id="_msg706"> + <trans-unit id="_msg723"> <source xml:space="preserve">Use available balance</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">187</context></context-group> </trans-unit> - <trans-unit id="_msg707"> + <trans-unit id="_msg724"> <source xml:space="preserve">Message:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">196</context></context-group> </trans-unit> - <trans-unit id="_msg708"> + <trans-unit id="_msg725"> <source xml:space="preserve">This is an unauthenticated payment request.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">639</context></context-group> </trans-unit> - <trans-unit id="_msg709"> + <trans-unit id="_msg726"> <source xml:space="preserve">This is an authenticated payment request.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">1168</context></context-group> </trans-unit> - <trans-unit id="_msg710"> + <trans-unit id="_msg727"> <source xml:space="preserve">Enter a label for this address to add it to the list of used addresses</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">145</context></context-group> <context-group purpose="location"><context context-type="linenumber">148</context></context-group> </trans-unit> - <trans-unit id="_msg711"> + <trans-unit id="_msg728"> <source xml:space="preserve">A message that was attached to the bitcoin: URI which will be stored with the transaction for your reference. Note: This message will not be sent over the Bitcoin network.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">206</context></context-group> </trans-unit> - <trans-unit id="_msg712"> + <trans-unit id="_msg729"> <source xml:space="preserve">Pay To:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">654</context></context-group> <context-group purpose="location"><context context-type="linenumber">1183</context></context-group> </trans-unit> - <trans-unit id="_msg713"> + <trans-unit id="_msg730"> <source xml:space="preserve">Memo:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">688</context></context-group> @@ -3932,128 +4023,128 @@ Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 </body></file> <file original="../forms/signverifymessagedialog.ui" datatype="x-trolltech-designer-ui" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="SignVerifyMessageDialog"> - <trans-unit id="_msg714" approved="yes"> + <trans-unit id="_msg731" approved="yes"> <source xml:space="preserve">Signatures - Sign / Verify a Message</source> <target xml:space="preserve">Signatures - Sign / Verify a Message</target> <context-group purpose="location"><context context-type="linenumber">14</context></context-group> </trans-unit> - <trans-unit id="_msg715" approved="yes"> + <trans-unit id="_msg732" approved="yes"> <source xml:space="preserve">&Sign Message</source> <target xml:space="preserve">&Sign Message</target> <context-group purpose="location"><context context-type="linenumber">27</context></context-group> </trans-unit> - <trans-unit id="_msg716"> + <trans-unit id="_msg733"> <source xml:space="preserve">You can sign messages/agreements with your addresses to prove you can receive bitcoins sent to them. Be careful not to sign anything vague or random, as phishing attacks may try to trick you into signing your identity over to them. Only sign fully-detailed statements you agree to.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">33</context></context-group> </trans-unit> - <trans-unit id="_msg717"> + <trans-unit id="_msg734"> <source xml:space="preserve">The Bitcoin address to sign the message with</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">51</context></context-group> </trans-unit> - <trans-unit id="_msg718"> + <trans-unit id="_msg735"> <source xml:space="preserve">Choose previously used address</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">58</context></context-group> <context-group purpose="location"><context context-type="linenumber">274</context></context-group> </trans-unit> - <trans-unit id="_msg719" approved="yes"> + <trans-unit id="_msg736" approved="yes"> <source xml:space="preserve">Alt+A</source> <target xml:space="preserve">Alt+A</target> <context-group purpose="location"><context context-type="linenumber">68</context></context-group> <context-group purpose="location"><context context-type="linenumber">284</context></context-group> </trans-unit> - <trans-unit id="_msg720" approved="yes"> + <trans-unit id="_msg737" approved="yes"> <source xml:space="preserve">Paste address from clipboard</source> <target xml:space="preserve">Paste address from clipboard</target> <context-group purpose="location"><context context-type="linenumber">78</context></context-group> </trans-unit> - <trans-unit id="_msg721" approved="yes"> + <trans-unit id="_msg738" approved="yes"> <source xml:space="preserve">Alt+P</source> <target xml:space="preserve">Alt+P</target> <context-group purpose="location"><context context-type="linenumber">88</context></context-group> </trans-unit> - <trans-unit id="_msg722" approved="yes"> + <trans-unit id="_msg739" approved="yes"> <source xml:space="preserve">Enter the message you want to sign here</source> <target xml:space="preserve">Enter the message you want to sign here</target> <context-group purpose="location"><context context-type="linenumber">100</context></context-group> <context-group purpose="location"><context context-type="linenumber">103</context></context-group> </trans-unit> - <trans-unit id="_msg723" approved="yes"> + <trans-unit id="_msg740" approved="yes"> <source xml:space="preserve">Signature</source> <target xml:space="preserve">Signature</target> <context-group purpose="location"><context context-type="linenumber">110</context></context-group> </trans-unit> - <trans-unit id="_msg724" approved="yes"> + <trans-unit id="_msg741" approved="yes"> <source xml:space="preserve">Copy the current signature to the system clipboard</source> <target xml:space="preserve">Copy the current signature to the system clipboard</target> <context-group purpose="location"><context context-type="linenumber">140</context></context-group> </trans-unit> - <trans-unit id="_msg725" approved="yes"> + <trans-unit id="_msg742" approved="yes"> <source xml:space="preserve">Sign the message to prove you own this Bitcoin address</source> <target xml:space="preserve">Sign the message to prove you own this Bitcoin address</target> <context-group purpose="location"><context context-type="linenumber">161</context></context-group> </trans-unit> - <trans-unit id="_msg726" approved="yes"> + <trans-unit id="_msg743" approved="yes"> <source xml:space="preserve">Sign &Message</source> <target xml:space="preserve">Sign &Message</target> <context-group purpose="location"><context context-type="linenumber">164</context></context-group> </trans-unit> - <trans-unit id="_msg727" approved="yes"> + <trans-unit id="_msg744" approved="yes"> <source xml:space="preserve">Reset all sign message fields</source> <target xml:space="preserve">Reset all sign message fields</target> <context-group purpose="location"><context context-type="linenumber">178</context></context-group> </trans-unit> - <trans-unit id="_msg728" approved="yes"> + <trans-unit id="_msg745" approved="yes"> <source xml:space="preserve">Clear &All</source> <target xml:space="preserve">Clear &All</target> <context-group purpose="location"><context context-type="linenumber">181</context></context-group> <context-group purpose="location"><context context-type="linenumber">338</context></context-group> </trans-unit> - <trans-unit id="_msg729" approved="yes"> + <trans-unit id="_msg746" approved="yes"> <source xml:space="preserve">&Verify Message</source> <target xml:space="preserve">&Verify Message</target> <context-group purpose="location"><context context-type="linenumber">240</context></context-group> </trans-unit> - <trans-unit id="_msg730"> + <trans-unit id="_msg747"> <source xml:space="preserve">Enter the receiver's address, message (ensure you copy line breaks, spaces, tabs, etc. exactly) and signature below to verify the message. Be careful not to read more into the signature than what is in the signed message itself, to avoid being tricked by a man-in-the-middle attack. Note that this only proves the signing party receives with the address, it cannot prove sendership of any transaction!</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">246</context></context-group> </trans-unit> - <trans-unit id="_msg731"> + <trans-unit id="_msg748"> <source xml:space="preserve">The Bitcoin address the message was signed with</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">267</context></context-group> </trans-unit> - <trans-unit id="_msg732"> + <trans-unit id="_msg749"> <source xml:space="preserve">The signed message to verify</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">296</context></context-group> <context-group purpose="location"><context context-type="linenumber">299</context></context-group> </trans-unit> - <trans-unit id="_msg733"> + <trans-unit id="_msg750"> <source xml:space="preserve">The signature given when the message was signed</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">306</context></context-group> <context-group purpose="location"><context context-type="linenumber">309</context></context-group> </trans-unit> - <trans-unit id="_msg734" approved="yes"> + <trans-unit id="_msg751" approved="yes"> <source xml:space="preserve">Verify the message to ensure it was signed with the specified Bitcoin address</source> <target xml:space="preserve">Verify the message to ensure it was signed with the specified Bitcoin address</target> <context-group purpose="location"><context context-type="linenumber">318</context></context-group> </trans-unit> - <trans-unit id="_msg735" approved="yes"> + <trans-unit id="_msg752" approved="yes"> <source xml:space="preserve">Verify &Message</source> <target xml:space="preserve">Verify &Message</target> <context-group purpose="location"><context context-type="linenumber">321</context></context-group> </trans-unit> - <trans-unit id="_msg736" approved="yes"> + <trans-unit id="_msg753" approved="yes"> <source xml:space="preserve">Reset all verify message fields</source> <target xml:space="preserve">Reset all verify message fields</target> <context-group purpose="location"><context context-type="linenumber">335</context></context-group> </trans-unit> - <trans-unit id="_msg737"> + <trans-unit id="_msg754"> <source xml:space="preserve">Click "Sign Message" to generate signature</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">125</context></context-group> @@ -4062,13 +4153,13 @@ Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 </body></file> <file original="../signverifymessagedialog.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="SignVerifyMessageDialog"> - <trans-unit id="_msg738"> + <trans-unit id="_msg755"> <source xml:space="preserve">The entered address is invalid.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">120</context></context-group> <context-group purpose="location"><context context-type="linenumber">219</context></context-group> </trans-unit> - <trans-unit id="_msg739"> + <trans-unit id="_msg756"> <source xml:space="preserve">Please check the address and try again.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">120</context></context-group> @@ -4076,59 +4167,59 @@ Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 <context-group purpose="location"><context context-type="linenumber">220</context></context-group> <context-group purpose="location"><context context-type="linenumber">227</context></context-group> </trans-unit> - <trans-unit id="_msg740"> + <trans-unit id="_msg757"> <source xml:space="preserve">The entered address does not refer to a key.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">127</context></context-group> <context-group purpose="location"><context context-type="linenumber">226</context></context-group> </trans-unit> - <trans-unit id="_msg741"> + <trans-unit id="_msg758"> <source xml:space="preserve">Wallet unlock was cancelled.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">135</context></context-group> </trans-unit> - <trans-unit id="_msg742"> + <trans-unit id="_msg759"> <source xml:space="preserve">No error</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">146</context></context-group> </trans-unit> - <trans-unit id="_msg743"> + <trans-unit id="_msg760"> <source xml:space="preserve">Private key for the entered address is not available.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">149</context></context-group> </trans-unit> - <trans-unit id="_msg744"> + <trans-unit id="_msg761"> <source xml:space="preserve">Message signing failed.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">152</context></context-group> </trans-unit> - <trans-unit id="_msg745"> + <trans-unit id="_msg762"> <source xml:space="preserve">Message signed.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">164</context></context-group> </trans-unit> - <trans-unit id="_msg746"> + <trans-unit id="_msg763"> <source xml:space="preserve">The signature could not be decoded.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">233</context></context-group> </trans-unit> - <trans-unit id="_msg747"> + <trans-unit id="_msg764"> <source xml:space="preserve">Please check the signature and try again.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">234</context></context-group> <context-group purpose="location"><context context-type="linenumber">241</context></context-group> </trans-unit> - <trans-unit id="_msg748"> + <trans-unit id="_msg765"> <source xml:space="preserve">The signature did not match the message digest.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">240</context></context-group> </trans-unit> - <trans-unit id="_msg749"> + <trans-unit id="_msg766"> <source xml:space="preserve">Message verification failed.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">246</context></context-group> </trans-unit> - <trans-unit id="_msg750"> + <trans-unit id="_msg767"> <source xml:space="preserve">Message verified.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">214</context></context-group> @@ -4137,7 +4228,7 @@ Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 </body></file> <file original="../trafficgraphwidget.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="TrafficGraphWidget"> - <trans-unit id="_msg751"> + <trans-unit id="_msg768"> <source xml:space="preserve">kB/s</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">82</context></context-group> @@ -4148,111 +4239,111 @@ Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 <group restype="x-trolltech-linguist-context" resname="TransactionDesc"> <group restype="x-gettext-plurals"> <context-group purpose="location"><context context-type="linenumber">36</context></context-group> - <trans-unit id="_msg752[0]" approved="yes"> + <trans-unit id="_msg769[0]" approved="yes"> <source xml:space="preserve">Open for %n more block(s)</source> <target xml:space="preserve">Open for %n more block</target> </trans-unit> - <trans-unit id="_msg752[1]" approved="yes"> + <trans-unit id="_msg769[1]" approved="yes"> <source xml:space="preserve">Open for %n more block(s)</source> <target xml:space="preserve">Open for %n more blocks</target> </trans-unit> </group> - <trans-unit id="_msg753"> + <trans-unit id="_msg770"> <source xml:space="preserve">Open until %1</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">38</context></context-group> </trans-unit> - <trans-unit id="_msg754"> + <trans-unit id="_msg771"> <source xml:space="preserve">conflicted with a transaction with %1 confirmations</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">44</context></context-group> </trans-unit> - <trans-unit id="_msg755"> + <trans-unit id="_msg772"> <source xml:space="preserve">0/unconfirmed, %1</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">47</context></context-group> </trans-unit> - <trans-unit id="_msg756"> + <trans-unit id="_msg773"> <source xml:space="preserve">in memory pool</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">47</context></context-group> </trans-unit> - <trans-unit id="_msg757"> + <trans-unit id="_msg774"> <source xml:space="preserve">not in memory pool</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">47</context></context-group> </trans-unit> - <trans-unit id="_msg758"> + <trans-unit id="_msg775"> <source xml:space="preserve">abandoned</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">46</context></context-group> </trans-unit> - <trans-unit id="_msg759"> + <trans-unit id="_msg776"> <source xml:space="preserve">%1/unconfirmed</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">49</context></context-group> </trans-unit> - <trans-unit id="_msg760"> + <trans-unit id="_msg777"> <source xml:space="preserve">%1 confirmations</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">51</context></context-group> </trans-unit> - <trans-unit id="_msg761"> + <trans-unit id="_msg778"> <source xml:space="preserve">Status</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">102</context></context-group> </trans-unit> - <trans-unit id="_msg762"> + <trans-unit id="_msg779"> <source xml:space="preserve">Date</source> <target xml:space="preserve" state="needs-review-translation">Date</target> <context-group purpose="location"><context context-type="linenumber">105</context></context-group> </trans-unit> - <trans-unit id="_msg763"> + <trans-unit id="_msg780"> <source xml:space="preserve">Source</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">112</context></context-group> </trans-unit> - <trans-unit id="_msg764"> + <trans-unit id="_msg781"> <source xml:space="preserve">Generated</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">112</context></context-group> </trans-unit> - <trans-unit id="_msg765"> + <trans-unit id="_msg782"> <source xml:space="preserve">From</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">117</context></context-group> <context-group purpose="location"><context context-type="linenumber">131</context></context-group> <context-group purpose="location"><context context-type="linenumber">203</context></context-group> </trans-unit> - <trans-unit id="_msg766"> + <trans-unit id="_msg783"> <source xml:space="preserve">unknown</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">131</context></context-group> </trans-unit> - <trans-unit id="_msg767"> + <trans-unit id="_msg784"> <source xml:space="preserve">To</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">132</context></context-group> <context-group purpose="location"><context context-type="linenumber">152</context></context-group> <context-group purpose="location"><context context-type="linenumber">222</context></context-group> </trans-unit> - <trans-unit id="_msg768"> + <trans-unit id="_msg785"> <source xml:space="preserve">own address</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">134</context></context-group> </trans-unit> - <trans-unit id="_msg769"> + <trans-unit id="_msg786"> <source xml:space="preserve">watch-only</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">134</context></context-group> <context-group purpose="location"><context context-type="linenumber">203</context></context-group> </trans-unit> - <trans-unit id="_msg770"> + <trans-unit id="_msg787"> <source xml:space="preserve">label</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">136</context></context-group> </trans-unit> - <trans-unit id="_msg771"> + <trans-unit id="_msg788"> <source xml:space="preserve">Credit</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">172</context></context-group> @@ -4263,120 +4354,120 @@ Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 </trans-unit> <group restype="x-gettext-plurals"> <context-group purpose="location"><context context-type="linenumber">174</context></context-group> - <trans-unit id="_msg772[0]" approved="yes"> + <trans-unit id="_msg789[0]" approved="yes"> <source xml:space="preserve">matures in %n more block(s)</source> <target xml:space="preserve">matures in %n more block</target> </trans-unit> - <trans-unit id="_msg772[1]" approved="yes"> + <trans-unit id="_msg789[1]" approved="yes"> <source xml:space="preserve">matures in %n more block(s)</source> <target xml:space="preserve">matures in %n more blocks</target> </trans-unit> </group> - <trans-unit id="_msg773"> + <trans-unit id="_msg790"> <source xml:space="preserve">not accepted</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">176</context></context-group> </trans-unit> - <trans-unit id="_msg774"> + <trans-unit id="_msg791"> <source xml:space="preserve">Debit</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">236</context></context-group> <context-group purpose="location"><context context-type="linenumber">262</context></context-group> <context-group purpose="location"><context context-type="linenumber">325</context></context-group> </trans-unit> - <trans-unit id="_msg775"> + <trans-unit id="_msg792"> <source xml:space="preserve">Total debit</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">246</context></context-group> </trans-unit> - <trans-unit id="_msg776"> + <trans-unit id="_msg793"> <source xml:space="preserve">Total credit</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">247</context></context-group> </trans-unit> - <trans-unit id="_msg777"> + <trans-unit id="_msg794"> <source xml:space="preserve">Transaction fee</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">252</context></context-group> </trans-unit> - <trans-unit id="_msg778"> + <trans-unit id="_msg795"> <source xml:space="preserve">Net amount</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">274</context></context-group> </trans-unit> - <trans-unit id="_msg779"> + <trans-unit id="_msg796"> <source xml:space="preserve">Message</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">280</context></context-group> <context-group purpose="location"><context context-type="linenumber">292</context></context-group> </trans-unit> - <trans-unit id="_msg780"> + <trans-unit id="_msg797"> <source xml:space="preserve">Comment</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">282</context></context-group> </trans-unit> - <trans-unit id="_msg781"> + <trans-unit id="_msg798"> <source xml:space="preserve">Transaction ID</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">284</context></context-group> </trans-unit> - <trans-unit id="_msg782"> + <trans-unit id="_msg799"> <source xml:space="preserve">Transaction total size</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">285</context></context-group> </trans-unit> - <trans-unit id="_msg783"> + <trans-unit id="_msg800"> <source xml:space="preserve">Transaction virtual size</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">286</context></context-group> </trans-unit> - <trans-unit id="_msg784"> + <trans-unit id="_msg801"> <source xml:space="preserve">Output index</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">287</context></context-group> </trans-unit> - <trans-unit id="_msg785"> + <trans-unit id="_msg802"> <source xml:space="preserve"> (Certificate was not verified)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">303</context></context-group> </trans-unit> - <trans-unit id="_msg786"> + <trans-unit id="_msg803"> <source xml:space="preserve">Merchant</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">306</context></context-group> </trans-unit> - <trans-unit id="_msg787"> + <trans-unit id="_msg804"> <source xml:space="preserve">Generated coins must mature %1 blocks before they can be spent. When you generated this block, it was broadcast to the network to be added to the block chain. If it fails to get into the chain, its state will change to "not accepted" and it won't be spendable. This may occasionally happen if another node generates a block within a few seconds of yours.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">314</context></context-group> </trans-unit> - <trans-unit id="_msg788"> + <trans-unit id="_msg805"> <source xml:space="preserve">Debug information</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">322</context></context-group> </trans-unit> - <trans-unit id="_msg789"> + <trans-unit id="_msg806"> <source xml:space="preserve">Transaction</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">330</context></context-group> </trans-unit> - <trans-unit id="_msg790"> + <trans-unit id="_msg807"> <source xml:space="preserve">Inputs</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">333</context></context-group> </trans-unit> - <trans-unit id="_msg791"> + <trans-unit id="_msg808"> <source xml:space="preserve">Amount</source> <target xml:space="preserve" state="needs-review-translation">Amount</target> <context-group purpose="location"><context context-type="linenumber">354</context></context-group> </trans-unit> - <trans-unit id="_msg792"> + <trans-unit id="_msg809"> <source xml:space="preserve">true</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">355</context></context-group> <context-group purpose="location"><context context-type="linenumber">356</context></context-group> </trans-unit> - <trans-unit id="_msg793"> + <trans-unit id="_msg810"> <source xml:space="preserve">false</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">355</context></context-group> @@ -4386,7 +4477,7 @@ Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 </body></file> <file original="../forms/transactiondescdialog.ui" datatype="x-trolltech-designer-ui" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="TransactionDescDialog"> - <trans-unit id="_msg794" approved="yes"> + <trans-unit id="_msg811" approved="yes"> <source xml:space="preserve">This pane shows a detailed description of the transaction</source> <target xml:space="preserve">This pane shows a detailed description of the transaction</target> <context-group purpose="location"><context context-type="linenumber">20</context></context-group> @@ -4395,7 +4486,7 @@ Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 </body></file> <file original="../transactiondescdialog.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="TransactionDescDialog"> - <trans-unit id="_msg795"> + <trans-unit id="_msg812"> <source xml:space="preserve">Details for %1</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">18</context></context-group> @@ -4404,138 +4495,138 @@ Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 </body></file> <file original="../transactiontablemodel.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="TransactionTableModel"> - <trans-unit id="_msg796"> + <trans-unit id="_msg813"> <source xml:space="preserve">Date</source> <target xml:space="preserve" state="needs-review-translation">Date</target> <context-group purpose="location"><context context-type="linenumber">260</context></context-group> </trans-unit> - <trans-unit id="_msg797"> + <trans-unit id="_msg814"> <source xml:space="preserve">Type</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">260</context></context-group> </trans-unit> - <trans-unit id="_msg798"> + <trans-unit id="_msg815"> <source xml:space="preserve">Label</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">260</context></context-group> </trans-unit> <group restype="x-gettext-plurals"> <context-group purpose="location"><context context-type="linenumber">320</context></context-group> - <trans-unit id="_msg799[0]" approved="yes"> + <trans-unit id="_msg816[0]" approved="yes"> <source xml:space="preserve">Open for %n more block(s)</source> <target xml:space="preserve">Open for %n more block</target> </trans-unit> - <trans-unit id="_msg799[1]" approved="yes"> + <trans-unit id="_msg816[1]" approved="yes"> <source xml:space="preserve">Open for %n more block(s)</source> <target xml:space="preserve">Open for %n more blocks</target> </trans-unit> </group> - <trans-unit id="_msg800"> + <trans-unit id="_msg817"> <source xml:space="preserve">Open until %1</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">323</context></context-group> </trans-unit> - <trans-unit id="_msg801"> + <trans-unit id="_msg818"> <source xml:space="preserve">Unconfirmed</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">326</context></context-group> </trans-unit> - <trans-unit id="_msg802"> + <trans-unit id="_msg819"> <source xml:space="preserve">Abandoned</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">329</context></context-group> </trans-unit> - <trans-unit id="_msg803"> + <trans-unit id="_msg820"> <source xml:space="preserve">Confirming (%1 of %2 recommended confirmations)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">332</context></context-group> </trans-unit> - <trans-unit id="_msg804"> + <trans-unit id="_msg821"> <source xml:space="preserve">Confirmed (%1 confirmations)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">335</context></context-group> </trans-unit> - <trans-unit id="_msg805"> + <trans-unit id="_msg822"> <source xml:space="preserve">Conflicted</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">338</context></context-group> </trans-unit> - <trans-unit id="_msg806"> + <trans-unit id="_msg823"> <source xml:space="preserve">Immature (%1 confirmations, will be available after %2)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">341</context></context-group> </trans-unit> - <trans-unit id="_msg807"> + <trans-unit id="_msg824"> <source xml:space="preserve">Generated but not accepted</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">344</context></context-group> </trans-unit> - <trans-unit id="_msg808"> + <trans-unit id="_msg825"> <source xml:space="preserve">Received with</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">383</context></context-group> </trans-unit> - <trans-unit id="_msg809"> + <trans-unit id="_msg826"> <source xml:space="preserve">Received from</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">385</context></context-group> </trans-unit> - <trans-unit id="_msg810"> + <trans-unit id="_msg827"> <source xml:space="preserve">Sent to</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">388</context></context-group> </trans-unit> - <trans-unit id="_msg811"> + <trans-unit id="_msg828"> <source xml:space="preserve">Payment to yourself</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">390</context></context-group> </trans-unit> - <trans-unit id="_msg812"> + <trans-unit id="_msg829"> <source xml:space="preserve">Mined</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">392</context></context-group> </trans-unit> - <trans-unit id="_msg813"> + <trans-unit id="_msg830"> <source xml:space="preserve">watch-only</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">420</context></context-group> </trans-unit> - <trans-unit id="_msg814"> + <trans-unit id="_msg831"> <source xml:space="preserve">(n/a)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">436</context></context-group> </trans-unit> - <trans-unit id="_msg815"> + <trans-unit id="_msg832"> <source xml:space="preserve">(no label)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">646</context></context-group> </trans-unit> - <trans-unit id="_msg816"> + <trans-unit id="_msg833"> <source xml:space="preserve">Transaction status. Hover over this field to show number of confirmations.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">685</context></context-group> </trans-unit> - <trans-unit id="_msg817"> + <trans-unit id="_msg834"> <source xml:space="preserve">Date and time that the transaction was received.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">687</context></context-group> </trans-unit> - <trans-unit id="_msg818"> + <trans-unit id="_msg835"> <source xml:space="preserve">Type of transaction.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">689</context></context-group> </trans-unit> - <trans-unit id="_msg819"> + <trans-unit id="_msg836"> <source xml:space="preserve">Whether or not a watch-only address is involved in this transaction.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">691</context></context-group> </trans-unit> - <trans-unit id="_msg820"> + <trans-unit id="_msg837"> <source xml:space="preserve">User-defined intent/purpose of the transaction.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">693</context></context-group> </trans-unit> - <trans-unit id="_msg821"> + <trans-unit id="_msg838"> <source xml:space="preserve">Amount removed from or added to balance.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">695</context></context-group> @@ -4544,199 +4635,199 @@ Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 </body></file> <file original="../transactionview.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="TransactionView"> - <trans-unit id="_msg822"> + <trans-unit id="_msg839"> <source xml:space="preserve">All</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">70</context></context-group> <context-group purpose="location"><context context-type="linenumber">86</context></context-group> </trans-unit> - <trans-unit id="_msg823"> + <trans-unit id="_msg840"> <source xml:space="preserve">Today</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">71</context></context-group> </trans-unit> - <trans-unit id="_msg824"> + <trans-unit id="_msg841"> <source xml:space="preserve">This week</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">72</context></context-group> </trans-unit> - <trans-unit id="_msg825"> + <trans-unit id="_msg842"> <source xml:space="preserve">This month</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">73</context></context-group> </trans-unit> - <trans-unit id="_msg826"> + <trans-unit id="_msg843"> <source xml:space="preserve">Last month</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">74</context></context-group> </trans-unit> - <trans-unit id="_msg827"> + <trans-unit id="_msg844"> <source xml:space="preserve">This year</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">75</context></context-group> </trans-unit> - <trans-unit id="_msg828"> + <trans-unit id="_msg845"> <source xml:space="preserve">Received with</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">87</context></context-group> </trans-unit> - <trans-unit id="_msg829"> + <trans-unit id="_msg846"> <source xml:space="preserve">Sent to</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">89</context></context-group> </trans-unit> - <trans-unit id="_msg830"> + <trans-unit id="_msg847"> <source xml:space="preserve">To yourself</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">91</context></context-group> </trans-unit> - <trans-unit id="_msg831"> + <trans-unit id="_msg848"> <source xml:space="preserve">Mined</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">92</context></context-group> </trans-unit> - <trans-unit id="_msg832"> + <trans-unit id="_msg849"> <source xml:space="preserve">Other</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">93</context></context-group> </trans-unit> - <trans-unit id="_msg833"> + <trans-unit id="_msg850"> <source xml:space="preserve">Enter address, transaction id, or label to search</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">98</context></context-group> </trans-unit> - <trans-unit id="_msg834"> + <trans-unit id="_msg851"> <source xml:space="preserve">Min amount</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">102</context></context-group> </trans-unit> - <trans-unit id="_msg835"> - <source xml:space="preserve">Abandon transaction</source> - <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">177</context></context-group> - </trans-unit> - <trans-unit id="_msg836"> - <source xml:space="preserve">Increase transaction fee</source> + <trans-unit id="_msg852"> + <source xml:space="preserve">Range…</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">174</context></context-group> + <context-group purpose="location"><context context-type="linenumber">76</context></context-group> </trans-unit> - <trans-unit id="_msg837"> - <source xml:space="preserve">Copy address</source> + <trans-unit id="_msg853"> + <source xml:space="preserve">&Copy address</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">166</context></context-group> </trans-unit> - <trans-unit id="_msg838"> - <source xml:space="preserve">Copy label</source> + <trans-unit id="_msg854"> + <source xml:space="preserve">Copy &label</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">167</context></context-group> </trans-unit> - <trans-unit id="_msg839"> - <source xml:space="preserve">Copy amount</source> + <trans-unit id="_msg855"> + <source xml:space="preserve">Copy &amount</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">168</context></context-group> </trans-unit> - <trans-unit id="_msg840"> - <source xml:space="preserve">Copy transaction ID</source> + <trans-unit id="_msg856"> + <source xml:space="preserve">Copy transaction &ID</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">169</context></context-group> </trans-unit> - <trans-unit id="_msg841"> - <source xml:space="preserve">Copy raw transaction</source> + <trans-unit id="_msg857"> + <source xml:space="preserve">Copy &raw transaction</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">170</context></context-group> </trans-unit> - <trans-unit id="_msg842"> - <source xml:space="preserve">Copy full transaction details</source> + <trans-unit id="_msg858"> + <source xml:space="preserve">Copy full transaction &details</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">171</context></context-group> </trans-unit> - <trans-unit id="_msg843"> - <source xml:space="preserve">Edit address label</source> + <trans-unit id="_msg859"> + <source xml:space="preserve">&Show transaction details</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">178</context></context-group> + <context-group purpose="location"><context context-type="linenumber">172</context></context-group> </trans-unit> - <trans-unit id="_msg844"> - <source xml:space="preserve">Show transaction details</source> + <trans-unit id="_msg860"> + <source xml:space="preserve">Increase transaction &fee</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">172</context></context-group> + <context-group purpose="location"><context context-type="linenumber">174</context></context-group> </trans-unit> - <trans-unit id="_msg845"> - <source xml:space="preserve">Range…</source> + <trans-unit id="_msg861"> + <source xml:space="preserve">A&bandon transaction</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">76</context></context-group> + <context-group purpose="location"><context context-type="linenumber">177</context></context-group> </trans-unit> - <trans-unit id="_msg846"> + <trans-unit id="_msg862"> + <source xml:space="preserve">&Edit address label</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">178</context></context-group> + </trans-unit> + <trans-unit id="_msg863"> <source xml:space="preserve">Export Transaction History</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">352</context></context-group> </trans-unit> - <trans-unit id="_msg847"> + <trans-unit id="_msg864"> <source xml:space="preserve">Comma separated file</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">355</context></context-group> <note annotates="source" from="developer">Expanded name of the CSV file format. See https://en.wikipedia.org/wiki/Comma-separated_values</note> </trans-unit> - <trans-unit id="_msg848"> + <trans-unit id="_msg865"> <source xml:space="preserve">Confirmed</source> <target xml:space="preserve" state="needs-review-translation">Confirmed</target> <context-group purpose="location"><context context-type="linenumber">364</context></context-group> </trans-unit> - <trans-unit id="_msg849"> + <trans-unit id="_msg866"> <source xml:space="preserve">Watch-only</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">366</context></context-group> </trans-unit> - <trans-unit id="_msg850"> + <trans-unit id="_msg867"> <source xml:space="preserve">Date</source> <target xml:space="preserve" state="needs-review-translation">Date</target> <context-group purpose="location"><context context-type="linenumber">367</context></context-group> </trans-unit> - <trans-unit id="_msg851"> + <trans-unit id="_msg868"> <source xml:space="preserve">Type</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">368</context></context-group> </trans-unit> - <trans-unit id="_msg852"> + <trans-unit id="_msg869"> <source xml:space="preserve">Label</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">369</context></context-group> </trans-unit> - <trans-unit id="_msg853"> + <trans-unit id="_msg870"> <source xml:space="preserve">Address</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">370</context></context-group> </trans-unit> - <trans-unit id="_msg854"> + <trans-unit id="_msg871"> <source xml:space="preserve">ID</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">372</context></context-group> </trans-unit> - <trans-unit id="_msg855"> + <trans-unit id="_msg872"> <source xml:space="preserve">Exporting Failed</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">375</context></context-group> </trans-unit> - <trans-unit id="_msg856"> + <trans-unit id="_msg873"> <source xml:space="preserve">There was an error trying to save the transaction history to %1.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">375</context></context-group> </trans-unit> - <trans-unit id="_msg857"> + <trans-unit id="_msg874"> <source xml:space="preserve">Exporting Successful</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">379</context></context-group> </trans-unit> - <trans-unit id="_msg858"> + <trans-unit id="_msg875"> <source xml:space="preserve">The transaction history was successfully saved to %1.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">379</context></context-group> </trans-unit> - <trans-unit id="_msg859"> + <trans-unit id="_msg876"> <source xml:space="preserve">Range:</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">551</context></context-group> </trans-unit> - <trans-unit id="_msg860"> + <trans-unit id="_msg877"> <source xml:space="preserve">to</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">559</context></context-group> @@ -4745,178 +4836,183 @@ Note: Since the fee is calculated on a per-byte basis, a fee rate of "100 </body></file> <file original="../walletframe.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="WalletFrame"> - <trans-unit id="_msg861"> + <trans-unit id="_msg878"> <source xml:space="preserve">No wallet has been loaded. Go to File > Open Wallet to load a wallet. - OR -</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">39</context></context-group> + <context-group purpose="location"><context context-type="linenumber">35</context></context-group> </trans-unit> - <trans-unit id="_msg862"> + <trans-unit id="_msg879"> <source xml:space="preserve">Create a new wallet</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">44</context></context-group> + <context-group purpose="location"><context context-type="linenumber">40</context></context-group> </trans-unit> </group> </body></file> <file original="../walletmodel.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="WalletModel"> - <trans-unit id="_msg863"> + <trans-unit id="_msg880"> <source xml:space="preserve">Send Coins</source> <target xml:space="preserve" state="needs-review-translation">Send Coins</target> <context-group purpose="location"><context context-type="linenumber">218</context></context-group> </trans-unit> - <trans-unit id="_msg864"> + <trans-unit id="_msg881"> <source xml:space="preserve">Fee bump error</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">497</context></context-group> - <context-group purpose="location"><context context-type="linenumber">549</context></context-group> - <context-group purpose="location"><context context-type="linenumber">562</context></context-group> - <context-group purpose="location"><context context-type="linenumber">567</context></context-group> + <context-group purpose="location"><context context-type="linenumber">478</context></context-group> + <context-group purpose="location"><context context-type="linenumber">530</context></context-group> + <context-group purpose="location"><context context-type="linenumber">543</context></context-group> + <context-group purpose="location"><context context-type="linenumber">548</context></context-group> </trans-unit> - <trans-unit id="_msg865"> + <trans-unit id="_msg882"> <source xml:space="preserve">Increasing transaction fee failed</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">497</context></context-group> + <context-group purpose="location"><context context-type="linenumber">478</context></context-group> </trans-unit> - <trans-unit id="_msg866"> + <trans-unit id="_msg883"> <source xml:space="preserve">Do you want to increase the fee?</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">505</context></context-group> + <context-group purpose="location"><context context-type="linenumber">486</context></context-group> </trans-unit> - <trans-unit id="_msg867"> + <trans-unit id="_msg884"> <source xml:space="preserve">Do you want to draft a transaction with fee increase?</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">505</context></context-group> + <context-group purpose="location"><context context-type="linenumber">486</context></context-group> </trans-unit> - <trans-unit id="_msg868"> + <trans-unit id="_msg885"> <source xml:space="preserve">Current fee:</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">509</context></context-group> + <context-group purpose="location"><context context-type="linenumber">490</context></context-group> </trans-unit> - <trans-unit id="_msg869"> + <trans-unit id="_msg886"> <source xml:space="preserve">Increase:</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">513</context></context-group> + <context-group purpose="location"><context context-type="linenumber">494</context></context-group> </trans-unit> - <trans-unit id="_msg870"> + <trans-unit id="_msg887"> <source xml:space="preserve">New fee:</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">517</context></context-group> + <context-group purpose="location"><context context-type="linenumber">498</context></context-group> </trans-unit> - <trans-unit id="_msg871"> + <trans-unit id="_msg888"> <source xml:space="preserve">Warning: This may pay the additional fee by reducing change outputs or adding inputs, when necessary. It may add a new change output if one does not already exist. These changes may potentially leak privacy.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">525</context></context-group> + <context-group purpose="location"><context context-type="linenumber">506</context></context-group> </trans-unit> - <trans-unit id="_msg872"> + <trans-unit id="_msg889"> <source xml:space="preserve">Confirm fee bump</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">528</context></context-group> + <context-group purpose="location"><context context-type="linenumber">509</context></context-group> </trans-unit> - <trans-unit id="_msg873"> + <trans-unit id="_msg890"> <source xml:space="preserve">Can't draft transaction.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">549</context></context-group> + <context-group purpose="location"><context context-type="linenumber">530</context></context-group> </trans-unit> - <trans-unit id="_msg874"> + <trans-unit id="_msg891"> <source xml:space="preserve">PSBT copied</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">556</context></context-group> + <context-group purpose="location"><context context-type="linenumber">537</context></context-group> </trans-unit> - <trans-unit id="_msg875"> + <trans-unit id="_msg892"> <source xml:space="preserve">Can't sign transaction.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">562</context></context-group> + <context-group purpose="location"><context context-type="linenumber">543</context></context-group> </trans-unit> - <trans-unit id="_msg876"> + <trans-unit id="_msg893"> <source xml:space="preserve">Could not commit transaction</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">567</context></context-group> + <context-group purpose="location"><context context-type="linenumber">548</context></context-group> </trans-unit> - <trans-unit id="_msg877"> + <trans-unit id="_msg894"> + <source xml:space="preserve">Can't display address</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">562</context></context-group> + </trans-unit> + <trans-unit id="_msg895"> <source xml:space="preserve">default wallet</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">587</context></context-group> + <context-group purpose="location"><context context-type="linenumber">580</context></context-group> </trans-unit> </group> </body></file> <file original="../walletview.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="WalletView"> - <trans-unit id="_msg878"> + <trans-unit id="_msg896"> <source xml:space="preserve">&Export</source> <target xml:space="preserve" state="needs-review-translation">&Export</target> <context-group purpose="location"><context context-type="linenumber">51</context></context-group> </trans-unit> - <trans-unit id="_msg879"> + <trans-unit id="_msg897"> <source xml:space="preserve">Export the data in the current tab to a file</source> <target xml:space="preserve" state="needs-review-translation">Export the data in the current tab to a file</target> <context-group purpose="location"><context context-type="linenumber">52</context></context-group> </trans-unit> - <trans-unit id="_msg880"> + <trans-unit id="_msg898"> <source xml:space="preserve">Error</source> <target xml:space="preserve" state="needs-review-translation">Error</target> <context-group purpose="location"><context context-type="linenumber">217</context></context-group> <context-group purpose="location"><context context-type="linenumber">226</context></context-group> <context-group purpose="location"><context context-type="linenumber">236</context></context-group> </trans-unit> - <trans-unit id="_msg881"> + <trans-unit id="_msg899"> <source xml:space="preserve">Unable to decode PSBT from clipboard (invalid base64)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">217</context></context-group> </trans-unit> - <trans-unit id="_msg882"> + <trans-unit id="_msg900"> <source xml:space="preserve">Load Transaction Data</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">222</context></context-group> </trans-unit> - <trans-unit id="_msg883"> + <trans-unit id="_msg901"> <source xml:space="preserve">Partially Signed Transaction (*.psbt)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">223</context></context-group> </trans-unit> - <trans-unit id="_msg884"> + <trans-unit id="_msg902"> <source xml:space="preserve">PSBT file must be smaller than 100 MiB</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">226</context></context-group> </trans-unit> - <trans-unit id="_msg885"> + <trans-unit id="_msg903"> <source xml:space="preserve">Unable to decode PSBT</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">236</context></context-group> </trans-unit> - <trans-unit id="_msg886"> + <trans-unit id="_msg904"> <source xml:space="preserve">Backup Wallet</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">275</context></context-group> </trans-unit> - <trans-unit id="_msg887"> + <trans-unit id="_msg905"> <source xml:space="preserve">Wallet Data</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">277</context></context-group> <note annotates="source" from="developer">Name of the wallet data file format.</note> </trans-unit> - <trans-unit id="_msg888"> + <trans-unit id="_msg906"> <source xml:space="preserve">Backup Failed</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">283</context></context-group> </trans-unit> - <trans-unit id="_msg889"> + <trans-unit id="_msg907"> <source xml:space="preserve">There was an error trying to save the wallet data to %1.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">283</context></context-group> </trans-unit> - <trans-unit id="_msg890"> + <trans-unit id="_msg908"> <source xml:space="preserve">Backup Successful</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">287</context></context-group> </trans-unit> - <trans-unit id="_msg891"> + <trans-unit id="_msg909"> <source xml:space="preserve">The wallet data was successfully saved to %1.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">287</context></context-group> </trans-unit> - <trans-unit id="_msg892"> + <trans-unit id="_msg910"> <source xml:space="preserve">Cancel</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">331</context></context-group> @@ -4925,766 +5021,776 @@ Go to File > Open Wallet to load a wallet. </body></file> <file original="../bitcoinstrings.cpp" datatype="cpp" source-language="en" target-language="en"><body> <group restype="x-trolltech-linguist-context" resname="bitcoin-core"> - <trans-unit id="_msg893"> + <trans-unit id="_msg911"> <source xml:space="preserve">The %s developers</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">12</context></context-group> </trans-unit> - <trans-unit id="_msg894"> + <trans-unit id="_msg912"> <source xml:space="preserve">%s corrupt. Try using the wallet tool bitcoin-wallet to salvage or restoring a backup.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">13</context></context-group> </trans-unit> - <trans-unit id="_msg895"> + <trans-unit id="_msg913"> <source xml:space="preserve">-maxtxfee is set very high! Fees this large could be paid on a single transaction.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">16</context></context-group> </trans-unit> - <trans-unit id="_msg896"> + <trans-unit id="_msg914"> <source xml:space="preserve">Cannot downgrade wallet from version %i to version %i. Wallet version unchanged.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">19</context></context-group> </trans-unit> - <trans-unit id="_msg897"> + <trans-unit id="_msg915"> <source xml:space="preserve">Cannot obtain a lock on data directory %s. %s is probably already running.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">22</context></context-group> </trans-unit> - <trans-unit id="_msg898"> + <trans-unit id="_msg916"> <source xml:space="preserve">Cannot provide specific connections and have addrman find outgoing connections at the same.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">24</context></context-group> </trans-unit> - <trans-unit id="_msg899"> + <trans-unit id="_msg917"> <source xml:space="preserve">Cannot upgrade a non HD split wallet from version %i to version %i without upgrading to support pre-split keypool. Please use version %i or no version specified.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">27</context></context-group> </trans-unit> - <trans-unit id="_msg900"> + <trans-unit id="_msg918"> <source xml:space="preserve">Distributed under the MIT software license, see the accompanying file %s or %s</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">31</context></context-group> </trans-unit> - <trans-unit id="_msg901"> + <trans-unit id="_msg919"> <source xml:space="preserve">Error reading %s! All keys read correctly, but transaction data or address book entries might be missing or incorrect.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">34</context></context-group> </trans-unit> - <trans-unit id="_msg902"> + <trans-unit id="_msg920"> <source xml:space="preserve">Error: Dumpfile format record is incorrect. Got "%s", expected "format".</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">37</context></context-group> </trans-unit> - <trans-unit id="_msg903"> + <trans-unit id="_msg921"> <source xml:space="preserve">Error: Dumpfile identifier record is incorrect. Got "%s", expected "%s".</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">39</context></context-group> </trans-unit> - <trans-unit id="_msg904"> + <trans-unit id="_msg922"> <source xml:space="preserve">Error: Dumpfile version is not supported. This version of bitcoin-wallet only supports version 1 dumpfiles. Got dumpfile with version %s</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">41</context></context-group> </trans-unit> - <trans-unit id="_msg905"> - <source xml:space="preserve">Error: Listening for incoming connections failed (listen returned error %s)</source> + <trans-unit id="_msg923"> + <source xml:space="preserve">Error: Legacy wallets only support the "legacy", "p2sh-segwit", and "bech32" address types</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">44</context></context-group> </trans-unit> - <trans-unit id="_msg906"> + <trans-unit id="_msg924"> + <source xml:space="preserve">Error: Listening for incoming connections failed (listen returned error %s)</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">47</context></context-group> + </trans-unit> + <trans-unit id="_msg925"> <source xml:space="preserve">Fee estimation failed. Fallbackfee is disabled. Wait a few blocks or enable -fallbackfee.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">46</context></context-group> + <context-group purpose="location"><context context-type="linenumber">49</context></context-group> </trans-unit> - <trans-unit id="_msg907"> + <trans-unit id="_msg926"> <source xml:space="preserve">File %s already exists. If you are sure this is what you want, move it out of the way first.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">49</context></context-group> + <context-group purpose="location"><context context-type="linenumber">52</context></context-group> </trans-unit> - <trans-unit id="_msg908"> + <trans-unit id="_msg927"> <source xml:space="preserve">Invalid amount for -maxtxfee=<amount>: '%s' (must be at least the minrelay fee of %s to prevent stuck transactions)</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">52</context></context-group> + <context-group purpose="location"><context context-type="linenumber">55</context></context-group> </trans-unit> - <trans-unit id="_msg909"> + <trans-unit id="_msg928"> <source xml:space="preserve">More than one onion bind address is provided. Using %s for the automatically created Tor onion service.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">55</context></context-group> + <context-group purpose="location"><context context-type="linenumber">58</context></context-group> </trans-unit> - <trans-unit id="_msg910"> + <trans-unit id="_msg929"> <source xml:space="preserve">No dump file provided. To use createfromdump, -dumpfile=<filename> must be provided.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">58</context></context-group> + <context-group purpose="location"><context context-type="linenumber">61</context></context-group> </trans-unit> - <trans-unit id="_msg911"> + <trans-unit id="_msg930"> <source xml:space="preserve">No dump file provided. To use dump, -dumpfile=<filename> must be provided.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">61</context></context-group> + <context-group purpose="location"><context context-type="linenumber">64</context></context-group> </trans-unit> - <trans-unit id="_msg912"> + <trans-unit id="_msg931"> <source xml:space="preserve">No wallet file format provided. To use createfromdump, -format=<format> must be provided.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">63</context></context-group> + <context-group purpose="location"><context context-type="linenumber">66</context></context-group> </trans-unit> - <trans-unit id="_msg913"> + <trans-unit id="_msg932"> <source xml:space="preserve">Please check that your computer's date and time are correct! If your clock is wrong, %s will not work properly.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">66</context></context-group> + <context-group purpose="location"><context context-type="linenumber">69</context></context-group> </trans-unit> - <trans-unit id="_msg914"> + <trans-unit id="_msg933"> <source xml:space="preserve">Please contribute if you find %s useful. Visit %s for further information about the software.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">69</context></context-group> + <context-group purpose="location"><context context-type="linenumber">72</context></context-group> </trans-unit> - <trans-unit id="_msg915"> + <trans-unit id="_msg934"> <source xml:space="preserve">Prune configured below the minimum of %d MiB. Please use a higher number.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">72</context></context-group> + <context-group purpose="location"><context context-type="linenumber">75</context></context-group> </trans-unit> - <trans-unit id="_msg916"> + <trans-unit id="_msg935"> <source xml:space="preserve">Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">74</context></context-group> + <context-group purpose="location"><context context-type="linenumber">77</context></context-group> </trans-unit> - <trans-unit id="_msg917"> + <trans-unit id="_msg936"> <source xml:space="preserve">SQLiteDatabase: Unknown sqlite wallet schema version %d. Only version %d is supported</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">77</context></context-group> + <context-group purpose="location"><context context-type="linenumber">80</context></context-group> </trans-unit> - <trans-unit id="_msg918"> + <trans-unit id="_msg937"> <source xml:space="preserve">The block database contains a block which appears to be from the future. This may be due to your computer's date and time being set incorrectly. Only rebuild the block database if you are sure that your computer's date and time are correct</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">80</context></context-group> + <context-group purpose="location"><context context-type="linenumber">83</context></context-group> </trans-unit> - <trans-unit id="_msg919"> + <trans-unit id="_msg938"> <source xml:space="preserve">The transaction amount is too small to send after the fee has been deducted</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">85</context></context-group> + <context-group purpose="location"><context context-type="linenumber">88</context></context-group> </trans-unit> - <trans-unit id="_msg920"> + <trans-unit id="_msg939"> <source xml:space="preserve">This error could occur if this wallet was not shutdown cleanly and was last loaded using a build with a newer version of Berkeley DB. If so, please use the software that last loaded this wallet</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">87</context></context-group> + <context-group purpose="location"><context context-type="linenumber">90</context></context-group> </trans-unit> - <trans-unit id="_msg921"> + <trans-unit id="_msg940"> <source xml:space="preserve">This is a pre-release test build - use at your own risk - do not use for mining or merchant applications</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">91</context></context-group> + <context-group purpose="location"><context context-type="linenumber">94</context></context-group> </trans-unit> - <trans-unit id="_msg922"> + <trans-unit id="_msg941"> <source xml:space="preserve">This is the maximum transaction fee you pay (in addition to the normal fee) to prioritize partial spend avoidance over regular coin selection.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">94</context></context-group> + <context-group purpose="location"><context context-type="linenumber">97</context></context-group> </trans-unit> - <trans-unit id="_msg923"> + <trans-unit id="_msg942"> <source xml:space="preserve">This is the transaction fee you may discard if change is smaller than dust at this level</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">97</context></context-group> + <context-group purpose="location"><context context-type="linenumber">100</context></context-group> </trans-unit> - <trans-unit id="_msg924"> + <trans-unit id="_msg943"> <source xml:space="preserve">This is the transaction fee you may pay when fee estimates are not available.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">100</context></context-group> + <context-group purpose="location"><context context-type="linenumber">103</context></context-group> </trans-unit> - <trans-unit id="_msg925"> + <trans-unit id="_msg944"> <source xml:space="preserve">Total length of network version string (%i) exceeds maximum length (%i). Reduce the number or size of uacomments.</source> <target xml:space="preserve"></target> - <context-group purpose="location"><context context-type="linenumber">102</context></context-group> - </trans-unit> - <trans-unit id="_msg926"> - <source xml:space="preserve">Transaction needs a change address, but we can't generate it. Please call keypoolrefill first.</source> - <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">105</context></context-group> </trans-unit> - <trans-unit id="_msg927"> + <trans-unit id="_msg945"> <source xml:space="preserve">Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">108</context></context-group> </trans-unit> - <trans-unit id="_msg928"> + <trans-unit id="_msg946"> <source xml:space="preserve">Unknown wallet file format "%s" provided. Please provide one of "bdb" or "sqlite".</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">111</context></context-group> </trans-unit> - <trans-unit id="_msg929"> + <trans-unit id="_msg947"> <source xml:space="preserve">Warning: Dumpfile wallet format "%s" does not match command line specified format "%s".</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">114</context></context-group> </trans-unit> - <trans-unit id="_msg930"> + <trans-unit id="_msg948"> <source xml:space="preserve">Warning: Private keys detected in wallet {%s} with disabled private keys</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">117</context></context-group> </trans-unit> - <trans-unit id="_msg931"> + <trans-unit id="_msg949"> <source xml:space="preserve">Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">119</context></context-group> </trans-unit> - <trans-unit id="_msg932"> + <trans-unit id="_msg950"> <source xml:space="preserve">Witness data for blocks after height %d requires validation. Please restart with -reindex.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">122</context></context-group> </trans-unit> - <trans-unit id="_msg933"> + <trans-unit id="_msg951"> <source xml:space="preserve">You need to rebuild the database using -reindex to go back to unpruned mode. This will redownload the entire blockchain</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">125</context></context-group> </trans-unit> - <trans-unit id="_msg934"> + <trans-unit id="_msg952"> <source xml:space="preserve">%s is set very high!</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">128</context></context-group> </trans-unit> - <trans-unit id="_msg935"> + <trans-unit id="_msg953"> <source xml:space="preserve">-maxmempool must be at least %d MB</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">129</context></context-group> </trans-unit> - <trans-unit id="_msg936"> + <trans-unit id="_msg954"> <source xml:space="preserve">A fatal internal error occurred, see debug.log for details</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">130</context></context-group> </trans-unit> - <trans-unit id="_msg937"> + <trans-unit id="_msg955"> <source xml:space="preserve">Cannot resolve -%s address: '%s'</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">131</context></context-group> </trans-unit> - <trans-unit id="_msg938"> + <trans-unit id="_msg956"> <source xml:space="preserve">Cannot set -peerblockfilters without -blockfilterindex.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">132</context></context-group> </trans-unit> - <trans-unit id="_msg939"> + <trans-unit id="_msg957"> <source xml:space="preserve">Cannot write to data directory '%s'; check permissions.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">133</context></context-group> </trans-unit> - <trans-unit id="_msg940"> + <trans-unit id="_msg958"> <source xml:space="preserve">Change index out of range</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">134</context></context-group> </trans-unit> - <trans-unit id="_msg941"> + <trans-unit id="_msg959"> <source xml:space="preserve">Config setting for %s only applied on %s network when in [%s] section.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">135</context></context-group> </trans-unit> - <trans-unit id="_msg942"> + <trans-unit id="_msg960"> <source xml:space="preserve">Copyright (C) %i-%i</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">136</context></context-group> </trans-unit> - <trans-unit id="_msg943"> + <trans-unit id="_msg961"> <source xml:space="preserve">Corrupted block database detected</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">137</context></context-group> </trans-unit> - <trans-unit id="_msg944"> + <trans-unit id="_msg962"> <source xml:space="preserve">Could not find asmap file %s</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">138</context></context-group> </trans-unit> - <trans-unit id="_msg945"> + <trans-unit id="_msg963"> <source xml:space="preserve">Could not parse asmap file %s</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">139</context></context-group> </trans-unit> - <trans-unit id="_msg946"> + <trans-unit id="_msg964"> <source xml:space="preserve">Disk space is too low!</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">140</context></context-group> </trans-unit> - <trans-unit id="_msg947"> + <trans-unit id="_msg965"> <source xml:space="preserve">Do you want to rebuild the block database now?</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">141</context></context-group> </trans-unit> - <trans-unit id="_msg948"> + <trans-unit id="_msg966"> <source xml:space="preserve">Done loading</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">142</context></context-group> </trans-unit> - <trans-unit id="_msg949"> + <trans-unit id="_msg967"> <source xml:space="preserve">Dump file %s does not exist.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">143</context></context-group> </trans-unit> - <trans-unit id="_msg950"> + <trans-unit id="_msg968"> <source xml:space="preserve">Error creating %s</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">144</context></context-group> </trans-unit> - <trans-unit id="_msg951"> + <trans-unit id="_msg969"> <source xml:space="preserve">Error initializing block database</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">145</context></context-group> </trans-unit> - <trans-unit id="_msg952"> + <trans-unit id="_msg970"> <source xml:space="preserve">Error initializing wallet database environment %s!</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">146</context></context-group> </trans-unit> - <trans-unit id="_msg953"> + <trans-unit id="_msg971"> <source xml:space="preserve">Error loading %s</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">147</context></context-group> </trans-unit> - <trans-unit id="_msg954"> + <trans-unit id="_msg972"> <source xml:space="preserve">Error loading %s: Private keys can only be disabled during creation</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">148</context></context-group> </trans-unit> - <trans-unit id="_msg955"> + <trans-unit id="_msg973"> <source xml:space="preserve">Error loading %s: Wallet corrupted</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">149</context></context-group> </trans-unit> - <trans-unit id="_msg956"> + <trans-unit id="_msg974"> <source xml:space="preserve">Error loading %s: Wallet requires newer version of %s</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">150</context></context-group> </trans-unit> - <trans-unit id="_msg957"> + <trans-unit id="_msg975"> <source xml:space="preserve">Error loading block database</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">151</context></context-group> </trans-unit> - <trans-unit id="_msg958"> + <trans-unit id="_msg976"> <source xml:space="preserve">Error opening block database</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">152</context></context-group> </trans-unit> - <trans-unit id="_msg959"> + <trans-unit id="_msg977"> <source xml:space="preserve">Error reading from database, shutting down.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">153</context></context-group> </trans-unit> - <trans-unit id="_msg960"> + <trans-unit id="_msg978"> <source xml:space="preserve">Error reading next record from wallet database</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">154</context></context-group> </trans-unit> - <trans-unit id="_msg961"> + <trans-unit id="_msg979"> <source xml:space="preserve">Error upgrading chainstate database</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">155</context></context-group> </trans-unit> - <trans-unit id="_msg962"> + <trans-unit id="_msg980"> <source xml:space="preserve">Error: Couldn't create cursor into database</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">156</context></context-group> </trans-unit> - <trans-unit id="_msg963"> + <trans-unit id="_msg981"> <source xml:space="preserve">Error: Disk space is low for %s</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">157</context></context-group> </trans-unit> - <trans-unit id="_msg964"> + <trans-unit id="_msg982"> <source xml:space="preserve">Error: Dumpfile checksum does not match. Computed %s, expected %s</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">158</context></context-group> </trans-unit> - <trans-unit id="_msg965"> + <trans-unit id="_msg983"> <source xml:space="preserve">Error: Got key that was not hex: %s</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">159</context></context-group> </trans-unit> - <trans-unit id="_msg966"> + <trans-unit id="_msg984"> <source xml:space="preserve">Error: Got value that was not hex: %s</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">160</context></context-group> </trans-unit> - <trans-unit id="_msg967"> + <trans-unit id="_msg985"> <source xml:space="preserve">Error: Keypool ran out, please call keypoolrefill first</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">161</context></context-group> </trans-unit> - <trans-unit id="_msg968"> + <trans-unit id="_msg986"> <source xml:space="preserve">Error: Missing checksum</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">162</context></context-group> </trans-unit> - <trans-unit id="_msg969"> - <source xml:space="preserve">Error: Unable to parse version %u as a uint32_t</source> + <trans-unit id="_msg987"> + <source xml:space="preserve">Error: No %s addresses available.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">163</context></context-group> </trans-unit> - <trans-unit id="_msg970"> - <source xml:space="preserve">Error: Unable to write record to new wallet</source> + <trans-unit id="_msg988"> + <source xml:space="preserve">Error: Unable to parse version %u as a uint32_t</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">164</context></context-group> </trans-unit> - <trans-unit id="_msg971"> - <source xml:space="preserve">Failed to listen on any port. Use -listen=0 if you want this.</source> + <trans-unit id="_msg989"> + <source xml:space="preserve">Error: Unable to write record to new wallet</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">165</context></context-group> </trans-unit> - <trans-unit id="_msg972"> - <source xml:space="preserve">Failed to rescan the wallet during initialization</source> + <trans-unit id="_msg990"> + <source xml:space="preserve">Failed to listen on any port. Use -listen=0 if you want this.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">166</context></context-group> </trans-unit> - <trans-unit id="_msg973"> - <source xml:space="preserve">Failed to verify database</source> + <trans-unit id="_msg991"> + <source xml:space="preserve">Failed to rescan the wallet during initialization</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">167</context></context-group> </trans-unit> - <trans-unit id="_msg974"> - <source xml:space="preserve">Fee rate (%s) is lower than the minimum fee rate setting (%s)</source> + <trans-unit id="_msg992"> + <source xml:space="preserve">Failed to verify database</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">168</context></context-group> </trans-unit> - <trans-unit id="_msg975"> - <source xml:space="preserve">Ignoring duplicate -wallet %s.</source> + <trans-unit id="_msg993"> + <source xml:space="preserve">Fee rate (%s) is lower than the minimum fee rate setting (%s)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">169</context></context-group> </trans-unit> - <trans-unit id="_msg976"> - <source xml:space="preserve">Importing…</source> + <trans-unit id="_msg994"> + <source xml:space="preserve">Ignoring duplicate -wallet %s.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">170</context></context-group> </trans-unit> - <trans-unit id="_msg977"> - <source xml:space="preserve">Incorrect or no genesis block found. Wrong datadir for network?</source> + <trans-unit id="_msg995"> + <source xml:space="preserve">Importing…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">171</context></context-group> </trans-unit> - <trans-unit id="_msg978"> - <source xml:space="preserve">Initialization sanity check failed. %s is shutting down.</source> + <trans-unit id="_msg996"> + <source xml:space="preserve">Incorrect or no genesis block found. Wrong datadir for network?</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">172</context></context-group> </trans-unit> - <trans-unit id="_msg979"> - <source xml:space="preserve">Insufficient funds</source> + <trans-unit id="_msg997"> + <source xml:space="preserve">Initialization sanity check failed. %s is shutting down.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">173</context></context-group> </trans-unit> - <trans-unit id="_msg980"> - <source xml:space="preserve">Invalid -i2psam address or hostname: '%s'</source> + <trans-unit id="_msg998"> + <source xml:space="preserve">Insufficient funds</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">174</context></context-group> </trans-unit> - <trans-unit id="_msg981"> - <source xml:space="preserve">Invalid -onion address or hostname: '%s'</source> + <trans-unit id="_msg999"> + <source xml:space="preserve">Invalid -i2psam address or hostname: '%s'</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">175</context></context-group> </trans-unit> - <trans-unit id="_msg982"> - <source xml:space="preserve">Invalid -proxy address or hostname: '%s'</source> + <trans-unit id="_msg1000"> + <source xml:space="preserve">Invalid -onion address or hostname: '%s'</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">176</context></context-group> </trans-unit> - <trans-unit id="_msg983"> - <source xml:space="preserve">Invalid P2P permission: '%s'</source> + <trans-unit id="_msg1001"> + <source xml:space="preserve">Invalid -proxy address or hostname: '%s'</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">177</context></context-group> </trans-unit> - <trans-unit id="_msg984"> - <source xml:space="preserve">Invalid amount for -%s=<amount>: '%s'</source> + <trans-unit id="_msg1002"> + <source xml:space="preserve">Invalid P2P permission: '%s'</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">178</context></context-group> </trans-unit> - <trans-unit id="_msg985"> - <source xml:space="preserve">Invalid amount for -discardfee=<amount>: '%s'</source> + <trans-unit id="_msg1003"> + <source xml:space="preserve">Invalid amount for -%s=<amount>: '%s'</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">179</context></context-group> </trans-unit> - <trans-unit id="_msg986"> - <source xml:space="preserve">Invalid amount for -fallbackfee=<amount>: '%s'</source> + <trans-unit id="_msg1004"> + <source xml:space="preserve">Invalid amount for -discardfee=<amount>: '%s'</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">180</context></context-group> </trans-unit> - <trans-unit id="_msg987"> - <source xml:space="preserve">Invalid amount for -paytxfee=<amount>: '%s' (must be at least %s)</source> + <trans-unit id="_msg1005"> + <source xml:space="preserve">Invalid amount for -fallbackfee=<amount>: '%s'</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">181</context></context-group> </trans-unit> - <trans-unit id="_msg988"> - <source xml:space="preserve">Invalid netmask specified in -whitelist: '%s'</source> + <trans-unit id="_msg1006"> + <source xml:space="preserve">Invalid amount for -paytxfee=<amount>: '%s' (must be at least %s)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">182</context></context-group> </trans-unit> - <trans-unit id="_msg989"> - <source xml:space="preserve">Loading P2P addresses…</source> + <trans-unit id="_msg1007"> + <source xml:space="preserve">Invalid netmask specified in -whitelist: '%s'</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">183</context></context-group> </trans-unit> - <trans-unit id="_msg990"> - <source xml:space="preserve">Loading banlist…</source> + <trans-unit id="_msg1008"> + <source xml:space="preserve">Loading P2P addresses…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">184</context></context-group> </trans-unit> - <trans-unit id="_msg991"> - <source xml:space="preserve">Loading block index…</source> + <trans-unit id="_msg1009"> + <source xml:space="preserve">Loading banlist…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">185</context></context-group> </trans-unit> - <trans-unit id="_msg992"> - <source xml:space="preserve">Loading wallet…</source> + <trans-unit id="_msg1010"> + <source xml:space="preserve">Loading block index…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">186</context></context-group> </trans-unit> - <trans-unit id="_msg993"> - <source xml:space="preserve">Need to specify a port with -whitebind: '%s'</source> + <trans-unit id="_msg1011"> + <source xml:space="preserve">Loading wallet…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">187</context></context-group> </trans-unit> - <trans-unit id="_msg994"> - <source xml:space="preserve">No proxy server specified. Use -proxy=<ip> or -proxy=<ip:port>.</source> + <trans-unit id="_msg1012"> + <source xml:space="preserve">Need to specify a port with -whitebind: '%s'</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">188</context></context-group> </trans-unit> - <trans-unit id="_msg995"> - <source xml:space="preserve">Not enough file descriptors available.</source> + <trans-unit id="_msg1013"> + <source xml:space="preserve">No proxy server specified. Use -proxy=<ip> or -proxy=<ip:port>.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">189</context></context-group> </trans-unit> - <trans-unit id="_msg996"> - <source xml:space="preserve">Prune cannot be configured with a negative value.</source> + <trans-unit id="_msg1014"> + <source xml:space="preserve">Not enough file descriptors available.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">190</context></context-group> </trans-unit> - <trans-unit id="_msg997"> - <source xml:space="preserve">Prune mode is incompatible with -coinstatsindex.</source> + <trans-unit id="_msg1015"> + <source xml:space="preserve">Prune cannot be configured with a negative value.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">191</context></context-group> </trans-unit> - <trans-unit id="_msg998"> - <source xml:space="preserve">Prune mode is incompatible with -txindex.</source> + <trans-unit id="_msg1016"> + <source xml:space="preserve">Prune mode is incompatible with -coinstatsindex.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">192</context></context-group> </trans-unit> - <trans-unit id="_msg999"> - <source xml:space="preserve">Pruning blockstore…</source> + <trans-unit id="_msg1017"> + <source xml:space="preserve">Prune mode is incompatible with -txindex.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">193</context></context-group> </trans-unit> - <trans-unit id="_msg1000"> - <source xml:space="preserve">Reducing -maxconnections from %d to %d, because of system limitations.</source> + <trans-unit id="_msg1018"> + <source xml:space="preserve">Pruning blockstore…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">194</context></context-group> </trans-unit> - <trans-unit id="_msg1001"> - <source xml:space="preserve">Replaying blocks…</source> + <trans-unit id="_msg1019"> + <source xml:space="preserve">Reducing -maxconnections from %d to %d, because of system limitations.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">195</context></context-group> </trans-unit> - <trans-unit id="_msg1002"> - <source xml:space="preserve">Rescanning…</source> + <trans-unit id="_msg1020"> + <source xml:space="preserve">Replaying blocks…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">196</context></context-group> </trans-unit> - <trans-unit id="_msg1003"> - <source xml:space="preserve">SQLiteDatabase: Failed to execute statement to verify database: %s</source> + <trans-unit id="_msg1021"> + <source xml:space="preserve">Rescanning…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">197</context></context-group> </trans-unit> - <trans-unit id="_msg1004"> - <source xml:space="preserve">SQLiteDatabase: Failed to prepare statement to verify database: %s</source> + <trans-unit id="_msg1022"> + <source xml:space="preserve">SQLiteDatabase: Failed to execute statement to verify database: %s</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">198</context></context-group> </trans-unit> - <trans-unit id="_msg1005"> - <source xml:space="preserve">SQLiteDatabase: Failed to read database verification error: %s</source> + <trans-unit id="_msg1023"> + <source xml:space="preserve">SQLiteDatabase: Failed to prepare statement to verify database: %s</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">199</context></context-group> </trans-unit> - <trans-unit id="_msg1006"> - <source xml:space="preserve">SQLiteDatabase: Unexpected application id. Expected %u, got %u</source> + <trans-unit id="_msg1024"> + <source xml:space="preserve">SQLiteDatabase: Failed to read database verification error: %s</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">200</context></context-group> </trans-unit> - <trans-unit id="_msg1007"> - <source xml:space="preserve">Section [%s] is not recognized.</source> + <trans-unit id="_msg1025"> + <source xml:space="preserve">SQLiteDatabase: Unexpected application id. Expected %u, got %u</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">201</context></context-group> </trans-unit> - <trans-unit id="_msg1008"> - <source xml:space="preserve">Signing transaction failed</source> + <trans-unit id="_msg1026"> + <source xml:space="preserve">Section [%s] is not recognized.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">202</context></context-group> </trans-unit> - <trans-unit id="_msg1009"> - <source xml:space="preserve">Specified -walletdir "%s" does not exist</source> + <trans-unit id="_msg1027"> + <source xml:space="preserve">Signing transaction failed</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">203</context></context-group> </trans-unit> - <trans-unit id="_msg1010"> - <source xml:space="preserve">Specified -walletdir "%s" is a relative path</source> + <trans-unit id="_msg1028"> + <source xml:space="preserve">Specified -walletdir "%s" does not exist</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">204</context></context-group> </trans-unit> - <trans-unit id="_msg1011"> - <source xml:space="preserve">Specified -walletdir "%s" is not a directory</source> + <trans-unit id="_msg1029"> + <source xml:space="preserve">Specified -walletdir "%s" is a relative path</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">205</context></context-group> </trans-unit> - <trans-unit id="_msg1012"> - <source xml:space="preserve">Specified blocks directory "%s" does not exist.</source> + <trans-unit id="_msg1030"> + <source xml:space="preserve">Specified -walletdir "%s" is not a directory</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">206</context></context-group> </trans-unit> - <trans-unit id="_msg1013"> - <source xml:space="preserve">Starting network threads…</source> + <trans-unit id="_msg1031"> + <source xml:space="preserve">Specified blocks directory "%s" does not exist.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">207</context></context-group> </trans-unit> - <trans-unit id="_msg1014"> - <source xml:space="preserve">The source code is available from %s.</source> + <trans-unit id="_msg1032"> + <source xml:space="preserve">Starting network threads…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">208</context></context-group> </trans-unit> - <trans-unit id="_msg1015"> - <source xml:space="preserve">The specified config file %s does not exist</source> + <trans-unit id="_msg1033"> + <source xml:space="preserve">The source code is available from %s.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">209</context></context-group> </trans-unit> - <trans-unit id="_msg1016"> - <source xml:space="preserve">The transaction amount is too small to pay the fee</source> + <trans-unit id="_msg1034"> + <source xml:space="preserve">The specified config file %s does not exist</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">210</context></context-group> </trans-unit> - <trans-unit id="_msg1017"> - <source xml:space="preserve">The wallet will avoid paying less than the minimum relay fee.</source> + <trans-unit id="_msg1035"> + <source xml:space="preserve">The transaction amount is too small to pay the fee</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">211</context></context-group> </trans-unit> - <trans-unit id="_msg1018"> - <source xml:space="preserve">This is experimental software.</source> + <trans-unit id="_msg1036"> + <source xml:space="preserve">The wallet will avoid paying less than the minimum relay fee.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">212</context></context-group> </trans-unit> - <trans-unit id="_msg1019"> - <source xml:space="preserve">This is the minimum transaction fee you pay on every transaction.</source> + <trans-unit id="_msg1037"> + <source xml:space="preserve">This is experimental software.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">213</context></context-group> </trans-unit> - <trans-unit id="_msg1020"> - <source xml:space="preserve">This is the transaction fee you will pay if you send a transaction.</source> + <trans-unit id="_msg1038"> + <source xml:space="preserve">This is the minimum transaction fee you pay on every transaction.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">214</context></context-group> </trans-unit> - <trans-unit id="_msg1021"> - <source xml:space="preserve">Transaction amount too small</source> + <trans-unit id="_msg1039"> + <source xml:space="preserve">This is the transaction fee you will pay if you send a transaction.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">215</context></context-group> </trans-unit> - <trans-unit id="_msg1022"> - <source xml:space="preserve">Transaction amounts must not be negative</source> + <trans-unit id="_msg1040"> + <source xml:space="preserve">Transaction amount too small</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">216</context></context-group> </trans-unit> - <trans-unit id="_msg1023"> - <source xml:space="preserve">Transaction has too long of a mempool chain</source> + <trans-unit id="_msg1041"> + <source xml:space="preserve">Transaction amounts must not be negative</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">217</context></context-group> </trans-unit> - <trans-unit id="_msg1024"> - <source xml:space="preserve">Transaction must have at least one recipient</source> + <trans-unit id="_msg1042"> + <source xml:space="preserve">Transaction has too long of a mempool chain</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">218</context></context-group> </trans-unit> - <trans-unit id="_msg1025"> - <source xml:space="preserve">Transaction too large</source> + <trans-unit id="_msg1043"> + <source xml:space="preserve">Transaction must have at least one recipient</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">219</context></context-group> </trans-unit> - <trans-unit id="_msg1026"> - <source xml:space="preserve">Unable to bind to %s on this computer (bind returned error %s)</source> + <trans-unit id="_msg1044"> + <source xml:space="preserve">Transaction needs a change address, but we can't generate it. %s</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">220</context></context-group> </trans-unit> - <trans-unit id="_msg1027"> - <source xml:space="preserve">Unable to bind to %s on this computer. %s is probably already running.</source> + <trans-unit id="_msg1045"> + <source xml:space="preserve">Transaction too large</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">221</context></context-group> </trans-unit> - <trans-unit id="_msg1028"> - <source xml:space="preserve">Unable to create the PID file '%s': %s</source> + <trans-unit id="_msg1046"> + <source xml:space="preserve">Unable to bind to %s on this computer (bind returned error %s)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">222</context></context-group> </trans-unit> - <trans-unit id="_msg1029"> - <source xml:space="preserve">Unable to generate initial keys</source> + <trans-unit id="_msg1047"> + <source xml:space="preserve">Unable to bind to %s on this computer. %s is probably already running.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">223</context></context-group> </trans-unit> - <trans-unit id="_msg1030"> - <source xml:space="preserve">Unable to generate keys</source> + <trans-unit id="_msg1048"> + <source xml:space="preserve">Unable to create the PID file '%s': %s</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">224</context></context-group> </trans-unit> - <trans-unit id="_msg1031"> - <source xml:space="preserve">Unable to open %s for writing</source> + <trans-unit id="_msg1049"> + <source xml:space="preserve">Unable to generate initial keys</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">225</context></context-group> </trans-unit> - <trans-unit id="_msg1032"> - <source xml:space="preserve">Unable to start HTTP server. See debug log for details.</source> + <trans-unit id="_msg1050"> + <source xml:space="preserve">Unable to generate keys</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">226</context></context-group> </trans-unit> - <trans-unit id="_msg1033"> - <source xml:space="preserve">Unknown -blockfilterindex value %s.</source> + <trans-unit id="_msg1051"> + <source xml:space="preserve">Unable to open %s for writing</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">227</context></context-group> </trans-unit> - <trans-unit id="_msg1034"> - <source xml:space="preserve">Unknown address type '%s'</source> + <trans-unit id="_msg1052"> + <source xml:space="preserve">Unable to start HTTP server. See debug log for details.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">228</context></context-group> </trans-unit> - <trans-unit id="_msg1035"> - <source xml:space="preserve">Unknown change type '%s'</source> + <trans-unit id="_msg1053"> + <source xml:space="preserve">Unknown -blockfilterindex value %s.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">229</context></context-group> </trans-unit> - <trans-unit id="_msg1036"> - <source xml:space="preserve">Unknown network specified in -onlynet: '%s'</source> + <trans-unit id="_msg1054"> + <source xml:space="preserve">Unknown address type '%s'</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">230</context></context-group> </trans-unit> - <trans-unit id="_msg1037"> - <source xml:space="preserve">Unsupported logging category %s=%s.</source> + <trans-unit id="_msg1055"> + <source xml:space="preserve">Unknown change type '%s'</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">231</context></context-group> </trans-unit> - <trans-unit id="_msg1038"> - <source xml:space="preserve">Upgrading UTXO database</source> + <trans-unit id="_msg1056"> + <source xml:space="preserve">Unknown network specified in -onlynet: '%s'</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">232</context></context-group> </trans-unit> - <trans-unit id="_msg1039"> - <source xml:space="preserve">Upgrading txindex database</source> + <trans-unit id="_msg1057"> + <source xml:space="preserve">Unknown new rules activated (versionbit %i)</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">233</context></context-group> </trans-unit> - <trans-unit id="_msg1040"> - <source xml:space="preserve">User Agent comment (%s) contains unsafe characters.</source> + <trans-unit id="_msg1058"> + <source xml:space="preserve">Unsupported logging category %s=%s.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">234</context></context-group> </trans-unit> - <trans-unit id="_msg1041"> - <source xml:space="preserve">Verifying blocks…</source> + <trans-unit id="_msg1059"> + <source xml:space="preserve">Upgrading UTXO database</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">235</context></context-group> </trans-unit> - <trans-unit id="_msg1042"> - <source xml:space="preserve">Verifying wallet(s)…</source> + <trans-unit id="_msg1060"> + <source xml:space="preserve">Upgrading txindex database</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">236</context></context-group> </trans-unit> - <trans-unit id="_msg1043"> - <source xml:space="preserve">Wallet needed to be rewritten: restart %s to complete</source> + <trans-unit id="_msg1061"> + <source xml:space="preserve">User Agent comment (%s) contains unsafe characters.</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">237</context></context-group> </trans-unit> - <trans-unit id="_msg1044"> - <source xml:space="preserve">Warning: unknown new rules activated (versionbit %i)</source> + <trans-unit id="_msg1062"> + <source xml:space="preserve">Verifying blocks…</source> <target xml:space="preserve"></target> <context-group purpose="location"><context context-type="linenumber">238</context></context-group> </trans-unit> + <trans-unit id="_msg1063"> + <source xml:space="preserve">Verifying wallet(s)…</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">239</context></context-group> + </trans-unit> + <trans-unit id="_msg1064"> + <source xml:space="preserve">Wallet needed to be rewritten: restart %s to complete</source> + <target xml:space="preserve"></target> + <context-group purpose="location"><context context-type="linenumber">240</context></context-group> + </trans-unit> </group> </body></file> </xliff> diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index b12fe96567..0cc2d61df6 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -210,6 +210,7 @@ void OptionsDialog::setModel(OptionsModel *_model) connect(ui->spendZeroConfChange, &QCheckBox::clicked, this, &OptionsDialog::showRestartWarning); /* Network */ connect(ui->allowIncoming, &QCheckBox::clicked, this, &OptionsDialog::showRestartWarning); + connect(ui->enableServer, &QCheckBox::clicked, this, &OptionsDialog::showRestartWarning); connect(ui->connectSocks, &QCheckBox::clicked, this, &OptionsDialog::showRestartWarning); connect(ui->connectSocksTor, &QCheckBox::clicked, this, &OptionsDialog::showRestartWarning); /* Display */ @@ -239,12 +240,14 @@ void OptionsDialog::setMapper() /* Wallet */ mapper->addMapping(ui->spendZeroConfChange, OptionsModel::SpendZeroConfChange); mapper->addMapping(ui->coinControlFeatures, OptionsModel::CoinControlFeatures); + mapper->addMapping(ui->subFeeFromAmount, OptionsModel::SubFeeFromAmount); mapper->addMapping(ui->externalSignerPath, OptionsModel::ExternalSignerPath); /* Network */ mapper->addMapping(ui->mapPortUpnp, OptionsModel::MapPortUPnP); mapper->addMapping(ui->mapPortNatpmp, OptionsModel::MapPortNatpmp); mapper->addMapping(ui->allowIncoming, OptionsModel::Listen); + mapper->addMapping(ui->enableServer, OptionsModel::Server); mapper->addMapping(ui->connectSocks, OptionsModel::ProxyUse); mapper->addMapping(ui->proxyIp, OptionsModel::ProxyIP); @@ -289,16 +292,29 @@ void OptionsDialog::on_resetButton_clicked() /* reset all options and close GUI */ model->Reset(); - QApplication::quit(); + close(); + Q_EMIT quitOnReset(); } } void OptionsDialog::on_openBitcoinConfButton_clicked() { - /* explain the purpose of the config file */ - QMessageBox::information(this, tr("Configuration options"), - tr("The configuration file is used to specify advanced user options which override GUI settings. " - "Additionally, any command-line options will override this configuration file.")); + QMessageBox config_msgbox(this); + config_msgbox.setIcon(QMessageBox::Information); + //: Window title text of pop-up box that allows opening up of configuration file. + config_msgbox.setWindowTitle(tr("Configuration options")); + /*: Explanatory text about the priority order of instructions considered by client. + The order from high to low being: command-line, configuration file, GUI settings. */ + config_msgbox.setText(tr("The configuration file is used to specify advanced user options which override GUI settings. " + "Additionally, any command-line options will override this configuration file.")); + + QPushButton* open_button = config_msgbox.addButton(tr("Continue"), QMessageBox::ActionRole); + config_msgbox.addButton(tr("Cancel"), QMessageBox::RejectRole); + open_button->setDefault(true); + + config_msgbox.exec(); + + if (config_msgbox.clickedButton() != open_button) return; /* show an error if there was some problem opening the file */ if (!GUIUtil::openBitcoinConf()) diff --git a/src/qt/optionsdialog.h b/src/qt/optionsdialog.h index ba35ff3b67..f14aec3449 100644 --- a/src/qt/optionsdialog.h +++ b/src/qt/optionsdialog.h @@ -68,6 +68,7 @@ private Q_SLOTS: Q_SIGNALS: void proxyIpChecks(QValidatedLineEdit *pUiProxyIp, uint16_t nProxyPort); + void quitOnReset(); private: Ui::OptionsDialog *ui; diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index 24a4e9ee96..9e2f38f7ec 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -124,6 +124,11 @@ void OptionsModel::Init(bool resetSettings) if (!gArgs.SoftSetArg("-signer", settings.value("external_signer_path").toString().toStdString())) { addOverriddenOption("-signer"); } + + if (!settings.contains("SubFeeFromAmount")) { + settings.setValue("SubFeeFromAmount", false); + } + m_sub_fee_from_amount = settings.value("SubFeeFromAmount", false).toBool(); #endif // Network @@ -144,6 +149,13 @@ void OptionsModel::Init(bool resetSettings) if (!gArgs.SoftSetBoolArg("-listen", settings.value("fListen").toBool())) addOverriddenOption("-listen"); + if (!settings.contains("server")) { + settings.setValue("server", false); + } + if (!gArgs.SoftSetBoolArg("-server", settings.value("server").toBool())) { + addOverriddenOption("-server"); + } + if (!settings.contains("fUseProxy")) settings.setValue("fUseProxy", false); if (!settings.contains("addrProxy")) @@ -335,6 +347,8 @@ QVariant OptionsModel::data(const QModelIndex & index, int role) const return settings.value("bSpendZeroConfChange"); case ExternalSignerPath: return settings.value("external_signer_path"); + case SubFeeFromAmount: + return m_sub_fee_from_amount; #endif case DisplayUnit: return nDisplayUnit; @@ -356,6 +370,8 @@ QVariant OptionsModel::data(const QModelIndex & index, int role) const return settings.value("nThreadsScriptVerif"); case Listen: return settings.value("fListen"); + case Server: + return settings.value("server"); default: return QVariant(); } @@ -460,6 +476,10 @@ bool OptionsModel::setData(const QModelIndex & index, const QVariant & value, in setRestartRequired(true); } break; + case SubFeeFromAmount: + m_sub_fee_from_amount = value.toBool(); + settings.setValue("SubFeeFromAmount", m_sub_fee_from_amount); + break; #endif case DisplayUnit: setDisplayUnit(value); @@ -517,6 +537,12 @@ bool OptionsModel::setData(const QModelIndex & index, const QVariant & value, in setRestartRequired(true); } break; + case Server: + if (settings.value("server") != value) { + settings.setValue("server", value); + setRestartRequired(true); + } + break; default: break; } diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index 535843e8ba..8f1513e48d 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -5,7 +5,6 @@ #ifndef BITCOIN_QT_OPTIONSMODEL_H #define BITCOIN_QT_OPTIONSMODEL_H -#include <amount.h> #include <cstdint> #include <qt/guiconstants.h> @@ -61,6 +60,7 @@ public: Language, // QString UseEmbeddedMonospacedFont, // bool CoinControlFeatures, // bool + SubFeeFromAmount, // bool ThreadsScriptVerif, // int Prune, // bool PruneSize, // int @@ -68,6 +68,7 @@ public: ExternalSignerPath, // QString SpendZeroConfChange, // bool Listen, // bool + Server, // bool OptionIDRowCount, }; @@ -88,6 +89,7 @@ public: QString getThirdPartyTxUrls() const { return strThirdPartyTxUrls; } bool getUseEmbeddedMonospacedFont() const { return m_use_embedded_monospaced_font; } bool getCoinControlFeatures() const { return fCoinControlFeatures; } + bool getSubFeeFromAmount() const { return m_sub_fee_from_amount; } const QString& getOverriddenByCommandLine() { return strOverriddenByCommandLine; } /* Explicit setters */ @@ -112,6 +114,7 @@ private: QString strThirdPartyTxUrls; bool m_use_embedded_monospaced_font; bool fCoinControlFeatures; + bool m_sub_fee_from_amount; /* settings that were overridden by command-line */ QString strOverriddenByCommandLine; diff --git a/src/qt/overviewpage.cpp b/src/qt/overviewpage.cpp index 26e3dd0d60..864a62edc8 100644 --- a/src/qt/overviewpage.cpp +++ b/src/qt/overviewpage.cpp @@ -69,20 +69,18 @@ public: foreground = brush.color(); } - painter->setPen(foreground); - QRect boundingRect; - painter->drawText(addressRect, Qt::AlignLeft | Qt::AlignVCenter, address, &boundingRect); - int address_rect_min_width = boundingRect.width(); - - if (index.data(TransactionTableModel::WatchonlyRole).toBool()) - { + if (index.data(TransactionTableModel::WatchonlyRole).toBool()) { QIcon iconWatchonly = qvariant_cast<QIcon>(index.data(TransactionTableModel::WatchonlyDecorationRole)); - QRect watchonlyRect(boundingRect.right() + 5, mainRect.top()+ypad+halfheight, 16, halfheight); + QRect watchonlyRect(addressRect.left(), addressRect.top(), 16, addressRect.height()); iconWatchonly = platformStyle->TextColorIcon(iconWatchonly); iconWatchonly.paint(painter, watchonlyRect); - address_rect_min_width += 5 + watchonlyRect.width(); + addressRect.setLeft(addressRect.left() + watchonlyRect.width() + 5); } + painter->setPen(foreground); + QRect boundingRect; + painter->drawText(addressRect, Qt::AlignLeft | Qt::AlignVCenter, address, &boundingRect); + if(amount < 0) { foreground = COLOR_NEGATIVE; @@ -109,7 +107,8 @@ public: QRect date_bounding_rect; painter->drawText(amountRect, Qt::AlignLeft | Qt::AlignVCenter, GUIUtil::dateTimeStr(date), &date_bounding_rect); - const int minimum_width = std::max(address_rect_min_width, amount_bounding_rect.width() + date_bounding_rect.width()); + // 0.4*date_bounding_rect.width() is used to visually distinguish a date from an amount. + const int minimum_width = 1.4 * date_bounding_rect.width() + amount_bounding_rect.width(); const auto search = m_minimum_width.find(index.row()); if (search == m_minimum_width.end() || search->second != minimum_width) { m_minimum_width[index.row()] = minimum_width; @@ -297,13 +296,13 @@ void OverviewPage::setWalletModel(WalletModel *model) void OverviewPage::changeEvent(QEvent* e) { -#ifdef Q_OS_MACOS if (e->type() == QEvent::PaletteChange) { QIcon icon = m_platform_style->SingleColorIcon(QStringLiteral(":/icons/warning")); ui->labelTransactionsStatus->setIcon(icon); ui->labelWalletStatus->setIcon(icon); } -#endif + + QWidget::changeEvent(e); } void OverviewPage::updateDisplayUnit() diff --git a/src/qt/peertablemodel.cpp b/src/qt/peertablemodel.cpp index b324693692..433a1ea934 100644 --- a/src/qt/peertablemodel.cpp +++ b/src/qt/peertablemodel.cpp @@ -72,8 +72,13 @@ QVariant PeerTableModel::data(const QModelIndex& index, int role) const case NetNodeId: return (qint64)rec->nodeStats.nodeid; case Address: - // prepend to peer address down-arrow symbol for inbound connection and up-arrow for outbound connection - return QString::fromStdString((rec->nodeStats.fInbound ? "↓ " : "↑ ") + rec->nodeStats.addrName); + return QString::fromStdString(rec->nodeStats.m_addr_name); + case Direction: + return QString(rec->nodeStats.fInbound ? + //: An Inbound Connection from a Peer. + tr("Inbound") : + //: An Outbound Connection to a Peer. + tr("Outbound")); case ConnectionType: return GUIUtil::ConnectionTypeToQString(rec->nodeStats.m_conn_type, /* prepend_direction */ false); case Network: @@ -94,6 +99,7 @@ QVariant PeerTableModel::data(const QModelIndex& index, int role) const return QVariant(Qt::AlignRight | Qt::AlignVCenter); case Address: return {}; + case Direction: case ConnectionType: case Network: return QVariant(Qt::AlignCenter); @@ -179,5 +185,7 @@ void PeerTableModel::refresh() m_peers_data.swap(new_peers_data); } - Q_EMIT changed(); + const auto top_left = index(0, 0); + const auto bottom_right = index(rowCount() - 1, columnCount() - 1); + Q_EMIT dataChanged(top_left, bottom_right); } diff --git a/src/qt/peertablemodel.h b/src/qt/peertablemodel.h index 0ff1b5dba7..40265ee266 100644 --- a/src/qt/peertablemodel.h +++ b/src/qt/peertablemodel.h @@ -48,6 +48,7 @@ public: enum ColumnIndex { NetNodeId = 0, Address, + Direction, ConnectionType, Network, Ping, @@ -73,9 +74,6 @@ public: public Q_SLOTS: void refresh(); -Q_SIGNALS: - void changed(); - private: //! Internal peer data structure. QList<CNodeCombinedStats> m_peers_data{}; @@ -87,6 +85,9 @@ private: /*: Title of Peers Table column which contains the IP/Onion/I2P address of the connected peer. */ tr("Address"), + /*: Title of Peers Table column which indicates the direction + the peer connection was initiated from. */ + tr("Direction"), /*: Title of Peers Table column which describes the type of peer connection. The "type" describes why the connection exists. */ tr("Type"), diff --git a/src/qt/peertablesortproxy.cpp b/src/qt/peertablesortproxy.cpp index 78932da8d4..419133bc32 100644 --- a/src/qt/peertablesortproxy.cpp +++ b/src/qt/peertablesortproxy.cpp @@ -25,7 +25,9 @@ bool PeerTableSortProxy::lessThan(const QModelIndex& left_index, const QModelInd case PeerTableModel::NetNodeId: return left_stats.nodeid < right_stats.nodeid; case PeerTableModel::Address: - return left_stats.addrName.compare(right_stats.addrName) < 0; + return left_stats.m_addr_name.compare(right_stats.m_addr_name) < 0; + case PeerTableModel::Direction: + return left_stats.fInbound > right_stats.fInbound; // default sort Inbound, then Outbound case PeerTableModel::ConnectionType: return left_stats.m_conn_type < right_stats.m_conn_type; case PeerTableModel::Network: diff --git a/src/qt/psbtoperationsdialog.cpp b/src/qt/psbtoperationsdialog.cpp index 2adfeeaaf0..34d56e5506 100644 --- a/src/qt/psbtoperationsdialog.cpp +++ b/src/qt/psbtoperationsdialog.cpp @@ -47,18 +47,22 @@ void PSBTOperationsDialog::openWithPSBT(PartiallySignedTransaction psbtx) { m_transaction_data = psbtx; - bool complete; - size_t n_could_sign; - FinalizePSBT(psbtx); // Make sure all existing signatures are fully combined before checking for completeness. - TransactionError err = m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, false /* sign */, true /* bip32derivs */, &n_could_sign, m_transaction_data, complete); - if (err != TransactionError::OK) { - showStatus(tr("Failed to load transaction: %1") - .arg(QString::fromStdString(TransactionErrorString(err).translated)), StatusLevel::ERR); - return; + bool complete = FinalizePSBT(psbtx); // Make sure all existing signatures are fully combined before checking for completeness. + if (m_wallet_model) { + size_t n_could_sign; + TransactionError err = m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, false /* sign */, true /* bip32derivs */, &n_could_sign, m_transaction_data, complete); + if (err != TransactionError::OK) { + showStatus(tr("Failed to load transaction: %1") + .arg(QString::fromStdString(TransactionErrorString(err).translated)), + StatusLevel::ERR); + return; + } + m_ui->signTransactionButton->setEnabled(!complete && !m_wallet_model->wallet().privateKeysDisabled() && n_could_sign > 0); + } else { + m_ui->signTransactionButton->setEnabled(false); } m_ui->broadcastTransactionButton->setEnabled(complete); - m_ui->signTransactionButton->setEnabled(!complete && !m_wallet_model->wallet().privateKeysDisabled() && n_could_sign > 0); updateTransactionDisplay(); } @@ -67,6 +71,9 @@ void PSBTOperationsDialog::signTransaction() { bool complete; size_t n_signed; + + WalletModel::UnlockContext ctx(m_wallet_model->requestUnlock()); + TransactionError err = m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, true /* sign */, true /* bip32derivs */, &n_signed, m_transaction_data, complete); if (err != TransactionError::OK) { @@ -77,7 +84,9 @@ void PSBTOperationsDialog::signTransaction() updateTransactionDisplay(); - if (!complete && n_signed < 1) { + if (!complete && !ctx.isValid()) { + showStatus(tr("Cannot sign inputs while wallet is locked."), StatusLevel::WARN); + } else if (!complete && n_signed < 1) { showStatus(tr("Could not sign any more inputs."), StatusLevel::WARN); } else if (!complete) { showStatus(tr("Signed %1 inputs, but more signatures are still required.").arg(n_signed), @@ -133,7 +142,7 @@ void PSBTOperationsDialog::saveTransaction() { } CTxDestination address; ExtractDestination(out.scriptPubKey, address); - QString amount = BitcoinUnits::format(m_wallet_model->getOptionsModel()->getDisplayUnit(), out.nValue); + QString amount = BitcoinUnits::format(m_client_model->getOptionsModel()->getDisplayUnit(), out.nValue); QString address_str = QString::fromStdString(EncodeDestination(address)); filename_suggestion.append(address_str + "-" + amount); first = false; @@ -224,6 +233,10 @@ void PSBTOperationsDialog::showStatus(const QString &msg, StatusLevel level) { } size_t PSBTOperationsDialog::couldSignInputs(const PartiallySignedTransaction &psbtx) { + if (!m_wallet_model) { + return 0; + } + size_t n_signed; bool complete; TransactionError err = m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, false /* sign */, false /* bip32derivs */, &n_signed, m_transaction_data, complete); @@ -246,7 +259,10 @@ void PSBTOperationsDialog::showTransactionStatus(const PartiallySignedTransactio case PSBTRole::SIGNER: { QString need_sig_text = tr("Transaction still needs signature(s)."); StatusLevel level = StatusLevel::INFO; - if (m_wallet_model->wallet().privateKeysDisabled()) { + if (!m_wallet_model) { + need_sig_text += " " + tr("(But no wallet is loaded.)"); + level = StatusLevel::WARN; + } else if (m_wallet_model->wallet().privateKeysDisabled()) { need_sig_text += " " + tr("(But this wallet cannot sign transactions.)"); level = StatusLevel::WARN; } else if (n_could_sign < 1) { diff --git a/src/qt/qrimagewidget.cpp b/src/qt/qrimagewidget.cpp index 7cdd568644..0799e01aac 100644 --- a/src/qt/qrimagewidget.cpp +++ b/src/qt/qrimagewidget.cpp @@ -119,7 +119,7 @@ void QRImageWidget::saveImage() QString fn = GUIUtil::getSaveFileName( this, tr("Save QR Code"), QString(), /*: Expanded name of the PNG file format. - See https://en.wikipedia.org/wiki/Portable_Network_Graphics */ + See: https://en.wikipedia.org/wiki/Portable_Network_Graphics. */ tr("PNG Image") + QLatin1String(" (*.png)"), nullptr); if (!fn.isEmpty()) { diff --git a/src/qt/recentrequeststablemodel.cpp b/src/qt/recentrequeststablemodel.cpp index ec3d970a7f..ab8225e19f 100644 --- a/src/qt/recentrequeststablemodel.cpp +++ b/src/qt/recentrequeststablemodel.cpp @@ -234,7 +234,7 @@ bool RecentRequestEntryLessThan::operator()(const RecentRequestEntry& left, cons switch(column) { case RecentRequestsTableModel::Date: - return pLeft->date.toTime_t() < pRight->date.toTime_t(); + return pLeft->date.toSecsSinceEpoch() < pRight->date.toSecsSinceEpoch(); case RecentRequestsTableModel::Label: return pLeft->recipient.label < pRight->recipient.label; case RecentRequestsTableModel::Message: diff --git a/src/qt/recentrequeststablemodel.h b/src/qt/recentrequeststablemodel.h index b817b64e77..c489c0eaf4 100644 --- a/src/qt/recentrequeststablemodel.h +++ b/src/qt/recentrequeststablemodel.h @@ -7,6 +7,8 @@ #include <qt/sendcoinsrecipient.h> +#include <string> + #include <QAbstractTableModel> #include <QStringList> #include <QDateTime> @@ -26,9 +28,9 @@ public: SERIALIZE_METHODS(RecentRequestEntry, obj) { unsigned int date_timet; - SER_WRITE(obj, date_timet = obj.date.toTime_t()); + SER_WRITE(obj, date_timet = obj.date.toSecsSinceEpoch()); READWRITE(obj.nVersion, obj.id, date_timet, obj.recipient); - SER_READ(obj, obj.date = QDateTime::fromTime_t(date_timet)); + SER_READ(obj, obj.date = QDateTime::fromSecsSinceEpoch(date_timet)); } }; diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index ff4bfb16f6..0c3332ab76 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -35,6 +35,7 @@ #endif #include <QAbstractButton> +#include <QAbstractItemModel> #include <QDateTime> #include <QFont> #include <QKeyEvent> @@ -246,10 +247,11 @@ bool RPCConsole::RPCParseCommandLine(interfaces::Node* node, std::string &strRes UniValue subelement; if (lastResult.isArray()) { - for(char argch: curarg) - if (!IsDigit(argch)) - throw std::runtime_error("Invalid result query"); - subelement = lastResult[atoi(curarg.c_str())]; + const auto parsed{ToIntegral<size_t>(curarg)}; + if (!parsed) { + throw std::runtime_error("Invalid result query"); + } + subelement = lastResult[parsed.value()]; } else if (lastResult.isObject()) subelement = find_value(lastResult, curarg); @@ -287,6 +289,7 @@ bool RPCConsole::RPCParseCommandLine(interfaces::Node* node, std::string &strRes } if (breakParsing) break; + [[fallthrough]]; } case STATE_ARGUMENT: // In or after argument case STATE_EATING_SPACES_IN_ARG: @@ -400,6 +403,7 @@ bool RPCConsole::RPCParseCommandLine(interfaces::Node* node, std::string &strRes strResult = lastResult.get_str(); else strResult = lastResult.write(2); + [[fallthrough]]; case STATE_ARGUMENT: case STATE_EATING_SPACES: return true; @@ -492,14 +496,28 @@ RPCConsole::RPCConsole(interfaces::Node& node, const PlatformStyle *_platformSty constexpr QChar nonbreaking_hyphen(8209); const std::vector<QString> CONNECTION_TYPE_DOC{ + //: Explanatory text for an inbound peer connection. tr("Inbound: initiated by peer"), + /*: Explanatory text for an outbound peer connection that + relays all network information. This is the default behavior for + outbound connections. */ tr("Outbound Full Relay: default"), + /*: Explanatory text for an outbound peer connection that relays + network information about blocks and not transactions or addresses. */ tr("Outbound Block Relay: does not relay transactions or addresses"), + /*: Explanatory text for an outbound peer connection that was + established manually through one of several methods. The numbered + arguments are stand-ins for the methods available to establish + manual connections. */ tr("Outbound Manual: added using RPC %1 or %2/%3 configuration options") .arg("addnode") .arg(QString(nonbreaking_hyphen) + "addnode") .arg(QString(nonbreaking_hyphen) + "connect"), + /*: Explanatory text for a short-lived outbound peer connection that + is used to test the aliveness of known addresses. */ tr("Outbound Feeler: short-lived, for testing addresses"), + /*: Explanatory text for a short-lived outbound peer connection that is used + to request addresses from a peer. */ tr("Outbound Address Fetch: short-lived, for soliciting addresses")}; const QString list{"<ul><li>" + Join(CONNECTION_TYPE_DOC, QString("</li><li>")) + "</li></ul>"}; ui->peerConnectionTypeLabel->setToolTip(ui->peerConnectionTypeLabel->toolTip().arg(list)); @@ -529,6 +547,8 @@ RPCConsole::RPCConsole(interfaces::Node& node, const PlatformStyle *_platformSty //: Secondary shortcut to decrease the RPC console font size. GUIUtil::AddButtonShortcut(ui->fontSmallerButton, tr("Ctrl+_")); + ui->promptIcon->setIcon(platformStyle->SingleColorIcon(QStringLiteral(":/icons/prompticon"))); + // Install event filter for up and down arrow ui->lineEdit->installEventFilter(this); ui->lineEdit->setMaxLength(16 * 1024 * 1024); @@ -646,7 +666,7 @@ void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_ setNumConnections(model->getNumConnections()); connect(model, &ClientModel::numConnectionsChanged, this, &RPCConsole::setNumConnections); - setNumBlocks(bestblock_height, QDateTime::fromTime_t(bestblock_date), verification_progress, false); + setNumBlocks(bestblock_height, QDateTime::fromSecsSinceEpoch(bestblock_date), verification_progress, false); connect(model, &ClientModel::numBlocksChanged, this, &RPCConsole::setNumBlocks); updateNetworkState(); @@ -675,6 +695,11 @@ void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_ // create peer table context menu peersTableContextMenu = new QMenu(this); + //: Context menu action to copy the address of a peer. + peersTableContextMenu->addAction(tr("&Copy address"), [this] { + GUIUtil::copyEntryData(ui->peerWidget, PeerTableModel::Address, Qt::DisplayRole); + }); + peersTableContextMenu->addSeparator(); peersTableContextMenu->addAction(tr("&Disconnect"), this, &RPCConsole::disconnectSelectedNode); peersTableContextMenu->addAction(ts.ban_for + " " + tr("1 &hour"), [this] { banSelectedNode(60 * 60); }); peersTableContextMenu->addAction(ts.ban_for + " " + tr("1 d&ay"), [this] { banSelectedNode(60 * 60 * 24); }); @@ -684,7 +709,7 @@ void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_ // peer table signal handling - update peer details when selecting new node connect(ui->peerWidget->selectionModel(), &QItemSelectionModel::selectionChanged, this, &RPCConsole::updateDetailWidget); - connect(model->getPeerTableModel(), &PeerTableModel::changed, this, &RPCConsole::updateDetailWidget); + connect(model->getPeerTableModel(), &QAbstractItemModel::dataChanged, [this] { updateDetailWidget(); }); // set up ban table ui->banlistWidget->setModel(model->getBanTableModel()); @@ -701,6 +726,13 @@ void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_ // create ban table context menu banTableContextMenu = new QMenu(this); + /*: Context menu action to copy the IP/Netmask of a banned peer. + IP/Netmask is the combination of a peer's IP address and its Netmask. + For IP address, see: https://en.wikipedia.org/wiki/IP_address. */ + banTableContextMenu->addAction(tr("&Copy IP/Netmask"), [this] { + GUIUtil::copyEntryData(ui->banlistWidget, BanTableModel::Address, Qt::DisplayRole); + }); + banTableContextMenu->addSeparator(); banTableContextMenu->addAction(tr("&Unban"), this, &RPCConsole::unbanSelectedNode); connect(ui->banlistWidget, &QTableView::customContextMenuRequested, this, &RPCConsole::showBanTableContextMenu); @@ -882,7 +914,6 @@ void RPCConsole::keyPressEvent(QKeyEvent *event) void RPCConsole::changeEvent(QEvent* e) { -#ifdef Q_OS_MACOS if (e->type() == QEvent::PaletteChange) { ui->clearButton->setIcon(platformStyle->SingleColorIcon(QStringLiteral(":/icons/remove"))); ui->fontBiggerButton->setIcon(platformStyle->SingleColorIcon(QStringLiteral(":/icons/fontbigger"))); @@ -896,7 +927,8 @@ void RPCConsole::changeEvent(QEvent* e) platformStyle->SingleColorImage(ICON_MAPPING[i].source).scaled(QSize(consoleFontSize * 2, consoleFontSize * 2), Qt::IgnoreAspectRatio, Qt::SmoothTransformation)); } } -#endif + + QWidget::changeEvent(e); } void RPCConsole::message(int category, const QString &message, bool html) @@ -1124,7 +1156,7 @@ void RPCConsole::updateDetailWidget() } const auto stats = selected_peers.first().data(PeerTableModel::StatsRole).value<CNodeCombinedStats*>(); // update the detail ui with latest node information - QString peerAddrDetails(QString::fromStdString(stats->nodeStats.addrName) + " "); + QString peerAddrDetails(QString::fromStdString(stats->nodeStats.m_addr_name) + " "); peerAddrDetails += tr("(peer: %1)").arg(QString::number(stats->nodeStats.nodeid)); if (!stats->nodeStats.addrLocal.empty()) peerAddrDetails += "<br />" + tr("via %1").arg(QString::fromStdString(stats->nodeStats.addrLocal)); diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index c9bf757dfc..2718392940 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -200,7 +200,7 @@ void SendCoinsDialog::setModel(WalletModel *_model) ui->optInRBF->setCheckState(Qt::Checked); if (model->wallet().hasExternalSigner()) { - //: "device" usually means a hardware wallet + //: "device" usually means a hardware wallet. ui->sendButton->setText(tr("Sign on device")); if (gArgs.GetArg("-signer", "") != "") { ui->sendButton->setEnabled(true); @@ -399,9 +399,10 @@ void SendCoinsDialog::sendButtonClicked([[maybe_unused]] bool checked) const QString confirmation = model->wallet().privateKeysDisabled() && !model->wallet().hasExternalSigner() ? tr("Confirm transaction proposal") : tr("Confirm send coins"); const QString confirmButtonText = model->wallet().privateKeysDisabled() && !model->wallet().hasExternalSigner() ? tr("Create Unsigned") : tr("Sign and send"); - SendConfirmationDialog confirmationDialog(confirmation, question_string, informative_text, detailed_text, SEND_CONFIRM_DELAY, confirmButtonText, this); - confirmationDialog.exec(); - QMessageBox::StandardButton retval = static_cast<QMessageBox::StandardButton>(confirmationDialog.result()); + auto confirmationDialog = new SendConfirmationDialog(confirmation, question_string, informative_text, detailed_text, SEND_CONFIRM_DELAY, confirmButtonText, this); + confirmationDialog->setAttribute(Qt::WA_DeleteOnClose); + // TODO: Replace QDialog::exec() with safer QDialog::show(). + const auto retval = static_cast<QMessageBox::StandardButton>(confirmationDialog->exec()); if(retval != QMessageBox::Yes) { @@ -914,9 +915,9 @@ void SendCoinsDialog::coinControlFeatureChanged(bool checked) // Coin Control: button inputs -> show actual coin control dialog void SendCoinsDialog::coinControlButtonClicked() { - CoinControlDialog dlg(*m_coin_control, model, platformStyle); - dlg.exec(); - coinControlUpdateLabels(); + auto dlg = new CoinControlDialog(*m_coin_control, model, platformStyle); + connect(dlg, &QDialog::finished, this, &SendCoinsDialog::coinControlUpdateLabels); + GUIUtil::ShowModalDialogAndDeleteOnClose(dlg); } // Coin Control: checkbox custom change address diff --git a/src/qt/sendcoinsentry.cpp b/src/qt/sendcoinsentry.cpp index f701bb9615..5fa5165615 100644 --- a/src/qt/sendcoinsentry.cpp +++ b/src/qt/sendcoinsentry.cpp @@ -97,7 +97,9 @@ void SendCoinsEntry::clear() ui->payTo->clear(); ui->addAsLabel->clear(); ui->payAmount->clear(); - ui->checkboxSubtractFeeFromAmount->setCheckState(Qt::Unchecked); + if (model && model->getOptionsModel()) { + ui->checkboxSubtractFeeFromAmount->setChecked(model->getOptionsModel()->getSubFeeFromAmount()); + } ui->messageTextLabel->clear(); ui->messageTextLabel->hide(); ui->messageLabel->hide(); @@ -238,7 +240,6 @@ void SendCoinsEntry::updateDisplayUnit() void SendCoinsEntry::changeEvent(QEvent* e) { -#ifdef Q_OS_MACOS if (e->type() == QEvent::PaletteChange) { ui->addressBookButton->setIcon(platformStyle->SingleColorIcon(QStringLiteral(":/icons/address-book"))); ui->pasteButton->setIcon(platformStyle->SingleColorIcon(QStringLiteral(":/icons/editpaste"))); @@ -246,7 +247,8 @@ void SendCoinsEntry::changeEvent(QEvent* e) ui->deleteButton_is->setIcon(platformStyle->SingleColorIcon(QStringLiteral(":/icons/remove"))); ui->deleteButton_s->setIcon(platformStyle->SingleColorIcon(QStringLiteral(":/icons/remove"))); } -#endif + + QStackedWidget::changeEvent(e); } bool SendCoinsEntry::updateLabel(const QString &address) diff --git a/src/qt/sendcoinsrecipient.h b/src/qt/sendcoinsrecipient.h index 01135cdfef..c23afcab24 100644 --- a/src/qt/sendcoinsrecipient.h +++ b/src/qt/sendcoinsrecipient.h @@ -9,7 +9,7 @@ #include <config/bitcoin-config.h> #endif -#include <amount.h> +#include <consensus/amount.h> #include <serialize.h> #include <string> diff --git a/src/qt/signverifymessagedialog.cpp b/src/qt/signverifymessagedialog.cpp index b982cc577d..33589f09bf 100644 --- a/src/qt/signverifymessagedialog.cpp +++ b/src/qt/signverifymessagedialog.cpp @@ -286,7 +286,6 @@ bool SignVerifyMessageDialog::eventFilter(QObject *object, QEvent *event) void SignVerifyMessageDialog::changeEvent(QEvent* e) { -#ifdef Q_OS_MACOS if (e->type() == QEvent::PaletteChange) { ui->addressBookButton_SM->setIcon(platformStyle->SingleColorIcon(QStringLiteral(":/icons/address-book"))); ui->pasteButton_SM->setIcon(platformStyle->SingleColorIcon(QStringLiteral(":/icons/editpaste"))); @@ -297,5 +296,6 @@ void SignVerifyMessageDialog::changeEvent(QEvent* e) ui->verifyMessageButton_VM->setIcon(platformStyle->SingleColorIcon(QStringLiteral(":/icons/transaction_0"))); ui->clearButton_VM->setIcon(platformStyle->SingleColorIcon(QStringLiteral(":/icons/remove"))); } -#endif + + QDialog::changeEvent(e); } diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 39c69fe184..0de781661a 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -60,6 +60,8 @@ void EditAddressAndSubmit( void TestAddAddressesToSendBook(interfaces::Node& node) { TestChain100Setup test; + auto wallet_client = interfaces::MakeWalletClient(*test.m_node.chain, *Assert(test.m_node.args)); + test.m_node.wallet_client = wallet_client.get(); node.setContext(&test.m_node); std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockWalletDatabase()); wallet->SetupLegacyScriptPubKeyMan(); @@ -109,9 +111,10 @@ void TestAddAddressesToSendBook(interfaces::Node& node) std::unique_ptr<const PlatformStyle> platformStyle(PlatformStyle::instantiate("other")); OptionsModel optionsModel; ClientModel clientModel(node, &optionsModel); - AddWallet(wallet); - WalletModel walletModel(interfaces::MakeWallet(wallet), clientModel, platformStyle.get()); - RemoveWallet(wallet, std::nullopt); + WalletContext& context = *node.walletClient().context(); + AddWallet(context, wallet); + WalletModel walletModel(interfaces::MakeWallet(context, wallet), clientModel, platformStyle.get()); + RemoveWallet(context, wallet, /* load_on_start= */ std::nullopt); EditAddressDialog editAddressDialog(EditAddressDialog::NewSendingAddress); editAddressDialog.setModel(walletModel.getAddressTableModel()); diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index 9c31cd50df..8489b33144 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -12,7 +12,6 @@ #include <qt/rpcconsole.h> #include <shutdown.h> #include <test/util/setup_common.h> -#include <univalue.h> #include <validation.h> #if defined(HAVE_CONFIG_H) @@ -21,8 +20,10 @@ #include <QAction> #include <QLineEdit> +#include <QRegularExpression> #include <QScopedPointer> #include <QSignalSpy> +#include <QString> #include <QTest> #include <QTextEdit> #include <QtGlobal> @@ -30,6 +31,13 @@ #include <QtTest/QtTestGui> namespace { +//! Regex find a string group inside of the console output +QString FindInConsole(const QString& output, const QString& pattern) +{ + const QRegularExpression re(pattern); + return re.match(output).captured(1); +} + //! Call getblockchaininfo RPC and check first field of JSON output. void TestRpcCommand(RPCConsole* console) { @@ -41,10 +49,9 @@ void TestRpcCommand(RPCConsole* console) QTest::keyClick(lineEdit, Qt::Key_Return); QVERIFY(mw_spy.wait(1000)); QCOMPARE(mw_spy.count(), 4); - QString output = messagesWidget->toPlainText(); - UniValue value; - value.read(output.right(output.size() - output.lastIndexOf(QChar::ObjectReplacementCharacter) - 1).toStdString()); - QCOMPARE(value["chain"].get_str(), std::string("regtest")); + const QString output = messagesWidget->toPlainText(); + const QString pattern = QStringLiteral("\"chain\": \"(\\w+)\""); + QCOMPARE(FindInConsole(output, pattern), QString("regtest")); } } // namespace diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index eb86f027ef..55d00bb37e 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -6,6 +6,7 @@ #include <config/bitcoin-config.h> #endif +#include <interfaces/init.h> #include <interfaces/node.h> #include <qt/bitcoin.h> #include <qt/test/apptests.h> @@ -52,7 +53,8 @@ int main(int argc, char* argv[]) } NodeContext node_context; - std::unique_ptr<interfaces::Node> node = interfaces::MakeNode(&node_context); + int unused_exit_status; + std::unique_ptr<interfaces::Init> init = interfaces::MakeNodeInit(node_context, argc, argv, unused_exit_status); gArgs.ForceSetArg("-listen", "0"); gArgs.ForceSetArg("-listenonion", "0"); gArgs.ForceSetArg("-discover", "0"); @@ -75,10 +77,9 @@ int main(int argc, char* argv[]) // Don't remove this, it's needed to access // QApplication:: and QCoreApplication:: in the tests BitcoinApplication app; - app.setNode(*node); app.setApplicationName("Bitcoin-Qt-test"); + app.createNode(*init); - app.node().context()->args = &gArgs; // Make gArgs available in the NodeContext AppTests app_tests(app); if (QTest::qExec(&app_tests) != 0) { fInvalid = true; diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 3e1a0e0fa9..62b135d3f1 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -69,7 +69,7 @@ uint256 SendCoins(CWallet& wallet, SendCoinsDialog& sendCoinsDialog, const CTxDe ->findChild<QCheckBox*>("optInRBF") ->setCheckState(rbf ? Qt::Checked : Qt::Unchecked); uint256 txid; - boost::signals2::scoped_connection c(wallet.NotifyTransactionChanged.connect([&txid](CWallet*, const uint256& hash, ChangeType status) { + boost::signals2::scoped_connection c(wallet.NotifyTransactionChanged.connect([&txid](const uint256& hash, ChangeType status) { if (status == CT_NEW) txid = hash; })); ConfirmSend(); @@ -138,6 +138,8 @@ void TestGUI(interfaces::Node& node) for (int i = 0; i < 5; ++i) { test.CreateAndProcessBlock({}, GetScriptForRawPubKey(test.coinbaseKey.GetPubKey())); } + auto wallet_client = interfaces::MakeWalletClient(*test.m_node.chain, *Assert(test.m_node.args)); + test.m_node.wallet_client = wallet_client.get(); node.setContext(&test.m_node); std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockWalletDatabase()); wallet->LoadWallet(); @@ -164,9 +166,10 @@ void TestGUI(interfaces::Node& node) TransactionView transactionView(platformStyle.get()); OptionsModel optionsModel; ClientModel clientModel(node, &optionsModel); - AddWallet(wallet); - WalletModel walletModel(interfaces::MakeWallet(wallet), clientModel, platformStyle.get()); - RemoveWallet(wallet, std::nullopt); + WalletContext& context = *node.walletClient().context(); + AddWallet(context, wallet); + WalletModel walletModel(interfaces::MakeWallet(context, wallet), clientModel, platformStyle.get()); + RemoveWallet(context, wallet, /* load_on_start= */ std::nullopt); sendCoinsDialog.setModel(&walletModel); transactionView.setModel(&walletModel); diff --git a/src/qt/transactionfilterproxy.cpp b/src/qt/transactionfilterproxy.cpp index a631f497af..57c05a647e 100644 --- a/src/qt/transactionfilterproxy.cpp +++ b/src/qt/transactionfilterproxy.cpp @@ -7,17 +7,12 @@ #include <qt/transactiontablemodel.h> #include <qt/transactionrecord.h> +#include <algorithm> #include <cstdlib> - -// Earliest date that can be represented (far in the past) -const QDateTime TransactionFilterProxy::MIN_DATE = QDateTime::fromTime_t(0); -// Last date that can be represented (far in the future) -const QDateTime TransactionFilterProxy::MAX_DATE = QDateTime::fromTime_t(0xFFFFFFFF); +#include <optional> TransactionFilterProxy::TransactionFilterProxy(QObject *parent) : QSortFilterProxyModel(parent), - dateFrom(MIN_DATE), - dateTo(MAX_DATE), m_search_string(), typeFilter(ALL_TYPES), watchOnlyFilter(WatchOnlyFilter_All), @@ -46,8 +41,8 @@ bool TransactionFilterProxy::filterAcceptsRow(int sourceRow, const QModelIndex & return false; QDateTime datetime = index.data(TransactionTableModel::DateRole).toDateTime(); - if (datetime < dateFrom || datetime > dateTo) - return false; + if (dateFrom && datetime < *dateFrom) return false; + if (dateTo && datetime > *dateTo) return false; QString address = index.data(TransactionTableModel::AddressRole).toString(); QString label = index.data(TransactionTableModel::LabelRole).toString(); @@ -65,10 +60,10 @@ bool TransactionFilterProxy::filterAcceptsRow(int sourceRow, const QModelIndex & return true; } -void TransactionFilterProxy::setDateRange(const QDateTime &from, const QDateTime &to) +void TransactionFilterProxy::setDateRange(const std::optional<QDateTime>& from, const std::optional<QDateTime>& to) { - this->dateFrom = from; - this->dateTo = to; + dateFrom = from; + dateTo = to; invalidateFilter(); } diff --git a/src/qt/transactionfilterproxy.h b/src/qt/transactionfilterproxy.h index 693b363692..270b8ef78c 100644 --- a/src/qt/transactionfilterproxy.h +++ b/src/qt/transactionfilterproxy.h @@ -5,11 +5,13 @@ #ifndef BITCOIN_QT_TRANSACTIONFILTERPROXY_H #define BITCOIN_QT_TRANSACTIONFILTERPROXY_H -#include <amount.h> +#include <consensus/amount.h> #include <QDateTime> #include <QSortFilterProxyModel> +#include <optional> + /** Filter the transaction list according to pre-specified rules. */ class TransactionFilterProxy : public QSortFilterProxyModel { @@ -18,10 +20,6 @@ class TransactionFilterProxy : public QSortFilterProxyModel public: explicit TransactionFilterProxy(QObject *parent = nullptr); - /** Earliest date that can be represented (far in the past) */ - static const QDateTime MIN_DATE; - /** Last date that can be represented (far in the future) */ - static const QDateTime MAX_DATE; /** Type filter bit field (all types) */ static const quint32 ALL_TYPES = 0xFFFFFFFF; @@ -34,7 +32,8 @@ public: WatchOnlyFilter_No }; - void setDateRange(const QDateTime &from, const QDateTime &to); + /** Filter transactions between date range. Use std::nullopt for open range. */ + void setDateRange(const std::optional<QDateTime>& from, const std::optional<QDateTime>& to); void setSearchString(const QString &); /** @note Type filter takes a bit field created with TYPE() or ALL_TYPES @@ -55,8 +54,8 @@ protected: bool filterAcceptsRow(int source_row, const QModelIndex & source_parent) const override; private: - QDateTime dateFrom; - QDateTime dateTo; + std::optional<QDateTime> dateFrom; + std::optional<QDateTime> dateTo; QString m_search_string; quint32 typeFilter; WatchOnlyFilter watchOnlyFilter; diff --git a/src/qt/transactionrecord.h b/src/qt/transactionrecord.h index e10243a28a..fb88ca424f 100644 --- a/src/qt/transactionrecord.h +++ b/src/qt/transactionrecord.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_TRANSACTIONRECORD_H #define BITCOIN_QT_TRANSACTIONRECORD_H -#include <amount.h> +#include <consensus/amount.h> #include <uint256.h> #include <QList> diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index b68ceaedbb..23590ea4d2 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -610,7 +610,7 @@ QVariant TransactionTableModel::data(const QModelIndex &index, int role) const case TypeRole: return rec->type; case DateRole: - return QDateTime::fromTime_t(static_cast<uint>(rec->time)); + return QDateTime::fromSecsSinceEpoch(rec->time); case WatchonlyRole: return rec->involvesWatchAddress; case WatchonlyDecorationRole: @@ -630,7 +630,7 @@ QVariant TransactionTableModel::data(const QModelIndex &index, int role) const case TxPlainTextRole: { QString details; - QDateTime date = QDateTime::fromTime_t(static_cast<uint>(rec->time)); + QDateTime date = QDateTime::fromSecsSinceEpoch(rec->time); QString txLabel = walletModel->getAddressTableModel()->labelForAddress(QString::fromStdString(rec->address)); details.append(date.toString("M/d/yy HH:mm")); diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 4b1a546c7c..653f3dda6d 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -19,6 +19,8 @@ #include <node/ui_interface.h> +#include <optional> + #include <QApplication> #include <QComboBox> #include <QDateTimeEdit> @@ -220,17 +222,21 @@ void TransactionView::setModel(WalletModel *_model) { // Add third party transaction URLs to context menu QStringList listUrls = GUIUtil::SplitSkipEmptyParts(_model->getOptionsModel()->getThirdPartyTxUrls(), "|"); + bool actions_created = false; for (int i = 0; i < listUrls.size(); ++i) { QString url = listUrls[i].trimmed(); QString host = QUrl(url, QUrl::StrictMode).host(); if (!host.isEmpty()) { - QAction *thirdPartyTxUrlAction = new QAction(host, this); // use host as menu item label - if (i == 0) + if (!actions_created) { contextMenu->addSeparator(); - contextMenu->addAction(thirdPartyTxUrlAction); - connect(thirdPartyTxUrlAction, &QAction::triggered, [this, url] { openThirdPartyTxUrl(url); }); + actions_created = true; + } + /*: Transactions table context menu action to show the + selected transaction in a third-party block explorer. + %1 is a stand-in argument for the URL of the explorer. */ + contextMenu->addAction(tr("Show in %1").arg(host), [this, url] { openThirdPartyTxUrl(url); }); } } } @@ -245,7 +251,6 @@ void TransactionView::setModel(WalletModel *_model) void TransactionView::changeEvent(QEvent* e) { -#ifdef Q_OS_MACOS if (e->type() == QEvent::PaletteChange) { watchOnlyWidget->setItemIcon( TransactionFilterProxy::WatchOnlyFilter_Yes, @@ -254,7 +259,8 @@ void TransactionView::changeEvent(QEvent* e) TransactionFilterProxy::WatchOnlyFilter_No, m_platform_style->SingleColorIcon(QStringLiteral(":/icons/eye_minus"))); } -#endif + + QWidget::changeEvent(e); } void TransactionView::chooseDate(int idx) @@ -266,26 +272,26 @@ void TransactionView::chooseDate(int idx) { case All: transactionProxyModel->setDateRange( - TransactionFilterProxy::MIN_DATE, - TransactionFilterProxy::MAX_DATE); + std::nullopt, + std::nullopt); break; case Today: transactionProxyModel->setDateRange( GUIUtil::StartOfDay(current), - TransactionFilterProxy::MAX_DATE); + std::nullopt); break; case ThisWeek: { // Find last Monday QDate startOfWeek = current.addDays(-(current.dayOfWeek()-1)); transactionProxyModel->setDateRange( GUIUtil::StartOfDay(startOfWeek), - TransactionFilterProxy::MAX_DATE); + std::nullopt); } break; case ThisMonth: transactionProxyModel->setDateRange( GUIUtil::StartOfDay(QDate(current.year(), current.month(), 1)), - TransactionFilterProxy::MAX_DATE); + std::nullopt); break; case LastMonth: transactionProxyModel->setDateRange( @@ -295,7 +301,7 @@ void TransactionView::chooseDate(int idx) case ThisYear: transactionProxyModel->setDateRange( GUIUtil::StartOfDay(QDate(current.year(), 1, 1)), - TransactionFilterProxy::MAX_DATE); + std::nullopt); break; case Range: dateRangeWidget->setVisible(true); @@ -351,7 +357,7 @@ void TransactionView::exportClicked() QString filename = GUIUtil::getSaveFileName(this, tr("Export Transaction History"), QString(), /*: Expanded name of the CSV file format. - See https://en.wikipedia.org/wiki/Comma-separated_values */ + See: https://en.wikipedia.org/wiki/Comma-separated_values. */ tr("Comma separated file") + QLatin1String(" (*.csv)"), nullptr); if (filename.isNull()) @@ -498,22 +504,22 @@ void TransactionView::editLabel() // Determine type of address, launch appropriate editor dialog type QString type = modelIdx.data(AddressTableModel::TypeRole).toString(); - EditAddressDialog dlg( + auto dlg = new EditAddressDialog( type == AddressTableModel::Receive ? EditAddressDialog::EditReceivingAddress : EditAddressDialog::EditSendingAddress, this); - dlg.setModel(addressBook); - dlg.loadRow(idx); - dlg.exec(); + dlg->setModel(addressBook); + dlg->loadRow(idx); + GUIUtil::ShowModalDialogAndDeleteOnClose(dlg); } else { // Add sending address - EditAddressDialog dlg(EditAddressDialog::NewSendingAddress, + auto dlg = new EditAddressDialog(EditAddressDialog::NewSendingAddress, this); - dlg.setModel(addressBook); - dlg.setAddress(address); - dlg.exec(); + dlg->setModel(addressBook); + dlg->setAddress(address); + GUIUtil::ShowModalDialogAndDeleteOnClose(dlg); } } } diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index 3cceb5ca5a..a0ad59f12a 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -41,10 +41,6 @@ WalletController::WalletController(ClientModel& client_model, const PlatformStyl getOrCreateWallet(std::move(wallet)); }); - for (std::unique_ptr<interfaces::Wallet>& wallet : m_node.walletClient().getWallets()) { - getOrCreateWallet(std::move(wallet)); - } - m_activity_worker->moveToThread(m_activity_thread); m_activity_thread->start(); QTimer::singleShot(0, m_activity_worker, []() { @@ -61,12 +57,6 @@ WalletController::~WalletController() delete m_activity_worker; } -std::vector<WalletModel*> WalletController::getOpenWallets() const -{ - QMutexLocker locker(&m_mutex); - return m_wallets; -} - std::map<std::string, bool> WalletController::listWalletDir() const { QMutexLocker locker(&m_mutex); @@ -191,33 +181,24 @@ WalletControllerActivity::WalletControllerActivity(WalletController* wallet_cont , m_wallet_controller(wallet_controller) , m_parent_widget(parent_widget) { + connect(this, &WalletControllerActivity::finished, this, &QObject::deleteLater); } -WalletControllerActivity::~WalletControllerActivity() +void WalletControllerActivity::showProgressDialog(const QString& title_text, const QString& label_text) { - delete m_progress_dialog; -} - -void WalletControllerActivity::showProgressDialog(const QString& label_text) -{ - assert(!m_progress_dialog); - m_progress_dialog = new QProgressDialog(m_parent_widget); - - m_progress_dialog->setLabelText(label_text); - m_progress_dialog->setRange(0, 0); - m_progress_dialog->setCancelButton(nullptr); - m_progress_dialog->setWindowModality(Qt::ApplicationModal); - GUIUtil::PolishProgressDialog(m_progress_dialog); + auto progress_dialog = new QProgressDialog(m_parent_widget); + progress_dialog->setAttribute(Qt::WA_DeleteOnClose); + connect(this, &WalletControllerActivity::finished, progress_dialog, &QWidget::close); + + progress_dialog->setWindowTitle(title_text); + progress_dialog->setLabelText(label_text); + progress_dialog->setRange(0, 0); + progress_dialog->setCancelButton(nullptr); + progress_dialog->setWindowModality(Qt::ApplicationModal); + GUIUtil::PolishProgressDialog(progress_dialog); // The setValue call forces QProgressDialog to start the internal duration estimation. // See details in https://bugreports.qt.io/browse/QTBUG-47042. - m_progress_dialog->setValue(0); -} - -void WalletControllerActivity::destroyProgressDialog() -{ - assert(m_progress_dialog); - delete m_progress_dialog; - m_progress_dialog = nullptr; + progress_dialog->setValue(0); } CreateWalletActivity::CreateWalletActivity(WalletController* wallet_controller, QWidget* parent_widget) @@ -251,7 +232,12 @@ void CreateWalletActivity::askPassphrase() void CreateWalletActivity::createWallet() { - showProgressDialog(tr("Creating Wallet <b>%1</b>…").arg(m_create_wallet_dialog->walletName().toHtmlEscaped())); + showProgressDialog( + //: Title of window indicating the progress of creation of a new wallet. + tr("Create Wallet"), + /*: Descriptive text of the create wallet progress window which indicates + to the user which wallet is currently being created. */ + tr("Creating Wallet <b>%1</b>…").arg(m_create_wallet_dialog->walletName().toHtmlEscaped())); std::string name = m_create_wallet_dialog->walletName().toStdString(); uint64_t flags = 0; @@ -279,8 +265,6 @@ void CreateWalletActivity::createWallet() void CreateWalletActivity::finish() { - destroyProgressDialog(); - if (!m_error_message.empty()) { QMessageBox::critical(m_parent_widget, tr("Create wallet failed"), QString::fromStdString(m_error_message.translated)); } else if (!m_warning_message.empty()) { @@ -329,8 +313,6 @@ OpenWalletActivity::OpenWalletActivity(WalletController* wallet_controller, QWid void OpenWalletActivity::finish() { - destroyProgressDialog(); - if (!m_error_message.empty()) { QMessageBox::critical(m_parent_widget, tr("Open wallet failed"), QString::fromStdString(m_error_message.translated)); } else if (!m_warning_message.empty()) { @@ -346,7 +328,12 @@ void OpenWalletActivity::open(const std::string& path) { QString name = path.empty() ? QString("["+tr("default wallet")+"]") : QString::fromStdString(path); - showProgressDialog(tr("Opening Wallet <b>%1</b>…").arg(name.toHtmlEscaped())); + showProgressDialog( + //: Title of window indicating the progress of opening of a wallet. + tr("Open Wallet"), + /*: Descriptive text of the open wallet progress window which indicates + to the user which wallet is currently being opened. */ + tr("Opening Wallet <b>%1</b>…").arg(name.toHtmlEscaped())); QTimer::singleShot(0, worker(), [this, path] { std::unique_ptr<interfaces::Wallet> wallet = node().walletClient().loadWallet(path, m_error_message, m_warning_message); @@ -356,3 +343,26 @@ void OpenWalletActivity::open(const std::string& path) QTimer::singleShot(0, this, &OpenWalletActivity::finish); }); } + +LoadWalletsActivity::LoadWalletsActivity(WalletController* wallet_controller, QWidget* parent_widget) + : WalletControllerActivity(wallet_controller, parent_widget) +{ +} + +void LoadWalletsActivity::load() +{ + showProgressDialog( + //: Title of progress window which is displayed when wallets are being loaded. + tr("Load Wallets"), + /*: Descriptive text of the load wallets progress window which indicates to + the user that wallets are currently being loaded.*/ + tr("Loading wallets…")); + + QTimer::singleShot(0, worker(), [this] { + for (auto& wallet : node().walletClient().getWallets()) { + m_wallet_controller->getOrCreateWallet(std::move(wallet)); + } + + QTimer::singleShot(0, this, [this] { Q_EMIT finished(); }); + }); +} diff --git a/src/qt/walletcontroller.h b/src/qt/walletcontroller.h index f7e366878d..bbd990228f 100644 --- a/src/qt/walletcontroller.h +++ b/src/qt/walletcontroller.h @@ -52,9 +52,6 @@ public: WalletController(ClientModel& client_model, const PlatformStyle* platform_style, QObject* parent); ~WalletController(); - //! Returns wallet models currently open. - std::vector<WalletModel*> getOpenWallets() const; - WalletModel* getOrCreateWallet(std::unique_ptr<interfaces::Wallet> wallet); //! Returns all wallet names in the wallet dir mapped to whether the wallet @@ -90,7 +87,7 @@ class WalletControllerActivity : public QObject public: WalletControllerActivity(WalletController* wallet_controller, QWidget* parent_widget); - virtual ~WalletControllerActivity(); + virtual ~WalletControllerActivity() = default; Q_SIGNALS: void finished(); @@ -99,12 +96,10 @@ protected: interfaces::Node& node() const { return m_wallet_controller->m_node; } QObject* worker() const { return m_wallet_controller->m_activity_worker; } - void showProgressDialog(const QString& label_text); - void destroyProgressDialog(); + void showProgressDialog(const QString& title_text, const QString& label_text); WalletController* const m_wallet_controller; QWidget* const m_parent_widget; - QProgressDialog* m_progress_dialog{nullptr}; WalletModel* m_wallet_model{nullptr}; bilingual_str m_error_message; std::vector<bilingual_str> m_warning_message; @@ -150,4 +145,14 @@ private: void finish(); }; +class LoadWalletsActivity : public WalletControllerActivity +{ + Q_OBJECT + +public: + LoadWalletsActivity(WalletController* wallet_controller, QWidget* parent_widget); + + void load(); +}; + #endif // BITCOIN_QT_WALLETCONTROLLER_H diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index a1f357e0db..4ff92bf82c 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -4,12 +4,19 @@ #include <qt/walletframe.h> +#include <node/ui_interface.h> +#include <psbt.h> +#include <qt/guiutil.h> #include <qt/overviewpage.h> +#include <qt/psbtoperationsdialog.h> #include <qt/walletmodel.h> #include <qt/walletview.h> +#include <util/system.h> #include <cassert> +#include <QApplication> +#include <QClipboard> #include <QGroupBox> #include <QHBoxLayout> #include <QLabel> @@ -58,14 +65,13 @@ void WalletFrame::setClientModel(ClientModel *_clientModel) } } -bool WalletFrame::addWallet(WalletModel* walletModel, WalletView* walletView) +bool WalletFrame::addView(WalletView* walletView) { - if (!clientModel || !walletModel) return false; + if (!clientModel) return false; - if (mapWalletViews.count(walletModel) > 0) return false; + if (mapWalletViews.count(walletView->getWalletModel()) > 0) return false; walletView->setClientModel(clientModel); - walletView->setWalletModel(walletModel); walletView->showOutOfSyncWarning(bOutOfSync); WalletView* current_wallet_view = currentWalletView(); @@ -76,7 +82,7 @@ bool WalletFrame::addWallet(WalletModel* walletModel, WalletView* walletView) } walletStack->addWidget(walletView); - mapWalletViews[walletModel] = walletView; + mapWalletViews[walletView->getWalletModel()] = walletView; return true; } @@ -103,7 +109,8 @@ void WalletFrame::setCurrentWallet(WalletModel* wallet_model) walletView->updateGeometry(); walletStack->setCurrentWidget(walletView); - walletView->updateEncryptionStatus(); + + Q_EMIT currentWalletSet(); } void WalletFrame::removeWallet(WalletModel* wallet_model) @@ -184,10 +191,39 @@ void WalletFrame::gotoVerifyMessageTab(QString addr) void WalletFrame::gotoLoadPSBT(bool from_clipboard) { - WalletView *walletView = currentWalletView(); - if (walletView) { - walletView->gotoLoadPSBT(from_clipboard); + std::string data; + + if (from_clipboard) { + std::string raw = QApplication::clipboard()->text().toStdString(); + bool invalid; + data = DecodeBase64(raw, &invalid); + if (invalid) { + Q_EMIT message(tr("Error"), tr("Unable to decode PSBT from clipboard (invalid base64)"), CClientUIInterface::MSG_ERROR); + return; + } + } else { + QString filename = GUIUtil::getOpenFileName(this, + tr("Load Transaction Data"), QString(), + tr("Partially Signed Transaction (*.psbt)"), nullptr); + if (filename.isEmpty()) return; + if (GetFileSize(filename.toLocal8Bit().data(), MAX_FILE_SIZE_PSBT) == MAX_FILE_SIZE_PSBT) { + Q_EMIT message(tr("Error"), tr("PSBT file must be smaller than 100 MiB"), CClientUIInterface::MSG_ERROR); + return; + } + std::ifstream in(filename.toLocal8Bit().data(), std::ios::binary); + data = std::string(std::istreambuf_iterator<char>{in}, {}); } + + std::string error; + PartiallySignedTransaction psbtx; + if (!DecodeRawPSBT(psbtx, data, error)) { + Q_EMIT message(tr("Error"), tr("Unable to decode PSBT") + "\n" + QString::fromStdString(error), CClientUIInterface::MSG_ERROR); + return; + } + + auto dlg = new PSBTOperationsDialog(this, currentWalletModel(), clientModel); + dlg->openWithPSBT(psbtx); + GUIUtil::ShowModalDialogAndDeleteOnClose(dlg); } void WalletFrame::encryptWallet() diff --git a/src/qt/walletframe.h b/src/qt/walletframe.h index 4f77bd716f..cfca5c4c5c 100644 --- a/src/qt/walletframe.h +++ b/src/qt/walletframe.h @@ -35,7 +35,7 @@ public: void setClientModel(ClientModel *clientModel); - bool addWallet(WalletModel* walletModel, WalletView* walletView); + bool addView(WalletView* walletView); void setCurrentWallet(WalletModel* wallet_model); void removeWallet(WalletModel* wallet_model); void removeAllWallets(); @@ -48,6 +48,8 @@ public: Q_SIGNALS: void createWalletButtonClicked(); + void message(const QString& title, const QString& message, unsigned int style); + void currentWalletSet(); private: QStackedWidget *walletStack; diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 967dd588b4..052453cf65 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -506,9 +506,10 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) questionString.append(tr("Warning: This may pay the additional fee by reducing change outputs or adding inputs, when necessary. It may add a new change output if one does not already exist. These changes may potentially leak privacy.")); } - SendConfirmationDialog confirmationDialog(tr("Confirm fee bump"), questionString); - confirmationDialog.exec(); - QMessageBox::StandardButton retval = static_cast<QMessageBox::StandardButton>(confirmationDialog.result()); + auto confirmationDialog = new SendConfirmationDialog(tr("Confirm fee bump"), questionString); + confirmationDialog->setAttribute(Qt::WA_DeleteOnClose); + // TODO: Replace QDialog::exec() with safer QDialog::show(). + const auto retval = static_cast<QMessageBox::StandardButton>(confirmationDialog->exec()); // cancel sign&broadcast if user doesn't want to bump the fee if (retval != QMessageBox::Yes) { diff --git a/src/qt/walletmodeltransaction.h b/src/qt/walletmodeltransaction.h index 120d240d91..0bae4bade3 100644 --- a/src/qt/walletmodeltransaction.h +++ b/src/qt/walletmodeltransaction.h @@ -8,7 +8,7 @@ #include <primitives/transaction.h> #include <qt/sendcoinsrecipient.h> -#include <amount.h> +#include <consensus/amount.h> #include <QObject> diff --git a/src/qt/walletview.cpp b/src/qt/walletview.cpp index 3b8cf4c7ed..7813b89e41 100644 --- a/src/qt/walletview.cpp +++ b/src/qt/walletview.cpp @@ -8,7 +8,6 @@ #include <qt/askpassphrasedialog.h> #include <qt/clientmodel.h> #include <qt/guiutil.h> -#include <qt/psbtoperationsdialog.h> #include <qt/optionsmodel.h> #include <qt/overviewpage.h> #include <qt/platformstyle.h> @@ -21,32 +20,34 @@ #include <interfaces/node.h> #include <node/ui_interface.h> -#include <psbt.h> #include <util/strencodings.h> #include <QAction> #include <QActionGroup> -#include <QApplication> -#include <QClipboard> #include <QFileDialog> #include <QHBoxLayout> #include <QProgressDialog> #include <QPushButton> #include <QVBoxLayout> -WalletView::WalletView(const PlatformStyle *_platformStyle, QWidget *parent): - QStackedWidget(parent), - clientModel(nullptr), - walletModel(nullptr), - platformStyle(_platformStyle) +WalletView::WalletView(WalletModel* wallet_model, const PlatformStyle* _platformStyle, QWidget* parent) + : QStackedWidget(parent), + clientModel(nullptr), + walletModel(wallet_model), + platformStyle(_platformStyle) { + assert(walletModel); + // Create tabs overviewPage = new OverviewPage(platformStyle); + overviewPage->setWalletModel(walletModel); transactionsPage = new QWidget(this); QVBoxLayout *vbox = new QVBoxLayout(); QHBoxLayout *hbox_buttons = new QHBoxLayout(); transactionView = new TransactionView(platformStyle, this); + transactionView->setModel(walletModel); + vbox->addWidget(transactionView); QPushButton *exportButton = new QPushButton(tr("&Export"), this); exportButton->setToolTip(tr("Export the data in the current tab to a file")); @@ -59,10 +60,16 @@ WalletView::WalletView(const PlatformStyle *_platformStyle, QWidget *parent): transactionsPage->setLayout(vbox); receiveCoinsPage = new ReceiveCoinsDialog(platformStyle); + receiveCoinsPage->setModel(walletModel); + sendCoinsPage = new SendCoinsDialog(platformStyle); + sendCoinsPage->setModel(walletModel); usedSendingAddressesPage = new AddressBookPage(platformStyle, AddressBookPage::ForEditing, AddressBookPage::SendingTab, this); + usedSendingAddressesPage->setModel(walletModel->getAddressTableModel()); + usedReceivingAddressesPage = new AddressBookPage(platformStyle, AddressBookPage::ForEditing, AddressBookPage::ReceivingTab, this); + usedReceivingAddressesPage->setModel(walletModel->getAddressTableModel()); addWidget(overviewPage); addWidget(transactionsPage); @@ -88,6 +95,21 @@ WalletView::WalletView(const PlatformStyle *_platformStyle, QWidget *parent): connect(transactionView, &TransactionView::message, this, &WalletView::message); connect(this, &WalletView::setPrivacy, overviewPage, &OverviewPage::setPrivacy); + + // Receive and pass through messages from wallet model + connect(walletModel, &WalletModel::message, this, &WalletView::message); + + // Handle changes in encryption status + connect(walletModel, &WalletModel::encryptionStatusChanged, this, &WalletView::encryptionStatusChanged); + + // Balloon pop-up for new transaction + connect(walletModel->getTransactionTableModel(), &TransactionTableModel::rowsInserted, this, &WalletView::processNewTransaction); + + // Ask for passphrase if needed + connect(walletModel, &WalletModel::requireUnlock, this, &WalletView::unlockWallet); + + // Show progress dialog + connect(walletModel, &WalletModel::showProgress, this, &WalletView::showProgress); } WalletView::~WalletView() @@ -100,49 +122,15 @@ void WalletView::setClientModel(ClientModel *_clientModel) overviewPage->setClientModel(_clientModel); sendCoinsPage->setClientModel(_clientModel); - if (walletModel) walletModel->setClientModel(_clientModel); -} - -void WalletView::setWalletModel(WalletModel *_walletModel) -{ - this->walletModel = _walletModel; - - // Put transaction list in tabs - transactionView->setModel(_walletModel); - overviewPage->setWalletModel(_walletModel); - receiveCoinsPage->setModel(_walletModel); - sendCoinsPage->setModel(_walletModel); - usedReceivingAddressesPage->setModel(_walletModel ? _walletModel->getAddressTableModel() : nullptr); - usedSendingAddressesPage->setModel(_walletModel ? _walletModel->getAddressTableModel() : nullptr); - - if (_walletModel) - { - // Receive and pass through messages from wallet model - connect(_walletModel, &WalletModel::message, this, &WalletView::message); - - // Handle changes in encryption status - connect(_walletModel, &WalletModel::encryptionStatusChanged, this, &WalletView::encryptionStatusChanged); - updateEncryptionStatus(); - - // update HD status - Q_EMIT hdEnabledStatusChanged(); - - // Balloon pop-up for new transaction - connect(_walletModel->getTransactionTableModel(), &TransactionTableModel::rowsInserted, this, &WalletView::processNewTransaction); - - // Ask for passphrase if needed - connect(_walletModel, &WalletModel::requireUnlock, this, &WalletView::unlockWallet); - - // Show progress dialog - connect(_walletModel, &WalletModel::showProgress, this, &WalletView::showProgress); - } + walletModel->setClientModel(_clientModel); } void WalletView::processNewTransaction(const QModelIndex& parent, int start, int /*end*/) { // Prevent balloon-spam when initial block download is in progress - if (!walletModel || !clientModel || clientModel->node().isInitialBlockDownload()) + if (!clientModel || clientModel->node().isInitialBlockDownload()) { return; + } TransactionTableModel *ttm = walletModel->getTransactionTableModel(); if (!ttm || ttm->processingQueuedTransactions()) @@ -205,44 +193,6 @@ void WalletView::gotoVerifyMessageTab(QString addr) signVerifyMessageDialog->setAddress_VM(addr); } -void WalletView::gotoLoadPSBT(bool from_clipboard) -{ - std::string data; - - if (from_clipboard) { - std::string raw = QApplication::clipboard()->text().toStdString(); - bool invalid; - data = DecodeBase64(raw, &invalid); - if (invalid) { - Q_EMIT message(tr("Error"), tr("Unable to decode PSBT from clipboard (invalid base64)"), CClientUIInterface::MSG_ERROR); - return; - } - } else { - QString filename = GUIUtil::getOpenFileName(this, - tr("Load Transaction Data"), QString(), - tr("Partially Signed Transaction (*.psbt)"), nullptr); - if (filename.isEmpty()) return; - if (GetFileSize(filename.toLocal8Bit().data(), MAX_FILE_SIZE_PSBT) == MAX_FILE_SIZE_PSBT) { - Q_EMIT message(tr("Error"), tr("PSBT file must be smaller than 100 MiB"), CClientUIInterface::MSG_ERROR); - return; - } - std::ifstream in(filename.toLocal8Bit().data(), std::ios::binary); - data = std::string(std::istreambuf_iterator<char>{in}, {}); - } - - std::string error; - PartiallySignedTransaction psbtx; - if (!DecodeRawPSBT(psbtx, data, error)) { - Q_EMIT message(tr("Error"), tr("Unable to decode PSBT") + "\n" + QString::fromStdString(error), CClientUIInterface::MSG_ERROR); - return; - } - - PSBTOperationsDialog* dlg = new PSBTOperationsDialog(this, walletModel, clientModel); - dlg->openWithPSBT(psbtx); - dlg->setAttribute(Qt::WA_DeleteOnClose); - dlg->exec(); -} - bool WalletView::handlePaymentRequest(const SendCoinsRecipient& recipient) { return sendCoinsPage->handlePaymentRequest(recipient); @@ -253,20 +203,12 @@ void WalletView::showOutOfSyncWarning(bool fShow) overviewPage->showOutOfSyncWarning(fShow); } -void WalletView::updateEncryptionStatus() -{ - Q_EMIT encryptionStatusChanged(); -} - void WalletView::encryptWallet() { - if(!walletModel) - return; - AskPassphraseDialog dlg(AskPassphraseDialog::Encrypt, this); - dlg.setModel(walletModel); - dlg.exec(); - - updateEncryptionStatus(); + auto dlg = new AskPassphraseDialog(AskPassphraseDialog::Encrypt, this); + dlg->setModel(walletModel); + connect(dlg, &QDialog::finished, this, &WalletView::encryptionStatusChanged); + GUIUtil::ShowModalDialogAndDeleteOnClose(dlg); } void WalletView::backupWallet() @@ -291,37 +233,28 @@ void WalletView::backupWallet() void WalletView::changePassphrase() { - AskPassphraseDialog dlg(AskPassphraseDialog::ChangePass, this); - dlg.setModel(walletModel); - dlg.exec(); + auto dlg = new AskPassphraseDialog(AskPassphraseDialog::ChangePass, this); + dlg->setModel(walletModel); + GUIUtil::ShowModalDialogAndDeleteOnClose(dlg); } void WalletView::unlockWallet() { - if(!walletModel) - return; // Unlock wallet when requested by wallet model - if (walletModel->getEncryptionStatus() == WalletModel::Locked) - { - AskPassphraseDialog dlg(AskPassphraseDialog::Unlock, this); - dlg.setModel(walletModel); - dlg.exec(); + if (walletModel->getEncryptionStatus() == WalletModel::Locked) { + auto dlg = new AskPassphraseDialog(AskPassphraseDialog::Unlock, this); + dlg->setModel(walletModel); + GUIUtil::ShowModalDialogAndDeleteOnClose(dlg); } } void WalletView::usedSendingAddresses() { - if(!walletModel) - return; - GUIUtil::bringToFront(usedSendingAddressesPage); } void WalletView::usedReceivingAddresses() { - if(!walletModel) - return; - GUIUtil::bringToFront(usedReceivingAddressesPage); } diff --git a/src/qt/walletview.h b/src/qt/walletview.h index fedf06b710..86a835c484 100644 --- a/src/qt/walletview.h +++ b/src/qt/walletview.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_WALLETVIEW_H #define BITCOIN_QT_WALLETVIEW_H -#include <amount.h> +#include <consensus/amount.h> #include <QStackedWidget> @@ -35,19 +35,14 @@ class WalletView : public QStackedWidget Q_OBJECT public: - explicit WalletView(const PlatformStyle *platformStyle, QWidget *parent); + explicit WalletView(WalletModel* wallet_model, const PlatformStyle* platformStyle, QWidget* parent); ~WalletView(); /** Set the client model. The client model represents the part of the core that communicates with the P2P network, and is wallet-agnostic. */ void setClientModel(ClientModel *clientModel); - WalletModel *getWalletModel() { return walletModel; } - /** Set the wallet model. - The wallet model represents a bitcoin wallet, and offers access to the list of transactions, address book and sending - functionality. - */ - void setWalletModel(WalletModel *walletModel); + WalletModel* getWalletModel() const noexcept { return walletModel; } bool handlePaymentRequest(const SendCoinsRecipient& recipient); @@ -55,7 +50,12 @@ public: private: ClientModel *clientModel; - WalletModel *walletModel; + + //! + //! The wallet model represents a bitcoin wallet, and offers access to + //! the list of transactions, address book and sending functionality. + //! + WalletModel* const walletModel; OverviewPage *overviewPage; QWidget *transactionsPage; @@ -83,8 +83,6 @@ public Q_SLOTS: void gotoSignMessageTab(QString addr = ""); /** Show Sign/Verify Message dialog and switch to verify message tab */ void gotoVerifyMessageTab(QString addr = ""); - /** Load Partially Signed Bitcoin Transaction */ - void gotoLoadPSBT(bool from_clipboard = false); /** Show incoming transaction notification for new transactions. @@ -105,9 +103,6 @@ public Q_SLOTS: /** Show used receiving addresses */ void usedReceivingAddresses(); - /** Re-emit encryption status signal */ - void updateEncryptionStatus(); - /** Show progress dialog e.g. for rescan */ void showProgress(const QString &title, int nProgress); @@ -119,8 +114,6 @@ Q_SIGNALS: void message(const QString &title, const QString &message, unsigned int style); /** Encryption status of wallet changed */ void encryptionStatusChanged(); - /** HD-Enabled status of wallet changed (only possible during startup) */ - void hdEnabledStatusChanged(); /** Notify that a new transaction appeared */ void incomingTransaction(const QString& date, int unit, const CAmount& amount, const QString& type, const QString& address, const QString& label, const QString& walletName); /** Notify that the out of sync warning icon has been pressed */ diff --git a/src/qt/winshutdownmonitor.h b/src/qt/winshutdownmonitor.h index 8edb98c744..bf399edcf3 100644 --- a/src/qt/winshutdownmonitor.h +++ b/src/qt/winshutdownmonitor.h @@ -17,7 +17,7 @@ class WinShutdownMonitor : public QAbstractNativeEventFilter { public: /** Implements QAbstractNativeEventFilter interface for processing Windows messages */ - bool nativeEventFilter(const QByteArray &eventType, void *pMessage, long *pnResult); + bool nativeEventFilter(const QByteArray &eventType, void *pMessage, long *pnResult) override; /** Register the reason for blocking shutdown on Windows to allow clean client exit */ static void registerShutdownBlockReason(const QString& strReason, const HWND& mainWinId); diff --git a/src/randomenv.cpp b/src/randomenv.cpp index fa2a3a0607..bf23ea4a12 100644 --- a/src/randomenv.cpp +++ b/src/randomenv.cpp @@ -53,7 +53,7 @@ #include <sys/vmmeter.h> #endif #endif -#if defined(HAVE_STRONG_GETAUXVAL) || defined(HAVE_WEAK_GETAUXVAL) +#if defined(HAVE_STRONG_GETAUXVAL) #include <sys/auxv.h> #endif @@ -326,7 +326,7 @@ void RandAddStaticEnv(CSHA512& hasher) // Bitcoin client version hasher << CLIENT_VERSION; -#if defined(HAVE_STRONG_GETAUXVAL) || defined(HAVE_WEAK_GETAUXVAL) +#if defined(HAVE_STRONG_GETAUXVAL) // Information available through getauxval() # ifdef AT_HWCAP hasher << getauxval(AT_HWCAP); @@ -346,7 +346,7 @@ void RandAddStaticEnv(CSHA512& hasher) const char* exec_str = (const char*)getauxval(AT_EXECFN); if (exec_str) hasher.Write((const unsigned char*)exec_str, strlen(exec_str) + 1); # endif -#endif // HAVE_STRONG_GETAUXVAL || HAVE_WEAK_GETAUXVAL +#endif // HAVE_STRONG_GETAUXVAL #ifdef HAVE_GETCPUID AddAllCPUID(hasher); diff --git a/src/rest.cpp b/src/rest.cpp index d599f381e3..e50ab33e54 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -524,6 +524,7 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: // convert hex to bin, continue then with bin part std::vector<unsigned char> strRequestV = ParseHex(strRequestMutable); strRequestMutable.assign(strRequestV.begin(), strRequestV.end()); + [[fallthrough]]; } case RetFormat::BINARY: { diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index f996aa9c76..ac746de32f 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -5,13 +5,16 @@ #include <rpc/blockchain.h> -#include <amount.h> #include <blockfilter.h> #include <chain.h> #include <chainparams.h> #include <coins.h> +#include <consensus/amount.h> +#include <consensus/params.h> #include <consensus/validation.h> #include <core_io.h> +#include <deploymentinfo.h> +#include <deploymentstatus.h> #include <hash.h> #include <index/blockfilterindex.h> #include <index/coinstatsindex.h> @@ -33,10 +36,12 @@ #include <txmempool.h> #include <undo.h> #include <util/strencodings.h> +#include <util/string.h> #include <util/system.h> #include <util/translation.h> #include <validation.h> #include <validationinterface.h> +#include <versionbits.h> #include <warnings.h> #include <stdint.h> @@ -1110,19 +1115,19 @@ static RPCHelpMan gettxoutsetinfo() {RPCResult::Type::NUM, "bogosize", "Database-independent, meaningless metric indicating the UTXO set size"}, {RPCResult::Type::STR_HEX, "hash_serialized_2", /* optional */ true, "The serialized hash (only present if 'hash_serialized_2' hash_type is chosen)"}, {RPCResult::Type::STR_HEX, "muhash", /* optional */ true, "The serialized hash (only present if 'muhash' hash_type is chosen)"}, - {RPCResult::Type::NUM, "transactions", "The number of transactions with unspent outputs (not available when coinstatsindex is used)"}, - {RPCResult::Type::NUM, "disk_size", "The estimated size of the chainstate on disk (not available when coinstatsindex is used)"}, + {RPCResult::Type::NUM, "transactions", /* optional */ true, "The number of transactions with unspent outputs (not available when coinstatsindex is used)"}, + {RPCResult::Type::NUM, "disk_size", /* optional */ true, "The estimated size of the chainstate on disk (not available when coinstatsindex is used)"}, {RPCResult::Type::STR_AMOUNT, "total_amount", "The total amount of coins in the UTXO set"}, - {RPCResult::Type::STR_AMOUNT, "total_unspendable_amount", "The total amount of coins permanently excluded from the UTXO set (only available if coinstatsindex is used)"}, - {RPCResult::Type::OBJ, "block_info", "Info on amounts in the block at this block height (only available if coinstatsindex is used)", + {RPCResult::Type::STR_AMOUNT, "total_unspendable_amount", /* optional */ true, "The total amount of coins permanently excluded from the UTXO set (only available if coinstatsindex is used)"}, + {RPCResult::Type::OBJ, "block_info", /* optional */ true, "Info on amounts in the block at this block height (only available if coinstatsindex is used)", { - {RPCResult::Type::STR_AMOUNT, "prevout_spent", ""}, - {RPCResult::Type::STR_AMOUNT, "coinbase", ""}, - {RPCResult::Type::STR_AMOUNT, "new_outputs_ex_coinbase", ""}, - {RPCResult::Type::STR_AMOUNT, "unspendable", ""}, + {RPCResult::Type::STR_AMOUNT, "prevout_spent", "Total amount of all prevouts spent in this block"}, + {RPCResult::Type::STR_AMOUNT, "coinbase", "Coinbase subsidy amount of this block"}, + {RPCResult::Type::STR_AMOUNT, "new_outputs_ex_coinbase", "Total amount of new outputs created by this block"}, + {RPCResult::Type::STR_AMOUNT, "unspendable", "Total amount of unspendable outputs created in this block"}, {RPCResult::Type::OBJ, "unspendables", "Detailed view of the unspendable categories", { - {RPCResult::Type::STR_AMOUNT, "genesis_block", ""}, + {RPCResult::Type::STR_AMOUNT, "genesis_block", "The unspendable amount of the Genesis block subsidy"}, {RPCResult::Type::STR_AMOUNT, "bip30", "Transactions overridden by duplicates (no longer possible with BIP30)"}, {RPCResult::Type::STR_AMOUNT, "scripts", "Amounts sent to scripts that are unspendable (for example OP_RETURN outputs)"}, {RPCResult::Type::STR_AMOUNT, "unclaimed_rewards", "Fee rewards that miners did not claim in their coinbase transaction"}, @@ -1174,6 +1179,18 @@ static RPCHelpMan gettxoutsetinfo() pindex = ParseHashOrHeight(request.params[1], chainman); } + if (stats.index_requested && g_coin_stats_index) { + if (!g_coin_stats_index->BlockUntilSyncedToCurrentChain()) { + const IndexSummary summary{g_coin_stats_index->GetSummary()}; + + // If a specific block was requested and the index has already synced past that height, we can return the + // data already even though the index is not fully synced yet. + if (pindex->nHeight > summary.best_block_height) { + throw JSONRPCError(RPC_INTERNAL_ERROR, strprintf("Unable to get data because coinstatsindex is still syncing. Current height: %d", summary.best_block_height)); + } + } + } + if (GetUTXOStats(coins_view, *blockman, stats, node.rpc_interruption_point, pindex)) { ret.pushKV("height", (int64_t)stats.nHeight); ret.pushKV("bestblock", stats.hashBlock.GetHex()); @@ -1190,7 +1207,7 @@ static RPCHelpMan gettxoutsetinfo() ret.pushKV("transactions", static_cast<int64_t>(stats.nTransactions)); ret.pushKV("disk_size", stats.nDiskSize); } else { - ret.pushKV("total_unspendable_amount", ValueFromAmount(stats.block_unspendable_amount)); + ret.pushKV("total_unspendable_amount", ValueFromAmount(stats.total_unspendable_amount)); CCoinsStats prev_stats{hash_type}; @@ -1199,28 +1216,21 @@ static RPCHelpMan gettxoutsetinfo() } UniValue block_info(UniValue::VOBJ); - block_info.pushKV("prevout_spent", ValueFromAmount(stats.block_prevout_spent_amount - prev_stats.block_prevout_spent_amount)); - block_info.pushKV("coinbase", ValueFromAmount(stats.block_coinbase_amount - prev_stats.block_coinbase_amount)); - block_info.pushKV("new_outputs_ex_coinbase", ValueFromAmount(stats.block_new_outputs_ex_coinbase_amount - prev_stats.block_new_outputs_ex_coinbase_amount)); - block_info.pushKV("unspendable", ValueFromAmount(stats.block_unspendable_amount - prev_stats.block_unspendable_amount)); + block_info.pushKV("prevout_spent", ValueFromAmount(stats.total_prevout_spent_amount - prev_stats.total_prevout_spent_amount)); + block_info.pushKV("coinbase", ValueFromAmount(stats.total_coinbase_amount - prev_stats.total_coinbase_amount)); + block_info.pushKV("new_outputs_ex_coinbase", ValueFromAmount(stats.total_new_outputs_ex_coinbase_amount - prev_stats.total_new_outputs_ex_coinbase_amount)); + block_info.pushKV("unspendable", ValueFromAmount(stats.total_unspendable_amount - prev_stats.total_unspendable_amount)); UniValue unspendables(UniValue::VOBJ); - unspendables.pushKV("genesis_block", ValueFromAmount(stats.unspendables_genesis_block - prev_stats.unspendables_genesis_block)); - unspendables.pushKV("bip30", ValueFromAmount(stats.unspendables_bip30 - prev_stats.unspendables_bip30)); - unspendables.pushKV("scripts", ValueFromAmount(stats.unspendables_scripts - prev_stats.unspendables_scripts)); - unspendables.pushKV("unclaimed_rewards", ValueFromAmount(stats.unspendables_unclaimed_rewards - prev_stats.unspendables_unclaimed_rewards)); + unspendables.pushKV("genesis_block", ValueFromAmount(stats.total_unspendables_genesis_block - prev_stats.total_unspendables_genesis_block)); + unspendables.pushKV("bip30", ValueFromAmount(stats.total_unspendables_bip30 - prev_stats.total_unspendables_bip30)); + unspendables.pushKV("scripts", ValueFromAmount(stats.total_unspendables_scripts - prev_stats.total_unspendables_scripts)); + unspendables.pushKV("unclaimed_rewards", ValueFromAmount(stats.total_unspendables_unclaimed_rewards - prev_stats.total_unspendables_unclaimed_rewards)); block_info.pushKV("unspendables", unspendables); ret.pushKV("block_info", block_info); } } else { - if (g_coin_stats_index) { - const IndexSummary summary{g_coin_stats_index->GetSummary()}; - - if (!summary.synced) { - throw JSONRPCError(RPC_INTERNAL_ERROR, strprintf("Unable to read UTXO set because coinstatsindex is still syncing. Current height: %d", summary.best_block_height)); - } - } throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); } return ret; @@ -1246,11 +1256,8 @@ static RPCHelpMan gettxout() {RPCResult::Type::OBJ, "scriptPubKey", "", { {RPCResult::Type::STR, "asm", ""}, {RPCResult::Type::STR_HEX, "hex", ""}, - {RPCResult::Type::NUM, "reqSigs", /* optional */ true, "(DEPRECATED, returned only if config option -deprecatedrpc=addresses is passed) Number of required signatures"}, {RPCResult::Type::STR, "type", "The type, eg pubkeyhash"}, - {RPCResult::Type::STR, "address", /* optional */ true, "bitcoin address (only if a well-defined address exists)"}, - {RPCResult::Type::ARR, "addresses", /* optional */ true, "(DEPRECATED, returned only if config option -deprecatedrpc=addresses is passed) Array of bitcoin addresses", - {{RPCResult::Type::STR, "address", "bitcoin address"}}}, + {RPCResult::Type::STR, "address", /* optional */ true, "The Bitcoin address (only if a well-defined address exists)"}, }}, {RPCResult::Type::BOOL, "coinbase", "Coinbase or not"}, }}, @@ -1319,7 +1326,7 @@ static RPCHelpMan verifychain() "\nVerifies blockchain database.\n", { {"checklevel", RPCArg::Type::NUM, RPCArg::DefaultHint{strprintf("%d, range=0-4", DEFAULT_CHECKLEVEL)}, - strprintf("How thorough the block verification is:\n - %s", Join(CHECKLEVEL_DOC, "\n- "))}, + strprintf("How thorough the block verification is:\n%s", MakeUnorderedList(CHECKLEVEL_DOC))}, {"nblocks", RPCArg::Type::NUM, RPCArg::DefaultHint{strprintf("%d, 0=all", DEFAULT_CHECKBLOCKS)}, "The number of blocks to check."}, }, RPCResult{ @@ -1343,32 +1350,29 @@ static RPCHelpMan verifychain() }; } -static void BuriedForkDescPushBack(UniValue& softforks, const std::string &name, int softfork_height, int tip_height) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +static void SoftForkDescPushBack(const CBlockIndex* active_chain_tip, UniValue& softforks, const Consensus::Params& params, Consensus::BuriedDeployment dep) { // For buried deployments. - // A buried deployment is one where the height of the activation has been hardcoded into - // the client implementation long after the consensus change has activated. See BIP 90. - // Buried deployments with activation height value of - // std::numeric_limits<int>::max() are disabled and thus hidden. - if (softfork_height == std::numeric_limits<int>::max()) return; + + if (!DeploymentEnabled(params, dep)) return; UniValue rv(UniValue::VOBJ); rv.pushKV("type", "buried"); // getblockchaininfo reports the softfork as active from when the chain height is // one below the activation height - rv.pushKV("active", tip_height + 1 >= softfork_height); - rv.pushKV("height", softfork_height); - softforks.pushKV(name, rv); + rv.pushKV("active", DeploymentActiveAfter(active_chain_tip, params, dep)); + rv.pushKV("height", params.DeploymentHeight(dep)); + softforks.pushKV(DeploymentName(dep), rv); } -static void BIP9SoftForkDescPushBack(const CBlockIndex* active_chain_tip, UniValue& softforks, const std::string &name, const Consensus::Params& consensusParams, Consensus::DeploymentPos id) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +static void SoftForkDescPushBack(const CBlockIndex* active_chain_tip, UniValue& softforks, const Consensus::Params& consensusParams, Consensus::DeploymentPos id) { // For BIP9 deployments. - // Deployments that are never active are hidden. - if (consensusParams.vDeployments[id].nStartTime == Consensus::BIP9Deployment::NEVER_ACTIVE) return; + + if (!DeploymentEnabled(consensusParams, id)) return; UniValue bip9(UniValue::VOBJ); - const ThresholdState thresholdState = VersionBitsState(active_chain_tip, consensusParams, id, versionbitscache); + const ThresholdState thresholdState = g_versionbitscache.State(active_chain_tip, consensusParams, id); switch (thresholdState) { case ThresholdState::DEFINED: bip9.pushKV("status", "defined"); break; case ThresholdState::STARTED: bip9.pushKV("status", "started"); break; @@ -1376,23 +1380,24 @@ static void BIP9SoftForkDescPushBack(const CBlockIndex* active_chain_tip, UniVal case ThresholdState::ACTIVE: bip9.pushKV("status", "active"); break; case ThresholdState::FAILED: bip9.pushKV("status", "failed"); break; } - if (ThresholdState::STARTED == thresholdState) - { + const bool has_signal = (ThresholdState::STARTED == thresholdState || ThresholdState::LOCKED_IN == thresholdState); + if (has_signal) { bip9.pushKV("bit", consensusParams.vDeployments[id].bit); } bip9.pushKV("start_time", consensusParams.vDeployments[id].nStartTime); bip9.pushKV("timeout", consensusParams.vDeployments[id].nTimeout); - int64_t since_height = VersionBitsStateSinceHeight(active_chain_tip, consensusParams, id, versionbitscache); + int64_t since_height = g_versionbitscache.StateSinceHeight(active_chain_tip, consensusParams, id); bip9.pushKV("since", since_height); - if (ThresholdState::STARTED == thresholdState) - { + if (has_signal) { UniValue statsUV(UniValue::VOBJ); - BIP9Stats statsStruct = VersionBitsStatistics(active_chain_tip, consensusParams, id); + BIP9Stats statsStruct = g_versionbitscache.Statistics(active_chain_tip, consensusParams, id); statsUV.pushKV("period", statsStruct.period); - statsUV.pushKV("threshold", statsStruct.threshold); statsUV.pushKV("elapsed", statsStruct.elapsed); statsUV.pushKV("count", statsStruct.count); - statsUV.pushKV("possible", statsStruct.possible); + if (ThresholdState::LOCKED_IN != thresholdState) { + statsUV.pushKV("threshold", statsStruct.threshold); + statsUV.pushKV("possible", statsStruct.possible); + } bip9.pushKV("statistics", statsUV); } bip9.pushKV("min_activation_height", consensusParams.vDeployments[id].min_activation_height); @@ -1405,7 +1410,7 @@ static void BIP9SoftForkDescPushBack(const CBlockIndex* active_chain_tip, UniVal } rv.pushKV("active", ThresholdState::ACTIVE == thresholdState); - softforks.pushKV(name, rv); + softforks.pushKV(DeploymentName(id), rv); } RPCHelpMan getblockchaininfo() @@ -1421,38 +1426,39 @@ RPCHelpMan getblockchaininfo() {RPCResult::Type::NUM, "headers", "the current number of headers we have validated"}, {RPCResult::Type::STR, "bestblockhash", "the hash of the currently best block"}, {RPCResult::Type::NUM, "difficulty", "the current difficulty"}, - {RPCResult::Type::NUM, "mediantime", "median time for the current best block"}, + {RPCResult::Type::NUM_TIME, "time", "The block time expressed in " + UNIX_EPOCH_TIME}, + {RPCResult::Type::NUM_TIME, "mediantime", "The median block time expressed in " + UNIX_EPOCH_TIME}, {RPCResult::Type::NUM, "verificationprogress", "estimate of verification progress [0..1]"}, {RPCResult::Type::BOOL, "initialblockdownload", "(debug information) estimate of whether this node is in Initial Block Download mode"}, {RPCResult::Type::STR_HEX, "chainwork", "total amount of work in active chain, in hexadecimal"}, {RPCResult::Type::NUM, "size_on_disk", "the estimated size of the block and undo files on disk"}, {RPCResult::Type::BOOL, "pruned", "if the blocks are subject to pruning"}, - {RPCResult::Type::NUM, "pruneheight", "lowest-height complete block stored (only present if pruning is enabled)"}, - {RPCResult::Type::BOOL, "automatic_pruning", "whether automatic pruning is enabled (only present if pruning is enabled)"}, - {RPCResult::Type::NUM, "prune_target_size", "the target size used by pruning (only present if automatic pruning is enabled)"}, + {RPCResult::Type::NUM, "pruneheight", /* optional */ true, "lowest-height complete block stored (only present if pruning is enabled)"}, + {RPCResult::Type::BOOL, "automatic_pruning", /* optional */ true, "whether automatic pruning is enabled (only present if pruning is enabled)"}, + {RPCResult::Type::NUM, "prune_target_size", /* optional */ true, "the target size used by pruning (only present if automatic pruning is enabled)"}, {RPCResult::Type::OBJ_DYN, "softforks", "status of softforks", { {RPCResult::Type::OBJ, "xxxx", "name of the softfork", { {RPCResult::Type::STR, "type", "one of \"buried\", \"bip9\""}, - {RPCResult::Type::OBJ, "bip9", "status of bip9 softforks (only for \"bip9\" type)", + {RPCResult::Type::OBJ, "bip9", /* optional */ true, "status of bip9 softforks (only for \"bip9\" type)", { {RPCResult::Type::STR, "status", "one of \"defined\", \"started\", \"locked_in\", \"active\", \"failed\""}, - {RPCResult::Type::NUM, "bit", "the bit (0-28) in the block version field used to signal this softfork (only for \"started\" status)"}, + {RPCResult::Type::NUM, "bit", /* optional */ true, "the bit (0-28) in the block version field used to signal this softfork (only for \"started\" and \"locked_in\" status)"}, {RPCResult::Type::NUM_TIME, "start_time", "the minimum median time past of a block at which the bit gains its meaning"}, {RPCResult::Type::NUM_TIME, "timeout", "the median time past of a block at which the deployment is considered failed if not yet locked in"}, {RPCResult::Type::NUM, "since", "height of the first block to which the status applies"}, {RPCResult::Type::NUM, "min_activation_height", "minimum height of blocks for which the rules may be enforced"}, - {RPCResult::Type::OBJ, "statistics", "numeric statistics about BIP9 signalling for a softfork (only for \"started\" status)", + {RPCResult::Type::OBJ, "statistics", /* optional */ true, "numeric statistics about signalling for a softfork (only for \"started\" and \"locked_in\" status)", { - {RPCResult::Type::NUM, "period", "the length in blocks of the BIP9 signalling period"}, - {RPCResult::Type::NUM, "threshold", "the number of blocks with the version bit set required to activate the feature"}, + {RPCResult::Type::NUM, "period", "the length in blocks of the signalling period"}, + {RPCResult::Type::NUM, "threshold", /* optional */ true, "the number of blocks with the version bit set required to activate the feature (only for \"started\" status)"}, {RPCResult::Type::NUM, "elapsed", "the number of blocks elapsed since the beginning of the current period"}, {RPCResult::Type::NUM, "count", "the number of blocks with the version bit set in the current period"}, - {RPCResult::Type::BOOL, "possible", "returns false if there are not enough blocks left in this period to pass activation threshold"}, + {RPCResult::Type::BOOL, "possible", /* optional */ true, "returns false if there are not enough blocks left in this period to pass activation threshold (only for \"started\" status)"}, }}, }}, - {RPCResult::Type::NUM, "height", "height of the first block which the rules are or will be enforced (only for \"buried\" type, or \"bip9\" type with \"active\" status)"}, + {RPCResult::Type::NUM, "height", /* optional */ true, "height of the first block which the rules are or will be enforced (only for \"buried\" type, or \"bip9\" type with \"active\" status)"}, {RPCResult::Type::BOOL, "active", "true if the rules are enforced for the mempool and the next block"}, }}, }}, @@ -1477,6 +1483,7 @@ RPCHelpMan getblockchaininfo() obj.pushKV("headers", pindexBestHeader ? pindexBestHeader->nHeight : -1); obj.pushKV("bestblockhash", tip->GetBlockHash().GetHex()); obj.pushKV("difficulty", (double)GetDifficulty(tip)); + obj.pushKV("time", (int64_t)tip->nTime); obj.pushKV("mediantime", (int64_t)tip->GetMedianTimePast()); obj.pushKV("verificationprogress", GuessVerificationProgress(Params().TxData(), tip)); obj.pushKV("initialblockdownload", active_chainstate.IsInitialBlockDownload()); @@ -1493,7 +1500,7 @@ RPCHelpMan getblockchaininfo() obj.pushKV("pruneheight", block->nHeight); // if 0, execution bypasses the whole if block. - bool automatic_pruning = (gArgs.GetArg("-prune", 0) != 1); + bool automatic_pruning = (gArgs.GetIntArg("-prune", 0) != 1); obj.pushKV("automatic_pruning", automatic_pruning); if (automatic_pruning) { obj.pushKV("prune_target_size", nPruneTarget); @@ -1502,14 +1509,14 @@ RPCHelpMan getblockchaininfo() const Consensus::Params& consensusParams = Params().GetConsensus(); UniValue softforks(UniValue::VOBJ); - BuriedForkDescPushBack(softforks, "bip34", consensusParams.BIP34Height, height); - BuriedForkDescPushBack(softforks, "bip66", consensusParams.BIP66Height, height); - BuriedForkDescPushBack(softforks, "bip65", consensusParams.BIP65Height, height); - BuriedForkDescPushBack(softforks, "csv", consensusParams.CSVHeight, height); - BuriedForkDescPushBack(softforks, "segwit", consensusParams.SegwitHeight, height); - BIP9SoftForkDescPushBack(tip, softforks, "testdummy", consensusParams, Consensus::DEPLOYMENT_TESTDUMMY); - BIP9SoftForkDescPushBack(tip, softforks, "taproot", consensusParams, Consensus::DEPLOYMENT_TAPROOT); - obj.pushKV("softforks", softforks); + SoftForkDescPushBack(tip, softforks, consensusParams, Consensus::DEPLOYMENT_HEIGHTINCB); + SoftForkDescPushBack(tip, softforks, consensusParams, Consensus::DEPLOYMENT_DERSIG); + SoftForkDescPushBack(tip, softforks, consensusParams, Consensus::DEPLOYMENT_CLTV); + SoftForkDescPushBack(tip, softforks, consensusParams, Consensus::DEPLOYMENT_CSV); + SoftForkDescPushBack(tip, softforks, consensusParams, Consensus::DEPLOYMENT_SEGWIT); + SoftForkDescPushBack(tip, softforks, consensusParams, Consensus::DEPLOYMENT_TESTDUMMY); + SoftForkDescPushBack(tip, softforks, consensusParams, Consensus::DEPLOYMENT_TAPROOT); + obj.pushKV("softforks", softforks); obj.pushKV("warnings", GetWarnings(false).original); return obj; @@ -1640,7 +1647,7 @@ UniValue MempoolInfoToJSON(const CTxMemPool& pool) ret.pushKV("bytes", (int64_t)pool.GetTotalTxSize()); ret.pushKV("usage", (int64_t)pool.DynamicMemoryUsage()); ret.pushKV("total_fee", ValueFromAmount(pool.GetTotalFee())); - size_t maxmempool = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; + size_t maxmempool = gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; ret.pushKV("maxmempool", (int64_t) maxmempool); ret.pushKV("mempoolminfee", ValueFromAmount(std::max(pool.GetMinFee(maxmempool), ::minRelayTxFee).GetFeePerK())); ret.pushKV("minrelaytxfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); @@ -1706,7 +1713,7 @@ static RPCHelpMan preciousblock() } BlockValidationState state; - chainman.ActiveChainstate().PreciousBlock(state, Params(), pblockindex); + chainman.ActiveChainstate().PreciousBlock(state, pblockindex); if (!state.IsValid()) { throw JSONRPCError(RPC_DATABASE_ERROR, state.ToString()); @@ -1743,10 +1750,10 @@ static RPCHelpMan invalidateblock() throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } } - chainman.ActiveChainstate().InvalidateBlock(state, Params(), pblockindex); + chainman.ActiveChainstate().InvalidateBlock(state, pblockindex); if (state.IsValid()) { - chainman.ActiveChainstate().ActivateBestChain(state, Params()); + chainman.ActiveChainstate().ActivateBestChain(state); } if (!state.IsValid()) { @@ -1787,7 +1794,7 @@ static RPCHelpMan reconsiderblock() } BlockValidationState state; - chainman.ActiveChainstate().ActivateBestChain(state, Params()); + chainman.ActiveChainstate().ActivateBestChain(state); if (!state.IsValid()) { throw JSONRPCError(RPC_DATABASE_ERROR, state.ToString()); @@ -1923,16 +1930,6 @@ void CalculatePercentilesByWeight(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], } } -void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex) -{ - ScriptPubKeyToUniv(scriptPubKey, out, fIncludeHex, IsDeprecatedRPCEnabled("addresses")); -} - -void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex, int serialize_flags, const CTxUndo* txundo) -{ - TxToUniv(tx, hashBlock, IsDeprecatedRPCEnabled("addresses"), entry, include_hex, serialize_flags, txundo); -} - template<typename T> static inline bool SetHasKeys(const std::set<T>& set) {return false;} template<typename T, typename Tk, typename... Args> @@ -1961,11 +1958,11 @@ static RPCHelpMan getblockstats() RPCResult{ RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::NUM, "avgfee", "Average fee in the block"}, - {RPCResult::Type::NUM, "avgfeerate", "Average feerate (in satoshis per virtual byte)"}, - {RPCResult::Type::NUM, "avgtxsize", "Average transaction size"}, - {RPCResult::Type::STR_HEX, "blockhash", "The block hash (to check for potential reorgs)"}, - {RPCResult::Type::ARR_FIXED, "feerate_percentiles", "Feerates at the 10th, 25th, 50th, 75th, and 90th percentile weight unit (in satoshis per virtual byte)", + {RPCResult::Type::NUM, "avgfee", /* optional */ true, "Average fee in the block"}, + {RPCResult::Type::NUM, "avgfeerate", /* optional */ true, "Average feerate (in satoshis per virtual byte)"}, + {RPCResult::Type::NUM, "avgtxsize", /* optional */ true, "Average transaction size"}, + {RPCResult::Type::STR_HEX, "blockhash", /* optional */ true, "The block hash (to check for potential reorgs)"}, + {RPCResult::Type::ARR_FIXED, "feerate_percentiles", /* optional */ true, "Feerates at the 10th, 25th, 50th, 75th, and 90th percentile weight unit (in satoshis per virtual byte)", { {RPCResult::Type::NUM, "10th_percentile_feerate", "The 10th percentile feerate"}, {RPCResult::Type::NUM, "25th_percentile_feerate", "The 25th percentile feerate"}, @@ -1973,30 +1970,30 @@ static RPCHelpMan getblockstats() {RPCResult::Type::NUM, "75th_percentile_feerate", "The 75th percentile feerate"}, {RPCResult::Type::NUM, "90th_percentile_feerate", "The 90th percentile feerate"}, }}, - {RPCResult::Type::NUM, "height", "The height of the block"}, - {RPCResult::Type::NUM, "ins", "The number of inputs (excluding coinbase)"}, - {RPCResult::Type::NUM, "maxfee", "Maximum fee in the block"}, - {RPCResult::Type::NUM, "maxfeerate", "Maximum feerate (in satoshis per virtual byte)"}, - {RPCResult::Type::NUM, "maxtxsize", "Maximum transaction size"}, - {RPCResult::Type::NUM, "medianfee", "Truncated median fee in the block"}, - {RPCResult::Type::NUM, "mediantime", "The block median time past"}, - {RPCResult::Type::NUM, "mediantxsize", "Truncated median transaction size"}, - {RPCResult::Type::NUM, "minfee", "Minimum fee in the block"}, - {RPCResult::Type::NUM, "minfeerate", "Minimum feerate (in satoshis per virtual byte)"}, - {RPCResult::Type::NUM, "mintxsize", "Minimum transaction size"}, - {RPCResult::Type::NUM, "outs", "The number of outputs"}, - {RPCResult::Type::NUM, "subsidy", "The block subsidy"}, - {RPCResult::Type::NUM, "swtotal_size", "Total size of all segwit transactions"}, - {RPCResult::Type::NUM, "swtotal_weight", "Total weight of all segwit transactions"}, - {RPCResult::Type::NUM, "swtxs", "The number of segwit transactions"}, - {RPCResult::Type::NUM, "time", "The block time"}, - {RPCResult::Type::NUM, "total_out", "Total amount in all outputs (excluding coinbase and thus reward [ie subsidy + totalfee])"}, - {RPCResult::Type::NUM, "total_size", "Total size of all non-coinbase transactions"}, - {RPCResult::Type::NUM, "total_weight", "Total weight of all non-coinbase transactions"}, - {RPCResult::Type::NUM, "totalfee", "The fee total"}, - {RPCResult::Type::NUM, "txs", "The number of transactions (including coinbase)"}, - {RPCResult::Type::NUM, "utxo_increase", "The increase/decrease in the number of unspent outputs"}, - {RPCResult::Type::NUM, "utxo_size_inc", "The increase/decrease in size for the utxo index (not discounting op_return and similar)"}, + {RPCResult::Type::NUM, "height", /* optional */ true, "The height of the block"}, + {RPCResult::Type::NUM, "ins", /* optional */ true, "The number of inputs (excluding coinbase)"}, + {RPCResult::Type::NUM, "maxfee", /* optional */ true, "Maximum fee in the block"}, + {RPCResult::Type::NUM, "maxfeerate", /* optional */ true, "Maximum feerate (in satoshis per virtual byte)"}, + {RPCResult::Type::NUM, "maxtxsize", /* optional */ true, "Maximum transaction size"}, + {RPCResult::Type::NUM, "medianfee", /* optional */ true, "Truncated median fee in the block"}, + {RPCResult::Type::NUM, "mediantime", /* optional */ true, "The block median time past"}, + {RPCResult::Type::NUM, "mediantxsize", /* optional */ true, "Truncated median transaction size"}, + {RPCResult::Type::NUM, "minfee", /* optional */ true, "Minimum fee in the block"}, + {RPCResult::Type::NUM, "minfeerate", /* optional */ true, "Minimum feerate (in satoshis per virtual byte)"}, + {RPCResult::Type::NUM, "mintxsize", /* optional */ true, "Minimum transaction size"}, + {RPCResult::Type::NUM, "outs", /* optional */ true, "The number of outputs"}, + {RPCResult::Type::NUM, "subsidy", /* optional */ true, "The block subsidy"}, + {RPCResult::Type::NUM, "swtotal_size", /* optional */ true, "Total size of all segwit transactions"}, + {RPCResult::Type::NUM, "swtotal_weight", /* optional */ true, "Total weight of all segwit transactions"}, + {RPCResult::Type::NUM, "swtxs", /* optional */ true, "The number of segwit transactions"}, + {RPCResult::Type::NUM, "time", /* optional */ true, "The block time"}, + {RPCResult::Type::NUM, "total_out", /* optional */ true, "Total amount in all outputs (excluding coinbase and thus reward [ie subsidy + totalfee])"}, + {RPCResult::Type::NUM, "total_size", /* optional */ true, "Total size of all non-coinbase transactions"}, + {RPCResult::Type::NUM, "total_weight", /* optional */ true, "Total weight of all non-coinbase transactions"}, + {RPCResult::Type::NUM, "totalfee", /* optional */ true, "The fee total"}, + {RPCResult::Type::NUM, "txs", /* optional */ true, "The number of transactions (including coinbase)"}, + {RPCResult::Type::NUM, "utxo_increase", /* optional */ true, "The increase/decrease in the number of unspent outputs"}, + {RPCResult::Type::NUM, "utxo_size_inc", /* optional */ true, "The increase/decrease in size for the utxo index (not discounting op_return and similar)"}, }}, RPCExamples{ HelpExampleCli("getblockstats", R"('"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09"' '["minfeerate","avgfeerate"]')") + diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index ffb6f03b47..09e471afdd 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -5,8 +5,7 @@ #ifndef BITCOIN_RPC_BLOCKCHAIN_H #define BITCOIN_RPC_BLOCKCHAIN_H -#include <amount.h> -#include <core_io.h> +#include <consensus/amount.h> #include <streams.h> #include <sync.h> @@ -53,9 +52,6 @@ UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex /** Used by getblockstats to get feerates at different percentiles by weight */ void CalculatePercentilesByWeight(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], std::vector<std::pair<CAmount, int64_t>>& scores, int64_t total_weight); -void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); -void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex = true, int serialize_flags = 0, const CTxUndo* txundo = nullptr); - NodeContext& EnsureAnyNodeContext(const std::any& context); CTxMemPool& EnsureMemPool(const NodeContext& node); CTxMemPool& EnsureAnyMemPool(const std::any& context); diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 9c8582c7a3..93e49cb9a8 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -131,6 +131,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "gettxoutsetinfo", 2, "use_index"}, { "lockunspent", 0, "unlock" }, { "lockunspent", 1, "transactions" }, + { "lockunspent", 2, "persistent" }, { "send", 0, "outputs" }, { "send", 1, "conf_target" }, { "send", 3, "fee_rate"}, @@ -142,6 +143,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "importmulti", 0, "requests" }, { "importmulti", 1, "options" }, { "importdescriptors", 0, "requests" }, + { "listdescriptors", 0, "private" }, { "verifychain", 0, "checklevel" }, { "verifychain", 1, "nblocks" }, { "getblockstats", 0, "hash_or_height" }, @@ -186,10 +188,12 @@ static const CRPCConvertParam vRPCConvertParams[] = { "createwallet", 5, "descriptors"}, { "createwallet", 6, "load_on_startup"}, { "createwallet", 7, "external_signer"}, + { "restorewallet", 2, "load_on_startup"}, { "loadwallet", 1, "load_on_startup"}, { "unloadwallet", 1, "load_on_startup"}, { "getnodeaddresses", 0, "count"}, { "addpeeraddress", 1, "port"}, + { "addpeeraddress", 2, "tried"}, { "stop", 0, "wait" }, }; // clang-format on diff --git a/src/rpc/external_signer.cpp b/src/rpc/external_signer.cpp index 6ec2b1a07f..60ec15e904 100644 --- a/src/rpc/external_signer.cpp +++ b/src/rpc/external_signer.cpp @@ -24,8 +24,11 @@ static RPCHelpMan enumeratesigners() { {RPCResult::Type::ARR, "signers", /* optional */ false, "", { - {RPCResult::Type::STR_HEX, "masterkeyfingerprint", "Master key fingerprint"}, - {RPCResult::Type::STR, "name", "Device name"}, + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR_HEX, "fingerprint", "Master key fingerprint"}, + {RPCResult::Type::STR, "name", "Device name"}, + }}, }, } } diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 327f961196..518c41d12a 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -3,13 +3,15 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> #include <chain.h> #include <chainparams.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/params.h> #include <consensus/validation.h> #include <core_io.h> +#include <deploymentinfo.h> +#include <deploymentstatus.h> #include <key_io.h> #include <miner.h> #include <net.h> @@ -34,7 +36,6 @@ #include <util/translation.h> #include <validation.h> #include <validationinterface.h> -#include <versionbitsinfo.h> #include <warnings.h> #include <memory> @@ -552,6 +553,10 @@ static RPCHelpMan getblocktemplate() { {RPCResult::Type::NUM, "rulename", "identifies the bit number as indicating acceptance and readiness for the named softfork rule"}, }}, + {RPCResult::Type::ARR, "capabilities", "", + { + {RPCResult::Type::STR, "value", "A supported feature, for example 'proposal'"}, + }}, {RPCResult::Type::NUM, "vbrequired", "bit mask of versionbits the server requires set in submissions"}, {RPCResult::Type::STR, "previousblockhash", "The hash of current highest block"}, {RPCResult::Type::ARR, "transactions", "contents of non-coinbase transactions that should be included in the next block", @@ -585,11 +590,12 @@ static RPCHelpMan getblocktemplate() {RPCResult::Type::STR_HEX, "noncerange", "A range of valid nonces"}, {RPCResult::Type::NUM, "sigoplimit", "limit of sigops in blocks"}, {RPCResult::Type::NUM, "sizelimit", "limit of block size"}, - {RPCResult::Type::NUM, "weightlimit", "limit of block weight"}, + {RPCResult::Type::NUM, "weightlimit", /* optional */ true, "limit of block weight"}, {RPCResult::Type::NUM_TIME, "curtime", "current timestamp in " + UNIX_EPOCH_TIME}, {RPCResult::Type::STR, "bits", "compressed target of next block"}, {RPCResult::Type::NUM, "height", "The height of the next block"}, - {RPCResult::Type::STR, "default_witness_commitment", /* optional */ true, "a valid witness commitment for the unmodified block template"}, + {RPCResult::Type::STR_HEX, "signet_challenge", /* optional */ true, "Only on signet"}, + {RPCResult::Type::STR_HEX, "default_witness_commitment", /* optional */ true, "a valid witness commitment for the unmodified block template"}, }}, }, RPCExamples{ @@ -696,7 +702,7 @@ static RPCHelpMan getblocktemplate() std::string lpstr = lpval.get_str(); hashWatchedChain = ParseHashV(lpstr.substr(0, 64), "longpollid"); - nTransactionsUpdatedLastLP = atoi64(lpstr.substr(64)); + nTransactionsUpdatedLastLP = LocaleIndependentAtoi<int64_t>(lpstr.substr(64)); } else { @@ -774,7 +780,7 @@ static RPCHelpMan getblocktemplate() pblock->nNonce = 0; // NOTE: If at some point we support pre-segwit miners post-segwit-activation, this needs to take segwit support into consideration - const bool fPreSegWit = (pindexPrev->nHeight + 1 < consensusParams.SegwitHeight); + const bool fPreSegWit = !DeploymentActiveAfter(pindexPrev, consensusParams, Consensus::DEPLOYMENT_SEGWIT); UniValue aCaps(UniValue::VARR); aCaps.push_back("proposal"); @@ -840,7 +846,7 @@ static RPCHelpMan getblocktemplate() UniValue vbavailable(UniValue::VOBJ); for (int j = 0; j < (int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; ++j) { Consensus::DeploymentPos pos = Consensus::DeploymentPos(j); - ThresholdState state = VersionBitsState(pindexPrev, consensusParams, pos, versionbitscache); + ThresholdState state = g_versionbitscache.State(pindexPrev, consensusParams, pos); switch (state) { case ThresholdState::DEFINED: case ThresholdState::FAILED: @@ -848,8 +854,8 @@ static RPCHelpMan getblocktemplate() break; case ThresholdState::LOCKED_IN: // Ensure bit is set in block version - pblock->nVersion |= VersionBitsMask(consensusParams, pos); - // FALL THROUGH to get vbavailable set... + pblock->nVersion |= g_versionbitscache.Mask(consensusParams, pos); + [[fallthrough]]; case ThresholdState::STARTED: { const struct VBDeploymentInfo& vbinfo = VersionBitsDeploymentInfo[pos]; @@ -857,7 +863,7 @@ static RPCHelpMan getblocktemplate() if (setClientRules.find(vbinfo.name) == setClientRules.end()) { if (!vbinfo.gbt_force) { // If the client doesn't support this, don't indicate it in the [default] version - pblock->nVersion &= ~VersionBitsMask(consensusParams, pos); + pblock->nVersion &= ~g_versionbitscache.Mask(consensusParams, pos); } } break; @@ -1088,7 +1094,8 @@ static RPCHelpMan estimatesmartfee() "have been observed to make an estimate for any number of blocks."}, }}, RPCExamples{ - HelpExampleCli("estimatesmartfee", "6") + HelpExampleCli("estimatesmartfee", "6") + + HelpExampleRpc("estimatesmartfee", "6") }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { @@ -1096,6 +1103,8 @@ static RPCHelpMan estimatesmartfee() RPCTypeCheckArgument(request.params[0], UniValue::VNUM); CBlockPolicyEstimator& fee_estimator = EnsureAnyFeeEstimator(request.context); + const NodeContext& node = EnsureAnyNodeContext(request.context); + const CTxMemPool& mempool = EnsureMemPool(node); unsigned int max_target = fee_estimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); unsigned int conf_target = ParseConfirmTarget(request.params[0], max_target); @@ -1111,7 +1120,10 @@ static RPCHelpMan estimatesmartfee() UniValue result(UniValue::VOBJ); UniValue errors(UniValue::VARR); FeeCalculation feeCalc; - CFeeRate feeRate = fee_estimator.estimateSmartFee(conf_target, &feeCalc, conservative); + CFeeRate feeRate{fee_estimator.estimateSmartFee(conf_target, &feeCalc, conservative)}; + CFeeRate min_mempool_feerate{mempool.GetMinFee(gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000)}; + CFeeRate min_relay_feerate{::minRelayTxFee}; + feeRate = std::max({feeRate, min_mempool_feerate, min_relay_feerate}); if (feeRate != CFeeRate(0)) { result.pushKV("feerate", ValueFromAmount(feeRate.GetFeePerK())); } else { diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 5178ce60e8..39bd9c6091 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -22,8 +22,10 @@ #include <util/check.h> #include <util/message.h> // For MessageSign(), MessageVerify() #include <util/strencodings.h> +#include <util/syscall_sandbox.h> #include <util/system.h> +#include <optional> #include <stdint.h> #include <tuple> #ifdef HAVE_MALLOC_INFO @@ -43,10 +45,10 @@ static RPCHelpMan validateaddress() RPCResult::Type::OBJ, "", "", { {RPCResult::Type::BOOL, "isvalid", "If the address is valid or not"}, - {RPCResult::Type::STR, "address", "The bitcoin address validated"}, - {RPCResult::Type::STR_HEX, "scriptPubKey", "The hex-encoded scriptPubKey generated by the address"}, - {RPCResult::Type::BOOL, "isscript", "If the key is a script"}, - {RPCResult::Type::BOOL, "iswitness", "If the address is a witness address"}, + {RPCResult::Type::STR, "address", /* optional */ true, "The bitcoin address validated"}, + {RPCResult::Type::STR_HEX, "scriptPubKey", /* optional */ true, "The hex-encoded scriptPubKey generated by the address"}, + {RPCResult::Type::BOOL, "isscript", /* optional */ true, "If the key is a script"}, + {RPCResult::Type::BOOL, "iswitness", /* optional */ true, "If the address is a witness address"}, {RPCResult::Type::NUM, "witness_version", /* optional */ true, "The version number of the witness program"}, {RPCResult::Type::STR_HEX, "witness_program", /* optional */ true, "The hex value of the witness program"}, {RPCResult::Type::STR, "error", /* optional */ true, "Error message, if any"}, @@ -108,7 +110,7 @@ static RPCHelpMan createmultisig() "\nCreate a multisig address from 2 public keys\n" + HelpExampleCli("createmultisig", "2 \"[\\\"03789ed0bb717d88f7d321a368d905e7430207ebbd82bd342cf11ae157a7ace5fd\\\",\\\"03dbc6764b8884a92e871274b87583e6d5c2a58819473e17e107ef3f6aa5a61626\\\"]\"") + "\nAs a JSON-RPC call\n" - + HelpExampleRpc("createmultisig", "2, \"[\\\"03789ed0bb717d88f7d321a368d905e7430207ebbd82bd342cf11ae157a7ace5fd\\\",\\\"03dbc6764b8884a92e871274b87583e6d5c2a58819473e17e107ef3f6aa5a61626\\\"]\"") + + HelpExampleRpc("createmultisig", "2, [\"03789ed0bb717d88f7d321a368d905e7430207ebbd82bd342cf11ae157a7ace5fd\",\"03dbc6764b8884a92e871274b87583e6d5c2a58819473e17e107ef3f6aa5a61626\"]") }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { @@ -128,12 +130,13 @@ static RPCHelpMan createmultisig() // Get the output type OutputType output_type = OutputType::LEGACY; if (!request.params[2].isNull()) { - if (!ParseOutputType(request.params[2].get_str(), output_type)) { + std::optional<OutputType> parsed = ParseOutputType(request.params[2].get_str()); + if (!parsed) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Unknown address type '%s'", request.params[2].get_str())); - } - if (output_type == OutputType::BECH32M) { + } else if (parsed.value() == OutputType::BECH32M) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "createmultisig cannot create bech32m multisig addresses"); } + output_type = parsed.value(); } // Construct using pay-to-script-hash: @@ -156,6 +159,8 @@ static RPCHelpMan createmultisig() static RPCHelpMan getdescriptorinfo() { + const std::string EXAMPLE_DESCRIPTOR = "wpkh([d34db33f/84h/0h/0h]0279be667ef9dcbbac55a06295Ce870b07029Bfcdb2dce28d959f2815b16f81798)"; + return RPCHelpMan{"getdescriptorinfo", {"\nAnalyses a descriptor.\n"}, { @@ -173,7 +178,8 @@ static RPCHelpMan getdescriptorinfo() }, RPCExamples{ "Analyse a descriptor\n" + - HelpExampleCli("getdescriptorinfo", "\"wpkh([d34db33f/84h/0h/0h]0279be667ef9dcbbac55a06295Ce870b07029Bfcdb2dce28d959f2815b16f81798)\"") + HelpExampleCli("getdescriptorinfo", "\"" + EXAMPLE_DESCRIPTOR + "\"") + + HelpExampleRpc("getdescriptorinfo", "\"" + EXAMPLE_DESCRIPTOR + "\"") }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { @@ -199,6 +205,8 @@ static RPCHelpMan getdescriptorinfo() static RPCHelpMan deriveaddresses() { + const std::string EXAMPLE_DESCRIPTOR = "wpkh([d34db33f/84h/0h/0h]xpub6DJ2dNUysrn5Vt36jH2KLBT2i1auw1tTSSomg8PhqNiUtx8QX2SvC9nrHu81fT41fvDUnhMjEzQgXnQjKEu3oaqMSzhSrHMxyyoEAmUHQbY/0/*)#cjjspncu"; + return RPCHelpMan{"deriveaddresses", {"\nDerives one or more addresses corresponding to an output descriptor.\n" "Examples of output descriptors are:\n" @@ -221,7 +229,8 @@ static RPCHelpMan deriveaddresses() }, RPCExamples{ "First three native segwit receive addresses\n" + - HelpExampleCli("deriveaddresses", "\"wpkh([d34db33f/84h/0h/0h]xpub6DJ2dNUysrn5Vt36jH2KLBT2i1auw1tTSSomg8PhqNiUtx8QX2SvC9nrHu81fT41fvDUnhMjEzQgXnQjKEu3oaqMSzhSrHMxyyoEAmUHQbY/0/*)#cjjspncu\" \"[0,2]\"") + HelpExampleCli("deriveaddresses", "\"" + EXAMPLE_DESCRIPTOR + "\" \"[0,2]\"") + + HelpExampleRpc("deriveaddresses", "\"" + EXAMPLE_DESCRIPTOR + "\", \"[0,2]\"") }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { @@ -409,6 +418,27 @@ static RPCHelpMan setmocktime() }; } +#if defined(USE_SYSCALL_SANDBOX) +static RPCHelpMan invokedisallowedsyscall() +{ + return RPCHelpMan{ + "invokedisallowedsyscall", + "\nInvoke a disallowed syscall to trigger a syscall sandbox violation. Used for testing purposes.\n", + {}, + RPCResult{RPCResult::Type::NONE, "", ""}, + RPCExamples{ + HelpExampleCli("invokedisallowedsyscall", "") + HelpExampleRpc("invokedisallowedsyscall", "")}, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + if (!Params().IsTestChain()) { + throw std::runtime_error("invokedisallowedsyscall is used for testing only."); + } + TestDisallowedSandboxCall(); + return NullUniValue; + }, + }; +} +#endif // USE_SYSCALL_SANDBOX + static RPCHelpMan mockscheduler() { return RPCHelpMan{"mockscheduler", @@ -662,8 +692,9 @@ static RPCHelpMan echoipc() RPCExamples{HelpExampleCli("echo", "\"Hello world\"") + HelpExampleRpc("echo", "\"Hello world\"")}, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + interfaces::Init& local_init = *EnsureAnyNodeContext(request.context).init; std::unique_ptr<interfaces::Echo> echo; - if (interfaces::Ipc* ipc = Assert(EnsureAnyNodeContext(request.context).init)->ipc()) { + if (interfaces::Ipc* ipc = local_init.ipc()) { // Spawn a new bitcoin-node process and call makeEcho to get a // client pointer to a interfaces::Echo instance running in // that process. This is just for testing. A slightly more @@ -681,7 +712,7 @@ static RPCHelpMan echoipc() // interfaces::Echo object and return it so the `echoipc` RPC // method will work, and the python test calling `echoipc` // can expect the same result. - echo = interfaces::MakeEcho(); + echo = local_init.makeEcho(); } return echo->echo(request.params[0].get_str()); }, @@ -708,7 +739,7 @@ static RPCHelpMan getindexinfo() {"index_name", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "Filter results for an index with a specific name."}, }, RPCResult{ - RPCResult::Type::OBJ, "", "", { + RPCResult::Type::OBJ_DYN, "", "", { { RPCResult::Type::OBJ, "name", "The name of the index", { @@ -768,6 +799,9 @@ static const CRPCCommand commands[] = { "hidden", &echo, }, { "hidden", &echojson, }, { "hidden", &echoipc, }, +#if defined(USE_SYSCALL_SANDBOX) + { "hidden", &invokedisallowedsyscall, }, +#endif // USE_SYSCALL_SANDBOX }; // clang-format on for (const auto& c : commands) { diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 3013c76825..a9bee33c5d 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -116,10 +116,10 @@ static RPCHelpMan getpeerinfo() { {RPCResult::Type::NUM, "id", "Peer index"}, {RPCResult::Type::STR, "addr", "(host:port) The IP address and port of the peer"}, - {RPCResult::Type::STR, "addrbind", "(ip:port) Bind address of the connection to the peer"}, - {RPCResult::Type::STR, "addrlocal", "(ip:port) Local address as reported by the peer"}, + {RPCResult::Type::STR, "addrbind", /* optional */ true, "(ip:port) Bind address of the connection to the peer"}, + {RPCResult::Type::STR, "addrlocal", /* optional */ true, "(ip:port) Local address as reported by the peer"}, {RPCResult::Type::STR, "network", "Network (" + Join(GetNetworkNames(/* append_unroutable */ true), ", ") + ")"}, - {RPCResult::Type::NUM, "mapped_as", "The AS in the BGP route to the peer used for diversifying\n" + {RPCResult::Type::NUM, "mapped_as", /* optional */ true, "The AS in the BGP route to the peer used for diversifying\n" "peer selection (only available if the asmap config flag is set)"}, {RPCResult::Type::STR_HEX, "services", "The services offered"}, {RPCResult::Type::ARR, "servicesnames", "the services offered, in human-readable form", @@ -135,9 +135,9 @@ static RPCHelpMan getpeerinfo() {RPCResult::Type::NUM, "bytesrecv", "The total bytes received"}, {RPCResult::Type::NUM_TIME, "conntime", "The " + UNIX_EPOCH_TIME + " of the connection"}, {RPCResult::Type::NUM, "timeoffset", "The time offset in seconds"}, - {RPCResult::Type::NUM, "pingtime", "ping time (if available)"}, - {RPCResult::Type::NUM, "minping", "minimum observed ping time (if any at all)"}, - {RPCResult::Type::NUM, "pingwait", "ping wait (if non-zero)"}, + {RPCResult::Type::NUM, "pingtime", /* optional */ true, "ping time (if available)"}, + {RPCResult::Type::NUM, "minping", /* optional */ true, "minimum observed ping time (if any at all)"}, + {RPCResult::Type::NUM, "pingwait", /* optional */ true, "ping wait (if non-zero)"}, {RPCResult::Type::NUM, "version", "The peer version, such as 70001"}, {RPCResult::Type::STR, "subver", "The string version"}, {RPCResult::Type::BOOL, "inbound", "Inbound (true) or Outbound (false)"}, @@ -150,6 +150,9 @@ static RPCHelpMan getpeerinfo() { {RPCResult::Type::NUM, "n", "The heights of blocks we're currently asking from this peer"}, }}, + {RPCResult::Type::BOOL, "addr_relay_enabled", "Whether we participate in address relay with this peer"}, + {RPCResult::Type::NUM, "addr_processed", "The total number of addresses processed, excluding those dropped due to rate limiting"}, + {RPCResult::Type::NUM, "addr_rate_limited", "The total number of addresses dropped due to rate limiting"}, {RPCResult::Type::ARR, "permissions", "Any special permissions that have been granted to this peer", { {RPCResult::Type::STR, "permission_type", Join(NET_PERMISSIONS_DOC, ",\n") + ".\n"}, @@ -194,7 +197,7 @@ static RPCHelpMan getpeerinfo() CNodeStateStats statestats; bool fStateStats = peerman.GetNodeStateStats(stats.nodeid, statestats); obj.pushKV("id", stats.nodeid); - obj.pushKV("addr", stats.addrName); + obj.pushKV("addr", stats.m_addr_name); if (stats.addrBind.IsValid()) { obj.pushKV("addrbind", stats.addrBind.ToString()); } @@ -242,6 +245,9 @@ static RPCHelpMan getpeerinfo() heights.push_back(height); } obj.pushKV("inflight", heights); + obj.pushKV("addr_relay_enabled", statestats.m_addr_relay_enabled); + obj.pushKV("addr_processed", statestats.m_addr_processed); + obj.pushKV("addr_rate_limited", statestats.m_addr_rate_limited); } UniValue permissions(UniValue::VARR); for (const auto& permission : NetPermissions::ToStrings(stats.m_permissionFlags)) { @@ -337,7 +343,7 @@ static RPCHelpMan addconnection() "\nOpen an outbound connection to a specified node. This RPC is for testing only.\n", { {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The IP address and port to attempt connecting to."}, - {"connection_type", RPCArg::Type::STR, RPCArg::Optional::NO, "Type of connection to open, either \"outbound-full-relay\" or \"block-relay-only\"."}, + {"connection_type", RPCArg::Type::STR, RPCArg::Optional::NO, "Type of connection to open (\"outbound-full-relay\", \"block-relay-only\" or \"addr-fetch\")."}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -363,6 +369,8 @@ static RPCHelpMan addconnection() conn_type = ConnectionType::OUTBOUND_FULL_RELAY; } else if (conn_type_in == "block-relay-only") { conn_type = ConnectionType::BLOCK_RELAY; + } else if (conn_type_in == "addr-fetch") { + conn_type = ConnectionType::ADDR_FETCH; } else { throw JSONRPCError(RPC_INVALID_PARAMETER, self.ToString()); } @@ -913,6 +921,7 @@ static RPCHelpMan addpeeraddress() { {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The IP address of the peer"}, {"port", RPCArg::Type::NUM, RPCArg::Optional::NO, "The port of the peer"}, + {"tried", RPCArg::Type::BOOL, RPCArg::Default{false}, "If true, attempt to add the peer to the tried addresses table"}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -921,8 +930,8 @@ static RPCHelpMan addpeeraddress() }, }, RPCExamples{ - HelpExampleCli("addpeeraddress", "\"1.2.3.4\" 8333") - + HelpExampleRpc("addpeeraddress", "\"1.2.3.4\", 8333") + HelpExampleCli("addpeeraddress", "\"1.2.3.4\" 8333 true") + + HelpExampleRpc("addpeeraddress", "\"1.2.3.4\", 8333, true") }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { @@ -933,6 +942,7 @@ static RPCHelpMan addpeeraddress() const std::string& addr_string{request.params[0].get_str()}; const uint16_t port{static_cast<uint16_t>(request.params[1].get_int())}; + const bool tried{request.params[2].isTrue()}; UniValue obj(UniValue::VOBJ); CNetAddr net_addr; @@ -943,7 +953,13 @@ static RPCHelpMan addpeeraddress() address.nTime = GetAdjustedTime(); // The source address is set equal to the address. This is equivalent to the peer // announcing itself. - if (node.addrman->Add(address, address)) success = true; + if (node.addrman->Add({address}, address)) { + success = true; + if (tried) { + // Attempt to move the address to the tried addresses table. + node.addrman->Good(address); + } + } } obj.pushKV("success", success); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index ccb3123714..483717aa7a 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -5,6 +5,7 @@ #include <chain.h> #include <coins.h> +#include <consensus/amount.h> #include <consensus/validation.h> #include <core_io.h> #include <index/txindex.h> @@ -74,12 +75,10 @@ static RPCHelpMan getrawtransaction() "getrawtransaction", "\nReturn the raw transaction data.\n" - "\nBy default this function only works for mempool transactions. When called with a blockhash\n" - "argument, getrawtransaction will return the transaction if the specified block is available and\n" - "the transaction is found in that block. When called without a blockhash argument, getrawtransaction\n" - "will return the transaction if it is in the mempool, or if -txindex is enabled and the transaction\n" - "is in a block in the blockchain.\n" - + "\nBy default, this call only returns a transaction if it is in the mempool. If -txindex is enabled\n" + "and no blockhash argument is passed, it will return the transaction if it is in the mempool or any block.\n" + "If a blockhash argument is passed, it will return the transaction if\n" + "the specified block is available and the transaction is in that block.\n" "\nHint: Use gettransaction for wallet transactions.\n" "\nIf verbose is 'true', returns an Object with information about 'txid'.\n" @@ -96,7 +95,7 @@ static RPCHelpMan getrawtransaction() RPCResult{"if verbose is set to true", RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::BOOL, "in_active_chain", "Whether specified block is in the active chain or not (only present with explicit \"blockhash\" argument)"}, + {RPCResult::Type::BOOL, "in_active_chain", /* optional */ true, "Whether specified block is in the active chain or not (only present with explicit \"blockhash\" argument)"}, {RPCResult::Type::STR_HEX, "hex", "The serialized, hex-encoded data for 'txid'"}, {RPCResult::Type::STR_HEX, "txid", "The transaction id (same as provided)"}, {RPCResult::Type::STR_HEX, "hash", "The transaction hash (differs from txid for witness transactions)"}, @@ -117,7 +116,7 @@ static RPCHelpMan getrawtransaction() {RPCResult::Type::STR_HEX, "hex", "hex"}, }}, {RPCResult::Type::NUM, "sequence", "The script sequence number"}, - {RPCResult::Type::ARR, "txinwitness", "", + {RPCResult::Type::ARR, "txinwitness", /* optional */ true, "", { {RPCResult::Type::STR_HEX, "hex", "hex-encoded witness data (if any)"}, }}, @@ -133,20 +132,15 @@ static RPCHelpMan getrawtransaction() { {RPCResult::Type::STR, "asm", "the asm"}, {RPCResult::Type::STR, "hex", "the hex"}, - {RPCResult::Type::NUM, "reqSigs", /* optional */ true, "(DEPRECATED, returned only if config option -deprecatedrpc=addresses is passed) Number of required signatures"}, {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, - {RPCResult::Type::STR, "address", /* optional */ true, "bitcoin address (only if a well-defined address exists)"}, - {RPCResult::Type::ARR, "addresses", /* optional */ true, "(DEPRECATED, returned only if config option -deprecatedrpc=addresses is passed) Array of bitcoin addresses", - { - {RPCResult::Type::STR, "address", "bitcoin address"}, - }}, + {RPCResult::Type::STR, "address", /* optional */ true, "The Bitcoin address (only if a well-defined address exists)"}, }}, }}, }}, - {RPCResult::Type::STR_HEX, "blockhash", "the block hash"}, - {RPCResult::Type::NUM, "confirmations", "The confirmations"}, - {RPCResult::Type::NUM_TIME, "blocktime", "The block time expressed in " + UNIX_EPOCH_TIME}, - {RPCResult::Type::NUM, "time", "Same as \"blocktime\""}, + {RPCResult::Type::STR_HEX, "blockhash", /* optional */ true, "the block hash"}, + {RPCResult::Type::NUM, "confirmations", /* optional */ true, "The confirmations"}, + {RPCResult::Type::NUM_TIME, "blocktime", /* optional */ true, "The block time expressed in " + UNIX_EPOCH_TIME}, + {RPCResult::Type::NUM, "time", /* optional */ true, "Same as \"blocktime\""}, } }, }, @@ -472,14 +466,15 @@ static RPCHelpMan decoderawtransaction() { {RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::STR_HEX, "txid", "The transaction id"}, - {RPCResult::Type::NUM, "vout", "The output number"}, - {RPCResult::Type::OBJ, "scriptSig", "The script", + {RPCResult::Type::STR_HEX, "coinbase", /* optional */ true, ""}, + {RPCResult::Type::STR_HEX, "txid", /* optional */ true, "The transaction id"}, + {RPCResult::Type::NUM, "vout", /* optional */ true, "The output number"}, + {RPCResult::Type::OBJ, "scriptSig", /* optional */ true, "The script", { {RPCResult::Type::STR, "asm", "asm"}, {RPCResult::Type::STR_HEX, "hex", "hex"}, }}, - {RPCResult::Type::ARR, "txinwitness", "", + {RPCResult::Type::ARR, "txinwitness", /* optional */ true, "", { {RPCResult::Type::STR_HEX, "hex", "hex-encoded witness data (if any)"}, }}, @@ -496,13 +491,8 @@ static RPCHelpMan decoderawtransaction() { {RPCResult::Type::STR, "asm", "the asm"}, {RPCResult::Type::STR_HEX, "hex", "the hex"}, - {RPCResult::Type::NUM, "reqSigs", /* optional */ true, "(DEPRECATED, returned only if config option -deprecatedrpc=addresses is passed) Number of required signatures"}, {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, - {RPCResult::Type::STR, "address", /* optional */ true, "bitcoin address (only if a well-defined address exists)"}, - {RPCResult::Type::ARR, "addresses", /* optional */ true, "(DEPRECATED, returned only if config option -deprecatedrpc=addresses is passed) Array of bitcoin addresses", - { - {RPCResult::Type::STR, "address", "bitcoin address"}, - }}, + {RPCResult::Type::STR, "address", /* optional */ true, "The Bitcoin address (only if a well-defined address exists)"}, }}, }}, }}, @@ -555,24 +545,14 @@ static RPCHelpMan decodescript() { {RPCResult::Type::STR, "asm", "Script public key"}, {RPCResult::Type::STR, "type", "The output type (e.g. "+GetAllOutputTypes()+")"}, - {RPCResult::Type::STR, "address", /* optional */ true, "bitcoin address (only if a well-defined address exists)"}, - {RPCResult::Type::NUM, "reqSigs", /* optional */ true, "(DEPRECATED, returned only if config option -deprecatedrpc=addresses is passed) Number of required signatures"}, - {RPCResult::Type::ARR, "addresses", /* optional */ true, "(DEPRECATED, returned only if config option -deprecatedrpc=addresses is passed) Array of bitcoin addresses", - { - {RPCResult::Type::STR, "address", "bitcoin address"}, - }}, - {RPCResult::Type::STR, "p2sh", "address of P2SH script wrapping this redeem script (not returned if the script is already a P2SH)"}, - {RPCResult::Type::OBJ, "segwit", "Result of a witness script public key wrapping this redeem script (not returned if the script is a P2SH or witness)", + {RPCResult::Type::STR, "address", /* optional */ true, "The Bitcoin address (only if a well-defined address exists)"}, + {RPCResult::Type::STR, "p2sh", /* optional */ true, "address of P2SH script wrapping this redeem script (not returned if the script is already a P2SH)"}, + {RPCResult::Type::OBJ, "segwit", /* optional */ true, "Result of a witness script public key wrapping this redeem script (not returned if the script is a P2SH or witness)", { {RPCResult::Type::STR, "asm", "String representation of the script public key"}, {RPCResult::Type::STR_HEX, "hex", "Hex string of the script public key"}, {RPCResult::Type::STR, "type", "The type of the script public key (e.g. witness_v0_keyhash or witness_v0_scripthash)"}, - {RPCResult::Type::STR, "address", /* optional */ true, "bitcoin address (only if a well-defined address exists)"}, - {RPCResult::Type::NUM, "reqSigs", /* optional */ true, "(DEPRECATED, returned only if config option -deprecatedrpc=addresses is passed) Number of required signatures"}, - {RPCResult::Type::ARR, "addresses", /* optional */ true, "(DEPRECATED, returned only if config option -deprecatedrpc=addresses is passed) Array of bitcoin addresses", - { - {RPCResult::Type::STR, "address", "segwit address"}, - }}, + {RPCResult::Type::STR, "address", /* optional */ true, "The Bitcoin address (only if a well-defined address exists)"}, {RPCResult::Type::STR, "p2sh-segwit", "address of the P2SH script wrapping this witness redeem script"}, }}, } @@ -593,7 +573,7 @@ static RPCHelpMan decodescript() } else { // Empty scripts are valid } - ScriptPubKeyToUniv(script, r, /* fIncludeHex */ false); + ScriptPubKeyToUniv(script, r, /* include_hex */ false); UniValue type; type = find_value(r, "type"); @@ -627,7 +607,7 @@ static RPCHelpMan decodescript() // Newer segwit program versions should be considered when then become available. segwitScr = GetScriptForDestination(WitnessV0ScriptHash(script)); } - ScriptPubKeyToUniv(segwitScr, sr, /* fIncludeHex */ true); + ScriptPubKeyToUniv(segwitScr, sr, /* include_hex */ true); sr.pushKV("p2sh-segwit", EncodeDestination(ScriptHash(segwitScr))); r.pushKV("segwit", sr); } @@ -774,6 +754,10 @@ static RPCHelpMan signrawtransactionwithkey() { {RPCResult::Type::STR_HEX, "txid", "The hash of the referenced, previous transaction"}, {RPCResult::Type::NUM, "vout", "The index of the output to spent and used as input"}, + {RPCResult::Type::ARR, "witness", "", + { + {RPCResult::Type::STR_HEX, "witness", ""}, + }}, {RPCResult::Type::STR_HEX, "scriptSig", "The hex-encoded signature script"}, {RPCResult::Type::NUM, "sequence", "Script sequence number"}, {RPCResult::Type::STR, "error", "Verification or signing error related to the input"}, @@ -894,8 +878,7 @@ static RPCHelpMan testmempoolaccept() "\nThis checks if transactions violate the consensus or policy rules.\n" "\nSee sendrawtransaction call.\n", { - {"rawtxs", RPCArg::Type::ARR, RPCArg::Optional::NO, "An array of hex strings of raw transactions.\n" - " Length must be one for now.", + {"rawtxs", RPCArg::Type::ARR, RPCArg::Optional::NO, "An array of hex strings of raw transactions.", { {"rawtx", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, }, @@ -906,21 +889,21 @@ static RPCHelpMan testmempoolaccept() RPCResult{ RPCResult::Type::ARR, "", "The result of the mempool acceptance test for each raw transaction in the input array.\n" "Returns results for each transaction in the same order they were passed in.\n" - "It is possible for transactions to not be fully validated ('allowed' unset) if another transaction failed.\n", + "Transactions that cannot be fully validated due to failures in other transactions will not contain an 'allowed' result.\n", { {RPCResult::Type::OBJ, "", "", { {RPCResult::Type::STR_HEX, "txid", "The transaction hash in hex"}, {RPCResult::Type::STR_HEX, "wtxid", "The transaction witness hash in hex"}, - {RPCResult::Type::STR, "package-error", "Package validation error, if any (only possible if rawtxs had more than 1 transaction)."}, - {RPCResult::Type::BOOL, "allowed", "Whether this tx would be accepted to the mempool and pass client-specified maxfeerate." + {RPCResult::Type::STR, "package-error", /* optional */ true, "Package validation error, if any (only possible if rawtxs had more than 1 transaction)."}, + {RPCResult::Type::BOOL, "allowed", /* optional */ true, "Whether this tx would be accepted to the mempool and pass client-specified maxfeerate. " "If not present, the tx was not fully validated due to a failure in another tx in the list."}, - {RPCResult::Type::NUM, "vsize", "Virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted (only present when 'allowed' is true)"}, - {RPCResult::Type::OBJ, "fees", "Transaction fees (only present if 'allowed' is true)", + {RPCResult::Type::NUM, "vsize", /* optional */ true, "Virtual transaction size as defined in BIP 141. This is different from actual serialized size for witness transactions as witness data is discounted (only present when 'allowed' is true)"}, + {RPCResult::Type::OBJ, "fees", /* optional */ true, "Transaction fees (only present if 'allowed' is true)", { {RPCResult::Type::STR_AMOUNT, "base", "transaction fee in " + CURRENCY_UNIT}, }}, - {RPCResult::Type::STR, "reject-reason", "Rejection string (only present when 'allowed' is false)"}, + {RPCResult::Type::STR, "reject-reason", /* optional */ true, "Rejection string (only present when 'allowed' is false)"}, }}, } }, @@ -1059,7 +1042,7 @@ static RPCHelpMan decodepsbt() {RPCResult::Type::STR, "asm", "The asm"}, {RPCResult::Type::STR_HEX, "hex", "The hex"}, {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, - {RPCResult::Type::STR, "address"," Bitcoin address if there is one"}, + {RPCResult::Type::STR, "address", /* optional */ true, "The Bitcoin address (only if a well-defined address exists)"}, }}, }}, {RPCResult::Type::OBJ_DYN, "partial_signatures", /* optional */ true, "", @@ -1081,22 +1064,23 @@ static RPCHelpMan decodepsbt() }}, {RPCResult::Type::ARR, "bip32_derivs", /* optional */ true, "", { - {RPCResult::Type::OBJ, "pubkey", /* optional */ true, "The public key with the derivation path as the value.", + {RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::STR, "pubkey", "The public key with the derivation path as the value."}, {RPCResult::Type::STR, "master_fingerprint", "The fingerprint of the master key"}, {RPCResult::Type::STR, "path", "The path"}, }}, }}, - {RPCResult::Type::OBJ, "final_scriptsig", /* optional */ true, "", + {RPCResult::Type::OBJ, "final_scriptSig", /* optional */ true, "", { {RPCResult::Type::STR, "asm", "The asm"}, {RPCResult::Type::STR, "hex", "The hex"}, }}, - {RPCResult::Type::ARR, "final_scriptwitness", "", + {RPCResult::Type::ARR, "final_scriptwitness", /* optional */ true, "", { {RPCResult::Type::STR_HEX, "", "hex-encoded witness data (if any)"}, }}, - {RPCResult::Type::OBJ_DYN, "unknown", "The unknown global fields", + {RPCResult::Type::OBJ_DYN, "unknown", /* optional */ true, "The unknown global fields", { {RPCResult::Type::STR_HEX, "key", "(key-value pair) An unknown key-value pair"}, }}, @@ -1127,7 +1111,7 @@ static RPCHelpMan decodepsbt() {RPCResult::Type::STR, "path", "The path"}, }}, }}, - {RPCResult::Type::OBJ_DYN, "unknown", "The unknown global fields", + {RPCResult::Type::OBJ_DYN, "unknown", /* optional */ true, "The unknown global fields", { {RPCResult::Type::STR_HEX, "key", "(key-value pair) An unknown key-value pair"}, }}, @@ -1178,7 +1162,7 @@ static RPCHelpMan decodepsbt() txout = input.witness_utxo; UniValue o(UniValue::VOBJ); - ScriptToUniv(txout.scriptPubKey, o, true); + ScriptPubKeyToUniv(txout.scriptPubKey, o, /* include_hex */ true); UniValue out(UniValue::VOBJ); out.pushKV("amount", ValueFromAmount(txout.nValue)); @@ -1225,12 +1209,12 @@ static RPCHelpMan decodepsbt() // Redeem script and witness script if (!input.redeem_script.empty()) { UniValue r(UniValue::VOBJ); - ScriptToUniv(input.redeem_script, r, false); + ScriptToUniv(input.redeem_script, r); in.pushKV("redeem_script", r); } if (!input.witness_script.empty()) { UniValue r(UniValue::VOBJ); - ScriptToUniv(input.witness_script, r, false); + ScriptToUniv(input.witness_script, r); in.pushKV("witness_script", r); } @@ -1285,12 +1269,12 @@ static RPCHelpMan decodepsbt() // Redeem script and witness script if (!output.redeem_script.empty()) { UniValue r(UniValue::VOBJ); - ScriptToUniv(output.redeem_script, r, false); + ScriptToUniv(output.redeem_script, r); out.pushKV("redeem_script", r); } if (!output.witness_script.empty()) { UniValue r(UniValue::VOBJ); - ScriptToUniv(output.witness_script, r, false); + ScriptToUniv(output.witness_script, r); out.pushKV("witness_script", r); } @@ -1401,8 +1385,8 @@ static RPCHelpMan finalizepsbt() RPCResult{ RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::STR, "psbt", "The base64-encoded partially signed transaction if not extracted"}, - {RPCResult::Type::STR_HEX, "hex", "The hex-encoded network transaction if extracted"}, + {RPCResult::Type::STR, "psbt", /* optional */ true, "The base64-encoded partially signed transaction if not extracted"}, + {RPCResult::Type::STR_HEX, "hex", /* optional */ true, "The hex-encoded network transaction if extracted"}, {RPCResult::Type::BOOL, "complete", "If the transaction has a complete set of signatures"}, } }, @@ -1794,7 +1778,7 @@ static RPCHelpMan analyzepsbt() RPCResult { RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::ARR, "inputs", "", + {RPCResult::Type::ARR, "inputs", /* optional */ true, "", { {RPCResult::Type::OBJ, "", "", { diff --git a/src/rpc/rawtransaction_util.cpp b/src/rpc/rawtransaction_util.cpp index 122a92f084..d550160260 100644 --- a/src/rpc/rawtransaction_util.cpp +++ b/src/rpc/rawtransaction_util.cpp @@ -6,6 +6,7 @@ #include <rpc/rawtransaction_util.h> #include <coins.h> +#include <consensus/amount.h> #include <core_io.h> #include <key_io.h> #include <policy/policy.h> @@ -18,6 +19,7 @@ #include <univalue.h> #include <util/rbf.h> #include <util/strencodings.h> +#include <util/translation.h> CMutableTransaction ConstructTransaction(const UniValue& inputs_in, const UniValue& outputs_in, const UniValue& locktime, bool rbf) { @@ -280,22 +282,22 @@ void SignTransaction(CMutableTransaction& mtx, const SigningProvider* keystore, int nHashType = ParseSighashString(hashType); // Script verification errors - std::map<int, std::string> input_errors; + std::map<int, bilingual_str> input_errors; bool complete = SignTransaction(mtx, keystore, coins, nHashType, input_errors); SignTransactionResultToJSON(mtx, complete, coins, input_errors, result); } -void SignTransactionResultToJSON(CMutableTransaction& mtx, bool complete, const std::map<COutPoint, Coin>& coins, const std::map<int, std::string>& input_errors, UniValue& result) +void SignTransactionResultToJSON(CMutableTransaction& mtx, bool complete, const std::map<COutPoint, Coin>& coins, const std::map<int, bilingual_str>& input_errors, UniValue& result) { // Make errors UniValue UniValue vErrors(UniValue::VARR); for (const auto& err_pair : input_errors) { - if (err_pair.second == "Missing amount") { + if (err_pair.second.original == "Missing amount") { // This particular error needs to be an exception for some reason throw JSONRPCError(RPC_TYPE_ERROR, strprintf("Missing amount for %s", coins.at(mtx.vin.at(err_pair.first).prevout).out.ToString())); } - TxInErrorToJSON(mtx.vin.at(err_pair.first), vErrors, err_pair.second); + TxInErrorToJSON(mtx.vin.at(err_pair.first), vErrors, err_pair.second.original); } result.pushKV("hex", EncodeHexTx(CTransaction(mtx))); diff --git a/src/rpc/rawtransaction_util.h b/src/rpc/rawtransaction_util.h index ce7d5834fa..d2e116f7ee 100644 --- a/src/rpc/rawtransaction_util.h +++ b/src/rpc/rawtransaction_util.h @@ -8,6 +8,7 @@ #include <map> #include <string> +struct bilingual_str; class FillableSigningProvider; class UniValue; struct CMutableTransaction; @@ -25,7 +26,7 @@ class SigningProvider; * @param result JSON object where signed transaction results accumulate */ void SignTransaction(CMutableTransaction& mtx, const SigningProvider* keystore, const std::map<COutPoint, Coin>& coins, const UniValue& hashType, UniValue& result); -void SignTransactionResultToJSON(CMutableTransaction& mtx, bool complete, const std::map<COutPoint, Coin>& coins, const std::map<int, std::string>& input_errors, UniValue& result); +void SignTransactionResultToJSON(CMutableTransaction& mtx, bool complete, const std::map<COutPoint, Coin>& coins, const std::map<int, bilingual_str>& input_errors, UniValue& result); /** * Parse a prevtxs UniValue array and get the map of coins from it diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index cf80b08b96..0d02ec5c47 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -540,7 +540,7 @@ void RPCRunLater(const std::string& name, std::function<void()> func, int64_t nS int RPCSerializationFlags() { int flag = 0; - if (gArgs.GetArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) == 0) + if (gArgs.GetIntArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) == 0) flag |= SERIALIZE_TRANSACTION_NO_WITNESS; return flag; } diff --git a/src/rpc/server.h b/src/rpc/server.h index 03967020c2..e6bb35fc33 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -6,7 +6,6 @@ #ifndef BITCOIN_RPC_SERVER_H #define BITCOIN_RPC_SERVER_H -#include <amount.h> #include <rpc/request.h> #include <rpc/util.h> diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 2059628b54..2d7f5f2894 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <consensus/amount.h> #include <key_io.h> #include <outputtype.h> #include <rpc/util.h> diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 02ada969a4..162cced6c7 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -5,6 +5,7 @@ #include <scheduler.h> #include <random.h> +#include <util/syscall_sandbox.h> #include <util/time.h> #include <assert.h> @@ -24,6 +25,7 @@ CScheduler::~CScheduler() void CScheduler::serviceQueue() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::SCHEDULER); WAIT_LOCK(newTaskMutex, lock); ++nThreadsServicingQueue; diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index fdbd2d7fc7..621a1b9fd6 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -166,7 +166,7 @@ public: * write_cache is the cache to write keys to (if not nullptr) * Caches are not exclusive but this is not tested. Currently we use them exclusively */ - virtual bool GetPubKey(int pos, const SigningProvider& arg, CPubKey& key, KeyOriginInfo& info, const DescriptorCache* read_cache = nullptr, DescriptorCache* write_cache = nullptr) = 0; + virtual bool GetPubKey(int pos, const SigningProvider& arg, CPubKey& key, KeyOriginInfo& info, const DescriptorCache* read_cache = nullptr, DescriptorCache* write_cache = nullptr) const = 0; /** Whether this represent multiple public keys at different positions. */ virtual bool IsRange() const = 0; @@ -181,7 +181,7 @@ public: virtual bool ToPrivateString(const SigningProvider& arg, std::string& out) const = 0; /** Get the descriptor string form with the xpub at the last hardened derivation */ - virtual bool ToNormalizedString(const SigningProvider& arg, std::string& out, bool priv) const = 0; + virtual bool ToNormalizedString(const SigningProvider& arg, std::string& out, const DescriptorCache* cache = nullptr) const = 0; /** Derive a private key, if private data is available in arg. */ virtual bool GetPrivKey(int pos, const SigningProvider& arg, CKey& key) const = 0; @@ -199,7 +199,7 @@ class OriginPubkeyProvider final : public PubkeyProvider public: OriginPubkeyProvider(uint32_t exp_index, KeyOriginInfo info, std::unique_ptr<PubkeyProvider> provider) : PubkeyProvider(exp_index), m_origin(std::move(info)), m_provider(std::move(provider)) {} - bool GetPubKey(int pos, const SigningProvider& arg, CPubKey& key, KeyOriginInfo& info, const DescriptorCache* read_cache = nullptr, DescriptorCache* write_cache = nullptr) override + bool GetPubKey(int pos, const SigningProvider& arg, CPubKey& key, KeyOriginInfo& info, const DescriptorCache* read_cache = nullptr, DescriptorCache* write_cache = nullptr) const override { if (!m_provider->GetPubKey(pos, arg, key, info, read_cache, write_cache)) return false; std::copy(std::begin(m_origin.fingerprint), std::end(m_origin.fingerprint), info.fingerprint); @@ -216,10 +216,10 @@ public: ret = "[" + OriginString() + "]" + std::move(sub); return true; } - bool ToNormalizedString(const SigningProvider& arg, std::string& ret, bool priv) const override + bool ToNormalizedString(const SigningProvider& arg, std::string& ret, const DescriptorCache* cache) const override { std::string sub; - if (!m_provider->ToNormalizedString(arg, sub, priv)) return false; + if (!m_provider->ToNormalizedString(arg, sub, cache)) return false; // If m_provider is a BIP32PubkeyProvider, we may get a string formatted like a OriginPubkeyProvider // In that case, we need to strip out the leading square bracket and fingerprint from the substring, // and append that to our own origin string. @@ -245,7 +245,7 @@ class ConstPubkeyProvider final : public PubkeyProvider public: ConstPubkeyProvider(uint32_t exp_index, const CPubKey& pubkey, bool xonly) : PubkeyProvider(exp_index), m_pubkey(pubkey), m_xonly(xonly) {} - bool GetPubKey(int pos, const SigningProvider& arg, CPubKey& key, KeyOriginInfo& info, const DescriptorCache* read_cache = nullptr, DescriptorCache* write_cache = nullptr) override + bool GetPubKey(int pos, const SigningProvider& arg, CPubKey& key, KeyOriginInfo& info, const DescriptorCache* read_cache = nullptr, DescriptorCache* write_cache = nullptr) const override { key = m_pubkey; info.path.clear(); @@ -263,9 +263,8 @@ public: ret = EncodeSecret(key); return true; } - bool ToNormalizedString(const SigningProvider& arg, std::string& ret, bool priv) const override + bool ToNormalizedString(const SigningProvider& arg, std::string& ret, const DescriptorCache* cache) const override { - if (priv) return ToPrivateString(arg, ret); ret = ToString(); return true; } @@ -288,9 +287,6 @@ class BIP32PubkeyProvider final : public PubkeyProvider CExtPubKey m_root_extkey; KeyPath m_path; DeriveType m_derive; - // Cache of the parent of the final derived pubkeys. - // Primarily useful for situations when no read_cache is provided - CExtPubKey m_cached_xpub; bool GetExtKey(const SigningProvider& arg, CExtKey& ret) const { @@ -305,11 +301,14 @@ class BIP32PubkeyProvider final : public PubkeyProvider } // Derives the last xprv - bool GetDerivedExtKey(const SigningProvider& arg, CExtKey& xprv) const + bool GetDerivedExtKey(const SigningProvider& arg, CExtKey& xprv, CExtKey& last_hardened) const { if (!GetExtKey(arg, xprv)) return false; for (auto entry : m_path) { xprv.Derive(xprv, entry); + if (entry >> 31) { + last_hardened = xprv; + } } return true; } @@ -327,7 +326,7 @@ public: BIP32PubkeyProvider(uint32_t exp_index, const CExtPubKey& extkey, KeyPath path, DeriveType derive) : PubkeyProvider(exp_index), m_root_extkey(extkey), m_path(std::move(path)), m_derive(derive) {} bool IsRange() const override { return m_derive != DeriveType::NO; } size_t GetSize() const override { return 33; } - bool GetPubKey(int pos, const SigningProvider& arg, CPubKey& key_out, KeyOriginInfo& final_info_out, const DescriptorCache* read_cache = nullptr, DescriptorCache* write_cache = nullptr) override + bool GetPubKey(int pos, const SigningProvider& arg, CPubKey& key_out, KeyOriginInfo& final_info_out, const DescriptorCache* read_cache = nullptr, DescriptorCache* write_cache = nullptr) const override { // Info of parent of the to be derived pubkey KeyOriginInfo parent_info; @@ -343,6 +342,7 @@ public: // Derive keys or fetch them from cache CExtPubKey final_extkey = m_root_extkey; CExtPubKey parent_extkey = m_root_extkey; + CExtPubKey last_hardened_extkey; bool der = true; if (read_cache) { if (!read_cache->GetCachedDerivedExtPubKey(m_expr_index, pos, final_extkey)) { @@ -352,16 +352,17 @@ public: final_extkey = parent_extkey; if (m_derive == DeriveType::UNHARDENED) der = parent_extkey.Derive(final_extkey, pos); } - } else if (m_cached_xpub.pubkey.IsValid() && m_derive != DeriveType::HARDENED) { - parent_extkey = final_extkey = m_cached_xpub; - if (m_derive == DeriveType::UNHARDENED) der = parent_extkey.Derive(final_extkey, pos); } else if (IsHardened()) { CExtKey xprv; - if (!GetDerivedExtKey(arg, xprv)) return false; + CExtKey lh_xprv; + if (!GetDerivedExtKey(arg, xprv, lh_xprv)) return false; parent_extkey = xprv.Neuter(); if (m_derive == DeriveType::UNHARDENED) der = xprv.Derive(xprv, pos); if (m_derive == DeriveType::HARDENED) der = xprv.Derive(xprv, pos | 0x80000000UL); final_extkey = xprv.Neuter(); + if (lh_xprv.key.IsValid()) { + last_hardened_extkey = lh_xprv.Neuter(); + } } else { for (auto entry : m_path) { der = parent_extkey.Derive(parent_extkey, entry); @@ -376,15 +377,14 @@ public: final_info_out = final_info_out_tmp; key_out = final_extkey.pubkey; - // We rely on the consumer to check that m_derive isn't HARDENED as above - // But we can't have already cached something in case we read something from the cache - // and parent_extkey isn't actually the parent. - if (!m_cached_xpub.pubkey.IsValid()) m_cached_xpub = parent_extkey; - if (write_cache) { // Only cache parent if there is any unhardened derivation if (m_derive != DeriveType::HARDENED) { write_cache->CacheParentExtPubKey(m_expr_index, parent_extkey); + // Cache last hardened xpub if we have it + if (last_hardened_extkey.pubkey.IsValid()) { + write_cache->CacheLastHardenedExtPubKey(m_expr_index, last_hardened_extkey); + } } else if (final_info_out.path.size() > 0) { write_cache->CacheDerivedExtPubKey(m_expr_index, pos, final_extkey); } @@ -412,11 +412,10 @@ public: } return true; } - bool ToNormalizedString(const SigningProvider& arg, std::string& out, bool priv) const override + bool ToNormalizedString(const SigningProvider& arg, std::string& out, const DescriptorCache* cache) const override { // For hardened derivation type, just return the typical string, nothing to normalize if (m_derive == DeriveType::HARDENED) { - if (priv) return ToPrivateString(arg, out); out = ToString(); return true; } @@ -429,33 +428,42 @@ public: } // Either no derivation or all unhardened derivation if (i == -1) { - if (priv) return ToPrivateString(arg, out); out = ToString(); return true; } - // Derive the xpub at the last hardened step - CExtKey xprv; - if (!GetExtKey(arg, xprv)) return false; + // Get the path to the last hardened stup KeyOriginInfo origin; int k = 0; for (; k <= i; ++k) { - // Derive - xprv.Derive(xprv, m_path.at(k)); // Add to the path origin.path.push_back(m_path.at(k)); - // First derivation element, get the fingerprint for origin - if (k == 0) { - std::copy(xprv.vchFingerprint, xprv.vchFingerprint + 4, origin.fingerprint); - } } // Build the remaining path KeyPath end_path; for (; k < (int)m_path.size(); ++k) { end_path.push_back(m_path.at(k)); } + // Get the fingerprint + CKeyID id = m_root_extkey.pubkey.GetID(); + std::copy(id.begin(), id.begin() + 4, origin.fingerprint); + + CExtPubKey xpub; + CExtKey lh_xprv; + // If we have the cache, just get the parent xpub + if (cache != nullptr) { + cache->GetCachedLastHardenedExtPubKey(m_expr_index, xpub); + } + if (!xpub.pubkey.IsValid()) { + // Cache miss, or nor cache, or need privkey + CExtKey xprv; + if (!GetDerivedExtKey(arg, xprv, lh_xprv)) return false; + xpub = lh_xprv.Neuter(); + } + assert(xpub.pubkey.IsValid()); + // Build the string std::string origin_str = HexStr(origin.fingerprint) + FormatHDKeypath(origin.path); - out = "[" + origin_str + "]" + (priv ? EncodeExtKey(xprv) : EncodeExtPubKey(xprv.Neuter())) + FormatHDKeypath(end_path); + out = "[" + origin_str + "]" + EncodeExtPubKey(xpub) + FormatHDKeypath(end_path); if (IsRange()) { out += "/*"; assert(m_derive == DeriveType::UNHARDENED); @@ -465,7 +473,8 @@ public: bool GetPrivKey(int pos, const SigningProvider& arg, CKey& key) const override { CExtKey extkey; - if (!GetDerivedExtKey(arg, extkey)) return false; + CExtKey dummy; + if (!GetDerivedExtKey(arg, extkey, dummy)) return false; if (m_derive == DeriveType::UNHARDENED) extkey.Derive(extkey, pos); if (m_derive == DeriveType::HARDENED) extkey.Derive(extkey, pos | 0x80000000UL); key = extkey.key; @@ -508,6 +517,13 @@ public: DescriptorImpl(std::vector<std::unique_ptr<PubkeyProvider>> pubkeys, std::unique_ptr<DescriptorImpl> script, const std::string& name) : m_pubkey_args(std::move(pubkeys)), m_name(name), m_subdescriptor_args(Vector(std::move(script))) {} DescriptorImpl(std::vector<std::unique_ptr<PubkeyProvider>> pubkeys, std::vector<std::unique_ptr<DescriptorImpl>> scripts, const std::string& name) : m_pubkey_args(std::move(pubkeys)), m_name(name), m_subdescriptor_args(std::move(scripts)) {} + enum class StringType + { + PUBLIC, + PRIVATE, + NORMALIZED, + }; + bool IsSolvable() const override { for (const auto& arg : m_subdescriptor_args) { @@ -527,19 +543,19 @@ public: return false; } - virtual bool ToStringSubScriptHelper(const SigningProvider* arg, std::string& ret, bool priv, bool normalized) const + virtual bool ToStringSubScriptHelper(const SigningProvider* arg, std::string& ret, const StringType type, const DescriptorCache* cache = nullptr) const { size_t pos = 0; for (const auto& scriptarg : m_subdescriptor_args) { if (pos++) ret += ","; std::string tmp; - if (!scriptarg->ToStringHelper(arg, tmp, priv, normalized)) return false; + if (!scriptarg->ToStringHelper(arg, tmp, type, cache)) return false; ret += std::move(tmp); } return true; } - bool ToStringHelper(const SigningProvider* arg, std::string& out, bool priv, bool normalized) const + bool ToStringHelper(const SigningProvider* arg, std::string& out, const StringType type, const DescriptorCache* cache = nullptr) const { std::string extra = ToStringExtra(); size_t pos = extra.size() > 0 ? 1 : 0; @@ -547,17 +563,21 @@ public: for (const auto& pubkey : m_pubkey_args) { if (pos++) ret += ","; std::string tmp; - if (normalized) { - if (!pubkey->ToNormalizedString(*arg, tmp, priv)) return false; - } else if (priv) { - if (!pubkey->ToPrivateString(*arg, tmp)) return false; - } else { - tmp = pubkey->ToString(); + switch (type) { + case StringType::NORMALIZED: + if (!pubkey->ToNormalizedString(*arg, tmp, cache)) return false; + break; + case StringType::PRIVATE: + if (!pubkey->ToPrivateString(*arg, tmp)) return false; + break; + case StringType::PUBLIC: + tmp = pubkey->ToString(); + break; } ret += std::move(tmp); } std::string subscript; - if (!ToStringSubScriptHelper(arg, subscript, priv, normalized)) return false; + if (!ToStringSubScriptHelper(arg, subscript, type, cache)) return false; if (pos && subscript.size()) ret += ','; out = std::move(ret) + std::move(subscript) + ")"; return true; @@ -566,20 +586,20 @@ public: std::string ToString() const final { std::string ret; - ToStringHelper(nullptr, ret, false, false); + ToStringHelper(nullptr, ret, StringType::PUBLIC); return AddChecksum(ret); } bool ToPrivateString(const SigningProvider& arg, std::string& out) const final { - bool ret = ToStringHelper(&arg, out, true, false); + bool ret = ToStringHelper(&arg, out, StringType::PRIVATE); out = AddChecksum(out); return ret; } - bool ToNormalizedString(const SigningProvider& arg, std::string& out, bool priv) const override final + bool ToNormalizedString(const SigningProvider& arg, std::string& out, const DescriptorCache* cache) const override final { - bool ret = ToStringHelper(&arg, out, priv, true); + bool ret = ToStringHelper(&arg, out, StringType::NORMALIZED, cache); out = AddChecksum(out); return ret; } @@ -833,7 +853,7 @@ protected: out.tr_spenddata[output].Merge(builder.GetSpendData()); return Vector(GetScriptForDestination(output)); } - bool ToStringSubScriptHelper(const SigningProvider* arg, std::string& ret, bool priv, bool normalized) const override + bool ToStringSubScriptHelper(const SigningProvider* arg, std::string& ret, const StringType type, const DescriptorCache* cache = nullptr) const override { if (m_depths.empty()) return true; std::vector<bool> path; @@ -844,7 +864,7 @@ protected: path.push_back(false); } std::string tmp; - if (!m_subdescriptor_args[pos]->ToStringHelper(arg, tmp, priv, normalized)) return false; + if (!m_subdescriptor_args[pos]->ToStringHelper(arg, tmp, type, cache)) return false; ret += std::move(tmp); while (!path.empty() && path.back()) { if (path.size() > 1) ret += '}'; @@ -1222,14 +1242,8 @@ std::unique_ptr<PubkeyProvider> InferXOnlyPubkey(const XOnlyPubKey& xkey, ParseS CPubKey pubkey(full_key); std::unique_ptr<PubkeyProvider> key_provider = std::make_unique<ConstPubkeyProvider>(0, pubkey, true); KeyOriginInfo info; - if (provider.GetKeyOrigin(pubkey.GetID(), info)) { + if (provider.GetKeyOriginByXOnly(xkey, info)) { return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider)); - } else { - full_key[0] = 0x03; - pubkey = CPubKey(full_key); - if (provider.GetKeyOrigin(pubkey.GetID(), info)) { - return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider)); - } } return key_provider; } @@ -1411,6 +1425,11 @@ void DescriptorCache::CacheDerivedExtPubKey(uint32_t key_exp_pos, uint32_t der_i xpubs[der_index] = xpub; } +void DescriptorCache::CacheLastHardenedExtPubKey(uint32_t key_exp_pos, const CExtPubKey& xpub) +{ + m_last_hardened_xpubs[key_exp_pos] = xpub; +} + bool DescriptorCache::GetCachedParentExtPubKey(uint32_t key_exp_pos, CExtPubKey& xpub) const { const auto& it = m_parent_xpubs.find(key_exp_pos); @@ -1429,6 +1448,55 @@ bool DescriptorCache::GetCachedDerivedExtPubKey(uint32_t key_exp_pos, uint32_t d return true; } +bool DescriptorCache::GetCachedLastHardenedExtPubKey(uint32_t key_exp_pos, CExtPubKey& xpub) const +{ + const auto& it = m_last_hardened_xpubs.find(key_exp_pos); + if (it == m_last_hardened_xpubs.end()) return false; + xpub = it->second; + return true; +} + +DescriptorCache DescriptorCache::MergeAndDiff(const DescriptorCache& other) +{ + DescriptorCache diff; + for (const auto& parent_xpub_pair : other.GetCachedParentExtPubKeys()) { + CExtPubKey xpub; + if (GetCachedParentExtPubKey(parent_xpub_pair.first, xpub)) { + if (xpub != parent_xpub_pair.second) { + throw std::runtime_error(std::string(__func__) + ": New cached parent xpub does not match already cached parent xpub"); + } + continue; + } + CacheParentExtPubKey(parent_xpub_pair.first, parent_xpub_pair.second); + diff.CacheParentExtPubKey(parent_xpub_pair.first, parent_xpub_pair.second); + } + for (const auto& derived_xpub_map_pair : other.GetCachedDerivedExtPubKeys()) { + for (const auto& derived_xpub_pair : derived_xpub_map_pair.second) { + CExtPubKey xpub; + if (GetCachedDerivedExtPubKey(derived_xpub_map_pair.first, derived_xpub_pair.first, xpub)) { + if (xpub != derived_xpub_pair.second) { + throw std::runtime_error(std::string(__func__) + ": New cached derived xpub does not match already cached derived xpub"); + } + continue; + } + CacheDerivedExtPubKey(derived_xpub_map_pair.first, derived_xpub_pair.first, derived_xpub_pair.second); + diff.CacheDerivedExtPubKey(derived_xpub_map_pair.first, derived_xpub_pair.first, derived_xpub_pair.second); + } + } + for (const auto& lh_xpub_pair : other.GetCachedLastHardenedExtPubKeys()) { + CExtPubKey xpub; + if (GetCachedLastHardenedExtPubKey(lh_xpub_pair.first, xpub)) { + if (xpub != lh_xpub_pair.second) { + throw std::runtime_error(std::string(__func__) + ": New cached last hardened xpub does not match already cached last hardened xpub"); + } + continue; + } + CacheLastHardenedExtPubKey(lh_xpub_pair.first, lh_xpub_pair.second); + diff.CacheLastHardenedExtPubKey(lh_xpub_pair.first, lh_xpub_pair.second); + } + return diff; +} + const ExtPubKeyMap DescriptorCache::GetCachedParentExtPubKeys() const { return m_parent_xpubs; @@ -1438,3 +1506,8 @@ const std::unordered_map<uint32_t, ExtPubKeyMap> DescriptorCache::GetCachedDeriv { return m_derived_xpubs; } + +const ExtPubKeyMap DescriptorCache::GetCachedLastHardenedExtPubKeys() const +{ + return m_last_hardened_xpubs; +} diff --git a/src/script/descriptor.h b/src/script/descriptor.h index 332ae2f230..ecd7c4eea5 100644 --- a/src/script/descriptor.h +++ b/src/script/descriptor.h @@ -22,6 +22,8 @@ private: std::unordered_map<uint32_t, ExtPubKeyMap> m_derived_xpubs; /** Map key expression index -> parent xpub */ ExtPubKeyMap m_parent_xpubs; + /** Map key expression index -> last hardened xpub */ + ExtPubKeyMap m_last_hardened_xpubs; public: /** Cache a parent xpub @@ -50,11 +52,30 @@ public: * @param[in] xpub The CExtPubKey to get from cache */ bool GetCachedDerivedExtPubKey(uint32_t key_exp_pos, uint32_t der_index, CExtPubKey& xpub) const; + /** Cache a last hardened xpub + * + * @param[in] key_exp_pos Position of the key expression within the descriptor + * @param[in] xpub The CExtPubKey to cache + */ + void CacheLastHardenedExtPubKey(uint32_t key_exp_pos, const CExtPubKey& xpub); + /** Retrieve a cached last hardened xpub + * + * @param[in] key_exp_pos Position of the key expression within the descriptor + * @param[in] xpub The CExtPubKey to get from cache + */ + bool GetCachedLastHardenedExtPubKey(uint32_t key_exp_pos, CExtPubKey& xpub) const; /** Retrieve all cached parent xpubs */ const ExtPubKeyMap GetCachedParentExtPubKeys() const; /** Retrieve all cached derived xpubs */ const std::unordered_map<uint32_t, ExtPubKeyMap> GetCachedDerivedExtPubKeys() const; + /** Retrieve all cached last hardened xpubs */ + const ExtPubKeyMap GetCachedLastHardenedExtPubKeys() const; + + /** Combine another DescriptorCache into this one. + * Returns a cache containing the items from the other cache unknown to current cache + */ + DescriptorCache MergeAndDiff(const DescriptorCache& other); }; /** \brief Interface for parsed descriptor objects. @@ -94,7 +115,7 @@ struct Descriptor { virtual bool ToPrivateString(const SigningProvider& provider, std::string& out) const = 0; /** Convert the descriptor to a normalized string. Normalized descriptors have the xpub at the last hardened step. This fails if the provided provider does not have the private keys to derive that xpub. */ - virtual bool ToNormalizedString(const SigningProvider& provider, std::string& out, bool priv) const = 0; + virtual bool ToNormalizedString(const SigningProvider& provider, std::string& out, const DescriptorCache* cache = nullptr) const = 0; /** Expand a descriptor at a specified position. * diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 8cebbc5d10..eafa9840d7 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1740,9 +1740,9 @@ bool GenericTransactionSignatureChecker<T>::CheckLockTime(const CScriptNum& nLoc if (nLockTime > (int64_t)txTo->nLockTime) return false; - // Finally the nLockTime feature can be disabled and thus - // CHECKLOCKTIMEVERIFY bypassed if every txin has been - // finalized by setting nSequence to maxint. The + // Finally the nLockTime feature can be disabled in IsFinalTx() + // and thus CHECKLOCKTIMEVERIFY bypassed if every txin has + // been finalized by setting nSequence to maxint. The // transaction would be allowed into the blockchain, making // the opcode ineffective. // @@ -1807,16 +1807,16 @@ bool GenericTransactionSignatureChecker<T>::CheckSequence(const CScriptNum& nSeq template class GenericTransactionSignatureChecker<CTransaction>; template class GenericTransactionSignatureChecker<CMutableTransaction>; -static bool ExecuteWitnessScript(const Span<const valtype>& stack_span, const CScript& scriptPubKey, unsigned int flags, SigVersion sigversion, const BaseSignatureChecker& checker, ScriptExecutionData& execdata, ScriptError* serror) +static bool ExecuteWitnessScript(const Span<const valtype>& stack_span, const CScript& exec_script, unsigned int flags, SigVersion sigversion, const BaseSignatureChecker& checker, ScriptExecutionData& execdata, ScriptError* serror) { std::vector<valtype> stack{stack_span.begin(), stack_span.end()}; if (sigversion == SigVersion::TAPSCRIPT) { // OP_SUCCESSx processing overrides everything, including stack element size limits - CScript::const_iterator pc = scriptPubKey.begin(); - while (pc < scriptPubKey.end()) { + CScript::const_iterator pc = exec_script.begin(); + while (pc < exec_script.end()) { opcodetype opcode; - if (!scriptPubKey.GetOp(pc, opcode)) { + if (!exec_script.GetOp(pc, opcode)) { // Note how this condition would not be reached if an unknown OP_SUCCESSx was found return set_error(serror, SCRIPT_ERR_BAD_OPCODE); } @@ -1839,7 +1839,7 @@ static bool ExecuteWitnessScript(const Span<const valtype>& stack_span, const CS } // Run the script interpreter. - if (!EvalScript(stack, scriptPubKey, flags, checker, sigversion, execdata, serror)) return false; + if (!EvalScript(stack, exec_script, flags, checker, sigversion, execdata, serror)) return false; // Scripts inside witness implicitly require cleanstack behaviour if (stack.size() != 1) return set_error(serror, SCRIPT_ERR_CLEANSTACK); @@ -1874,9 +1874,9 @@ static bool VerifyTaprootCommitment(const std::vector<unsigned char>& control, c assert(control.size() >= TAPROOT_CONTROL_BASE_SIZE); assert(program.size() >= uint256::size()); //! The internal pubkey (x-only, so no Y coordinate parity). - const XOnlyPubKey p{uint256(std::vector<unsigned char>(control.begin() + 1, control.begin() + TAPROOT_CONTROL_BASE_SIZE))}; + const XOnlyPubKey p{Span<const unsigned char>{control.data() + 1, control.data() + TAPROOT_CONTROL_BASE_SIZE}}; //! The output pubkey (taken from the scriptPubKey). - const XOnlyPubKey q{uint256(program)}; + const XOnlyPubKey q{program}; // Compute the Merkle root from the leaf and the provided path. const uint256 merkle_root = ComputeTaprootMerkleRoot(control, tapleaf_hash); // Verify that the output pubkey matches the tweaked internal pubkey, after correcting for parity. diff --git a/src/script/interpreter.h b/src/script/interpreter.h index 034c937b99..ab49e84577 100644 --- a/src/script/interpreter.h +++ b/src/script/interpreter.h @@ -39,8 +39,7 @@ enum * All flags are intended to be soft forks: the set of acceptable scripts under * flags (A | B) is a subset of the acceptable scripts under flag (A). */ -enum -{ +enum : uint32_t { SCRIPT_VERIFY_NONE = 0, // Evaluate P2SH subscripts (BIP16). @@ -140,6 +139,10 @@ enum // Making unknown public key versions (in BIP 342 scripts) non-standard SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_PUBKEYTYPE = (1U << 20), + + // Constants to point to the highest flag in use. Add new flags above this line. + // + SCRIPT_VERIFY_END_MARKER }; bool CheckSignatureEncoding(const std::vector<unsigned char> &vchSig, unsigned int flags, ScriptError* serror); @@ -167,6 +170,13 @@ struct PrecomputedTransactionData PrecomputedTransactionData() = default; + /** Initialize this PrecomputedTransactionData with transaction data. + * + * @param[in] tx The transaction for which data is being precomputed. + * @param[in] spent_outputs The CTxOuts being spent, one for each tx.vin, in order. + * @param[in] force Whether to precompute data for all optional features, + * regardless of what is in the inputs (used at signing + * time, when the inputs aren't filled in yet). */ template <class T> void Init(const T& tx, std::vector<CTxOut>&& spent_outputs, bool force = false); diff --git a/src/script/script.h b/src/script/script.h index 974cde4984..8cd1cc3855 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -6,6 +6,7 @@ #ifndef BITCOIN_SCRIPT_SCRIPT_H #define BITCOIN_SCRIPT_SCRIPT_H +#include <attributes.h> #include <crypto/common.h> #include <prevector.h> #include <serialize.h> @@ -438,9 +439,9 @@ public: /** Delete non-existent operator to defend against future introduction */ CScript& operator<<(const CScript& b) = delete; - CScript& operator<<(int64_t b) { return push_int64(b); } + CScript& operator<<(int64_t b) LIFETIMEBOUND { return push_int64(b); } - CScript& operator<<(opcodetype opcode) + CScript& operator<<(opcodetype opcode) LIFETIMEBOUND { if (opcode < 0 || opcode > 0xff) throw std::runtime_error("CScript::operator<<(): invalid opcode"); @@ -448,13 +449,13 @@ public: return *this; } - CScript& operator<<(const CScriptNum& b) + CScript& operator<<(const CScriptNum& b) LIFETIMEBOUND { *this << b.getvch(); return *this; } - CScript& operator<<(const std::vector<unsigned char>& b) + CScript& operator<<(const std::vector<unsigned char>& b) LIFETIMEBOUND { if (b.size() < OP_PUSHDATA1) { diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index 65867c1c14..6f911f4fe7 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -96,7 +96,7 @@ void InitSignatureCache() { // nMaxCacheSize is unsigned. If -maxsigcachesize is set to zero, // setup_bytes creates the minimum possible cache (2 elements). - size_t nMaxCacheSize = std::min(std::max((int64_t)0, gArgs.GetArg("-maxsigcachesize", DEFAULT_MAX_SIG_CACHE_SIZE) / 2), MAX_MAX_SIG_CACHE_SIZE) * ((size_t) 1 << 20); + size_t nMaxCacheSize = std::min(std::max((int64_t)0, gArgs.GetIntArg("-maxsigcachesize", DEFAULT_MAX_SIG_CACHE_SIZE) / 2), MAX_MAX_SIG_CACHE_SIZE) * ((size_t) 1 << 20); size_t nElems = signatureCache.setup_bytes(nMaxCacheSize); LogPrintf("Using %zu MiB out of %zu/2 requested for signature cache, able to store %zu elements\n", (nElems*sizeof(uint256)) >>20, (nMaxCacheSize*2)>>20, nElems); diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 65276f641f..4cb2125747 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -5,12 +5,14 @@ #include <script/sign.h> +#include <consensus/amount.h> #include <key.h> #include <policy/policy.h> #include <primitives/transaction.h> #include <script/signingprovider.h> #include <script/standard.h> #include <uint256.h> +#include <util/translation.h> #include <util/vector.h> typedef std::vector<unsigned char> valtype; @@ -59,22 +61,7 @@ bool MutableTransactionSignatureCreator::CreateSchnorrSig(const SigningProvider& assert(sigversion == SigVersion::TAPROOT || sigversion == SigVersion::TAPSCRIPT); CKey key; - { - // For now, use the old full pubkey-based key derivation logic. As it indexed by - // Hash160(full pubkey), we need to try both a version prefixed with 0x02, and one - // with 0x03. - unsigned char b[33] = {0x02}; - std::copy(pubkey.begin(), pubkey.end(), b + 1); - CPubKey fullpubkey; - fullpubkey.Set(b, b + 33); - CKeyID keyid = fullpubkey.GetID(); - if (!provider.GetKey(keyid, key)) { - b[0] = 0x03; - fullpubkey.Set(b, b + 33); - CKeyID keyid = fullpubkey.GetID(); - if (!provider.GetKey(keyid, key)) return false; - } - } + if (!provider.GetKeyByXOnly(pubkey, key)) return false; // BIP341/BIP342 signing needs lots of precomputed transaction data. While some // (non-SIGHASH_DEFAULT) sighash modes exist that can work with just some subset @@ -612,21 +599,24 @@ bool IsSolvable(const SigningProvider& provider, const CScript& script) bool IsSegWitOutput(const SigningProvider& provider, const CScript& script) { - std::vector<valtype> solutions; - auto whichtype = Solver(script, solutions); - if (whichtype == TxoutType::WITNESS_V0_SCRIPTHASH || whichtype == TxoutType::WITNESS_V0_KEYHASH || whichtype == TxoutType::WITNESS_UNKNOWN) return true; - if (whichtype == TxoutType::SCRIPTHASH) { - auto h160 = uint160(solutions[0]); - CScript subscript; - if (provider.GetCScript(CScriptID{h160}, subscript)) { - whichtype = Solver(subscript, solutions); - if (whichtype == TxoutType::WITNESS_V0_SCRIPTHASH || whichtype == TxoutType::WITNESS_V0_KEYHASH || whichtype == TxoutType::WITNESS_UNKNOWN) return true; + int version; + valtype program; + if (script.IsWitnessProgram(version, program)) return true; + if (script.IsPayToScriptHash()) { + std::vector<valtype> solutions; + auto whichtype = Solver(script, solutions); + if (whichtype == TxoutType::SCRIPTHASH) { + auto h160 = uint160(solutions[0]); + CScript subscript; + if (provider.GetCScript(CScriptID{h160}, subscript)) { + if (subscript.IsWitnessProgram(version, program)) return true; + } } } return false; } -bool SignTransaction(CMutableTransaction& mtx, const SigningProvider* keystore, const std::map<COutPoint, Coin>& coins, int nHashType, std::map<int, std::string>& input_errors) +bool SignTransaction(CMutableTransaction& mtx, const SigningProvider* keystore, const std::map<COutPoint, Coin>& coins, int nHashType, std::map<int, bilingual_str>& input_errors) { bool fHashSingle = ((nHashType & ~SIGHASH_ANYONECANPAY) == SIGHASH_SINGLE); @@ -636,29 +626,26 @@ bool SignTransaction(CMutableTransaction& mtx, const SigningProvider* keystore, PrecomputedTransactionData txdata; std::vector<CTxOut> spent_outputs; - spent_outputs.resize(mtx.vin.size()); - bool have_all_spent_outputs = true; - for (unsigned int i = 0; i < mtx.vin.size(); i++) { + for (unsigned int i = 0; i < mtx.vin.size(); ++i) { CTxIn& txin = mtx.vin[i]; auto coin = coins.find(txin.prevout); if (coin == coins.end() || coin->second.IsSpent()) { - have_all_spent_outputs = false; + txdata.Init(txConst, /* spent_outputs */ {}, /* force */ true); + break; } else { - spent_outputs[i] = CTxOut(coin->second.out.nValue, coin->second.out.scriptPubKey); + spent_outputs.emplace_back(coin->second.out.nValue, coin->second.out.scriptPubKey); } } - if (have_all_spent_outputs) { + if (spent_outputs.size() == mtx.vin.size()) { txdata.Init(txConst, std::move(spent_outputs), true); - } else { - txdata.Init(txConst, {}, true); } // Sign what we can: - for (unsigned int i = 0; i < mtx.vin.size(); i++) { + for (unsigned int i = 0; i < mtx.vin.size(); ++i) { CTxIn& txin = mtx.vin[i]; auto coin = coins.find(txin.prevout); if (coin == coins.end() || coin->second.IsSpent()) { - input_errors[i] = "Input not found or already spent"; + input_errors[i] = _("Input not found or already spent"); continue; } const CScript& prevPubKey = coin->second.out.scriptPubKey; @@ -674,7 +661,7 @@ bool SignTransaction(CMutableTransaction& mtx, const SigningProvider* keystore, // amount must be specified for valid segwit signature if (amount == MAX_MONEY && !txin.scriptWitness.IsNull()) { - input_errors[i] = "Missing amount"; + input_errors[i] = _("Missing amount"); continue; } @@ -682,12 +669,12 @@ bool SignTransaction(CMutableTransaction& mtx, const SigningProvider* keystore, if (!VerifyScript(txin.scriptSig, prevPubKey, &txin.scriptWitness, STANDARD_SCRIPT_VERIFY_FLAGS, TransactionSignatureChecker(&txConst, i, amount, txdata, MissingDataBehavior::FAIL), &serror)) { if (serror == SCRIPT_ERR_INVALID_STACK_OPERATION) { // Unable to sign input and verification failed (possible attempt to partially sign). - input_errors[i] = "Unable to sign input, invalid stack size (possibly missing key)"; + input_errors[i] = Untranslated("Unable to sign input, invalid stack size (possibly missing key)"); } else if (serror == SCRIPT_ERR_SIG_NULLFAIL) { // Verification failed (possibly due to insufficient signatures). - input_errors[i] = "CHECK(MULTI)SIG failing with non-zero signature (possibly need more signatures)"; + input_errors[i] = Untranslated("CHECK(MULTI)SIG failing with non-zero signature (possibly need more signatures)"); } else { - input_errors[i] = ScriptErrorString(serror); + input_errors[i] = Untranslated(ScriptErrorString(serror)); } } else { // If this input succeeds, make sure there is no error set for it diff --git a/src/script/sign.h b/src/script/sign.h index b4e7318892..6d3479c143 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -21,6 +21,7 @@ class CScript; class CTransaction; class SigningProvider; +struct bilingual_str; struct CMutableTransaction; /** Interface for signature creators. */ @@ -44,8 +45,8 @@ class MutableTransactionSignatureCreator : public BaseSignatureCreator { const PrecomputedTransactionData* m_txdata; public: - MutableTransactionSignatureCreator(const CMutableTransaction* txToIn, unsigned int nInIn, const CAmount& amountIn, int nHashTypeIn = SIGHASH_ALL); - MutableTransactionSignatureCreator(const CMutableTransaction* txToIn, unsigned int nInIn, const CAmount& amountIn, const PrecomputedTransactionData* txdata, int nHashTypeIn = SIGHASH_ALL); + MutableTransactionSignatureCreator(const CMutableTransaction* txToIn, unsigned int nInIn, const CAmount& amountIn, int nHashTypeIn); + MutableTransactionSignatureCreator(const CMutableTransaction* txToIn, unsigned int nInIn, const CAmount& amountIn, const PrecomputedTransactionData* txdata, int nHashTypeIn); const BaseSignatureChecker& Checker() const override { return checker; } bool CreateSig(const SigningProvider& provider, std::vector<unsigned char>& vchSig, const CKeyID& keyid, const CScript& scriptCode, SigVersion sigversion) const override; bool CreateSchnorrSig(const SigningProvider& provider, std::vector<unsigned char>& sig, const XOnlyPubKey& pubkey, const uint256* leaf_hash, const uint256* merkle_root, SigVersion sigversion) const override; @@ -178,6 +179,6 @@ bool IsSolvable(const SigningProvider& provider, const CScript& script); bool IsSegWitOutput(const SigningProvider& provider, const CScript& script); /** Sign the CMutableTransaction */ -bool SignTransaction(CMutableTransaction& mtx, const SigningProvider* provider, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors); +bool SignTransaction(CMutableTransaction& mtx, const SigningProvider* provider, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, bilingual_str>& input_errors); #endif // BITCOIN_SCRIPT_SIGN_H diff --git a/src/script/signingprovider.h b/src/script/signingprovider.h index 939ae10622..fbce61c6a9 100644 --- a/src/script/signingprovider.h +++ b/src/script/signingprovider.h @@ -26,6 +26,30 @@ public: virtual bool HaveKey(const CKeyID &address) const { return false; } virtual bool GetKeyOrigin(const CKeyID& keyid, KeyOriginInfo& info) const { return false; } virtual bool GetTaprootSpendData(const XOnlyPubKey& output_key, TaprootSpendData& spenddata) const { return false; } + + bool GetKeyByXOnly(const XOnlyPubKey& pubkey, CKey& key) const + { + for (const auto& id : pubkey.GetKeyIDs()) { + if (GetKey(id, key)) return true; + } + return false; + } + + bool GetPubKeyByXOnly(const XOnlyPubKey& pubkey, CPubKey& out) const + { + for (const auto& id : pubkey.GetKeyIDs()) { + if (GetPubKey(id, out)) return true; + } + return false; + } + + bool GetKeyOriginByXOnly(const XOnlyPubKey& pubkey, KeyOriginInfo& info) const + { + for (const auto& id : pubkey.GetKeyIDs()) { + if (GetKeyOrigin(id, info)) return true; + } + return false; + } }; extern const SigningProvider& DUMMY_SIGNING_PROVIDER; diff --git a/src/script/standard.cpp b/src/script/standard.cpp index b8349bb9ab..d9656c781d 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -266,47 +266,6 @@ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) assert(false); } -// TODO: from v23 ("addresses" and "reqSigs" deprecated) "ExtractDestinations" should be removed -bool ExtractDestinations(const CScript& scriptPubKey, TxoutType& typeRet, std::vector<CTxDestination>& addressRet, int& nRequiredRet) -{ - addressRet.clear(); - std::vector<valtype> vSolutions; - typeRet = Solver(scriptPubKey, vSolutions); - if (typeRet == TxoutType::NONSTANDARD) { - return false; - } else if (typeRet == TxoutType::NULL_DATA) { - // This is data, not addresses - return false; - } - - if (typeRet == TxoutType::MULTISIG) - { - nRequiredRet = vSolutions.front()[0]; - for (unsigned int i = 1; i < vSolutions.size()-1; i++) - { - CPubKey pubKey(vSolutions[i]); - if (!pubKey.IsValid()) - continue; - - CTxDestination address = PKHash(pubKey); - addressRet.push_back(address); - } - - if (addressRet.empty()) - return false; - } - else - { - nRequiredRet = 1; - CTxDestination address; - if (!ExtractDestination(scriptPubKey, address)) - return false; - addressRet.push_back(address); - } - - return true; -} - namespace { class CScriptVisitor { @@ -504,6 +463,7 @@ WitnessV1Taproot TaprootBuilder::GetOutput() { return WitnessV1Taproot{m_output_ TaprootSpendData TaprootBuilder::GetSpendData() const { + assert(IsComplete()); TaprootSpendData spd; spd.merkle_root = m_branch.size() == 0 ? uint256() : m_branch[0]->hash; spd.internal_key = m_internal_key; diff --git a/src/script/standard.h b/src/script/standard.h index ac4e2f3276..a8e57231bf 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -176,28 +176,12 @@ TxoutType Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned c /** * Parse a standard scriptPubKey for the destination address. Assigns result to - * the addressRet parameter and returns true if successful. For multisig - * scripts, instead use ExtractDestinations. Currently only works for P2PK, + * the addressRet parameter and returns true if successful. Currently only works for P2PK, * P2PKH, P2SH, P2WPKH, and P2WSH scripts. */ bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet); /** - * Parse a standard scriptPubKey with one or more destination addresses. For - * multisig scripts, this populates the addressRet vector with the pubkey IDs - * and nRequiredRet with the n required to spend. For other destinations, - * addressRet is populated with a single value and nRequiredRet is set to 1. - * Returns true if successful. - * - * Note: this function confuses destinations (a subset of CScripts that are - * encodable as an address) with key identifiers (of keys involved in a - * CScript), and its use should be phased out. - * - * TODO: from v23 ("addresses" and "reqSigs" deprecated) "ExtractDestinations" should be removed - */ -bool ExtractDestinations(const CScript& scriptPubKey, TxoutType& typeRet, std::vector<CTxDestination>& addressRet, int& nRequiredRet); - -/** * Generate a Bitcoin scriptPubKey for the given CTxDestination. Returns a P2PKH * script for a CKeyID destination, a P2SH script for a CScriptID, and an empty * script for CNoDestination. @@ -227,8 +211,11 @@ struct TaprootSpendData /** The Merkle root of the script tree (0 if no scripts). */ uint256 merkle_root; /** Map from (script, leaf_version) to (sets of) control blocks. - * The control blocks are sorted by size, so that the signing logic can - * easily prefer the cheapest one. */ + * More than one control block for a given script is only possible if it + * appears in multiple branches of the tree. We keep them all so that + * inference can reconstruct the full tree. Within each set, the control + * blocks are sorted by size, so that the signing logic can easily + * prefer the cheapest one. */ std::map<std::pair<CScript, int>, std::set<std::vector<unsigned char>, ShortestVectorFirstComparator>> scripts; /** Merge other TaprootSpendData (for the same scriptPubKey) into this. */ void Merge(TaprootSpendData other); @@ -252,7 +239,7 @@ private: /** Merkle hash of this node. */ uint256 hash; /** Tracked leaves underneath this node (either from the node itself, or its children). - * The merkle_branch field for each is the partners to get to *this* node. */ + * The merkle_branch field of each is the partners to get to *this* node. */ std::vector<LeafInfo> leaves; }; /** Whether the builder is in a valid state so far. */ diff --git a/src/secp256k1/.cirrus.yml b/src/secp256k1/.cirrus.yml index 506a860336..bf71a70839 100644 --- a/src/secp256k1/.cirrus.yml +++ b/src/secp256k1/.cirrus.yml @@ -1,21 +1,28 @@ env: - WIDEMUL: auto + ### compiler options + HOST: + # Specific warnings can be disabled with -Wno-error=foo. + # -pedantic-errors is not equivalent to -Werror=pedantic and thus not implied by -Werror according to the GCC manual. + WERROR_CFLAGS: -Werror -pedantic-errors + MAKEFLAGS: -j2 + BUILD: check + ### secp256k1 config STATICPRECOMPUTATION: yes ECMULTGENPRECISION: auto ASM: no - BUILD: check + WIDEMUL: auto WITH_VALGRIND: yes - RUN_VALGRIND: no EXTRAFLAGS: - HOST: + ### secp256k1 modules + EXPERIMENTAL: no ECDH: no RECOVERY: no SCHNORRSIG: no - EXPERIMENTAL: no - CTIMETEST: yes + ### test options + TEST_ITERS: BENCH: yes - ITERS: 2 - MAKEFLAGS: -j2 + BENCH_ITERS: 2 + CTIMETEST: yes cat_logs_snippet: &CAT_LOGS always: @@ -63,27 +70,8 @@ task: - env: {BUILD: distcheck, WITH_VALGRIND: no, CTIMETEST: no, BENCH: no} - env: {CPPFLAGS: -DDETERMINISTIC} - env: {CFLAGS: -O0, CTIMETEST: no} - - env: - CFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer" - LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer" - UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1" - ASM: x86_64 - ECDH: yes - RECOVERY: yes - EXPERIMENTAL: yes - SCHNORRSIG: yes - CTIMETEST: no - env: { ECMULTGENPRECISION: 2 } - env: { ECMULTGENPRECISION: 8 } - - env: - RUN_VALGRIND: yes - ASM: x86_64 - ECDH: yes - RECOVERY: yes - EXPERIMENTAL: yes - SCHNORRSIG: yes - EXTRAFLAGS: "--disable-openssl-tests" - BUILD: matrix: - env: CC: gcc @@ -111,6 +99,7 @@ task: CC: i686-linux-gnu-gcc - env: CC: clang --target=i686-pc-linux-gnu -isystem /usr/i686-linux-gnu/include + << : *MERGE_BASE test_script: - ./ci/cirrus.sh << : *CAT_LOGS @@ -181,9 +170,9 @@ task: cpu: 1 memory: 1G env: - QEMU_CMD: qemu-s390x + WRAPPER_CMD: qemu-s390x + TEST_ITERS: 16 HOST: s390x-linux-gnu - BUILD: WITH_VALGRIND: no ECDH: yes RECOVERY: yes @@ -196,3 +185,158 @@ task: - rm /etc/ld.so.cache - ./ci/cirrus.sh << : *CAT_LOGS + +task: + name: "ARM32: Linux (Debian stable, QEMU)" + container: + dockerfile: ci/linux-debian.Dockerfile + cpu: 1 + memory: 1G + env: + WRAPPER_CMD: qemu-arm + TEST_ITERS: 16 + HOST: arm-linux-gnueabihf + WITH_VALGRIND: no + ECDH: yes + RECOVERY: yes + EXPERIMENTAL: yes + SCHNORRSIG: yes + CTIMETEST: no + matrix: + - env: {} + - env: {ASM: arm} + << : *MERGE_BASE + test_script: + - ./ci/cirrus.sh + << : *CAT_LOGS + +task: + name: "ARM64: Linux (Debian stable, QEMU)" + container: + dockerfile: ci/linux-debian.Dockerfile + cpu: 1 + memory: 1G + env: + WRAPPER_CMD: qemu-aarch64 + TEST_ITERS: 16 + HOST: aarch64-linux-gnu + WITH_VALGRIND: no + ECDH: yes + RECOVERY: yes + EXPERIMENTAL: yes + SCHNORRSIG: yes + CTIMETEST: no + << : *MERGE_BASE + test_script: + - ./ci/cirrus.sh + << : *CAT_LOGS + +task: + name: "ppc64le: Linux (Debian stable, QEMU)" + container: + dockerfile: ci/linux-debian.Dockerfile + cpu: 1 + memory: 1G + env: + WRAPPER_CMD: qemu-ppc64le + TEST_ITERS: 16 + HOST: powerpc64le-linux-gnu + WITH_VALGRIND: no + ECDH: yes + RECOVERY: yes + EXPERIMENTAL: yes + SCHNORRSIG: yes + CTIMETEST: no + << : *MERGE_BASE + test_script: + - ./ci/cirrus.sh + << : *CAT_LOGS + +task: + name: "x86_64 (mingw32-w64): Windows (Debian stable, Wine)" + container: + dockerfile: ci/linux-debian.Dockerfile + cpu: 1 + memory: 1G + env: + WRAPPER_CMD: wine64-stable + TEST_ITERS: 16 + HOST: x86_64-w64-mingw32 + WITH_VALGRIND: no + ECDH: yes + RECOVERY: yes + EXPERIMENTAL: yes + SCHNORRSIG: yes + CTIMETEST: no + << : *MERGE_BASE + test_script: + - ./ci/cirrus.sh + << : *CAT_LOGS + +# Sanitizers +task: + container: + dockerfile: ci/linux-debian.Dockerfile + cpu: 1 + memory: 1G + env: + ECDH: yes + RECOVERY: yes + EXPERIMENTAL: yes + SCHNORRSIG: yes + CTIMETEST: no + EXTRAFLAGS: "--disable-openssl-tests" + matrix: + - name: "Valgrind (memcheck)" + env: + # The `--error-exitcode` is required to make the test fail if valgrind found errors, otherwise it'll return 0 (https://www.valgrind.org/docs/manual/manual-core.html) + WRAPPER_CMD: "valgrind --error-exitcode=42" + TEST_ITERS: 16 + - name: "UBSan, ASan, LSan" + env: + CFLAGS: "-fsanitize=undefined,address" + CFLAGS_FOR_BUILD: "-fsanitize=undefined,address" + UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1" + ASAN_OPTIONS: "strict_string_checks=1:detect_stack_use_after_return=1:detect_leaks=1" + LSAN_OPTIONS: "use_unaligned=1" + TEST_ITERS: 32 + # Try to cover many configurations with just a tiny matrix. + matrix: + - env: + ASM: auto + STATICPRECOMPUTATION: yes + - env: + ASM: no + STATICPRECOMPUTATION: no + ECMULTGENPRECISION: 2 + matrix: + - env: + CC: clang + - env: + HOST: i686-linux-gnu + CC: i686-linux-gnu-gcc + << : *MERGE_BASE + test_script: + - ./ci/cirrus.sh + << : *CAT_LOGS + +task: + name: "C++ -fpermissive" + container: + dockerfile: ci/linux-debian.Dockerfile + cpu: 1 + memory: 1G + env: + # ./configure correctly errors out when given CC=g++. + # We hack around this by passing CC=g++ only to make. + CC: gcc + MAKEFLAGS: -j2 CC=g++ CFLAGS=-fpermissive + WERROR_CFLAGS: + EXPERIMENTAL: yes + ECDH: yes + RECOVERY: yes + SCHNORRSIG: yes + << : *MERGE_BASE + test_script: + - ./ci/cirrus.sh + << : *CAT_LOGS diff --git a/src/secp256k1/.gitignore b/src/secp256k1/.gitignore index ccdef02b29..79b740db8a 100644 --- a/src/secp256k1/.gitignore +++ b/src/secp256k1/.gitignore @@ -23,6 +23,7 @@ aclocal.m4 autom4te.cache/ config.log config.status +conftest* *.tar.gz *.la libtool @@ -33,6 +34,14 @@ libtool *~ *.log *.trs + +coverage/ +coverage.html +coverage.*.html +*.gcda +*.gcno +*.gcov + src/libsecp256k1-config.h src/libsecp256k1-config.h.in src/ecmult_static_context.h diff --git a/src/secp256k1/Makefile.am b/src/secp256k1/Makefile.am index 58c9635e53..1e03560884 100644 --- a/src/secp256k1/Makefile.am +++ b/src/secp256k1/Makefile.am @@ -1,5 +1,9 @@ ACLOCAL_AMFLAGS = -I build-aux/m4 +# AM_CFLAGS will be automatically prepended to CFLAGS by Automake when compiling some foo +# which does not have an explicit foo_CFLAGS variable set. +AM_CFLAGS = $(SECP_CFLAGS) + lib_LTLIBRARIES = libsecp256k1.la include_HEADERS = include/secp256k1.h include_HEADERS += include/secp256k1_preallocated.h @@ -68,7 +72,7 @@ endif endif libsecp256k1_la_SOURCES = src/secp256k1.c -libsecp256k1_la_CPPFLAGS = -DSECP256K1_BUILD -I$(top_srcdir)/include -I$(top_srcdir)/src $(SECP_INCLUDES) +libsecp256k1_la_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/src $(SECP_INCLUDES) libsecp256k1_la_LIBADD = $(SECP_LIBS) $(COMMON_LIB) if VALGRIND_ENABLED @@ -81,27 +85,27 @@ noinst_PROGRAMS += bench_verify bench_sign bench_internal bench_ecmult bench_verify_SOURCES = src/bench_verify.c bench_verify_LDADD = libsecp256k1.la $(SECP_LIBS) $(SECP_TEST_LIBS) $(COMMON_LIB) # SECP_TEST_INCLUDES are only used here for CRYPTO_CPPFLAGS -bench_verify_CPPFLAGS = -DSECP256K1_BUILD $(SECP_TEST_INCLUDES) +bench_verify_CPPFLAGS = $(SECP_TEST_INCLUDES) bench_sign_SOURCES = src/bench_sign.c bench_sign_LDADD = libsecp256k1.la $(SECP_LIBS) $(SECP_TEST_LIBS) $(COMMON_LIB) bench_internal_SOURCES = src/bench_internal.c bench_internal_LDADD = $(SECP_LIBS) $(COMMON_LIB) -bench_internal_CPPFLAGS = -DSECP256K1_BUILD $(SECP_INCLUDES) +bench_internal_CPPFLAGS = $(SECP_INCLUDES) bench_ecmult_SOURCES = src/bench_ecmult.c bench_ecmult_LDADD = $(SECP_LIBS) $(COMMON_LIB) -bench_ecmult_CPPFLAGS = -DSECP256K1_BUILD $(SECP_INCLUDES) +bench_ecmult_CPPFLAGS = $(SECP_INCLUDES) endif TESTS = if USE_TESTS noinst_PROGRAMS += tests tests_SOURCES = src/tests.c -tests_CPPFLAGS = -DSECP256K1_BUILD -I$(top_srcdir)/src -I$(top_srcdir)/include $(SECP_INCLUDES) $(SECP_TEST_INCLUDES) +tests_CPPFLAGS = -I$(top_srcdir)/src -I$(top_srcdir)/include $(SECP_INCLUDES) $(SECP_TEST_INCLUDES) if VALGRIND_ENABLED tests_CPPFLAGS += -DVALGRIND noinst_PROGRAMS += valgrind_ctime_test valgrind_ctime_test_SOURCES = src/valgrind_ctime_test.c -valgrind_ctime_test_LDADD = libsecp256k1.la $(SECP_LIBS) $(SECP_LIBS) $(COMMON_LIB) +valgrind_ctime_test_LDADD = libsecp256k1.la $(SECP_LIBS) $(COMMON_LIB) endif if !ENABLE_COVERAGE tests_CPPFLAGS += -DVERIFY @@ -114,7 +118,7 @@ endif if USE_EXHAUSTIVE_TESTS noinst_PROGRAMS += exhaustive_tests exhaustive_tests_SOURCES = src/tests_exhaustive.c -exhaustive_tests_CPPFLAGS = -DSECP256K1_BUILD -I$(top_srcdir)/src $(SECP_INCLUDES) +exhaustive_tests_CPPFLAGS = -I$(top_srcdir)/src $(SECP_INCLUDES) if !ENABLE_COVERAGE exhaustive_tests_CPPFLAGS += -DVERIFY endif @@ -129,10 +133,10 @@ CPPFLAGS_FOR_BUILD +=-I$(top_srcdir) -I$(builddir)/src gen_context_OBJECTS = gen_context.o gen_context_BIN = gen_context$(BUILD_EXEEXT) gen_%.o: src/gen_%.c src/libsecp256k1-config.h - $(CC_FOR_BUILD) $(CPPFLAGS_FOR_BUILD) $(CFLAGS_FOR_BUILD) -c $< -o $@ + $(CC_FOR_BUILD) $(DEFS) $(CPPFLAGS_FOR_BUILD) $(SECP_CFLAGS_FOR_BUILD) $(CFLAGS_FOR_BUILD) -c $< -o $@ $(gen_context_BIN): $(gen_context_OBJECTS) - $(CC_FOR_BUILD) $(CFLAGS_FOR_BUILD) $(LDFLAGS_FOR_BUILD) $^ -o $@ + $(CC_FOR_BUILD) $(SECP_CFLAGS_FOR_BUILD) $(CFLAGS_FOR_BUILD) $(LDFLAGS_FOR_BUILD) $^ -o $@ $(libsecp256k1_la_OBJECTS): src/ecmult_static_context.h $(tests_OBJECTS): src/ecmult_static_context.h diff --git a/src/secp256k1/README.md b/src/secp256k1/README.md index 197a56fff8..182c29d9ce 100644 --- a/src/secp256k1/README.md +++ b/src/secp256k1/README.md @@ -17,6 +17,7 @@ Features: * Suitable for embedded systems. * Optional module for public key recovery. * Optional module for ECDH key exchange. +* Optional module for Schnorr signatures according to [BIP-340](https://github.com/bitcoin/bips/blob/master/bip-0340.mediawiki) (experimental). Experimental features have not received enough scrutiny to satisfy the standard of quality of this library but are made available for testing and review by the community. The APIs of these features should not be considered stable. @@ -96,7 +97,8 @@ To create a report, `gcovr` is recommended, as it includes branch coverage repor To create a HTML report with coloured and annotated source code: - $ gcovr --exclude 'src/bench*' --html --html-details -o coverage.html + $ mkdir -p coverage + $ gcovr --exclude 'src/bench*' --html --html-details -o coverage/coverage.html Reporting a vulnerability ------------ diff --git a/src/secp256k1/build-aux/m4/bitcoin_secp.m4 b/src/secp256k1/build-aux/m4/bitcoin_secp.m4 index e57888ca18..8245b2b863 100644 --- a/src/secp256k1/build-aux/m4/bitcoin_secp.m4 +++ b/src/secp256k1/build-aux/m4/bitcoin_secp.m4 @@ -82,3 +82,19 @@ if test x"$has_valgrind" != x"yes"; then AC_CHECK_HEADER([valgrind/memcheck.h], [has_valgrind=yes; AC_DEFINE(HAVE_VALGRIND,1,[Define this symbol if valgrind is installed])]) fi ]) + +dnl SECP_TRY_APPEND_CFLAGS(flags, VAR) +dnl Append flags to VAR if CC accepts them. +AC_DEFUN([SECP_TRY_APPEND_CFLAGS], [ + AC_MSG_CHECKING([if ${CC} supports $1]) + SECP_TRY_APPEND_CFLAGS_saved_CFLAGS="$CFLAGS" + CFLAGS="$1 $CFLAGS" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([[char foo;]])], [flag_works=yes], [flag_works=no]) + AC_MSG_RESULT($flag_works) + CFLAGS="$SECP_TRY_APPEND_CFLAGS_saved_CFLAGS" + if test x"$flag_works" = x"yes"; then + $2="$$2 $1" + fi + unset flag_works + AC_SUBST($2) +]) diff --git a/src/secp256k1/ci/cirrus.sh b/src/secp256k1/ci/cirrus.sh index f26ca98d1d..27db1e6779 100755 --- a/src/secp256k1/ci/cirrus.sh +++ b/src/secp256k1/ci/cirrus.sh @@ -25,42 +25,27 @@ valgrind --version || true make # Print information about binaries so that we can see that the architecture is correct -file *tests || true +file *tests* || true file bench_* || true file .libs/* || true -if [ -n "$BUILD" ] -then - make "$BUILD" -fi +# This tells `make check` to wrap test invocations. +export LOG_COMPILER="$WRAPPER_CMD" -if [ "$RUN_VALGRIND" = "yes" ] -then - # the `--error-exitcode` is required to make the test fail if valgrind found errors, otherwise it'll return 0 (https://www.valgrind.org/docs/manual/manual-core.html) - valgrind --error-exitcode=42 ./tests 16 - valgrind --error-exitcode=42 ./exhaustive_tests -fi +# This limits the iterations in the tests and benchmarks. +export SECP256K1_TEST_ITERS="$TEST_ITERS" +export SECP256K1_BENCH_ITERS="$BENCH_ITERS" -if [ -n "$QEMU_CMD" ] -then - $QEMU_CMD ./tests 16 - $QEMU_CMD ./exhaustive_tests -fi +make "$BUILD" if [ "$BENCH" = "yes" ] then # Using the local `libtool` because on macOS the system's libtool has nothing to do with GNU libtool EXEC='./libtool --mode=execute' - if [ -n "$QEMU_CMD" ] - then - EXEC="$EXEC $QEMU_CMD" - fi - if [ "$RUN_VALGRIND" = "yes" ] + if [ -n "$WRAPPER_CMD" ] then - EXEC="$EXEC valgrind --error-exitcode=42" + EXEC="$EXEC $WRAPPER_CMD" fi - # This limits the iterations in the benchmarks below to ITER iterations. - export SECP256K1_BENCH_ITERS="$ITERS" { $EXEC ./bench_ecmult $EXEC ./bench_internal diff --git a/src/secp256k1/ci/linux-debian.Dockerfile b/src/secp256k1/ci/linux-debian.Dockerfile index 5967cf8b31..2c02ed69d0 100644 --- a/src/secp256k1/ci/linux-debian.Dockerfile +++ b/src/secp256k1/ci/linux-debian.Dockerfile @@ -2,12 +2,24 @@ FROM debian:stable RUN dpkg --add-architecture i386 RUN dpkg --add-architecture s390x +RUN dpkg --add-architecture armhf +RUN dpkg --add-architecture arm64 +RUN dpkg --add-architecture ppc64el RUN apt-get update # dkpg-dev: to make pkg-config work in cross-builds +# llvm: for llvm-symbolizer, which is used by clang's UBSan for symbolized stack traces RUN apt-get install --no-install-recommends --no-upgrade -y \ git ca-certificates \ make automake libtool pkg-config dpkg-dev valgrind qemu-user \ - gcc clang libc6-dbg \ - gcc-i686-linux-gnu libc6-dev-i386-cross libc6-dbg:i386 \ - gcc-s390x-linux-gnu libc6-dev-s390x-cross libc6-dbg:s390x + gcc clang llvm libc6-dbg \ + g++ \ + gcc-i686-linux-gnu libc6-dev-i386-cross libc6-dbg:i386 libubsan1:i386 libasan5:i386 \ + gcc-s390x-linux-gnu libc6-dev-s390x-cross libc6-dbg:s390x \ + gcc-arm-linux-gnueabihf libc6-dev-armhf-cross libc6-dbg:armhf \ + gcc-aarch64-linux-gnu libc6-dev-arm64-cross libc6-dbg:arm64 \ + gcc-powerpc64le-linux-gnu libc6-dev-ppc64el-cross libc6-dbg:ppc64el \ + wine gcc-mingw-w64-x86-64 + +# Run a dummy command in wine to make it set up configuration +RUN wine64-stable xcopy || true diff --git a/src/secp256k1/configure.ac b/src/secp256k1/configure.ac index 1ed991afa7..9969cfa343 100644 --- a/src/secp256k1/configure.ac +++ b/src/secp256k1/configure.ac @@ -8,10 +8,6 @@ AH_TOP([#define LIBSECP256K1_CONFIG_H]) AH_BOTTOM([#endif /*LIBSECP256K1_CONFIG_H*/]) AM_INIT_AUTOMAKE([foreign subdir-objects]) -# Set -g if CFLAGS are not already set, which matches the default autoconf -# behavior (see PROG_CC in the Autoconf manual) with the exception that we don't -# set -O2 here because we set it in any case (see further down). -: ${CFLAGS="-g"} LT_INIT # Make the compilation flags quiet unless V=1 is used. @@ -42,8 +38,8 @@ AM_PROG_AS case $host_os in *darwin*) if test x$cross_compiling != xyes; then - AC_PATH_PROG([BREW],brew,) - if test x$BREW != x; then + AC_CHECK_PROG([BREW], brew, brew) + if test x$BREW = xbrew; then # These Homebrew packages may be keg-only, meaning that they won't be found # in expected paths because they may conflict with system files. Ask # Homebrew where each one is located, then adjust paths accordingly. @@ -58,10 +54,10 @@ case $host_os in VALGRIND_CPPFLAGS="-I$valgrind_prefix/include" fi else - AC_PATH_PROG([PORT],port,) + AC_CHECK_PROG([PORT], port, port) # If homebrew isn't installed and macports is, add the macports default paths # as a last resort. - if test x$PORT != x; then + if test x$PORT = xport; then CPPFLAGS="$CPPFLAGS -isystem /opt/local/include" LDFLAGS="$LDFLAGS -L/opt/local/lib" fi @@ -70,35 +66,41 @@ case $host_os in ;; esac -CFLAGS="-W $CFLAGS" - -warn_CFLAGS="-std=c89 -pedantic -Wall -Wextra -Wcast-align -Wnested-externs -Wshadow -Wstrict-prototypes -Wundef -Wno-unused-function -Wno-long-long -Wno-overlength-strings" -saved_CFLAGS="$CFLAGS" -CFLAGS="$warn_CFLAGS $CFLAGS" -AC_MSG_CHECKING([if ${CC} supports ${warn_CFLAGS}]) -AC_COMPILE_IFELSE([AC_LANG_SOURCE([[char foo;]])], - [ AC_MSG_RESULT([yes]) ], - [ AC_MSG_RESULT([no]) - CFLAGS="$saved_CFLAGS" - ]) - -saved_CFLAGS="$CFLAGS" -CFLAGS="-Wconditional-uninitialized $CFLAGS" -AC_MSG_CHECKING([if ${CC} supports -Wconditional-uninitialized]) -AC_COMPILE_IFELSE([AC_LANG_SOURCE([[char foo;]])], - [ AC_MSG_RESULT([yes]) ], - [ AC_MSG_RESULT([no]) - CFLAGS="$saved_CFLAGS" - ]) - -saved_CFLAGS="$CFLAGS" -CFLAGS="-fvisibility=hidden $CFLAGS" -AC_MSG_CHECKING([if ${CC} supports -fvisibility=hidden]) -AC_COMPILE_IFELSE([AC_LANG_SOURCE([[char foo;]])], - [ AC_MSG_RESULT([yes]) ], - [ AC_MSG_RESULT([no]) - CFLAGS="$saved_CFLAGS" - ]) +# Try if some desirable compiler flags are supported and append them to SECP_CFLAGS. +# +# These are our own flags, so we append them to our own SECP_CFLAGS variable (instead of CFLAGS) as +# recommended in the automake manual (Section "Flag Variables Ordering"). CFLAGS belongs to the user +# and we are not supposed to touch it. In the Makefile, we will need to ensure that SECP_CFLAGS +# is prepended to CFLAGS when invoking the compiler so that the user always has the last word (flag). +# +# Another advantage of not touching CFLAGS is that the contents of CFLAGS will be picked up by +# libtool for compiling helper executables. For example, when compiling for Windows, libtool will +# generate entire wrapper executables (instead of simple wrapper scripts as on Unix) to ensure +# proper operation of uninstalled programs linked by libtool against the uninstalled shared library. +# These executables are compiled from C source file for which our flags may not be appropriate, +# e.g., -std=c89 flag has lead to undesirable warnings in the past. +# +# TODO We should analogously not touch CPPFLAGS and LDFLAGS but currently there are no issues. +AC_DEFUN([SECP_TRY_APPEND_DEFAULT_CFLAGS], [ + # Try to append -Werror=unknown-warning-option to CFLAGS temporarily. Otherwise clang will + # not error out if it gets unknown warning flags and the checks here will always succeed + # no matter if clang knows the flag or not. + SECP_TRY_APPEND_DEFAULT_CFLAGS_saved_CFLAGS="$CFLAGS" + SECP_TRY_APPEND_CFLAGS([-Werror=unknown-warning-option], CFLAGS) + + SECP_TRY_APPEND_CFLAGS([-std=c89 -pedantic -Wno-long-long -Wnested-externs -Wshadow -Wstrict-prototypes -Wundef], $1) # GCC >= 3.0, -Wlong-long is implied by -pedantic. + SECP_TRY_APPEND_CFLAGS([-Wno-overlength-strings], $1) # GCC >= 4.2, -Woverlength-strings is implied by -pedantic. + SECP_TRY_APPEND_CFLAGS([-Wall], $1) # GCC >= 2.95 and probably many other compilers + SECP_TRY_APPEND_CFLAGS([-Wno-unused-function], $1) # GCC >= 3.0, -Wunused-function is implied by -Wall. + SECP_TRY_APPEND_CFLAGS([-Wextra], $1) # GCC >= 3.4, this is the newer name of -W, which we don't use because older GCCs will warn about unused functions. + SECP_TRY_APPEND_CFLAGS([-Wcast-align], $1) # GCC >= 2.95 + SECP_TRY_APPEND_CFLAGS([-Wcast-align=strict], $1) # GCC >= 8.0 + SECP_TRY_APPEND_CFLAGS([-Wconditional-uninitialized], $1) # Clang >= 3.0 only + SECP_TRY_APPEND_CFLAGS([-fvisibility=hidden], $1) # GCC >= 4.0 + + CFLAGS="$SECP_TRY_APPEND_DEFAULT_CFLAGS_saved_CFLAGS" +]) +SECP_TRY_APPEND_DEFAULT_CFLAGS(SECP_CFLAGS) ### ### Define config arguments @@ -213,10 +215,14 @@ AM_CONDITIONAL([VALGRIND_ENABLED],[test "$enable_valgrind" = "yes"]) if test x"$enable_coverage" = x"yes"; then AC_DEFINE(COVERAGE, 1, [Define this symbol to compile out all VERIFY code]) - CFLAGS="-O0 --coverage $CFLAGS" + SECP_CFLAGS="-O0 --coverage $SECP_CFLAGS" LDFLAGS="--coverage $LDFLAGS" else - CFLAGS="-O2 $CFLAGS" + # Most likely the CFLAGS already contain -O2 because that is autoconf's default. + # We still add it here because passing it twice is not an issue, and handling + # this case would just add unnecessary complexity (see #896). + SECP_CFLAGS="-O2 $SECP_CFLAGS" + SECP_CFLAGS_FOR_BUILD="-O2 $SECP_CFLAGS_FOR_BUILD" fi if test x"$req_asm" = x"auto"; then @@ -351,6 +357,9 @@ if test x"$enable_valgrind" = x"yes"; then SECP_INCLUDES="$SECP_INCLUDES $VALGRIND_CPPFLAGS" fi +# Add -Werror and similar flags passed from the outside (for testing, e.g., in CI) +SECP_CFLAGS="$SECP_CFLAGS $WERROR_CFLAGS" + # Handle static precomputation (after everything which modifies CFLAGS and friends) if test x"$use_ecmult_static_precomputation" != x"no"; then if test x"$cross_compiling" = x"no"; then @@ -360,8 +369,9 @@ if test x"$use_ecmult_static_precomputation" != x"no"; then fi # If we're not cross-compiling, simply use the same compiler for building the static precompation code. CC_FOR_BUILD="$CC" - CFLAGS_FOR_BUILD="$CFLAGS" CPPFLAGS_FOR_BUILD="$CPPFLAGS" + SECP_CFLAGS_FOR_BUILD="$SECP_CFLAGS" + CFLAGS_FOR_BUILD="$CFLAGS" LDFLAGS_FOR_BUILD="$LDFLAGS" else AX_PROG_CC_FOR_BUILD @@ -371,22 +381,14 @@ if test x"$use_ecmult_static_precomputation" != x"no"; then cross_compiling=no SAVE_CC="$CC" CC="$CC_FOR_BUILD" - SAVE_CFLAGS="$CFLAGS" - CFLAGS="$CFLAGS_FOR_BUILD" SAVE_CPPFLAGS="$CPPFLAGS" CPPFLAGS="$CPPFLAGS_FOR_BUILD" + SAVE_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS_FOR_BUILD" SAVE_LDFLAGS="$LDFLAGS" LDFLAGS="$LDFLAGS_FOR_BUILD" - warn_CFLAGS_FOR_BUILD="-Wall -Wextra -Wno-unused-function" - saved_CFLAGS="$CFLAGS" - CFLAGS="$warn_CFLAGS_FOR_BUILD $CFLAGS" - AC_MSG_CHECKING([if native ${CC_FOR_BUILD} supports ${warn_CFLAGS_FOR_BUILD}]) - AC_COMPILE_IFELSE([AC_LANG_SOURCE([[char foo;]])], - [ AC_MSG_RESULT([yes]) ], - [ AC_MSG_RESULT([no]) - CFLAGS="$saved_CFLAGS" - ]) + SECP_TRY_APPEND_DEFAULT_CFLAGS(SECP_CFLAGS_FOR_BUILD) AC_MSG_CHECKING([for working native compiler: ${CC_FOR_BUILD}]) AC_RUN_IFELSE( @@ -394,19 +396,17 @@ if test x"$use_ecmult_static_precomputation" != x"no"; then [working_native_cc=yes], [working_native_cc=no],[:]) - CFLAGS_FOR_BUILD="$CFLAGS" - # Restore the environment cross_compiling=$save_cross_compiling CC="$SAVE_CC" - CFLAGS="$SAVE_CFLAGS" CPPFLAGS="$SAVE_CPPFLAGS" + CFLAGS="$SAVE_CFLAGS" LDFLAGS="$SAVE_LDFLAGS" if test x"$working_native_cc" = x"no"; then AC_MSG_RESULT([no]) set_precomp=no - m4_define([please_set_for_build], [Please set CC_FOR_BUILD, CFLAGS_FOR_BUILD, CPPFLAGS_FOR_BUILD, and/or LDFLAGS_FOR_BUILD.]) + m4_define([please_set_for_build], [Please set CC_FOR_BUILD, CPPFLAGS_FOR_BUILD, CFLAGS_FOR_BUILD, and/or LDFLAGS_FOR_BUILD.]) if test x"$use_ecmult_static_precomputation" = x"yes"; then AC_MSG_ERROR([native compiler ${CC_FOR_BUILD} does not produce working binaries. please_set_for_build]) else @@ -419,8 +419,9 @@ if test x"$use_ecmult_static_precomputation" != x"no"; then fi AC_SUBST(CC_FOR_BUILD) - AC_SUBST(CFLAGS_FOR_BUILD) AC_SUBST(CPPFLAGS_FOR_BUILD) + AC_SUBST(SECP_CFLAGS_FOR_BUILD) + AC_SUBST(CFLAGS_FOR_BUILD) AC_SUBST(LDFLAGS_FOR_BUILD) else set_precomp=no @@ -490,6 +491,7 @@ AC_SUBST(SECP_INCLUDES) AC_SUBST(SECP_LIBS) AC_SUBST(SECP_TEST_LIBS) AC_SUBST(SECP_TEST_INCLUDES) +AC_SUBST(SECP_CFLAGS) AM_CONDITIONAL([ENABLE_COVERAGE], [test x"$enable_coverage" = x"yes"]) AM_CONDITIONAL([USE_TESTS], [test x"$use_tests" != x"no"]) AM_CONDITIONAL([USE_EXHAUSTIVE_TESTS], [test x"$use_exhaustive_tests" != x"no"]) @@ -532,13 +534,15 @@ fi echo echo " valgrind = $enable_valgrind" echo " CC = $CC" -echo " CFLAGS = $CFLAGS" echo " CPPFLAGS = $CPPFLAGS" +echo " SECP_CFLAGS = $SECP_CFLAGS" +echo " CFLAGS = $CFLAGS" echo " LDFLAGS = $LDFLAGS" echo if test x"$set_precomp" = x"yes"; then echo " CC_FOR_BUILD = $CC_FOR_BUILD" -echo " CFLAGS_FOR_BUILD = $CFLAGS_FOR_BUILD" echo " CPPFLAGS_FOR_BUILD = $CPPFLAGS_FOR_BUILD" +echo " SECP_CFLAGS_FOR_BUILD = $SECP_CFLAGS_FOR_BUILD" +echo " CFLAGS_FOR_BUILD = $CFLAGS_FOR_BUILD" echo " LDFLAGS_FOR_BUILD = $LDFLAGS_FOR_BUILD" fi diff --git a/src/secp256k1/contrib/lax_der_parsing.c b/src/secp256k1/contrib/lax_der_parsing.c index c1627e37e9..bf562303ed 100644 --- a/src/secp256k1/contrib/lax_der_parsing.c +++ b/src/secp256k1/contrib/lax_der_parsing.c @@ -5,7 +5,6 @@ ***********************************************************************/ #include <string.h> -#include <secp256k1.h> #include "lax_der_parsing.h" @@ -121,7 +120,7 @@ int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1_ecdsa_ /* Copy R value */ if (rlen > 32) { overflow = 1; - } else { + } else if (rlen) { memcpy(tmpsig + 32 - rlen, input + rpos, rlen); } @@ -133,7 +132,7 @@ int ecdsa_signature_parse_der_lax(const secp256k1_context* ctx, secp256k1_ecdsa_ /* Copy S value */ if (slen > 32) { overflow = 1; - } else { + } else if (slen) { memcpy(tmpsig + 64 - slen, input + spos, slen); } diff --git a/src/secp256k1/contrib/lax_der_parsing.h b/src/secp256k1/contrib/lax_der_parsing.h index 6b7255e28f..034a38e6a0 100644 --- a/src/secp256k1/contrib/lax_der_parsing.h +++ b/src/secp256k1/contrib/lax_der_parsing.h @@ -51,7 +51,13 @@ #ifndef SECP256K1_CONTRIB_LAX_DER_PARSING_H #define SECP256K1_CONTRIB_LAX_DER_PARSING_H +/* #include secp256k1.h only when it hasn't been included yet. + This enables this file to be #included directly in other project + files (such as tests.c) without the need to set an explicit -I flag, + which would be necessary to locate secp256k1.h. */ +#ifndef SECP256K1_H #include <secp256k1.h> +#endif #ifdef __cplusplus extern "C" { diff --git a/src/secp256k1/contrib/lax_der_privatekey_parsing.c b/src/secp256k1/contrib/lax_der_privatekey_parsing.c index 429760fbb6..a1b8200079 100644 --- a/src/secp256k1/contrib/lax_der_privatekey_parsing.c +++ b/src/secp256k1/contrib/lax_der_privatekey_parsing.c @@ -5,7 +5,6 @@ ***********************************************************************/ #include <string.h> -#include <secp256k1.h> #include "lax_der_privatekey_parsing.h" @@ -45,7 +44,7 @@ int ec_privkey_import_der(const secp256k1_context* ctx, unsigned char *out32, co if (end < privkey+2 || privkey[0] != 0x04 || privkey[1] > 0x20 || end < privkey+2+privkey[1]) { return 0; } - memcpy(out32 + 32 - privkey[1], privkey + 2, privkey[1]); + if (privkey[1]) memcpy(out32 + 32 - privkey[1], privkey + 2, privkey[1]); if (!secp256k1_ec_seckey_verify(ctx, out32)) { memset(out32, 0, 32); return 0; diff --git a/src/secp256k1/contrib/lax_der_privatekey_parsing.h b/src/secp256k1/contrib/lax_der_privatekey_parsing.h index 602c7c556a..1a8ad8ae0c 100644 --- a/src/secp256k1/contrib/lax_der_privatekey_parsing.h +++ b/src/secp256k1/contrib/lax_der_privatekey_parsing.h @@ -28,7 +28,13 @@ #ifndef SECP256K1_CONTRIB_BER_PRIVATEKEY_H #define SECP256K1_CONTRIB_BER_PRIVATEKEY_H +/* #include secp256k1.h only when it hasn't been included yet. + This enables this file to be #included directly in other project + files (such as tests.c) without the need to set an explicit -I flag, + which would be necessary to locate secp256k1.h. */ +#ifndef SECP256K1_H #include <secp256k1.h> +#endif #ifdef __cplusplus extern "C" { diff --git a/src/secp256k1/include/secp256k1.h b/src/secp256k1/include/secp256k1.h index d368488af2..7be7fd5723 100644 --- a/src/secp256k1/include/secp256k1.h +++ b/src/secp256k1/include/secp256k1.h @@ -7,7 +7,9 @@ extern "C" { #include <stddef.h> -/* These rules specify the order of arguments in API calls: +/* Unless explicitly stated all pointer arguments must not be NULL. + * + * The following rules specify the order of arguments in API calls: * * 1. Context pointers go first, followed by output arguments, combined * output/input arguments, and finally input-only arguments. @@ -61,8 +63,9 @@ typedef struct secp256k1_scratch_space_struct secp256k1_scratch_space; * The exact representation of data inside is implementation defined and not * guaranteed to be portable between different platforms or versions. It is * however guaranteed to be 64 bytes in size, and can be safely copied/moved. - * If you need to convert to a format suitable for storage, transmission, or - * comparison, use secp256k1_ec_pubkey_serialize and secp256k1_ec_pubkey_parse. + * If you need to convert to a format suitable for storage or transmission, + * use secp256k1_ec_pubkey_serialize and secp256k1_ec_pubkey_parse. To + * compare keys, use secp256k1_ec_pubkey_cmp. */ typedef struct { unsigned char data[64]; @@ -127,6 +130,17 @@ typedef int (*secp256k1_nonce_function)( # define SECP256K1_INLINE inline # endif +/** When this header is used at build-time the SECP256K1_BUILD define needs to be set + * to correctly setup export attributes and nullness checks. This is normally done + * by secp256k1.c but to guard against this header being included before secp256k1.c + * has had a chance to set the define (e.g. via test harnesses that just includes + * secp256k1.c) we set SECP256K1_NO_BUILD when this header is processed without the + * BUILD define so this condition can be caught. + */ +#ifndef SECP256K1_BUILD +# define SECP256K1_NO_BUILD +#endif + #ifndef SECP256K1_API # if defined(_WIN32) # ifdef SECP256K1_BUILD @@ -370,6 +384,21 @@ SECP256K1_API int secp256k1_ec_pubkey_serialize( unsigned int flags ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); +/** Compare two public keys using lexicographic (of compressed serialization) order + * + * Returns: <0 if the first public key is less than the second + * >0 if the first public key is greater than the second + * 0 if the two public keys are equal + * Args: ctx: a secp256k1 context object. + * In: pubkey1: first public key to compare + * pubkey2: second public key to compare + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_cmp( + const secp256k1_context* ctx, + const secp256k1_pubkey* pubkey1, + const secp256k1_pubkey* pubkey2 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + /** Parse an ECDSA signature in compact (64 bytes) format. * * Returns: 1 when the signature could be parsed, 0 otherwise. @@ -764,6 +793,31 @@ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_ec_pubkey_combine( size_t n ) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); +/** Compute a tagged hash as defined in BIP-340. + * + * This is useful for creating a message hash and achieving domain separation + * through an application-specific tag. This function returns + * SHA256(SHA256(tag)||SHA256(tag)||msg). Therefore, tagged hash + * implementations optimized for a specific tag can precompute the SHA256 state + * after hashing the tag hashes. + * + * Returns 0 if the arguments are invalid and 1 otherwise. + * Args: ctx: pointer to a context object + * Out: hash32: pointer to a 32-byte array to store the resulting hash + * In: tag: pointer to an array containing the tag + * taglen: length of the tag array + * msg: pointer to an array containing the message + * msglen: length of the message array + */ +SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_tagged_sha256( + const secp256k1_context* ctx, + unsigned char *hash32, + const unsigned char *tag, + size_t taglen, + const unsigned char *msg, + size_t msglen +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(5); + #ifdef __cplusplus } #endif diff --git a/src/secp256k1/include/secp256k1_extrakeys.h b/src/secp256k1/include/secp256k1_extrakeys.h index 6fc7b290f8..0a37fb6b9d 100644 --- a/src/secp256k1/include/secp256k1_extrakeys.h +++ b/src/secp256k1/include/secp256k1_extrakeys.h @@ -15,9 +15,9 @@ extern "C" { * The exact representation of data inside is implementation defined and not * guaranteed to be portable between different platforms or versions. It is * however guaranteed to be 64 bytes in size, and can be safely copied/moved. - * If you need to convert to a format suitable for storage, transmission, or - * comparison, use secp256k1_xonly_pubkey_serialize and - * secp256k1_xonly_pubkey_parse. + * If you need to convert to a format suitable for storage, transmission, use + * use secp256k1_xonly_pubkey_serialize and secp256k1_xonly_pubkey_parse. To + * compare keys, use secp256k1_xonly_pubkey_cmp. */ typedef struct { unsigned char data[64]; @@ -67,6 +67,21 @@ SECP256K1_API int secp256k1_xonly_pubkey_serialize( const secp256k1_xonly_pubkey* pubkey ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); +/** Compare two x-only public keys using lexicographic order + * + * Returns: <0 if the first public key is less than the second + * >0 if the first public key is greater than the second + * 0 if the two public keys are equal + * Args: ctx: a secp256k1 context object. + * In: pubkey1: first public key to compare + * pubkey2: second public key to compare + */ +SECP256K1_API int secp256k1_xonly_pubkey_cmp( + const secp256k1_context* ctx, + const secp256k1_xonly_pubkey* pk1, + const secp256k1_xonly_pubkey* pk2 +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3); + /** Converts a secp256k1_pubkey into a secp256k1_xonly_pubkey. * * Returns: 1 if the public key was successfully converted diff --git a/src/secp256k1/include/secp256k1_schnorrsig.h b/src/secp256k1/include/secp256k1_schnorrsig.h index 0150cd3395..74cbcac45e 100644 --- a/src/secp256k1/include/secp256k1_schnorrsig.h +++ b/src/secp256k1/include/secp256k1_schnorrsig.h @@ -23,24 +23,29 @@ extern "C" { * * Returns: 1 if a nonce was successfully generated. 0 will cause signing to * return an error. - * Out: nonce32: pointer to a 32-byte array to be filled by the function. - * In: msg32: the 32-byte message hash being verified (will not be NULL) - * key32: pointer to a 32-byte secret key (will not be NULL) - * xonly_pk32: the 32-byte serialized xonly pubkey corresponding to key32 - * (will not be NULL) - * algo16: pointer to a 16-byte array describing the signature - * algorithm (will not be NULL). - * data: Arbitrary data pointer that is passed through. + * Out: nonce32: pointer to a 32-byte array to be filled by the function + * In: msg: the message being verified. Is NULL if and only if msglen + * is 0. + * msglen: the length of the message + * key32: pointer to a 32-byte secret key (will not be NULL) + * xonly_pk32: the 32-byte serialized xonly pubkey corresponding to key32 + * (will not be NULL) + * algo: pointer to an array describing the signature + * algorithm (will not be NULL) + * algolen: the length of the algo array + * data: arbitrary data pointer that is passed through * * Except for test cases, this function should compute some cryptographic hash of * the message, the key, the pubkey, the algorithm description, and data. */ typedef int (*secp256k1_nonce_function_hardened)( unsigned char *nonce32, - const unsigned char *msg32, + const unsigned char *msg, + size_t msglen, const unsigned char *key32, const unsigned char *xonly_pk32, - const unsigned char *algo16, + const unsigned char *algo, + size_t algolen, void *data ); @@ -50,59 +55,113 @@ typedef int (*secp256k1_nonce_function_hardened)( * * If a data pointer is passed, it is assumed to be a pointer to 32 bytes of * auxiliary random data as defined in BIP-340. If the data pointer is NULL, - * schnorrsig_sign does not produce BIP-340 compliant signatures. The algo16 - * argument must be non-NULL, otherwise the function will fail and return 0. - * The hash will be tagged with algo16 after removing all terminating null - * bytes. Therefore, to create BIP-340 compliant signatures, algo16 must be set - * to "BIP0340/nonce\0\0\0" + * the nonce derivation procedure follows BIP-340 by setting the auxiliary + * random data to zero. The algo argument must be non-NULL, otherwise the + * function will fail and return 0. The hash will be tagged with algo. + * Therefore, to create BIP-340 compliant signatures, algo must be set to + * "BIP0340/nonce" and algolen to 13. */ SECP256K1_API extern const secp256k1_nonce_function_hardened secp256k1_nonce_function_bip340; +/** Data structure that contains additional arguments for schnorrsig_sign_custom. + * + * A schnorrsig_extraparams structure object can be initialized correctly by + * setting it to SECP256K1_SCHNORRSIG_EXTRAPARAMS_INIT. + * + * Members: + * magic: set to SECP256K1_SCHNORRSIG_EXTRAPARAMS_MAGIC at initialization + * and has no other function than making sure the object is + * initialized. + * noncefp: pointer to a nonce generation function. If NULL, + * secp256k1_nonce_function_bip340 is used + * ndata: pointer to arbitrary data used by the nonce generation function + * (can be NULL). If it is non-NULL and + * secp256k1_nonce_function_bip340 is used, then ndata must be a + * pointer to 32-byte auxiliary randomness as per BIP-340. + */ +typedef struct { + unsigned char magic[4]; + secp256k1_nonce_function_hardened noncefp; + void* ndata; +} secp256k1_schnorrsig_extraparams; + +#define SECP256K1_SCHNORRSIG_EXTRAPARAMS_MAGIC { 0xda, 0x6f, 0xb3, 0x8c } +#define SECP256K1_SCHNORRSIG_EXTRAPARAMS_INIT {\ + SECP256K1_SCHNORRSIG_EXTRAPARAMS_MAGIC,\ + NULL,\ + NULL\ +} + /** Create a Schnorr signature. * * Does _not_ strictly follow BIP-340 because it does not verify the resulting * signature. Instead, you can manually use secp256k1_schnorrsig_verify and * abort if it fails. * - * Otherwise BIP-340 compliant if the noncefp argument is NULL or - * secp256k1_nonce_function_bip340 and the ndata argument is 32-byte auxiliary - * randomness. + * This function only signs 32-byte messages. If you have messages of a + * different size (or the same size but without a context-specific tag + * prefix), it is recommended to create a 32-byte message hash with + * secp256k1_tagged_sha256 and then sign the hash. Tagged hashing allows + * providing an context-specific tag for domain separation. This prevents + * signatures from being valid in multiple contexts by accident. * * Returns 1 on success, 0 on failure. * Args: ctx: pointer to a context object, initialized for signing (cannot be NULL) * Out: sig64: pointer to a 64-byte array to store the serialized signature (cannot be NULL) * In: msg32: the 32-byte message being signed (cannot be NULL) * keypair: pointer to an initialized keypair (cannot be NULL) - * noncefp: pointer to a nonce generation function. If NULL, secp256k1_nonce_function_bip340 is used - * ndata: pointer to arbitrary data used by the nonce generation - * function (can be NULL). If it is non-NULL and - * secp256k1_nonce_function_bip340 is used, then ndata must be a - * pointer to 32-byte auxiliary randomness as per BIP-340. + * aux_rand32: 32 bytes of fresh randomness. While recommended to provide + * this, it is only supplemental to security and can be NULL. See + * BIP-340 "Default Signing" for a full explanation of this + * argument and for guidance if randomness is expensive. */ SECP256K1_API int secp256k1_schnorrsig_sign( const secp256k1_context* ctx, unsigned char *sig64, const unsigned char *msg32, const secp256k1_keypair *keypair, - secp256k1_nonce_function_hardened noncefp, - void *ndata + unsigned char *aux_rand32 ) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); +/** Create a Schnorr signature with a more flexible API. + * + * Same arguments as secp256k1_schnorrsig_sign except that it allows signing + * variable length messages and accepts a pointer to an extraparams object that + * allows customizing signing by passing additional arguments. + * + * Creates the same signatures as schnorrsig_sign if msglen is 32 and the + * extraparams.ndata is the same as aux_rand32. + * + * In: msg: the message being signed. Can only be NULL if msglen is 0. + * msglen: length of the message + * extraparams: pointer to a extraparams object (can be NULL) + */ +SECP256K1_API int secp256k1_schnorrsig_sign_custom( + const secp256k1_context* ctx, + unsigned char *sig64, + const unsigned char *msg, + size_t msglen, + const secp256k1_keypair *keypair, + secp256k1_schnorrsig_extraparams *extraparams +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(5); + /** Verify a Schnorr signature. * * Returns: 1: correct signature * 0: incorrect signature * Args: ctx: a secp256k1 context object, initialized for verification. * In: sig64: pointer to the 64-byte signature to verify (cannot be NULL) - * msg32: the 32-byte message being verified (cannot be NULL) + * msg: the message being verified. Can only be NULL if msglen is 0. + * msglen: length of the message * pubkey: pointer to an x-only public key to verify with (cannot be NULL) */ SECP256K1_API SECP256K1_WARN_UNUSED_RESULT int secp256k1_schnorrsig_verify( const secp256k1_context* ctx, const unsigned char *sig64, - const unsigned char *msg32, + const unsigned char *msg, + size_t msglen, const secp256k1_xonly_pubkey *pubkey -) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(3) SECP256K1_ARG_NONNULL(4); +) SECP256K1_ARG_NONNULL(1) SECP256K1_ARG_NONNULL(2) SECP256K1_ARG_NONNULL(5); #ifdef __cplusplus } diff --git a/src/secp256k1/obj/.gitignore b/src/secp256k1/obj/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 --- a/src/secp256k1/obj/.gitignore +++ /dev/null diff --git a/src/secp256k1/src/bench_ecdh.c b/src/secp256k1/src/bench_ecdh.c index ab4b8f4244..cb020d26b4 100644 --- a/src/secp256k1/src/bench_ecdh.c +++ b/src/secp256k1/src/bench_ecdh.c @@ -6,8 +6,8 @@ #include <string.h> -#include "include/secp256k1.h" -#include "include/secp256k1_ecdh.h" +#include "../include/secp256k1.h" +#include "../include/secp256k1_ecdh.h" #include "util.h" #include "bench.h" diff --git a/src/secp256k1/src/bench_ecmult.c b/src/secp256k1/src/bench_ecmult.c index 204e85a5dd..1d463f92d0 100644 --- a/src/secp256k1/src/bench_ecmult.c +++ b/src/secp256k1/src/bench_ecmult.c @@ -5,7 +5,8 @@ ***********************************************************************/ #include <stdio.h> -#include "include/secp256k1.h" +#include "secp256k1.c" +#include "../include/secp256k1.h" #include "util.h" #include "hash_impl.h" @@ -14,33 +15,177 @@ #include "scalar_impl.h" #include "ecmult_impl.h" #include "bench.h" -#include "secp256k1.c" #define POINTS 32768 +void help(char **argv) { + printf("Benchmark EC multiplication algorithms\n"); + printf("\n"); + printf("Usage: %s <help|pippenger_wnaf|strauss_wnaf|simple>\n", argv[0]); + printf("The output shows the number of multiplied and summed points right after the\n"); + printf("function name. The letter 'g' indicates that one of the points is the generator.\n"); + printf("The benchmarks are divided by the number of points.\n"); + printf("\n"); + printf("default (ecmult_multi): picks pippenger_wnaf or strauss_wnaf depending on the\n"); + printf(" batch size\n"); + printf("pippenger_wnaf: for all batch sizes\n"); + printf("strauss_wnaf: for all batch sizes\n"); + printf("simple: multiply and sum each point individually\n"); +} + typedef struct { /* Setup once in advance */ secp256k1_context* ctx; secp256k1_scratch_space* scratch; secp256k1_scalar* scalars; secp256k1_ge* pubkeys; + secp256k1_gej* pubkeys_gej; secp256k1_scalar* seckeys; secp256k1_gej* expected_output; secp256k1_ecmult_multi_func ecmult_multi; - /* Changes per test */ + /* Changes per benchmark */ size_t count; int includes_g; - /* Changes per test iteration */ + /* Changes per benchmark iteration, used to pick different scalars and pubkeys + * in each run. */ size_t offset1; size_t offset2; - /* Test output. */ + /* Benchmark output. */ secp256k1_gej* output; } bench_data; -static int bench_callback(secp256k1_scalar* sc, secp256k1_ge* ge, size_t idx, void* arg) { +/* Hashes x into [0, POINTS) twice and store the result in offset1 and offset2. */ +static void hash_into_offset(bench_data* data, size_t x) { + data->offset1 = (x * 0x537b7f6f + 0x8f66a481) % POINTS; + data->offset2 = (x * 0x7f6f537b + 0x6a1a8f49) % POINTS; +} + +/* Check correctness of the benchmark by computing + * sum(outputs) ?= (sum(scalars_gen) + sum(seckeys)*sum(scalars))*G */ +static void bench_ecmult_teardown_helper(bench_data* data, size_t* seckey_offset, size_t* scalar_offset, size_t* scalar_gen_offset, int iters) { + int i; + secp256k1_gej sum_output, tmp; + secp256k1_scalar sum_scalars; + + secp256k1_gej_set_infinity(&sum_output); + secp256k1_scalar_clear(&sum_scalars); + for (i = 0; i < iters; ++i) { + secp256k1_gej_add_var(&sum_output, &sum_output, &data->output[i], NULL); + if (scalar_gen_offset != NULL) { + secp256k1_scalar_add(&sum_scalars, &sum_scalars, &data->scalars[(*scalar_gen_offset+i) % POINTS]); + } + if (seckey_offset != NULL) { + secp256k1_scalar s = data->seckeys[(*seckey_offset+i) % POINTS]; + secp256k1_scalar_mul(&s, &s, &data->scalars[(*scalar_offset+i) % POINTS]); + secp256k1_scalar_add(&sum_scalars, &sum_scalars, &s); + } + } + secp256k1_ecmult_gen(&data->ctx->ecmult_gen_ctx, &tmp, &sum_scalars); + secp256k1_gej_neg(&tmp, &tmp); + secp256k1_gej_add_var(&tmp, &tmp, &sum_output, NULL); + CHECK(secp256k1_gej_is_infinity(&tmp)); +} + +static void bench_ecmult_setup(void* arg) { + bench_data* data = (bench_data*)arg; + /* Re-randomize offset to ensure that we're using different scalars and + * group elements in each run. */ + hash_into_offset(data, data->offset1); +} + +static void bench_ecmult_gen(void* arg, int iters) { + bench_data* data = (bench_data*)arg; + int i; + + for (i = 0; i < iters; ++i) { + secp256k1_ecmult_gen(&data->ctx->ecmult_gen_ctx, &data->output[i], &data->scalars[(data->offset1+i) % POINTS]); + } +} + +static void bench_ecmult_gen_teardown(void* arg, int iters) { + bench_data* data = (bench_data*)arg; + bench_ecmult_teardown_helper(data, NULL, NULL, &data->offset1, iters); +} + +static void bench_ecmult_const(void* arg, int iters) { + bench_data* data = (bench_data*)arg; + int i; + + for (i = 0; i < iters; ++i) { + secp256k1_ecmult_const(&data->output[i], &data->pubkeys[(data->offset1+i) % POINTS], &data->scalars[(data->offset2+i) % POINTS], 256); + } +} + +static void bench_ecmult_const_teardown(void* arg, int iters) { + bench_data* data = (bench_data*)arg; + bench_ecmult_teardown_helper(data, &data->offset1, &data->offset2, NULL, iters); +} + +static void bench_ecmult_1(void* arg, int iters) { + bench_data* data = (bench_data*)arg; + int i; + + for (i = 0; i < iters; ++i) { + secp256k1_ecmult(&data->ctx->ecmult_ctx, &data->output[i], &data->pubkeys_gej[(data->offset1+i) % POINTS], &data->scalars[(data->offset2+i) % POINTS], NULL); + } +} + +static void bench_ecmult_1_teardown(void* arg, int iters) { + bench_data* data = (bench_data*)arg; + bench_ecmult_teardown_helper(data, &data->offset1, &data->offset2, NULL, iters); +} + +static void bench_ecmult_1g(void* arg, int iters) { + bench_data* data = (bench_data*)arg; + secp256k1_scalar zero; + int i; + + secp256k1_scalar_set_int(&zero, 0); + for (i = 0; i < iters; ++i) { + secp256k1_ecmult(&data->ctx->ecmult_ctx, &data->output[i], NULL, &zero, &data->scalars[(data->offset1+i) % POINTS]); + } +} + +static void bench_ecmult_1g_teardown(void* arg, int iters) { + bench_data* data = (bench_data*)arg; + bench_ecmult_teardown_helper(data, NULL, NULL, &data->offset1, iters); +} + +static void bench_ecmult_2g(void* arg, int iters) { + bench_data* data = (bench_data*)arg; + int i; + + for (i = 0; i < iters/2; ++i) { + secp256k1_ecmult(&data->ctx->ecmult_ctx, &data->output[i], &data->pubkeys_gej[(data->offset1+i) % POINTS], &data->scalars[(data->offset2+i) % POINTS], &data->scalars[(data->offset1+i) % POINTS]); + } +} + +static void bench_ecmult_2g_teardown(void* arg, int iters) { + bench_data* data = (bench_data*)arg; + bench_ecmult_teardown_helper(data, &data->offset1, &data->offset2, &data->offset1, iters/2); +} + +static void run_ecmult_bench(bench_data* data, int iters) { + char str[32]; + sprintf(str, "ecmult_gen"); + run_benchmark(str, bench_ecmult_gen, bench_ecmult_setup, bench_ecmult_gen_teardown, data, 10, iters); + sprintf(str, "ecmult_const"); + run_benchmark(str, bench_ecmult_const, bench_ecmult_setup, bench_ecmult_const_teardown, data, 10, iters); + /* ecmult with non generator point */ + sprintf(str, "ecmult 1"); + run_benchmark(str, bench_ecmult_1, bench_ecmult_setup, bench_ecmult_1_teardown, data, 10, iters); + /* ecmult with generator point */ + sprintf(str, "ecmult 1g"); + run_benchmark(str, bench_ecmult_1g, bench_ecmult_setup, bench_ecmult_1g_teardown, data, 10, iters); + /* ecmult with generator and non-generator point. The reported time is per point. */ + sprintf(str, "ecmult 2g"); + run_benchmark(str, bench_ecmult_2g, bench_ecmult_setup, bench_ecmult_2g_teardown, data, 10, 2*iters); +} + +static int bench_ecmult_multi_callback(secp256k1_scalar* sc, secp256k1_ge* ge, size_t idx, void* arg) { bench_data* data = (bench_data*)arg; if (data->includes_g) ++idx; if (idx == 0) { @@ -53,7 +198,7 @@ static int bench_callback(secp256k1_scalar* sc, secp256k1_ge* ge, size_t idx, vo return 1; } -static void bench_ecmult(void* arg, int iters) { +static void bench_ecmult_multi(void* arg, int iters) { bench_data* data = (bench_data*)arg; int includes_g = data->includes_g; @@ -62,19 +207,18 @@ static void bench_ecmult(void* arg, int iters) { iters = iters / data->count; for (iter = 0; iter < iters; ++iter) { - data->ecmult_multi(&data->ctx->error_callback, &data->ctx->ecmult_ctx, data->scratch, &data->output[iter], data->includes_g ? &data->scalars[data->offset1] : NULL, bench_callback, arg, count - includes_g); + data->ecmult_multi(&data->ctx->error_callback, &data->ctx->ecmult_ctx, data->scratch, &data->output[iter], data->includes_g ? &data->scalars[data->offset1] : NULL, bench_ecmult_multi_callback, arg, count - includes_g); data->offset1 = (data->offset1 + count) % POINTS; data->offset2 = (data->offset2 + count - 1) % POINTS; } } -static void bench_ecmult_setup(void* arg) { +static void bench_ecmult_multi_setup(void* arg) { bench_data* data = (bench_data*)arg; - data->offset1 = (data->count * 0x537b7f6f + 0x8f66a481) % POINTS; - data->offset2 = (data->count * 0x7f6f537b + 0x6a1a8f49) % POINTS; + hash_into_offset(data, data->count); } -static void bench_ecmult_teardown(void* arg, int iters) { +static void bench_ecmult_multi_teardown(void* arg, int iters) { bench_data* data = (bench_data*)arg; int iter; iters = iters / data->count; @@ -88,7 +232,7 @@ static void bench_ecmult_teardown(void* arg, int iters) { static void generate_scalar(uint32_t num, secp256k1_scalar* scalar) { secp256k1_sha256 sha256; - unsigned char c[11] = {'e', 'c', 'm', 'u', 'l', 't', 0, 0, 0, 0}; + unsigned char c[10] = {'e', 'c', 'm', 'u', 'l', 't', 0, 0, 0, 0}; unsigned char buf[32]; int overflow = 0; c[6] = num; @@ -102,7 +246,7 @@ static void generate_scalar(uint32_t num, secp256k1_scalar* scalar) { CHECK(!overflow); } -static void run_test(bench_data* data, size_t count, int includes_g, int num_iters) { +static void run_ecmult_multi_bench(bench_data* data, size_t count, int includes_g, int num_iters) { char str[32]; static const secp256k1_scalar zero = SECP256K1_SCALAR_CONST(0, 0, 0, 0, 0, 0, 0, 0); size_t iters = 1 + num_iters / count; @@ -112,8 +256,7 @@ static void run_test(bench_data* data, size_t count, int includes_g, int num_ite data->includes_g = includes_g; /* Compute (the negation of) the expected results directly. */ - data->offset1 = (data->count * 0x537b7f6f + 0x8f66a481) % POINTS; - data->offset2 = (data->count * 0x7f6f537b + 0x6a1a8f49) % POINTS; + hash_into_offset(data, data->count); for (iter = 0; iter < iters; ++iter) { secp256k1_scalar tmp; secp256k1_scalar total = data->scalars[(data->offset1++) % POINTS]; @@ -127,25 +270,26 @@ static void run_test(bench_data* data, size_t count, int includes_g, int num_ite } /* Run the benchmark. */ - sprintf(str, includes_g ? "ecmult_%ig" : "ecmult_%i", (int)count); - run_benchmark(str, bench_ecmult, bench_ecmult_setup, bench_ecmult_teardown, data, 10, count * iters); + sprintf(str, includes_g ? "ecmult_multi %ig" : "ecmult_multi %i", (int)count); + run_benchmark(str, bench_ecmult_multi, bench_ecmult_multi_setup, bench_ecmult_multi_teardown, data, 10, count * iters); } int main(int argc, char **argv) { bench_data data; int i, p; - secp256k1_gej* pubkeys_gej; size_t scratch_size; int iters = get_iters(10000); - data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); - scratch_size = secp256k1_strauss_scratch_size(POINTS) + STRAUSS_SCRATCH_OBJECTS*16; - data.scratch = secp256k1_scratch_space_create(data.ctx, scratch_size); data.ecmult_multi = secp256k1_ecmult_multi_var; if (argc > 1) { - if(have_flag(argc, argv, "pippenger_wnaf")) { + if(have_flag(argc, argv, "-h") + || have_flag(argc, argv, "--help") + || have_flag(argc, argv, "help")) { + help(argv); + return 1; + } else if(have_flag(argc, argv, "pippenger_wnaf")) { printf("Using pippenger_wnaf:\n"); data.ecmult_multi = secp256k1_ecmult_pippenger_batch_single; } else if(have_flag(argc, argv, "strauss_wnaf")) { @@ -153,39 +297,48 @@ int main(int argc, char **argv) { data.ecmult_multi = secp256k1_ecmult_strauss_batch_single; } else if(have_flag(argc, argv, "simple")) { printf("Using simple algorithm:\n"); - data.ecmult_multi = secp256k1_ecmult_multi_var; - secp256k1_scratch_space_destroy(data.ctx, data.scratch); - data.scratch = NULL; } else { - fprintf(stderr, "%s: unrecognized argument '%s'.\n", argv[0], argv[1]); - fprintf(stderr, "Use 'pippenger_wnaf', 'strauss_wnaf', 'simple' or no argument to benchmark a combined algorithm.\n"); + fprintf(stderr, "%s: unrecognized argument '%s'.\n\n", argv[0], argv[1]); + help(argv); return 1; } } + data.ctx = secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY); + scratch_size = secp256k1_strauss_scratch_size(POINTS) + STRAUSS_SCRATCH_OBJECTS*16; + if (!have_flag(argc, argv, "simple")) { + data.scratch = secp256k1_scratch_space_create(data.ctx, scratch_size); + } else { + data.scratch = NULL; + } + /* Allocate stuff */ data.scalars = malloc(sizeof(secp256k1_scalar) * POINTS); data.seckeys = malloc(sizeof(secp256k1_scalar) * POINTS); data.pubkeys = malloc(sizeof(secp256k1_ge) * POINTS); + data.pubkeys_gej = malloc(sizeof(secp256k1_gej) * POINTS); data.expected_output = malloc(sizeof(secp256k1_gej) * (iters + 1)); data.output = malloc(sizeof(secp256k1_gej) * (iters + 1)); /* Generate a set of scalars, and private/public keypairs. */ - pubkeys_gej = malloc(sizeof(secp256k1_gej) * POINTS); - secp256k1_gej_set_ge(&pubkeys_gej[0], &secp256k1_ge_const_g); + secp256k1_gej_set_ge(&data.pubkeys_gej[0], &secp256k1_ge_const_g); secp256k1_scalar_set_int(&data.seckeys[0], 1); for (i = 0; i < POINTS; ++i) { generate_scalar(i, &data.scalars[i]); if (i) { - secp256k1_gej_double_var(&pubkeys_gej[i], &pubkeys_gej[i - 1], NULL); + secp256k1_gej_double_var(&data.pubkeys_gej[i], &data.pubkeys_gej[i - 1], NULL); secp256k1_scalar_add(&data.seckeys[i], &data.seckeys[i - 1], &data.seckeys[i - 1]); } } - secp256k1_ge_set_all_gej_var(data.pubkeys, pubkeys_gej, POINTS); - free(pubkeys_gej); + secp256k1_ge_set_all_gej_var(data.pubkeys, data.pubkeys_gej, POINTS); + + + /* Initialize offset1 and offset2 */ + hash_into_offset(&data, 0); + run_ecmult_bench(&data, iters); for (i = 1; i <= 8; ++i) { - run_test(&data, i, 1, iters); + run_ecmult_multi_bench(&data, i, 1, iters); } /* This is disabled with low count of iterations because the loop runs 77 times even with iters=1 @@ -194,7 +347,7 @@ int main(int argc, char **argv) { if (iters > 2) { for (p = 0; p <= 11; ++p) { for (i = 9; i <= 16; ++i) { - run_test(&data, i << p, 1, iters); + run_ecmult_multi_bench(&data, i << p, 1, iters); } } } @@ -205,6 +358,7 @@ int main(int argc, char **argv) { secp256k1_context_destroy(data.ctx); free(data.scalars); free(data.pubkeys); + free(data.pubkeys_gej); free(data.seckeys); free(data.output); free(data.expected_output); diff --git a/src/secp256k1/src/bench_internal.c b/src/secp256k1/src/bench_internal.c index 73b8a24ccb..161b1c4a47 100644 --- a/src/secp256k1/src/bench_internal.c +++ b/src/secp256k1/src/bench_internal.c @@ -5,7 +5,8 @@ ***********************************************************************/ #include <stdio.h> -#include "include/secp256k1.h" +#include "secp256k1.c" +#include "../include/secp256k1.h" #include "assumptions.h" #include "util.h" @@ -16,7 +17,6 @@ #include "ecmult_const_impl.h" #include "ecmult_impl.h" #include "bench.h" -#include "secp256k1.c" typedef struct { secp256k1_scalar scalar[2]; diff --git a/src/secp256k1/src/bench_recover.c b/src/secp256k1/src/bench_recover.c index 3f6270ce84..4bcac19dc0 100644 --- a/src/secp256k1/src/bench_recover.c +++ b/src/secp256k1/src/bench_recover.c @@ -4,8 +4,8 @@ * file COPYING or https://www.opensource.org/licenses/mit-license.php.* ***********************************************************************/ -#include "include/secp256k1.h" -#include "include/secp256k1_recovery.h" +#include "../include/secp256k1.h" +#include "../include/secp256k1_recovery.h" #include "util.h" #include "bench.h" diff --git a/src/secp256k1/src/bench_schnorrsig.c b/src/secp256k1/src/bench_schnorrsig.c index f7f591c41d..d95bc00f48 100644 --- a/src/secp256k1/src/bench_schnorrsig.c +++ b/src/secp256k1/src/bench_schnorrsig.c @@ -8,11 +8,13 @@ #include <stdlib.h> -#include "include/secp256k1.h" -#include "include/secp256k1_schnorrsig.h" +#include "../include/secp256k1.h" +#include "../include/secp256k1_schnorrsig.h" #include "util.h" #include "bench.h" +#define MSGLEN 32 + typedef struct { secp256k1_context *ctx; int n; @@ -26,13 +28,13 @@ typedef struct { void bench_schnorrsig_sign(void* arg, int iters) { bench_schnorrsig_data *data = (bench_schnorrsig_data *)arg; int i; - unsigned char msg[32] = "benchmarkexamplemessagetemplate"; + unsigned char msg[MSGLEN] = {0}; unsigned char sig[64]; for (i = 0; i < iters; i++) { msg[0] = i; msg[1] = i >> 8; - CHECK(secp256k1_schnorrsig_sign(data->ctx, sig, msg, data->keypairs[i], NULL, NULL)); + CHECK(secp256k1_schnorrsig_sign_custom(data->ctx, sig, msg, MSGLEN, data->keypairs[i], NULL)); } } @@ -43,7 +45,7 @@ void bench_schnorrsig_verify(void* arg, int iters) { for (i = 0; i < iters; i++) { secp256k1_xonly_pubkey pk; CHECK(secp256k1_xonly_pubkey_parse(data->ctx, &pk, data->pk[i]) == 1); - CHECK(secp256k1_schnorrsig_verify(data->ctx, data->sigs[i], data->msgs[i], &pk)); + CHECK(secp256k1_schnorrsig_verify(data->ctx, data->sigs[i], data->msgs[i], MSGLEN, &pk)); } } @@ -58,9 +60,10 @@ int main(void) { data.msgs = (const unsigned char **)malloc(iters * sizeof(unsigned char *)); data.sigs = (const unsigned char **)malloc(iters * sizeof(unsigned char *)); + CHECK(MSGLEN >= 4); for (i = 0; i < iters; i++) { unsigned char sk[32]; - unsigned char *msg = (unsigned char *)malloc(32); + unsigned char *msg = (unsigned char *)malloc(MSGLEN); unsigned char *sig = (unsigned char *)malloc(64); secp256k1_keypair *keypair = (secp256k1_keypair *)malloc(sizeof(*keypair)); unsigned char *pk_char = (unsigned char *)malloc(32); @@ -69,7 +72,7 @@ int main(void) { msg[1] = sk[1] = i >> 8; msg[2] = sk[2] = i >> 16; msg[3] = sk[3] = i >> 24; - memset(&msg[4], 'm', 28); + memset(&msg[4], 'm', MSGLEN - 4); memset(&sk[4], 's', 28); data.keypairs[i] = keypair; @@ -78,7 +81,7 @@ int main(void) { data.sigs[i] = sig; CHECK(secp256k1_keypair_create(data.ctx, keypair, sk)); - CHECK(secp256k1_schnorrsig_sign(data.ctx, sig, msg, keypair, NULL, NULL)); + CHECK(secp256k1_schnorrsig_sign_custom(data.ctx, sig, msg, MSGLEN, keypair, NULL)); CHECK(secp256k1_keypair_xonly_pub(data.ctx, &pk, NULL, keypair)); CHECK(secp256k1_xonly_pubkey_serialize(data.ctx, pk_char, &pk) == 1); } diff --git a/src/secp256k1/src/bench_sign.c b/src/secp256k1/src/bench_sign.c index 933f367c4b..f659c18c92 100644 --- a/src/secp256k1/src/bench_sign.c +++ b/src/secp256k1/src/bench_sign.c @@ -4,7 +4,7 @@ * file COPYING or https://www.opensource.org/licenses/mit-license.php.* ***********************************************************************/ -#include "include/secp256k1.h" +#include "../include/secp256k1.h" #include "util.h" #include "bench.h" diff --git a/src/secp256k1/src/bench_verify.c b/src/secp256k1/src/bench_verify.c index c56aefd369..565ae4beec 100644 --- a/src/secp256k1/src/bench_verify.c +++ b/src/secp256k1/src/bench_verify.c @@ -7,7 +7,7 @@ #include <stdio.h> #include <string.h> -#include "include/secp256k1.h" +#include "../include/secp256k1.h" #include "util.h" #include "bench.h" diff --git a/src/secp256k1/src/ecdsa_impl.h b/src/secp256k1/src/ecdsa_impl.h index 156a33d112..c32141e887 100644 --- a/src/secp256k1/src/ecdsa_impl.h +++ b/src/secp256k1/src/ecdsa_impl.h @@ -140,7 +140,7 @@ static int secp256k1_der_parse_integer(secp256k1_scalar *r, const unsigned char overflow = 1; } if (!overflow) { - memcpy(ra + 32 - rlen, *sig, rlen); + if (rlen) memcpy(ra + 32 - rlen, *sig, rlen); secp256k1_scalar_set_b32(r, ra, &overflow); } if (overflow) { diff --git a/src/secp256k1/src/ecmult.h b/src/secp256k1/src/ecmult.h index 7ab617e20e..84537bbfed 100644 --- a/src/secp256k1/src/ecmult.h +++ b/src/secp256k1/src/ecmult.h @@ -17,7 +17,6 @@ typedef struct { secp256k1_ge_storage (*pre_g_128)[]; /* odd multiples of 2^128*generator */ } secp256k1_ecmult_context; -static const size_t SECP256K1_ECMULT_CONTEXT_PREALLOCATED_SIZE; static void secp256k1_ecmult_context_init(secp256k1_ecmult_context *ctx); static void secp256k1_ecmult_context_build(secp256k1_ecmult_context *ctx, void **prealloc); static void secp256k1_ecmult_context_finalize_memcpy(secp256k1_ecmult_context *dst, const secp256k1_ecmult_context *src); diff --git a/src/secp256k1/src/ecmult_gen.h b/src/secp256k1/src/ecmult_gen.h index 539618dcbb..05cf4d52cc 100644 --- a/src/secp256k1/src/ecmult_gen.h +++ b/src/secp256k1/src/ecmult_gen.h @@ -35,7 +35,6 @@ typedef struct { secp256k1_gej initial; } secp256k1_ecmult_gen_context; -static const size_t SECP256K1_ECMULT_GEN_CONTEXT_PREALLOCATED_SIZE; static void secp256k1_ecmult_gen_context_init(secp256k1_ecmult_gen_context* ctx); static void secp256k1_ecmult_gen_context_build(secp256k1_ecmult_gen_context* ctx, void **prealloc); static void secp256k1_ecmult_gen_context_finalize_memcpy(secp256k1_ecmult_gen_context *dst, const secp256k1_ecmult_gen_context* src); diff --git a/src/secp256k1/src/gen_context.c b/src/secp256k1/src/gen_context.c index 024c557261..f9176eb996 100644 --- a/src/secp256k1/src/gen_context.c +++ b/src/secp256k1/src/gen_context.c @@ -13,7 +13,13 @@ /* We can't require the precomputed tables when creating them. */ #undef USE_ECMULT_STATIC_PRECOMPUTATION -#include "include/secp256k1.h" +/* In principle we could use ASM, but this yields only a minor speedup in + build time and it's very complicated. In particular when cross-compiling, we'd + need to build the ASM for the build and the host machine. */ +#undef USE_EXTERNAL_ASM +#undef USE_ASM_X86_64 + +#include "../include/secp256k1.h" #include "assumptions.h" #include "util.h" #include "field_impl.h" diff --git a/src/secp256k1/src/group_impl.h b/src/secp256k1/src/group_impl.h index 19ebd8f44e..47aea32be1 100644 --- a/src/secp256k1/src/group_impl.h +++ b/src/secp256k1/src/group_impl.h @@ -100,8 +100,8 @@ static void secp256k1_ge_set_gej(secp256k1_ge *r, secp256k1_gej *a) { static void secp256k1_ge_set_gej_var(secp256k1_ge *r, secp256k1_gej *a) { secp256k1_fe z2, z3; - r->infinity = a->infinity; if (a->infinity) { + secp256k1_ge_set_infinity(r); return; } secp256k1_fe_inv_var(&a->z, &a->z); @@ -110,8 +110,7 @@ static void secp256k1_ge_set_gej_var(secp256k1_ge *r, secp256k1_gej *a) { secp256k1_fe_mul(&a->x, &a->x, &z2); secp256k1_fe_mul(&a->y, &a->y, &z3); secp256k1_fe_set_int(&a->z, 1); - r->x = a->x; - r->y = a->y; + secp256k1_ge_set_xy(r, &a->x, &a->y); } static void secp256k1_ge_set_all_gej_var(secp256k1_ge *r, const secp256k1_gej *a, size_t len) { @@ -120,7 +119,9 @@ static void secp256k1_ge_set_all_gej_var(secp256k1_ge *r, const secp256k1_gej *a size_t last_i = SIZE_MAX; for (i = 0; i < len; i++) { - if (!a[i].infinity) { + if (a[i].infinity) { + secp256k1_ge_set_infinity(&r[i]); + } else { /* Use destination's x coordinates as scratch space */ if (last_i == SIZE_MAX) { r[i].x = a[i].z; @@ -148,7 +149,6 @@ static void secp256k1_ge_set_all_gej_var(secp256k1_ge *r, const secp256k1_gej *a r[last_i].x = u; for (i = 0; i < len; i++) { - r[i].infinity = a[i].infinity; if (!a[i].infinity) { secp256k1_ge_set_gej_zinv(&r[i], &a[i], &r[i].x); } @@ -311,7 +311,7 @@ static void secp256k1_gej_double_var(secp256k1_gej *r, const secp256k1_gej *a, s * point will be gibberish (z = 0 but infinity = 0). */ if (a->infinity) { - r->infinity = 1; + secp256k1_gej_set_infinity(r); if (rzr != NULL) { secp256k1_fe_set_int(rzr, 1); } diff --git a/src/secp256k1/src/modules/ecdh/main_impl.h b/src/secp256k1/src/modules/ecdh/main_impl.h index 1ac67086be..5408c9de70 100644 --- a/src/secp256k1/src/modules/ecdh/main_impl.h +++ b/src/secp256k1/src/modules/ecdh/main_impl.h @@ -7,8 +7,8 @@ #ifndef SECP256K1_MODULE_ECDH_MAIN_H #define SECP256K1_MODULE_ECDH_MAIN_H -#include "include/secp256k1_ecdh.h" -#include "ecmult_const_impl.h" +#include "../../../include/secp256k1_ecdh.h" +#include "../../ecmult_const_impl.h" static int ecdh_hash_function_sha256(unsigned char *output, const unsigned char *x32, const unsigned char *y32, void *data) { unsigned char version = (y32[31] & 0x01) | 0x02; diff --git a/src/secp256k1/src/modules/extrakeys/main_impl.h b/src/secp256k1/src/modules/extrakeys/main_impl.h index 7390b22718..8607bbede7 100644 --- a/src/secp256k1/src/modules/extrakeys/main_impl.h +++ b/src/secp256k1/src/modules/extrakeys/main_impl.h @@ -7,8 +7,8 @@ #ifndef SECP256K1_MODULE_EXTRAKEYS_MAIN_H #define SECP256K1_MODULE_EXTRAKEYS_MAIN_H -#include "include/secp256k1.h" -#include "include/secp256k1_extrakeys.h" +#include "../../../include/secp256k1.h" +#include "../../../include/secp256k1_extrakeys.h" static SECP256K1_INLINE int secp256k1_xonly_pubkey_load(const secp256k1_context* ctx, secp256k1_ge *ge, const secp256k1_xonly_pubkey *pubkey) { return secp256k1_pubkey_load(ctx, ge, (const secp256k1_pubkey *) pubkey); @@ -55,6 +55,32 @@ int secp256k1_xonly_pubkey_serialize(const secp256k1_context* ctx, unsigned char return 1; } +int secp256k1_xonly_pubkey_cmp(const secp256k1_context* ctx, const secp256k1_xonly_pubkey* pk0, const secp256k1_xonly_pubkey* pk1) { + unsigned char out[2][32]; + const secp256k1_xonly_pubkey* pk[2]; + int i; + + VERIFY_CHECK(ctx != NULL); + pk[0] = pk0; pk[1] = pk1; + for (i = 0; i < 2; i++) { + /* If the public key is NULL or invalid, xonly_pubkey_serialize will + * call the illegal_callback and return 0. In that case we will + * serialize the key as all zeros which is less than any valid public + * key. This results in consistent comparisons even if NULL or invalid + * pubkeys are involved and prevents edge cases such as sorting + * algorithms that use this function and do not terminate as a + * result. */ + if (!secp256k1_xonly_pubkey_serialize(ctx, out[i], pk[i])) { + /* Note that xonly_pubkey_serialize should already set the output to + * zero in that case, but it's not guaranteed by the API, we can't + * test it and writing a VERIFY_CHECK is more complex than + * explicitly memsetting (again). */ + memset(out[i], 0, sizeof(out[i])); + } + } + return secp256k1_memcmp_var(out[0], out[1], sizeof(out[1])); +} + /** Keeps a group element as is if it has an even Y and otherwise negates it. * y_parity is set to 0 in the former case and to 1 in the latter case. * Requires that the coordinates of r are normalized. */ diff --git a/src/secp256k1/src/modules/extrakeys/tests_exhaustive_impl.h b/src/secp256k1/src/modules/extrakeys/tests_exhaustive_impl.h index 0aca4fb72d..d4a2f5bdf4 100644 --- a/src/secp256k1/src/modules/extrakeys/tests_exhaustive_impl.h +++ b/src/secp256k1/src/modules/extrakeys/tests_exhaustive_impl.h @@ -8,7 +8,7 @@ #define SECP256K1_MODULE_EXTRAKEYS_TESTS_EXHAUSTIVE_H #include "src/modules/extrakeys/main_impl.h" -#include "include/secp256k1_extrakeys.h" +#include "../../../include/secp256k1_extrakeys.h" static void test_exhaustive_extrakeys(const secp256k1_context *ctx, const secp256k1_ge* group) { secp256k1_keypair keypair[EXHAUSTIVE_TEST_ORDER - 1]; diff --git a/src/secp256k1/src/modules/extrakeys/tests_impl.h b/src/secp256k1/src/modules/extrakeys/tests_impl.h index 9473a7dd48..4a5952714c 100644 --- a/src/secp256k1/src/modules/extrakeys/tests_impl.h +++ b/src/secp256k1/src/modules/extrakeys/tests_impl.h @@ -7,7 +7,7 @@ #ifndef SECP256K1_MODULE_EXTRAKEYS_TESTS_H #define SECP256K1_MODULE_EXTRAKEYS_TESTS_H -#include "secp256k1_extrakeys.h" +#include "../../../include/secp256k1_extrakeys.h" static secp256k1_context* api_test_context(int flags, int *ecount) { secp256k1_context *ctx0 = secp256k1_context_create(flags); @@ -137,6 +137,43 @@ void test_xonly_pubkey(void) { secp256k1_context_destroy(verify); } +void test_xonly_pubkey_comparison(void) { + unsigned char pk1_ser[32] = { + 0x58, 0x84, 0xb3, 0xa2, 0x4b, 0x97, 0x37, 0x88, 0x92, 0x38, 0xa6, 0x26, 0x62, 0x52, 0x35, 0x11, + 0xd0, 0x9a, 0xa1, 0x1b, 0x80, 0x0b, 0x5e, 0x93, 0x80, 0x26, 0x11, 0xef, 0x67, 0x4b, 0xd9, 0x23 + }; + const unsigned char pk2_ser[32] = { + 0xde, 0x36, 0x0e, 0x87, 0x59, 0x8f, 0x3c, 0x01, 0x36, 0x2a, 0x2a, 0xb8, 0xc6, 0xf4, 0x5e, 0x4d, + 0xb2, 0xc2, 0xd5, 0x03, 0xa7, 0xf9, 0xf1, 0x4f, 0xa8, 0xfa, 0x95, 0xa8, 0xe9, 0x69, 0x76, 0x1c + }; + secp256k1_xonly_pubkey pk1; + secp256k1_xonly_pubkey pk2; + int ecount = 0; + secp256k1_context *none = api_test_context(SECP256K1_CONTEXT_NONE, &ecount); + + CHECK(secp256k1_xonly_pubkey_parse(none, &pk1, pk1_ser) == 1); + CHECK(secp256k1_xonly_pubkey_parse(none, &pk2, pk2_ser) == 1); + + CHECK(secp256k1_xonly_pubkey_cmp(none, NULL, &pk2) < 0); + CHECK(ecount == 1); + CHECK(secp256k1_xonly_pubkey_cmp(none, &pk1, NULL) > 0); + CHECK(ecount == 2); + CHECK(secp256k1_xonly_pubkey_cmp(none, &pk1, &pk2) < 0); + CHECK(secp256k1_xonly_pubkey_cmp(none, &pk2, &pk1) > 0); + CHECK(secp256k1_xonly_pubkey_cmp(none, &pk1, &pk1) == 0); + CHECK(secp256k1_xonly_pubkey_cmp(none, &pk2, &pk2) == 0); + CHECK(ecount == 2); + memset(&pk1, 0, sizeof(pk1)); /* illegal pubkey */ + CHECK(secp256k1_xonly_pubkey_cmp(none, &pk1, &pk2) < 0); + CHECK(ecount == 3); + CHECK(secp256k1_xonly_pubkey_cmp(none, &pk1, &pk1) == 0); + CHECK(ecount == 5); + CHECK(secp256k1_xonly_pubkey_cmp(none, &pk2, &pk1) > 0); + CHECK(ecount == 6); + + secp256k1_context_destroy(none); +} + void test_xonly_pubkey_tweak(void) { unsigned char zeros64[64] = { 0 }; unsigned char overflows[32]; @@ -540,6 +577,7 @@ void run_extrakeys_tests(void) { test_xonly_pubkey_tweak(); test_xonly_pubkey_tweak_check(); test_xonly_pubkey_tweak_recursive(); + test_xonly_pubkey_comparison(); /* keypair tests */ test_keypair(); diff --git a/src/secp256k1/src/modules/recovery/main_impl.h b/src/secp256k1/src/modules/recovery/main_impl.h index 7a440a729b..9e19f2a2dc 100644 --- a/src/secp256k1/src/modules/recovery/main_impl.h +++ b/src/secp256k1/src/modules/recovery/main_impl.h @@ -7,7 +7,7 @@ #ifndef SECP256K1_MODULE_RECOVERY_MAIN_H #define SECP256K1_MODULE_RECOVERY_MAIN_H -#include "include/secp256k1_recovery.h" +#include "../../../include/secp256k1_recovery.h" static void secp256k1_ecdsa_recoverable_signature_load(const secp256k1_context* ctx, secp256k1_scalar* r, secp256k1_scalar* s, int* recid, const secp256k1_ecdsa_recoverable_signature* sig) { (void)ctx; diff --git a/src/secp256k1/src/modules/recovery/tests_exhaustive_impl.h b/src/secp256k1/src/modules/recovery/tests_exhaustive_impl.h index 0ba9409c69..590a972ed3 100644 --- a/src/secp256k1/src/modules/recovery/tests_exhaustive_impl.h +++ b/src/secp256k1/src/modules/recovery/tests_exhaustive_impl.h @@ -8,7 +8,7 @@ #define SECP256K1_MODULE_RECOVERY_EXHAUSTIVE_TESTS_H #include "src/modules/recovery/main_impl.h" -#include "include/secp256k1_recovery.h" +#include "../../../include/secp256k1_recovery.h" void test_exhaustive_recovery_sign(const secp256k1_context *ctx, const secp256k1_ge *group) { int i, j, k; diff --git a/src/secp256k1/src/modules/schnorrsig/main_impl.h b/src/secp256k1/src/modules/schnorrsig/main_impl.h index 22e1b33a5a..693b78f034 100644 --- a/src/secp256k1/src/modules/schnorrsig/main_impl.h +++ b/src/secp256k1/src/modules/schnorrsig/main_impl.h @@ -7,9 +7,9 @@ #ifndef SECP256K1_MODULE_SCHNORRSIG_MAIN_H #define SECP256K1_MODULE_SCHNORRSIG_MAIN_H -#include "include/secp256k1.h" -#include "include/secp256k1_schnorrsig.h" -#include "hash.h" +#include "../../../include/secp256k1.h" +#include "../../../include/secp256k1_schnorrsig.h" +#include "../../hash.h" /* Initializes SHA256 with fixed midstate. This midstate was computed by applying * SHA256 to SHA256("BIP0340/nonce")||SHA256("BIP0340/nonce"). */ @@ -43,16 +43,18 @@ static void secp256k1_nonce_function_bip340_sha256_tagged_aux(secp256k1_sha256 * sha->bytes = 64; } -/* algo16 argument for nonce_function_bip340 to derive the nonce exactly as stated in BIP-340 +/* algo argument for nonce_function_bip340 to derive the nonce exactly as stated in BIP-340 * by using the correct tagged hash function. */ -static const unsigned char bip340_algo16[16] = "BIP0340/nonce\0\0\0"; +static const unsigned char bip340_algo[13] = "BIP0340/nonce"; -static int nonce_function_bip340(unsigned char *nonce32, const unsigned char *msg32, const unsigned char *key32, const unsigned char *xonly_pk32, const unsigned char *algo16, void *data) { +static const unsigned char schnorrsig_extraparams_magic[4] = SECP256K1_SCHNORRSIG_EXTRAPARAMS_MAGIC; + +static int nonce_function_bip340(unsigned char *nonce32, const unsigned char *msg, size_t msglen, const unsigned char *key32, const unsigned char *xonly_pk32, const unsigned char *algo, size_t algolen, void *data) { secp256k1_sha256 sha; unsigned char masked_key[32]; int i; - if (algo16 == NULL) { + if (algo == NULL) { return 0; } @@ -65,18 +67,14 @@ static int nonce_function_bip340(unsigned char *nonce32, const unsigned char *ms } } - /* Tag the hash with algo16 which is important to avoid nonce reuse across + /* Tag the hash with algo which is important to avoid nonce reuse across * algorithms. If this nonce function is used in BIP-340 signing as defined * in the spec, an optimized tagging implementation is used. */ - if (secp256k1_memcmp_var(algo16, bip340_algo16, 16) == 0) { + if (algolen == sizeof(bip340_algo) + && secp256k1_memcmp_var(algo, bip340_algo, algolen) == 0) { secp256k1_nonce_function_bip340_sha256_tagged(&sha); } else { - int algo16_len = 16; - /* Remove terminating null bytes */ - while (algo16_len > 0 && !algo16[algo16_len - 1]) { - algo16_len--; - } - secp256k1_sha256_initialize_tagged(&sha, algo16, algo16_len); + secp256k1_sha256_initialize_tagged(&sha, algo, algolen); } /* Hash (masked-)key||pk||msg using the tagged hash as per the spec */ @@ -86,7 +84,7 @@ static int nonce_function_bip340(unsigned char *nonce32, const unsigned char *ms secp256k1_sha256_write(&sha, key32, 32); } secp256k1_sha256_write(&sha, xonly_pk32, 32); - secp256k1_sha256_write(&sha, msg32, 32); + secp256k1_sha256_write(&sha, msg, msglen); secp256k1_sha256_finalize(&sha, nonce32); return 1; } @@ -108,23 +106,23 @@ static void secp256k1_schnorrsig_sha256_tagged(secp256k1_sha256 *sha) { sha->bytes = 64; } -static void secp256k1_schnorrsig_challenge(secp256k1_scalar* e, const unsigned char *r32, const unsigned char *msg32, const unsigned char *pubkey32) +static void secp256k1_schnorrsig_challenge(secp256k1_scalar* e, const unsigned char *r32, const unsigned char *msg, size_t msglen, const unsigned char *pubkey32) { unsigned char buf[32]; secp256k1_sha256 sha; - /* tagged hash(r.x, pk.x, msg32) */ + /* tagged hash(r.x, pk.x, msg) */ secp256k1_schnorrsig_sha256_tagged(&sha); secp256k1_sha256_write(&sha, r32, 32); secp256k1_sha256_write(&sha, pubkey32, 32); - secp256k1_sha256_write(&sha, msg32, 32); + secp256k1_sha256_write(&sha, msg, msglen); secp256k1_sha256_finalize(&sha, buf); /* Set scalar e to the challenge hash modulo the curve order as per * BIP340. */ secp256k1_scalar_set_b32(e, buf, NULL); } -int secp256k1_schnorrsig_sign(const secp256k1_context* ctx, unsigned char *sig64, const unsigned char *msg32, const secp256k1_keypair *keypair, secp256k1_nonce_function_hardened noncefp, void *ndata) { +int secp256k1_schnorrsig_sign_internal(const secp256k1_context* ctx, unsigned char *sig64, const unsigned char *msg, size_t msglen, const secp256k1_keypair *keypair, secp256k1_nonce_function_hardened noncefp, void *ndata) { secp256k1_scalar sk; secp256k1_scalar e; secp256k1_scalar k; @@ -139,7 +137,7 @@ int secp256k1_schnorrsig_sign(const secp256k1_context* ctx, unsigned char *sig64 VERIFY_CHECK(ctx != NULL); ARG_CHECK(secp256k1_ecmult_gen_context_is_built(&ctx->ecmult_gen_ctx)); ARG_CHECK(sig64 != NULL); - ARG_CHECK(msg32 != NULL); + ARG_CHECK(msg != NULL || msglen == 0); ARG_CHECK(keypair != NULL); if (noncefp == NULL) { @@ -156,7 +154,7 @@ int secp256k1_schnorrsig_sign(const secp256k1_context* ctx, unsigned char *sig64 secp256k1_scalar_get_b32(seckey, &sk); secp256k1_fe_get_b32(pk_buf, &pk.x); - ret &= !!noncefp(buf, msg32, seckey, pk_buf, bip340_algo16, ndata); + ret &= !!noncefp(buf, msg, msglen, seckey, pk_buf, bip340_algo, sizeof(bip340_algo), ndata); secp256k1_scalar_set_b32(&k, buf, NULL); ret &= !secp256k1_scalar_is_zero(&k); secp256k1_scalar_cmov(&k, &secp256k1_scalar_one, !ret); @@ -174,7 +172,7 @@ int secp256k1_schnorrsig_sign(const secp256k1_context* ctx, unsigned char *sig64 secp256k1_fe_normalize_var(&r.x); secp256k1_fe_get_b32(&sig64[0], &r.x); - secp256k1_schnorrsig_challenge(&e, &sig64[0], msg32, pk_buf); + secp256k1_schnorrsig_challenge(&e, &sig64[0], msg, msglen, pk_buf); secp256k1_scalar_mul(&e, &e, &sk); secp256k1_scalar_add(&e, &e, &k); secp256k1_scalar_get_b32(&sig64[32], &e); @@ -187,7 +185,26 @@ int secp256k1_schnorrsig_sign(const secp256k1_context* ctx, unsigned char *sig64 return ret; } -int secp256k1_schnorrsig_verify(const secp256k1_context* ctx, const unsigned char *sig64, const unsigned char *msg32, const secp256k1_xonly_pubkey *pubkey) { +int secp256k1_schnorrsig_sign(const secp256k1_context* ctx, unsigned char *sig64, const unsigned char *msg32, const secp256k1_keypair *keypair, unsigned char *aux_rand32) { + return secp256k1_schnorrsig_sign_internal(ctx, sig64, msg32, 32, keypair, secp256k1_nonce_function_bip340, aux_rand32); +} + +int secp256k1_schnorrsig_sign_custom(const secp256k1_context* ctx, unsigned char *sig64, const unsigned char *msg, size_t msglen, const secp256k1_keypair *keypair, secp256k1_schnorrsig_extraparams *extraparams) { + secp256k1_nonce_function_hardened noncefp = NULL; + void *ndata = NULL; + VERIFY_CHECK(ctx != NULL); + + if (extraparams != NULL) { + ARG_CHECK(secp256k1_memcmp_var(extraparams->magic, + schnorrsig_extraparams_magic, + sizeof(extraparams->magic)) == 0); + noncefp = extraparams->noncefp; + ndata = extraparams->ndata; + } + return secp256k1_schnorrsig_sign_internal(ctx, sig64, msg, msglen, keypair, noncefp, ndata); +} + +int secp256k1_schnorrsig_verify(const secp256k1_context* ctx, const unsigned char *sig64, const unsigned char *msg, size_t msglen, const secp256k1_xonly_pubkey *pubkey) { secp256k1_scalar s; secp256k1_scalar e; secp256k1_gej rj; @@ -201,7 +218,7 @@ int secp256k1_schnorrsig_verify(const secp256k1_context* ctx, const unsigned cha VERIFY_CHECK(ctx != NULL); ARG_CHECK(secp256k1_ecmult_context_is_built(&ctx->ecmult_ctx)); ARG_CHECK(sig64 != NULL); - ARG_CHECK(msg32 != NULL); + ARG_CHECK(msg != NULL || msglen == 0); ARG_CHECK(pubkey != NULL); if (!secp256k1_fe_set_b32(&rx, &sig64[0])) { @@ -219,7 +236,7 @@ int secp256k1_schnorrsig_verify(const secp256k1_context* ctx, const unsigned cha /* Compute e. */ secp256k1_fe_get_b32(buf, &pk.x); - secp256k1_schnorrsig_challenge(&e, &sig64[0], msg32, buf); + secp256k1_schnorrsig_challenge(&e, &sig64[0], msg, msglen, buf); /* Compute rj = s*G + (-e)*pkj */ secp256k1_scalar_negate(&e, &e); diff --git a/src/secp256k1/src/modules/schnorrsig/tests_exhaustive_impl.h b/src/secp256k1/src/modules/schnorrsig/tests_exhaustive_impl.h index b4a428729f..d8df9dd2df 100644 --- a/src/secp256k1/src/modules/schnorrsig/tests_exhaustive_impl.h +++ b/src/secp256k1/src/modules/schnorrsig/tests_exhaustive_impl.h @@ -7,7 +7,7 @@ #ifndef SECP256K1_MODULE_SCHNORRSIG_TESTS_EXHAUSTIVE_H #define SECP256K1_MODULE_SCHNORRSIG_TESTS_EXHAUSTIVE_H -#include "include/secp256k1_schnorrsig.h" +#include "../../../include/secp256k1_schnorrsig.h" #include "src/modules/schnorrsig/main_impl.h" static const unsigned char invalid_pubkey_bytes[][32] = { @@ -58,15 +58,19 @@ static const unsigned char invalid_pubkey_bytes[][32] = { #define NUM_INVALID_KEYS (sizeof(invalid_pubkey_bytes) / sizeof(invalid_pubkey_bytes[0])) -static int secp256k1_hardened_nonce_function_smallint(unsigned char *nonce32, const unsigned char *msg32, +static int secp256k1_hardened_nonce_function_smallint(unsigned char *nonce32, const unsigned char *msg, + size_t msglen, const unsigned char *key32, const unsigned char *xonly_pk32, - const unsigned char *algo16, void* data) { + const unsigned char *algo, size_t algolen, + void* data) { secp256k1_scalar s; int *idata = data; - (void)msg32; + (void)msg; + (void)msglen; (void)key32; (void)xonly_pk32; - (void)algo16; + (void)algo; + (void)algolen; secp256k1_scalar_set_int(&s, *idata); secp256k1_scalar_get_b32(nonce32, &s); return 1; @@ -101,7 +105,7 @@ static void test_exhaustive_schnorrsig_verify(const secp256k1_context *ctx, cons secp256k1_scalar e; unsigned char msg32[32]; secp256k1_testrand256(msg32); - secp256k1_schnorrsig_challenge(&e, sig64, msg32, pk32); + secp256k1_schnorrsig_challenge(&e, sig64, msg32, sizeof(msg32), pk32); /* Only do work if we hit a challenge we haven't tried before. */ if (!e_done[e]) { /* Iterate over the possible valid last 32 bytes in the signature. @@ -119,7 +123,7 @@ static void test_exhaustive_schnorrsig_verify(const secp256k1_context *ctx, cons secp256k1_testrand256(sig64 + 32); expect_valid = 0; } - valid = secp256k1_schnorrsig_verify(ctx, sig64, msg32, &pubkeys[d - 1]); + valid = secp256k1_schnorrsig_verify(ctx, sig64, msg32, sizeof(msg32), &pubkeys[d - 1]); CHECK(valid == expect_valid); count_valid += valid; } @@ -137,6 +141,8 @@ static void test_exhaustive_schnorrsig_verify(const secp256k1_context *ctx, cons static void test_exhaustive_schnorrsig_sign(const secp256k1_context *ctx, unsigned char (*xonly_pubkey_bytes)[32], const secp256k1_keypair* keypairs, const int* parities) { int d, k; uint64_t iter = 0; + secp256k1_schnorrsig_extraparams extraparams = SECP256K1_SCHNORRSIG_EXTRAPARAMS_INIT; + /* Loop over keys. */ for (d = 1; d < EXHAUSTIVE_TEST_ORDER; ++d) { int actual_d = d; @@ -149,19 +155,21 @@ static void test_exhaustive_schnorrsig_sign(const secp256k1_context *ctx, unsign unsigned char sig64[64]; int actual_k = k; if (skip_section(&iter)) continue; + extraparams.noncefp = secp256k1_hardened_nonce_function_smallint; + extraparams.ndata = &k; if (parities[k - 1]) actual_k = EXHAUSTIVE_TEST_ORDER - k; /* Generate random messages until all challenges have been tried. */ while (e_count_done < EXHAUSTIVE_TEST_ORDER) { secp256k1_scalar e; secp256k1_testrand256(msg32); - secp256k1_schnorrsig_challenge(&e, xonly_pubkey_bytes[k - 1], msg32, xonly_pubkey_bytes[d - 1]); + secp256k1_schnorrsig_challenge(&e, xonly_pubkey_bytes[k - 1], msg32, sizeof(msg32), xonly_pubkey_bytes[d - 1]); /* Only do work if we hit a challenge we haven't tried before. */ if (!e_done[e]) { secp256k1_scalar expected_s = (actual_k + e * actual_d) % EXHAUSTIVE_TEST_ORDER; unsigned char expected_s_bytes[32]; secp256k1_scalar_get_b32(expected_s_bytes, &expected_s); /* Invoke the real function to construct a signature. */ - CHECK(secp256k1_schnorrsig_sign(ctx, sig64, msg32, &keypairs[d - 1], secp256k1_hardened_nonce_function_smallint, &k)); + CHECK(secp256k1_schnorrsig_sign_custom(ctx, sig64, msg32, sizeof(msg32), &keypairs[d - 1], &extraparams)); /* The first 32 bytes must match the xonly pubkey for the specified k. */ CHECK(secp256k1_memcmp_var(sig64, xonly_pubkey_bytes[k - 1], 32) == 0); /* The last 32 bytes must match the expected s value. */ diff --git a/src/secp256k1/src/modules/schnorrsig/tests_impl.h b/src/secp256k1/src/modules/schnorrsig/tests_impl.h index 338462fc9d..59357afa99 100644 --- a/src/secp256k1/src/modules/schnorrsig/tests_impl.h +++ b/src/secp256k1/src/modules/schnorrsig/tests_impl.h @@ -7,16 +7,16 @@ #ifndef SECP256K1_MODULE_SCHNORRSIG_TESTS_H #define SECP256K1_MODULE_SCHNORRSIG_TESTS_H -#include "secp256k1_schnorrsig.h" +#include "../../../include/secp256k1_schnorrsig.h" /* Checks that a bit flip in the n_flip-th argument (that has n_bytes many * bytes) changes the hash function */ -void nonce_function_bip340_bitflip(unsigned char **args, size_t n_flip, size_t n_bytes) { +void nonce_function_bip340_bitflip(unsigned char **args, size_t n_flip, size_t n_bytes, size_t msglen, size_t algolen) { unsigned char nonces[2][32]; - CHECK(nonce_function_bip340(nonces[0], args[0], args[1], args[2], args[3], args[4]) == 1); + CHECK(nonce_function_bip340(nonces[0], args[0], msglen, args[1], args[2], args[3], algolen, args[4]) == 1); secp256k1_testrand_flip(args[n_flip], n_bytes); - CHECK(nonce_function_bip340(nonces[1], args[0], args[1], args[2], args[3], args[4]) == 1); + CHECK(nonce_function_bip340(nonces[1], args[0], msglen, args[1], args[2], args[3], algolen, args[4]) == 1); CHECK(secp256k1_memcmp_var(nonces[0], nonces[1], 32) != 0); } @@ -34,11 +34,13 @@ void test_sha256_eq(const secp256k1_sha256 *sha1, const secp256k1_sha256 *sha2) void run_nonce_function_bip340_tests(void) { unsigned char tag[13] = "BIP0340/nonce"; unsigned char aux_tag[11] = "BIP0340/aux"; - unsigned char algo16[16] = "BIP0340/nonce\0\0\0"; + unsigned char algo[13] = "BIP0340/nonce"; + size_t algolen = sizeof(algo); secp256k1_sha256 sha; secp256k1_sha256 sha_optimized; unsigned char nonce[32]; unsigned char msg[32]; + size_t msglen = sizeof(msg); unsigned char key[32]; unsigned char pk[32]; unsigned char aux_rand[32]; @@ -68,33 +70,45 @@ void run_nonce_function_bip340_tests(void) { args[0] = msg; args[1] = key; args[2] = pk; - args[3] = algo16; + args[3] = algo; args[4] = aux_rand; for (i = 0; i < count; i++) { - nonce_function_bip340_bitflip(args, 0, 32); - nonce_function_bip340_bitflip(args, 1, 32); - nonce_function_bip340_bitflip(args, 2, 32); - /* Flip algo16 special case "BIP0340/nonce" */ - nonce_function_bip340_bitflip(args, 3, 16); - /* Flip algo16 again */ - nonce_function_bip340_bitflip(args, 3, 16); - nonce_function_bip340_bitflip(args, 4, 32); + nonce_function_bip340_bitflip(args, 0, 32, msglen, algolen); + nonce_function_bip340_bitflip(args, 1, 32, msglen, algolen); + nonce_function_bip340_bitflip(args, 2, 32, msglen, algolen); + /* Flip algo special case "BIP0340/nonce" */ + nonce_function_bip340_bitflip(args, 3, algolen, msglen, algolen); + /* Flip algo again */ + nonce_function_bip340_bitflip(args, 3, algolen, msglen, algolen); + nonce_function_bip340_bitflip(args, 4, 32, msglen, algolen); } - /* NULL algo16 is disallowed */ - CHECK(nonce_function_bip340(nonce, msg, key, pk, NULL, NULL) == 0); - /* Empty algo16 is fine */ - memset(algo16, 0x00, 16); - CHECK(nonce_function_bip340(nonce, msg, key, pk, algo16, NULL) == 1); - /* algo16 with terminating null bytes is fine */ - algo16[1] = 65; - CHECK(nonce_function_bip340(nonce, msg, key, pk, algo16, NULL) == 1); - /* Other algo16 is fine */ - memset(algo16, 0xFF, 16); - CHECK(nonce_function_bip340(nonce, msg, key, pk, algo16, NULL) == 1); + /* NULL algo is disallowed */ + CHECK(nonce_function_bip340(nonce, msg, msglen, key, pk, NULL, 0, NULL) == 0); + CHECK(nonce_function_bip340(nonce, msg, msglen, key, pk, algo, algolen, NULL) == 1); + /* Other algo is fine */ + secp256k1_rfc6979_hmac_sha256_generate(&secp256k1_test_rng, algo, algolen); + CHECK(nonce_function_bip340(nonce, msg, msglen, key, pk, algo, algolen, NULL) == 1); + + for (i = 0; i < count; i++) { + unsigned char nonce2[32]; + uint32_t offset = secp256k1_testrand_int(msglen - 1); + size_t msglen_tmp = (msglen + offset) % msglen; + size_t algolen_tmp; + + /* Different msglen gives different nonce */ + CHECK(nonce_function_bip340(nonce2, msg, msglen_tmp, key, pk, algo, algolen, NULL) == 1); + CHECK(secp256k1_memcmp_var(nonce, nonce2, 32) != 0); + + /* Different algolen gives different nonce */ + offset = secp256k1_testrand_int(algolen - 1); + algolen_tmp = (algolen + offset) % algolen; + CHECK(nonce_function_bip340(nonce2, msg, msglen, key, pk, algo, algolen_tmp, NULL) == 1); + CHECK(secp256k1_memcmp_var(nonce, nonce2, 32) != 0); + } /* NULL aux_rand argument is allowed. */ - CHECK(nonce_function_bip340(nonce, msg, key, pk, algo16, NULL) == 1); + CHECK(nonce_function_bip340(nonce, msg, msglen, key, pk, algo, algolen, NULL) == 1); } void test_schnorrsig_api(void) { @@ -103,10 +117,12 @@ void test_schnorrsig_api(void) { unsigned char sk3[32]; unsigned char msg[32]; secp256k1_keypair keypairs[3]; - secp256k1_keypair invalid_keypair = { 0 }; + secp256k1_keypair invalid_keypair = {{ 0 }}; secp256k1_xonly_pubkey pk[3]; secp256k1_xonly_pubkey zero_pk; unsigned char sig[64]; + secp256k1_schnorrsig_extraparams extraparams = SECP256K1_SCHNORRSIG_EXTRAPARAMS_INIT; + secp256k1_schnorrsig_extraparams invalid_extraparams = {{ 0 }, NULL, NULL}; /** setup **/ secp256k1_context *none = secp256k1_context_create(SECP256K1_CONTEXT_NONE); @@ -138,36 +154,60 @@ void test_schnorrsig_api(void) { /** main test body **/ ecount = 0; - CHECK(secp256k1_schnorrsig_sign(none, sig, msg, &keypairs[0], NULL, NULL) == 0); + CHECK(secp256k1_schnorrsig_sign(none, sig, msg, &keypairs[0], NULL) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_schnorrsig_sign(vrfy, sig, msg, &keypairs[0], NULL) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_schnorrsig_sign(sign, sig, msg, &keypairs[0], NULL) == 1); + CHECK(ecount == 2); + CHECK(secp256k1_schnorrsig_sign(sign, NULL, msg, &keypairs[0], NULL) == 0); + CHECK(ecount == 3); + CHECK(secp256k1_schnorrsig_sign(sign, sig, NULL, &keypairs[0], NULL) == 0); + CHECK(ecount == 4); + CHECK(secp256k1_schnorrsig_sign(sign, sig, msg, NULL, NULL) == 0); + CHECK(ecount == 5); + CHECK(secp256k1_schnorrsig_sign(sign, sig, msg, &invalid_keypair, NULL) == 0); + CHECK(ecount == 6); + + ecount = 0; + CHECK(secp256k1_schnorrsig_sign_custom(none, sig, msg, sizeof(msg), &keypairs[0], &extraparams) == 0); CHECK(ecount == 1); - CHECK(secp256k1_schnorrsig_sign(vrfy, sig, msg, &keypairs[0], NULL, NULL) == 0); + CHECK(secp256k1_schnorrsig_sign_custom(vrfy, sig, msg, sizeof(msg), &keypairs[0], &extraparams) == 0); CHECK(ecount == 2); - CHECK(secp256k1_schnorrsig_sign(sign, sig, msg, &keypairs[0], NULL, NULL) == 1); + CHECK(secp256k1_schnorrsig_sign_custom(sign, sig, msg, sizeof(msg), &keypairs[0], &extraparams) == 1); CHECK(ecount == 2); - CHECK(secp256k1_schnorrsig_sign(sign, NULL, msg, &keypairs[0], NULL, NULL) == 0); + CHECK(secp256k1_schnorrsig_sign_custom(sign, NULL, msg, sizeof(msg), &keypairs[0], &extraparams) == 0); CHECK(ecount == 3); - CHECK(secp256k1_schnorrsig_sign(sign, sig, NULL, &keypairs[0], NULL, NULL) == 0); + CHECK(secp256k1_schnorrsig_sign_custom(sign, sig, NULL, sizeof(msg), &keypairs[0], &extraparams) == 0); CHECK(ecount == 4); - CHECK(secp256k1_schnorrsig_sign(sign, sig, msg, NULL, NULL, NULL) == 0); + CHECK(secp256k1_schnorrsig_sign_custom(sign, sig, NULL, 0, &keypairs[0], &extraparams) == 1); + CHECK(ecount == 4); + CHECK(secp256k1_schnorrsig_sign_custom(sign, sig, msg, sizeof(msg), NULL, &extraparams) == 0); CHECK(ecount == 5); - CHECK(secp256k1_schnorrsig_sign(sign, sig, msg, &invalid_keypair, NULL, NULL) == 0); + CHECK(secp256k1_schnorrsig_sign_custom(sign, sig, msg, sizeof(msg), &invalid_keypair, &extraparams) == 0); + CHECK(ecount == 6); + CHECK(secp256k1_schnorrsig_sign_custom(sign, sig, msg, sizeof(msg), &keypairs[0], NULL) == 1); CHECK(ecount == 6); + CHECK(secp256k1_schnorrsig_sign_custom(sign, sig, msg, sizeof(msg), &keypairs[0], &invalid_extraparams) == 0); + CHECK(ecount == 7); ecount = 0; - CHECK(secp256k1_schnorrsig_sign(sign, sig, msg, &keypairs[0], NULL, NULL) == 1); - CHECK(secp256k1_schnorrsig_verify(none, sig, msg, &pk[0]) == 0); + CHECK(secp256k1_schnorrsig_sign(sign, sig, msg, &keypairs[0], NULL) == 1); + CHECK(secp256k1_schnorrsig_verify(none, sig, msg, sizeof(msg), &pk[0]) == 0); CHECK(ecount == 1); - CHECK(secp256k1_schnorrsig_verify(sign, sig, msg, &pk[0]) == 0); + CHECK(secp256k1_schnorrsig_verify(sign, sig, msg, sizeof(msg), &pk[0]) == 0); CHECK(ecount == 2); - CHECK(secp256k1_schnorrsig_verify(vrfy, sig, msg, &pk[0]) == 1); + CHECK(secp256k1_schnorrsig_verify(vrfy, sig, msg, sizeof(msg), &pk[0]) == 1); CHECK(ecount == 2); - CHECK(secp256k1_schnorrsig_verify(vrfy, NULL, msg, &pk[0]) == 0); + CHECK(secp256k1_schnorrsig_verify(vrfy, NULL, msg, sizeof(msg), &pk[0]) == 0); CHECK(ecount == 3); - CHECK(secp256k1_schnorrsig_verify(vrfy, sig, NULL, &pk[0]) == 0); + CHECK(secp256k1_schnorrsig_verify(vrfy, sig, NULL, sizeof(msg), &pk[0]) == 0); + CHECK(ecount == 4); + CHECK(secp256k1_schnorrsig_verify(vrfy, sig, NULL, 0, &pk[0]) == 0); CHECK(ecount == 4); - CHECK(secp256k1_schnorrsig_verify(vrfy, sig, msg, NULL) == 0); + CHECK(secp256k1_schnorrsig_verify(vrfy, sig, msg, sizeof(msg), NULL) == 0); CHECK(ecount == 5); - CHECK(secp256k1_schnorrsig_verify(vrfy, sig, msg, &zero_pk) == 0); + CHECK(secp256k1_schnorrsig_verify(vrfy, sig, msg, sizeof(msg), &zero_pk) == 0); CHECK(ecount == 6); secp256k1_context_destroy(none); @@ -179,7 +219,7 @@ void test_schnorrsig_api(void) { /* Checks that hash initialized by secp256k1_schnorrsig_sha256_tagged has the * expected state. */ void test_schnorrsig_sha256_tagged(void) { - char tag[17] = "BIP0340/challenge"; + unsigned char tag[17] = "BIP0340/challenge"; secp256k1_sha256 sha; secp256k1_sha256 sha_optimized; @@ -190,19 +230,19 @@ void test_schnorrsig_sha256_tagged(void) { /* Helper function for schnorrsig_bip_vectors * Signs the message and checks that it's the same as expected_sig. */ -void test_schnorrsig_bip_vectors_check_signing(const unsigned char *sk, const unsigned char *pk_serialized, unsigned char *aux_rand, const unsigned char *msg, const unsigned char *expected_sig) { +void test_schnorrsig_bip_vectors_check_signing(const unsigned char *sk, const unsigned char *pk_serialized, unsigned char *aux_rand, const unsigned char *msg32, const unsigned char *expected_sig) { unsigned char sig[64]; secp256k1_keypair keypair; secp256k1_xonly_pubkey pk, pk_expected; CHECK(secp256k1_keypair_create(ctx, &keypair, sk)); - CHECK(secp256k1_schnorrsig_sign(ctx, sig, msg, &keypair, NULL, aux_rand)); + CHECK(secp256k1_schnorrsig_sign(ctx, sig, msg32, &keypair, aux_rand)); CHECK(secp256k1_memcmp_var(sig, expected_sig, 64) == 0); CHECK(secp256k1_xonly_pubkey_parse(ctx, &pk_expected, pk_serialized)); CHECK(secp256k1_keypair_xonly_pub(ctx, &pk, NULL, &keypair)); CHECK(secp256k1_memcmp_var(&pk, &pk_expected, sizeof(pk)) == 0); - CHECK(secp256k1_schnorrsig_verify(ctx, sig, msg, &pk)); + CHECK(secp256k1_schnorrsig_verify(ctx, sig, msg32, 32, &pk)); } /* Helper function for schnorrsig_bip_vectors @@ -211,7 +251,7 @@ void test_schnorrsig_bip_vectors_check_verify(const unsigned char *pk_serialized secp256k1_xonly_pubkey pk; CHECK(secp256k1_xonly_pubkey_parse(ctx, &pk, pk_serialized)); - CHECK(expected == secp256k1_schnorrsig_verify(ctx, sig, msg32, &pk)); + CHECK(expected == secp256k1_schnorrsig_verify(ctx, sig, msg32, 32, &pk)); } /* Test vectors according to BIP-340 ("Schnorr Signatures for secp256k1"). See @@ -634,22 +674,26 @@ void test_schnorrsig_bip_vectors(void) { } /* Nonce function that returns constant 0 */ -static int nonce_function_failing(unsigned char *nonce32, const unsigned char *msg32, const unsigned char *key32, const unsigned char *xonly_pk32, const unsigned char *algo16, void *data) { - (void) msg32; +static int nonce_function_failing(unsigned char *nonce32, const unsigned char *msg, size_t msglen, const unsigned char *key32, const unsigned char *xonly_pk32, const unsigned char *algo, size_t algolen, void *data) { + (void) msg; + (void) msglen; (void) key32; (void) xonly_pk32; - (void) algo16; + (void) algo; + (void) algolen; (void) data; (void) nonce32; return 0; } /* Nonce function that sets nonce to 0 */ -static int nonce_function_0(unsigned char *nonce32, const unsigned char *msg32, const unsigned char *key32, const unsigned char *xonly_pk32, const unsigned char *algo16, void *data) { - (void) msg32; +static int nonce_function_0(unsigned char *nonce32, const unsigned char *msg, size_t msglen, const unsigned char *key32, const unsigned char *xonly_pk32, const unsigned char *algo, size_t algolen, void *data) { + (void) msg; + (void) msglen; (void) key32; (void) xonly_pk32; - (void) algo16; + (void) algo; + (void) algolen; (void) data; memset(nonce32, 0, 32); @@ -657,11 +701,13 @@ static int nonce_function_0(unsigned char *nonce32, const unsigned char *msg32, } /* Nonce function that sets nonce to 0xFF...0xFF */ -static int nonce_function_overflowing(unsigned char *nonce32, const unsigned char *msg32, const unsigned char *key32, const unsigned char *xonly_pk32, const unsigned char *algo16, void *data) { - (void) msg32; +static int nonce_function_overflowing(unsigned char *nonce32, const unsigned char *msg, size_t msglen, const unsigned char *key32, const unsigned char *xonly_pk32, const unsigned char *algo, size_t algolen, void *data) { + (void) msg; + (void) msglen; (void) key32; (void) xonly_pk32; - (void) algo16; + (void) algo; + (void) algolen; (void) data; memset(nonce32, 0xFF, 32); @@ -670,24 +716,45 @@ static int nonce_function_overflowing(unsigned char *nonce32, const unsigned cha void test_schnorrsig_sign(void) { unsigned char sk[32]; + secp256k1_xonly_pubkey pk; secp256k1_keypair keypair; const unsigned char msg[32] = "this is a msg for a schnorrsig.."; unsigned char sig[64]; + unsigned char sig2[64]; unsigned char zeros64[64] = { 0 }; + secp256k1_schnorrsig_extraparams extraparams = SECP256K1_SCHNORRSIG_EXTRAPARAMS_INIT; + unsigned char aux_rand[32]; secp256k1_testrand256(sk); + secp256k1_testrand256(aux_rand); CHECK(secp256k1_keypair_create(ctx, &keypair, sk)); - CHECK(secp256k1_schnorrsig_sign(ctx, sig, msg, &keypair, NULL, NULL) == 1); + CHECK(secp256k1_keypair_xonly_pub(ctx, &pk, NULL, &keypair)); + CHECK(secp256k1_schnorrsig_sign(ctx, sig, msg, &keypair, NULL) == 1); + CHECK(secp256k1_schnorrsig_verify(ctx, sig, msg, sizeof(msg), &pk)); /* Test different nonce functions */ + CHECK(secp256k1_schnorrsig_sign_custom(ctx, sig, msg, sizeof(msg), &keypair, &extraparams) == 1); + CHECK(secp256k1_schnorrsig_verify(ctx, sig, msg, sizeof(msg), &pk)); memset(sig, 1, sizeof(sig)); - CHECK(secp256k1_schnorrsig_sign(ctx, sig, msg, &keypair, nonce_function_failing, NULL) == 0); + extraparams.noncefp = nonce_function_failing; + CHECK(secp256k1_schnorrsig_sign_custom(ctx, sig, msg, sizeof(msg), &keypair, &extraparams) == 0); CHECK(secp256k1_memcmp_var(sig, zeros64, sizeof(sig)) == 0); memset(&sig, 1, sizeof(sig)); - CHECK(secp256k1_schnorrsig_sign(ctx, sig, msg, &keypair, nonce_function_0, NULL) == 0); + extraparams.noncefp = nonce_function_0; + CHECK(secp256k1_schnorrsig_sign_custom(ctx, sig, msg, sizeof(msg), &keypair, &extraparams) == 0); CHECK(secp256k1_memcmp_var(sig, zeros64, sizeof(sig)) == 0); - CHECK(secp256k1_schnorrsig_sign(ctx, sig, msg, &keypair, nonce_function_overflowing, NULL) == 1); - CHECK(secp256k1_memcmp_var(sig, zeros64, sizeof(sig)) != 0); + memset(&sig, 1, sizeof(sig)); + extraparams.noncefp = nonce_function_overflowing; + CHECK(secp256k1_schnorrsig_sign_custom(ctx, sig, msg, sizeof(msg), &keypair, &extraparams) == 1); + CHECK(secp256k1_schnorrsig_verify(ctx, sig, msg, sizeof(msg), &pk)); + + /* When using the default nonce function, schnorrsig_sign_custom produces + * the same result as schnorrsig_sign with aux_rand = extraparams.ndata */ + extraparams.noncefp = NULL; + extraparams.ndata = aux_rand; + CHECK(secp256k1_schnorrsig_sign_custom(ctx, sig, msg, sizeof(msg), &keypair, &extraparams) == 1); + CHECK(secp256k1_schnorrsig_sign(ctx, sig2, msg, &keypair, extraparams.ndata) == 1); + CHECK(secp256k1_memcmp_var(sig, sig2, sizeof(sig)) == 0); } #define N_SIGS 3 @@ -709,8 +776,8 @@ void test_schnorrsig_sign_verify(void) { for (i = 0; i < N_SIGS; i++) { secp256k1_testrand256(msg[i]); - CHECK(secp256k1_schnorrsig_sign(ctx, sig[i], msg[i], &keypair, NULL, NULL)); - CHECK(secp256k1_schnorrsig_verify(ctx, sig[i], msg[i], &pk)); + CHECK(secp256k1_schnorrsig_sign(ctx, sig[i], msg[i], &keypair, NULL)); + CHECK(secp256k1_schnorrsig_verify(ctx, sig[i], msg[i], sizeof(msg[i]), &pk)); } { @@ -720,36 +787,54 @@ void test_schnorrsig_sign_verify(void) { size_t byte_idx = secp256k1_testrand_int(32); unsigned char xorbyte = secp256k1_testrand_int(254)+1; sig[sig_idx][byte_idx] ^= xorbyte; - CHECK(!secp256k1_schnorrsig_verify(ctx, sig[sig_idx], msg[sig_idx], &pk)); + CHECK(!secp256k1_schnorrsig_verify(ctx, sig[sig_idx], msg[sig_idx], sizeof(msg[sig_idx]), &pk)); sig[sig_idx][byte_idx] ^= xorbyte; byte_idx = secp256k1_testrand_int(32); sig[sig_idx][32+byte_idx] ^= xorbyte; - CHECK(!secp256k1_schnorrsig_verify(ctx, sig[sig_idx], msg[sig_idx], &pk)); + CHECK(!secp256k1_schnorrsig_verify(ctx, sig[sig_idx], msg[sig_idx], sizeof(msg[sig_idx]), &pk)); sig[sig_idx][32+byte_idx] ^= xorbyte; byte_idx = secp256k1_testrand_int(32); msg[sig_idx][byte_idx] ^= xorbyte; - CHECK(!secp256k1_schnorrsig_verify(ctx, sig[sig_idx], msg[sig_idx], &pk)); + CHECK(!secp256k1_schnorrsig_verify(ctx, sig[sig_idx], msg[sig_idx], sizeof(msg[sig_idx]), &pk)); msg[sig_idx][byte_idx] ^= xorbyte; /* Check that above bitflips have been reversed correctly */ - CHECK(secp256k1_schnorrsig_verify(ctx, sig[sig_idx], msg[sig_idx], &pk)); + CHECK(secp256k1_schnorrsig_verify(ctx, sig[sig_idx], msg[sig_idx], sizeof(msg[sig_idx]), &pk)); } /* Test overflowing s */ - CHECK(secp256k1_schnorrsig_sign(ctx, sig[0], msg[0], &keypair, NULL, NULL)); - CHECK(secp256k1_schnorrsig_verify(ctx, sig[0], msg[0], &pk)); + CHECK(secp256k1_schnorrsig_sign(ctx, sig[0], msg[0], &keypair, NULL)); + CHECK(secp256k1_schnorrsig_verify(ctx, sig[0], msg[0], sizeof(msg[0]), &pk)); memset(&sig[0][32], 0xFF, 32); - CHECK(!secp256k1_schnorrsig_verify(ctx, sig[0], msg[0], &pk)); + CHECK(!secp256k1_schnorrsig_verify(ctx, sig[0], msg[0], sizeof(msg[0]), &pk)); /* Test negative s */ - CHECK(secp256k1_schnorrsig_sign(ctx, sig[0], msg[0], &keypair, NULL, NULL)); - CHECK(secp256k1_schnorrsig_verify(ctx, sig[0], msg[0], &pk)); + CHECK(secp256k1_schnorrsig_sign(ctx, sig[0], msg[0], &keypair, NULL)); + CHECK(secp256k1_schnorrsig_verify(ctx, sig[0], msg[0], sizeof(msg[0]), &pk)); secp256k1_scalar_set_b32(&s, &sig[0][32], NULL); secp256k1_scalar_negate(&s, &s); secp256k1_scalar_get_b32(&sig[0][32], &s); - CHECK(!secp256k1_schnorrsig_verify(ctx, sig[0], msg[0], &pk)); + CHECK(!secp256k1_schnorrsig_verify(ctx, sig[0], msg[0], sizeof(msg[0]), &pk)); + + /* The empty message can be signed & verified */ + CHECK(secp256k1_schnorrsig_sign_custom(ctx, sig[0], NULL, 0, &keypair, NULL) == 1); + CHECK(secp256k1_schnorrsig_verify(ctx, sig[0], NULL, 0, &pk) == 1); + + { + /* Test varying message lengths */ + unsigned char msg_large[32 * 8]; + uint32_t msglen = secp256k1_testrand_int(sizeof(msg_large)); + for (i = 0; i < sizeof(msg_large); i += 32) { + secp256k1_testrand256(&msg_large[i]); + } + CHECK(secp256k1_schnorrsig_sign_custom(ctx, sig[0], msg_large, msglen, &keypair, NULL) == 1); + CHECK(secp256k1_schnorrsig_verify(ctx, sig[0], msg_large, msglen, &pk) == 1); + /* Verification for a random wrong message length fails */ + msglen = (msglen + (sizeof(msg_large) - 1)) % sizeof(msg_large); + CHECK(secp256k1_schnorrsig_verify(ctx, sig[0], msg_large, msglen, &pk) == 0); + } } #undef N_SIGS @@ -777,10 +862,10 @@ void test_schnorrsig_taproot(void) { /* Key spend */ secp256k1_testrand256(msg); - CHECK(secp256k1_schnorrsig_sign(ctx, sig, msg, &keypair, NULL, NULL) == 1); + CHECK(secp256k1_schnorrsig_sign(ctx, sig, msg, &keypair, NULL) == 1); /* Verify key spend */ CHECK(secp256k1_xonly_pubkey_parse(ctx, &output_pk, output_pk_bytes) == 1); - CHECK(secp256k1_schnorrsig_verify(ctx, sig, msg, &output_pk) == 1); + CHECK(secp256k1_schnorrsig_verify(ctx, sig, msg, sizeof(msg), &output_pk) == 1); /* Script spend */ CHECK(secp256k1_xonly_pubkey_serialize(ctx, internal_pk_bytes, &internal_pk) == 1); diff --git a/src/secp256k1/src/secp256k1.c b/src/secp256k1/src/secp256k1.c index aef3f99ac3..9908cab864 100644 --- a/src/secp256k1/src/secp256k1.c +++ b/src/secp256k1/src/secp256k1.c @@ -4,8 +4,10 @@ * file COPYING or https://www.opensource.org/licenses/mit-license.php.* ***********************************************************************/ -#include "include/secp256k1.h" -#include "include/secp256k1_preallocated.h" +#define SECP256K1_BUILD + +#include "../include/secp256k1.h" +#include "../include/secp256k1_preallocated.h" #include "assumptions.h" #include "util.h" @@ -21,6 +23,10 @@ #include "scratch_impl.h" #include "selftest.h" +#ifdef SECP256K1_NO_BUILD +# error "secp256k1.h processed without SECP256K1_BUILD defined while building secp256k1.c" +#endif + #if defined(VALGRIND) # include <valgrind/memcheck.h> #endif @@ -316,6 +322,32 @@ int secp256k1_ec_pubkey_serialize(const secp256k1_context* ctx, unsigned char *o return ret; } +int secp256k1_ec_pubkey_cmp(const secp256k1_context* ctx, const secp256k1_pubkey* pubkey0, const secp256k1_pubkey* pubkey1) { + unsigned char out[2][33]; + const secp256k1_pubkey* pk[2]; + int i; + + VERIFY_CHECK(ctx != NULL); + pk[0] = pubkey0; pk[1] = pubkey1; + for (i = 0; i < 2; i++) { + size_t out_size = sizeof(out[i]); + /* If the public key is NULL or invalid, ec_pubkey_serialize will call + * the illegal_callback and return 0. In that case we will serialize the + * key as all zeros which is less than any valid public key. This + * results in consistent comparisons even if NULL or invalid pubkeys are + * involved and prevents edge cases such as sorting algorithms that use + * this function and do not terminate as a result. */ + if (!secp256k1_ec_pubkey_serialize(ctx, out[i], &out_size, pk[i], SECP256K1_EC_COMPRESSED)) { + /* Note that ec_pubkey_serialize should already set the output to + * zero in that case, but it's not guaranteed by the API, we can't + * test it and writing a VERIFY_CHECK is more complex than + * explicitly memsetting (again). */ + memset(out[i], 0, sizeof(out[i])); + } + } + return secp256k1_memcmp_var(out[0], out[1], sizeof(out[0])); +} + static void secp256k1_ecdsa_signature_load(const secp256k1_context* ctx, secp256k1_scalar* r, secp256k1_scalar* s, const secp256k1_ecdsa_signature* sig) { (void)ctx; if (sizeof(secp256k1_scalar) == 32) { @@ -758,6 +790,19 @@ int secp256k1_ec_pubkey_combine(const secp256k1_context* ctx, secp256k1_pubkey * return 1; } +int secp256k1_tagged_sha256(const secp256k1_context* ctx, unsigned char *hash32, const unsigned char *tag, size_t taglen, const unsigned char *msg, size_t msglen) { + secp256k1_sha256 sha; + VERIFY_CHECK(ctx != NULL); + ARG_CHECK(hash32 != NULL); + ARG_CHECK(tag != NULL); + ARG_CHECK(msg != NULL); + + secp256k1_sha256_initialize_tagged(&sha, tag, taglen); + secp256k1_sha256_write(&sha, msg, msglen); + secp256k1_sha256_finalize(&sha, hash32); + return 1; +} + #ifdef ENABLE_MODULE_ECDH # include "modules/ecdh/main_impl.h" #endif diff --git a/src/secp256k1/src/testrand_impl.h b/src/secp256k1/src/testrand_impl.h index e643778f36..c8d30ef6a8 100644 --- a/src/secp256k1/src/testrand_impl.h +++ b/src/secp256k1/src/testrand_impl.h @@ -127,7 +127,7 @@ static void secp256k1_testrand_init(const char* hexseed) { pos++; } } else { - FILE *frand = fopen("/dev/urandom", "r"); + FILE *frand = fopen("/dev/urandom", "rb"); if ((frand == NULL) || fread(&seed16, 1, sizeof(seed16), frand) != sizeof(seed16)) { uint64_t t = time(NULL) * (uint64_t)1337; fprintf(stderr, "WARNING: could not read 16 bytes from /dev/urandom; falling back to insecure PRNG\n"); diff --git a/src/secp256k1/src/tests.c b/src/secp256k1/src/tests.c index a146394305..99d9468e29 100644 --- a/src/secp256k1/src/tests.c +++ b/src/secp256k1/src/tests.c @@ -15,8 +15,8 @@ #include <time.h> #include "secp256k1.c" -#include "include/secp256k1.h" -#include "include/secp256k1_preallocated.h" +#include "../include/secp256k1.h" +#include "../include/secp256k1_preallocated.h" #include "testrand_impl.h" #include "util.h" @@ -30,8 +30,8 @@ void ECDSA_SIG_get0(const ECDSA_SIG *sig, const BIGNUM **pr, const BIGNUM **ps) # endif #endif -#include "contrib/lax_der_parsing.c" -#include "contrib/lax_der_privatekey_parsing.c" +#include "../contrib/lax_der_parsing.c" +#include "../contrib/lax_der_privatekey_parsing.c" #include "modinv32_impl.h" #ifdef SECP256K1_WIDEMUL_INT128 @@ -564,6 +564,38 @@ void run_rfc6979_hmac_sha256_tests(void) { secp256k1_rfc6979_hmac_sha256_finalize(&rng); } +void run_tagged_sha256_tests(void) { + int ecount = 0; + secp256k1_context *none = secp256k1_context_create(SECP256K1_CONTEXT_NONE); + unsigned char tag[32] = { 0 }; + unsigned char msg[32] = { 0 }; + unsigned char hash32[32]; + unsigned char hash_expected[32] = { + 0x04, 0x7A, 0x5E, 0x17, 0xB5, 0x86, 0x47, 0xC1, + 0x3C, 0xC6, 0xEB, 0xC0, 0xAA, 0x58, 0x3B, 0x62, + 0xFB, 0x16, 0x43, 0x32, 0x68, 0x77, 0x40, 0x6C, + 0xE2, 0x76, 0x55, 0x9A, 0x3B, 0xDE, 0x55, 0xB3 + }; + + secp256k1_context_set_illegal_callback(none, counting_illegal_callback_fn, &ecount); + + /* API test */ + CHECK(secp256k1_tagged_sha256(none, hash32, tag, sizeof(tag), msg, sizeof(msg)) == 1); + CHECK(secp256k1_tagged_sha256(none, NULL, tag, sizeof(tag), msg, sizeof(msg)) == 0); + CHECK(ecount == 1); + CHECK(secp256k1_tagged_sha256(none, hash32, NULL, 0, msg, sizeof(msg)) == 0); + CHECK(ecount == 2); + CHECK(secp256k1_tagged_sha256(none, hash32, tag, sizeof(tag), NULL, 0) == 0); + CHECK(ecount == 3); + + /* Static test vector */ + memcpy(tag, "tag", 3); + memcpy(msg, "msg", 3); + CHECK(secp256k1_tagged_sha256(none, hash32, tag, 3, msg, 3) == 1); + CHECK(secp256k1_memcmp_var(hash32, hash_expected, sizeof(hash32)) == 0); + secp256k1_context_destroy(none); +} + /***** RANDOM TESTS *****/ void test_rand_bits(int rand32, int bits) { @@ -2508,6 +2540,70 @@ void run_field_misc(void) { } } +void test_fe_mul(const secp256k1_fe* a, const secp256k1_fe* b, int use_sqr) +{ + secp256k1_fe c, an, bn; + /* Variables in BE 32-byte format. */ + unsigned char a32[32], b32[32], c32[32]; + /* Variables in LE 16x uint16_t format. */ + uint16_t a16[16], b16[16], c16[16]; + /* Field modulus in LE 16x uint16_t format. */ + static const uint16_t m16[16] = { + 0xfc2f, 0xffff, 0xfffe, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + }; + uint16_t t16[32]; + int i; + + /* Compute C = A * B in fe format. */ + c = *a; + if (use_sqr) { + secp256k1_fe_sqr(&c, &c); + } else { + secp256k1_fe_mul(&c, &c, b); + } + + /* Convert A, B, C into LE 16x uint16_t format. */ + an = *a; + bn = *b; + secp256k1_fe_normalize_var(&c); + secp256k1_fe_normalize_var(&an); + secp256k1_fe_normalize_var(&bn); + secp256k1_fe_get_b32(a32, &an); + secp256k1_fe_get_b32(b32, &bn); + secp256k1_fe_get_b32(c32, &c); + for (i = 0; i < 16; ++i) { + a16[i] = a32[31 - 2*i] + ((uint16_t)a32[30 - 2*i] << 8); + b16[i] = b32[31 - 2*i] + ((uint16_t)b32[30 - 2*i] << 8); + c16[i] = c32[31 - 2*i] + ((uint16_t)c32[30 - 2*i] << 8); + } + /* Compute T = A * B in LE 16x uint16_t format. */ + mulmod256(t16, a16, b16, m16); + /* Compare */ + CHECK(secp256k1_memcmp_var(t16, c16, 32) == 0); +} + +void run_fe_mul(void) { + int i; + for (i = 0; i < 100 * count; ++i) { + secp256k1_fe a, b, c, d; + random_fe(&a); + random_field_element_magnitude(&a); + random_fe(&b); + random_field_element_magnitude(&b); + random_fe_test(&c); + random_field_element_magnitude(&c); + random_fe_test(&d); + random_field_element_magnitude(&d); + test_fe_mul(&a, &a, 1); + test_fe_mul(&c, &c, 1); + test_fe_mul(&a, &b, 0); + test_fe_mul(&a, &c, 0); + test_fe_mul(&c, &b, 0); + test_fe_mul(&c, &d, 0); + } +} + void run_sqr(void) { secp256k1_fe x, s; @@ -2595,7 +2691,7 @@ void test_inverse_scalar(secp256k1_scalar* out, const secp256k1_scalar* x, int v { secp256k1_scalar l, r, t; - (var ? secp256k1_scalar_inverse_var : secp256k1_scalar_inverse_var)(&l, x); /* l = 1/x */ + (var ? secp256k1_scalar_inverse_var : secp256k1_scalar_inverse)(&l, x); /* l = 1/x */ if (out) *out = l; if (secp256k1_scalar_is_zero(x)) { CHECK(secp256k1_scalar_is_zero(&l)); @@ -2605,9 +2701,9 @@ void test_inverse_scalar(secp256k1_scalar* out, const secp256k1_scalar* x, int v CHECK(secp256k1_scalar_is_one(&t)); /* x*(1/x) == 1 */ secp256k1_scalar_add(&r, x, &scalar_minus_one); /* r = x-1 */ if (secp256k1_scalar_is_zero(&r)) return; - (var ? secp256k1_scalar_inverse_var : secp256k1_scalar_inverse_var)(&r, &r); /* r = 1/(x-1) */ + (var ? secp256k1_scalar_inverse_var : secp256k1_scalar_inverse)(&r, &r); /* r = 1/(x-1) */ secp256k1_scalar_add(&l, &scalar_minus_one, &l); /* l = 1/x-1 */ - (var ? secp256k1_scalar_inverse_var : secp256k1_scalar_inverse_var)(&l, &l); /* l = 1/(1/x-1) */ + (var ? secp256k1_scalar_inverse_var : secp256k1_scalar_inverse)(&l, &l); /* l = 1/(1/x-1) */ secp256k1_scalar_add(&l, &l, &secp256k1_scalar_one); /* l = 1/(1/x-1)+1 */ secp256k1_scalar_add(&l, &r, &l); /* l = 1/(1/x-1)+1 + 1/(x-1) */ CHECK(secp256k1_scalar_is_zero(&l)); /* l == 0 */ @@ -3101,20 +3197,34 @@ void test_ge(void) { /* Test batch gej -> ge conversion with many infinities. */ for (i = 0; i < 4 * runs + 1; i++) { + int odd; random_group_element_test(&ge[i]); + odd = secp256k1_fe_is_odd(&ge[i].x); + CHECK(odd == 0 || odd == 1); /* randomly set half the points to infinity */ - if(secp256k1_fe_is_odd(&ge[i].x)) { + if (odd == i % 2) { secp256k1_ge_set_infinity(&ge[i]); } secp256k1_gej_set_ge(&gej[i], &ge[i]); } - /* batch invert */ + /* batch convert */ secp256k1_ge_set_all_gej_var(ge, gej, 4 * runs + 1); /* check result */ for (i = 0; i < 4 * runs + 1; i++) { ge_equals_gej(&ge[i], &gej[i]); } + /* Test batch gej -> ge conversion with all infinities. */ + for (i = 0; i < 4 * runs + 1; i++) { + secp256k1_gej_set_infinity(&gej[i]); + } + /* batch convert */ + secp256k1_ge_set_all_gej_var(ge, gej, 4 * runs + 1); + /* check result */ + for (i = 0; i < 4 * runs + 1; i++) { + CHECK(secp256k1_ge_is_infinity(&ge[i])); + } + free(ge); free(gej); } @@ -5434,6 +5544,55 @@ void test_random_pubkeys(void) { } } +void run_pubkey_comparison(void) { + unsigned char pk1_ser[33] = { + 0x02, + 0x58, 0x84, 0xb3, 0xa2, 0x4b, 0x97, 0x37, 0x88, 0x92, 0x38, 0xa6, 0x26, 0x62, 0x52, 0x35, 0x11, + 0xd0, 0x9a, 0xa1, 0x1b, 0x80, 0x0b, 0x5e, 0x93, 0x80, 0x26, 0x11, 0xef, 0x67, 0x4b, 0xd9, 0x23 + }; + const unsigned char pk2_ser[33] = { + 0x02, + 0xde, 0x36, 0x0e, 0x87, 0x59, 0x8f, 0x3c, 0x01, 0x36, 0x2a, 0x2a, 0xb8, 0xc6, 0xf4, 0x5e, 0x4d, + 0xb2, 0xc2, 0xd5, 0x03, 0xa7, 0xf9, 0xf1, 0x4f, 0xa8, 0xfa, 0x95, 0xa8, 0xe9, 0x69, 0x76, 0x1c + }; + secp256k1_pubkey pk1; + secp256k1_pubkey pk2; + int32_t ecount = 0; + + CHECK(secp256k1_ec_pubkey_parse(ctx, &pk1, pk1_ser, sizeof(pk1_ser)) == 1); + CHECK(secp256k1_ec_pubkey_parse(ctx, &pk2, pk2_ser, sizeof(pk2_ser)) == 1); + + secp256k1_context_set_illegal_callback(ctx, counting_illegal_callback_fn, &ecount); + CHECK(secp256k1_ec_pubkey_cmp(ctx, NULL, &pk2) < 0); + CHECK(ecount == 1); + CHECK(secp256k1_ec_pubkey_cmp(ctx, &pk1, NULL) > 0); + CHECK(ecount == 2); + CHECK(secp256k1_ec_pubkey_cmp(ctx, &pk1, &pk2) < 0); + CHECK(secp256k1_ec_pubkey_cmp(ctx, &pk2, &pk1) > 0); + CHECK(secp256k1_ec_pubkey_cmp(ctx, &pk1, &pk1) == 0); + CHECK(secp256k1_ec_pubkey_cmp(ctx, &pk2, &pk2) == 0); + CHECK(ecount == 2); + { + secp256k1_pubkey pk_tmp; + memset(&pk_tmp, 0, sizeof(pk_tmp)); /* illegal pubkey */ + CHECK(secp256k1_ec_pubkey_cmp(ctx, &pk_tmp, &pk2) < 0); + CHECK(ecount == 3); + CHECK(secp256k1_ec_pubkey_cmp(ctx, &pk_tmp, &pk_tmp) == 0); + CHECK(ecount == 5); + CHECK(secp256k1_ec_pubkey_cmp(ctx, &pk2, &pk_tmp) > 0); + CHECK(ecount == 6); + } + + secp256k1_context_set_illegal_callback(ctx, NULL, NULL); + + /* Make pk2 the same as pk1 but with 3 rather than 2. Note that in + * an uncompressed encoding, these would have the opposite ordering */ + pk1_ser[0] = 3; + CHECK(secp256k1_ec_pubkey_parse(ctx, &pk2, pk1_ser, sizeof(pk1_ser)) == 1); + CHECK(secp256k1_ec_pubkey_cmp(ctx, &pk1, &pk2) < 0); + CHECK(secp256k1_ec_pubkey_cmp(ctx, &pk2, &pk1) > 0); +} + void run_random_pubkeys(void) { int i; for (i = 0; i < 10*count; i++) { @@ -6408,7 +6567,7 @@ int main(int argc, char **argv) { count = strtol(argv[1], NULL, 0); } else { const char* env = getenv("SECP256K1_TEST_ITERS"); - if (env) { + if (env && strlen(env) > 0) { count = strtol(env, NULL, 0); } } @@ -6442,6 +6601,7 @@ int main(int argc, char **argv) { run_sha256_tests(); run_hmac_sha256_tests(); run_rfc6979_hmac_sha256_tests(); + run_tagged_sha256_tests(); /* scalar tests */ run_scalar_tests(); @@ -6449,6 +6609,7 @@ int main(int argc, char **argv) { /* field tests */ run_field_misc(); run_field_convert(); + run_fe_mul(); run_sqr(); run_sqrt(); @@ -6485,6 +6646,7 @@ int main(int argc, char **argv) { #endif /* ecdsa tests */ + run_pubkey_comparison(); run_random_pubkeys(); run_ecdsa_der_parse(); run_ecdsa_sign_verify(); diff --git a/src/secp256k1/src/tests_exhaustive.c b/src/secp256k1/src/tests_exhaustive.c index 2bb5381446..5b9a3035d9 100644 --- a/src/secp256k1/src/tests_exhaustive.c +++ b/src/secp256k1/src/tests_exhaustive.c @@ -10,7 +10,6 @@ #include <stdio.h> #include <stdlib.h> - #include <time.h> #undef USE_ECMULT_STATIC_PRECOMPUTATION @@ -20,10 +19,10 @@ #define EXHAUSTIVE_TEST_ORDER 13 #endif -#include "include/secp256k1.h" +#include "secp256k1.c" +#include "../include/secp256k1.h" #include "assumptions.h" #include "group.h" -#include "secp256k1.c" #include "testrand_impl.h" static int count = 2; @@ -303,6 +302,7 @@ void test_exhaustive_sign(const secp256k1_context *ctx, const secp256k1_ge *grou if (skip_section(&iter)) continue; for (k = 1; k < EXHAUSTIVE_TEST_ORDER; k++) { /* nonce */ const int starting_k = k; + int ret; secp256k1_ecdsa_signature sig; secp256k1_scalar sk, msg, r, s, expected_r; unsigned char sk32[32], msg32[32]; @@ -311,7 +311,8 @@ void test_exhaustive_sign(const secp256k1_context *ctx, const secp256k1_ge *grou secp256k1_scalar_get_b32(sk32, &sk); secp256k1_scalar_get_b32(msg32, &msg); - secp256k1_ecdsa_sign(ctx, &sig, msg32, sk32, secp256k1_nonce_function_smallint, &k); + ret = secp256k1_ecdsa_sign(ctx, &sig, msg32, sk32, secp256k1_nonce_function_smallint, &k); + CHECK(ret == 1); secp256k1_ecdsa_signature_load(ctx, &r, &s, &sig); /* Note that we compute expected_r *after* signing -- this is important diff --git a/src/secp256k1/src/valgrind_ctime_test.c b/src/secp256k1/src/valgrind_ctime_test.c index cfca5a196e..ea6d4b3deb 100644 --- a/src/secp256k1/src/valgrind_ctime_test.c +++ b/src/secp256k1/src/valgrind_ctime_test.c @@ -7,24 +7,24 @@ #include <valgrind/memcheck.h> #include <stdio.h> -#include "include/secp256k1.h" +#include "../include/secp256k1.h" #include "assumptions.h" #include "util.h" #ifdef ENABLE_MODULE_ECDH -# include "include/secp256k1_ecdh.h" +# include "../include/secp256k1_ecdh.h" #endif #ifdef ENABLE_MODULE_RECOVERY -# include "include/secp256k1_recovery.h" +# include "../include/secp256k1_recovery.h" #endif #ifdef ENABLE_MODULE_EXTRAKEYS -# include "include/secp256k1_extrakeys.h" +# include "../include/secp256k1_extrakeys.h" #endif #ifdef ENABLE_MODULE_SCHNORRSIG -#include "include/secp256k1_schnorrsig.h" +#include "../include/secp256k1_schnorrsig.h" #endif void run_tests(secp256k1_context *ctx, unsigned char *key); @@ -166,7 +166,7 @@ void run_tests(secp256k1_context *ctx, unsigned char *key) { ret = secp256k1_keypair_create(ctx, &keypair, key); VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); CHECK(ret == 1); - ret = secp256k1_schnorrsig_sign(ctx, sig, msg, &keypair, NULL, NULL); + ret = secp256k1_schnorrsig_sign(ctx, sig, msg, &keypair, NULL); VALGRIND_MAKE_MEM_DEFINED(&ret, sizeof(ret)); CHECK(ret == 1); #endif diff --git a/src/signet.cpp b/src/signet.cpp index 1ba8502287..aafd1999ee 100644 --- a/src/signet.cpp +++ b/src/signet.cpp @@ -141,7 +141,7 @@ bool CheckSignetBlockSolution(const CBlock& block, const Consensus::Params& cons PrecomputedTransactionData txdata; txdata.Init(signet_txs->m_to_sign, {signet_txs->m_to_spend.vout[0]}); - TransactionSignatureChecker sigcheck(&signet_txs->m_to_sign, /*nIn=*/ 0, /*amount=*/ signet_txs->m_to_spend.vout[0].nValue, txdata, MissingDataBehavior::ASSERT_FAIL); + TransactionSignatureChecker sigcheck(&signet_txs->m_to_sign, /* nInIn= */ 0, /* amountIn= */ signet_txs->m_to_spend.vout[0].nValue, txdata, MissingDataBehavior::ASSERT_FAIL); if (!VerifyScript(scriptSig, signet_txs->m_to_spend.vout[0].scriptPubKey, &witness, BLOCK_SCRIPT_VERIFY_FLAGS, sigcheck)) { LogPrint(BCLog::VALIDATION, "CheckSignetBlockSolution: Errors in block (block solution invalid)\n"); diff --git a/src/sync.cpp b/src/sync.cpp index a2b62c2286..c9fd8e347e 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -23,17 +23,6 @@ #include <utility> #include <vector> -#ifdef DEBUG_LOCKCONTENTION -#if !defined(HAVE_THREAD_LOCAL) -static_assert(false, "thread_local is not supported"); -#endif -void PrintLockContention(const char* pszName, const char* pszFile, int nLine) -{ - LogPrintf("LOCKCONTENTION: %s\n", pszName); - LogPrintf("Locker: %s:%d\n", pszFile, nLine); -} -#endif /* DEBUG_LOCKCONTENTION */ - #ifdef DEBUG_LOCKORDER // // Early deadlock detection. @@ -108,27 +97,29 @@ static void potential_deadlock_detected(const LockPair& mismatch, const LockStac LogPrintf("POTENTIAL DEADLOCK DETECTED\n"); LogPrintf("Previous lock order was:\n"); for (const LockStackItem& i : s1) { + std::string prefix{}; if (i.first == mismatch.first) { - LogPrintf(" (1)"); /* Continued */ + prefix = " (1)"; } if (i.first == mismatch.second) { - LogPrintf(" (2)"); /* Continued */ + prefix = " (2)"; } - LogPrintf(" %s\n", i.second.ToString()); + LogPrintf("%s %s\n", prefix, i.second.ToString()); } std::string mutex_a, mutex_b; LogPrintf("Current lock order is:\n"); for (const LockStackItem& i : s2) { + std::string prefix{}; if (i.first == mismatch.first) { - LogPrintf(" (1)"); /* Continued */ + prefix = " (1)"; mutex_a = i.second.Name(); } if (i.first == mismatch.second) { - LogPrintf(" (2)"); /* Continued */ + prefix = " (2)"; mutex_b = i.second.Name(); } - LogPrintf(" %s\n", i.second.ToString()); + LogPrintf("%s %s\n", prefix, i.second.ToString()); } if (g_debug_lockorder_abort) { tfm::format(std::cerr, "Assertion failed: detected inconsistent lock order for %s, details in debug log.\n", s2.back().second.ToString()); @@ -142,10 +133,11 @@ static void double_lock_detected(const void* mutex, const LockStack& lock_stack) LogPrintf("DOUBLE LOCK DETECTED\n"); LogPrintf("Lock order:\n"); for (const LockStackItem& i : lock_stack) { + std::string prefix{}; if (i.first == mutex) { - LogPrintf(" (*)"); /* Continued */ + prefix = " (*)"; } - LogPrintf(" %s\n", i.second.ToString()); + LogPrintf("%s %s\n", prefix, i.second.ToString()); } if (g_debug_lockorder_abort) { tfm::format(std::cerr, diff --git a/src/sync.h b/src/sync.h index 146c228592..6ba63d5e4d 100644 --- a/src/sync.h +++ b/src/sync.h @@ -6,6 +6,8 @@ #ifndef BITCOIN_SYNC_H #define BITCOIN_SYNC_H +#include <logging.h> +#include <logging/timer.h> #include <threadsafety.h> #include <util/macros.h> @@ -126,10 +128,6 @@ using RecursiveMutex = AnnotatedMixin<std::recursive_mutex>; /** Wrapped mutex: supports waiting but not recursive locking */ typedef AnnotatedMixin<std::mutex> Mutex; -#ifdef DEBUG_LOCKCONTENTION -void PrintLockContention(const char* pszName, const char* pszFile, int nLine); -#endif - /** Wrapper around std::unique_lock style lock for Mutex. */ template <typename Mutex, typename Base = typename Mutex::UniqueLock> class SCOPED_LOCKABLE UniqueLock : public Base @@ -138,22 +136,18 @@ private: void Enter(const char* pszName, const char* pszFile, int nLine) { EnterCritical(pszName, pszFile, nLine, Base::mutex()); -#ifdef DEBUG_LOCKCONTENTION - if (!Base::try_lock()) { - PrintLockContention(pszName, pszFile, nLine); -#endif - Base::lock(); -#ifdef DEBUG_LOCKCONTENTION - } -#endif + if (Base::try_lock()) return; + LOG_TIME_MICROS_WITH_CATEGORY(strprintf("lock contention %s, %s:%d", pszName, pszFile, nLine), BCLog::LOCK); + Base::lock(); } bool TryEnter(const char* pszName, const char* pszFile, int nLine) { EnterCritical(pszName, pszFile, nLine, Base::mutex(), true); Base::try_lock(); - if (!Base::owns_lock()) + if (!Base::owns_lock()) { LeaveCritical(); + } return Base::owns_lock(); } diff --git a/src/test/README.md b/src/test/README.md index 57cda26d7c..d03411c3ed 100644 --- a/src/test/README.md +++ b/src/test/README.md @@ -74,3 +74,29 @@ start debugging, just like you would with any other program: ```bash gdb src/test/test_bitcoin ``` + +#### Segmentation faults + +If you hit a segmentation fault during a test run, you can diagnose where the fault +is happening by running `gdb ./src/test/test_bitcoin` and then using the `bt` command +within gdb. + +Another tool that can be used to resolve segmentation faults is +[valgrind](https://valgrind.org/). + +If for whatever reason you want to produce a core dump file for this fault, you can do +that as well. By default, the boost test runner will intercept system errors and not +produce a core file. To bypass this, add `--catch_system_errors=no` to the +`test_bitcoin` arguments and ensure that your ulimits are set properly (e.g. `ulimit -c +unlimited`). + +Running the tests and hitting a segmentation fault should now produce a file called `core` +(on Linux platforms, the file name will likely depend on the contents of +`/proc/sys/kernel/core_pattern`). + +You can then explore the core dump using +``` bash +gdb src/test/test_bitcoin core + +(gbd) bt # produce a backtrace for where a segfault occurred +``` diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index eb5c37b34d..bd6f470219 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -1,69 +1,120 @@ // Copyright (c) 2012-2020 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <addrdb.h> #include <addrman.h> +#include <addrman_impl.h> +#include <chainparams.h> +#include <clientversion.h> +#include <hash.h> +#include <netbase.h> +#include <random.h> #include <test/data/asmap.raw.h> #include <test/util/setup_common.h> #include <util/asmap.h> #include <util/string.h> -#include <hash.h> -#include <netbase.h> -#include <random.h> #include <boost/test/unit_test.hpp> #include <optional> #include <string> -class CAddrManTest : public CAddrMan +using namespace std::literals; + +class AddrManSerializationMock : public AddrMan { -private: - bool deterministic; public: - explicit CAddrManTest(bool makeDeterministic = true, - std::vector<bool> asmap = std::vector<bool>()) + virtual void Serialize(CDataStream& s) const = 0; + + AddrManSerializationMock() + : AddrMan(/* asmap */ std::vector<bool>(), /* deterministic */ true, /* consistency_check_ratio */ 100) + {} +}; + +class AddrManUncorrupted : public AddrManSerializationMock +{ +public: + void Serialize(CDataStream& s) const override { - if (makeDeterministic) { - // Set addrman addr placement to be deterministic. - MakeDeterministic(); - } - deterministic = makeDeterministic; - m_asmap = asmap; + AddrMan::Serialize(s); } +}; - //! Ensure that bucket placement is always the same for testing purposes. - void MakeDeterministic() +class AddrManCorrupted : public AddrManSerializationMock +{ +public: + void Serialize(CDataStream& s) const override + { + // Produces corrupt output that claims addrman has 20 addrs when it only has one addr. + unsigned char nVersion = 1; + s << nVersion; + s << ((unsigned char)32); + s << uint256::ONE; + s << 10; // nNew + s << 10; // nTried + + int nUBuckets = ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30); + s << nUBuckets; + + CService serv; + BOOST_CHECK(Lookup("252.1.1.1", serv, 7777, false)); + CAddress addr = CAddress(serv, NODE_NONE); + CNetAddr resolved; + BOOST_CHECK(LookupHost("252.2.2.2", resolved, false)); + AddrInfo info = AddrInfo(addr, resolved); + s << info; + } +}; + +static CDataStream AddrmanToStream(const AddrManSerializationMock& _addrman) +{ + CDataStream ssPeersIn(SER_DISK, CLIENT_VERSION); + ssPeersIn << Params().MessageStart(); + ssPeersIn << _addrman; + std::string str = ssPeersIn.str(); + std::vector<unsigned char> vchData(str.begin(), str.end()); + return CDataStream(vchData, SER_DISK, CLIENT_VERSION); +} + +class AddrManTest : public AddrMan +{ +private: + bool deterministic; +public: + explicit AddrManTest(bool makeDeterministic = true, + std::vector<bool> asmap = std::vector<bool>()) + : AddrMan(asmap, makeDeterministic, /* consistency_check_ratio */ 100) { - nKey.SetNull(); - insecure_rand = FastRandomContext(true); + deterministic = makeDeterministic; } - CAddrInfo* Find(const CNetAddr& addr, int* pnId = nullptr) + AddrInfo* Find(const CNetAddr& addr, int* pnId = nullptr) { - LOCK(cs); - return CAddrMan::Find(addr, pnId); + LOCK(m_impl->cs); + return m_impl->Find(addr, pnId); } - CAddrInfo* Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId = nullptr) + AddrInfo* Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId = nullptr) { - LOCK(cs); - return CAddrMan::Create(addr, addrSource, pnId); + LOCK(m_impl->cs); + return m_impl->Create(addr, addrSource, pnId); } void Delete(int nId) { - LOCK(cs); - CAddrMan::Delete(nId); + LOCK(m_impl->cs); + m_impl->Delete(nId); } // Used to test deserialization std::pair<int, int> GetBucketAndEntry(const CAddress& addr) { - LOCK(cs); - int nId = mapAddr[addr]; + LOCK(m_impl->cs); + int nId = m_impl->mapAddr[addr]; for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; ++bucket) { for (int entry = 0; entry < ADDRMAN_BUCKET_SIZE; ++entry) { - if (nId == vvNew[bucket][entry]) { + if (nId == m_impl->vvNew[bucket][entry]) { return std::pair<int, int>(bucket, entry); } } @@ -76,22 +127,12 @@ public: { int64_t nLastSuccess = 1; // Set last good connection in the deep past. - Good(addr, true, nLastSuccess); + Good(addr, nLastSuccess); bool count_failure = false; int64_t nLastTry = GetAdjustedTime()-61; Attempt(addr, count_failure, nLastTry); } - - void Clear() - { - CAddrMan::Clear(); - if (deterministic) { - nKey.SetNull(); - insecure_rand = FastRandomContext(true); - } - } - }; static CNetAddr ResolveIP(const std::string& ip) @@ -125,27 +166,27 @@ BOOST_FIXTURE_TEST_SUITE(addrman_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(addrman_simple) { - CAddrManTest addrman; + auto addrman = std::make_unique<AddrManTest>(); CNetAddr source = ResolveIP("252.2.2.2"); // Test: Does Addrman respond correctly when empty. - BOOST_CHECK_EQUAL(addrman.size(), 0U); - CAddrInfo addr_null = addrman.Select(); + BOOST_CHECK_EQUAL(addrman->size(), 0U); + auto addr_null = addrman->Select().first; BOOST_CHECK_EQUAL(addr_null.ToString(), "[::]:0"); // Test: Does Addrman::Add work as expected. CService addr1 = ResolveService("250.1.1.1", 8333); - BOOST_CHECK(addrman.Add(CAddress(addr1, NODE_NONE), source)); - BOOST_CHECK_EQUAL(addrman.size(), 1U); - CAddrInfo addr_ret1 = addrman.Select(); + BOOST_CHECK(addrman->Add({CAddress(addr1, NODE_NONE)}, source)); + BOOST_CHECK_EQUAL(addrman->size(), 1U); + auto addr_ret1 = addrman->Select().first; BOOST_CHECK_EQUAL(addr_ret1.ToString(), "250.1.1.1:8333"); // Test: Does IP address deduplication work correctly. // Expected dup IP should not be added. CService addr1_dup = ResolveService("250.1.1.1", 8333); - BOOST_CHECK(!addrman.Add(CAddress(addr1_dup, NODE_NONE), source)); - BOOST_CHECK_EQUAL(addrman.size(), 1U); + BOOST_CHECK(!addrman->Add({CAddress(addr1_dup, NODE_NONE)}, source)); + BOOST_CHECK_EQUAL(addrman->size(), 1U); // Test: New table has one addr and we add a diff addr we should @@ -155,26 +196,21 @@ BOOST_AUTO_TEST_CASE(addrman_simple) // success. CService addr2 = ResolveService("250.1.1.2", 8333); - BOOST_CHECK(addrman.Add(CAddress(addr2, NODE_NONE), source)); - BOOST_CHECK(addrman.size() >= 1); - - // Test: AddrMan::Clear() should empty the new table. - addrman.Clear(); - BOOST_CHECK_EQUAL(addrman.size(), 0U); - CAddrInfo addr_null2 = addrman.Select(); - BOOST_CHECK_EQUAL(addr_null2.ToString(), "[::]:0"); + BOOST_CHECK(addrman->Add({CAddress(addr2, NODE_NONE)}, source)); + BOOST_CHECK(addrman->size() >= 1); - // Test: AddrMan::Add multiple addresses works as expected + // Test: reset addrman and test AddrMan::Add multiple addresses works as expected + addrman = std::make_unique<AddrManTest>(); std::vector<CAddress> vAddr; vAddr.push_back(CAddress(ResolveService("250.1.1.3", 8333), NODE_NONE)); vAddr.push_back(CAddress(ResolveService("250.1.1.4", 8333), NODE_NONE)); - BOOST_CHECK(addrman.Add(vAddr, source)); - BOOST_CHECK(addrman.size() >= 1); + BOOST_CHECK(addrman->Add(vAddr, source)); + BOOST_CHECK(addrman->size() >= 1); } BOOST_AUTO_TEST_CASE(addrman_ports) { - CAddrManTest addrman; + AddrManTest addrman; CNetAddr source = ResolveIP("252.2.2.2"); @@ -182,13 +218,13 @@ BOOST_AUTO_TEST_CASE(addrman_ports) // Test 7; Addr with same IP but diff port does not replace existing addr. CService addr1 = ResolveService("250.1.1.1", 8333); - BOOST_CHECK(addrman.Add(CAddress(addr1, NODE_NONE), source)); + BOOST_CHECK(addrman.Add({CAddress(addr1, NODE_NONE)}, source)); BOOST_CHECK_EQUAL(addrman.size(), 1U); CService addr1_port = ResolveService("250.1.1.1", 8334); - BOOST_CHECK(!addrman.Add(CAddress(addr1_port, NODE_NONE), source)); + BOOST_CHECK(!addrman.Add({CAddress(addr1_port, NODE_NONE)}, source)); BOOST_CHECK_EQUAL(addrman.size(), 1U); - CAddrInfo addr_ret2 = addrman.Select(); + auto addr_ret2 = addrman.Select().first; BOOST_CHECK_EQUAL(addr_ret2.ToString(), "250.1.1.1:8333"); // Test: Add same IP but diff port to tried table, it doesn't get added. @@ -196,33 +232,33 @@ BOOST_AUTO_TEST_CASE(addrman_ports) addrman.Good(CAddress(addr1_port, NODE_NONE)); BOOST_CHECK_EQUAL(addrman.size(), 1U); bool newOnly = true; - CAddrInfo addr_ret3 = addrman.Select(newOnly); + auto addr_ret3 = addrman.Select(newOnly).first; BOOST_CHECK_EQUAL(addr_ret3.ToString(), "250.1.1.1:8333"); } BOOST_AUTO_TEST_CASE(addrman_select) { - CAddrManTest addrman; + AddrManTest addrman; CNetAddr source = ResolveIP("252.2.2.2"); // Test: Select from new with 1 addr in new. CService addr1 = ResolveService("250.1.1.1", 8333); - BOOST_CHECK(addrman.Add(CAddress(addr1, NODE_NONE), source)); + BOOST_CHECK(addrman.Add({CAddress(addr1, NODE_NONE)}, source)); BOOST_CHECK_EQUAL(addrman.size(), 1U); bool newOnly = true; - CAddrInfo addr_ret1 = addrman.Select(newOnly); + auto addr_ret1 = addrman.Select(newOnly).first; BOOST_CHECK_EQUAL(addr_ret1.ToString(), "250.1.1.1:8333"); // Test: move addr to tried, select from new expected nothing returned. addrman.Good(CAddress(addr1, NODE_NONE)); BOOST_CHECK_EQUAL(addrman.size(), 1U); - CAddrInfo addr_ret2 = addrman.Select(newOnly); + auto addr_ret2 = addrman.Select(newOnly).first; BOOST_CHECK_EQUAL(addr_ret2.ToString(), "[::]:0"); - CAddrInfo addr_ret3 = addrman.Select(); + auto addr_ret3 = addrman.Select().first; BOOST_CHECK_EQUAL(addr_ret3.ToString(), "250.1.1.1:8333"); BOOST_CHECK_EQUAL(addrman.size(), 1U); @@ -233,20 +269,20 @@ BOOST_AUTO_TEST_CASE(addrman_select) CService addr3 = ResolveService("250.3.2.2", 9999); CService addr4 = ResolveService("250.3.3.3", 9999); - BOOST_CHECK(addrman.Add(CAddress(addr2, NODE_NONE), ResolveService("250.3.1.1", 8333))); - BOOST_CHECK(addrman.Add(CAddress(addr3, NODE_NONE), ResolveService("250.3.1.1", 8333))); - BOOST_CHECK(addrman.Add(CAddress(addr4, NODE_NONE), ResolveService("250.4.1.1", 8333))); + BOOST_CHECK(addrman.Add({CAddress(addr2, NODE_NONE)}, ResolveService("250.3.1.1", 8333))); + BOOST_CHECK(addrman.Add({CAddress(addr3, NODE_NONE)}, ResolveService("250.3.1.1", 8333))); + BOOST_CHECK(addrman.Add({CAddress(addr4, NODE_NONE)}, ResolveService("250.4.1.1", 8333))); // Add three addresses to tried table. CService addr5 = ResolveService("250.4.4.4", 8333); CService addr6 = ResolveService("250.4.5.5", 7777); CService addr7 = ResolveService("250.4.6.6", 8333); - BOOST_CHECK(addrman.Add(CAddress(addr5, NODE_NONE), ResolveService("250.3.1.1", 8333))); + BOOST_CHECK(addrman.Add({CAddress(addr5, NODE_NONE)}, ResolveService("250.3.1.1", 8333))); addrman.Good(CAddress(addr5, NODE_NONE)); - BOOST_CHECK(addrman.Add(CAddress(addr6, NODE_NONE), ResolveService("250.3.1.1", 8333))); + BOOST_CHECK(addrman.Add({CAddress(addr6, NODE_NONE)}, ResolveService("250.3.1.1", 8333))); addrman.Good(CAddress(addr6, NODE_NONE)); - BOOST_CHECK(addrman.Add(CAddress(addr7, NODE_NONE), ResolveService("250.1.1.3", 8333))); + BOOST_CHECK(addrman.Add({CAddress(addr7, NODE_NONE)}, ResolveService("250.1.1.3", 8333))); addrman.Good(CAddress(addr7, NODE_NONE)); // Test: 6 addrs + 1 addr from last test = 7. @@ -255,67 +291,73 @@ BOOST_AUTO_TEST_CASE(addrman_select) // Test: Select pulls from new and tried regardless of port number. std::set<uint16_t> ports; for (int i = 0; i < 20; ++i) { - ports.insert(addrman.Select().GetPort()); + ports.insert(addrman.Select().first.GetPort()); } BOOST_CHECK_EQUAL(ports.size(), 3U); } BOOST_AUTO_TEST_CASE(addrman_new_collisions) { - CAddrManTest addrman; + AddrManTest addrman; CNetAddr source = ResolveIP("252.2.2.2"); - BOOST_CHECK_EQUAL(addrman.size(), 0U); + uint32_t num_addrs{0}; + + BOOST_CHECK_EQUAL(addrman.size(), num_addrs); - for (unsigned int i = 1; i < 18; i++) { - CService addr = ResolveService("250.1.1." + ToString(i)); - BOOST_CHECK(addrman.Add(CAddress(addr, NODE_NONE), source)); + while (num_addrs < 22) { // Magic number! 250.1.1.1 - 250.1.1.22 do not collide with deterministic key = 1 + CService addr = ResolveService("250.1.1." + ToString(++num_addrs)); + BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); //Test: No collision in new table yet. - BOOST_CHECK_EQUAL(addrman.size(), i); + BOOST_CHECK_EQUAL(addrman.size(), num_addrs); } //Test: new table collision! - CService addr1 = ResolveService("250.1.1.18"); - BOOST_CHECK(addrman.Add(CAddress(addr1, NODE_NONE), source)); - BOOST_CHECK_EQUAL(addrman.size(), 17U); - - CService addr2 = ResolveService("250.1.1.19"); - BOOST_CHECK(addrman.Add(CAddress(addr2, NODE_NONE), source)); - BOOST_CHECK_EQUAL(addrman.size(), 18U); + CService addr1 = ResolveService("250.1.1." + ToString(++num_addrs)); + uint32_t collisions{1}; + BOOST_CHECK(addrman.Add({CAddress(addr1, NODE_NONE)}, source)); + BOOST_CHECK_EQUAL(addrman.size(), num_addrs - collisions); + + CService addr2 = ResolveService("250.1.1." + ToString(++num_addrs)); + BOOST_CHECK(addrman.Add({CAddress(addr2, NODE_NONE)}, source)); + BOOST_CHECK_EQUAL(addrman.size(), num_addrs - collisions); } BOOST_AUTO_TEST_CASE(addrman_tried_collisions) { - CAddrManTest addrman; + AddrManTest addrman; CNetAddr source = ResolveIP("252.2.2.2"); - BOOST_CHECK_EQUAL(addrman.size(), 0U); + uint32_t num_addrs{0}; + + BOOST_CHECK_EQUAL(addrman.size(), num_addrs); - for (unsigned int i = 1; i < 80; i++) { - CService addr = ResolveService("250.1.1." + ToString(i)); - BOOST_CHECK(addrman.Add(CAddress(addr, NODE_NONE), source)); + while (num_addrs < 64) { // Magic number! 250.1.1.1 - 250.1.1.64 do not collide with deterministic key = 1 + CService addr = ResolveService("250.1.1." + ToString(++num_addrs)); + BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); addrman.Good(CAddress(addr, NODE_NONE)); //Test: No collision in tried table yet. - BOOST_CHECK_EQUAL(addrman.size(), i); + BOOST_CHECK_EQUAL(addrman.size(), num_addrs); } //Test: tried table collision! - CService addr1 = ResolveService("250.1.1.80"); - BOOST_CHECK(addrman.Add(CAddress(addr1, NODE_NONE), source)); - BOOST_CHECK_EQUAL(addrman.size(), 79U); - - CService addr2 = ResolveService("250.1.1.81"); - BOOST_CHECK(addrman.Add(CAddress(addr2, NODE_NONE), source)); - BOOST_CHECK_EQUAL(addrman.size(), 80U); + CService addr1 = ResolveService("250.1.1." + ToString(++num_addrs)); + uint32_t collisions{1}; + BOOST_CHECK(addrman.Add({CAddress(addr1, NODE_NONE)}, source)); + BOOST_CHECK_EQUAL(addrman.size(), num_addrs - collisions); + + CService addr2 = ResolveService("250.1.1." + ToString(++num_addrs)); + BOOST_CHECK(addrman.Add({CAddress(addr2, NODE_NONE)}, source)); + BOOST_CHECK_EQUAL(addrman.size(), num_addrs - collisions); } BOOST_AUTO_TEST_CASE(addrman_find) { - CAddrManTest addrman; + AddrManTest addrman; BOOST_CHECK_EQUAL(addrman.size(), 0U); @@ -326,29 +368,29 @@ BOOST_AUTO_TEST_CASE(addrman_find) CNetAddr source1 = ResolveIP("250.1.2.1"); CNetAddr source2 = ResolveIP("250.1.2.2"); - BOOST_CHECK(addrman.Add(addr1, source1)); - BOOST_CHECK(!addrman.Add(addr2, source2)); - BOOST_CHECK(addrman.Add(addr3, source1)); + BOOST_CHECK(addrman.Add({addr1}, source1)); + BOOST_CHECK(!addrman.Add({addr2}, source2)); + BOOST_CHECK(addrman.Add({addr3}, source1)); // Test: ensure Find returns an IP matching what we searched on. - CAddrInfo* info1 = addrman.Find(addr1); + AddrInfo* info1 = addrman.Find(addr1); BOOST_REQUIRE(info1); BOOST_CHECK_EQUAL(info1->ToString(), "250.1.2.1:8333"); // Test 18; Find does not discriminate by port number. - CAddrInfo* info2 = addrman.Find(addr2); + AddrInfo* info2 = addrman.Find(addr2); BOOST_REQUIRE(info2); BOOST_CHECK_EQUAL(info2->ToString(), info1->ToString()); // Test: Find returns another IP matching what we searched on. - CAddrInfo* info3 = addrman.Find(addr3); + AddrInfo* info3 = addrman.Find(addr3); BOOST_REQUIRE(info3); BOOST_CHECK_EQUAL(info3->ToString(), "251.255.2.1:8333"); } BOOST_AUTO_TEST_CASE(addrman_create) { - CAddrManTest addrman; + AddrManTest addrman; BOOST_CHECK_EQUAL(addrman.size(), 0U); @@ -356,19 +398,19 @@ BOOST_AUTO_TEST_CASE(addrman_create) CNetAddr source1 = ResolveIP("250.1.2.1"); int nId; - CAddrInfo* pinfo = addrman.Create(addr1, source1, &nId); + AddrInfo* pinfo = addrman.Create(addr1, source1, &nId); // Test: The result should be the same as the input addr. BOOST_CHECK_EQUAL(pinfo->ToString(), "250.1.2.1:8333"); - CAddrInfo* info2 = addrman.Find(addr1); + AddrInfo* info2 = addrman.Find(addr1); BOOST_CHECK_EQUAL(info2->ToString(), "250.1.2.1:8333"); } BOOST_AUTO_TEST_CASE(addrman_delete) { - CAddrManTest addrman; + AddrManTest addrman; BOOST_CHECK_EQUAL(addrman.size(), 0U); @@ -382,13 +424,13 @@ BOOST_AUTO_TEST_CASE(addrman_delete) BOOST_CHECK_EQUAL(addrman.size(), 1U); addrman.Delete(nId); BOOST_CHECK_EQUAL(addrman.size(), 0U); - CAddrInfo* info2 = addrman.Find(addr1); + AddrInfo* info2 = addrman.Find(addr1); BOOST_CHECK(info2 == nullptr); } BOOST_AUTO_TEST_CASE(addrman_getaddr) { - CAddrManTest addrman; + AddrManTest addrman; // Test: Sanity check, GetAddr should never return anything if addrman // is empty. @@ -410,11 +452,8 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) CNetAddr source2 = ResolveIP("250.2.3.3"); // Test: Ensure GetAddr works with new addresses. - BOOST_CHECK(addrman.Add(addr1, source1)); - BOOST_CHECK(addrman.Add(addr2, source2)); - BOOST_CHECK(addrman.Add(addr3, source1)); - BOOST_CHECK(addrman.Add(addr4, source2)); - BOOST_CHECK(addrman.Add(addr5, source1)); + BOOST_CHECK(addrman.Add({addr1, addr3, addr5}, source1)); + BOOST_CHECK(addrman.Add({addr2, addr4}, source2)); BOOST_CHECK_EQUAL(addrman.GetAddr(/* max_addresses */ 0, /* max_pct */ 0, /* network */ std::nullopt).size(), 5U); // Net processing asks for 23% of addresses. 23% of 5 is 1 rounded down. @@ -435,7 +474,7 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) // Ensure that for all addrs in addrman, isTerrible == false. addr.nTime = GetAdjustedTime(); - addrman.Add(addr, ResolveIP(strAddr)); + addrman.Add({addr}, ResolveIP(strAddr)); if (i % 8 == 0) addrman.Good(addr); } @@ -451,7 +490,7 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) { - CAddrManTest addrman; + AddrManTest addrman; CAddress addr1 = CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.1.1", 9999), NODE_NONE); @@ -459,7 +498,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) CNetAddr source1 = ResolveIP("250.1.1.1"); - CAddrInfo info1 = CAddrInfo(addr1, source1); + AddrInfo info1 = AddrInfo(addr1, source1); uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); @@ -474,14 +513,14 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) // Test: Two addresses with same IP but different ports can map to // different buckets because they have different keys. - CAddrInfo info2 = CAddrInfo(addr2, source1); + AddrInfo info2 = AddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); std::set<int> buckets; for (int i = 0; i < 255; i++) { - CAddrInfo infoi = CAddrInfo( + AddrInfo infoi = AddrInfo( CAddress(ResolveService("250.1.1." + ToString(i)), NODE_NONE), ResolveIP("250.1.1." + ToString(i))); int bucket = infoi.GetTriedBucket(nKey1, asmap); @@ -493,7 +532,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) buckets.clear(); for (int j = 0; j < 255; j++) { - CAddrInfo infoj = CAddrInfo( + AddrInfo infoj = AddrInfo( CAddress(ResolveService("250." + ToString(j) + ".1.1"), NODE_NONE), ResolveIP("250." + ToString(j) + ".1.1")); int bucket = infoj.GetTriedBucket(nKey1, asmap); @@ -506,14 +545,14 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) { - CAddrManTest addrman; + AddrManTest addrman; CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); CNetAddr source1 = ResolveIP("250.1.2.1"); - CAddrInfo info1 = CAddrInfo(addr1, source1); + AddrInfo info1 = AddrInfo(addr1, source1); uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); @@ -529,13 +568,13 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); // Test: Ports should not affect bucket placement in the addr - CAddrInfo info2 = CAddrInfo(addr2, source1); + AddrInfo info2 = AddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); std::set<int> buckets; for (int i = 0; i < 255; i++) { - CAddrInfo infoi = CAddrInfo( + AddrInfo infoi = AddrInfo( CAddress(ResolveService("250.1.1." + ToString(i)), NODE_NONE), ResolveIP("250.1.1." + ToString(i))); int bucket = infoi.GetNewBucket(nKey1, asmap); @@ -547,7 +586,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) buckets.clear(); for (int j = 0; j < 4 * 255; j++) { - CAddrInfo infoj = CAddrInfo(CAddress( + AddrInfo infoj = AddrInfo(CAddress( ResolveService( ToString(250 + (j / 255)) + "." + ToString(j % 256) + ".1.1"), NODE_NONE), ResolveIP("251.4.1.1")); @@ -560,7 +599,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) buckets.clear(); for (int p = 0; p < 255; p++) { - CAddrInfo infoj = CAddrInfo( + AddrInfo infoj = AddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), ResolveIP("250." + ToString(p) + ".1.1")); int bucket = infoj.GetNewBucket(nKey1, asmap); @@ -584,7 +623,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) // 101.8.0.0/16 AS8 BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) { - CAddrManTest addrman; + AddrManTest addrman; CAddress addr1 = CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.1.1", 9999), NODE_NONE); @@ -592,7 +631,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) CNetAddr source1 = ResolveIP("250.1.1.1"); - CAddrInfo info1 = CAddrInfo(addr1, source1); + AddrInfo info1 = AddrInfo(addr1, source1); uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); @@ -607,14 +646,14 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) // Test: Two addresses with same IP but different ports can map to // different buckets because they have different keys. - CAddrInfo info2 = CAddrInfo(addr2, source1); + AddrInfo info2 = AddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); std::set<int> buckets; for (int j = 0; j < 255; j++) { - CAddrInfo infoj = CAddrInfo( + AddrInfo infoj = AddrInfo( CAddress(ResolveService("101." + ToString(j) + ".1.1"), NODE_NONE), ResolveIP("101." + ToString(j) + ".1.1")); int bucket = infoj.GetTriedBucket(nKey1, asmap); @@ -626,7 +665,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) buckets.clear(); for (int j = 0; j < 255; j++) { - CAddrInfo infoj = CAddrInfo( + AddrInfo infoj = AddrInfo( CAddress(ResolveService("250." + ToString(j) + ".1.1"), NODE_NONE), ResolveIP("250." + ToString(j) + ".1.1")); int bucket = infoj.GetTriedBucket(nKey1, asmap); @@ -639,14 +678,14 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) { - CAddrManTest addrman; + AddrManTest addrman; CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); CNetAddr source1 = ResolveIP("250.1.2.1"); - CAddrInfo info1 = CAddrInfo(addr1, source1); + AddrInfo info1 = AddrInfo(addr1, source1); uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); @@ -662,13 +701,13 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); // Test: Ports should not affect bucket placement in the addr - CAddrInfo info2 = CAddrInfo(addr2, source1); + AddrInfo info2 = AddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); std::set<int> buckets; for (int i = 0; i < 255; i++) { - CAddrInfo infoi = CAddrInfo( + AddrInfo infoi = AddrInfo( CAddress(ResolveService("250.1.1." + ToString(i)), NODE_NONE), ResolveIP("250.1.1." + ToString(i))); int bucket = infoi.GetNewBucket(nKey1, asmap); @@ -680,7 +719,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) buckets.clear(); for (int j = 0; j < 4 * 255; j++) { - CAddrInfo infoj = CAddrInfo(CAddress( + AddrInfo infoj = AddrInfo(CAddress( ResolveService( ToString(250 + (j / 255)) + "." + ToString(j % 256) + ".1.1"), NODE_NONE), ResolveIP("251.4.1.1")); @@ -693,7 +732,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) buckets.clear(); for (int p = 0; p < 255; p++) { - CAddrInfo infoj = CAddrInfo( + AddrInfo infoj = AddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), ResolveIP("101." + ToString(p) + ".1.1")); int bucket = infoj.GetNewBucket(nKey1, asmap); @@ -705,7 +744,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) buckets.clear(); for (int p = 0; p < 255; p++) { - CAddrInfo infoj = CAddrInfo( + AddrInfo infoj = AddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), ResolveIP("250." + ToString(p) + ".1.1")); int bucket = infoj.GetNewBucket(nKey1, asmap); @@ -721,23 +760,23 @@ BOOST_AUTO_TEST_CASE(addrman_serialization) { std::vector<bool> asmap1 = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); - CAddrManTest addrman_asmap1(true, asmap1); - CAddrManTest addrman_asmap1_dup(true, asmap1); - CAddrManTest addrman_noasmap; + auto addrman_asmap1 = std::make_unique<AddrManTest>(true, asmap1); + auto addrman_asmap1_dup = std::make_unique<AddrManTest>(true, asmap1); + auto addrman_noasmap = std::make_unique<AddrManTest>(); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); CAddress addr = CAddress(ResolveService("250.1.1.1"), NODE_NONE); CNetAddr default_source; - addrman_asmap1.Add(addr, default_source); + addrman_asmap1->Add({addr}, default_source); - stream << addrman_asmap1; + stream << *addrman_asmap1; // serizalizing/deserializing addrman with the same asmap - stream >> addrman_asmap1_dup; + stream >> *addrman_asmap1_dup; - std::pair<int, int> bucketAndEntry_asmap1 = addrman_asmap1.GetBucketAndEntry(addr); - std::pair<int, int> bucketAndEntry_asmap1_dup = addrman_asmap1_dup.GetBucketAndEntry(addr); + std::pair<int, int> bucketAndEntry_asmap1 = addrman_asmap1->GetBucketAndEntry(addr); + std::pair<int, int> bucketAndEntry_asmap1_dup = addrman_asmap1_dup->GetBucketAndEntry(addr); BOOST_CHECK(bucketAndEntry_asmap1.second != -1); BOOST_CHECK(bucketAndEntry_asmap1_dup.second != -1); @@ -745,64 +784,103 @@ BOOST_AUTO_TEST_CASE(addrman_serialization) BOOST_CHECK(bucketAndEntry_asmap1.second == bucketAndEntry_asmap1_dup.second); // deserializing asmaped peers.dat to non-asmaped addrman - stream << addrman_asmap1; - stream >> addrman_noasmap; - std::pair<int, int> bucketAndEntry_noasmap = addrman_noasmap.GetBucketAndEntry(addr); + stream << *addrman_asmap1; + stream >> *addrman_noasmap; + std::pair<int, int> bucketAndEntry_noasmap = addrman_noasmap->GetBucketAndEntry(addr); BOOST_CHECK(bucketAndEntry_noasmap.second != -1); BOOST_CHECK(bucketAndEntry_asmap1.first != bucketAndEntry_noasmap.first); BOOST_CHECK(bucketAndEntry_asmap1.second != bucketAndEntry_noasmap.second); // deserializing non-asmaped peers.dat to asmaped addrman - addrman_asmap1.Clear(); - addrman_noasmap.Clear(); - addrman_noasmap.Add(addr, default_source); - stream << addrman_noasmap; - stream >> addrman_asmap1; - std::pair<int, int> bucketAndEntry_asmap1_deser = addrman_asmap1.GetBucketAndEntry(addr); + addrman_asmap1 = std::make_unique<AddrManTest>(true, asmap1); + addrman_noasmap = std::make_unique<AddrManTest>(); + addrman_noasmap->Add({addr}, default_source); + stream << *addrman_noasmap; + stream >> *addrman_asmap1; + std::pair<int, int> bucketAndEntry_asmap1_deser = addrman_asmap1->GetBucketAndEntry(addr); BOOST_CHECK(bucketAndEntry_asmap1_deser.second != -1); BOOST_CHECK(bucketAndEntry_asmap1_deser.first != bucketAndEntry_noasmap.first); BOOST_CHECK(bucketAndEntry_asmap1_deser.first == bucketAndEntry_asmap1_dup.first); BOOST_CHECK(bucketAndEntry_asmap1_deser.second == bucketAndEntry_asmap1_dup.second); // used to map to different buckets, now maps to the same bucket. - addrman_asmap1.Clear(); - addrman_noasmap.Clear(); + addrman_asmap1 = std::make_unique<AddrManTest>(true, asmap1); + addrman_noasmap = std::make_unique<AddrManTest>(); CAddress addr1 = CAddress(ResolveService("250.1.1.1"), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.2.1.1"), NODE_NONE); - addrman_noasmap.Add(addr, default_source); - addrman_noasmap.Add(addr2, default_source); - std::pair<int, int> bucketAndEntry_noasmap_addr1 = addrman_noasmap.GetBucketAndEntry(addr1); - std::pair<int, int> bucketAndEntry_noasmap_addr2 = addrman_noasmap.GetBucketAndEntry(addr2); + addrman_noasmap->Add({addr, addr2}, default_source); + std::pair<int, int> bucketAndEntry_noasmap_addr1 = addrman_noasmap->GetBucketAndEntry(addr1); + std::pair<int, int> bucketAndEntry_noasmap_addr2 = addrman_noasmap->GetBucketAndEntry(addr2); BOOST_CHECK(bucketAndEntry_noasmap_addr1.first != bucketAndEntry_noasmap_addr2.first); BOOST_CHECK(bucketAndEntry_noasmap_addr1.second != bucketAndEntry_noasmap_addr2.second); - stream << addrman_noasmap; - stream >> addrman_asmap1; - std::pair<int, int> bucketAndEntry_asmap1_deser_addr1 = addrman_asmap1.GetBucketAndEntry(addr1); - std::pair<int, int> bucketAndEntry_asmap1_deser_addr2 = addrman_asmap1.GetBucketAndEntry(addr2); + stream << *addrman_noasmap; + stream >> *addrman_asmap1; + std::pair<int, int> bucketAndEntry_asmap1_deser_addr1 = addrman_asmap1->GetBucketAndEntry(addr1); + std::pair<int, int> bucketAndEntry_asmap1_deser_addr2 = addrman_asmap1->GetBucketAndEntry(addr2); BOOST_CHECK(bucketAndEntry_asmap1_deser_addr1.first == bucketAndEntry_asmap1_deser_addr2.first); BOOST_CHECK(bucketAndEntry_asmap1_deser_addr1.second != bucketAndEntry_asmap1_deser_addr2.second); } +BOOST_AUTO_TEST_CASE(remove_invalid) +{ + // Confirm that invalid addresses are ignored in unserialization. + + auto addrman = std::make_unique<AddrManTest>(); + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + + const CAddress new1{ResolveService("5.5.5.5"), NODE_NONE}; + const CAddress new2{ResolveService("6.6.6.6"), NODE_NONE}; + const CAddress tried1{ResolveService("7.7.7.7"), NODE_NONE}; + const CAddress tried2{ResolveService("8.8.8.8"), NODE_NONE}; + + addrman->Add({new1, tried1, new2, tried2}, CNetAddr{}); + addrman->Good(tried1); + addrman->Good(tried2); + BOOST_REQUIRE_EQUAL(addrman->size(), 4); + + stream << *addrman; + + const std::string str{stream.str()}; + size_t pos; + + const char new2_raw[]{6, 6, 6, 6}; + const uint8_t new2_raw_replacement[]{0, 0, 0, 0}; // 0.0.0.0 is !IsValid() + pos = str.find(new2_raw, 0, sizeof(new2_raw)); + BOOST_REQUIRE(pos != std::string::npos); + BOOST_REQUIRE(pos + sizeof(new2_raw_replacement) <= stream.size()); + memcpy(stream.data() + pos, new2_raw_replacement, sizeof(new2_raw_replacement)); + + const char tried2_raw[]{8, 8, 8, 8}; + const uint8_t tried2_raw_replacement[]{255, 255, 255, 255}; // 255.255.255.255 is !IsValid() + pos = str.find(tried2_raw, 0, sizeof(tried2_raw)); + BOOST_REQUIRE(pos != std::string::npos); + BOOST_REQUIRE(pos + sizeof(tried2_raw_replacement) <= stream.size()); + memcpy(stream.data() + pos, tried2_raw_replacement, sizeof(tried2_raw_replacement)); + + addrman = std::make_unique<AddrManTest>(); + stream >> *addrman; + BOOST_CHECK_EQUAL(addrman->size(), 2); +} BOOST_AUTO_TEST_CASE(addrman_selecttriedcollision) { - CAddrManTest addrman; + AddrManTest addrman; BOOST_CHECK(addrman.size() == 0); // Empty addrman should return blank addrman info. - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); // Add twenty two addresses. CNetAddr source = ResolveIP("252.2.2.2"); for (unsigned int i = 1; i < 23; i++) { CService addr = ResolveService("250.1.1."+ToString(i)); - BOOST_CHECK(addrman.Add(CAddress(addr, NODE_NONE), source)); + BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); addrman.Good(addr); // No collisions yet. BOOST_CHECK(addrman.size() == i); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } // Ensure Good handles duplicates well. @@ -811,119 +889,192 @@ BOOST_AUTO_TEST_CASE(addrman_selecttriedcollision) addrman.Good(addr); BOOST_CHECK(addrman.size() == 22); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } } BOOST_AUTO_TEST_CASE(addrman_noevict) { - CAddrManTest addrman; + AddrManTest addrman; - // Add twenty two addresses. + // Add 35 addresses. CNetAddr source = ResolveIP("252.2.2.2"); - for (unsigned int i = 1; i < 23; i++) { + for (unsigned int i = 1; i < 36; i++) { CService addr = ResolveService("250.1.1."+ToString(i)); - BOOST_CHECK(addrman.Add(CAddress(addr, NODE_NONE), source)); + BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); addrman.Good(addr); // No collision yet. BOOST_CHECK(addrman.size() == i); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } - // Collision between 23 and 19. - CService addr23 = ResolveService("250.1.1.23"); - BOOST_CHECK(addrman.Add(CAddress(addr23, NODE_NONE), source)); - addrman.Good(addr23); + // Collision between 36 and 19. + CService addr36 = ResolveService("250.1.1.36"); + BOOST_CHECK(addrman.Add({CAddress(addr36, NODE_NONE)}, source)); + addrman.Good(addr36); - BOOST_CHECK(addrman.size() == 23); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "250.1.1.19:0"); + BOOST_CHECK(addrman.size() == 36); + BOOST_CHECK_EQUAL(addrman.SelectTriedCollision().first.ToString(), "250.1.1.19:0"); - // 23 should be discarded and 19 not evicted. + // 36 should be discarded and 19 not evicted. addrman.ResolveCollisions(); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); // Lets create two collisions. - for (unsigned int i = 24; i < 33; i++) { + for (unsigned int i = 37; i < 59; i++) { CService addr = ResolveService("250.1.1."+ToString(i)); - BOOST_CHECK(addrman.Add(CAddress(addr, NODE_NONE), source)); + BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); addrman.Good(addr); BOOST_CHECK(addrman.size() == i); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } // Cause a collision. - CService addr33 = ResolveService("250.1.1.33"); - BOOST_CHECK(addrman.Add(CAddress(addr33, NODE_NONE), source)); - addrman.Good(addr33); - BOOST_CHECK(addrman.size() == 33); + CService addr59 = ResolveService("250.1.1.59"); + BOOST_CHECK(addrman.Add({CAddress(addr59, NODE_NONE)}, source)); + addrman.Good(addr59); + BOOST_CHECK(addrman.size() == 59); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "250.1.1.27:0"); + BOOST_CHECK_EQUAL(addrman.SelectTriedCollision().first.ToString(), "250.1.1.10:0"); // Cause a second collision. - BOOST_CHECK(!addrman.Add(CAddress(addr23, NODE_NONE), source)); - addrman.Good(addr23); - BOOST_CHECK(addrman.size() == 33); + BOOST_CHECK(!addrman.Add({CAddress(addr36, NODE_NONE)}, source)); + addrman.Good(addr36); + BOOST_CHECK(addrman.size() == 59); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() != "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() != "[::]:0"); addrman.ResolveCollisions(); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } BOOST_AUTO_TEST_CASE(addrman_evictionworks) { - CAddrManTest addrman; + AddrManTest addrman; BOOST_CHECK(addrman.size() == 0); // Empty addrman should return blank addrman info. - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); - // Add twenty two addresses. + // Add 35 addresses CNetAddr source = ResolveIP("252.2.2.2"); - for (unsigned int i = 1; i < 23; i++) { + for (unsigned int i = 1; i < 36; i++) { CService addr = ResolveService("250.1.1."+ToString(i)); - BOOST_CHECK(addrman.Add(CAddress(addr, NODE_NONE), source)); + BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); addrman.Good(addr); // No collision yet. BOOST_CHECK(addrman.size() == i); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } - // Collision between 23 and 19. - CService addr = ResolveService("250.1.1.23"); - BOOST_CHECK(addrman.Add(CAddress(addr, NODE_NONE), source)); + // Collision between 36 and 19. + CService addr = ResolveService("250.1.1.36"); + BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); addrman.Good(addr); - BOOST_CHECK(addrman.size() == 23); - CAddrInfo info = addrman.SelectTriedCollision(); - BOOST_CHECK(info.ToString() == "250.1.1.19:0"); + BOOST_CHECK_EQUAL(addrman.size(), 36); + auto info = addrman.SelectTriedCollision().first; + BOOST_CHECK_EQUAL(info.ToString(), "250.1.1.19:0"); // Ensure test of address fails, so that it is evicted. addrman.SimConnFail(info); - // Should swap 23 for 19. + // Should swap 36 for 19. addrman.ResolveCollisions(); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); - // If 23 was swapped for 19, then this should cause no collisions. - BOOST_CHECK(!addrman.Add(CAddress(addr, NODE_NONE), source)); + // If 36 was swapped for 19, then this should cause no collisions. + BOOST_CHECK(!addrman.Add({CAddress(addr, NODE_NONE)}, source)); addrman.Good(addr); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); - // If we insert 19 is should collide with 23. + // If we insert 19 it should collide with 36 CService addr19 = ResolveService("250.1.1.19"); - BOOST_CHECK(!addrman.Add(CAddress(addr19, NODE_NONE), source)); + BOOST_CHECK(!addrman.Add({CAddress(addr19, NODE_NONE)}, source)); addrman.Good(addr19); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "250.1.1.23:0"); + BOOST_CHECK_EQUAL(addrman.SelectTriedCollision().first.ToString(), "250.1.1.36:0"); addrman.ResolveCollisions(); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); +} + +BOOST_AUTO_TEST_CASE(load_addrman) +{ + AddrManUncorrupted addrmanUncorrupted; + + CService addr1, addr2, addr3; + BOOST_CHECK(Lookup("250.7.1.1", addr1, 8333, false)); + BOOST_CHECK(Lookup("250.7.2.2", addr2, 9999, false)); + BOOST_CHECK(Lookup("250.7.3.3", addr3, 9999, false)); + BOOST_CHECK(Lookup("250.7.3.3"s, addr3, 9999, false)); + BOOST_CHECK(!Lookup("250.7.3.3\0example.com"s, addr3, 9999, false)); + + // Add three addresses to new table. + CService source; + BOOST_CHECK(Lookup("252.5.1.1", source, 8333, false)); + std::vector<CAddress> addresses{CAddress(addr1, NODE_NONE), CAddress(addr2, NODE_NONE), CAddress(addr3, NODE_NONE)}; + BOOST_CHECK(addrmanUncorrupted.Add(addresses, source)); + BOOST_CHECK(addrmanUncorrupted.size() == 3); + + // Test that the de-serialization does not throw an exception. + CDataStream ssPeers1 = AddrmanToStream(addrmanUncorrupted); + bool exceptionThrown = false; + AddrMan addrman1(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); + + BOOST_CHECK(addrman1.size() == 0); + try { + unsigned char pchMsgTmp[4]; + ssPeers1 >> pchMsgTmp; + ssPeers1 >> addrman1; + } catch (const std::exception&) { + exceptionThrown = true; + } + + BOOST_CHECK(addrman1.size() == 3); + BOOST_CHECK(exceptionThrown == false); + + // Test that ReadFromStream creates an addrman with the correct number of addrs. + CDataStream ssPeers2 = AddrmanToStream(addrmanUncorrupted); + + AddrMan addrman2(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); + BOOST_CHECK(addrman2.size() == 0); + ReadFromStream(addrman2, ssPeers2); + BOOST_CHECK(addrman2.size() == 3); +} + + +BOOST_AUTO_TEST_CASE(load_addrman_corrupted) +{ + AddrManCorrupted addrmanCorrupted; + + // Test that the de-serialization of corrupted addrman throws an exception. + CDataStream ssPeers1 = AddrmanToStream(addrmanCorrupted); + bool exceptionThrown = false; + AddrMan addrman1(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); + BOOST_CHECK(addrman1.size() == 0); + try { + unsigned char pchMsgTmp[4]; + ssPeers1 >> pchMsgTmp; + ssPeers1 >> addrman1; + } catch (const std::exception&) { + exceptionThrown = true; + } + // Even though de-serialization failed addrman is not left in a clean state. + BOOST_CHECK(addrman1.size() == 1); + BOOST_CHECK(exceptionThrown); + + // Test that ReadFromStream fails if peers.dat is corrupt + CDataStream ssPeers2 = AddrmanToStream(addrmanCorrupted); + + AddrMan addrman2(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); + BOOST_CHECK(addrman2.size() == 0); + BOOST_CHECK_THROW(ReadFromStream(addrman2, ssPeers2), std::ios_base::failure); } diff --git a/src/test/amount_tests.cpp b/src/test/amount_tests.cpp index 77b7758a17..114fe3907c 100644 --- a/src/test/amount_tests.cpp +++ b/src/test/amount_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> +#include <consensus/amount.h> #include <policy/feerate.h> #include <limits> diff --git a/src/test/bip32_tests.cpp b/src/test/bip32_tests.cpp index fb16c92647..a89868e1ef 100644 --- a/src/test/bip32_tests.cpp +++ b/src/test/bip32_tests.cpp @@ -14,6 +14,8 @@ #include <string> #include <vector> +namespace { + struct TestDerivation { std::string pub; std::string prv; @@ -99,7 +101,26 @@ TestVector test4 = "xprv9xJocDuwtYCMNAo3Zw76WENQeAS6WGXQ55RCy7tDJ8oALr4FWkuVoHJeHVAcAqiZLE7Je3vZJHxspZdFHfnBEjHqU5hG1Jaj32dVoS6XLT1", 0); -static void RunTest(const TestVector &test) { +const std::vector<std::string> TEST5 = { + "xpub661MyMwAqRbcEYS8w7XLSVeEsBXy79zSzH1J8vCdxAZningWLdN3zgtU6LBpB85b3D2yc8sfvZU521AAwdZafEz7mnzBBsz4wKY5fTtTQBm", + "xprv9s21ZrQH143K24Mfq5zL5MhWK9hUhhGbd45hLXo2Pq2oqzMMo63oStZzFGTQQD3dC4H2D5GBj7vWvSQaaBv5cxi9gafk7NF3pnBju6dwKvH", + "xpub661MyMwAqRbcEYS8w7XLSVeEsBXy79zSzH1J8vCdxAZningWLdN3zgtU6Txnt3siSujt9RCVYsx4qHZGc62TG4McvMGcAUjeuwZdduYEvFn", + "xprv9s21ZrQH143K24Mfq5zL5MhWK9hUhhGbd45hLXo2Pq2oqzMMo63oStZzFGpWnsj83BHtEy5Zt8CcDr1UiRXuWCmTQLxEK9vbz5gPstX92JQ", + "xpub661MyMwAqRbcEYS8w7XLSVeEsBXy79zSzH1J8vCdxAZningWLdN3zgtU6N8ZMMXctdiCjxTNq964yKkwrkBJJwpzZS4HS2fxvyYUA4q2Xe4", + "xprv9s21ZrQH143K24Mfq5zL5MhWK9hUhhGbd45hLXo2Pq2oqzMMo63oStZzFAzHGBP2UuGCqWLTAPLcMtD9y5gkZ6Eq3Rjuahrv17fEQ3Qen6J", + "xprv9s2SPatNQ9Vc6GTbVMFPFo7jsaZySyzk7L8n2uqKXJen3KUmvQNTuLh3fhZMBoG3G4ZW1N2kZuHEPY53qmbZzCHshoQnNf4GvELZfqTUrcv", + "xpub661no6RGEX3uJkY4bNnPcw4URcQTrSibUZ4NqJEw5eBkv7ovTwgiT91XX27VbEXGENhYRCf7hyEbWrR3FewATdCEebj6znwMfQkhRYHRLpJ", + "xprv9s21ZrQH4r4TsiLvyLXqM9P7k1K3EYhA1kkD6xuquB5i39AU8KF42acDyL3qsDbU9NmZn6MsGSUYZEsuoePmjzsB3eFKSUEh3Gu1N3cqVUN", + "xpub661MyMwAuDcm6CRQ5N4qiHKrJ39Xe1R1NyfouMKTTWcguwVcfrZJaNvhpebzGerh7gucBvzEQWRugZDuDXjNDRmXzSZe4c7mnTK97pTvGS8", + "DMwo58pR1QLEFihHiXPVykYB6fJmsTeHvyTp7hRThAtCX8CvYzgPcn8XnmdfHGMQzT7ayAmfo4z3gY5KfbrZWZ6St24UVf2Qgo6oujFktLHdHY4", + "DMwo58pR1QLEFihHiXPVykYB6fJmsTeHvyTp7hRThAtCX8CvYzgPcn8XnmdfHPmHJiEDXkTiJTVV9rHEBUem2mwVbbNfvT2MTcAqj3nesx8uBf9", + "xprv9s21ZrQH143K24Mfq5zL5MhWK9hUhhGbd45hLXo2Pq2oqzMMo63oStZzF93Y5wvzdUayhgkkFoicQZcP3y52uPPxFnfoLZB21Teqt1VvEHx", + "xprv9s21ZrQH143K24Mfq5zL5MhWK9hUhhGbd45hLXo2Pq2oqzMMo63oStZzFAzHGBP2UuGCqWLTAPLcMtD5SDKr24z3aiUvKr9bJpdrcLg1y3G", + "xpub661MyMwAqRbcEYS8w7XLSVeEsBXy79zSzH1J8vCdxAZningWLdN3zgtU6Q5JXayek4PRsn35jii4veMimro1xefsM58PgBMrvdYre8QyULY", + "xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHL" +}; + +void RunTest(const TestVector &test) { std::vector<unsigned char> seed = ParseHex(test.strHexMaster); CExtKey key; CExtPubKey pubkey; @@ -133,6 +154,8 @@ static void RunTest(const TestVector &test) { } } +} // namespace + BOOST_FIXTURE_TEST_SUITE(bip32_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(bip32_test1) { @@ -151,4 +174,13 @@ BOOST_AUTO_TEST_CASE(bip32_test4) { RunTest(test4); } +BOOST_AUTO_TEST_CASE(bip32_test5) { + for (const auto& str : TEST5) { + auto dec_extkey = DecodeExtKey(str); + auto dec_extpubkey = DecodeExtPubKey(str); + BOOST_CHECK_MESSAGE(!dec_extkey.key.IsValid(), "Decoding '" + str + "' as xprv should fail"); + BOOST_CHECK_MESSAGE(!dec_extpubkey.pubkey.IsValid(), "Decoding '" + str + "' as xpub should fail"); + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 5a98558240..23ef2062ef 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -83,7 +83,7 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_key) CBloomFilter filter(2, 0.001, 0, BLOOM_UPDATE_ALL); filter.insert(vchPubKey); uint160 hash = pubkey.GetID(); - filter.insert(std::vector<unsigned char>(hash.begin(), hash.end())); + filter.insert(hash); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << filter; diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index edec5f0a31..5b3b39fdb8 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -617,7 +617,7 @@ static void TestChaCha20Poly1305AEAD(bool must_succeed, unsigned int expected_aa ChaCha20Poly1305AEAD aead(aead_K_1.data(), aead_K_1.size(), aead_K_2.data(), aead_K_2.size()); // create a chacha20 instance to compare against - ChaCha20 cmp_ctx(aead_K_2.data(), 32); + ChaCha20 cmp_ctx(aead_K_1.data(), 32); // encipher bool res = aead.Crypt(seqnr_payload, seqnr_aad, aad_pos, ciphertext_buf.data(), ciphertext_buf.size(), plaintext_buf.data(), plaintext_buf.size(), true); @@ -708,8 +708,8 @@ BOOST_AUTO_TEST_CASE(chacha20_poly1305_aead_testvector) "b1a03d5bd2855d60699e7d3a3133fa47be740fe4e4c1f967555e2d9271f31c3a8bd94d54b5ecabbc41ffbb0c90924080"); TestChaCha20Poly1305AEAD(true, 255, "ff0000f195e66982105ffb640bb7757f579da31602fc93ec01ac56f85ac3c134a4547b733b46413042c9440049176905d3be59ea1c53f15916155c2be8241a38008b9a26bc35941e2444177c8ade6689de95264986d95889fb60e84629c9bd9a5acb1cc118be563eb9b3a4a472f82e09a7e778492b562ef7130e88dfe031c79db9d4f7c7a899151b9a475032b63fc385245fe054e3dd5a97a5f576fe064025d3ce042c566ab2c507b138db853e3d6959660996546cc9c4a6eafdc777c040d70eaf46f76dad3979e5c5360c3317166a1c894c94a371876a94df7628fe4eaaf2ccb27d5aaae0ad7ad0f9d4b6ad3b54098746d4524d38407a6deb3ab78fab78c9", - "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", "ff0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", "c640c1711e3ee904ac35c57ab9791c8a1c408603a90b77a83b54f6c844cb4b06d94e7fc6c800e165acd66147e80ec45a567f6ce66d05ec0cae679dceeb890017", "3940c1e92da4582ff6f92a776aeb14d014d384eeb30f660dacf70a14a23fd31e91212701334e2ce1acf5199dc84f4d61ddbe6571bca5af874b4c9226c26e650995d157644e1848b96ed6c2102d5489a050e71d29a5a66ece11de5fb5c9558d54da28fe45b0bc4db4e5b88030bfc4a352b4b7068eccf656bae7ad6a35615315fc7c49d4200388d5eca67c2e822e069336c69b40db67e0f3c81209c50f3216a4b89fb3ae1b984b7851a2ec6f68ab12b101ab120e1ea7313bb93b5a0f71185c7fea017ddb92769861c29dba4fbc432280d5dff21b36d1c4c790128b22699950bb18bf74c448cdfe547d8ed4f657d8005fdc0cd7a050c2d46050a44c4376355858981fbe8b184288276e7a93eabc899c4a", "f039c6689eaeef0456685200feaab9d54bbd9acde4410a3b6f4321296f4a8ca2604b49727d8892c57e005d799b2a38e85e809f20146e08eec75169691c8d4f54a0d51a1e1c7b381e0474eb02f994be9415ef3ffcbd2343f0601e1f3b172a1d494f838824e4df570f8e3b0c04e27966e36c82abd352d07054ef7bd36b84c63f9369afe7ed79b94f953873006b920c3fa251a771de1b63da927058ade119aa898b8c97e42a606b2f6df1e2d957c22f7593c1e2002f4252f4c9ae4bf773499e5cfcfe14dfc1ede26508953f88553bf4a76a802f6a0068d59295b01503fd9a600067624203e880fdf53933b96e1f4d9eb3f4e363dd8165a278ff667a41ee42b9892b077cefff92b93441f7be74cf10e6cd"); diff --git a/src/test/data/README.md b/src/test/data/README.md index 2463daa42a..a05d9c668b 100644 --- a/src/test/data/README.md +++ b/src/test/data/README.md @@ -8,5 +8,5 @@ License The data files in this directory are distributed under the MIT software license, see the accompanying file COPYING or -http://www.opensource.org/licenses/mit-license.php. +https://www.opensource.org/licenses/mit-license.php. diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 5668ead1fb..0bfe6eecd9 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -53,7 +53,7 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) const CChainParams& chainparams = Params(); auto connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman); auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, nullptr, - *m_node.scheduler, *m_node.chainman, *m_node.mempool, false); + *m_node.chainman, *m_node.mempool, false); // Mock an outbound peer CAddress addr1(ip(0xa0b0c001), NODE_NONE); @@ -121,7 +121,7 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) const CChainParams& chainparams = Params(); auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman); auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, nullptr, - *m_node.scheduler, *m_node.chainman, *m_node.mempool, false); + *m_node.chainman, *m_node.mempool, false); constexpr int max_outbound_full_relay = MAX_OUTBOUND_FULL_RELAY_CONNECTIONS; CConnman::Options options; @@ -194,7 +194,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) auto banman = std::make_unique<BanMan>(m_args.GetDataDirBase() / "banlist", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman); auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, banman.get(), - *m_node.scheduler, *m_node.chainman, *m_node.mempool, false); + *m_node.chainman, *m_node.mempool, false); CNetAddr tor_netaddr; BOOST_REQUIRE( @@ -288,7 +288,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) auto banman = std::make_unique<BanMan>(m_args.GetDataDirBase() / "banlist", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman); auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, banman.get(), - *m_node.scheduler, *m_node.chainman, *m_node.mempool, false); + *m_node.chainman, *m_node.mempool, false); banman->ClearBanned(); int64_t nStartTime = GetTime(); diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 36e2dac3ff..8553f80a17 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -124,14 +124,10 @@ void DoCheck(const std::string& prv, const std::string& pub, const std::string& // Check that private can produce the normalized descriptors std::string norm1; - BOOST_CHECK(parse_priv->ToNormalizedString(keys_priv, norm1, false)); + BOOST_CHECK(parse_priv->ToNormalizedString(keys_priv, norm1)); BOOST_CHECK(EqualDescriptor(norm1, norm_pub)); - BOOST_CHECK(parse_pub->ToNormalizedString(keys_priv, norm1, false)); + BOOST_CHECK(parse_pub->ToNormalizedString(keys_priv, norm1)); BOOST_CHECK(EqualDescriptor(norm1, norm_pub)); - BOOST_CHECK(parse_priv->ToNormalizedString(keys_priv, norm1, true)); - BOOST_CHECK(EqualDescriptor(norm1, norm_prv)); - BOOST_CHECK(parse_pub->ToNormalizedString(keys_priv, norm1, true)); - BOOST_CHECK(EqualDescriptor(norm1, norm_prv)); // Check whether IsRange on both returns the expected result BOOST_CHECK_EQUAL(parse_pub->IsRange(), (flags & RANGE) != 0); diff --git a/src/test/fuzz/addrdb.cpp b/src/test/fuzz/addrdb.cpp deleted file mode 100644 index d15c785673..0000000000 --- a/src/test/fuzz/addrdb.cpp +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) 2020 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <addrdb.h> -#include <test/fuzz/FuzzedDataProvider.h> -#include <test/fuzz/fuzz.h> -#include <test/fuzz/util.h> - -#include <cassert> -#include <cstdint> -#include <optional> -#include <string> -#include <vector> - -FUZZ_TARGET(addrdb) -{ - FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - - // The point of this code is to exercise all CBanEntry constructors. - const CBanEntry ban_entry = [&] { - switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 2)) { - case 0: - return CBanEntry{fuzzed_data_provider.ConsumeIntegral<int64_t>()}; - break; - case 1: { - const std::optional<CBanEntry> ban_entry = ConsumeDeserializable<CBanEntry>(fuzzed_data_provider); - if (ban_entry) { - return *ban_entry; - } - break; - } - } - return CBanEntry{}; - }(); - (void)ban_entry; // currently unused -} diff --git a/src/test/fuzz/addrman.cpp b/src/test/fuzz/addrman.cpp index db0b461873..8df3707fc9 100644 --- a/src/test/fuzz/addrman.cpp +++ b/src/test/fuzz/addrman.cpp @@ -4,14 +4,17 @@ #include <addrdb.h> #include <addrman.h> +#include <addrman_impl.h> #include <chainparams.h> #include <merkleblock.h> +#include <random.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <time.h> #include <util/asmap.h> +#include <cassert> #include <cstdint> #include <optional> #include <string> @@ -22,48 +25,222 @@ void initialize_addrman() SelectParams(CBaseChainParams::REGTEST); } -class CAddrManDeterministic : public CAddrMan +FUZZ_TARGET_INIT(data_stream_addr_man, initialize_addrman) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + CDataStream data_stream = ConsumeDataStream(fuzzed_data_provider); + AddrMan addr_man(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + try { + ReadFromStream(addr_man, data_stream); + } catch (const std::exception&) { + } +} + +/** + * Generate a random address. Always returns a valid address. + */ +CNetAddr RandAddr(FuzzedDataProvider& fuzzed_data_provider, FastRandomContext& fast_random_context) +{ + CNetAddr addr; + if (fuzzed_data_provider.remaining_bytes() > 1 && fuzzed_data_provider.ConsumeBool()) { + addr = ConsumeNetAddr(fuzzed_data_provider); + } else { + // The networks [1..6] correspond to CNetAddr::BIP155Network (private). + static const std::map<uint8_t, uint8_t> net_len_map = {{1, ADDR_IPV4_SIZE}, + {2, ADDR_IPV6_SIZE}, + {4, ADDR_TORV3_SIZE}, + {5, ADDR_I2P_SIZE}, + {6, ADDR_CJDNS_SIZE}}; + uint8_t net = fast_random_context.randrange(5) + 1; // [1..5] + if (net == 3) { + net = 6; + } + + CDataStream s(SER_NETWORK, PROTOCOL_VERSION | ADDRV2_FORMAT); + + s << net; + s << fast_random_context.randbytes(net_len_map.at(net)); + + s >> addr; + } + + // Return a dummy IPv4 5.5.5.5 if we generated an invalid address. + if (!addr.IsValid()) { + in_addr v4_addr = {}; + v4_addr.s_addr = 0x05050505; + addr = CNetAddr{v4_addr}; + } + + return addr; +} + +/** Fill addrman with lots of addresses from lots of sources. */ +void FillAddrman(AddrMan& addrman, FuzzedDataProvider& fuzzed_data_provider) +{ + // Add a fraction of the addresses to the "tried" table. + // 0, 1, 2, 3 corresponding to 0%, 100%, 50%, 33% + const size_t n = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 3); + + const size_t num_sources = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 50); + CNetAddr prev_source; + // Generate a FastRandomContext seed to use inside the loops instead of + // fuzzed_data_provider. When fuzzed_data_provider is exhausted it + // just returns 0. + FastRandomContext fast_random_context{ConsumeUInt256(fuzzed_data_provider)}; + for (size_t i = 0; i < num_sources; ++i) { + const auto source = RandAddr(fuzzed_data_provider, fast_random_context); + const size_t num_addresses = fast_random_context.randrange(500) + 1; // [1..500] + + for (size_t j = 0; j < num_addresses; ++j) { + const auto addr = CAddress{CService{RandAddr(fuzzed_data_provider, fast_random_context), 8333}, NODE_NETWORK}; + const auto time_penalty = fast_random_context.randrange(100000001); + addrman.Add({addr}, source, time_penalty); + + if (n > 0 && addrman.size() % n == 0) { + addrman.Good(addr, GetTime()); + } + + // Add 10% of the addresses from more than one source. + if (fast_random_context.randrange(10) == 0 && prev_source.IsValid()) { + addrman.Add({addr}, prev_source, time_penalty); + } + } + prev_source = source; + } +} + +class AddrManDeterministic : public AddrMan { public: - void MakeDeterministic(const uint256& random_seed) + explicit AddrManDeterministic(std::vector<bool> asmap, FuzzedDataProvider& fuzzed_data_provider) + : AddrMan(std::move(asmap), /* deterministic */ true, /* consistency_check_ratio */ 0) { - insecure_rand = FastRandomContext{random_seed}; - Clear(); + WITH_LOCK(m_impl->cs, m_impl->insecure_rand = FastRandomContext{ConsumeUInt256(fuzzed_data_provider)}); + } + + /** + * Compare with another AddrMan. + * This compares: + * - the values in `mapInfo` (the keys aka ids are ignored) + * - vvNew entries refer to the same addresses + * - vvTried entries refer to the same addresses + */ + bool operator==(const AddrManDeterministic& other) + { + LOCK2(m_impl->cs, other.m_impl->cs); + + if (m_impl->mapInfo.size() != other.m_impl->mapInfo.size() || m_impl->nNew != other.m_impl->nNew || + m_impl->nTried != other.m_impl->nTried) { + return false; + } + + // Check that all values in `mapInfo` are equal to all values in `other.mapInfo`. + // Keys may be different. + + using AddrInfoHasher = std::function<size_t(const AddrInfo&)>; + using AddrInfoEq = std::function<bool(const AddrInfo&, const AddrInfo&)>; + + CNetAddrHash netaddr_hasher; + + AddrInfoHasher addrinfo_hasher = [&netaddr_hasher](const AddrInfo& a) { + return netaddr_hasher(static_cast<CNetAddr>(a)) ^ netaddr_hasher(a.source) ^ + a.nLastSuccess ^ a.nAttempts ^ a.nRefCount ^ a.fInTried; + }; + + AddrInfoEq addrinfo_eq = [](const AddrInfo& lhs, const AddrInfo& rhs) { + return static_cast<CNetAddr>(lhs) == static_cast<CNetAddr>(rhs) && + lhs.source == rhs.source && lhs.nLastSuccess == rhs.nLastSuccess && + lhs.nAttempts == rhs.nAttempts && lhs.nRefCount == rhs.nRefCount && + lhs.fInTried == rhs.fInTried; + }; + + using Addresses = std::unordered_set<AddrInfo, AddrInfoHasher, AddrInfoEq>; + + const size_t num_addresses{m_impl->mapInfo.size()}; + + Addresses addresses{num_addresses, addrinfo_hasher, addrinfo_eq}; + for (const auto& [id, addr] : m_impl->mapInfo) { + addresses.insert(addr); + } + + Addresses other_addresses{num_addresses, addrinfo_hasher, addrinfo_eq}; + for (const auto& [id, addr] : other.m_impl->mapInfo) { + other_addresses.insert(addr); + } + + if (addresses != other_addresses) { + return false; + } + + auto IdsReferToSameAddress = [&](int id, int other_id) EXCLUSIVE_LOCKS_REQUIRED(m_impl->cs, other.m_impl->cs) { + if (id == -1 && other_id == -1) { + return true; + } + if ((id == -1 && other_id != -1) || (id != -1 && other_id == -1)) { + return false; + } + return m_impl->mapInfo.at(id) == other.m_impl->mapInfo.at(other_id); + }; + + // Check that `vvNew` contains the same addresses as `other.vvNew`. Notice - `vvNew[i][j]` + // contains just an id and the address is to be found in `mapInfo.at(id)`. The ids + // themselves may differ between `vvNew` and `other.vvNew`. + for (size_t i = 0; i < ADDRMAN_NEW_BUCKET_COUNT; ++i) { + for (size_t j = 0; j < ADDRMAN_BUCKET_SIZE; ++j) { + if (!IdsReferToSameAddress(m_impl->vvNew[i][j], other.m_impl->vvNew[i][j])) { + return false; + } + } + } + + // Same for `vvTried`. + for (size_t i = 0; i < ADDRMAN_TRIED_BUCKET_COUNT; ++i) { + for (size_t j = 0; j < ADDRMAN_BUCKET_SIZE; ++j) { + if (!IdsReferToSameAddress(m_impl->vvTried[i][j], other.m_impl->vvTried[i][j])) { + return false; + } + } + } + + return true; } }; +[[nodiscard]] inline std::vector<bool> ConsumeAsmap(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + std::vector<bool> asmap = ConsumeRandomLengthBitVector(fuzzed_data_provider); + if (!SanityCheckASMap(asmap, 128)) asmap.clear(); + return asmap; +} + FUZZ_TARGET_INIT(addrman, initialize_addrman) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); SetMockTime(ConsumeTime(fuzzed_data_provider)); - CAddrManDeterministic addr_man; - addr_man.MakeDeterministic(ConsumeUInt256(fuzzed_data_provider)); + std::vector<bool> asmap = ConsumeAsmap(fuzzed_data_provider); + auto addr_man_ptr = std::make_unique<AddrManDeterministic>(asmap, fuzzed_data_provider); if (fuzzed_data_provider.ConsumeBool()) { - addr_man.m_asmap = ConsumeRandomLengthBitVector(fuzzed_data_provider); - if (!SanityCheckASMap(addr_man.m_asmap)) { - addr_man.m_asmap.clear(); + const std::vector<uint8_t> serialized_data{ConsumeRandomLengthByteVector(fuzzed_data_provider)}; + CDataStream ds(serialized_data, SER_DISK, INIT_PROTO_VERSION); + const auto ser_version{fuzzed_data_provider.ConsumeIntegral<int32_t>()}; + ds.SetVersion(ser_version); + try { + ds >> *addr_man_ptr; + } catch (const std::ios_base::failure&) { + addr_man_ptr = std::make_unique<AddrManDeterministic>(asmap, fuzzed_data_provider); } } + AddrManDeterministic& addr_man = *addr_man_ptr; while (fuzzed_data_provider.ConsumeBool()) { CallOneOf( fuzzed_data_provider, [&] { - addr_man.Clear(); - }, - [&] { addr_man.ResolveCollisions(); }, [&] { (void)addr_man.SelectTriedCollision(); }, [&] { - const std::optional<CAddress> opt_address = ConsumeDeserializable<CAddress>(fuzzed_data_provider); - const std::optional<CNetAddr> opt_net_addr = ConsumeDeserializable<CNetAddr>(fuzzed_data_provider); - if (opt_address && opt_net_addr) { - addr_man.Add(*opt_address, *opt_net_addr, fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(0, 100000000)); - } - }, - [&] { std::vector<CAddress> addresses; while (fuzzed_data_provider.ConsumeBool()) { const std::optional<CAddress> opt_address = ConsumeDeserializable<CAddress>(fuzzed_data_provider); @@ -80,7 +257,7 @@ FUZZ_TARGET_INIT(addrman, initialize_addrman) [&] { const std::optional<CService> opt_service = ConsumeDeserializable<CService>(fuzzed_data_provider); if (opt_service) { - addr_man.Good(*opt_service, fuzzed_data_provider.ConsumeBool(), ConsumeTime(fuzzed_data_provider)); + addr_man.Good(*opt_service, ConsumeTime(fuzzed_data_provider)); } }, [&] { @@ -102,13 +279,31 @@ FUZZ_TARGET_INIT(addrman, initialize_addrman) } }); } - const CAddrMan& const_addr_man{addr_man}; - (void)/*const_*/addr_man.GetAddr( + const AddrMan& const_addr_man{addr_man}; + (void)const_addr_man.GetAddr( /* max_addresses */ fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096), /* max_pct */ fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096), /* network */ std::nullopt); - (void)/*const_*/addr_man.Select(fuzzed_data_provider.ConsumeBool()); + (void)const_addr_man.Select(fuzzed_data_provider.ConsumeBool()); (void)const_addr_man.size(); CDataStream data_stream(SER_NETWORK, PROTOCOL_VERSION); data_stream << const_addr_man; } + +// Check that serialize followed by unserialize produces the same addrman. +FUZZ_TARGET_INIT(addrman_serdeser, initialize_addrman) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + SetMockTime(ConsumeTime(fuzzed_data_provider)); + + std::vector<bool> asmap = ConsumeAsmap(fuzzed_data_provider); + AddrManDeterministic addr_man1{asmap, fuzzed_data_provider}; + AddrManDeterministic addr_man2{asmap, fuzzed_data_provider}; + + CDataStream data_stream(SER_NETWORK, PROTOCOL_VERSION); + + FillAddrman(addr_man1, fuzzed_data_provider); + data_stream << addr_man1; + data_stream >> addr_man2; + assert(addr_man1 == addr_man2); +} diff --git a/src/test/fuzz/asmap.cpp b/src/test/fuzz/asmap.cpp index 4c5bc0cbf2..d402f8632c 100644 --- a/src/test/fuzz/asmap.cpp +++ b/src/test/fuzz/asmap.cpp @@ -4,6 +4,7 @@ #include <netaddress.h> #include <test/fuzz/fuzz.h> +#include <util/asmap.h> #include <cstdint> #include <vector> @@ -42,7 +43,7 @@ FUZZ_TARGET(asmap) asmap.push_back((buffer[1 + i] >> j) & 1); } } - if (!SanityCheckASMap(asmap)) return; + if (!SanityCheckASMap(asmap, 128)) return; const uint8_t* addr_data = buffer.data() + 1 + asmap_size; CNetAddr net_addr; diff --git a/src/test/fuzz/banman.cpp b/src/test/fuzz/banman.cpp index 182aabc79b..561cc83c72 100644 --- a/src/test/fuzz/banman.cpp +++ b/src/test/fuzz/banman.cpp @@ -41,10 +41,6 @@ static bool operator==(const CBanEntry& lhs, const CBanEntry& rhs) FUZZ_TARGET_INIT(banman, initialize_banman) { - // The complexity is O(N^2), where N is the input size, because each call - // might call DumpBanlist (or other methods that are at least linear - // complexity of the input size). - int limit_max_ops{300}; FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; SetMockTime(ConsumeTime(fuzzed_data_provider)); fs::path banlist_file = gArgs.GetDataDirNet() / "fuzzed_banlist"; @@ -52,8 +48,7 @@ FUZZ_TARGET_INIT(banman, initialize_banman) const bool start_with_corrupted_banlist{fuzzed_data_provider.ConsumeBool()}; bool force_read_and_write_to_err{false}; if (start_with_corrupted_banlist) { - const std::string sfx{fuzzed_data_provider.ConsumeBool() ? ".dat" : ".json"}; - assert(WriteBinaryFile(banlist_file.string() + sfx, + assert(WriteBinaryFile(banlist_file.string() + ".json", fuzzed_data_provider.ConsumeRandomLengthString())); } else { force_read_and_write_to_err = fuzzed_data_provider.ConsumeBool(); @@ -64,7 +59,11 @@ FUZZ_TARGET_INIT(banman, initialize_banman) { BanMan ban_man{banlist_file, /* client_interface */ nullptr, /* default_ban_time */ ConsumeBanTimeOffset(fuzzed_data_provider)}; - while (--limit_max_ops >= 0 && fuzzed_data_provider.ConsumeBool()) { + // The complexity is O(N^2), where N is the input size, because each call + // might call DumpBanlist (or other methods that are at least linear + // complexity of the input size). + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 300) + { CallOneOf( fuzzed_data_provider, [&] { @@ -112,6 +111,5 @@ FUZZ_TARGET_INIT(banman, initialize_banman) assert(banmap == banmap_read); } } - fs::remove(banlist_file.string() + ".dat"); fs::remove(banlist_file.string() + ".json"); } diff --git a/src/test/fuzz/blockfilter.cpp b/src/test/fuzz/blockfilter.cpp index 7fa06085f8..96f049625d 100644 --- a/src/test/fuzz/blockfilter.cpp +++ b/src/test/fuzz/blockfilter.cpp @@ -36,9 +36,10 @@ FUZZ_TARGET(blockfilter) (void)gcs_filter.GetEncoded(); (void)gcs_filter.Match(ConsumeRandomLengthByteVector(fuzzed_data_provider)); GCSFilter::ElementSet element_set; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 30000) + { element_set.insert(ConsumeRandomLengthByteVector(fuzzed_data_provider)); - gcs_filter.MatchAny(element_set); } + gcs_filter.MatchAny(element_set); } } diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index f452696689..87e70861fa 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> #include <chainparams.h> #include <chainparamsbase.h> #include <coins.h> +#include <consensus/amount.h> #include <consensus/tx_check.h> #include <consensus/tx_verify.h> #include <consensus/validation.h> @@ -258,7 +258,7 @@ FUZZ_TARGET_INIT(coins_view, initialize_coins_view) // consensus/tx_verify.cpp:130: unsigned int GetP2SHSigOpCount(const CTransaction &, const CCoinsViewCache &): Assertion `!coin.IsSpent()' failed. return; } - const int flags = fuzzed_data_provider.ConsumeIntegral<int>(); + const auto flags{fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; if (!transaction.vin.empty() && (flags & SCRIPT_VERIFY_WITNESS) != 0 && (flags & SCRIPT_VERIFY_P2SH) == 0) { // Avoid: // script/interpreter.cpp:1705: size_t CountWitnessSigOps(const CScript &, const CScript &, const CScriptWitness *, unsigned int): Assertion `(flags & SCRIPT_VERIFY_P2SH) != 0' failed. diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index bbec5943af..d381345a0d 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -25,7 +25,7 @@ FUZZ_TARGET_INIT(connman, initialize_connman) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; SetMockTime(ConsumeTime(fuzzed_data_provider)); - CAddrMan addrman; + AddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); CConnman connman{fuzzed_data_provider.ConsumeIntegral<uint64_t>(), fuzzed_data_provider.ConsumeIntegral<uint64_t>(), addrman, fuzzed_data_provider.ConsumeBool()}; CNetAddr random_netaddr; CNode random_node = ConsumeNode(fuzzed_data_provider); @@ -104,12 +104,6 @@ FUZZ_TARGET_INIT(connman, initialize_connman) connman.RemoveAddedNode(random_string); }, [&] { - const std::vector<bool> asmap = ConsumeRandomLengthBitVector(fuzzed_data_provider); - if (SanityCheckASMap(asmap)) { - connman.SetAsmap(asmap); - } - }, - [&] { connman.SetNetworkActive(fuzzed_data_provider.ConsumeBool()); }, [&] { diff --git a/src/test/fuzz/crypto.cpp b/src/test/fuzz/crypto.cpp index f83747e424..84b95117e2 100644 --- a/src/test/fuzz/crypto.cpp +++ b/src/test/fuzz/crypto.cpp @@ -19,10 +19,6 @@ FUZZ_TARGET(crypto) { - // Hashing is expensive with sanitizers enabled, so limit the number of - // calls - int limit_max_ops{30}; - FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; std::vector<uint8_t> data = ConsumeRandomLengthByteVector(fuzzed_data_provider); if (data.empty()) { @@ -40,7 +36,8 @@ FUZZ_TARGET(crypto) SHA3_256 sha3; CSipHasher sip_hasher{fuzzed_data_provider.ConsumeIntegral<uint64_t>(), fuzzed_data_provider.ConsumeIntegral<uint64_t>()}; - while (--limit_max_ops >= 0 && fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 30) + { CallOneOf( fuzzed_data_provider, [&] { diff --git a/src/test/fuzz/data_stream.cpp b/src/test/fuzz/data_stream.cpp deleted file mode 100644 index 473caec6ff..0000000000 --- a/src/test/fuzz/data_stream.cpp +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) 2020-2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <addrman.h> -#include <net.h> -#include <test/fuzz/FuzzedDataProvider.h> -#include <test/fuzz/fuzz.h> -#include <test/fuzz/util.h> -#include <test/util/setup_common.h> - -#include <cstdint> -#include <vector> - -void initialize_data_stream_addr_man() -{ - static const auto testing_setup = MakeNoLogFileContext<>(); -} - -FUZZ_TARGET_INIT(data_stream_addr_man, initialize_data_stream_addr_man) -{ - FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - CDataStream data_stream = ConsumeDataStream(fuzzed_data_provider); - CAddrMan addr_man; - CAddrDB::Read(addr_man, data_stream); -} diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index 721e4360d0..a9325fa738 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -4,6 +4,7 @@ #include <addrdb.h> #include <addrman.h> +#include <addrman_impl.h> #include <blockencodings.h> #include <blockfilter.h> #include <chain.h> @@ -100,228 +101,213 @@ void AssertEqualAfterSerializeDeserialize(const T& obj, const int version = INIT } // namespace FUZZ_TARGET_DESERIALIZE(block_filter_deserialize, { - BlockFilter block_filter; - DeserializeFromFuzzingInput(buffer, block_filter); + BlockFilter block_filter; + DeserializeFromFuzzingInput(buffer, block_filter); }) FUZZ_TARGET_DESERIALIZE(addr_info_deserialize, { - CAddrInfo addr_info; - DeserializeFromFuzzingInput(buffer, addr_info); + AddrInfo addr_info; + DeserializeFromFuzzingInput(buffer, addr_info); }) FUZZ_TARGET_DESERIALIZE(block_file_info_deserialize, { - CBlockFileInfo block_file_info; - DeserializeFromFuzzingInput(buffer, block_file_info); + CBlockFileInfo block_file_info; + DeserializeFromFuzzingInput(buffer, block_file_info); }) FUZZ_TARGET_DESERIALIZE(block_header_and_short_txids_deserialize, { - CBlockHeaderAndShortTxIDs block_header_and_short_txids; - DeserializeFromFuzzingInput(buffer, block_header_and_short_txids); + CBlockHeaderAndShortTxIDs block_header_and_short_txids; + DeserializeFromFuzzingInput(buffer, block_header_and_short_txids); }) FUZZ_TARGET_DESERIALIZE(fee_rate_deserialize, { - CFeeRate fee_rate; - DeserializeFromFuzzingInput(buffer, fee_rate); - AssertEqualAfterSerializeDeserialize(fee_rate); + CFeeRate fee_rate; + DeserializeFromFuzzingInput(buffer, fee_rate); + AssertEqualAfterSerializeDeserialize(fee_rate); }) FUZZ_TARGET_DESERIALIZE(merkle_block_deserialize, { - CMerkleBlock merkle_block; - DeserializeFromFuzzingInput(buffer, merkle_block); + CMerkleBlock merkle_block; + DeserializeFromFuzzingInput(buffer, merkle_block); }) FUZZ_TARGET_DESERIALIZE(out_point_deserialize, { - COutPoint out_point; - DeserializeFromFuzzingInput(buffer, out_point); - AssertEqualAfterSerializeDeserialize(out_point); + COutPoint out_point; + DeserializeFromFuzzingInput(buffer, out_point); + AssertEqualAfterSerializeDeserialize(out_point); }) FUZZ_TARGET_DESERIALIZE(partial_merkle_tree_deserialize, { - CPartialMerkleTree partial_merkle_tree; - DeserializeFromFuzzingInput(buffer, partial_merkle_tree); + CPartialMerkleTree partial_merkle_tree; + DeserializeFromFuzzingInput(buffer, partial_merkle_tree); }) FUZZ_TARGET_DESERIALIZE(pub_key_deserialize, { - CPubKey pub_key; - DeserializeFromFuzzingInput(buffer, pub_key); - AssertEqualAfterSerializeDeserialize(pub_key); + CPubKey pub_key; + DeserializeFromFuzzingInput(buffer, pub_key); + AssertEqualAfterSerializeDeserialize(pub_key); }) FUZZ_TARGET_DESERIALIZE(script_deserialize, { - CScript script; - DeserializeFromFuzzingInput(buffer, script); -}) -FUZZ_TARGET_DESERIALIZE(sub_net_deserialize, { - CSubNet sub_net_1; - DeserializeFromFuzzingInput(buffer, sub_net_1, INIT_PROTO_VERSION); - AssertEqualAfterSerializeDeserialize(sub_net_1, INIT_PROTO_VERSION); - CSubNet sub_net_2; - DeserializeFromFuzzingInput(buffer, sub_net_2, INIT_PROTO_VERSION | ADDRV2_FORMAT); - AssertEqualAfterSerializeDeserialize(sub_net_2, INIT_PROTO_VERSION | ADDRV2_FORMAT); - CSubNet sub_net_3; - DeserializeFromFuzzingInput(buffer, sub_net_3); - AssertEqualAfterSerializeDeserialize(sub_net_3, INIT_PROTO_VERSION | ADDRV2_FORMAT); + CScript script; + DeserializeFromFuzzingInput(buffer, script); }) FUZZ_TARGET_DESERIALIZE(tx_in_deserialize, { - CTxIn tx_in; - DeserializeFromFuzzingInput(buffer, tx_in); - AssertEqualAfterSerializeDeserialize(tx_in); + CTxIn tx_in; + DeserializeFromFuzzingInput(buffer, tx_in); + AssertEqualAfterSerializeDeserialize(tx_in); }) FUZZ_TARGET_DESERIALIZE(flat_file_pos_deserialize, { - FlatFilePos flat_file_pos; - DeserializeFromFuzzingInput(buffer, flat_file_pos); - AssertEqualAfterSerializeDeserialize(flat_file_pos); + FlatFilePos flat_file_pos; + DeserializeFromFuzzingInput(buffer, flat_file_pos); + AssertEqualAfterSerializeDeserialize(flat_file_pos); }) FUZZ_TARGET_DESERIALIZE(key_origin_info_deserialize, { - KeyOriginInfo key_origin_info; - DeserializeFromFuzzingInput(buffer, key_origin_info); - AssertEqualAfterSerializeDeserialize(key_origin_info); + KeyOriginInfo key_origin_info; + DeserializeFromFuzzingInput(buffer, key_origin_info); + AssertEqualAfterSerializeDeserialize(key_origin_info); }) FUZZ_TARGET_DESERIALIZE(partially_signed_transaction_deserialize, { - PartiallySignedTransaction partially_signed_transaction; - DeserializeFromFuzzingInput(buffer, partially_signed_transaction); + PartiallySignedTransaction partially_signed_transaction; + DeserializeFromFuzzingInput(buffer, partially_signed_transaction); }) FUZZ_TARGET_DESERIALIZE(prefilled_transaction_deserialize, { - PrefilledTransaction prefilled_transaction; - DeserializeFromFuzzingInput(buffer, prefilled_transaction); + PrefilledTransaction prefilled_transaction; + DeserializeFromFuzzingInput(buffer, prefilled_transaction); }) FUZZ_TARGET_DESERIALIZE(psbt_input_deserialize, { - PSBTInput psbt_input; - DeserializeFromFuzzingInput(buffer, psbt_input); + PSBTInput psbt_input; + DeserializeFromFuzzingInput(buffer, psbt_input); }) FUZZ_TARGET_DESERIALIZE(psbt_output_deserialize, { - PSBTOutput psbt_output; - DeserializeFromFuzzingInput(buffer, psbt_output); + PSBTOutput psbt_output; + DeserializeFromFuzzingInput(buffer, psbt_output); }) FUZZ_TARGET_DESERIALIZE(block_deserialize, { - CBlock block; - DeserializeFromFuzzingInput(buffer, block); + CBlock block; + DeserializeFromFuzzingInput(buffer, block); }) FUZZ_TARGET_DESERIALIZE(blocklocator_deserialize, { - CBlockLocator bl; - DeserializeFromFuzzingInput(buffer, bl); + CBlockLocator bl; + DeserializeFromFuzzingInput(buffer, bl); }) FUZZ_TARGET_DESERIALIZE(blockmerkleroot, { - CBlock block; - DeserializeFromFuzzingInput(buffer, block); - bool mutated; - BlockMerkleRoot(block, &mutated); + CBlock block; + DeserializeFromFuzzingInput(buffer, block); + bool mutated; + BlockMerkleRoot(block, &mutated); }) FUZZ_TARGET_DESERIALIZE(addrman_deserialize, { - CAddrMan am; - DeserializeFromFuzzingInput(buffer, am); + AddrMan am(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + DeserializeFromFuzzingInput(buffer, am); }) FUZZ_TARGET_DESERIALIZE(blockheader_deserialize, { - CBlockHeader bh; - DeserializeFromFuzzingInput(buffer, bh); -}) -FUZZ_TARGET_DESERIALIZE(banentry_deserialize, { - CBanEntry be; - DeserializeFromFuzzingInput(buffer, be); + CBlockHeader bh; + DeserializeFromFuzzingInput(buffer, bh); }) FUZZ_TARGET_DESERIALIZE(txundo_deserialize, { - CTxUndo tu; - DeserializeFromFuzzingInput(buffer, tu); + CTxUndo tu; + DeserializeFromFuzzingInput(buffer, tu); }) FUZZ_TARGET_DESERIALIZE(blockundo_deserialize, { - CBlockUndo bu; - DeserializeFromFuzzingInput(buffer, bu); + CBlockUndo bu; + DeserializeFromFuzzingInput(buffer, bu); }) FUZZ_TARGET_DESERIALIZE(coins_deserialize, { - Coin coin; - DeserializeFromFuzzingInput(buffer, coin); + Coin coin; + DeserializeFromFuzzingInput(buffer, coin); }) FUZZ_TARGET_DESERIALIZE(netaddr_deserialize, { - CNetAddr na; - DeserializeFromFuzzingInput(buffer, na); - if (na.IsAddrV1Compatible()) { - AssertEqualAfterSerializeDeserialize(na); - } - AssertEqualAfterSerializeDeserialize(na, INIT_PROTO_VERSION | ADDRV2_FORMAT); + CNetAddr na; + DeserializeFromFuzzingInput(buffer, na); + if (na.IsAddrV1Compatible()) { + AssertEqualAfterSerializeDeserialize(na); + } + AssertEqualAfterSerializeDeserialize(na, INIT_PROTO_VERSION | ADDRV2_FORMAT); }) FUZZ_TARGET_DESERIALIZE(service_deserialize, { - CService s; - DeserializeFromFuzzingInput(buffer, s); - if (s.IsAddrV1Compatible()) { - AssertEqualAfterSerializeDeserialize(s); - } - AssertEqualAfterSerializeDeserialize(s, INIT_PROTO_VERSION | ADDRV2_FORMAT); - CService s1; - DeserializeFromFuzzingInput(buffer, s1, INIT_PROTO_VERSION); - AssertEqualAfterSerializeDeserialize(s1, INIT_PROTO_VERSION); - assert(s1.IsAddrV1Compatible()); - CService s2; - DeserializeFromFuzzingInput(buffer, s2, INIT_PROTO_VERSION | ADDRV2_FORMAT); - AssertEqualAfterSerializeDeserialize(s2, INIT_PROTO_VERSION | ADDRV2_FORMAT); + CService s; + DeserializeFromFuzzingInput(buffer, s); + if (s.IsAddrV1Compatible()) { + AssertEqualAfterSerializeDeserialize(s); + } + AssertEqualAfterSerializeDeserialize(s, INIT_PROTO_VERSION | ADDRV2_FORMAT); + CService s1; + DeserializeFromFuzzingInput(buffer, s1, INIT_PROTO_VERSION); + AssertEqualAfterSerializeDeserialize(s1, INIT_PROTO_VERSION); + assert(s1.IsAddrV1Compatible()); + CService s2; + DeserializeFromFuzzingInput(buffer, s2, INIT_PROTO_VERSION | ADDRV2_FORMAT); + AssertEqualAfterSerializeDeserialize(s2, INIT_PROTO_VERSION | ADDRV2_FORMAT); }) FUZZ_TARGET_DESERIALIZE(messageheader_deserialize, { - CMessageHeader mh; - DeserializeFromFuzzingInput(buffer, mh); - (void)mh.IsCommandValid(); + CMessageHeader mh; + DeserializeFromFuzzingInput(buffer, mh); + (void)mh.IsCommandValid(); }) FUZZ_TARGET_DESERIALIZE(address_deserialize_v1_notime, { - CAddress a; - DeserializeFromFuzzingInput(buffer, a, INIT_PROTO_VERSION); - // A CAddress without nTime (as is expected under INIT_PROTO_VERSION) will roundtrip - // in all 5 formats (with/without nTime, v1/v2, network/disk) - AssertEqualAfterSerializeDeserialize(a, INIT_PROTO_VERSION); - AssertEqualAfterSerializeDeserialize(a, PROTOCOL_VERSION); - AssertEqualAfterSerializeDeserialize(a, 0, SER_DISK); - AssertEqualAfterSerializeDeserialize(a, PROTOCOL_VERSION | ADDRV2_FORMAT); - AssertEqualAfterSerializeDeserialize(a, ADDRV2_FORMAT, SER_DISK); + CAddress a; + DeserializeFromFuzzingInput(buffer, a, INIT_PROTO_VERSION); + // A CAddress without nTime (as is expected under INIT_PROTO_VERSION) will roundtrip + // in all 5 formats (with/without nTime, v1/v2, network/disk) + AssertEqualAfterSerializeDeserialize(a, INIT_PROTO_VERSION); + AssertEqualAfterSerializeDeserialize(a, PROTOCOL_VERSION); + AssertEqualAfterSerializeDeserialize(a, 0, SER_DISK); + AssertEqualAfterSerializeDeserialize(a, PROTOCOL_VERSION | ADDRV2_FORMAT); + AssertEqualAfterSerializeDeserialize(a, ADDRV2_FORMAT, SER_DISK); }) FUZZ_TARGET_DESERIALIZE(address_deserialize_v1_withtime, { - CAddress a; - DeserializeFromFuzzingInput(buffer, a, PROTOCOL_VERSION); - // A CAddress in V1 mode will roundtrip in all 4 formats that have nTime. - AssertEqualAfterSerializeDeserialize(a, PROTOCOL_VERSION); - AssertEqualAfterSerializeDeserialize(a, 0, SER_DISK); - AssertEqualAfterSerializeDeserialize(a, PROTOCOL_VERSION | ADDRV2_FORMAT); - AssertEqualAfterSerializeDeserialize(a, ADDRV2_FORMAT, SER_DISK); + CAddress a; + DeserializeFromFuzzingInput(buffer, a, PROTOCOL_VERSION); + // A CAddress in V1 mode will roundtrip in all 4 formats that have nTime. + AssertEqualAfterSerializeDeserialize(a, PROTOCOL_VERSION); + AssertEqualAfterSerializeDeserialize(a, 0, SER_DISK); + AssertEqualAfterSerializeDeserialize(a, PROTOCOL_VERSION | ADDRV2_FORMAT); + AssertEqualAfterSerializeDeserialize(a, ADDRV2_FORMAT, SER_DISK); }) FUZZ_TARGET_DESERIALIZE(address_deserialize_v2, { - CAddress a; - DeserializeFromFuzzingInput(buffer, a, PROTOCOL_VERSION | ADDRV2_FORMAT); - // A CAddress in V2 mode will roundtrip in both V2 formats, and also in the V1 formats - // with time if it's V1 compatible. - if (a.IsAddrV1Compatible()) { - AssertEqualAfterSerializeDeserialize(a, PROTOCOL_VERSION); - AssertEqualAfterSerializeDeserialize(a, 0, SER_DISK); - } - AssertEqualAfterSerializeDeserialize(a, PROTOCOL_VERSION | ADDRV2_FORMAT); - AssertEqualAfterSerializeDeserialize(a, ADDRV2_FORMAT, SER_DISK); + CAddress a; + DeserializeFromFuzzingInput(buffer, a, PROTOCOL_VERSION | ADDRV2_FORMAT); + // A CAddress in V2 mode will roundtrip in both V2 formats, and also in the V1 formats + // with time if it's V1 compatible. + if (a.IsAddrV1Compatible()) { + AssertEqualAfterSerializeDeserialize(a, PROTOCOL_VERSION); + AssertEqualAfterSerializeDeserialize(a, 0, SER_DISK); + } + AssertEqualAfterSerializeDeserialize(a, PROTOCOL_VERSION | ADDRV2_FORMAT); + AssertEqualAfterSerializeDeserialize(a, ADDRV2_FORMAT, SER_DISK); }) FUZZ_TARGET_DESERIALIZE(inv_deserialize, { - CInv i; - DeserializeFromFuzzingInput(buffer, i); + CInv i; + DeserializeFromFuzzingInput(buffer, i); }) FUZZ_TARGET_DESERIALIZE(bloomfilter_deserialize, { - CBloomFilter bf; - DeserializeFromFuzzingInput(buffer, bf); + CBloomFilter bf; + DeserializeFromFuzzingInput(buffer, bf); }) FUZZ_TARGET_DESERIALIZE(diskblockindex_deserialize, { - CDiskBlockIndex dbi; - DeserializeFromFuzzingInput(buffer, dbi); + CDiskBlockIndex dbi; + DeserializeFromFuzzingInput(buffer, dbi); }) FUZZ_TARGET_DESERIALIZE(txoutcompressor_deserialize, { - CTxOut to; - auto toc = Using<TxOutCompression>(to); - DeserializeFromFuzzingInput(buffer, toc); + CTxOut to; + auto toc = Using<TxOutCompression>(to); + DeserializeFromFuzzingInput(buffer, toc); }) FUZZ_TARGET_DESERIALIZE(blocktransactions_deserialize, { - BlockTransactions bt; - DeserializeFromFuzzingInput(buffer, bt); + BlockTransactions bt; + DeserializeFromFuzzingInput(buffer, bt); }) FUZZ_TARGET_DESERIALIZE(blocktransactionsrequest_deserialize, { - BlockTransactionsRequest btr; - DeserializeFromFuzzingInput(buffer, btr); + BlockTransactionsRequest btr; + DeserializeFromFuzzingInput(buffer, btr); }) FUZZ_TARGET_DESERIALIZE(snapshotmetadata_deserialize, { - SnapshotMetadata snapshot_metadata; - DeserializeFromFuzzingInput(buffer, snapshot_metadata); + SnapshotMetadata snapshot_metadata; + DeserializeFromFuzzingInput(buffer, snapshot_metadata); }) FUZZ_TARGET_DESERIALIZE(uint160_deserialize, { - uint160 u160; - DeserializeFromFuzzingInput(buffer, u160); - AssertEqualAfterSerializeDeserialize(u160); + uint160 u160; + DeserializeFromFuzzingInput(buffer, u160); + AssertEqualAfterSerializeDeserialize(u160); }) FUZZ_TARGET_DESERIALIZE(uint256_deserialize, { - uint256 u256; - DeserializeFromFuzzingInput(buffer, u256); - AssertEqualAfterSerializeDeserialize(u256); -}) - // Classes intentionally not covered in this file since their deserialization code is - // fuzzed elsewhere: - // * Deserialization of CTxOut is fuzzed in test/fuzz/tx_out.cpp - // * Deserialization of CMutableTransaction is fuzzed in src/test/fuzz/transaction.cpp + uint256 u256; + DeserializeFromFuzzingInput(buffer, u256); + AssertEqualAfterSerializeDeserialize(u256); +}) +// Classes intentionally not covered in this file since their deserialization code is +// fuzzed elsewhere: +// * Deserialization of CTxOut is fuzzed in test/fuzz/tx_out.cpp +// * Deserialization of CMutableTransaction is fuzzed in src/test/fuzz/transaction.cpp diff --git a/src/test/fuzz/fee_rate.cpp b/src/test/fuzz/fee_rate.cpp index dff0e58000..a852f8fb60 100644 --- a/src/test/fuzz/fee_rate.cpp +++ b/src/test/fuzz/fee_rate.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> +#include <consensus/amount.h> #include <policy/feerate.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> diff --git a/src/test/fuzz/fees.cpp b/src/test/fuzz/fees.cpp index 61c7681bf9..b5a07c7ba3 100644 --- a/src/test/fuzz/fees.cpp +++ b/src/test/fuzz/fees.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> +#include <consensus/amount.h> #include <policy/fees.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> diff --git a/src/test/fuzz/fuzz.h b/src/test/fuzz/fuzz.h index 2bad77bdc1..c91c33da67 100644 --- a/src/test/fuzz/fuzz.h +++ b/src/test/fuzz/fuzz.h @@ -11,6 +11,13 @@ #include <functional> #include <string_view> +/** + * Can be used to limit a theoretically unbounded loop. This caps the runtime + * to avoid timeouts or OOMs. + */ +#define LIMITED_WHILE(condition, limit) \ + for (unsigned _count{limit}; (condition) && _count; --_count) + using FuzzBufferType = Span<const uint8_t>; using TypeTestOneInput = std::function<void(FuzzBufferType)>; diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index e9fa343896..b6c40809e3 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> #include <arith_uint256.h> #include <compressor.h> +#include <consensus/amount.h> #include <consensus/merkle.h> #include <core_io.h> #include <crypto/common.h> @@ -16,8 +16,6 @@ #include <pow.h> #include <protocol.h> #include <pubkey.h> -#include <rpc/util.h> -#include <script/signingprovider.h> #include <script/standard.h> #include <serialize.h> #include <streams.h> @@ -25,6 +23,7 @@ #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <uint256.h> +#include <univalue.h> #include <util/check.h> #include <util/moneystr.h> #include <util/strencodings.h> @@ -85,9 +84,8 @@ FUZZ_TARGET_INIT(integer, initialize_integer) (void)FormatISO8601Date(i64); (void)FormatISO8601DateTime(i64); { - int64_t parsed_money; - if (ParseMoney(FormatMoney(i64), parsed_money)) { - assert(parsed_money == i64); + if (std::optional<CAmount> parsed = ParseMoney(FormatMoney(i64))) { + assert(parsed.value() == i64); } } (void)GetSizeOfCompactSize(u64); @@ -128,9 +126,8 @@ FUZZ_TARGET_INIT(integer, initialize_integer) (void)ToLower(ch); (void)ToUpper(ch); { - int64_t parsed_money; - if (ParseMoney(ValueFromAmount(i64).getValStr(), parsed_money)) { - assert(parsed_money == i64); + if (std::optional<CAmount> parsed = ParseMoney(ValueFromAmount(i64).getValStr())) { + assert(parsed.value() == i64); } } if (i32 >= 0 && i32 <= 16) { @@ -158,20 +155,6 @@ FUZZ_TARGET_INIT(integer, initialize_integer) const CKeyID key_id{u160}; const CScriptID script_id{u160}; - // CTxDestination = CNoDestination ∪ PKHash ∪ ScriptHash ∪ WitnessV0ScriptHash ∪ WitnessV0KeyHash ∪ WitnessUnknown - const PKHash pk_hash{u160}; - const ScriptHash script_hash{u160}; - const WitnessV0KeyHash witness_v0_key_hash{u160}; - const WitnessV0ScriptHash witness_v0_script_hash{u256}; - const std::vector<CTxDestination> destinations{pk_hash, script_hash, witness_v0_key_hash, witness_v0_script_hash}; - const SigningProvider store; - for (const CTxDestination& destination : destinations) { - (void)DescribeAddress(destination); - (void)EncodeDestination(destination); - (void)GetKeyForDestination(store, destination); - (void)GetScriptForDestination(destination); - (void)IsValidDestination(destination); - } { CDataStream stream(SER_NETWORK, INIT_PROTO_VERSION); diff --git a/src/test/fuzz/key_io.cpp b/src/test/fuzz/key_io.cpp index 665ca01fa1..f58bf8b316 100644 --- a/src/test/fuzz/key_io.cpp +++ b/src/test/fuzz/key_io.cpp @@ -4,9 +4,6 @@ #include <chainparams.h> #include <key_io.h> -#include <rpc/util.h> -#include <script/signingprovider.h> -#include <script/standard.h> #include <test/fuzz/fuzz.h> #include <cassert> @@ -39,12 +36,4 @@ FUZZ_TARGET_INIT(key_io, initialize_key_io) if (ext_pub_key.pubkey.size() == CPubKey::COMPRESSED_SIZE) { assert(ext_pub_key == DecodeExtPubKey(EncodeExtPubKey(ext_pub_key))); } - - const CTxDestination tx_destination = DecodeDestination(random_string); - (void)DescribeAddress(tx_destination); - (void)GetKeyForDestination(/* store */ {}, tx_destination); - (void)GetScriptForDestination(tx_destination); - (void)IsValidDestination(tx_destination); - - (void)IsValidDestinationString(random_string); } diff --git a/src/test/fuzz/kitchen_sink.cpp b/src/test/fuzz/kitchen_sink.cpp index 908e9a1c83..82f3a306c5 100644 --- a/src/test/fuzz/kitchen_sink.cpp +++ b/src/test/fuzz/kitchen_sink.cpp @@ -13,6 +13,7 @@ #include <array> #include <cstdint> +#include <optional> #include <vector> namespace { @@ -46,11 +47,10 @@ FUZZ_TARGET(kitchen_sink) const OutputType output_type = fuzzed_data_provider.PickValueInArray(OUTPUT_TYPES); const std::string& output_type_string = FormatOutputType(output_type); - OutputType output_type_parsed; - const bool parsed = ParseOutputType(output_type_string, output_type_parsed); + const std::optional<OutputType> parsed = ParseOutputType(output_type_string); assert(parsed); - assert(output_type == output_type_parsed); - (void)ParseOutputType(fuzzed_data_provider.ConsumeRandomLengthString(64), output_type_parsed); + assert(output_type == parsed.value()); + (void)ParseOutputType(fuzzed_data_provider.ConsumeRandomLengthString(64)); const std::vector<uint8_t> bytes = ConsumeRandomLengthByteVector(fuzzed_data_provider); const std::vector<bool> bits = BytesToBits(bytes); diff --git a/src/test/fuzz/load_external_block_file.cpp b/src/test/fuzz/load_external_block_file.cpp index a7770c90e8..bfa977520b 100644 --- a/src/test/fuzz/load_external_block_file.cpp +++ b/src/test/fuzz/load_external_block_file.cpp @@ -32,5 +32,5 @@ FUZZ_TARGET_INIT(load_external_block_file, initialize_load_external_block_file) return; } FlatFilePos flat_file_pos; - g_setup->m_node.chainman->ActiveChainstate().LoadExternalBlockFile(Params(), fuzzed_block_file, fuzzed_data_provider.ConsumeBool() ? &flat_file_pos : nullptr); + g_setup->m_node.chainman->ActiveChainstate().LoadExternalBlockFile(fuzzed_block_file, fuzzed_data_provider.ConsumeBool() ? &flat_file_pos : nullptr); } diff --git a/src/test/fuzz/locale.cpp b/src/test/fuzz/locale.cpp index 5b1acae57b..4ad8123554 100644 --- a/src/test/fuzz/locale.cpp +++ b/src/test/fuzz/locale.cpp @@ -50,8 +50,6 @@ FUZZ_TARGET(locale) const bool parseint32_without_locale = ParseInt32(random_string, &parseint32_out_without_locale); int64_t parseint64_out_without_locale; const bool parseint64_without_locale = ParseInt64(random_string, &parseint64_out_without_locale); - const int64_t atoi64_without_locale = atoi64(random_string); - const int atoi_without_locale = atoi(random_string); const int64_t random_int64 = fuzzed_data_provider.ConsumeIntegral<int64_t>(); const std::string tostring_without_locale = ToString(random_int64); // The variable `random_int32` is no longer used, but the harness still needs to @@ -77,10 +75,6 @@ FUZZ_TARGET(locale) if (parseint64_without_locale) { assert(parseint64_out_without_locale == parseint64_out_with_locale); } - const int64_t atoi64_with_locale = atoi64(random_string); - assert(atoi64_without_locale == atoi64_with_locale); - const int atoi_with_locale = atoi(random_string); - assert(atoi_without_locale == atoi_with_locale); const std::string tostring_with_locale = ToString(random_int64); assert(tostring_without_locale == tostring_with_locale); const std::string strprintf_int_with_locale = strprintf("%d", random_int64); diff --git a/src/test/fuzz/muhash.cpp b/src/test/fuzz/muhash.cpp index 4ea9511870..8304e6fdb8 100644 --- a/src/test/fuzz/muhash.cpp +++ b/src/test/fuzz/muhash.cpp @@ -12,52 +12,47 @@ FUZZ_TARGET(muhash) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - std::vector<uint8_t> data = ConsumeRandomLengthByteVector(fuzzed_data_provider); - std::vector<uint8_t> data2 = ConsumeRandomLengthByteVector(fuzzed_data_provider); - if (data.empty()) { - data.resize(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 4096), fuzzed_data_provider.ConsumeIntegral<uint8_t>()); - } - if (data2.empty()) { - data2.resize(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 4096), fuzzed_data_provider.ConsumeIntegral<uint8_t>()); - } - - data = ConsumeRandomLengthByteVector(fuzzed_data_provider); - data2 = ConsumeRandomLengthByteVector(fuzzed_data_provider); + std::vector<uint8_t> data{ConsumeRandomLengthByteVector(fuzzed_data_provider)}; + std::vector<uint8_t> data2{ConsumeRandomLengthByteVector(fuzzed_data_provider)}; MuHash3072 muhash; - // Test that MuHash result is consistent independent of order of operations muhash.Insert(data); muhash.Insert(data2); + const std::string initial_state_hash{"dd5ad2a105c2d29495f577245c357409002329b9f4d6182c0af3dc2f462555c8"}; uint256 out; - muhash.Finalize(out); - - muhash = MuHash3072(); - muhash.Insert(data2); - muhash.Insert(data); - uint256 out2; - muhash.Finalize(out2); - + CallOneOf( + fuzzed_data_provider, + [&] { + // Test that MuHash result is consistent independent of order of operations + muhash.Finalize(out); + + muhash = MuHash3072(); + muhash.Insert(data2); + muhash.Insert(data); + muhash.Finalize(out2); + }, + [&] { + // Test that multiplication with the initial state never changes the finalized result + muhash.Finalize(out); + MuHash3072 muhash3; + muhash3 *= muhash; + muhash3.Finalize(out2); + }, + [&] { + // Test that dividing a MuHash by itself brings it back to it's initial state + muhash /= muhash; + muhash.Finalize(out); + out2 = uint256S(initial_state_hash); + }, + [&] { + // Test that removing all added elements brings the object back to it's initial state + muhash.Remove(data); + muhash.Remove(data2); + muhash.Finalize(out); + out2 = uint256S(initial_state_hash); + }); assert(out == out2); - MuHash3072 muhash3; - muhash3 *= muhash; - uint256 out3; - muhash3.Finalize(out3); - assert(out == out3); - - // Test that removing all added elements brings the object back to it's initial state - muhash /= muhash; - muhash.Finalize(out); - - MuHash3072 muhash2; - muhash2.Finalize(out2); - - assert(out == out2); - - muhash3.Remove(data); - muhash3.Remove(data2); - muhash3.Finalize(out3); - assert(out == out3); } diff --git a/src/test/fuzz/multiplication_overflow.cpp b/src/test/fuzz/multiplication_overflow.cpp index 0f054529a6..c7251650c2 100644 --- a/src/test/fuzz/multiplication_overflow.cpp +++ b/src/test/fuzz/multiplication_overflow.cpp @@ -2,6 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> @@ -10,14 +14,6 @@ #include <string> #include <vector> -#if defined(__has_builtin) -#if __has_builtin(__builtin_mul_overflow) -#define HAVE_BUILTIN_MUL_OVERFLOW -#endif -#elif defined(__GNUC__) -#define HAVE_BUILTIN_MUL_OVERFLOW -#endif - namespace { template <typename T> void TestMultiplicationOverflow(FuzzedDataProvider& fuzzed_data_provider) diff --git a/src/test/fuzz/net.cpp b/src/test/fuzz/net.cpp index 20d8581312..bd1bb79d0e 100644 --- a/src/test/fuzz/net.cpp +++ b/src/test/fuzz/net.cpp @@ -14,6 +14,7 @@ #include <test/fuzz/util.h> #include <test/util/net.h> #include <test/util/setup_common.h> +#include <util/asmap.h> #include <cstdint> #include <optional> @@ -38,15 +39,8 @@ FUZZ_TARGET_INIT(net, initialize_net) node.CloseSocketDisconnect(); }, [&] { - node.MaybeSetAddrName(fuzzed_data_provider.ConsumeRandomLengthString(32)); - }, - [&] { - const std::vector<bool> asmap = ConsumeRandomLengthBitVector(fuzzed_data_provider); - if (!SanityCheckASMap(asmap)) { - return; - } CNodeStats stats; - node.copyStats(stats, asmap); + node.CopyStats(stats); }, [&] { const CNode* add_ref_node = node.AddRef(); @@ -82,7 +76,6 @@ FUZZ_TARGET_INIT(net, initialize_net) } (void)node.GetAddrLocal(); - (void)node.GetAddrName(); (void)node.GetId(); (void)node.GetLocalNonce(); (void)node.GetLocalServices(); diff --git a/src/test/fuzz/netaddress.cpp b/src/test/fuzz/netaddress.cpp index f9d8129ca9..6cb81901cb 100644 --- a/src/test/fuzz/netaddress.cpp +++ b/src/test/fuzz/netaddress.cpp @@ -54,7 +54,7 @@ FUZZ_TARGET(netaddress) (void)net_addr.IsRFC3927(); (void)net_addr.IsRFC3964(); if (net_addr.IsRFC4193()) { - assert(net_addr.GetNetwork() == Network::NET_ONION || net_addr.GetNetwork() == Network::NET_INTERNAL || net_addr.GetNetwork() == Network::NET_UNROUTABLE); + assert(net_addr.GetNetwork() == Network::NET_INTERNAL || net_addr.GetNetwork() == Network::NET_UNROUTABLE); } (void)net_addr.IsRFC4380(); (void)net_addr.IsRFC4843(); diff --git a/src/test/fuzz/parse_numbers.cpp b/src/test/fuzz/parse_numbers.cpp index 2c546e9b4a..85fee062f0 100644 --- a/src/test/fuzz/parse_numbers.cpp +++ b/src/test/fuzz/parse_numbers.cpp @@ -12,11 +12,7 @@ FUZZ_TARGET(parse_numbers) { const std::string random_string(buffer.begin(), buffer.end()); - CAmount amount; - (void)ParseMoney(random_string, amount); - - double d; - (void)ParseDouble(random_string, &d); + (void)ParseMoney(random_string); uint8_t u8; (void)ParseUInt8(random_string, &u8); @@ -26,13 +22,13 @@ FUZZ_TARGET(parse_numbers) int32_t i32; (void)ParseInt32(random_string, &i32); - (void)atoi(random_string); + (void)LocaleIndependentAtoi<int>(random_string); uint32_t u32; (void)ParseUInt32(random_string, &u32); int64_t i64; - (void)atoi64(random_string); + (void)LocaleIndependentAtoi<int64_t>(random_string); (void)ParseFixedPoint(random_string, 3, &i64); (void)ParseInt64(random_string, &i64); diff --git a/src/test/fuzz/prevector.cpp b/src/test/fuzz/prevector.cpp index 51956bbe9e..d4b3ed501f 100644 --- a/src/test/fuzz/prevector.cpp +++ b/src/test/fuzz/prevector.cpp @@ -209,7 +209,8 @@ FUZZ_TARGET(prevector) FuzzedDataProvider prov(buffer.data(), buffer.size()); prevector_tester<8, int> test; - while (prov.remaining_bytes()) { + LIMITED_WHILE(prov.remaining_bytes(), 3000) + { switch (prov.ConsumeIntegralInRange<int>(0, 13 + 3 * (test.size() > 0))) { case 0: test.insert(prov.ConsumeIntegralInRange<size_t>(0, test.size()), prov.ConsumeIntegral<int>()); diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index c4e4d4c785..7b99193ad0 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -58,19 +58,7 @@ void initialize_process_message() static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); g_setup = testing_setup.get(); - - // Temporary debug for https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=35027 - { - LOCK(::cs_main); - assert(CheckDiskSpace(gArgs.GetDataDirNet())); - assert(CheckDiskSpace(gArgs.GetDataDirNet(), 48 * 2 * 2 * g_setup->m_node.chainman->ActiveChainstate().CoinsTip().GetCacheSize())); - } for (int i = 0; i < 2 * COINBASE_MATURITY; i++) { - { - LOCK(::cs_main); - assert(CheckDiskSpace(gArgs.GetDataDirNet())); - assert(CheckDiskSpace(gArgs.GetDataDirNet(), 48 * 2 * 2 * g_setup->m_node.chainman->ActiveChainstate().CoinsTip().GetCacheSize())); - } MineBlock(g_setup->m_node, CScript() << OP_TRUE); } SyncWithValidationInterfaceQueue(); diff --git a/src/test/fuzz/rolling_bloom_filter.cpp b/src/test/fuzz/rolling_bloom_filter.cpp index 07059cce76..b9ed497e68 100644 --- a/src/test/fuzz/rolling_bloom_filter.cpp +++ b/src/test/fuzz/rolling_bloom_filter.cpp @@ -21,7 +21,8 @@ FUZZ_TARGET(rolling_bloom_filter) CRollingBloomFilter rolling_bloom_filter{ fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(1, 1000), 0.999 / fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(1, std::numeric_limits<unsigned int>::max())}; - while (fuzzed_data_provider.remaining_bytes() > 0) { + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes() > 0, 3000) + { CallOneOf( fuzzed_data_provider, [&] { @@ -32,13 +33,10 @@ FUZZ_TARGET(rolling_bloom_filter) assert(present); }, [&] { - const std::optional<uint256> u256 = ConsumeDeserializable<uint256>(fuzzed_data_provider); - if (!u256) { - return; - } - (void)rolling_bloom_filter.contains(*u256); - rolling_bloom_filter.insert(*u256); - const bool present = rolling_bloom_filter.contains(*u256); + const uint256 u256{ConsumeUInt256(fuzzed_data_provider)}; + (void)rolling_bloom_filter.contains(u256); + rolling_bloom_filter.insert(u256); + const bool present = rolling_bloom_filter.contains(u256); assert(present); }, [&] { diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index b87bcf2ef5..74c576322a 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -6,8 +6,10 @@ #include <compressor.h> #include <core_io.h> #include <core_memusage.h> +#include <key_io.h> #include <policy/policy.h> #include <pubkey.h> +#include <rpc/util.h> #include <script/descriptor.h> #include <script/interpreter.h> #include <script/script.h> @@ -54,46 +56,8 @@ FUZZ_TARGET_INIT(script, initialize_script) assert(script == decompressed_script); } - CTxDestination address; - TxoutType type_ret; - std::vector<CTxDestination> addresses; - int required_ret; - bool extract_destinations_ret = ExtractDestinations(script, type_ret, addresses, required_ret); - bool extract_destination_ret = ExtractDestination(script, address); - if (!extract_destinations_ret) { - assert(!extract_destination_ret); - if (type_ret == TxoutType::MULTISIG) { - assert(addresses.empty() && required_ret == 0); - } else { - assert(type_ret == TxoutType::PUBKEY || - type_ret == TxoutType::NONSTANDARD || - type_ret == TxoutType::NULL_DATA); - } - } else { - assert(required_ret >= 1 && required_ret <= 16); - assert((unsigned long)required_ret == addresses.size()); - assert(type_ret == TxoutType::MULTISIG || required_ret == 1); - } - if (type_ret == TxoutType::NONSTANDARD || type_ret == TxoutType::NULL_DATA) { - assert(!extract_destinations_ret); - } - if (!extract_destination_ret) { - assert(type_ret == TxoutType::PUBKEY || - type_ret == TxoutType::NONSTANDARD || - type_ret == TxoutType::NULL_DATA || - type_ret == TxoutType::MULTISIG); - } else { - assert(address == addresses[0]); - } - if (type_ret == TxoutType::NONSTANDARD || - type_ret == TxoutType::NULL_DATA || - type_ret == TxoutType::MULTISIG) { - assert(!extract_destination_ret); - } - TxoutType which_type; bool is_standard_ret = IsStandard(script, which_type); - assert(type_ret == which_type); if (!is_standard_ret) { assert(which_type == TxoutType::NONSTANDARD || which_type == TxoutType::NULL_DATA || @@ -110,6 +74,20 @@ FUZZ_TARGET_INIT(script, initialize_script) which_type == TxoutType::NONSTANDARD); } + CTxDestination address; + bool extract_destination_ret = ExtractDestination(script, address); + if (!extract_destination_ret) { + assert(which_type == TxoutType::PUBKEY || + which_type == TxoutType::NONSTANDARD || + which_type == TxoutType::NULL_DATA || + which_type == TxoutType::MULTISIG); + } + if (which_type == TxoutType::NONSTANDARD || + which_type == TxoutType::NULL_DATA || + which_type == TxoutType::MULTISIG) { + assert(!extract_destination_ret); + } + const FlatSigningProvider signing_provider; (void)InferDescriptor(script, signing_provider); (void)IsSegWitOutput(signing_provider, script); @@ -131,15 +109,11 @@ FUZZ_TARGET_INIT(script, initialize_script) (void)ScriptToAsmStr(script, true); UniValue o1(UniValue::VOBJ); - ScriptPubKeyToUniv(script, o1, true, true); - ScriptPubKeyToUniv(script, o1, true, false); + ScriptPubKeyToUniv(script, o1, true); UniValue o2(UniValue::VOBJ); - ScriptPubKeyToUniv(script, o2, false, true); - ScriptPubKeyToUniv(script, o2, false, false); + ScriptPubKeyToUniv(script, o2, false); UniValue o3(UniValue::VOBJ); - ScriptToUniv(script, o3, true); - UniValue o4(UniValue::VOBJ); - ScriptToUniv(script, o4, false); + ScriptToUniv(script, o3); { const std::vector<uint8_t> bytes = ConsumeRandomLengthByteVector(fuzzed_data_provider); @@ -184,26 +158,26 @@ FUZZ_TARGET_INIT(script, initialize_script) } { - WitnessUnknown witness_unknown_1{}; - witness_unknown_1.version = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); - const std::vector<uint8_t> witness_unknown_program_1 = fuzzed_data_provider.ConsumeBytes<uint8_t>(40); - witness_unknown_1.length = witness_unknown_program_1.size(); - std::copy(witness_unknown_program_1.begin(), witness_unknown_program_1.end(), witness_unknown_1.program); - - WitnessUnknown witness_unknown_2{}; - witness_unknown_2.version = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); - const std::vector<uint8_t> witness_unknown_program_2 = fuzzed_data_provider.ConsumeBytes<uint8_t>(40); - witness_unknown_2.length = witness_unknown_program_2.size(); - std::copy(witness_unknown_program_2.begin(), witness_unknown_program_2.end(), witness_unknown_2.program); - - (void)(witness_unknown_1 == witness_unknown_2); - (void)(witness_unknown_1 < witness_unknown_2); - } + const CTxDestination tx_destination_1{ + fuzzed_data_provider.ConsumeBool() ? + DecodeDestination(fuzzed_data_provider.ConsumeRandomLengthString()) : + ConsumeTxDestination(fuzzed_data_provider)}; + const CTxDestination tx_destination_2{ConsumeTxDestination(fuzzed_data_provider)}; + const std::string encoded_dest{EncodeDestination(tx_destination_1)}; + const UniValue json_dest{DescribeAddress(tx_destination_1)}; + Assert(tx_destination_1 == DecodeDestination(encoded_dest)); + (void)GetKeyForDestination(/* store */ {}, tx_destination_1); + const CScript dest{GetScriptForDestination(tx_destination_1)}; + const bool valid{IsValidDestination(tx_destination_1)}; + Assert(dest.empty() != valid); + + Assert(valid == IsValidDestinationString(encoded_dest)); - { - const CTxDestination tx_destination_1 = ConsumeTxDestination(fuzzed_data_provider); - const CTxDestination tx_destination_2 = ConsumeTxDestination(fuzzed_data_provider); - (void)(tx_destination_1 == tx_destination_2); (void)(tx_destination_1 < tx_destination_2); + if (tx_destination_1 == tx_destination_2) { + Assert(encoded_dest == EncodeDestination(tx_destination_2)); + Assert(json_dest.write() == DescribeAddress(tx_destination_2).write()); + Assert(dest == GetScriptForDestination(tx_destination_2)); + } } } diff --git a/src/test/fuzz/script_flags.cpp b/src/test/fuzz/script_flags.cpp index 1278dc87d4..43927772ae 100644 --- a/src/test/fuzz/script_flags.cpp +++ b/src/test/fuzz/script_flags.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <consensus/amount.h> #include <pubkey.h> #include <script/interpreter.h> #include <streams.h> diff --git a/src/test/fuzz/script_sign.cpp b/src/test/fuzz/script_sign.cpp index fe850a6959..684324c36e 100644 --- a/src/test/fuzz/script_sign.cpp +++ b/src/test/fuzz/script_sign.cpp @@ -13,6 +13,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <util/translation.h> #include <cassert> #include <cstdint> @@ -135,7 +136,7 @@ FUZZ_TARGET_INIT(script_sign, initialize_script_sign) } coins[*outpoint] = *coin; } - std::map<int, std::string> input_errors; + std::map<int, bilingual_str> input_errors; (void)SignTransaction(sign_transaction_tx_to, &provider, coins, fuzzed_data_provider.ConsumeIntegral<int>(), input_errors); } } diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index 286375f7ae..ab646c68fc 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -31,9 +31,105 @@ #include <version.h> #include <cstdint> +#include <cstdlib> #include <string> #include <vector> +namespace { +bool LegacyParsePrechecks(const std::string& str) +{ + if (str.empty()) // No empty string allowed + return false; + if (str.size() >= 1 && (IsSpace(str[0]) || IsSpace(str[str.size() - 1]))) // No padding allowed + return false; + if (!ValidAsCString(str)) // No embedded NUL characters allowed + return false; + return true; +} + +bool LegacyParseInt32(const std::string& str, int32_t* out) +{ + if (!LegacyParsePrechecks(str)) + return false; + char* endp = nullptr; + errno = 0; // strtol will not set errno if valid + long int n = strtol(str.c_str(), &endp, 10); + if (out) *out = (int32_t)n; + // Note that strtol returns a *long int*, so even if strtol doesn't report an over/underflow + // we still have to check that the returned value is within the range of an *int32_t*. On 64-bit + // platforms the size of these types may be different. + return endp && *endp == 0 && !errno && + n >= std::numeric_limits<int32_t>::min() && + n <= std::numeric_limits<int32_t>::max(); +} + +bool LegacyParseInt64(const std::string& str, int64_t* out) +{ + if (!LegacyParsePrechecks(str)) + return false; + char* endp = nullptr; + errno = 0; // strtoll will not set errno if valid + long long int n = strtoll(str.c_str(), &endp, 10); + if (out) *out = (int64_t)n; + // Note that strtoll returns a *long long int*, so even if strtol doesn't report an over/underflow + // we still have to check that the returned value is within the range of an *int64_t*. + return endp && *endp == 0 && !errno && + n >= std::numeric_limits<int64_t>::min() && + n <= std::numeric_limits<int64_t>::max(); +} + +bool LegacyParseUInt32(const std::string& str, uint32_t* out) +{ + if (!LegacyParsePrechecks(str)) + return false; + if (str.size() >= 1 && str[0] == '-') // Reject negative values, unfortunately strtoul accepts these by default if they fit in the range + return false; + char* endp = nullptr; + errno = 0; // strtoul will not set errno if valid + unsigned long int n = strtoul(str.c_str(), &endp, 10); + if (out) *out = (uint32_t)n; + // Note that strtoul returns a *unsigned long int*, so even if it doesn't report an over/underflow + // we still have to check that the returned value is within the range of an *uint32_t*. On 64-bit + // platforms the size of these types may be different. + return endp && *endp == 0 && !errno && + n <= std::numeric_limits<uint32_t>::max(); +} + +bool LegacyParseUInt8(const std::string& str, uint8_t* out) +{ + uint32_t u32; + if (!LegacyParseUInt32(str, &u32) || u32 > std::numeric_limits<uint8_t>::max()) { + return false; + } + if (out != nullptr) { + *out = static_cast<uint8_t>(u32); + } + return true; +} + +bool LegacyParseUInt64(const std::string& str, uint64_t* out) +{ + if (!LegacyParsePrechecks(str)) + return false; + if (str.size() >= 1 && str[0] == '-') // Reject negative values, unfortunately strtoull accepts these by default if they fit in the range + return false; + char* endp = nullptr; + errno = 0; // strtoull will not set errno if valid + unsigned long long int n = strtoull(str.c_str(), &endp, 10); + if (out) *out = (uint64_t)n; + // Note that strtoull returns a *unsigned long long int*, so even if it doesn't report an over/underflow + // we still have to check that the returned value is within the range of an *uint64_t*. + return endp && *endp == 0 && !errno && + n <= std::numeric_limits<uint64_t>::max(); +} + +// For backwards compatibility checking. +int64_t atoi64_legacy(const std::string& str) +{ + return strtoll(str.c_str(), nullptr, 10); +} +}; // namespace + FUZZ_TARGET(string) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); @@ -66,8 +162,7 @@ FUZZ_TARGET(string) (void)ParseNonRFCJSONValue(random_string_1); } catch (const std::runtime_error&) { } - OutputType output_type; - (void)ParseOutputType(random_string_1, output_type); + (void)ParseOutputType(random_string_1); (void)RemovePrefix(random_string_1, random_string_2); (void)ResolveErrMsg(random_string_1, random_string_2); try { @@ -134,4 +229,67 @@ FUZZ_TARGET(string) const bilingual_str bs2{random_string_2, random_string_1}; (void)(bs1 + bs2); } + { + int32_t i32; + int64_t i64; + uint32_t u32; + uint64_t u64; + uint8_t u8; + const bool ok_i32 = ParseInt32(random_string_1, &i32); + const bool ok_i64 = ParseInt64(random_string_1, &i64); + const bool ok_u32 = ParseUInt32(random_string_1, &u32); + const bool ok_u64 = ParseUInt64(random_string_1, &u64); + const bool ok_u8 = ParseUInt8(random_string_1, &u8); + + int32_t i32_legacy; + int64_t i64_legacy; + uint32_t u32_legacy; + uint64_t u64_legacy; + uint8_t u8_legacy; + const bool ok_i32_legacy = LegacyParseInt32(random_string_1, &i32_legacy); + const bool ok_i64_legacy = LegacyParseInt64(random_string_1, &i64_legacy); + const bool ok_u32_legacy = LegacyParseUInt32(random_string_1, &u32_legacy); + const bool ok_u64_legacy = LegacyParseUInt64(random_string_1, &u64_legacy); + const bool ok_u8_legacy = LegacyParseUInt8(random_string_1, &u8_legacy); + + assert(ok_i32 == ok_i32_legacy); + assert(ok_i64 == ok_i64_legacy); + assert(ok_u32 == ok_u32_legacy); + assert(ok_u64 == ok_u64_legacy); + assert(ok_u8 == ok_u8_legacy); + + if (ok_i32) { + assert(i32 == i32_legacy); + } + if (ok_i64) { + assert(i64 == i64_legacy); + } + if (ok_u32) { + assert(u32 == u32_legacy); + } + if (ok_u64) { + assert(u64 == u64_legacy); + } + if (ok_u8) { + assert(u8 == u8_legacy); + } + } + + { + const int atoi_result = atoi(random_string_1.c_str()); + const int locale_independent_atoi_result = LocaleIndependentAtoi<int>(random_string_1); + const int64_t atoi64_result = atoi64_legacy(random_string_1); + const bool out_of_range = atoi64_result < std::numeric_limits<int>::min() || atoi64_result > std::numeric_limits<int>::max(); + if (out_of_range) { + assert(locale_independent_atoi_result == 0); + } else { + assert(atoi_result == locale_independent_atoi_result); + } + } + + { + const int64_t atoi64_result = atoi64_legacy(random_string_1); + const int64_t locale_independent_atoi_result = LocaleIndependentAtoi<int64_t>(random_string_1); + assert(atoi64_result == locale_independent_atoi_result || locale_independent_atoi_result == 0); + } } diff --git a/src/test/fuzz/system.cpp b/src/test/fuzz/system.cpp index b25dcfcd3b..dc3f9c8b8f 100644 --- a/src/test/fuzz/system.cpp +++ b/src/test/fuzz/system.cpp @@ -5,6 +5,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/util/setup_common.h> #include <util/system.h> #include <cstdint> @@ -12,6 +13,11 @@ #include <vector> namespace { +void initialize_system() +{ + static const auto testing_setup = MakeNoLogFileContext<>(); +} + std::string GetArgumentName(const std::string& name) { size_t idx = name.find('='); @@ -20,9 +26,8 @@ std::string GetArgumentName(const std::string& name) } return name.substr(0, idx); } -} // namespace -FUZZ_TARGET(system) +FUZZ_TARGET_INIT(system, initialize_system) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); ArgsManager args_manager{}; @@ -31,7 +36,8 @@ FUZZ_TARGET(system) SetupHelpOptions(args_manager); } - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 3000) + { CallOneOf( fuzzed_data_provider, [&] { @@ -96,7 +102,7 @@ FUZZ_TARGET(system) const int64_t i64 = fuzzed_data_provider.ConsumeIntegral<int64_t>(); const bool b = fuzzed_data_provider.ConsumeBool(); - (void)args_manager.GetArg(s1, i64); + (void)args_manager.GetIntArg(s1, i64); (void)args_manager.GetArg(s1, s2); (void)args_manager.GetArgFlags(s1); (void)args_manager.GetArgs(s1); @@ -113,3 +119,4 @@ FUZZ_TARGET(system) (void)HelpRequested(args_manager); } +} // namespace diff --git a/src/test/fuzz/transaction.cpp b/src/test/fuzz/transaction.cpp index ff34cc87b2..a21e5cea0c 100644 --- a/src/test/fuzz/transaction.cpp +++ b/src/test/fuzz/transaction.cpp @@ -103,6 +103,6 @@ FUZZ_TARGET_INIT(transaction, initialize_transaction) (void)IsWitnessStandard(tx, coins_view_cache); UniValue u(UniValue::VOBJ); - TxToUniv(tx, /* hashBlock */ uint256::ZERO, /* include_addresses */ true, u); - TxToUniv(tx, /* hashBlock */ uint256::ONE, /* include_addresses */ false, u); + TxToUniv(tx, /* hashBlock */ uint256::ZERO, u); + TxToUniv(tx, /* hashBlock */ uint256::ONE, u); } diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index bab34ea340..6201cc813c 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -142,7 +142,8 @@ FUZZ_TARGET_INIT(tx_pool_standard, initialize_tx_pool) return c.out.nValue; }; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 300) + { { // Total supply is the mempool fee + all outpoints CAmount supply_now{WITH_LOCK(tx_pool.cs, return tx_pool.GetTotalFee())}; @@ -305,7 +306,8 @@ FUZZ_TARGET_INIT(tx_pool, initialize_tx_pool) CTxMemPool tx_pool_{/* estimator */ nullptr, /* check_ratio */ 1}; MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(&tx_pool_); - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 300) + { const auto mut_tx = ConsumeTransaction(fuzzed_data_provider, txids); if (fuzzed_data_provider.ConsumeBool()) { diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index bcf0b0ce72..d83d2924bb 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -2,6 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <consensus/amount.h> +#include <pubkey.h> #include <test/fuzz/util.h> #include <test/util/script.h> #include <util/rbf.h> @@ -304,3 +306,196 @@ uint32_t ConsumeSequence(FuzzedDataProvider& fuzzed_data_provider) noexcept }) : fuzzed_data_provider.ConsumeIntegral<uint32_t>(); } + +CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + CTxDestination tx_destination; + const size_t call_size{CallOneOf( + fuzzed_data_provider, + [&] { + tx_destination = CNoDestination{}; + }, + [&] { + tx_destination = PKHash{ConsumeUInt160(fuzzed_data_provider)}; + }, + [&] { + tx_destination = ScriptHash{ConsumeUInt160(fuzzed_data_provider)}; + }, + [&] { + tx_destination = WitnessV0ScriptHash{ConsumeUInt256(fuzzed_data_provider)}; + }, + [&] { + tx_destination = WitnessV0KeyHash{ConsumeUInt160(fuzzed_data_provider)}; + }, + [&] { + tx_destination = WitnessV1Taproot{XOnlyPubKey{ConsumeUInt256(fuzzed_data_provider)}}; + }, + [&] { + WitnessUnknown witness_unknown{}; + witness_unknown.version = fuzzed_data_provider.ConsumeIntegralInRange(2, 16); + std::vector<uint8_t> witness_unknown_program_1{fuzzed_data_provider.ConsumeBytes<uint8_t>(40)}; + if (witness_unknown_program_1.size() < 2) { + witness_unknown_program_1 = {0, 0}; + } + witness_unknown.length = witness_unknown_program_1.size(); + std::copy(witness_unknown_program_1.begin(), witness_unknown_program_1.end(), witness_unknown.program); + tx_destination = witness_unknown; + })}; + Assert(call_size == std::variant_size_v<CTxDestination>); + return tx_destination; +} + +CTxMemPoolEntry ConsumeTxMemPoolEntry(FuzzedDataProvider& fuzzed_data_provider, const CTransaction& tx) noexcept +{ + // Avoid: + // policy/feerate.cpp:28:34: runtime error: signed integer overflow: 34873208148477500 * 1000 cannot be represented in type 'long' + // + // Reproduce using CFeeRate(348732081484775, 10).GetFeePerK() + const CAmount fee = std::min<CAmount>(ConsumeMoney(fuzzed_data_provider), std::numeric_limits<CAmount>::max() / static_cast<CAmount>(100000)); + assert(MoneyRange(fee)); + const int64_t time = fuzzed_data_provider.ConsumeIntegral<int64_t>(); + const unsigned int entry_height = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); + const bool spends_coinbase = fuzzed_data_provider.ConsumeBool(); + const unsigned int sig_op_cost = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, MAX_BLOCK_SIGOPS_COST); + return CTxMemPoolEntry{MakeTransactionRef(tx), fee, time, entry_height, spends_coinbase, sig_op_cost, {}}; +} + +bool ContainsSpentInput(const CTransaction& tx, const CCoinsViewCache& inputs) noexcept +{ + for (const CTxIn& tx_in : tx.vin) { + const Coin& coin = inputs.AccessCoin(tx_in.prevout); + if (coin.IsSpent()) { + return true; + } + } + return false; +} + +CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + const Network network = fuzzed_data_provider.PickValueInArray({Network::NET_IPV4, Network::NET_IPV6, Network::NET_INTERNAL, Network::NET_ONION}); + CNetAddr net_addr; + if (network == Network::NET_IPV4) { + in_addr v4_addr = {}; + v4_addr.s_addr = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); + net_addr = CNetAddr{v4_addr}; + } else if (network == Network::NET_IPV6) { + if (fuzzed_data_provider.remaining_bytes() >= 16) { + in6_addr v6_addr = {}; + memcpy(v6_addr.s6_addr, fuzzed_data_provider.ConsumeBytes<uint8_t>(16).data(), 16); + net_addr = CNetAddr{v6_addr, fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; + } + } else if (network == Network::NET_INTERNAL) { + net_addr.SetInternal(fuzzed_data_provider.ConsumeBytesAsString(32)); + } else if (network == Network::NET_ONION) { + net_addr.SetSpecial(fuzzed_data_provider.ConsumeBytesAsString(32)); + } + return net_addr; +} + +FILE* FuzzedFileProvider::open() +{ + SetFuzzedErrNo(m_fuzzed_data_provider); + if (m_fuzzed_data_provider.ConsumeBool()) { + return nullptr; + } + std::string mode; + CallOneOf( + m_fuzzed_data_provider, + [&] { + mode = "r"; + }, + [&] { + mode = "r+"; + }, + [&] { + mode = "w"; + }, + [&] { + mode = "w+"; + }, + [&] { + mode = "a"; + }, + [&] { + mode = "a+"; + }); +#if defined _GNU_SOURCE && !defined __ANDROID__ + const cookie_io_functions_t io_hooks = { + FuzzedFileProvider::read, + FuzzedFileProvider::write, + FuzzedFileProvider::seek, + FuzzedFileProvider::close, + }; + return fopencookie(this, mode.c_str(), io_hooks); +#else + (void)mode; + return nullptr; +#endif +} + +ssize_t FuzzedFileProvider::read(void* cookie, char* buf, size_t size) +{ + FuzzedFileProvider* fuzzed_file = (FuzzedFileProvider*)cookie; + SetFuzzedErrNo(fuzzed_file->m_fuzzed_data_provider); + if (buf == nullptr || size == 0 || fuzzed_file->m_fuzzed_data_provider.ConsumeBool()) { + return fuzzed_file->m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; + } + const std::vector<uint8_t> random_bytes = fuzzed_file->m_fuzzed_data_provider.ConsumeBytes<uint8_t>(size); + if (random_bytes.empty()) { + return 0; + } + std::memcpy(buf, random_bytes.data(), random_bytes.size()); + if (AdditionOverflow(fuzzed_file->m_offset, (int64_t)random_bytes.size())) { + return fuzzed_file->m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; + } + fuzzed_file->m_offset += random_bytes.size(); + return random_bytes.size(); +} + +ssize_t FuzzedFileProvider::write(void* cookie, const char* buf, size_t size) +{ + FuzzedFileProvider* fuzzed_file = (FuzzedFileProvider*)cookie; + SetFuzzedErrNo(fuzzed_file->m_fuzzed_data_provider); + const ssize_t n = fuzzed_file->m_fuzzed_data_provider.ConsumeIntegralInRange<ssize_t>(0, size); + if (AdditionOverflow(fuzzed_file->m_offset, (int64_t)n)) { + return fuzzed_file->m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; + } + fuzzed_file->m_offset += n; + return n; +} + +int FuzzedFileProvider::seek(void* cookie, int64_t* offset, int whence) +{ + assert(whence == SEEK_SET || whence == SEEK_CUR || whence == SEEK_END); + FuzzedFileProvider* fuzzed_file = (FuzzedFileProvider*)cookie; + SetFuzzedErrNo(fuzzed_file->m_fuzzed_data_provider); + int64_t new_offset = 0; + if (whence == SEEK_SET) { + new_offset = *offset; + } else if (whence == SEEK_CUR) { + if (AdditionOverflow(fuzzed_file->m_offset, *offset)) { + return -1; + } + new_offset = fuzzed_file->m_offset + *offset; + } else if (whence == SEEK_END) { + const int64_t n = fuzzed_file->m_fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(0, 4096); + if (AdditionOverflow(n, *offset)) { + return -1; + } + new_offset = n + *offset; + } + if (new_offset < 0) { + return -1; + } + fuzzed_file->m_offset = new_offset; + *offset = new_offset; + return fuzzed_file->m_fuzzed_data_provider.ConsumeIntegralInRange<int>(-1, 0); +} + +int FuzzedFileProvider::close(void* cookie) +{ + FuzzedFileProvider* fuzzed_file = (FuzzedFileProvider*)cookie; + SetFuzzedErrNo(fuzzed_file->m_fuzzed_data_provider); + return fuzzed_file->m_fuzzed_data_provider.ConsumeIntegralInRange<int>(-1, 0); +} diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 023dcdb3e5..1bc6f1db45 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -5,12 +5,12 @@ #ifndef BITCOIN_TEST_FUZZ_UTIL_H #define BITCOIN_TEST_FUZZ_UTIL_H -#include <amount.h> #include <arith_uint256.h> #include <attributes.h> #include <chainparamsbase.h> #include <coins.h> #include <compat.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <merkleblock.h> #include <net.h> @@ -37,7 +37,7 @@ #include <vector> template <typename... Callables> -void CallOneOf(FuzzedDataProvider& fuzzed_data_provider, Callables... callables) +size_t CallOneOf(FuzzedDataProvider& fuzzed_data_provider, Callables... callables) { constexpr size_t call_size{sizeof...(callables)}; static_assert(call_size >= 1); @@ -45,6 +45,7 @@ void CallOneOf(FuzzedDataProvider& fuzzed_data_provider, Callables... callables) size_t i{0}; ((i++ == call_index ? callables() : void()), ...); + return call_size; } template <typename Collection> @@ -163,51 +164,9 @@ template <typename WeakEnumType, size_t size> return UintToArith256(ConsumeUInt256(fuzzed_data_provider)); } -[[nodiscard]] inline CTxMemPoolEntry ConsumeTxMemPoolEntry(FuzzedDataProvider& fuzzed_data_provider, const CTransaction& tx) noexcept -{ - // Avoid: - // policy/feerate.cpp:28:34: runtime error: signed integer overflow: 34873208148477500 * 1000 cannot be represented in type 'long' - // - // Reproduce using CFeeRate(348732081484775, 10).GetFeePerK() - const CAmount fee = std::min<CAmount>(ConsumeMoney(fuzzed_data_provider), std::numeric_limits<CAmount>::max() / static_cast<CAmount>(100000)); - assert(MoneyRange(fee)); - const int64_t time = fuzzed_data_provider.ConsumeIntegral<int64_t>(); - const unsigned int entry_height = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); - const bool spends_coinbase = fuzzed_data_provider.ConsumeBool(); - const unsigned int sig_op_cost = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, MAX_BLOCK_SIGOPS_COST); - return CTxMemPoolEntry{MakeTransactionRef(tx), fee, time, entry_height, spends_coinbase, sig_op_cost, {}}; -} +[[nodiscard]] CTxMemPoolEntry ConsumeTxMemPoolEntry(FuzzedDataProvider& fuzzed_data_provider, const CTransaction& tx) noexcept; -[[nodiscard]] inline CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) noexcept -{ - CTxDestination tx_destination; - CallOneOf( - fuzzed_data_provider, - [&] { - tx_destination = CNoDestination{}; - }, - [&] { - tx_destination = PKHash{ConsumeUInt160(fuzzed_data_provider)}; - }, - [&] { - tx_destination = ScriptHash{ConsumeUInt160(fuzzed_data_provider)}; - }, - [&] { - tx_destination = WitnessV0ScriptHash{ConsumeUInt256(fuzzed_data_provider)}; - }, - [&] { - tx_destination = WitnessV0KeyHash{ConsumeUInt160(fuzzed_data_provider)}; - }, - [&] { - WitnessUnknown witness_unknown{}; - witness_unknown.version = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); - const std::vector<uint8_t> witness_unknown_program_1 = fuzzed_data_provider.ConsumeBytes<uint8_t>(40); - witness_unknown.length = witness_unknown_program_1.size(); - std::copy(witness_unknown_program_1.begin(), witness_unknown_program_1.end(), witness_unknown.program); - tx_destination = witness_unknown; - }); - return tx_destination; -} +[[nodiscard]] CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) noexcept; template <typename T> [[nodiscard]] bool MultiplicationOverflow(const T i, const T j) noexcept @@ -243,16 +202,7 @@ template <class T> return std::numeric_limits<T>::max() - i < j; } -[[nodiscard]] inline bool ContainsSpentInput(const CTransaction& tx, const CCoinsViewCache& inputs) noexcept -{ - for (const CTxIn& tx_in : tx.vin) { - const Coin& coin = inputs.AccessCoin(tx_in.prevout); - if (coin.IsSpent()) { - return true; - } - } - return false; -} +[[nodiscard]] bool ContainsSpentInput(const CTransaction& tx, const CCoinsViewCache& inputs) noexcept; /** * Sets errno to a value selected from the given std::array `errnos`. @@ -287,27 +237,7 @@ inline void SetFuzzedErrNo(FuzzedDataProvider& fuzzed_data_provider) noexcept return result; } -inline CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcept -{ - const Network network = fuzzed_data_provider.PickValueInArray({Network::NET_IPV4, Network::NET_IPV6, Network::NET_INTERNAL, Network::NET_ONION}); - CNetAddr net_addr; - if (network == Network::NET_IPV4) { - in_addr v4_addr = {}; - v4_addr.s_addr = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); - net_addr = CNetAddr{v4_addr}; - } else if (network == Network::NET_IPV6) { - if (fuzzed_data_provider.remaining_bytes() >= 16) { - in6_addr v6_addr = {}; - memcpy(v6_addr.s6_addr, fuzzed_data_provider.ConsumeBytes<uint8_t>(16).data(), 16); - net_addr = CNetAddr{v6_addr, fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; - } - } else if (network == Network::NET_INTERNAL) { - net_addr.SetInternal(fuzzed_data_provider.ConsumeBytesAsString(32)); - } else if (network == Network::NET_ONION) { - net_addr.SetSpecial(fuzzed_data_provider.ConsumeBytesAsString(32)); - } - return net_addr; -} +CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcept; inline CSubNet ConsumeSubNet(FuzzedDataProvider& fuzzed_data_provider) noexcept { @@ -357,112 +287,15 @@ public: { } - FILE* open() - { - SetFuzzedErrNo(m_fuzzed_data_provider); - if (m_fuzzed_data_provider.ConsumeBool()) { - return nullptr; - } - std::string mode; - CallOneOf( - m_fuzzed_data_provider, - [&] { - mode = "r"; - }, - [&] { - mode = "r+"; - }, - [&] { - mode = "w"; - }, - [&] { - mode = "w+"; - }, - [&] { - mode = "a"; - }, - [&] { - mode = "a+"; - }); -#if defined _GNU_SOURCE && !defined __ANDROID__ - const cookie_io_functions_t io_hooks = { - FuzzedFileProvider::read, - FuzzedFileProvider::write, - FuzzedFileProvider::seek, - FuzzedFileProvider::close, - }; - return fopencookie(this, mode.c_str(), io_hooks); -#else - (void)mode; - return nullptr; -#endif - } + FILE* open(); - static ssize_t read(void* cookie, char* buf, size_t size) - { - FuzzedFileProvider* fuzzed_file = (FuzzedFileProvider*)cookie; - SetFuzzedErrNo(fuzzed_file->m_fuzzed_data_provider); - if (buf == nullptr || size == 0 || fuzzed_file->m_fuzzed_data_provider.ConsumeBool()) { - return fuzzed_file->m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; - } - const std::vector<uint8_t> random_bytes = fuzzed_file->m_fuzzed_data_provider.ConsumeBytes<uint8_t>(size); - if (random_bytes.empty()) { - return 0; - } - std::memcpy(buf, random_bytes.data(), random_bytes.size()); - if (AdditionOverflow(fuzzed_file->m_offset, (int64_t)random_bytes.size())) { - return fuzzed_file->m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; - } - fuzzed_file->m_offset += random_bytes.size(); - return random_bytes.size(); - } + static ssize_t read(void* cookie, char* buf, size_t size); - static ssize_t write(void* cookie, const char* buf, size_t size) - { - FuzzedFileProvider* fuzzed_file = (FuzzedFileProvider*)cookie; - SetFuzzedErrNo(fuzzed_file->m_fuzzed_data_provider); - const ssize_t n = fuzzed_file->m_fuzzed_data_provider.ConsumeIntegralInRange<ssize_t>(0, size); - if (AdditionOverflow(fuzzed_file->m_offset, (int64_t)n)) { - return fuzzed_file->m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; - } - fuzzed_file->m_offset += n; - return n; - } + static ssize_t write(void* cookie, const char* buf, size_t size); - static int seek(void* cookie, int64_t* offset, int whence) - { - assert(whence == SEEK_SET || whence == SEEK_CUR || whence == SEEK_END); - FuzzedFileProvider* fuzzed_file = (FuzzedFileProvider*)cookie; - SetFuzzedErrNo(fuzzed_file->m_fuzzed_data_provider); - int64_t new_offset = 0; - if (whence == SEEK_SET) { - new_offset = *offset; - } else if (whence == SEEK_CUR) { - if (AdditionOverflow(fuzzed_file->m_offset, *offset)) { - return -1; - } - new_offset = fuzzed_file->m_offset + *offset; - } else if (whence == SEEK_END) { - const int64_t n = fuzzed_file->m_fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(0, 4096); - if (AdditionOverflow(n, *offset)) { - return -1; - } - new_offset = n + *offset; - } - if (new_offset < 0) { - return -1; - } - fuzzed_file->m_offset = new_offset; - *offset = new_offset; - return fuzzed_file->m_fuzzed_data_provider.ConsumeIntegralInRange<int>(-1, 0); - } + static int seek(void* cookie, int64_t* offset, int whence); - static int close(void* cookie) - { - FuzzedFileProvider* fuzzed_file = (FuzzedFileProvider*)cookie; - SetFuzzedErrNo(fuzzed_file->m_fuzzed_data_provider); - return fuzzed_file->m_fuzzed_data_provider.ConsumeIntegralInRange<int>(-1, 0); - } + static int close(void* cookie); }; [[nodiscard]] inline FuzzedFileProvider ConsumeFile(FuzzedDataProvider& fuzzed_data_provider) noexcept diff --git a/src/test/fuzz/utxo_snapshot.cpp b/src/test/fuzz/utxo_snapshot.cpp index 6f2bc081c6..8d2a06f11a 100644 --- a/src/test/fuzz/utxo_snapshot.cpp +++ b/src/test/fuzz/utxo_snapshot.cpp @@ -4,6 +4,7 @@ #include <chainparams.h> #include <consensus/validation.h> +#include <node/utxo_snapshot.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> diff --git a/src/test/fuzz/versionbits.cpp b/src/test/fuzz/versionbits.cpp index 9186821836..73a7d24971 100644 --- a/src/test/fuzz/versionbits.cpp +++ b/src/test/fuzz/versionbits.cpp @@ -6,6 +6,7 @@ #include <chainparams.h> #include <consensus/params.h> #include <primitives/block.h> +#include <util/system.h> #include <versionbits.h> #include <test/fuzz/FuzzedDataProvider.h> diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index 2a217f3455..17e904fcff 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -137,20 +137,20 @@ BOOST_AUTO_TEST_CASE(intarg) const auto bar = std::make_pair("-bar", ArgsManager::ALLOW_ANY); SetupArgs({foo, bar}); ResetArgs(""); - BOOST_CHECK_EQUAL(m_local_args.GetArg("-foo", 11), 11); - BOOST_CHECK_EQUAL(m_local_args.GetArg("-foo", 0), 0); + BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-foo", 11), 11); + BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-foo", 0), 0); ResetArgs("-foo -bar"); - BOOST_CHECK_EQUAL(m_local_args.GetArg("-foo", 11), 0); - BOOST_CHECK_EQUAL(m_local_args.GetArg("-bar", 11), 0); + BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-foo", 11), 0); + BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-bar", 11), 0); ResetArgs("-foo=11 -bar=12"); - BOOST_CHECK_EQUAL(m_local_args.GetArg("-foo", 0), 11); - BOOST_CHECK_EQUAL(m_local_args.GetArg("-bar", 11), 12); + BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-foo", 0), 11); + BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-bar", 11), 12); ResetArgs("-foo=NaN -bar=NotANumber"); - BOOST_CHECK_EQUAL(m_local_args.GetArg("-foo", 1), 0); - BOOST_CHECK_EQUAL(m_local_args.GetArg("-bar", 11), 0); + BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-foo", 1), 0); + BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-bar", 11), 0); } BOOST_AUTO_TEST_CASE(doubledash) @@ -163,7 +163,7 @@ BOOST_AUTO_TEST_CASE(doubledash) ResetArgs("--foo=verbose --bar=1"); BOOST_CHECK_EQUAL(m_local_args.GetArg("-foo", ""), "verbose"); - BOOST_CHECK_EQUAL(m_local_args.GetArg("-bar", 0), 1); + BOOST_CHECK_EQUAL(m_local_args.GetIntArg("-bar", 0), 1); } BOOST_AUTO_TEST_CASE(boolargno) diff --git a/src/test/interfaces_tests.cpp b/src/test/interfaces_tests.cpp index 42a7c7798c..44779f7d7c 100644 --- a/src/test/interfaces_tests.cpp +++ b/src/test/interfaces_tests.cpp @@ -98,7 +98,7 @@ BOOST_AUTO_TEST_CASE(findCommonAncestor) auto* orig_tip = active.Tip(); for (int i = 0; i < 10; ++i) { BlockValidationState state; - m_node.chainman->ActiveChainstate().InvalidateBlock(state, Params(), active.Tip()); + m_node.chainman->ActiveChainstate().InvalidateBlock(state, active.Tip()); } BOOST_CHECK_EQUAL(active.Height(), orig_tip->nHeight - 10); coinbaseKey.MakeNewKey(true); diff --git a/src/test/logging_tests.cpp b/src/test/logging_tests.cpp index e99c6e0fc8..84ddbc50c6 100644 --- a/src/test/logging_tests.cpp +++ b/src/test/logging_tests.cpp @@ -15,9 +15,9 @@ BOOST_FIXTURE_TEST_SUITE(logging_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(logging_timer) { SetMockTime(1); - auto sec_timer = BCLog::Timer<std::chrono::seconds>("tests", "end_msg"); + auto micro_timer = BCLog::Timer<std::chrono::microseconds>("tests", "end_msg"); SetMockTime(2); - BOOST_CHECK_EQUAL(sec_timer.LogMsg("test secs"), "tests: test secs (1.00s)"); + BOOST_CHECK_EQUAL(micro_timer.LogMsg("test micros"), "tests: test micros (1000000μs)"); SetMockTime(1); auto ms_timer = BCLog::Timer<std::chrono::milliseconds>("tests", "end_msg"); @@ -25,9 +25,9 @@ BOOST_AUTO_TEST_CASE(logging_timer) BOOST_CHECK_EQUAL(ms_timer.LogMsg("test ms"), "tests: test ms (1000.00ms)"); SetMockTime(1); - auto micro_timer = BCLog::Timer<std::chrono::microseconds>("tests", "end_msg"); + auto sec_timer = BCLog::Timer<std::chrono::seconds>("tests", "end_msg"); SetMockTime(2); - BOOST_CHECK_EQUAL(micro_timer.LogMsg("test micros"), "tests: test micros (1000000.00μs)"); + BOOST_CHECK_EQUAL(sec_timer.LogMsg("test secs"), "tests: test secs (1.00s)"); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index e20c5e4e8f..7f44dcf20e 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2011-2020 The Bitcoin Core developers +// Copyright (c) 2011-2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -16,6 +16,7 @@ #include <util/system.h> #include <util/time.h> #include <validation.h> +#include <versionbits.h> #include <test/util/setup_common.h> @@ -51,36 +52,25 @@ BlockAssembler MinerTestingSetup::AssemblerForTest(const CChainParams& params) constexpr static struct { unsigned char extranonce; unsigned int nonce; -} blockinfo[] = { - {4, 0xa4a3e223}, {2, 0x15c32f9e}, {1, 0x0375b547}, {1, 0x7004a8a5}, - {2, 0xce440296}, {2, 0x52cfe198}, {1, 0x77a72cd0}, {2, 0xbb5d6f84}, - {2, 0x83f30c2c}, {1, 0x48a73d5b}, {1, 0xef7dcd01}, {2, 0x6809c6c4}, - {2, 0x0883ab3c}, {1, 0x087bbbe2}, {2, 0x2104a814}, {2, 0xdffb6daa}, - {1, 0xee8a0a08}, {2, 0xba4237c1}, {1, 0xa70349dc}, {1, 0x344722bb}, - {3, 0xd6294733}, {2, 0xec9f5c94}, {2, 0xca2fbc28}, {1, 0x6ba4f406}, - {2, 0x015d4532}, {1, 0x6e119b7c}, {2, 0x43e8f314}, {2, 0x27962f38}, - {2, 0xb571b51b}, {2, 0xb36bee23}, {2, 0xd17924a8}, {2, 0x6bc212d9}, - {1, 0x630d4948}, {2, 0x9a4c4ebb}, {2, 0x554be537}, {1, 0xd63ddfc7}, - {2, 0xa10acc11}, {1, 0x759a8363}, {2, 0xfb73090d}, {1, 0xe82c6a34}, - {1, 0xe33e92d7}, {3, 0x658ef5cb}, {2, 0xba32ff22}, {5, 0x0227a10c}, - {1, 0xa9a70155}, {5, 0xd096d809}, {1, 0x37176174}, {1, 0x830b8d0f}, - {1, 0xc6e3910e}, {2, 0x823f3ca8}, {1, 0x99850849}, {1, 0x7521fb81}, - {1, 0xaacaabab}, {1, 0xd645a2eb}, {5, 0x7aea1781}, {5, 0x9d6e4b78}, - {1, 0x4ce90fd8}, {1, 0xabdc832d}, {6, 0x4a34f32a}, {2, 0xf2524c1c}, - {2, 0x1bbeb08a}, {1, 0xad47f480}, {1, 0x9f026aeb}, {1, 0x15a95049}, - {2, 0xd1cb95b2}, {2, 0xf84bbda5}, {1, 0x0fa62cd1}, {1, 0xe05f9169}, - {1, 0x78d194a9}, {5, 0x3e38147b}, {5, 0x737ba0d4}, {1, 0x63378e10}, - {1, 0x6d5f91cf}, {2, 0x88612eb8}, {2, 0xe9639484}, {1, 0xb7fabc9d}, - {2, 0x19b01592}, {1, 0x5a90dd31}, {2, 0x5bd7e028}, {2, 0x94d00323}, - {1, 0xa9b9c01a}, {1, 0x3a40de61}, {1, 0x56e7eec7}, {5, 0x859f7ef6}, - {1, 0xfd8e5630}, {1, 0x2b0c9f7f}, {1, 0xba700e26}, {1, 0x7170a408}, - {1, 0x70de86a8}, {1, 0x74d64cd5}, {1, 0x49e738a1}, {2, 0x6910b602}, - {0, 0x643c565f}, {1, 0x54264b3f}, {2, 0x97ea6396}, {2, 0x55174459}, - {2, 0x03e8779a}, {1, 0x98f34d8f}, {1, 0xc07b2b07}, {1, 0xdfe29668}, - {1, 0x3141c7c1}, {1, 0xb3b595f4}, {1, 0x735abf08}, {5, 0x623bfbce}, - {2, 0xd351e722}, {1, 0xf4ca48c9}, {1, 0x5b19c670}, {1, 0xa164bf0e}, - {2, 0xbbbeb305}, {2, 0xfe1c810a}, -}; +} BLOCKINFO[]{{8, 582909131}, {0, 971462344}, {2, 1169481553}, {6, 66147495}, {7, 427785981}, {8, 80538907}, + {8, 207348013}, {2, 1951240923}, {4, 215054351}, {1, 491520534}, {8, 1282281282}, {4, 639565734}, + {3, 248274685}, {8, 1160085976}, {6, 396349768}, {5, 393780549}, {5, 1096899528}, {4, 965381630}, + {0, 728758712}, {5, 318638310}, {3, 164591898}, {2, 274234550}, {2, 254411237}, {7, 561761812}, + {2, 268342573}, {0, 402816691}, {1, 221006382}, {6, 538872455}, {7, 393315655}, {4, 814555937}, + {7, 504879194}, {6, 467769648}, {3, 925972193}, {2, 200581872}, {3, 168915404}, {8, 430446262}, + {5, 773507406}, {3, 1195366164}, {0, 433361157}, {3, 297051771}, {0, 558856551}, {2, 501614039}, + {3, 528488272}, {2, 473587734}, {8, 230125274}, {2, 494084400}, {4, 357314010}, {8, 60361686}, + {7, 640624687}, {3, 480441695}, {8, 1424447925}, {4, 752745419}, {1, 288532283}, {6, 669170574}, + {5, 1900907591}, {3, 555326037}, {3, 1121014051}, {0, 545835650}, {8, 189196651}, {5, 252371575}, + {0, 199163095}, {6, 558895874}, {6, 1656839784}, {6, 815175452}, {6, 718677851}, {5, 544000334}, + {0, 340113484}, {6, 850744437}, {4, 496721063}, {8, 524715182}, {6, 574361898}, {6, 1642305743}, + {6, 355110149}, {5, 1647379658}, {8, 1103005356}, {7, 556460625}, {3, 1139533992}, {5, 304736030}, + {2, 361539446}, {2, 143720360}, {6, 201939025}, {7, 423141476}, {4, 574633709}, {3, 1412254823}, + {4, 873254135}, {0, 341817335}, {6, 53501687}, {3, 179755410}, {5, 172209688}, {8, 516810279}, + {4, 1228391489}, {8, 325372589}, {6, 550367589}, {0, 876291812}, {7, 412454120}, {7, 717202854}, + {2, 222677843}, {6, 251778867}, {7, 842004420}, {7, 194762829}, {4, 96668841}, {1, 925485796}, + {0, 792342903}, {6, 678455063}, {6, 773251385}, {5, 186617471}, {6, 883189502}, {7, 396077336}, + {8, 254702874}, {0, 455592851}}; static CBlockIndex CreateBlockIndex(int nHeight, CBlockIndex* active_chain_tip) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { @@ -220,20 +210,18 @@ BOOST_AUTO_TEST_CASE(CreateNewBlock_validity) // We can't make transactions until we have inputs // Therefore, load 110 blocks :) - static_assert(std::size(blockinfo) == 110, "Should have 110 blocks to import"); + static_assert(std::size(BLOCKINFO) == 110, "Should have 110 blocks to import"); int baseheight = 0; std::vector<CTransactionRef> txFirst; - for (const auto& bi : blockinfo) { + for (const auto& bi : BLOCKINFO) { CBlock *pblock = &pblocktemplate->block; // pointer for convenience { LOCK(cs_main); - pblock->nVersion = 1; + pblock->nVersion = VERSIONBITS_TOP_BITS; pblock->nTime = m_node.chainman->ActiveChain().Tip()->GetMedianTimePast()+1; CMutableTransaction txCoinbase(*pblock->vtx[0]); txCoinbase.nVersion = 1; - txCoinbase.vin[0].scriptSig = CScript(); - txCoinbase.vin[0].scriptSig.push_back(bi.extranonce); - txCoinbase.vin[0].scriptSig.push_back(m_node.chainman->ActiveChain().Height()); + txCoinbase.vin[0].scriptSig = CScript{} << (m_node.chainman->ActiveChain().Height() + 1) << bi.extranonce; txCoinbase.vout.resize(1); // Ignore the (optional) segwit commitment added by CreateNewBlock (as the hardcoded nonces don't account for this) txCoinbase.vout[0].scriptPubKey = CScript(); pblock->vtx[0] = MakeTransactionRef(std::move(txCoinbase)); diff --git a/src/test/net_peer_eviction_tests.cpp b/src/test/net_peer_eviction_tests.cpp index 4bfd487b86..5eb280b498 100644 --- a/src/test/net_peer_eviction_tests.cpp +++ b/src/test/net_peer_eviction_tests.cpp @@ -17,28 +17,6 @@ BOOST_FIXTURE_TEST_SUITE(net_peer_eviction_tests, BasicTestingSetup) -std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(const int n_candidates, FastRandomContext& random_context) -{ - std::vector<NodeEvictionCandidate> candidates; - for (int id = 0; id < n_candidates; ++id) { - candidates.push_back({ - /* id */ id, - /* nTimeConnected */ static_cast<int64_t>(random_context.randrange(100)), - /* m_min_ping_time */ std::chrono::microseconds{random_context.randrange(100)}, - /* nLastBlockTime */ static_cast<int64_t>(random_context.randrange(100)), - /* nLastTXTime */ static_cast<int64_t>(random_context.randrange(100)), - /* fRelevantServices */ random_context.randbool(), - /* fRelayTxes */ random_context.randbool(), - /* fBloomFilter */ random_context.randbool(), - /* nKeyedNetGroup */ random_context.randrange(100), - /* prefer_evict */ random_context.randbool(), - /* m_is_local */ random_context.randbool(), - /* m_network */ ALL_NETWORKS[random_context.randrange(ALL_NETWORKS.size())], - }); - } - return candidates; -} - // Create `num_peers` random nodes, apply setup function `candidate_setup_fn`, // call ProtectEvictionCandidatesByRatio() to apply protection logic, and then // return true if all of `protected_peer_ids` and none of `unprotected_peer_ids` diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 46f88c1282..29938d4ede 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -2,8 +2,6 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <addrdb.h> -#include <addrman.h> #include <chainparams.h> #include <clientversion.h> #include <cstdint> @@ -29,64 +27,6 @@ using namespace std::literals; -class CAddrManSerializationMock : public CAddrMan -{ -public: - virtual void Serialize(CDataStream& s) const = 0; - - //! Ensure that bucket placement is always the same for testing purposes. - void MakeDeterministic() - { - nKey.SetNull(); - insecure_rand = FastRandomContext(true); - } -}; - -class CAddrManUncorrupted : public CAddrManSerializationMock -{ -public: - void Serialize(CDataStream& s) const override - { - CAddrMan::Serialize(s); - } -}; - -class CAddrManCorrupted : public CAddrManSerializationMock -{ -public: - void Serialize(CDataStream& s) const override - { - // Produces corrupt output that claims addrman has 20 addrs when it only has one addr. - unsigned char nVersion = 1; - s << nVersion; - s << ((unsigned char)32); - s << nKey; - s << 10; // nNew - s << 10; // nTried - - int nUBuckets = ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30); - s << nUBuckets; - - CService serv; - BOOST_CHECK(Lookup("252.1.1.1", serv, 7777, false)); - CAddress addr = CAddress(serv, NODE_NONE); - CNetAddr resolved; - BOOST_CHECK(LookupHost("252.2.2.2", resolved, false)); - CAddrInfo info = CAddrInfo(addr, resolved); - s << info; - } -}; - -static CDataStream AddrmanToStream(const CAddrManSerializationMock& _addrman) -{ - CDataStream ssPeersIn(SER_DISK, CLIENT_VERSION); - ssPeersIn << Params().MessageStart(); - ssPeersIn << _addrman; - std::string str = ssPeersIn.str(); - std::vector<unsigned char> vchData(str.begin(), str.end()); - return CDataStream(vchData, SER_DISK, CLIENT_VERSION); -} - BOOST_FIXTURE_TEST_SUITE(net_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(cnode_listen_port) @@ -101,82 +41,6 @@ BOOST_AUTO_TEST_CASE(cnode_listen_port) BOOST_CHECK(port == altPort); } -BOOST_AUTO_TEST_CASE(caddrdb_read) -{ - CAddrManUncorrupted addrmanUncorrupted; - addrmanUncorrupted.MakeDeterministic(); - - CService addr1, addr2, addr3; - BOOST_CHECK(Lookup("250.7.1.1", addr1, 8333, false)); - BOOST_CHECK(Lookup("250.7.2.2", addr2, 9999, false)); - BOOST_CHECK(Lookup("250.7.3.3", addr3, 9999, false)); - BOOST_CHECK(Lookup("250.7.3.3"s, addr3, 9999, false)); - BOOST_CHECK(!Lookup("250.7.3.3\0example.com"s, addr3, 9999, false)); - - // Add three addresses to new table. - CService source; - BOOST_CHECK(Lookup("252.5.1.1", source, 8333, false)); - BOOST_CHECK(addrmanUncorrupted.Add(CAddress(addr1, NODE_NONE), source)); - BOOST_CHECK(addrmanUncorrupted.Add(CAddress(addr2, NODE_NONE), source)); - BOOST_CHECK(addrmanUncorrupted.Add(CAddress(addr3, NODE_NONE), source)); - - // Test that the de-serialization does not throw an exception. - CDataStream ssPeers1 = AddrmanToStream(addrmanUncorrupted); - bool exceptionThrown = false; - CAddrMan addrman1; - - BOOST_CHECK(addrman1.size() == 0); - try { - unsigned char pchMsgTmp[4]; - ssPeers1 >> pchMsgTmp; - ssPeers1 >> addrman1; - } catch (const std::exception&) { - exceptionThrown = true; - } - - BOOST_CHECK(addrman1.size() == 3); - BOOST_CHECK(exceptionThrown == false); - - // Test that CAddrDB::Read creates an addrman with the correct number of addrs. - CDataStream ssPeers2 = AddrmanToStream(addrmanUncorrupted); - - CAddrMan addrman2; - BOOST_CHECK(addrman2.size() == 0); - BOOST_CHECK(CAddrDB::Read(addrman2, ssPeers2)); - BOOST_CHECK(addrman2.size() == 3); -} - - -BOOST_AUTO_TEST_CASE(caddrdb_read_corrupted) -{ - CAddrManCorrupted addrmanCorrupted; - addrmanCorrupted.MakeDeterministic(); - - // Test that the de-serialization of corrupted addrman throws an exception. - CDataStream ssPeers1 = AddrmanToStream(addrmanCorrupted); - bool exceptionThrown = false; - CAddrMan addrman1; - BOOST_CHECK(addrman1.size() == 0); - try { - unsigned char pchMsgTmp[4]; - ssPeers1 >> pchMsgTmp; - ssPeers1 >> addrman1; - } catch (const std::exception&) { - exceptionThrown = true; - } - // Even through de-serialization failed addrman is not left in a clean state. - BOOST_CHECK(addrman1.size() == 1); - BOOST_CHECK(exceptionThrown); - - // Test that CAddrDB::Read leaves addrman in a clean state if de-serialization fails. - CDataStream ssPeers2 = AddrmanToStream(addrmanCorrupted); - - CAddrMan addrman2; - BOOST_CHECK(addrman2.size() == 0); - BOOST_CHECK(!CAddrDB::Read(addrman2, ssPeers2)); - BOOST_CHECK(addrman2.size() == 0); -} - BOOST_AUTO_TEST_CASE(cnode_simple_test) { SOCKET hSocket = INVALID_SOCKET; @@ -763,37 +627,42 @@ BOOST_AUTO_TEST_CASE(ipv4_peer_with_ipv6_addrMe_test) BOOST_AUTO_TEST_CASE(LimitedAndReachable_Network) { - BOOST_CHECK_EQUAL(IsReachable(NET_IPV4), true); - BOOST_CHECK_EQUAL(IsReachable(NET_IPV6), true); - BOOST_CHECK_EQUAL(IsReachable(NET_ONION), true); + BOOST_CHECK(IsReachable(NET_IPV4)); + BOOST_CHECK(IsReachable(NET_IPV6)); + BOOST_CHECK(IsReachable(NET_ONION)); + BOOST_CHECK(IsReachable(NET_I2P)); SetReachable(NET_IPV4, false); SetReachable(NET_IPV6, false); SetReachable(NET_ONION, false); + SetReachable(NET_I2P, false); - BOOST_CHECK_EQUAL(IsReachable(NET_IPV4), false); - BOOST_CHECK_EQUAL(IsReachable(NET_IPV6), false); - BOOST_CHECK_EQUAL(IsReachable(NET_ONION), false); + BOOST_CHECK(!IsReachable(NET_IPV4)); + BOOST_CHECK(!IsReachable(NET_IPV6)); + BOOST_CHECK(!IsReachable(NET_ONION)); + BOOST_CHECK(!IsReachable(NET_I2P)); SetReachable(NET_IPV4, true); SetReachable(NET_IPV6, true); SetReachable(NET_ONION, true); + SetReachable(NET_I2P, true); - BOOST_CHECK_EQUAL(IsReachable(NET_IPV4), true); - BOOST_CHECK_EQUAL(IsReachable(NET_IPV6), true); - BOOST_CHECK_EQUAL(IsReachable(NET_ONION), true); + BOOST_CHECK(IsReachable(NET_IPV4)); + BOOST_CHECK(IsReachable(NET_IPV6)); + BOOST_CHECK(IsReachable(NET_ONION)); + BOOST_CHECK(IsReachable(NET_I2P)); } BOOST_AUTO_TEST_CASE(LimitedAndReachable_NetworkCaseUnroutableAndInternal) { - BOOST_CHECK_EQUAL(IsReachable(NET_UNROUTABLE), true); - BOOST_CHECK_EQUAL(IsReachable(NET_INTERNAL), true); + BOOST_CHECK(IsReachable(NET_UNROUTABLE)); + BOOST_CHECK(IsReachable(NET_INTERNAL)); SetReachable(NET_UNROUTABLE, false); SetReachable(NET_INTERNAL, false); - BOOST_CHECK_EQUAL(IsReachable(NET_UNROUTABLE), true); // Ignored for both networks - BOOST_CHECK_EQUAL(IsReachable(NET_INTERNAL), true); + BOOST_CHECK(IsReachable(NET_UNROUTABLE)); // Ignored for both networks + BOOST_CHECK(IsReachable(NET_INTERNAL)); } CNetAddr UtilBuildAddress(unsigned char p1, unsigned char p2, unsigned char p3, unsigned char p4) @@ -812,10 +681,10 @@ BOOST_AUTO_TEST_CASE(LimitedAndReachable_CNetAddr) CNetAddr addr = UtilBuildAddress(0x001, 0x001, 0x001, 0x001); // 1.1.1.1 SetReachable(NET_IPV4, true); - BOOST_CHECK_EQUAL(IsReachable(addr), true); + BOOST_CHECK(IsReachable(addr)); SetReachable(NET_IPV4, false); - BOOST_CHECK_EQUAL(IsReachable(addr), false); + BOOST_CHECK(!IsReachable(addr)); SetReachable(NET_IPV4, true); // have to reset this, because this is stateful. } @@ -827,12 +696,12 @@ BOOST_AUTO_TEST_CASE(LocalAddress_BasicLifecycle) SetReachable(NET_IPV4, true); - BOOST_CHECK_EQUAL(IsLocal(addr), false); - BOOST_CHECK_EQUAL(AddLocal(addr, 1000), true); - BOOST_CHECK_EQUAL(IsLocal(addr), true); + BOOST_CHECK(!IsLocal(addr)); + BOOST_CHECK(AddLocal(addr, 1000)); + BOOST_CHECK(IsLocal(addr)); RemoveLocal(addr); - BOOST_CHECK_EQUAL(IsLocal(addr), false); + BOOST_CHECK(!IsLocal(addr)); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/policy_fee_tests.cpp b/src/test/policy_fee_tests.cpp index 4a15be6ca6..f9c7d04d6c 100644 --- a/src/test/policy_fee_tests.cpp +++ b/src/test/policy_fee_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> +#include <consensus/amount.h> #include <policy/fees.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/script_parse_tests.cpp b/src/test/script_parse_tests.cpp new file mode 100644 index 0000000000..5b8b6a725f --- /dev/null +++ b/src/test/script_parse_tests.cpp @@ -0,0 +1,55 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <core_io.h> +#include <script/script.h> +#include <util/strencodings.h> +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +BOOST_AUTO_TEST_SUITE(script_parse_tests) +BOOST_AUTO_TEST_CASE(parse_script) +{ + const std::vector<std::pair<std::string,std::string>> IN_OUT{ + // {IN: script string , OUT: hex string } + {"", ""}, + {"0", "00"}, + {"1", "51"}, + {"2", "52"}, + {"3", "53"}, + {"4", "54"}, + {"5", "55"}, + {"6", "56"}, + {"7", "57"}, + {"8", "58"}, + {"9", "59"}, + {"10", "5a"}, + {"11", "5b"}, + {"12", "5c"}, + {"13", "5d"}, + {"14", "5e"}, + {"15", "5f"}, + {"16", "60"}, + {"17", "0111"}, + {"-9", "0189"}, + {"0x17", "17"}, + {"'17'", "023137"}, + {"ELSE", "67"}, + {"NOP10", "b9"}, + {"11111111111111111111", "00"}, + }; + std::string all_in; + std::string all_out; + for (const auto& [in, out] : IN_OUT) { + BOOST_CHECK_EQUAL(HexStr(ParseScript(in)), out); + all_in += " " + in + " "; + all_out += out; + } + BOOST_CHECK_EQUAL(HexStr(ParseScript(all_in)), all_out); + + BOOST_CHECK_EXCEPTION(ParseScript("11111111111"), std::runtime_error, HasReason("script parse error: decimal numeric value only allowed in the range -0xFFFFFFFF...0xFFFFFFFF")); + BOOST_CHECK_EXCEPTION(ParseScript("OP_CHECKSIGADD"), std::runtime_error, HasReason("script parse error: unknown opcode")); +} +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index a01d3fa03a..bf8ff5f5e2 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -252,67 +252,6 @@ BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) BOOST_CHECK(std::get<WitnessUnknown>(address) == unk); } -BOOST_AUTO_TEST_CASE(script_standard_ExtractDestinations) -{ - CKey keys[3]; - CPubKey pubkeys[3]; - for (int i = 0; i < 3; i++) { - keys[i].MakeNewKey(true); - pubkeys[i] = keys[i].GetPubKey(); - } - - CScript s; - TxoutType whichType; - std::vector<CTxDestination> addresses; - int nRequired; - - // TxoutType::PUBKEY - s.clear(); - s << ToByteVector(pubkeys[0]) << OP_CHECKSIG; - BOOST_CHECK(ExtractDestinations(s, whichType, addresses, nRequired)); - BOOST_CHECK_EQUAL(whichType, TxoutType::PUBKEY); - BOOST_CHECK_EQUAL(addresses.size(), 1U); - BOOST_CHECK_EQUAL(nRequired, 1); - BOOST_CHECK(std::get<PKHash>(addresses[0]) == PKHash(pubkeys[0])); - - // TxoutType::PUBKEYHASH - s.clear(); - s << OP_DUP << OP_HASH160 << ToByteVector(pubkeys[0].GetID()) << OP_EQUALVERIFY << OP_CHECKSIG; - BOOST_CHECK(ExtractDestinations(s, whichType, addresses, nRequired)); - BOOST_CHECK_EQUAL(whichType, TxoutType::PUBKEYHASH); - BOOST_CHECK_EQUAL(addresses.size(), 1U); - BOOST_CHECK_EQUAL(nRequired, 1); - BOOST_CHECK(std::get<PKHash>(addresses[0]) == PKHash(pubkeys[0])); - - // TxoutType::SCRIPTHASH - CScript redeemScript(s); // initialize with leftover P2PKH script - s.clear(); - s << OP_HASH160 << ToByteVector(CScriptID(redeemScript)) << OP_EQUAL; - BOOST_CHECK(ExtractDestinations(s, whichType, addresses, nRequired)); - BOOST_CHECK_EQUAL(whichType, TxoutType::SCRIPTHASH); - BOOST_CHECK_EQUAL(addresses.size(), 1U); - BOOST_CHECK_EQUAL(nRequired, 1); - BOOST_CHECK(std::get<ScriptHash>(addresses[0]) == ScriptHash(redeemScript)); - - // TxoutType::MULTISIG - s.clear(); - s << OP_2 << - ToByteVector(pubkeys[0]) << - ToByteVector(pubkeys[1]) << - OP_2 << OP_CHECKMULTISIG; - BOOST_CHECK(ExtractDestinations(s, whichType, addresses, nRequired)); - BOOST_CHECK_EQUAL(whichType, TxoutType::MULTISIG); - BOOST_CHECK_EQUAL(addresses.size(), 2U); - BOOST_CHECK_EQUAL(nRequired, 2); - BOOST_CHECK(std::get<PKHash>(addresses[0]) == PKHash(pubkeys[0])); - BOOST_CHECK(std::get<PKHash>(addresses[1]) == PKHash(pubkeys[1])); - - // TxoutType::NULL_DATA - s.clear(); - s << OP_RETURN << std::vector<unsigned char>({75}); - BOOST_CHECK(!ExtractDestinations(s, whichType, addresses, nRequired)); -} - BOOST_AUTO_TEST_CASE(script_standard_GetScriptFor_) { CKey keys[3]; diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 62fd81673d..2c39cbffb9 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -123,7 +123,7 @@ static ScriptError_t ParseScriptError(const std::string& name) BOOST_FIXTURE_TEST_SUITE(script_tests, BasicTestingSetup) -void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScriptWitness& scriptWitness, int flags, const std::string& message, int scriptError, CAmount nValue = 0) +void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScriptWitness& scriptWitness, uint32_t flags, const std::string& message, int scriptError, CAmount nValue = 0) { bool expect = (scriptError == SCRIPT_ERR_OK); if (flags & SCRIPT_VERIFY_CLEANSTACK) { @@ -139,8 +139,8 @@ void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScript // Verify that removing flags from a passing test or adding flags to a failing test does not change the result. for (int i = 0; i < 16; ++i) { - int extra_flags = InsecureRandBits(16); - int combined_flags = expect ? (flags & ~extra_flags) : (flags | extra_flags); + uint32_t extra_flags(InsecureRandBits(16)); + uint32_t combined_flags{expect ? (flags & ~extra_flags) : (flags | extra_flags)}; // Weed out some invalid flag combinations. if (combined_flags & SCRIPT_VERIFY_CLEANSTACK && ~combined_flags & (SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS)) continue; if (combined_flags & SCRIPT_VERIFY_WITNESS && ~combined_flags & SCRIPT_VERIFY_P2SH) continue; @@ -150,7 +150,7 @@ void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScript #if defined(HAVE_CONSENSUS_LIB) CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << tx2; - int libconsensus_flags = flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL; + uint32_t libconsensus_flags{flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL}; if (libconsensus_flags == flags) { int expectedSuccessCode = expect ? 1 : 0; if (flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_WITNESS) { @@ -258,7 +258,7 @@ private: bool havePush; std::vector<unsigned char> push; std::string comment; - int flags; + uint32_t flags; int scriptError; CAmount nValue; @@ -278,7 +278,7 @@ private: } public: - TestBuilder(const CScript& script_, const std::string& comment_, int flags_, bool P2SH = false, WitnessMode wm = WitnessMode::NONE, int witnessversion = 0, CAmount nValue_ = 0) : script(script_), havePush(false), comment(comment_), flags(flags_), scriptError(SCRIPT_ERR_OK), nValue(nValue_) + TestBuilder(const CScript& script_, const std::string& comment_, uint32_t flags_, bool P2SH = false, WitnessMode wm = WitnessMode::NONE, int witnessversion = 0, CAmount nValue_ = 0) : script(script_), havePush(false), comment(comment_), flags(flags_), scriptError(SCRIPT_ERR_OK), nValue(nValue_) { CScript scriptPubKey = script; if (wm == WitnessMode::PKH) { @@ -1160,7 +1160,7 @@ SignatureData CombineSignatures(const CTxOut& txout, const CMutableTransaction& SignatureData data; data.MergeSignatureData(scriptSig1); data.MergeSignatureData(scriptSig2); - ProduceSignature(DUMMY_SIGNING_PROVIDER, MutableTransactionSignatureCreator(&tx, 0, txout.nValue), txout.scriptPubKey, data); + ProduceSignature(DUMMY_SIGNING_PROVIDER, MutableTransactionSignatureCreator(&tx, 0, txout.nValue, SIGHASH_DEFAULT), txout.scriptPubKey, data); return data; } @@ -1677,7 +1677,7 @@ static void AssetTest(const UniValue& test) const std::vector<CTxOut> prevouts = TxOutsFromJSON(test["prevouts"]); BOOST_CHECK(prevouts.size() == mtx.vin.size()); size_t idx = test["index"].get_int64(); - unsigned int test_flags = ParseScriptFlags(test["flags"].get_str()); + uint32_t test_flags{ParseScriptFlags(test["flags"].get_str())}; bool fin = test.exists("final") && test["final"].get_bool(); if (test.exists("success")) { diff --git a/src/test/serfloat_tests.cpp b/src/test/serfloat_tests.cpp index 54e07b0f61..15612e2950 100644 --- a/src/test/serfloat_tests.cpp +++ b/src/test/serfloat_tests.cpp @@ -36,9 +36,9 @@ uint64_t TestDouble(double f) { } // namespace BOOST_AUTO_TEST_CASE(double_serfloat_tests) { - BOOST_CHECK_EQUAL(TestDouble(0.0), 0); + BOOST_CHECK_EQUAL(TestDouble(0.0), 0U); BOOST_CHECK_EQUAL(TestDouble(-0.0), 0x8000000000000000); - BOOST_CHECK_EQUAL(TestDouble(std::numeric_limits<double>::infinity()), 0x7ff0000000000000); + BOOST_CHECK_EQUAL(TestDouble(std::numeric_limits<double>::infinity()), 0x7ff0000000000000U); BOOST_CHECK_EQUAL(TestDouble(-std::numeric_limits<double>::infinity()), 0xfff0000000000000); BOOST_CHECK_EQUAL(TestDouble(0.5), 0x3fe0000000000000ULL); BOOST_CHECK_EQUAL(TestDouble(1.0), 0x3ff0000000000000ULL); @@ -48,8 +48,8 @@ BOOST_AUTO_TEST_CASE(double_serfloat_tests) { // Roundtrip test on IEC559-compatible systems if (std::numeric_limits<double>::is_iec559) { - BOOST_CHECK_EQUAL(sizeof(double), 8); - BOOST_CHECK_EQUAL(sizeof(uint64_t), 8); + BOOST_CHECK_EQUAL(sizeof(double), 8U); + BOOST_CHECK_EQUAL(sizeof(uint64_t), 8U); // Test extreme values TestDouble(std::numeric_limits<double>::min()); TestDouble(-std::numeric_limits<double>::min()); @@ -102,11 +102,12 @@ BOOST_AUTO_TEST_CASE(double_serfloat_tests) { Python code to generate the below hashes: def reversed_hex(x): - return binascii.hexlify(''.join(reversed(x))) + return bytes(reversed(x)).hex() + def dsha256(x): return hashlib.sha256(hashlib.sha256(x).digest()).digest() - reversed_hex(dsha256(''.join(struct.pack('<d', x) for x in range(0,1000)))) == '43d0c82591953c4eafe114590d392676a01585d25b25d433557f0d7878b23f96' + reversed_hex(dsha256(b''.join(struct.pack('<d', x) for x in range(0,1000)))) == '43d0c82591953c4eafe114590d392676a01585d25b25d433557f0d7878b23f96' */ BOOST_AUTO_TEST_CASE(doubles) { diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index 12fc575c1e..db96fd4940 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -67,7 +67,7 @@ BOOST_AUTO_TEST_CASE(GetSigOpCount) * Verifies script execution of the zeroth scriptPubKey of tx output and * zeroth scriptSig and witness of tx input. */ -static ScriptError VerifyWithFlag(const CTransaction& output, const CMutableTransaction& input, int flags) +static ScriptError VerifyWithFlag(const CTransaction& output, const CMutableTransaction& input, uint32_t flags) { ScriptError error; CTransaction inputi(input); @@ -121,7 +121,7 @@ BOOST_AUTO_TEST_CASE(GetTxSigOpCost) key.MakeNewKey(true); CPubKey pubkey = key.GetPubKey(); // Default flags - int flags = SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH; + const uint32_t flags{SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH}; // Multisig script (legacy counting) { diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index 7af2b79f37..acd0151e1a 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -119,7 +119,7 @@ BOOST_AUTO_TEST_CASE(streams_vector_reader_rvalue) uint32_t varint = 0; // Deserialize into r-value reader >> VARINT(varint); - BOOST_CHECK_EQUAL(varint, 54321); + BOOST_CHECK_EQUAL(varint, 54321U); BOOST_CHECK(reader.empty()); } diff --git a/src/test/system_tests.cpp b/src/test/system_tests.cpp index 940145b84f..e97eab2c00 100644 --- a/src/test/system_tests.cpp +++ b/src/test/system_tests.cpp @@ -7,6 +7,11 @@ #include <univalue.h> #ifdef ENABLE_EXTERNAL_SIGNER +#if defined(WIN32) && !defined(__kernel_entry) +// A workaround for boost 1.71 incompatibility with mingw-w64 compiler. +// For details see https://github.com/bitcoin/bitcoin/pull/22348. +#define __kernel_entry +#endif #include <boost/process.hpp> #endif // ENABLE_EXTERNAL_SIGNER diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 40c53cb2ec..c813fbea32 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -8,6 +8,7 @@ #include <checkqueue.h> #include <clientversion.h> +#include <consensus/amount.h> #include <consensus/tx_check.h> #include <consensus/validation.h> #include <core_io.h> @@ -446,7 +447,7 @@ static void CreateCreditAndSpend(const FillableSigningProvider& keystore, const assert(input.vin[0].scriptWitness.stack == inputm.vin[0].scriptWitness.stack); } -static void CheckWithFlag(const CTransactionRef& output, const CMutableTransaction& input, int flags, bool success) +static void CheckWithFlag(const CTransactionRef& output, const CMutableTransaction& input, uint32_t flags, bool success) { ScriptError error; CTransaction inputi(input); @@ -561,7 +562,7 @@ SignatureData CombineSignatures(const CMutableTransaction& input1, const CMutabl SignatureData sigdata; sigdata = DataFromTransaction(input1, 0, tx->vout[0]); sigdata.MergeSignatureData(DataFromTransaction(input2, 0, tx->vout[0])); - ProduceSignature(DUMMY_SIGNING_PROVIDER, MutableTransactionSignatureCreator(&input1, 0, tx->vout[0].nValue), tx->vout[0].scriptPubKey, sigdata); + ProduceSignature(DUMMY_SIGNING_PROVIDER, MutableTransactionSignatureCreator(&input1, 0, tx->vout[0].nValue, SIGHASH_ALL), tx->vout[0].scriptPubKey, sigdata); return sigdata; } @@ -765,95 +766,89 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) key.MakeNewKey(true); t.vout[0].scriptPubKey = GetScriptForDestination(PKHash(key.GetPubKey())); - std::string reason; - BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + constexpr auto CheckIsStandard = [](const auto& t) { + std::string reason; + BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK(reason.empty()); + }; + constexpr auto CheckIsNotStandard = [](const auto& t, const std::string& reason_in) { + std::string reason; + BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); + BOOST_CHECK_EQUAL(reason_in, reason); + }; + + CheckIsStandard(t); // Check dust with default relay fee: - CAmount nDustThreshold = 182 * dustRelayFee.GetFeePerK()/1000; + CAmount nDustThreshold = 182 * dustRelayFee.GetFeePerK() / 1000; BOOST_CHECK_EQUAL(nDustThreshold, 546); // dust: t.vout[0].nValue = nDustThreshold - 1; - reason.clear(); - BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); - BOOST_CHECK_EQUAL(reason, "dust"); + CheckIsNotStandard(t, "dust"); // not dust: t.vout[0].nValue = nDustThreshold; - BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + CheckIsStandard(t); // Disallowed nVersion t.nVersion = -1; - reason.clear(); - BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); - BOOST_CHECK_EQUAL(reason, "version"); + CheckIsNotStandard(t, "version"); t.nVersion = 0; - reason.clear(); - BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); - BOOST_CHECK_EQUAL(reason, "version"); + CheckIsNotStandard(t, "version"); t.nVersion = 3; - reason.clear(); - BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); - BOOST_CHECK_EQUAL(reason, "version"); + CheckIsNotStandard(t, "version"); // Allowed nVersion t.nVersion = 1; - BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + CheckIsStandard(t); t.nVersion = 2; - BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + CheckIsStandard(t); // Check dust with odd relay fee to verify rounding: // nDustThreshold = 182 * 3702 / 1000 dustRelayFee = CFeeRate(3702); // dust: t.vout[0].nValue = 673 - 1; - reason.clear(); - BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); - BOOST_CHECK_EQUAL(reason, "dust"); + CheckIsNotStandard(t, "dust"); // not dust: t.vout[0].nValue = 673; - BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + CheckIsStandard(t); dustRelayFee = CFeeRate(DUST_RELAY_TX_FEE); t.vout[0].scriptPubKey = CScript() << OP_1; - reason.clear(); - BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); - BOOST_CHECK_EQUAL(reason, "scriptpubkey"); + CheckIsNotStandard(t, "scriptpubkey"); // MAX_OP_RETURN_RELAY-byte TxoutType::NULL_DATA (standard) t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3804678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"); BOOST_CHECK_EQUAL(MAX_OP_RETURN_RELAY, t.vout[0].scriptPubKey.size()); - BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + CheckIsStandard(t); // MAX_OP_RETURN_RELAY+1-byte TxoutType::NULL_DATA (non-standard) t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3804678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef3800"); BOOST_CHECK_EQUAL(MAX_OP_RETURN_RELAY + 1, t.vout[0].scriptPubKey.size()); - reason.clear(); - BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); - BOOST_CHECK_EQUAL(reason, "scriptpubkey"); + CheckIsNotStandard(t, "scriptpubkey"); // Data payload can be encoded in any way... t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex(""); - BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + CheckIsStandard(t); t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex("00") << ParseHex("01"); - BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + CheckIsStandard(t); // OP_RESERVED *is* considered to be a PUSHDATA type opcode by IsPushOnly()! t.vout[0].scriptPubKey = CScript() << OP_RETURN << OP_RESERVED << -1 << 0 << ParseHex("01") << 2 << 3 << 4 << 5 << 6 << 7 << 8 << 9 << 10 << 11 << 12 << 13 << 14 << 15 << 16; - BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + CheckIsStandard(t); t.vout[0].scriptPubKey = CScript() << OP_RETURN << 0 << ParseHex("01") << 2 << ParseHex("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); - BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + CheckIsStandard(t); // ...so long as it only contains PUSHDATA's t.vout[0].scriptPubKey = CScript() << OP_RETURN << OP_RETURN; - reason.clear(); - BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); - BOOST_CHECK_EQUAL(reason, "scriptpubkey"); + CheckIsNotStandard(t, "scriptpubkey"); // TxoutType::NULL_DATA w/o PUSHDATA t.vout.resize(1); t.vout[0].scriptPubKey = CScript() << OP_RETURN; - BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + CheckIsStandard(t); // Only one TxoutType::NULL_DATA permitted in all cases t.vout.resize(2); @@ -861,21 +856,15 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) t.vout[0].nValue = 0; t.vout[1].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"); t.vout[1].nValue = 0; - reason.clear(); - BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); - BOOST_CHECK_EQUAL(reason, "multi-op-return"); + CheckIsNotStandard(t, "multi-op-return"); t.vout[0].scriptPubKey = CScript() << OP_RETURN << ParseHex("04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38"); t.vout[1].scriptPubKey = CScript() << OP_RETURN; - reason.clear(); - BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); - BOOST_CHECK_EQUAL(reason, "multi-op-return"); + CheckIsNotStandard(t, "multi-op-return"); t.vout[0].scriptPubKey = CScript() << OP_RETURN; t.vout[1].scriptPubKey = CScript() << OP_RETURN; - reason.clear(); - BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); - BOOST_CHECK_EQUAL(reason, "multi-op-return"); + CheckIsNotStandard(t, "multi-op-return"); // Check large scriptSig (non-standard if size is >1650 bytes) t.vout.resize(1); @@ -883,12 +872,10 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) t.vout[0].scriptPubKey = GetScriptForDestination(PKHash(key.GetPubKey())); // OP_PUSHDATA2 with len (3 bytes) + data (1647 bytes) = 1650 bytes t.vin[0].scriptSig = CScript() << std::vector<unsigned char>(1647, 0); // 1650 - BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + CheckIsStandard(t); t.vin[0].scriptSig = CScript() << std::vector<unsigned char>(1648, 0); // 1651 - reason.clear(); - BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); - BOOST_CHECK_EQUAL(reason, "scriptsig-size"); + CheckIsNotStandard(t, "scriptsig-size"); // Check scriptSig format (non-standard if there are any other ops than just PUSHs) t.vin[0].scriptSig = CScript() @@ -897,7 +884,7 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) << std::vector<unsigned char>(235, 0) // OP_PUSHDATA1 x [...x bytes...] << std::vector<unsigned char>(1234, 0) // OP_PUSHDATA2 x [...x bytes...] << OP_9; - BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + CheckIsStandard(t); const std::vector<unsigned char> non_push_ops = { // arbitrary set of non-push operations OP_NOP, OP_VERIFY, OP_IF, OP_ROT, OP_3DUP, OP_SIZE, OP_EQUAL, OP_ADD, OP_SUB, @@ -917,11 +904,10 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) // replace current push-op with each non-push-op for (auto op : non_push_ops) { t.vin[0].scriptSig[index] = op; - BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); - BOOST_CHECK_EQUAL(reason, "scriptsig-not-pushonly"); + CheckIsNotStandard(t, "scriptsig-not-pushonly"); } t.vin[0].scriptSig[index] = orig_op; // restore op - BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + CheckIsStandard(t); } // Check tx-size (non-standard if transaction weight is > MAX_STANDARD_TX_WEIGHT) @@ -934,27 +920,47 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) // =============================== // total: 400000 vbytes BOOST_CHECK_EQUAL(GetTransactionWeight(CTransaction(t)), 400000); - BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + CheckIsStandard(t); // increase output size by one byte, so we end up with 400004 vbytes t.vout[0].scriptPubKey = CScript() << OP_RETURN << std::vector<unsigned char>(20, 0); // output size: 31 bytes BOOST_CHECK_EQUAL(GetTransactionWeight(CTransaction(t)), 400004); - reason.clear(); - BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); - BOOST_CHECK_EQUAL(reason, "tx-size"); + CheckIsNotStandard(t, "tx-size"); // Check bare multisig (standard if policy flag fIsBareMultisigStd is set) fIsBareMultisigStd = true; t.vout[0].scriptPubKey = GetScriptForMultisig(1, {key.GetPubKey()}); // simple 1-of-1 t.vin.resize(1); t.vin[0].scriptSig = CScript() << std::vector<unsigned char>(65, 0); - BOOST_CHECK(IsStandardTx(CTransaction(t), reason)); + CheckIsStandard(t); fIsBareMultisigStd = false; - reason.clear(); - BOOST_CHECK(!IsStandardTx(CTransaction(t), reason)); - BOOST_CHECK_EQUAL(reason, "bare-multisig"); + CheckIsNotStandard(t, "bare-multisig"); fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; + + // Check P2WPKH outputs dust threshold + t.vout[0].scriptPubKey = CScript() << OP_0 << ParseHex("ffffffffffffffffffffffffffffffffffffffff"); + t.vout[0].nValue = 294; + CheckIsStandard(t); + t.vout[0].nValue = 293; + CheckIsNotStandard(t, "dust"); + + // Check P2WSH outputs dust threshold + t.vout[0].scriptPubKey = CScript() << OP_0 << ParseHex("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"); + t.vout[0].nValue = 330; + CheckIsStandard(t); + t.vout[0].nValue = 329; + CheckIsNotStandard(t, "dust"); + + // Check future Witness Program versions dust threshold + for (int op = OP_2; op <= OP_16; op += 1) { + t.vout[0].scriptPubKey = CScript() << (opcodetype)op << ParseHex("ffff"); + t.vout[0].nValue = 240; + CheckIsStandard(t); + + t.vout[0].nValue = 239; + CheckIsNotStandard(t, "dust"); + } } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 23195c0a26..afb3ad0cfd 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -13,6 +13,11 @@ #include <boost/test/unit_test.hpp> +struct Dersig100Setup : public TestChain100Setup { + Dersig100Setup() + : TestChain100Setup{{"-testactivationheight=dersig@102"}} {} +}; + bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, const CCoinsViewCache& inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, @@ -20,7 +25,7 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, BOOST_AUTO_TEST_SUITE(txvalidationcache_tests) -BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) +BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, Dersig100Setup) { // Make sure skipping validation of transactions that were // validated going into the memory pool does not allow @@ -112,10 +117,15 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, TestChain100Setup) static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t failing_flags, bool add_to_cache, CCoinsViewCache& active_coins_tip) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { PrecomputedTransactionData txdata; - // If we add many more flags, this loop can get too expensive, but we can - // rewrite in the future to randomly pick a set of flags to evaluate. - for (uint32_t test_flags=0; test_flags < (1U << 16); test_flags += 1) { + + FastRandomContext insecure_rand(true); + + for (int count = 0; count < 10000; ++count) { TxValidationState state; + + // Randomly selects flag combinations + uint32_t test_flags = (uint32_t) insecure_rand.randrange((SCRIPT_VERIFY_END_MARKER - 1) << 1); + // Filter out incompatible flag choices if ((test_flags & SCRIPT_VERIFY_CLEANSTACK)) { // CLEANSTACK requires P2SH and WITNESS, see VerifyScript() in @@ -148,7 +158,7 @@ static void ValidateCheckInputsForAllFlags(const CTransaction &tx, uint32_t fail } } -BOOST_FIXTURE_TEST_CASE(checkinputs_test, TestChain100Setup) +BOOST_FIXTURE_TEST_CASE(checkinputs_test, Dersig100Setup) { // Test that passing CheckInputScripts with one set of script flags doesn't imply // that we would pass again with a different set of flags. diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h new file mode 100644 index 0000000000..81ea4c38f5 --- /dev/null +++ b/src/test/util/chainstate.h @@ -0,0 +1,54 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +#ifndef BITCOIN_TEST_UTIL_CHAINSTATE_H +#define BITCOIN_TEST_UTIL_CHAINSTATE_H + +#include <clientversion.h> +#include <fs.h> +#include <node/context.h> +#include <node/utxo_snapshot.h> +#include <rpc/blockchain.h> +#include <validation.h> + +#include <univalue.h> + +#include <boost/test/unit_test.hpp> + +const auto NoMalleation = [](CAutoFile& file, SnapshotMetadata& meta){}; + +/** + * Create and activate a UTXO snapshot, optionally providing a function to + * malleate the snapshot. + */ +template<typename F = decltype(NoMalleation)> +static bool +CreateAndActivateUTXOSnapshot(NodeContext& node, const fs::path root, F malleation = NoMalleation) +{ + // Write out a snapshot to the test's tempdir. + // + int height; + WITH_LOCK(::cs_main, height = node.chainman->ActiveHeight()); + fs::path snapshot_path = root / tfm::format("test_snapshot.%d.dat", height); + FILE* outfile{fsbridge::fopen(snapshot_path, "wb")}; + CAutoFile auto_outfile{outfile, SER_DISK, CLIENT_VERSION}; + + UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), auto_outfile); + BOOST_TEST_MESSAGE( + "Wrote UTXO snapshot to " << snapshot_path.make_preferred().string() << ": " << result.write()); + + // Read the written snapshot in and then activate it. + // + FILE* infile{fsbridge::fopen(snapshot_path, "rb")}; + CAutoFile auto_infile{infile, SER_DISK, CLIENT_VERSION}; + SnapshotMetadata metadata; + auto_infile >> metadata; + + malleation(auto_infile, metadata); + + return node.chainman->ActivateSnapshot(auto_infile, metadata, /*in_memory*/ true); +} + + +#endif // BITCOIN_TEST_UTIL_CHAINSTATE_H diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index 847a490e03..28d7967078 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -6,6 +6,9 @@ #include <chainparams.h> #include <net.h> +#include <span.h> + +#include <vector> void ConnmanTestMsg::NodeReceiveMsgBytes(CNode& node, Span<const uint8_t> msg_bytes, bool& complete) const { @@ -37,3 +40,25 @@ bool ConnmanTestMsg::ReceiveMsgFrom(CNode& node, CSerializedNetMsg& ser_msg) con NodeReceiveMsgBytes(node, ser_msg.data, complete); return complete; } + +std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(int n_candidates, FastRandomContext& random_context) +{ + std::vector<NodeEvictionCandidate> candidates; + for (int id = 0; id < n_candidates; ++id) { + candidates.push_back({ + /* id */ id, + /* nTimeConnected */ static_cast<int64_t>(random_context.randrange(100)), + /* m_min_ping_time */ std::chrono::microseconds{random_context.randrange(100)}, + /* nLastBlockTime */ static_cast<int64_t>(random_context.randrange(100)), + /* nLastTXTime */ static_cast<int64_t>(random_context.randrange(100)), + /* fRelevantServices */ random_context.randbool(), + /* fRelayTxes */ random_context.randbool(), + /* fBloomFilter */ random_context.randbool(), + /* nKeyedNetGroup */ random_context.randrange(100), + /* prefer_evict */ random_context.randbool(), + /* m_is_local */ random_context.randbool(), + /* m_network */ ALL_NETWORKS[random_context.randrange(ALL_NETWORKS.size())], + }); + } + return candidates; +} diff --git a/src/test/util/net.h b/src/test/util/net.h index 1b49a671bd..939ec322ed 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -141,4 +141,6 @@ private: mutable size_t m_consumed; }; +std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(int n_candidates, FastRandomContext& random_context); + #endif // BITCOIN_TEST_UTIL_NET_H diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index f71d9148b6..ebefa9974e 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -114,7 +114,6 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve InitSignatureCache(); InitScriptExecutionCache(); m_node.chain = interfaces::MakeChain(m_node); - g_wallet_init_interface.Construct(m_node); fCheckBlockIndex = true; static bool noui_connected = false; if (!noui_connected) { @@ -141,12 +140,11 @@ ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::ve m_node.scheduler->m_service_thread = std::thread(util::TraceThread, "scheduler", [&] { m_node.scheduler->serviceQueue(); }); GetMainSignals().RegisterBackgroundSignalScheduler(*m_node.scheduler); - pblocktree.reset(new CBlockTreeDB(1 << 20, true)); - m_node.fee_estimator = std::make_unique<CBlockPolicyEstimator>(); m_node.mempool = std::make_unique<CTxMemPool>(m_node.fee_estimator.get(), 1); m_node.chainman = std::make_unique<ChainstateManager>(); + m_node.chainman->m_blockman.m_block_tree_db = std::make_unique<CBlockTreeDB>(1 << 20, true); // Start script-checking threads. Set g_parallel_script_checks to true so they are used. constexpr int script_check_threads = 2; @@ -169,7 +167,6 @@ ChainTestingSetup::~ChainTestingSetup() m_node.scheduler.reset(); m_node.chainman->Reset(); m_node.chainman.reset(); - pblocktree.reset(); } TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args) @@ -180,26 +177,26 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const // instead of unit tests, but for now we need these here. RegisterAllCoreRPCCommands(tableRPC); - m_node.chainman->InitializeChainstate(*m_node.mempool); + m_node.chainman->InitializeChainstate(m_node.mempool.get()); m_node.chainman->ActiveChainstate().InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); assert(!m_node.chainman->ActiveChainstate().CanFlushToDisk()); m_node.chainman->ActiveChainstate().InitCoinsCache(1 << 23); assert(m_node.chainman->ActiveChainstate().CanFlushToDisk()); - if (!m_node.chainman->ActiveChainstate().LoadGenesisBlock(chainparams)) { + if (!m_node.chainman->ActiveChainstate().LoadGenesisBlock()) { throw std::runtime_error("LoadGenesisBlock failed."); } BlockValidationState state; - if (!m_node.chainman->ActiveChainstate().ActivateBestChain(state, chainparams)) { + if (!m_node.chainman->ActiveChainstate().ActivateBestChain(state)) { throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString())); } - m_node.addrman = std::make_unique<CAddrMan>(); + m_node.addrman = std::make_unique<AddrMan>(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); m_node.banman = std::make_unique<BanMan>(m_args.GetDataDirBase() / "banlist", nullptr, DEFAULT_MISBEHAVING_BANTIME); m_node.connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman); // Deterministic randomness for tests. m_node.peerman = PeerManager::make(chainparams, *m_node.connman, *m_node.addrman, - m_node.banman.get(), *m_node.scheduler, *m_node.chainman, + m_node.banman.get(), *m_node.chainman, *m_node.mempool, false); { CConnman::Options options; @@ -208,7 +205,8 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const } } -TestChain100Setup::TestChain100Setup() +TestChain100Setup::TestChain100Setup(const std::vector<const char*>& extra_args) + : TestingSetup{CBaseChainParams::REGTEST, extra_args} { SetMockTime(1598887952); constexpr std::array<unsigned char, 32> vchKey = { @@ -237,11 +235,14 @@ void TestChain100Setup::mineBlocks(int num_blocks) } } -CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey) +CBlock TestChain100Setup::CreateBlock( + const std::vector<CMutableTransaction>& txns, + const CScript& scriptPubKey, + CChainState& chainstate) { const CChainParams& chainparams = Params(); CTxMemPool empty_pool; - CBlock block = BlockAssembler(m_node.chainman->ActiveChainstate(), empty_pool, chainparams).CreateNewBlock(scriptPubKey)->block; + CBlock block = BlockAssembler(chainstate, empty_pool, chainparams).CreateNewBlock(scriptPubKey)->block; Assert(block.vtx.size() == 1); for (const CMutableTransaction& tx : txns) { @@ -251,6 +252,20 @@ CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransa while (!CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce; + return block; +} + +CBlock TestChain100Setup::CreateAndProcessBlock( + const std::vector<CMutableTransaction>& txns, + const CScript& scriptPubKey, + CChainState* chainstate) +{ + if (!chainstate) { + chainstate = &Assert(m_node.chainman)->ActiveChainstate(); + } + + const CChainParams& chainparams = Params(); + const CBlock block = this->CreateBlock(txns, scriptPubKey, *chainstate); std::shared_ptr<const CBlock> shared_pblock = std::make_shared<const CBlock>(block); Assert(m_node.chainman)->ProcessNewBlock(chainparams, shared_pblock, true, nullptr); @@ -294,7 +309,7 @@ CMutableTransaction TestChain100Setup::CreateValidMempoolTransaction(CTransactio input_coins.insert({outpoint_to_spend, utxo_to_spend}); // - Default signature hashing type int nHashType = SIGHASH_ALL; - std::map<int, std::string> input_errors; + std::map<int, bilingual_str> input_errors; assert(SignTransaction(mempool_txn, &keystore, input_coins, nHashType, input_errors)); // If submit=true, add transaction to the mempool. @@ -307,11 +322,6 @@ CMutableTransaction TestChain100Setup::CreateValidMempoolTransaction(CTransactio return mempool_txn; } -TestChain100Setup::~TestChain100Setup() -{ - gArgs.ForceSetArg("-segwitheight", "0"); -} - CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction& tx) const { return FromTx(MakeTransactionRef(tx)); diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 5d12dc2323..7518cdb042 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -113,15 +113,26 @@ class CScript; /** * Testing fixture that pre-creates a 100-block REGTEST-mode block chain */ -struct TestChain100Setup : public RegTestingSetup { - TestChain100Setup(); +struct TestChain100Setup : public TestingSetup { + TestChain100Setup(const std::vector<const char*>& extra_args = {}); /** * Create a new block with just given transactions, coinbase paying to * scriptPubKey, and try to add it to the current chain. + * If no chainstate is specified, default to the active. */ CBlock CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns, - const CScript& scriptPubKey); + const CScript& scriptPubKey, + CChainState* chainstate = nullptr); + + /** + * Create a new block with just given transactions, coinbase paying to + * scriptPubKey. + */ + CBlock CreateBlock( + const std::vector<CMutableTransaction>& txns, + const CScript& scriptPubKey, + CChainState& chainstate); //! Mine a series of new blocks on the active chain. void mineBlocks(int num_blocks); @@ -145,8 +156,6 @@ struct TestChain100Setup : public RegTestingSetup { CAmount output_amount = CAmount(1 * COIN), bool submit = true); - ~TestChain100Setup(); - std::vector<CTransactionRef> m_coinbase_txns; // For convenience, coinbase transactions CKey coinbaseKey; // private/public key needed to spend coinbase transactions }; diff --git a/src/test/util/wallet.cpp b/src/test/util/wallet.cpp index fd6012e9fe..061659818f 100644 --- a/src/test/util/wallet.cpp +++ b/src/test/util/wallet.cpp @@ -8,6 +8,7 @@ #include <outputtype.h> #include <script/standard.h> #ifdef ENABLE_WALLET +#include <util/translation.h> #include <wallet/wallet.h> #endif @@ -18,7 +19,7 @@ std::string getnewaddress(CWallet& w) { constexpr auto output_type = OutputType::BECH32; CTxDestination dest; - std::string error; + bilingual_str error; if (!w.GetNewDestination(output_type, "", dest, error)) assert(false); return EncodeDestination(dest); diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 7ce38519cf..b5088d3c33 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -173,6 +173,22 @@ BOOST_AUTO_TEST_CASE(util_Join) BOOST_CHECK_EQUAL(Join<std::string>({"foo", "bar"}, ", ", op_upper), "FOO, BAR"); } +BOOST_AUTO_TEST_CASE(util_TrimString) +{ + BOOST_CHECK_EQUAL(TrimString(" foo bar "), "foo bar"); + BOOST_CHECK_EQUAL(TrimString("\t \n \n \f\n\r\t\v\tfoo \n \f\n\r\t\v\tbar\t \n \f\n\r\t\v\t\n "), "foo \n \f\n\r\t\v\tbar"); + BOOST_CHECK_EQUAL(TrimString("\t \n foo \n\tbar\t \n "), "foo \n\tbar"); + BOOST_CHECK_EQUAL(TrimString("\t \n foo \n\tbar\t \n ", "fobar"), "\t \n foo \n\tbar\t \n "); + BOOST_CHECK_EQUAL(TrimString("foo bar"), "foo bar"); + BOOST_CHECK_EQUAL(TrimString("foo bar", "fobar"), " "); + BOOST_CHECK_EQUAL(TrimString(std::string("\0 foo \0 ", 8)), std::string("\0 foo \0", 7)); + BOOST_CHECK_EQUAL(TrimString(std::string(" foo ", 5)), std::string("foo", 3)); + BOOST_CHECK_EQUAL(TrimString(std::string("\t\t\0\0\n\n", 6)), std::string("\0\0", 2)); + BOOST_CHECK_EQUAL(TrimString(std::string("\x05\x04\x03\x02\x01\x00", 6)), std::string("\x05\x04\x03\x02\x01\x00", 6)); + BOOST_CHECK_EQUAL(TrimString(std::string("\x05\x04\x03\x02\x01\x00", 6), std::string("\x05\x04\x03\x02\x01", 5)), std::string("\0", 1)); + BOOST_CHECK_EQUAL(TrimString(std::string("\x05\x04\x03\x02\x01\x00", 6), std::string("\x05\x04\x03\x02\x01\x00", 6)), ""); +} + BOOST_AUTO_TEST_CASE(util_FormatParseISO8601DateTime) { BOOST_CHECK_EQUAL(FormatISO8601DateTime(1317425777), "2011-09-30T23:36:17Z"); @@ -284,9 +300,9 @@ public: } if (expect.default_int) { - BOOST_CHECK_EQUAL(test.GetArg("-value", 99999), 99999); + BOOST_CHECK_EQUAL(test.GetIntArg("-value", 99999), 99999); } else if (expect.int_value) { - BOOST_CHECK_EQUAL(test.GetArg("-value", 99999), *expect.int_value); + BOOST_CHECK_EQUAL(test.GetIntArg("-value", 99999), *expect.int_value); } else { BOOST_CHECK(!success); } @@ -416,8 +432,8 @@ static void TestParse(const std::string& str, bool expected_bool, int64_t expect BOOST_CHECK(test.ParseParameters(2, (char**)argv, error)); BOOST_CHECK_EQUAL(test.GetBoolArg("-value", false), expected_bool); BOOST_CHECK_EQUAL(test.GetBoolArg("-value", true), expected_bool); - BOOST_CHECK_EQUAL(test.GetArg("-value", 99998), expected_int); - BOOST_CHECK_EQUAL(test.GetArg("-value", 99999), expected_int); + BOOST_CHECK_EQUAL(test.GetIntArg("-value", 99998), expected_int); + BOOST_CHECK_EQUAL(test.GetIntArg("-value", 99999), expected_int); } // Test bool and int parsing. @@ -768,9 +784,9 @@ BOOST_AUTO_TEST_CASE(util_GetArg) BOOST_CHECK_EQUAL(testArgs.GetArg("strtest1", "default"), "string..."); BOOST_CHECK_EQUAL(testArgs.GetArg("strtest2", "default"), "default"); - BOOST_CHECK_EQUAL(testArgs.GetArg("inttest1", -1), 12345); - BOOST_CHECK_EQUAL(testArgs.GetArg("inttest2", -1), 81985529216486895LL); - BOOST_CHECK_EQUAL(testArgs.GetArg("inttest3", -1), -1); + BOOST_CHECK_EQUAL(testArgs.GetIntArg("inttest1", -1), 12345); + BOOST_CHECK_EQUAL(testArgs.GetIntArg("inttest2", -1), 81985529216486895LL); + BOOST_CHECK_EQUAL(testArgs.GetIntArg("inttest3", -1), -1); BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest1", false), true); BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest2", false), false); BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest3", false), false); @@ -1222,86 +1238,71 @@ BOOST_AUTO_TEST_CASE(util_FormatMoney) BOOST_AUTO_TEST_CASE(util_ParseMoney) { - CAmount ret = 0; - BOOST_CHECK(ParseMoney("0.0", ret)); - BOOST_CHECK_EQUAL(ret, 0); - - BOOST_CHECK(ParseMoney("12345.6789", ret)); - BOOST_CHECK_EQUAL(ret, (COIN/10000)*123456789); - - BOOST_CHECK(ParseMoney("100000000.00", ret)); - BOOST_CHECK_EQUAL(ret, COIN*100000000); - BOOST_CHECK(ParseMoney("10000000.00", ret)); - BOOST_CHECK_EQUAL(ret, COIN*10000000); - BOOST_CHECK(ParseMoney("1000000.00", ret)); - BOOST_CHECK_EQUAL(ret, COIN*1000000); - BOOST_CHECK(ParseMoney("100000.00", ret)); - BOOST_CHECK_EQUAL(ret, COIN*100000); - BOOST_CHECK(ParseMoney("10000.00", ret)); - BOOST_CHECK_EQUAL(ret, COIN*10000); - BOOST_CHECK(ParseMoney("1000.00", ret)); - BOOST_CHECK_EQUAL(ret, COIN*1000); - BOOST_CHECK(ParseMoney("100.00", ret)); - BOOST_CHECK_EQUAL(ret, COIN*100); - BOOST_CHECK(ParseMoney("10.00", ret)); - BOOST_CHECK_EQUAL(ret, COIN*10); - BOOST_CHECK(ParseMoney("1.00", ret)); - BOOST_CHECK_EQUAL(ret, COIN); - BOOST_CHECK(ParseMoney("1", ret)); - BOOST_CHECK_EQUAL(ret, COIN); - BOOST_CHECK(ParseMoney(" 1", ret)); - BOOST_CHECK_EQUAL(ret, COIN); - BOOST_CHECK(ParseMoney("1 ", ret)); - BOOST_CHECK_EQUAL(ret, COIN); - BOOST_CHECK(ParseMoney(" 1 ", ret)); - BOOST_CHECK_EQUAL(ret, COIN); - BOOST_CHECK(ParseMoney("0.1", ret)); - BOOST_CHECK_EQUAL(ret, COIN/10); - BOOST_CHECK(ParseMoney("0.01", ret)); - BOOST_CHECK_EQUAL(ret, COIN/100); - BOOST_CHECK(ParseMoney("0.001", ret)); - BOOST_CHECK_EQUAL(ret, COIN/1000); - BOOST_CHECK(ParseMoney("0.0001", ret)); - BOOST_CHECK_EQUAL(ret, COIN/10000); - BOOST_CHECK(ParseMoney("0.00001", ret)); - BOOST_CHECK_EQUAL(ret, COIN/100000); - BOOST_CHECK(ParseMoney("0.000001", ret)); - BOOST_CHECK_EQUAL(ret, COIN/1000000); - BOOST_CHECK(ParseMoney("0.0000001", ret)); - BOOST_CHECK_EQUAL(ret, COIN/10000000); - BOOST_CHECK(ParseMoney("0.00000001", ret)); - BOOST_CHECK_EQUAL(ret, COIN/100000000); - BOOST_CHECK(ParseMoney(" 0.00000001 ", ret)); - BOOST_CHECK_EQUAL(ret, COIN/100000000); - BOOST_CHECK(ParseMoney("0.00000001 ", ret)); - BOOST_CHECK_EQUAL(ret, COIN/100000000); - BOOST_CHECK(ParseMoney(" 0.00000001", ret)); - BOOST_CHECK_EQUAL(ret, COIN/100000000); - - // Parsing amount that can not be represented in ret should fail - BOOST_CHECK(!ParseMoney("0.000000001", ret)); + BOOST_CHECK_EQUAL(ParseMoney("0.0").value(), 0); + BOOST_CHECK_EQUAL(ParseMoney(".").value(), 0); + BOOST_CHECK_EQUAL(ParseMoney("0.").value(), 0); + BOOST_CHECK_EQUAL(ParseMoney(".0").value(), 0); + BOOST_CHECK_EQUAL(ParseMoney(".6789").value(), 6789'0000); + BOOST_CHECK_EQUAL(ParseMoney("12345.").value(), COIN * 12345); + + BOOST_CHECK_EQUAL(ParseMoney("12345.6789").value(), (COIN/10000)*123456789); + + BOOST_CHECK_EQUAL(ParseMoney("10000000.00").value(), COIN*10000000); + BOOST_CHECK_EQUAL(ParseMoney("1000000.00").value(), COIN*1000000); + BOOST_CHECK_EQUAL(ParseMoney("100000.00").value(), COIN*100000); + BOOST_CHECK_EQUAL(ParseMoney("10000.00").value(), COIN*10000); + BOOST_CHECK_EQUAL(ParseMoney("1000.00").value(), COIN*1000); + BOOST_CHECK_EQUAL(ParseMoney("100.00").value(), COIN*100); + BOOST_CHECK_EQUAL(ParseMoney("10.00").value(), COIN*10); + BOOST_CHECK_EQUAL(ParseMoney("1.00").value(), COIN); + BOOST_CHECK_EQUAL(ParseMoney("1").value(), COIN); + BOOST_CHECK_EQUAL(ParseMoney(" 1").value(), COIN); + BOOST_CHECK_EQUAL(ParseMoney("1 ").value(), COIN); + BOOST_CHECK_EQUAL(ParseMoney(" 1 ").value(), COIN); + BOOST_CHECK_EQUAL(ParseMoney("0.1").value(), COIN/10); + BOOST_CHECK_EQUAL(ParseMoney("0.01").value(), COIN/100); + BOOST_CHECK_EQUAL(ParseMoney("0.001").value(), COIN/1000); + BOOST_CHECK_EQUAL(ParseMoney("0.0001").value(), COIN/10000); + BOOST_CHECK_EQUAL(ParseMoney("0.00001").value(), COIN/100000); + BOOST_CHECK_EQUAL(ParseMoney("0.000001").value(), COIN/1000000); + BOOST_CHECK_EQUAL(ParseMoney("0.0000001").value(), COIN/10000000); + BOOST_CHECK_EQUAL(ParseMoney("0.00000001").value(), COIN/100000000); + BOOST_CHECK_EQUAL(ParseMoney(" 0.00000001 ").value(), COIN/100000000); + BOOST_CHECK_EQUAL(ParseMoney("0.00000001 ").value(), COIN/100000000); + BOOST_CHECK_EQUAL(ParseMoney(" 0.00000001").value(), COIN/100000000); + + // Parsing amount that can not be represented should fail + BOOST_CHECK(!ParseMoney("100000000.00")); + BOOST_CHECK(!ParseMoney("0.000000001")); // Parsing empty string should fail - BOOST_CHECK(!ParseMoney("", ret)); - BOOST_CHECK(!ParseMoney(" ", ret)); - BOOST_CHECK(!ParseMoney(" ", ret)); + BOOST_CHECK(!ParseMoney("")); + BOOST_CHECK(!ParseMoney(" ")); + BOOST_CHECK(!ParseMoney(" ")); // Parsing two numbers should fail - BOOST_CHECK(!ParseMoney("1 2", ret)); - BOOST_CHECK(!ParseMoney(" 1 2 ", ret)); - BOOST_CHECK(!ParseMoney(" 1.2 3 ", ret)); - BOOST_CHECK(!ParseMoney(" 1 2.3 ", ret)); + BOOST_CHECK(!ParseMoney("..")); + BOOST_CHECK(!ParseMoney("0..0")); + BOOST_CHECK(!ParseMoney("1 2")); + BOOST_CHECK(!ParseMoney(" 1 2 ")); + BOOST_CHECK(!ParseMoney(" 1.2 3 ")); + BOOST_CHECK(!ParseMoney(" 1 2.3 ")); + + // Embedded whitespace should fail + BOOST_CHECK(!ParseMoney(" -1 .2 ")); + BOOST_CHECK(!ParseMoney(" 1 .2 ")); + BOOST_CHECK(!ParseMoney(" +1 .2 ")); // Attempted 63 bit overflow should fail - BOOST_CHECK(!ParseMoney("92233720368.54775808", ret)); + BOOST_CHECK(!ParseMoney("92233720368.54775808")); // Parsing negative amounts must fail - BOOST_CHECK(!ParseMoney("-1", ret)); + BOOST_CHECK(!ParseMoney("-1")); // Parsing strings with embedded NUL characters should fail - BOOST_CHECK(!ParseMoney("\0-1"s, ret)); - BOOST_CHECK(!ParseMoney(STRING_WITH_EMBEDDED_NULL_CHAR, ret)); - BOOST_CHECK(!ParseMoney("1\0"s, ret)); + BOOST_CHECK(!ParseMoney("\0-1"s)); + BOOST_CHECK(!ParseMoney(STRING_WITH_EMBEDDED_NULL_CHAR)); + BOOST_CHECK(!ParseMoney("1\0"s)); } BOOST_AUTO_TEST_CASE(util_IsHex) @@ -1485,6 +1486,168 @@ BOOST_AUTO_TEST_CASE(test_ParseInt32) BOOST_CHECK(!ParseInt32("32482348723847471234", nullptr)); } +template <typename T> +static void RunToIntegralTests() +{ + BOOST_CHECK(!ToIntegral<T>(STRING_WITH_EMBEDDED_NULL_CHAR)); + BOOST_CHECK(!ToIntegral<T>(" 1")); + BOOST_CHECK(!ToIntegral<T>("1 ")); + BOOST_CHECK(!ToIntegral<T>("1a")); + BOOST_CHECK(!ToIntegral<T>("1.1")); + BOOST_CHECK(!ToIntegral<T>("1.9")); + BOOST_CHECK(!ToIntegral<T>("+01.9")); + BOOST_CHECK(!ToIntegral<T>("-")); + BOOST_CHECK(!ToIntegral<T>("+")); + BOOST_CHECK(!ToIntegral<T>(" -1")); + BOOST_CHECK(!ToIntegral<T>("-1 ")); + BOOST_CHECK(!ToIntegral<T>(" -1 ")); + BOOST_CHECK(!ToIntegral<T>("+1")); + BOOST_CHECK(!ToIntegral<T>(" +1")); + BOOST_CHECK(!ToIntegral<T>(" +1 ")); + BOOST_CHECK(!ToIntegral<T>("+-1")); + BOOST_CHECK(!ToIntegral<T>("-+1")); + BOOST_CHECK(!ToIntegral<T>("++1")); + BOOST_CHECK(!ToIntegral<T>("--1")); + BOOST_CHECK(!ToIntegral<T>("")); + BOOST_CHECK(!ToIntegral<T>("aap")); + BOOST_CHECK(!ToIntegral<T>("0x1")); + BOOST_CHECK(!ToIntegral<T>("-32482348723847471234")); + BOOST_CHECK(!ToIntegral<T>("32482348723847471234")); +} + +BOOST_AUTO_TEST_CASE(test_ToIntegral) +{ + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("1234").value(), 1'234); + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("0").value(), 0); + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("01234").value(), 1'234); + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("00000000000000001234").value(), 1'234); + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("-00000000000000001234").value(), -1'234); + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("00000000000000000000").value(), 0); + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("-00000000000000000000").value(), 0); + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("-1234").value(), -1'234); + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("-1").value(), -1); + + RunToIntegralTests<uint64_t>(); + RunToIntegralTests<int64_t>(); + RunToIntegralTests<uint32_t>(); + RunToIntegralTests<int32_t>(); + RunToIntegralTests<uint16_t>(); + RunToIntegralTests<int16_t>(); + RunToIntegralTests<uint8_t>(); + RunToIntegralTests<int8_t>(); + + BOOST_CHECK(!ToIntegral<int64_t>("-9223372036854775809")); + BOOST_CHECK_EQUAL(ToIntegral<int64_t>("-9223372036854775808").value(), -9'223'372'036'854'775'807LL - 1LL); + BOOST_CHECK_EQUAL(ToIntegral<int64_t>("9223372036854775807").value(), 9'223'372'036'854'775'807); + BOOST_CHECK(!ToIntegral<int64_t>("9223372036854775808")); + + BOOST_CHECK(!ToIntegral<uint64_t>("-1")); + BOOST_CHECK_EQUAL(ToIntegral<uint64_t>("0").value(), 0U); + BOOST_CHECK_EQUAL(ToIntegral<uint64_t>("18446744073709551615").value(), 18'446'744'073'709'551'615ULL); + BOOST_CHECK(!ToIntegral<uint64_t>("18446744073709551616")); + + BOOST_CHECK(!ToIntegral<int32_t>("-2147483649")); + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("-2147483648").value(), -2'147'483'648LL); + BOOST_CHECK_EQUAL(ToIntegral<int32_t>("2147483647").value(), 2'147'483'647); + BOOST_CHECK(!ToIntegral<int32_t>("2147483648")); + + BOOST_CHECK(!ToIntegral<uint32_t>("-1")); + BOOST_CHECK_EQUAL(ToIntegral<uint32_t>("0").value(), 0U); + BOOST_CHECK_EQUAL(ToIntegral<uint32_t>("4294967295").value(), 4'294'967'295U); + BOOST_CHECK(!ToIntegral<uint32_t>("4294967296")); + + BOOST_CHECK(!ToIntegral<int16_t>("-32769")); + BOOST_CHECK_EQUAL(ToIntegral<int16_t>("-32768").value(), -32'768); + BOOST_CHECK_EQUAL(ToIntegral<int16_t>("32767").value(), 32'767); + BOOST_CHECK(!ToIntegral<int16_t>("32768")); + + BOOST_CHECK(!ToIntegral<uint16_t>("-1")); + BOOST_CHECK_EQUAL(ToIntegral<uint16_t>("0").value(), 0U); + BOOST_CHECK_EQUAL(ToIntegral<uint16_t>("65535").value(), 65'535U); + BOOST_CHECK(!ToIntegral<uint16_t>("65536")); + + BOOST_CHECK(!ToIntegral<int8_t>("-129")); + BOOST_CHECK_EQUAL(ToIntegral<int8_t>("-128").value(), -128); + BOOST_CHECK_EQUAL(ToIntegral<int8_t>("127").value(), 127); + BOOST_CHECK(!ToIntegral<int8_t>("128")); + + BOOST_CHECK(!ToIntegral<uint8_t>("-1")); + BOOST_CHECK_EQUAL(ToIntegral<uint8_t>("0").value(), 0U); + BOOST_CHECK_EQUAL(ToIntegral<uint8_t>("255").value(), 255U); + BOOST_CHECK(!ToIntegral<uint8_t>("256")); +} + +BOOST_AUTO_TEST_CASE(test_LocaleIndependentAtoi) +{ + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("1234"), 1'234); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("0"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("01234"), 1'234); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-1234"), -1'234); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>(" 1"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("1 "), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("1a"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("1.1"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("1.9"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("+01.9"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-1"), -1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>(" -1"), -1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-1 "), -1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>(" -1 "), -1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("+1"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>(" +1"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>(" +1 "), 1); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("+-1"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-+1"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("++1"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("--1"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>(""), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("aap"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("0x1"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-32482348723847471234"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("32482348723847471234"), 0); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int64_t>("-9223372036854775809"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int64_t>("-9223372036854775808"), -9'223'372'036'854'775'807LL - 1LL); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int64_t>("9223372036854775807"), 9'223'372'036'854'775'807); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int64_t>("9223372036854775808"), 0); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint64_t>("-1"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint64_t>("0"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint64_t>("18446744073709551615"), 18'446'744'073'709'551'615ULL); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint64_t>("18446744073709551616"), 0U); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-2147483649"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-2147483648"), -2'147'483'648LL); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("2147483647"), 2'147'483'647); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("2147483648"), 0); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint32_t>("-1"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint32_t>("0"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint32_t>("4294967295"), 4'294'967'295U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint32_t>("4294967296"), 0U); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int16_t>("-32769"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int16_t>("-32768"), -32'768); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int16_t>("32767"), 32'767); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int16_t>("32768"), 0); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint16_t>("-1"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint16_t>("0"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint16_t>("65535"), 65'535U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint16_t>("65536"), 0U); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int8_t>("-129"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int8_t>("-128"), -128); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int8_t>("127"), 127); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int8_t>("128"), 0); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint8_t>("-1"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint8_t>("0"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint8_t>("255"), 255U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint8_t>("256"), 0U); +} + BOOST_AUTO_TEST_CASE(test_ParseInt64) { int64_t n; @@ -1650,32 +1813,6 @@ BOOST_AUTO_TEST_CASE(test_ParseUInt64) BOOST_CHECK(!ParseUInt64("-1234", &n)); } -BOOST_AUTO_TEST_CASE(test_ParseDouble) -{ - double n; - // Valid values - BOOST_CHECK(ParseDouble("1234", nullptr)); - BOOST_CHECK(ParseDouble("0", &n) && n == 0.0); - BOOST_CHECK(ParseDouble("1234", &n) && n == 1234.0); - BOOST_CHECK(ParseDouble("01234", &n) && n == 1234.0); // no octal - BOOST_CHECK(ParseDouble("2147483647", &n) && n == 2147483647.0); - BOOST_CHECK(ParseDouble("-2147483648", &n) && n == -2147483648.0); - BOOST_CHECK(ParseDouble("-1234", &n) && n == -1234.0); - BOOST_CHECK(ParseDouble("1e6", &n) && n == 1e6); - BOOST_CHECK(ParseDouble("-1e6", &n) && n == -1e6); - // Invalid values - BOOST_CHECK(!ParseDouble("", &n)); - BOOST_CHECK(!ParseDouble(" 1", &n)); // no padding inside - BOOST_CHECK(!ParseDouble("1 ", &n)); - BOOST_CHECK(!ParseDouble("1a", &n)); - BOOST_CHECK(!ParseDouble("aap", &n)); - BOOST_CHECK(!ParseDouble("0x1", &n)); // no hex - BOOST_CHECK(!ParseDouble(STRING_WITH_EMBEDDED_NULL_CHAR, &n)); - // Overflow and underflow - BOOST_CHECK(!ParseDouble("-1e10000", nullptr)); - BOOST_CHECK(!ParseDouble("1e10000", nullptr)); -} - BOOST_AUTO_TEST_CASE(test_FormatParagraph) { BOOST_CHECK_EQUAL(FormatParagraph("", 79, 0), ""); diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index e0bc10d660..8f4ff6815b 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -77,6 +77,8 @@ std::shared_ptr<CBlock> MinerTestingSetup::Block(const uint256& prev_hash) txCoinbase.vout[1].nValue = txCoinbase.vout[0].nValue; txCoinbase.vout[0].nValue = 0; txCoinbase.vin[0].scriptWitness.SetNull(); + // Always pad with OP_0 at the end to avoid bad-cb-length error + txCoinbase.vin[0].scriptSig = CScript{} << WITH_LOCK(::cs_main, return m_node.chainman->m_blockman.LookupBlockIndex(prev_hash)->nHeight + 1) << OP_0; pblock->vtx[0] = MakeTransactionRef(std::move(txCoinbase)); return pblock; @@ -84,8 +86,8 @@ std::shared_ptr<CBlock> MinerTestingSetup::Block(const uint256& prev_hash) std::shared_ptr<CBlock> MinerTestingSetup::FinalizeBlock(std::shared_ptr<CBlock> pblock) { - LOCK(cs_main); // For m_node.chainman->m_blockman.LookupBlockIndex - GenerateCoinbaseCommitment(*pblock, m_node.chainman->m_blockman.LookupBlockIndex(pblock->hashPrevBlock), Params().GetConsensus()); + const CBlockIndex* prev_block{WITH_LOCK(::cs_main, return m_node.chainman->m_blockman.LookupBlockIndex(pblock->hashPrevBlock))}; + GenerateCoinbaseCommitment(*pblock, prev_block, Params().GetConsensus()); pblock->hashMerkleRoot = BlockMerkleRoot(*pblock); @@ -93,6 +95,11 @@ std::shared_ptr<CBlock> MinerTestingSetup::FinalizeBlock(std::shared_ptr<CBlock> ++(pblock->nNonce); } + // submit block header, so that miner can get the block height from the + // global state and the node has the topology of the chain + BlockValidationState ignored; + BOOST_CHECK(Assert(m_node.chainman)->ProcessNewBlockHeaders({pblock->GetBlockHeader()}, ignored, Params())); + return pblock; } @@ -147,13 +154,6 @@ BOOST_AUTO_TEST_CASE(processnewblock_signals_ordering) } bool ignored; - BlockValidationState state; - std::vector<CBlockHeader> headers; - std::transform(blocks.begin(), blocks.end(), std::back_inserter(headers), [](std::shared_ptr<const CBlock> b) { return b->GetBlockHeader(); }); - - // Process all the headers so we understand the toplogy of the chain - BOOST_CHECK(Assert(m_node.chainman)->ProcessNewBlockHeaders(headers, state, Params())); - // Connect the genesis block and drain any outstanding events BOOST_CHECK(Assert(m_node.chainman)->ProcessNewBlock(Params(), std::make_shared<CBlock>(Params().GenesisBlock()), true, &ignored)); SyncWithValidationInterfaceQueue(); diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 92d8cf2e7d..726c9ebbb8 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -2,10 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. // +#include <chainparams.h> #include <random.h> #include <uint256.h> #include <consensus/validation.h> #include <sync.h> +#include <rpc/blockchain.h> +#include <test/util/chainstate.h> #include <test/util/setup_common.h> #include <validation.h> @@ -20,6 +23,7 @@ BOOST_FIXTURE_TEST_SUITE(validation_chainstate_tests, TestingSetup) BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) { ChainstateManager manager; + WITH_LOCK(::cs_main, manager.m_blockman.m_block_tree_db = std::make_unique<CBlockTreeDB>(1 << 20, true)); CTxMemPool mempool; //! Create and add a Coin with DynamicMemoryUsage of 80 bytes to the given view. @@ -35,7 +39,7 @@ BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) return outp; }; - CChainState& c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(mempool)); + CChainState& c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool)); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); @@ -72,4 +76,77 @@ BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) WITH_LOCK(::cs_main, manager.Unload()); } +//! Test UpdateTip behavior for both active and background chainstates. +//! +//! When run on the background chainstate, UpdateTip should do a subset +//! of what it does for the active chainstate. +BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) +{ + ChainstateManager& chainman = *Assert(m_node.chainman); + uint256 curr_tip = ::g_best_block; + + // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can + // be found. + mineBlocks(10); + + // After adding some blocks to the tip, best block should have changed. + BOOST_CHECK(::g_best_block != curr_tip); + + BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(m_node, m_path_root)); + + // Ensure our active chain is the snapshot chainstate. + BOOST_CHECK(chainman.IsSnapshotActive()); + + curr_tip = ::g_best_block; + + // Mine a new block on top of the activated snapshot chainstate. + mineBlocks(1); // Defined in TestChain100Setup. + + // After adding some blocks to the snapshot tip, best block should have changed. + BOOST_CHECK(::g_best_block != curr_tip); + + curr_tip = ::g_best_block; + + CChainState* background_cs; + + BOOST_CHECK_EQUAL(chainman.GetAll().size(), 2); + for (CChainState* cs : chainman.GetAll()) { + if (cs != &chainman.ActiveChainstate()) { + background_cs = cs; + } + } + BOOST_CHECK(background_cs); + + // Create a block to append to the validation chain. + std::vector<CMutableTransaction> noTxns; + CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; + CBlock validation_block = this->CreateBlock(noTxns, scriptPubKey, *background_cs); + auto pblock = std::make_shared<const CBlock>(validation_block); + BlockValidationState state; + CBlockIndex* pindex = nullptr; + const CChainParams& chainparams = Params(); + bool newblock = false; + + // TODO: much of this is inlined from ProcessNewBlock(); just reuse PNB() + // once it is changed to support multiple chainstates. + { + LOCK(::cs_main); + bool checked = CheckBlock(*pblock, state, chainparams.GetConsensus()); + BOOST_CHECK(checked); + bool accepted = background_cs->AcceptBlock( + pblock, state, &pindex, true, nullptr, &newblock); + BOOST_CHECK(accepted); + } + // UpdateTip is called here + bool block_added = background_cs->ActivateBestChain(state, pblock); + + // Ensure tip is as expected + BOOST_CHECK_EQUAL(background_cs->m_chain.Tip()->GetBlockHash(), validation_block.GetHash()); + + // g_best_block should be unchanged after adding a block to the background + // validation chain. + BOOST_CHECK(block_added); + BOOST_CHECK_EQUAL(curr_tip, ::g_best_block); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 0b912acb08..be9e05a65e 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -8,13 +8,13 @@ #include <random.h> #include <rpc/blockchain.h> #include <sync.h> +#include <test/util/chainstate.h> #include <test/util/setup_common.h> #include <uint256.h> #include <validation.h> #include <validationinterface.h> #include <tinyformat.h> -#include <univalue.h> #include <vector> @@ -31,13 +31,12 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) CTxMemPool& mempool = *m_node.mempool; std::vector<CChainState*> chainstates; - const CChainParams& chainparams = Params(); BOOST_CHECK(!manager.SnapshotBlockhash().has_value()); // Create a legacy (IBD) chainstate. // - CChainState& c1 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(mempool)); + CChainState& c1 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(&mempool)); chainstates.push_back(&c1); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); @@ -45,7 +44,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) BOOST_CHECK(!manager.IsSnapshotActive()); BOOST_CHECK(!manager.IsSnapshotValidated()); - BOOST_CHECK(!manager.IsBackgroundIBD(&c1)); auto all = manager.GetAll(); BOOST_CHECK_EQUAL_COLLECTIONS(all.begin(), all.end(), chainstates.begin(), chainstates.end()); @@ -58,16 +56,13 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) auto exp_tip = c1.m_chain.Tip(); BOOST_CHECK_EQUAL(active_tip, exp_tip); - auto& validated_cs = manager.ValidatedChainstate(); - BOOST_CHECK_EQUAL(&validated_cs, &c1); - BOOST_CHECK(!manager.SnapshotBlockhash().has_value()); // Create a snapshot-based chainstate. // const uint256 snapshot_blockhash = GetRandHash(); CChainState& c2 = WITH_LOCK(::cs_main, return manager.InitializeChainstate( - mempool, snapshot_blockhash)); + &mempool, snapshot_blockhash)); chainstates.push_back(&c2); BOOST_CHECK_EQUAL(manager.SnapshotBlockhash().value(), snapshot_blockhash); @@ -76,14 +71,14 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); WITH_LOCK(::cs_main, c2.InitCoinsCache(1 << 23)); // Unlike c1, which doesn't have any blocks. Gets us different tip, height. - c2.LoadGenesisBlock(chainparams); + c2.LoadGenesisBlock(); BlockValidationState _; - BOOST_CHECK(c2.ActivateBestChain(_, chainparams, nullptr)); + BOOST_CHECK(c2.ActivateBestChain(_, nullptr)); BOOST_CHECK(manager.IsSnapshotActive()); BOOST_CHECK(!manager.IsSnapshotValidated()); - BOOST_CHECK(manager.IsBackgroundIBD(&c1)); - BOOST_CHECK(!manager.IsBackgroundIBD(&c2)); + BOOST_CHECK_EQUAL(&c2, &manager.ActiveChainstate()); + BOOST_CHECK(&c1 != &manager.ActiveChainstate()); auto all2 = manager.GetAll(); BOOST_CHECK_EQUAL_COLLECTIONS(all2.begin(), all2.end(), chainstates.begin(), chainstates.end()); @@ -100,16 +95,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // CCoinsViewCache instances. BOOST_CHECK(exp_tip != exp_tip2); - auto& validated_cs2 = manager.ValidatedChainstate(); - BOOST_CHECK_EQUAL(&validated_cs2, &c1); - - auto& validated_chain = manager.ValidatedChain(); - BOOST_CHECK_EQUAL(&validated_chain, &c1.m_chain); - - auto validated_tip = manager.ValidatedTip(); - exp_tip = c1.m_chain.Tip(); - BOOST_CHECK_EQUAL(validated_tip, exp_tip); - // Let scheduler events finish running to avoid accessing memory that is going to be unloaded SyncWithValidationInterfaceQueue(); @@ -130,7 +115,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) // Create a legacy (IBD) chainstate. // - CChainState& c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(mempool)); + CChainState& c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool)); chainstates.push_back(&c1); c1.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); @@ -138,7 +123,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) { LOCK(::cs_main); c1.InitCoinsCache(1 << 23); - BOOST_REQUIRE(c1.LoadGenesisBlock(Params())); + BOOST_REQUIRE(c1.LoadGenesisBlock()); c1.CoinsTip().SetBestBlock(InsecureRand256()); manager.MaybeRebalanceCaches(); } @@ -148,7 +133,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) // Create a snapshot-based chainstate. // - CChainState& c2 = WITH_LOCK(cs_main, return manager.InitializeChainstate(mempool, GetRandHash())); + CChainState& c2 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool, GetRandHash())); chainstates.push_back(&c2); c2.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); @@ -156,7 +141,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) { LOCK(::cs_main); c2.InitCoinsCache(1 << 23); - BOOST_REQUIRE(c2.LoadGenesisBlock(Params())); + BOOST_REQUIRE(c2.LoadGenesisBlock()); c2.CoinsTip().SetBestBlock(InsecureRand256()); manager.MaybeRebalanceCaches(); } @@ -169,36 +154,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) BOOST_CHECK_CLOSE(c2.m_coinsdb_cache_size_bytes, max_cache * 0.95, 1); } -auto NoMalleation = [](CAutoFile& file, SnapshotMetadata& meta){}; - -template<typename F = decltype(NoMalleation)> -static bool -CreateAndActivateUTXOSnapshot(NodeContext& node, const fs::path root, F malleation = NoMalleation) -{ - // Write out a snapshot to the test's tempdir. - // - int height; - WITH_LOCK(::cs_main, height = node.chainman->ActiveHeight()); - fs::path snapshot_path = root / tfm::format("test_snapshot.%d.dat", height); - FILE* outfile{fsbridge::fopen(snapshot_path, "wb")}; - CAutoFile auto_outfile{outfile, SER_DISK, CLIENT_VERSION}; - - UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), auto_outfile); - BOOST_TEST_MESSAGE( - "Wrote UTXO snapshot to " << snapshot_path.make_preferred().string() << ": " << result.write()); - - // Read the written snapshot in and then activate it. - // - FILE* infile{fsbridge::fopen(snapshot_path, "rb")}; - CAutoFile auto_infile{infile, SER_DISK, CLIENT_VERSION}; - SnapshotMetadata metadata; - auto_infile >> metadata; - - malleation(auto_infile, metadata); - - return node.chainman->ActivateSnapshot(auto_infile, metadata, /*in_memory*/ true); -} - //! Test basic snapshot activation. BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) { @@ -322,27 +277,27 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) { LOCK(::cs_main); size_t coins_in_active{0}; - size_t coins_in_ibd{0}; - size_t coins_missing_ibd{0}; + size_t coins_in_background{0}; + size_t coins_missing_from_background{0}; for (CChainState* chainstate : chainman.GetAll()) { BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); CCoinsViewCache& coinscache = chainstate->CoinsTip(); - bool is_ibd = chainman.IsBackgroundIBD(chainstate); + bool is_background = chainstate != &chainman.ActiveChainstate(); for (CTransactionRef& txn : m_coinbase_txns) { COutPoint op{txn->GetHash(), 0}; if (coinscache.HaveCoin(op)) { - (is_ibd ? coins_in_ibd : coins_in_active)++; - } else if (is_ibd) { - coins_missing_ibd++; + (is_background ? coins_in_background : coins_in_active)++; + } else if (is_background) { + coins_missing_from_background++; } } } BOOST_CHECK_EQUAL(coins_in_active, initial_total_coins + new_coins); - BOOST_CHECK_EQUAL(coins_in_ibd, initial_total_coins); - BOOST_CHECK_EQUAL(coins_missing_ibd, new_coins); + BOOST_CHECK_EQUAL(coins_in_background, initial_total_coins); + BOOST_CHECK_EQUAL(coins_missing_from_background, new_coins); } // Snapshot should refuse to load after one has already loaded. diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp index a3b344d2c9..9136c497ea 100644 --- a/src/test/validation_flush_tests.cpp +++ b/src/test/validation_flush_tests.cpp @@ -9,7 +9,7 @@ #include <boost/test/unit_test.hpp> -BOOST_FIXTURE_TEST_SUITE(validation_flush_tests, BasicTestingSetup) +BOOST_FIXTURE_TEST_SUITE(validation_flush_tests, ChainTestingSetup) //! Test utilities for detecting when we need to flush the coins cache based //! on estimated memory usage. @@ -20,10 +20,9 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) { CTxMemPool mempool; BlockManager blockman{}; - CChainState chainstate{mempool, blockman}; + CChainState chainstate{&mempool, blockman, *Assert(m_node.chainman)}; chainstate.InitCoinsDB(/*cache_size_bytes*/ 1 << 10, /*in_memory*/ true, /*should_wipe*/ false); WITH_LOCK(::cs_main, chainstate.InitCoinsCache(1 << 10)); - CTxMemPool tx_pool{}; constexpr bool is_64_bit = sizeof(void*) == 8; @@ -57,7 +56,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) // Without any coins in the cache, we shouldn't need to flush. BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), CoinsCacheSizeState::OK); // If the initial memory allocations of cacheCoins don't match these common @@ -72,7 +71,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) } BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), CoinsCacheSizeState::CRITICAL); BOOST_TEST_MESSAGE("Exiting cache flush tests early due to unsupported arch"); @@ -93,7 +92,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) print_view_mem_usage(view); BOOST_CHECK_EQUAL(view.AccessCoin(res).DynamicMemoryUsage(), COIN_SIZE); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), CoinsCacheSizeState::OK); } @@ -101,26 +100,26 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) for (int i{0}; i < 4; ++i) { add_coin(view); print_view_mem_usage(view); - if (chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0) == + if (chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0) == CoinsCacheSizeState::CRITICAL) { break; } } BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), CoinsCacheSizeState::CRITICAL); // Passing non-zero max mempool usage should allow us more headroom. BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), CoinsCacheSizeState::OK); for (int i{0}; i < 3; ++i) { add_coin(view); print_view_mem_usage(view); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), CoinsCacheSizeState::OK); } @@ -136,7 +135,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) BOOST_CHECK(usage_percentage >= 0.9); BOOST_CHECK(usage_percentage < 1); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, 1 << 10), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, 1 << 10), CoinsCacheSizeState::LARGE); } @@ -144,7 +143,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) for (int i{0}; i < 1000; ++i) { add_coin(view); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool), + chainstate.GetCoinsCacheSizeState(), CoinsCacheSizeState::OK); } @@ -152,7 +151,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) // preallocated memory that doesn't get reclaimed even after flush. BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, 0), CoinsCacheSizeState::CRITICAL); view.SetBestBlock(InsecureRand256()); @@ -160,7 +159,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) print_view_mem_usage(view); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(&tx_pool, MAX_COINS_CACHE_BYTES, 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, 0), CoinsCacheSizeState::CRITICAL); } diff --git a/src/test/validation_tests.cpp b/src/test/validation_tests.cpp index a0c2e76f00..ca52ecba2f 100644 --- a/src/test/validation_tests.cpp +++ b/src/test/validation_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> +#include <consensus/amount.h> #include <net.h> #include <signet.h> #include <uint256.h> diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index 304cd8feb0..690031cdc1 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -5,6 +5,7 @@ #include <chain.h> #include <chainparams.h> #include <consensus/params.h> +#include <deploymentstatus.h> #include <test/util/setup_common.h> #include <validation.h> #include <versionbits.h> @@ -258,8 +259,8 @@ BOOST_AUTO_TEST_CASE(versionbits_test) /** Check that ComputeBlockVersion will set the appropriate bit correctly */ static void check_computeblockversion(const Consensus::Params& params, Consensus::DeploymentPos dep) { - // This implicitly uses versionbitscache, so clear it every time - versionbitscache.Clear(); + // This implicitly uses g_versionbitscache, so clear it every time + g_versionbitscache.Clear(); int64_t bit = params.vDeployments[dep].bit; int64_t nStartTime = params.vDeployments[dep].nStartTime; @@ -267,7 +268,7 @@ static void check_computeblockversion(const Consensus::Params& params, Consensus int min_activation_height = params.vDeployments[dep].min_activation_height; // should not be any signalling for first block - BOOST_CHECK_EQUAL(ComputeBlockVersion(nullptr, params), VERSIONBITS_TOP_BITS); + BOOST_CHECK_EQUAL(g_versionbitscache.ComputeBlockVersion(nullptr, params), VERSIONBITS_TOP_BITS); // always/never active deployments shouldn't need to be tested further if (nStartTime == Consensus::BIP9Deployment::ALWAYS_ACTIVE || @@ -287,7 +288,7 @@ static void check_computeblockversion(const Consensus::Params& params, Consensus // Check min_activation_height is on a retarget boundary BOOST_REQUIRE_EQUAL(min_activation_height % params.nMinerConfirmationWindow, 0U); - const uint32_t bitmask{VersionBitsMask(params, dep)}; + const uint32_t bitmask{g_versionbitscache.Mask(params, dep)}; BOOST_CHECK_EQUAL(bitmask, uint32_t{1} << bit); // In the first chain, test that the bit is set by CBV until it has failed. @@ -306,9 +307,9 @@ static void check_computeblockversion(const Consensus::Params& params, Consensus // earlier time, so will transition from DEFINED to STARTED at the // end of the first period by mining blocks at nTime == 0 lastBlock = firstChain.Mine(params.nMinerConfirmationWindow - 1, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK_EQUAL(ComputeBlockVersion(lastBlock, params) & (1<<bit), 0); + BOOST_CHECK_EQUAL(g_versionbitscache.ComputeBlockVersion(lastBlock, params) & (1 << bit), 0); lastBlock = firstChain.Mine(params.nMinerConfirmationWindow, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK((ComputeBlockVersion(lastBlock, params) & (1<<bit)) != 0); + BOOST_CHECK((g_versionbitscache.ComputeBlockVersion(lastBlock, params) & (1 << bit)) != 0); // then we'll keep mining at nStartTime... } else { // use a time 1s earlier than start time to check we stay DEFINED @@ -316,28 +317,28 @@ static void check_computeblockversion(const Consensus::Params& params, Consensus // Start generating blocks before nStartTime lastBlock = firstChain.Mine(params.nMinerConfirmationWindow, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK_EQUAL(ComputeBlockVersion(lastBlock, params) & (1<<bit), 0); + BOOST_CHECK_EQUAL(g_versionbitscache.ComputeBlockVersion(lastBlock, params) & (1 << bit), 0); // Mine more blocks (4 less than the adjustment period) at the old time, and check that CBV isn't setting the bit yet. for (uint32_t i = 1; i < params.nMinerConfirmationWindow - 4; i++) { lastBlock = firstChain.Mine(params.nMinerConfirmationWindow + i, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK_EQUAL(ComputeBlockVersion(lastBlock, params) & (1<<bit), 0); + BOOST_CHECK_EQUAL(g_versionbitscache.ComputeBlockVersion(lastBlock, params) & (1 << bit), 0); } // Now mine 5 more blocks at the start time -- MTP should not have passed yet, so // CBV should still not yet set the bit. nTime = nStartTime; for (uint32_t i = params.nMinerConfirmationWindow - 4; i <= params.nMinerConfirmationWindow; i++) { lastBlock = firstChain.Mine(params.nMinerConfirmationWindow + i, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK_EQUAL(ComputeBlockVersion(lastBlock, params) & (1<<bit), 0); + BOOST_CHECK_EQUAL(g_versionbitscache.ComputeBlockVersion(lastBlock, params) & (1 << bit), 0); } // Next we will advance to the next period and transition to STARTED, } lastBlock = firstChain.Mine(params.nMinerConfirmationWindow * 3, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); // so ComputeBlockVersion should now set the bit, - BOOST_CHECK((ComputeBlockVersion(lastBlock, params) & (1<<bit)) != 0); + BOOST_CHECK((g_versionbitscache.ComputeBlockVersion(lastBlock, params) & (1 << bit)) != 0); // and should also be using the VERSIONBITS_TOP_BITS. - BOOST_CHECK_EQUAL(ComputeBlockVersion(lastBlock, params) & VERSIONBITS_TOP_MASK, VERSIONBITS_TOP_BITS); + BOOST_CHECK_EQUAL(g_versionbitscache.ComputeBlockVersion(lastBlock, params) & VERSIONBITS_TOP_MASK, VERSIONBITS_TOP_BITS); // Check that ComputeBlockVersion will set the bit until nTimeout nTime += 600; @@ -346,8 +347,8 @@ static void check_computeblockversion(const Consensus::Params& params, Consensus // These blocks are all before nTimeout is reached. while (nTime < nTimeout && blocksToMine > 0) { lastBlock = firstChain.Mine(nHeight+1, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK((ComputeBlockVersion(lastBlock, params) & (1<<bit)) != 0); - BOOST_CHECK_EQUAL(ComputeBlockVersion(lastBlock, params) & VERSIONBITS_TOP_MASK, VERSIONBITS_TOP_BITS); + BOOST_CHECK((g_versionbitscache.ComputeBlockVersion(lastBlock, params) & (1 << bit)) != 0); + BOOST_CHECK_EQUAL(g_versionbitscache.ComputeBlockVersion(lastBlock, params) & VERSIONBITS_TOP_MASK, VERSIONBITS_TOP_BITS); blocksToMine--; nTime += 600; nHeight += 1; @@ -361,7 +362,7 @@ static void check_computeblockversion(const Consensus::Params& params, Consensus // finish the last period before we start timing out while (nHeight % params.nMinerConfirmationWindow != 0) { lastBlock = firstChain.Mine(nHeight+1, nTime - 1, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK((ComputeBlockVersion(lastBlock, params) & (1<<bit)) != 0); + BOOST_CHECK((g_versionbitscache.ComputeBlockVersion(lastBlock, params) & (1 << bit)) != 0); nHeight += 1; } @@ -369,12 +370,12 @@ static void check_computeblockversion(const Consensus::Params& params, Consensus // the bit until the period transition. for (uint32_t i = 0; i < params.nMinerConfirmationWindow - 1; i++) { lastBlock = firstChain.Mine(nHeight+1, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK((ComputeBlockVersion(lastBlock, params) & (1<<bit)) != 0); + BOOST_CHECK((g_versionbitscache.ComputeBlockVersion(lastBlock, params) & (1 << bit)) != 0); nHeight += 1; } // The next block should trigger no longer setting the bit. lastBlock = firstChain.Mine(nHeight+1, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK_EQUAL(ComputeBlockVersion(lastBlock, params) & (1<<bit), 0); + BOOST_CHECK_EQUAL(g_versionbitscache.ComputeBlockVersion(lastBlock, params) & (1 << bit), 0); } // On a new chain: @@ -385,30 +386,30 @@ static void check_computeblockversion(const Consensus::Params& params, Consensus // Mine one period worth of blocks, and check that the bit will be on for the // next period. lastBlock = secondChain.Mine(params.nMinerConfirmationWindow, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK((ComputeBlockVersion(lastBlock, params) & (1<<bit)) != 0); + BOOST_CHECK((g_versionbitscache.ComputeBlockVersion(lastBlock, params) & (1 << bit)) != 0); // Mine another period worth of blocks, signaling the new bit. lastBlock = secondChain.Mine(params.nMinerConfirmationWindow * 2, nTime, VERSIONBITS_TOP_BITS | (1<<bit)).Tip(); // After one period of setting the bit on each block, it should have locked in. // We keep setting the bit for one more period though, until activation. - BOOST_CHECK((ComputeBlockVersion(lastBlock, params) & (1<<bit)) != 0); + BOOST_CHECK((g_versionbitscache.ComputeBlockVersion(lastBlock, params) & (1 << bit)) != 0); // Now check that we keep mining the block until the end of this period, and // then stop at the beginning of the next period. lastBlock = secondChain.Mine((params.nMinerConfirmationWindow * 3) - 1, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK((ComputeBlockVersion(lastBlock, params) & (1 << bit)) != 0); + BOOST_CHECK((g_versionbitscache.ComputeBlockVersion(lastBlock, params) & (1 << bit)) != 0); lastBlock = secondChain.Mine(params.nMinerConfirmationWindow * 3, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); if (lastBlock->nHeight + 1 < min_activation_height) { // check signalling continues while min_activation_height is not reached lastBlock = secondChain.Mine(min_activation_height - 1, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); - BOOST_CHECK((ComputeBlockVersion(lastBlock, params) & (1 << bit)) != 0); + BOOST_CHECK((g_versionbitscache.ComputeBlockVersion(lastBlock, params) & (1 << bit)) != 0); // then reach min_activation_height, which was already REQUIRE'd to start a new period lastBlock = secondChain.Mine(min_activation_height, nTime, VERSIONBITS_LAST_OLD_BLOCK_VERSION).Tip(); } // Check that we don't signal after activation - BOOST_CHECK_EQUAL(ComputeBlockVersion(lastBlock, params) & (1<<bit), 0); + BOOST_CHECK_EQUAL(g_versionbitscache.ComputeBlockVersion(lastBlock, params) & (1 << bit), 0); } BOOST_AUTO_TEST_CASE(versionbits_computeblockversion) @@ -425,7 +426,7 @@ BOOST_AUTO_TEST_CASE(versionbits_computeblockversion) // not take precedence over STARTED/LOCKED_IN. So all softforks on // the same bit might overlap, even when non-overlapping start-end // times are picked. - const uint32_t dep_mask{VersionBitsMask(chainParams->GetConsensus(), dep)}; + const uint32_t dep_mask{g_versionbitscache.Mask(chainParams->GetConsensus(), dep)}; BOOST_CHECK(!(chain_all_vbits & dep_mask)); chain_all_vbits |= dep_mask; check_computeblockversion(chainParams->GetConsensus(), dep); diff --git a/src/timedata.cpp b/src/timedata.cpp index 354092752d..69d0273e79 100644 --- a/src/timedata.cpp +++ b/src/timedata.cpp @@ -11,6 +11,7 @@ #include <netaddress.h> #include <node/ui_interface.h> #include <sync.h> +#include <tinyformat.h> #include <util/system.h> #include <util/translation.h> #include <warnings.h> @@ -74,7 +75,7 @@ void AddTimeData(const CNetAddr& ip, int64_t nOffsetSample) int64_t nMedian = vTimeOffsets.median(); std::vector<int64_t> vSorted = vTimeOffsets.sorted(); // Only let other nodes change our time by so much - int64_t max_adjustment = std::max<int64_t>(0, gArgs.GetArg("-maxtimeadjustment", DEFAULT_MAX_TIME_ADJUSTMENT)); + int64_t max_adjustment = std::max<int64_t>(0, gArgs.GetIntArg("-maxtimeadjustment", DEFAULT_MAX_TIME_ADJUSTMENT)); if (nMedian >= -max_adjustment && nMedian <= max_adjustment) { nTimeOffset = nMedian; } else { @@ -98,11 +99,12 @@ void AddTimeData(const CNetAddr& ip, int64_t nOffsetSample) } if (LogAcceptCategory(BCLog::NET)) { + std::string log_message{"time data samples: "}; for (const int64_t n : vSorted) { - LogPrint(BCLog::NET, "%+d ", n); /* Continued */ + log_message += strprintf("%+d ", n); } - LogPrint(BCLog::NET, "| "); /* Continued */ - LogPrint(BCLog::NET, "nTimeOffset = %+d (%+d minutes)\n", nTimeOffset, nTimeOffset / 60); + log_message += strprintf("| median offset = %+d (%+d minutes)", nTimeOffset, nTimeOffset / 60); + LogPrint(BCLog::NET, "%s\n", log_message); } } } diff --git a/src/tinyformat.h b/src/tinyformat.h index bc893ccda5..bedaa14007 100644 --- a/src/tinyformat.h +++ b/src/tinyformat.h @@ -797,27 +797,27 @@ inline const char* streamStateFromFormat(std::ostream& out, bool& positionalMode break; case 'X': out.setf(std::ios::uppercase); - // Falls through + [[fallthrough]]; case 'x': case 'p': out.setf(std::ios::hex, std::ios::basefield); intConversion = true; break; case 'E': out.setf(std::ios::uppercase); - // Falls through + [[fallthrough]]; case 'e': out.setf(std::ios::scientific, std::ios::floatfield); out.setf(std::ios::dec, std::ios::basefield); break; case 'F': out.setf(std::ios::uppercase); - // Falls through + [[fallthrough]]; case 'f': out.setf(std::ios::fixed, std::ios::floatfield); break; case 'A': out.setf(std::ios::uppercase); - // Falls through + [[fallthrough]]; case 'a': # ifdef _MSC_VER // Workaround https://developercommunity.visualstudio.com/content/problem/520472/hexfloat-stream-output-does-not-ignore-precision-a.html @@ -829,7 +829,7 @@ inline const char* streamStateFromFormat(std::ostream& out, bool& positionalMode break; case 'G': out.setf(std::ios::uppercase); - // Falls through + [[fallthrough]]; case 'g': out.setf(std::ios::dec, std::ios::basefield); // As in boost::format, let stream decide float format. diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index a0499fa51f..f9caf48df8 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -14,6 +14,7 @@ #include <netbase.h> #include <util/readwritefile.h> #include <util/strencodings.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <util/thread.h> #include <util/time.h> @@ -83,7 +84,7 @@ void TorControlConnection::readcb(struct bufferevent *bev, void *ctx) if (s.size() < 4) // Short line continue; // <status>(-|+| )<data><CRLF> - self->message.code = atoi(s.substr(0,3)); + self->message.code = LocaleIndependentAtoi<int>(s.substr(0,3)); self->message.lines.push_back(s.substr(4)); char ch = s[3]; // '-','+' or ' ' if (ch == ' ') { @@ -132,28 +133,35 @@ void TorControlConnection::eventcb(struct bufferevent *bev, short what, void *ct bool TorControlConnection::Connect(const std::string& tor_control_center, const ConnectionCB& _connected, const ConnectionCB& _disconnected) { - if (b_conn) + if (b_conn) { Disconnect(); - // Parse tor_control_center address:port - struct sockaddr_storage connect_to_addr; - int connect_to_addrlen = sizeof(connect_to_addr); - if (evutil_parse_sockaddr_port(tor_control_center.c_str(), - (struct sockaddr*)&connect_to_addr, &connect_to_addrlen)<0) { + } + + CService control_service; + if (!Lookup(tor_control_center, control_service, 9051, fNameLookup)) { + LogPrintf("tor: Failed to look up control center %s\n", tor_control_center); + return false; + } + + struct sockaddr_storage control_address; + socklen_t control_address_len = sizeof(control_address); + if (!control_service.GetSockAddr(reinterpret_cast<struct sockaddr*>(&control_address), &control_address_len)) { LogPrintf("tor: Error parsing socket address %s\n", tor_control_center); return false; } // Create a new socket, set up callbacks and enable notification bits b_conn = bufferevent_socket_new(base, -1, BEV_OPT_CLOSE_ON_FREE); - if (!b_conn) + if (!b_conn) { return false; + } bufferevent_setcb(b_conn, TorControlConnection::readcb, nullptr, TorControlConnection::eventcb, this); bufferevent_enable(b_conn, EV_READ|EV_WRITE); this->connected = _connected; this->disconnected = _disconnected; // Finally, connect to tor_control_center - if (bufferevent_socket_connect(b_conn, (struct sockaddr*)&connect_to_addr, connect_to_addrlen) < 0) { + if (bufferevent_socket_connect(b_conn, reinterpret_cast<struct sockaddr*>(&control_address), control_address_len) < 0) { LogPrintf("tor: Error connecting to address %s\n", tor_control_center); return false; } @@ -578,6 +586,7 @@ static std::thread torControlThread; static void TorControlThread(CService onion_service_target) { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::TOR_CONTROL); TorController ctrl(gBase, gArgs.GetArg("-torcontrol", DEFAULT_TOR_CONTROL), onion_service_target); event_base_dispatch(gBase); diff --git a/src/txdb.cpp b/src/txdb.cpp index 4b76bee5ab..3839c9083c 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -5,6 +5,7 @@ #include <txdb.h> +#include <chain.h> #include <node/ui_interface.h> #include <pow.h> #include <random.h> @@ -27,6 +28,28 @@ static constexpr uint8_t DB_FLAG{'F'}; static constexpr uint8_t DB_REINDEX_FLAG{'R'}; static constexpr uint8_t DB_LAST_BLOCK{'l'}; +// Keys used in previous version that might still be found in the DB: +static constexpr uint8_t DB_TXINDEX_BLOCK{'T'}; +// uint8_t DB_TXINDEX{'t'} + +std::optional<bilingual_str> CheckLegacyTxindex(CBlockTreeDB& block_tree_db) +{ + CBlockLocator ignored{}; + if (block_tree_db.Read(DB_TXINDEX_BLOCK, ignored)) { + return _("The -txindex upgrade started by a previous version can not be completed. Restart with the previous version or run a full -reindex."); + } + bool txindex_legacy_flag{false}; + block_tree_db.ReadFlag("txindex", txindex_legacy_flag); + if (txindex_legacy_flag) { + // Disable legacy txindex and warn once about occupied disk space + if (!block_tree_db.WriteFlag("txindex", false)) { + return Untranslated("Failed to write block index db flag 'txindex'='0'"); + } + return _("The block index db contains a legacy 'txindex'. To clear the occupied disk space, run a full -reindex, otherwise ignore this error. This error message will not be displayed again."); + } + return std::nullopt; +} + namespace { struct CoinEntry { @@ -84,8 +107,8 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { CDBBatch batch(*m_db); size_t count = 0; size_t changed = 0; - size_t batch_size = (size_t)gArgs.GetArg("-dbbatchsize", nDefaultDbBatchSize); - int crash_simulate = gArgs.GetArg("-dbcrashratio", 0); + size_t batch_size = (size_t)gArgs.GetIntArg("-dbbatchsize", nDefaultDbBatchSize); + int crash_simulate = gArgs.GetIntArg("-dbcrashratio", 0); assert(!hashBlock.IsNull()); uint256 old_tip = GetBestBlock(); diff --git a/src/txdb.h b/src/txdb.h index 845d80788f..1bdce71126 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -8,17 +8,20 @@ #include <coins.h> #include <dbwrapper.h> -#include <chain.h> -#include <primitives/block.h> #include <memory> +#include <optional> #include <string> #include <utility> #include <vector> +class CBlockFileInfo; class CBlockIndex; -class CCoinsViewDBCursor; class uint256; +namespace Consensus { +struct Params; +}; +struct bilingual_str; //! -dbcache default (MiB) static const int64_t nDefaultDbCache = 450; @@ -86,4 +89,6 @@ public: bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex); }; +std::optional<bilingual_str> CheckLegacyTxindex(CBlockTreeDB& block_tree_db); + #endif // BITCOIN_TXDB_H diff --git a/src/txmempool.cpp b/src/txmempool.cpp index c5a4bbf1b0..5a93f30c8a 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -21,23 +21,23 @@ #include <cmath> #include <optional> -CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, - int64_t _nTime, unsigned int _entryHeight, - bool _spendsCoinbase, int64_t _sigOpsCost, LockPoints lp) - : tx(_tx), nFee(_nFee), nTxWeight(GetTransactionWeight(*tx)), nUsageSize(RecursiveDynamicUsage(tx)), nTime(_nTime), entryHeight(_entryHeight), - spendsCoinbase(_spendsCoinbase), sigOpCost(_sigOpsCost), lockPoints(lp) -{ - nCountWithDescendants = 1; - nSizeWithDescendants = GetTxSize(); - nModFeesWithDescendants = nFee; - - feeDelta = 0; - - nCountWithAncestors = 1; - nSizeWithAncestors = GetTxSize(); - nModFeesWithAncestors = nFee; - nSigOpCostWithAncestors = sigOpCost; -} +CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& tx, CAmount fee, + int64_t time, unsigned int entry_height, + bool spends_coinbase, int64_t sigops_cost, LockPoints lp) + : tx{tx}, + nFee{fee}, + nTxWeight(GetTransactionWeight(*tx)), + nUsageSize{RecursiveDynamicUsage(tx)}, + nTime{time}, + entryHeight{entry_height}, + spendsCoinbase{spends_coinbase}, + sigOpCost{sigops_cost}, + lockPoints{lp}, + nSizeWithDescendants{GetTxSize()}, + nModFeesWithDescendants{nFee}, + nSizeWithAncestors{GetTxSize()}, + nModFeesWithAncestors{nFee}, + nSigOpCostWithAncestors{sigOpCost} {} void CTxMemPoolEntry::UpdateFeeDelta(int64_t newFeeDelta) { @@ -151,33 +151,17 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector<uint256> &vHashes } } -bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, setEntries &setAncestors, uint64_t limitAncestorCount, uint64_t limitAncestorSize, uint64_t limitDescendantCount, uint64_t limitDescendantSize, std::string &errString, bool fSearchForParents /* = true */) const +bool CTxMemPool::CalculateAncestorsAndCheckLimits(size_t entry_size, + size_t entry_count, + setEntries& setAncestors, + CTxMemPoolEntry::Parents& staged_ancestors, + uint64_t limitAncestorCount, + uint64_t limitAncestorSize, + uint64_t limitDescendantCount, + uint64_t limitDescendantSize, + std::string &errString) const { - CTxMemPoolEntry::Parents staged_ancestors; - const CTransaction &tx = entry.GetTx(); - - if (fSearchForParents) { - // Get parents of this transaction that are in the mempool - // GetMemPoolParents() is only valid for entries in the mempool, so we - // iterate mapTx to find parents. - for (unsigned int i = 0; i < tx.vin.size(); i++) { - std::optional<txiter> piter = GetIter(tx.vin[i].prevout.hash); - if (piter) { - staged_ancestors.insert(**piter); - if (staged_ancestors.size() + 1 > limitAncestorCount) { - errString = strprintf("too many unconfirmed parents [limit: %u]", limitAncestorCount); - return false; - } - } - } - } else { - // If we're not searching for parents, we require this to be an - // entry in the mempool already. - txiter it = mapTx.iterator_to(entry); - staged_ancestors = it->GetMemPoolParentsConst(); - } - - size_t totalSizeWithAncestors = entry.GetTxSize(); + size_t totalSizeWithAncestors = entry_size; while (!staged_ancestors.empty()) { const CTxMemPoolEntry& stage = staged_ancestors.begin()->get(); @@ -187,10 +171,10 @@ bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, setEntr staged_ancestors.erase(stage); totalSizeWithAncestors += stageit->GetTxSize(); - if (stageit->GetSizeWithDescendants() + entry.GetTxSize() > limitDescendantSize) { + if (stageit->GetSizeWithDescendants() + entry_size > limitDescendantSize) { errString = strprintf("exceeds descendant size limit for tx %s [limit: %u]", stageit->GetTx().GetHash().ToString(), limitDescendantSize); return false; - } else if (stageit->GetCountWithDescendants() + 1 > limitDescendantCount) { + } else if (stageit->GetCountWithDescendants() + entry_count > limitDescendantCount) { errString = strprintf("too many descendants for tx %s [limit: %u]", stageit->GetTx().GetHash().ToString(), limitDescendantCount); return false; } else if (totalSizeWithAncestors > limitAncestorSize) { @@ -206,7 +190,7 @@ bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, setEntr if (setAncestors.count(parent_it) == 0) { staged_ancestors.insert(parent); } - if (staged_ancestors.size() + setAncestors.size() + 1 > limitAncestorCount) { + if (staged_ancestors.size() + setAncestors.size() + entry_count > limitAncestorCount) { errString = strprintf("too many unconfirmed ancestors [limit: %u]", limitAncestorCount); return false; } @@ -216,6 +200,80 @@ bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, setEntr return true; } +bool CTxMemPool::CheckPackageLimits(const Package& package, + uint64_t limitAncestorCount, + uint64_t limitAncestorSize, + uint64_t limitDescendantCount, + uint64_t limitDescendantSize, + std::string &errString) const +{ + CTxMemPoolEntry::Parents staged_ancestors; + size_t total_size = 0; + for (const auto& tx : package) { + total_size += GetVirtualTransactionSize(*tx); + for (const auto& input : tx->vin) { + std::optional<txiter> piter = GetIter(input.prevout.hash); + if (piter) { + staged_ancestors.insert(**piter); + if (staged_ancestors.size() + package.size() > limitAncestorCount) { + errString = strprintf("too many unconfirmed parents [limit: %u]", limitAncestorCount); + return false; + } + } + } + } + // When multiple transactions are passed in, the ancestors and descendants of all transactions + // considered together must be within limits even if they are not interdependent. This may be + // stricter than the limits for each individual transaction. + setEntries setAncestors; + const auto ret = CalculateAncestorsAndCheckLimits(total_size, package.size(), + setAncestors, staged_ancestors, + limitAncestorCount, limitAncestorSize, + limitDescendantCount, limitDescendantSize, errString); + // It's possible to overestimate the ancestor/descendant totals. + if (!ret) errString.insert(0, "possibly "); + return ret; +} + +bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, + setEntries &setAncestors, + uint64_t limitAncestorCount, + uint64_t limitAncestorSize, + uint64_t limitDescendantCount, + uint64_t limitDescendantSize, + std::string &errString, + bool fSearchForParents /* = true */) const +{ + CTxMemPoolEntry::Parents staged_ancestors; + const CTransaction &tx = entry.GetTx(); + + if (fSearchForParents) { + // Get parents of this transaction that are in the mempool + // GetMemPoolParents() is only valid for entries in the mempool, so we + // iterate mapTx to find parents. + for (unsigned int i = 0; i < tx.vin.size(); i++) { + std::optional<txiter> piter = GetIter(tx.vin[i].prevout.hash); + if (piter) { + staged_ancestors.insert(**piter); + if (staged_ancestors.size() + 1 > limitAncestorCount) { + errString = strprintf("too many unconfirmed parents [limit: %u]", limitAncestorCount); + return false; + } + } + } + } else { + // If we're not searching for parents, we require this to already be an + // entry in the mempool and use the entry's cached parents. + txiter it = mapTx.iterator_to(entry); + staged_ancestors = it->GetMemPoolParentsConst(); + } + + return CalculateAncestorsAndCheckLimits(entry.GetTxSize(), /* entry_count */ 1, + setAncestors, staged_ancestors, + limitAncestorCount, limitAncestorSize, + limitDescendantCount, limitDescendantSize, errString); +} + void CTxMemPool::UpdateAncestorsOf(bool add, txiter it, setEntries &setAncestors) { CTxMemPoolEntry::Parents parents = it->GetMemPoolParents(); @@ -866,7 +924,7 @@ void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeD ++nTransactionsUpdated; } } - LogPrintf("PrioritiseTransaction: %s feerate += %s\n", hash.ToString(), FormatMoney(nFeeDelta)); + LogPrintf("PrioritiseTransaction: %s fee += %s\n", hash.ToString(), FormatMoney(nFeeDelta)); } void CTxMemPool::ApplyDelta(const uint256& hash, CAmount &nFeeDelta) const @@ -1116,12 +1174,14 @@ uint64_t CTxMemPool::CalculateDescendantMaximum(txiter entry) const { return maximum; } -void CTxMemPool::GetTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants) const { +void CTxMemPool::GetTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants, size_t* const ancestorsize, CAmount* const ancestorfees) const { LOCK(cs); auto it = mapTx.find(txid); ancestors = descendants = 0; if (it != mapTx.end()) { ancestors = it->GetCountWithAncestors(); + if (ancestorsize) *ancestorsize = it->GetSizeWithAncestors(); + if (ancestorfees) *ancestorfees = it->GetModFeesWithAncestors(); descendants = CalculateDescendantMaximum(it); } } diff --git a/src/txmempool.h b/src/txmempool.h index ae4b16d377..0be51db181 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -14,10 +14,11 @@ #include <utility> #include <vector> -#include <amount.h> #include <coins.h> +#include <consensus/amount.h> #include <indirectmap.h> #include <policy/feerate.h> +#include <policy/packages.h> #include <primitives/transaction.h> #include <random.h> #include <sync.h> @@ -36,19 +37,16 @@ extern RecursiveMutex cs_main; /** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */ static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; -struct LockPoints -{ +struct LockPoints { // Will be set to the blockchain height and median time past // values that would be necessary to satisfy all relative locktime // constraints (BIP68) of this tx given our view of block chain history - int height; - int64_t time; + int height{0}; + int64_t time{0}; // As long as the current chain descends from the highest height block // containing one of the inputs used in the calculation, then the cached // values are still valid even after a reorg. - CBlockIndex* maxInputBlock; - - LockPoints() : height(0), time(0), maxInputBlock(nullptr) { } + CBlockIndex* maxInputBlock{nullptr}; }; struct CompareIteratorByHash { @@ -97,27 +95,27 @@ private: const unsigned int entryHeight; //!< Chain height when entering the mempool const bool spendsCoinbase; //!< keep track of transactions that spend a coinbase const int64_t sigOpCost; //!< Total sigop cost - int64_t feeDelta; //!< Used for determining the priority of the transaction for mining in a block + int64_t feeDelta{0}; //!< Used for determining the priority of the transaction for mining in a block LockPoints lockPoints; //!< Track the height and time at which tx was final // Information about descendants of this transaction that are in the // mempool; if we remove this transaction we must remove all of these // descendants as well. - uint64_t nCountWithDescendants; //!< number of descendant transactions + uint64_t nCountWithDescendants{1}; //!< number of descendant transactions uint64_t nSizeWithDescendants; //!< ... and size CAmount nModFeesWithDescendants; //!< ... and total fees (all including us) // Analogous statistics for ancestor transactions - uint64_t nCountWithAncestors; + uint64_t nCountWithAncestors{1}; uint64_t nSizeWithAncestors; CAmount nModFeesWithAncestors; int64_t nSigOpCostWithAncestors; public: - CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFee, - int64_t _nTime, unsigned int _entryHeight, - bool spendsCoinbase, - int64_t nSigOpsCost, LockPoints lp); + CTxMemPoolEntry(const CTransactionRef& tx, CAmount fee, + int64_t time, unsigned int entry_height, + bool spends_coinbase, + int64_t sigops_cost, LockPoints lp); const CTransaction& GetTx() const { return *this->tx; } CTransactionRef GetSharedTx() const { return this->tx; } @@ -585,6 +583,25 @@ private: */ std::set<uint256> m_unbroadcast_txids GUARDED_BY(cs); + + /** + * Helper function to calculate all in-mempool ancestors of staged_ancestors and apply ancestor + * and descendant limits (including staged_ancestors thsemselves, entry_size and entry_count). + * param@[in] entry_size Virtual size to include in the limits. + * param@[in] entry_count How many entries to include in the limits. + * param@[in] staged_ancestors Should contain entries in the mempool. + * param@[out] setAncestors Will be populated with all mempool ancestors. + */ + bool CalculateAncestorsAndCheckLimits(size_t entry_size, + size_t entry_count, + setEntries& setAncestors, + CTxMemPoolEntry::Parents &staged_ancestors, + uint64_t limitAncestorCount, + uint64_t limitAncestorSize, + uint64_t limitDescendantCount, + uint64_t limitDescendantSize, + std::string &errString) const EXCLUSIVE_LOCKS_REQUIRED(cs); + public: indirectmap<COutPoint, const CTransaction*> mapNextTx GUARDED_BY(cs); std::map<uint256, CAmount> mapDeltas GUARDED_BY(cs); @@ -681,6 +698,28 @@ public: */ bool CalculateMemPoolAncestors(const CTxMemPoolEntry& entry, setEntries& setAncestors, uint64_t limitAncestorCount, uint64_t limitAncestorSize, uint64_t limitDescendantCount, uint64_t limitDescendantSize, std::string& errString, bool fSearchForParents = true) const EXCLUSIVE_LOCKS_REQUIRED(cs); + /** Calculate all in-mempool ancestors of a set of transactions not already in the mempool and + * check ancestor and descendant limits. Heuristics are used to estimate the ancestor and + * descendant count of all entries if the package were to be added to the mempool. The limits + * are applied to the union of all package transactions. For example, if the package has 3 + * transactions and limitAncestorCount = 25, the union of all 3 sets of ancestors (including the + * transactions themselves) must be <= 22. + * @param[in] package Transaction package being evaluated for acceptance + * to mempool. The transactions need not be direct + * ancestors/descendants of each other. + * @param[in] limitAncestorCount Max number of txns including ancestors. + * @param[in] limitAncestorSize Max virtual size including ancestors. + * @param[in] limitDescendantCount Max number of txns including descendants. + * @param[in] limitDescendantSize Max virtual size including descendants. + * @param[out] errString Populated with error reason if a limit is hit. + */ + bool CheckPackageLimits(const Package& package, + uint64_t limitAncestorCount, + uint64_t limitAncestorSize, + uint64_t limitDescendantCount, + uint64_t limitDescendantSize, + std::string &errString) const EXCLUSIVE_LOCKS_REQUIRED(cs); + /** Populate setDescendants with all in-mempool descendants of hash. * Assumes that setDescendants includes all in-mempool descendants of anything * already in it. */ @@ -706,8 +745,10 @@ public: /** * Calculate the ancestor and descendant count for the given transaction. * The counts include the transaction itself. + * When ancestors is non-zero (ie, the transaction itself is in the mempool), + * ancestorsize and ancestorfees will also be set to the appropriate values. */ - void GetTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants) const; + void GetTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants, size_t* ancestorsize = nullptr, CAmount* ancestorfees = nullptr) const; /** @returns true if the mempool is fully loaded */ bool IsLoaded() const; diff --git a/src/txorphanage.h b/src/txorphanage.h index e4266e470a..24c8318f36 100644 --- a/src/txorphanage.h +++ b/src/txorphanage.h @@ -47,6 +47,13 @@ public: * (ie orphans that may have found their final missing parent, and so should be reconsidered for the mempool) */ void AddChildrenToWorkSet(const CTransaction& tx, std::set<uint256>& orphan_work_set) const EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans); + /** Return how many entries exist in the orphange */ + size_t Size() LOCKS_EXCLUDED(::g_cs_orphans) + { + LOCK(::g_cs_orphans); + return m_orphans.size(); + } + protected: struct OrphanTx { CTransactionRef tx; diff --git a/src/util/asmap.cpp b/src/util/asmap.cpp index bacc3690a2..5695c62012 100644 --- a/src/util/asmap.cpp +++ b/src/util/asmap.cpp @@ -2,10 +2,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <util/asmap.h> + +#include <clientversion.h> +#include <crypto/common.h> +#include <logging.h> +#include <streams.h> + +#include <cassert> #include <map> #include <vector> -#include <assert.h> -#include <crypto/common.h> namespace { @@ -183,3 +189,31 @@ bool SanityCheckASMap(const std::vector<bool>& asmap, int bits) } return false; // Reached EOF without RETURN instruction } + +std::vector<bool> DecodeAsmap(fs::path path) +{ + std::vector<bool> bits; + FILE *filestr = fsbridge::fopen(path, "rb"); + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + LogPrintf("Failed to open asmap file from disk\n"); + return bits; + } + fseek(filestr, 0, SEEK_END); + int length = ftell(filestr); + LogPrintf("Opened asmap file %s (%d bytes) from disk\n", path, length); + fseek(filestr, 0, SEEK_SET); + uint8_t cur_byte; + for (int i = 0; i < length; ++i) { + file >> cur_byte; + for (int bit = 0; bit < 8; ++bit) { + bits.push_back((cur_byte >> bit) & 1); + } + } + if (!SanityCheckASMap(bits, 128)) { + LogPrintf("Sanity check of asmap file %s failed\n", path); + return {}; + } + return bits; +} + diff --git a/src/util/asmap.h b/src/util/asmap.h index d0588bc8c3..810d70b9a1 100644 --- a/src/util/asmap.h +++ b/src/util/asmap.h @@ -5,11 +5,16 @@ #ifndef BITCOIN_UTIL_ASMAP_H #define BITCOIN_UTIL_ASMAP_H -#include <stdint.h> +#include <fs.h> + +#include <cstdint> #include <vector> uint32_t Interpret(const std::vector<bool> &asmap, const std::vector<bool> &ip); bool SanityCheckASMap(const std::vector<bool>& asmap, int bits); +/** Read asmap from provided binary file */ +std::vector<bool> DecodeAsmap(fs::path path); + #endif // BITCOIN_UTIL_ASMAP_H diff --git a/src/util/epochguard.h b/src/util/epochguard.h index 1570ec4eb4..3e63e093da 100644 --- a/src/util/epochguard.h +++ b/src/util/epochguard.h @@ -40,6 +40,9 @@ public: Epoch() = default; Epoch(const Epoch&) = delete; Epoch& operator=(const Epoch&) = delete; + Epoch(Epoch&&) = delete; + Epoch& operator=(Epoch&&) = delete; + ~Epoch() = default; bool guarded() const { return m_guarded; } @@ -51,6 +54,13 @@ public: // only allow modification via Epoch member functions friend class Epoch; Marker& operator=(const Marker&) = delete; + + public: + Marker() = default; + Marker(const Marker&) = default; + Marker(Marker&&) = delete; + Marker& operator=(Marker&&) = delete; + ~Marker() = default; }; class SCOPED_LOCKABLE Guard diff --git a/src/util/getuniquepath.cpp b/src/util/getuniquepath.cpp index 9839d2f624..6776e7785b 100644 --- a/src/util/getuniquepath.cpp +++ b/src/util/getuniquepath.cpp @@ -1,3 +1,7 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + #include <random.h> #include <fs.h> #include <util/strencodings.h> diff --git a/src/util/hasher.h b/src/util/hasher.h index fa2fea30d8..9b79a1b5f1 100644 --- a/src/util/hasher.h +++ b/src/util/hasher.h @@ -33,10 +33,6 @@ public: SaltedOutpointHasher(); /** - * This *must* return size_t. With Boost 1.46 on 32-bit systems the - * unordered_map will behave unpredictably if the custom hasher returns a - * uint64_t, resulting in failures when syncing the chain (#4634). - * * Having the hash noexcept allows libstdc++'s unordered_map to recalculate * the hash during rehash, so it does not have to cache the value. This * reduces node's memory by sizeof(size_t). The required recalculation has diff --git a/src/util/moneystr.cpp b/src/util/moneystr.cpp index 3f9ce7dce4..1aed7daacf 100644 --- a/src/util/moneystr.cpp +++ b/src/util/moneystr.cpp @@ -5,10 +5,13 @@ #include <util/moneystr.h> +#include <consensus/amount.h> #include <tinyformat.h> #include <util/strencodings.h> #include <util/string.h> +#include <optional> + std::string FormatMoney(const CAmount n) { // Note: not using straight sprintf here because we do NOT want @@ -35,14 +38,14 @@ std::string FormatMoney(const CAmount n) } -bool ParseMoney(const std::string& money_string, CAmount& nRet) +std::optional<CAmount> ParseMoney(const std::string& money_string) { if (!ValidAsCString(money_string)) { - return false; + return std::nullopt; } const std::string str = TrimString(money_string); if (str.empty()) { - return false; + return std::nullopt; } std::string strWhole; @@ -62,21 +65,24 @@ bool ParseMoney(const std::string& money_string, CAmount& nRet) break; } if (IsSpace(*p)) - return false; + return std::nullopt; if (!IsDigit(*p)) - return false; + return std::nullopt; strWhole.insert(strWhole.end(), *p); } if (*p) { - return false; + return std::nullopt; } if (strWhole.size() > 10) // guard against 63 bit overflow - return false; + return std::nullopt; if (nUnits < 0 || nUnits > COIN) - return false; - int64_t nWhole = atoi64(strWhole); - CAmount nValue = nWhole*COIN + nUnits; + return std::nullopt; + int64_t nWhole = LocaleIndependentAtoi<int64_t>(strWhole); + CAmount value = nWhole * COIN + nUnits; + + if (!MoneyRange(value)) { + return std::nullopt; + } - nRet = nValue; - return true; + return value; } diff --git a/src/util/moneystr.h b/src/util/moneystr.h index 2aedbee358..f37dc1cffd 100644 --- a/src/util/moneystr.h +++ b/src/util/moneystr.h @@ -9,9 +9,10 @@ #ifndef BITCOIN_UTIL_MONEYSTR_H #define BITCOIN_UTIL_MONEYSTR_H -#include <amount.h> #include <attributes.h> +#include <consensus/amount.h> +#include <optional> #include <string> /* Do not use these functions to represent or parse monetary amounts to or from @@ -19,6 +20,6 @@ */ std::string FormatMoney(const CAmount n); /** Parse an amount denoted in full coins. E.g. "0.0034" supplied on the command line. **/ -[[nodiscard]] bool ParseMoney(const std::string& str, CAmount& nRet); +std::optional<CAmount> ParseMoney(const std::string& str); #endif // BITCOIN_UTIL_MONEYSTR_H diff --git a/src/util/rbf.h b/src/util/rbf.h index 6a20b37de5..aa522d8bfb 100644 --- a/src/util/rbf.h +++ b/src/util/rbf.h @@ -9,10 +9,15 @@ class CTransaction; -static const uint32_t MAX_BIP125_RBF_SEQUENCE = 0xfffffffd; +static constexpr uint32_t MAX_BIP125_RBF_SEQUENCE{0xfffffffd}; -// Check whether the sequence numbers on this transaction are signaling -// opt-in to replace-by-fee, according to BIP 125 -bool SignalsOptInRBF(const CTransaction &tx); +/** Check whether the sequence numbers on this transaction are signaling opt-in to replace-by-fee, + * according to BIP 125. Allow opt-out of transaction replacement by setting nSequence > + * MAX_BIP125_RBF_SEQUENCE (SEQUENCE_FINAL-2) on all inputs. +* +* SEQUENCE_FINAL-1 is picked to still allow use of nLockTime by non-replaceable transactions. All +* inputs rather than just one is for the sake of multi-party protocols, where we don't want a single +* party to be able to disable replacement by opting out in their own input. */ +bool SignalsOptInRBF(const CTransaction& tx); #endif // BITCOIN_UTIL_RBF_H diff --git a/src/util/settings.cpp b/src/util/settings.cpp index b92b1d30c3..846b34089d 100644 --- a/src/util/settings.cpp +++ b/src/util/settings.cpp @@ -60,9 +60,15 @@ bool ReadSettings(const fs::path& path, std::map<std::string, SettingsValue>& va values.clear(); errors.clear(); + // Ok for file to not exist + if (!fs::exists(path)) return true; + fsbridge::ifstream file; file.open(path); - if (!file.is_open()) return true; // Ok for file not to exist. + if (!file.is_open()) { + errors.emplace_back(strprintf("%s. Please check permissions.", path.string())); + return false; + } SettingsValue in; if (!in.read(std::string{std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>()})) { diff --git a/src/util/sock.cpp b/src/util/sock.cpp index b6c2a47434..1a4d67a65e 100644 --- a/src/util/sock.cpp +++ b/src/util/sock.cpp @@ -10,12 +10,14 @@ #include <util/system.h> #include <util/time.h> -#include <codecvt> -#include <cwchar> -#include <locale> #include <stdexcept> #include <string> +#ifdef WIN32 +#include <codecvt> +#include <locale> +#endif + #ifdef USE_POLL #include <poll.h> #endif diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index f514613f0d..15bd07b374 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -11,8 +11,7 @@ #include <algorithm> #include <cstdlib> #include <cstring> -#include <errno.h> -#include <limits> +#include <optional> static const std::string CHARS_ALPHA_NUM = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; @@ -282,118 +281,55 @@ std::string DecodeBase32(const std::string& str, bool* pf_invalid) return std::string((const char*)vchRet.data(), vchRet.size()); } -[[nodiscard]] static bool ParsePrechecks(const std::string& str) +namespace { +template <typename T> +bool ParseIntegral(const std::string& str, T* out) { - if (str.empty()) // No empty string allowed + static_assert(std::is_integral<T>::value); + // Replicate the exact behavior of strtol/strtoll/strtoul/strtoull when + // handling leading +/- for backwards compatibility. + if (str.length() >= 2 && str[0] == '+' && str[1] == '-') { return false; - if (str.size() >= 1 && (IsSpace(str[0]) || IsSpace(str[str.size()-1]))) // No padding allowed - return false; - if (!ValidAsCString(str)) // No embedded NUL characters allowed + } + const std::optional<T> opt_int = ToIntegral<T>((!str.empty() && str[0] == '+') ? str.substr(1) : str); + if (!opt_int) { return false; + } + if (out != nullptr) { + *out = *opt_int; + } return true; } +}; // namespace -bool ParseInt32(const std::string& str, int32_t *out) +bool ParseInt32(const std::string& str, int32_t* out) { - if (!ParsePrechecks(str)) - return false; - char *endp = nullptr; - errno = 0; // strtol will not set errno if valid - long int n = strtol(str.c_str(), &endp, 10); - if(out) *out = (int32_t)n; - // Note that strtol returns a *long int*, so even if strtol doesn't report an over/underflow - // we still have to check that the returned value is within the range of an *int32_t*. On 64-bit - // platforms the size of these types may be different. - return endp && *endp == 0 && !errno && - n >= std::numeric_limits<int32_t>::min() && - n <= std::numeric_limits<int32_t>::max(); + return ParseIntegral<int32_t>(str, out); } -bool ParseInt64(const std::string& str, int64_t *out) +bool ParseInt64(const std::string& str, int64_t* out) { - if (!ParsePrechecks(str)) - return false; - char *endp = nullptr; - errno = 0; // strtoll will not set errno if valid - long long int n = strtoll(str.c_str(), &endp, 10); - if(out) *out = (int64_t)n; - // Note that strtoll returns a *long long int*, so even if strtol doesn't report an over/underflow - // we still have to check that the returned value is within the range of an *int64_t*. - return endp && *endp == 0 && !errno && - n >= std::numeric_limits<int64_t>::min() && - n <= std::numeric_limits<int64_t>::max(); + return ParseIntegral<int64_t>(str, out); } -bool ParseUInt8(const std::string& str, uint8_t *out) +bool ParseUInt8(const std::string& str, uint8_t* out) { - uint32_t u32; - if (!ParseUInt32(str, &u32) || u32 > std::numeric_limits<uint8_t>::max()) { - return false; - } - if (out != nullptr) { - *out = static_cast<uint8_t>(u32); - } - return true; + return ParseIntegral<uint8_t>(str, out); } bool ParseUInt16(const std::string& str, uint16_t* out) { - uint32_t u32; - if (!ParseUInt32(str, &u32) || u32 > std::numeric_limits<uint16_t>::max()) { - return false; - } - if (out != nullptr) { - *out = static_cast<uint16_t>(u32); - } - return true; + return ParseIntegral<uint16_t>(str, out); } -bool ParseUInt32(const std::string& str, uint32_t *out) +bool ParseUInt32(const std::string& str, uint32_t* out) { - if (!ParsePrechecks(str)) - return false; - if (str.size() >= 1 && str[0] == '-') // Reject negative values, unfortunately strtoul accepts these by default if they fit in the range - return false; - char *endp = nullptr; - errno = 0; // strtoul will not set errno if valid - unsigned long int n = strtoul(str.c_str(), &endp, 10); - if(out) *out = (uint32_t)n; - // Note that strtoul returns a *unsigned long int*, so even if it doesn't report an over/underflow - // we still have to check that the returned value is within the range of an *uint32_t*. On 64-bit - // platforms the size of these types may be different. - return endp && *endp == 0 && !errno && - n <= std::numeric_limits<uint32_t>::max(); -} - -bool ParseUInt64(const std::string& str, uint64_t *out) -{ - if (!ParsePrechecks(str)) - return false; - if (str.size() >= 1 && str[0] == '-') // Reject negative values, unfortunately strtoull accepts these by default if they fit in the range - return false; - char *endp = nullptr; - errno = 0; // strtoull will not set errno if valid - unsigned long long int n = strtoull(str.c_str(), &endp, 10); - if(out) *out = (uint64_t)n; - // Note that strtoull returns a *unsigned long long int*, so even if it doesn't report an over/underflow - // we still have to check that the returned value is within the range of an *uint64_t*. - return endp && *endp == 0 && !errno && - n <= std::numeric_limits<uint64_t>::max(); + return ParseIntegral<uint32_t>(str, out); } - -bool ParseDouble(const std::string& str, double *out) +bool ParseUInt64(const std::string& str, uint64_t* out) { - if (!ParsePrechecks(str)) - return false; - if (str.size() >= 2 && str[0] == '0' && str[1] == 'x') // No hexadecimal floats allowed - return false; - std::istringstream text(str); - text.imbue(std::locale::classic()); - double result; - text >> result; - if(out) *out = result; - return text.eof() && !text.fail(); + return ParseIntegral<uint64_t>(str, out); } std::string FormatParagraph(const std::string& in, size_t width, size_t indent) @@ -437,20 +373,6 @@ std::string FormatParagraph(const std::string& in, size_t width, size_t indent) return out.str(); } -int64_t atoi64(const std::string& str) -{ -#ifdef _MSC_VER - return _atoi64(str.c_str()); -#else - return strtoll(str.c_str(), nullptr, 10); -#endif -} - -int atoi(const std::string& str) -{ - return atoi(str.c_str()); -} - /** Upper bound for mantissa. * 10^18-1 is the largest arbitrary decimal that will fit in a signed 64-bit integer. * Larger integers cannot consist of arbitrary combinations of 0-9: diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 26dc0a0ce3..1f7762aeef 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -11,9 +11,12 @@ #include <attributes.h> #include <span.h> +#include <util/string.h> +#include <charconv> #include <cstdint> #include <iterator> +#include <optional> #include <string> #include <vector> @@ -66,8 +69,33 @@ std::string EncodeBase32(Span<const unsigned char> input, bool pad = true); std::string EncodeBase32(const std::string& str, bool pad = true); void SplitHostPort(std::string in, uint16_t& portOut, std::string& hostOut); -int64_t atoi64(const std::string& str); -int atoi(const std::string& str); + +// LocaleIndependentAtoi is provided for backwards compatibility reasons. +// +// New code should use the ParseInt64/ParseUInt64/ParseInt32/ParseUInt32 functions +// which provide parse error feedback. +// +// The goal of LocaleIndependentAtoi is to replicate the exact defined behaviour +// of atoi and atoi64 as they behave under the "C" locale. +template <typename T> +T LocaleIndependentAtoi(const std::string& str) +{ + static_assert(std::is_integral<T>::value); + T result; + // Emulate atoi(...) handling of white space and leading +/-. + std::string s = TrimString(str); + if (!s.empty() && s[0] == '+') { + if (s.length() >= 2 && s[1] == '-') { + return 0; + } + s = s.substr(1); + } + auto [_, error_condition] = std::from_chars(s.data(), s.data() + s.size(), result); + if (error_condition != std::errc{}) { + return 0; + } + return result; +} /** * Tests if the given character is a decimal digit. @@ -95,6 +123,26 @@ constexpr inline bool IsSpace(char c) noexcept { } /** + * Convert string to integral type T. Leading whitespace, a leading +, or any + * trailing character fail the parsing. The required format expressed as regex + * is `-?[0-9]+`. + * + * @returns std::nullopt if the entire string could not be parsed, or if the + * parsed value is not in the range representable by the type T. + */ +template <typename T> +std::optional<T> ToIntegral(const std::string& str) +{ + static_assert(std::is_integral<T>::value); + T result; + const auto [first_nonmatching, error_condition] = std::from_chars(str.data(), str.data() + str.size(), result); + if (first_nonmatching != str.data() + str.size() || error_condition != std::errc{}) { + return std::nullopt; + } + return result; +} + +/** * Convert string to signed 32-bit integer with strict parse error feedback. * @returns true if the entire string could be parsed as valid integer, * false if not the entire string could be parsed or when overflow or underflow occurred. @@ -137,13 +185,6 @@ constexpr inline bool IsSpace(char c) noexcept { [[nodiscard]] bool ParseUInt64(const std::string& str, uint64_t *out); /** - * Convert string to double with strict parse error feedback. - * @returns true if the entire string could be parsed as valid double, - * false if not the entire string could be parsed or when overflow or underflow occurred. - */ -[[nodiscard]] bool ParseDouble(const std::string& str, double *out); - -/** * Convert a span of bytes to a lower-case hexadecimal string. */ std::string HexStr(const Span<const uint8_t> s); diff --git a/src/util/string.h b/src/util/string.h index b26facc502..5617e4acc1 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -65,6 +65,14 @@ inline std::string Join(const std::vector<std::string>& list, const std::string& } /** + * Create an unordered multi-line list of items. + */ +inline std::string MakeUnorderedList(const std::vector<std::string>& items) +{ + return Join(items, "\n", [](const std::string& item) { return "- " + item; }); +} + +/** * Check if a string does not contain any embedded NUL (\0) characters */ [[nodiscard]] inline bool ValidAsCString(const std::string& str) noexcept diff --git a/src/util/syscall_sandbox.cpp b/src/util/syscall_sandbox.cpp new file mode 100644 index 0000000000..7e76ecbf3a --- /dev/null +++ b/src/util/syscall_sandbox.cpp @@ -0,0 +1,909 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif // defined(HAVE_CONFIG_H) + +#include <util/syscall_sandbox.h> + +#if defined(USE_SYSCALL_SANDBOX) +#include <array> +#include <cassert> +#include <cstdint> +#include <exception> +#include <map> +#include <new> +#include <set> +#include <string> +#include <vector> + +#include <logging.h> +#include <tinyformat.h> +#include <util/threadnames.h> + +#include <linux/audit.h> +#include <linux/filter.h> +#include <linux/seccomp.h> +#include <linux/unistd.h> +#include <signal.h> +#include <sys/prctl.h> +#include <sys/types.h> +#include <unistd.h> + +namespace { +bool g_syscall_sandbox_enabled{false}; +bool g_syscall_sandbox_log_violation_before_terminating{false}; + +#if !defined(__x86_64__) +#error Syscall sandbox is an experimental feature currently available only under Linux x86-64. +#endif // defined(__x86_64__) + +#ifndef SECCOMP_RET_KILL_PROCESS +#define SECCOMP_RET_KILL_PROCESS 0x80000000U +#endif + +// Define system call numbers for x86_64 that are referenced in the system call profile +// but not provided by the kernel headers used in the GUIX build. +#ifndef __NR_statx +#define __NR_statx 332 +#endif + +#ifndef __NR_getrandom +#define __NR_getrandom 318 +#endif + +#ifndef __NR_membarrier +#define __NR_membarrier 324 +#endif + +#ifndef __NR_copy_file_range +#define __NR_copy_file_range 326 +#endif + +// This list of syscalls in LINUX_SYSCALLS is only used to map syscall numbers to syscall names in +// order to be able to print user friendly error messages which include the syscall name in addition +// to the syscall number. +// +// Example output in case of a syscall violation where the syscall is present in LINUX_SYSCALLS: +// +// ``` +// 2021-06-09T12:34:56Z ERROR: The syscall "execve" (syscall number 59) is not allowed by the syscall sandbox in thread "msghand". Please report. +// ``` +// +// Example output in case of a syscall violation where the syscall is not present in LINUX_SYSCALLS: +// +// ``` +// 2021-06-09T12:34:56Z ERROR: The syscall "*unknown*" (syscall number 314) is not allowed by the syscall sandbox in thread "msghand". Please report. +// `` +// +// LINUX_SYSCALLS contains two types of syscalls: +// 1.) Syscalls that are present under all architectures or relevant Linux kernel versions for which +// we support the syscall sandbox feature (currently only Linux x86-64). Examples include read, +// write, open, close, etc. +// 2.) Syscalls that are present under a subset of architectures or relevant Linux kernel versions +// for which we support the syscall sandbox feature. This type of syscalls should be added to +// LINUX_SYSCALLS conditional on availability like in the following example: +// ... +// #if defined(__NR_arch_dependent_syscall) +// {__NR_arch_dependent_syscall, "arch_dependent_syscall"}, +// #endif // defined(__NR_arch_dependent_syscall) +// ... +const std::map<uint32_t, std::string> LINUX_SYSCALLS{ + {__NR_accept, "accept"}, + {__NR_accept4, "accept4"}, + {__NR_access, "access"}, + {__NR_acct, "acct"}, + {__NR_add_key, "add_key"}, + {__NR_adjtimex, "adjtimex"}, + {__NR_afs_syscall, "afs_syscall"}, + {__NR_alarm, "alarm"}, + {__NR_arch_prctl, "arch_prctl"}, + {__NR_bind, "bind"}, + {__NR_bpf, "bpf"}, + {__NR_brk, "brk"}, + {__NR_capget, "capget"}, + {__NR_capset, "capset"}, + {__NR_chdir, "chdir"}, + {__NR_chmod, "chmod"}, + {__NR_chown, "chown"}, + {__NR_chroot, "chroot"}, + {__NR_clock_adjtime, "clock_adjtime"}, + {__NR_clock_getres, "clock_getres"}, + {__NR_clock_gettime, "clock_gettime"}, + {__NR_clock_nanosleep, "clock_nanosleep"}, + {__NR_clock_settime, "clock_settime"}, + {__NR_clone, "clone"}, + {__NR_close, "close"}, + {__NR_connect, "connect"}, + {__NR_copy_file_range, "copy_file_range"}, + {__NR_creat, "creat"}, + {__NR_create_module, "create_module"}, + {__NR_delete_module, "delete_module"}, + {__NR_dup, "dup"}, + {__NR_dup2, "dup2"}, + {__NR_dup3, "dup3"}, + {__NR_epoll_create, "epoll_create"}, + {__NR_epoll_create1, "epoll_create1"}, + {__NR_epoll_ctl, "epoll_ctl"}, + {__NR_epoll_ctl_old, "epoll_ctl_old"}, + {__NR_epoll_pwait, "epoll_pwait"}, + {__NR_epoll_wait, "epoll_wait"}, + {__NR_epoll_wait_old, "epoll_wait_old"}, + {__NR_eventfd, "eventfd"}, + {__NR_eventfd2, "eventfd2"}, + {__NR_execve, "execve"}, + {__NR_execveat, "execveat"}, + {__NR_exit, "exit"}, + {__NR_exit_group, "exit_group"}, + {__NR_faccessat, "faccessat"}, + {__NR_fadvise64, "fadvise64"}, + {__NR_fallocate, "fallocate"}, + {__NR_fanotify_init, "fanotify_init"}, + {__NR_fanotify_mark, "fanotify_mark"}, + {__NR_fchdir, "fchdir"}, + {__NR_fchmod, "fchmod"}, + {__NR_fchmodat, "fchmodat"}, + {__NR_fchown, "fchown"}, + {__NR_fchownat, "fchownat"}, + {__NR_fcntl, "fcntl"}, + {__NR_fdatasync, "fdatasync"}, + {__NR_fgetxattr, "fgetxattr"}, + {__NR_finit_module, "finit_module"}, + {__NR_flistxattr, "flistxattr"}, + {__NR_flock, "flock"}, + {__NR_fork, "fork"}, + {__NR_fremovexattr, "fremovexattr"}, + {__NR_fsetxattr, "fsetxattr"}, + {__NR_fstat, "fstat"}, + {__NR_fstatfs, "fstatfs"}, + {__NR_fsync, "fsync"}, + {__NR_ftruncate, "ftruncate"}, + {__NR_futex, "futex"}, + {__NR_futimesat, "futimesat"}, + {__NR_getcpu, "getcpu"}, + {__NR_getcwd, "getcwd"}, + {__NR_getdents, "getdents"}, + {__NR_getdents64, "getdents64"}, + {__NR_getegid, "getegid"}, + {__NR_geteuid, "geteuid"}, + {__NR_getgid, "getgid"}, + {__NR_getgroups, "getgroups"}, + {__NR_getitimer, "getitimer"}, + {__NR_get_kernel_syms, "get_kernel_syms"}, + {__NR_get_mempolicy, "get_mempolicy"}, + {__NR_getpeername, "getpeername"}, + {__NR_getpgid, "getpgid"}, + {__NR_getpgrp, "getpgrp"}, + {__NR_getpid, "getpid"}, + {__NR_getpmsg, "getpmsg"}, + {__NR_getppid, "getppid"}, + {__NR_getpriority, "getpriority"}, + {__NR_getrandom, "getrandom"}, + {__NR_getresgid, "getresgid"}, + {__NR_getresuid, "getresuid"}, + {__NR_getrlimit, "getrlimit"}, + {__NR_get_robust_list, "get_robust_list"}, + {__NR_getrusage, "getrusage"}, + {__NR_getsid, "getsid"}, + {__NR_getsockname, "getsockname"}, + {__NR_getsockopt, "getsockopt"}, + {__NR_get_thread_area, "get_thread_area"}, + {__NR_gettid, "gettid"}, + {__NR_gettimeofday, "gettimeofday"}, + {__NR_getuid, "getuid"}, + {__NR_getxattr, "getxattr"}, + {__NR_init_module, "init_module"}, + {__NR_inotify_add_watch, "inotify_add_watch"}, + {__NR_inotify_init, "inotify_init"}, + {__NR_inotify_init1, "inotify_init1"}, + {__NR_inotify_rm_watch, "inotify_rm_watch"}, + {__NR_io_cancel, "io_cancel"}, + {__NR_ioctl, "ioctl"}, + {__NR_io_destroy, "io_destroy"}, + {__NR_io_getevents, "io_getevents"}, + {__NR_ioperm, "ioperm"}, + {__NR_iopl, "iopl"}, + {__NR_ioprio_get, "ioprio_get"}, + {__NR_ioprio_set, "ioprio_set"}, + {__NR_io_setup, "io_setup"}, + {__NR_io_submit, "io_submit"}, + {__NR_kcmp, "kcmp"}, + {__NR_kexec_file_load, "kexec_file_load"}, + {__NR_kexec_load, "kexec_load"}, + {__NR_keyctl, "keyctl"}, + {__NR_kill, "kill"}, + {__NR_lchown, "lchown"}, + {__NR_lgetxattr, "lgetxattr"}, + {__NR_link, "link"}, + {__NR_linkat, "linkat"}, + {__NR_listen, "listen"}, + {__NR_listxattr, "listxattr"}, + {__NR_llistxattr, "llistxattr"}, + {__NR_lookup_dcookie, "lookup_dcookie"}, + {__NR_lremovexattr, "lremovexattr"}, + {__NR_lseek, "lseek"}, + {__NR_lsetxattr, "lsetxattr"}, + {__NR_lstat, "lstat"}, + {__NR_madvise, "madvise"}, + {__NR_mbind, "mbind"}, + {__NR_membarrier, "membarrier"}, + {__NR_memfd_create, "memfd_create"}, + {__NR_migrate_pages, "migrate_pages"}, + {__NR_mincore, "mincore"}, + {__NR_mkdir, "mkdir"}, + {__NR_mkdirat, "mkdirat"}, + {__NR_mknod, "mknod"}, + {__NR_mknodat, "mknodat"}, + {__NR_mlock, "mlock"}, + {__NR_mlock2, "mlock2"}, + {__NR_mlockall, "mlockall"}, + {__NR_mmap, "mmap"}, + {__NR_modify_ldt, "modify_ldt"}, + {__NR_mount, "mount"}, + {__NR_move_pages, "move_pages"}, + {__NR_mprotect, "mprotect"}, + {__NR_mq_getsetattr, "mq_getsetattr"}, + {__NR_mq_notify, "mq_notify"}, + {__NR_mq_open, "mq_open"}, + {__NR_mq_timedreceive, "mq_timedreceive"}, + {__NR_mq_timedsend, "mq_timedsend"}, + {__NR_mq_unlink, "mq_unlink"}, + {__NR_mremap, "mremap"}, + {__NR_msgctl, "msgctl"}, + {__NR_msgget, "msgget"}, + {__NR_msgrcv, "msgrcv"}, + {__NR_msgsnd, "msgsnd"}, + {__NR_msync, "msync"}, + {__NR_munlock, "munlock"}, + {__NR_munlockall, "munlockall"}, + {__NR_munmap, "munmap"}, + {__NR_name_to_handle_at, "name_to_handle_at"}, + {__NR_nanosleep, "nanosleep"}, + {__NR_newfstatat, "newfstatat"}, + {__NR_nfsservctl, "nfsservctl"}, + {__NR_open, "open"}, + {__NR_openat, "openat"}, + {__NR_open_by_handle_at, "open_by_handle_at"}, + {__NR_pause, "pause"}, + {__NR_perf_event_open, "perf_event_open"}, + {__NR_personality, "personality"}, + {__NR_pipe, "pipe"}, + {__NR_pipe2, "pipe2"}, + {__NR_pivot_root, "pivot_root"}, +#ifdef __NR_pkey_alloc + {__NR_pkey_alloc, "pkey_alloc"}, +#endif +#ifdef __NR_pkey_free + {__NR_pkey_free, "pkey_free"}, +#endif +#ifdef __NR_pkey_mprotect + {__NR_pkey_mprotect, "pkey_mprotect"}, +#endif + {__NR_poll, "poll"}, + {__NR_ppoll, "ppoll"}, + {__NR_prctl, "prctl"}, + {__NR_pread64, "pread64"}, + {__NR_preadv, "preadv"}, +#ifdef __NR_preadv2 + {__NR_preadv2, "preadv2"}, +#endif + {__NR_prlimit64, "prlimit64"}, + {__NR_process_vm_readv, "process_vm_readv"}, + {__NR_process_vm_writev, "process_vm_writev"}, + {__NR_pselect6, "pselect6"}, + {__NR_ptrace, "ptrace"}, + {__NR_putpmsg, "putpmsg"}, + {__NR_pwrite64, "pwrite64"}, + {__NR_pwritev, "pwritev"}, +#ifdef __NR_pwritev2 + {__NR_pwritev2, "pwritev2"}, +#endif + {__NR_query_module, "query_module"}, + {__NR_quotactl, "quotactl"}, + {__NR_read, "read"}, + {__NR_readahead, "readahead"}, + {__NR_readlink, "readlink"}, + {__NR_readlinkat, "readlinkat"}, + {__NR_readv, "readv"}, + {__NR_reboot, "reboot"}, + {__NR_recvfrom, "recvfrom"}, + {__NR_recvmmsg, "recvmmsg"}, + {__NR_recvmsg, "recvmsg"}, + {__NR_remap_file_pages, "remap_file_pages"}, + {__NR_removexattr, "removexattr"}, + {__NR_rename, "rename"}, + {__NR_renameat, "renameat"}, + {__NR_renameat2, "renameat2"}, + {__NR_request_key, "request_key"}, + {__NR_restart_syscall, "restart_syscall"}, + {__NR_rmdir, "rmdir"}, + {__NR_rt_sigaction, "rt_sigaction"}, + {__NR_rt_sigpending, "rt_sigpending"}, + {__NR_rt_sigprocmask, "rt_sigprocmask"}, + {__NR_rt_sigqueueinfo, "rt_sigqueueinfo"}, + {__NR_rt_sigreturn, "rt_sigreturn"}, + {__NR_rt_sigsuspend, "rt_sigsuspend"}, + {__NR_rt_sigtimedwait, "rt_sigtimedwait"}, + {__NR_rt_tgsigqueueinfo, "rt_tgsigqueueinfo"}, + {__NR_sched_getaffinity, "sched_getaffinity"}, + {__NR_sched_getattr, "sched_getattr"}, + {__NR_sched_getparam, "sched_getparam"}, + {__NR_sched_get_priority_max, "sched_get_priority_max"}, + {__NR_sched_get_priority_min, "sched_get_priority_min"}, + {__NR_sched_getscheduler, "sched_getscheduler"}, + {__NR_sched_rr_get_interval, "sched_rr_get_interval"}, + {__NR_sched_setaffinity, "sched_setaffinity"}, + {__NR_sched_setattr, "sched_setattr"}, + {__NR_sched_setparam, "sched_setparam"}, + {__NR_sched_setscheduler, "sched_setscheduler"}, + {__NR_sched_yield, "sched_yield"}, + {__NR_seccomp, "seccomp"}, + {__NR_security, "security"}, + {__NR_select, "select"}, + {__NR_semctl, "semctl"}, + {__NR_semget, "semget"}, + {__NR_semop, "semop"}, + {__NR_semtimedop, "semtimedop"}, + {__NR_sendfile, "sendfile"}, + {__NR_sendmmsg, "sendmmsg"}, + {__NR_sendmsg, "sendmsg"}, + {__NR_sendto, "sendto"}, + {__NR_setdomainname, "setdomainname"}, + {__NR_setfsgid, "setfsgid"}, + {__NR_setfsuid, "setfsuid"}, + {__NR_setgid, "setgid"}, + {__NR_setgroups, "setgroups"}, + {__NR_sethostname, "sethostname"}, + {__NR_setitimer, "setitimer"}, + {__NR_set_mempolicy, "set_mempolicy"}, + {__NR_setns, "setns"}, + {__NR_setpgid, "setpgid"}, + {__NR_setpriority, "setpriority"}, + {__NR_setregid, "setregid"}, + {__NR_setresgid, "setresgid"}, + {__NR_setresuid, "setresuid"}, + {__NR_setreuid, "setreuid"}, + {__NR_setrlimit, "setrlimit"}, + {__NR_set_robust_list, "set_robust_list"}, + {__NR_setsid, "setsid"}, + {__NR_setsockopt, "setsockopt"}, + {__NR_set_thread_area, "set_thread_area"}, + {__NR_set_tid_address, "set_tid_address"}, + {__NR_settimeofday, "settimeofday"}, + {__NR_setuid, "setuid"}, + {__NR_setxattr, "setxattr"}, + {__NR_shmat, "shmat"}, + {__NR_shmctl, "shmctl"}, + {__NR_shmdt, "shmdt"}, + {__NR_shmget, "shmget"}, + {__NR_shutdown, "shutdown"}, + {__NR_sigaltstack, "sigaltstack"}, + {__NR_signalfd, "signalfd"}, + {__NR_signalfd4, "signalfd4"}, + {__NR_socket, "socket"}, + {__NR_socketpair, "socketpair"}, + {__NR_splice, "splice"}, + {__NR_stat, "stat"}, + {__NR_statfs, "statfs"}, + {__NR_statx, "statx"}, + {__NR_swapoff, "swapoff"}, + {__NR_swapon, "swapon"}, + {__NR_symlink, "symlink"}, + {__NR_symlinkat, "symlinkat"}, + {__NR_sync, "sync"}, + {__NR_sync_file_range, "sync_file_range"}, + {__NR_syncfs, "syncfs"}, + {__NR__sysctl, "_sysctl"}, + {__NR_sysfs, "sysfs"}, + {__NR_sysinfo, "sysinfo"}, + {__NR_syslog, "syslog"}, + {__NR_tee, "tee"}, + {__NR_tgkill, "tgkill"}, + {__NR_time, "time"}, + {__NR_timer_create, "timer_create"}, + {__NR_timer_delete, "timer_delete"}, + {__NR_timerfd_create, "timerfd_create"}, + {__NR_timerfd_gettime, "timerfd_gettime"}, + {__NR_timerfd_settime, "timerfd_settime"}, + {__NR_timer_getoverrun, "timer_getoverrun"}, + {__NR_timer_gettime, "timer_gettime"}, + {__NR_timer_settime, "timer_settime"}, + {__NR_times, "times"}, + {__NR_tkill, "tkill"}, + {__NR_truncate, "truncate"}, + {__NR_tuxcall, "tuxcall"}, + {__NR_umask, "umask"}, + {__NR_umount2, "umount2"}, + {__NR_uname, "uname"}, + {__NR_unlink, "unlink"}, + {__NR_unlinkat, "unlinkat"}, + {__NR_unshare, "unshare"}, + {__NR_uselib, "uselib"}, + {__NR_userfaultfd, "userfaultfd"}, + {__NR_ustat, "ustat"}, + {__NR_utime, "utime"}, + {__NR_utimensat, "utimensat"}, + {__NR_utimes, "utimes"}, + {__NR_vfork, "vfork"}, + {__NR_vhangup, "vhangup"}, + {__NR_vmsplice, "vmsplice"}, + {__NR_vserver, "vserver"}, + {__NR_wait4, "wait4"}, + {__NR_waitid, "waitid"}, + {__NR_write, "write"}, + {__NR_writev, "writev"}, +}; + +std::string GetLinuxSyscallName(uint32_t syscall_number) +{ + const auto element = LINUX_SYSCALLS.find(syscall_number); + if (element != LINUX_SYSCALLS.end()) { + return element->second; + } + return "*unknown*"; +} + +// See Linux kernel developer Kees Cook's seccomp guide at <https://outflux.net/teach-seccomp/> for +// an accessible introduction to using seccomp. +// +// This function largely follows <https://outflux.net/teach-seccomp/step-3/syscall-reporter.c> and +// <https://outflux.net/teach-seccomp/step-3/seccomp-bpf.h>. +// +// Seccomp BPF resources: +// * Seccomp BPF documentation: <https://www.kernel.org/doc/html/latest/userspace-api/seccomp_filter.html> +// * seccomp(2) manual page: <https://www.kernel.org/doc/man-pages/online/pages/man2/seccomp.2.html> +// * Seccomp BPF demo code samples: <https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/samples/seccomp> +void SyscallSandboxDebugSignalHandler(int, siginfo_t* signal_info, void* void_signal_context) +{ + // The si_code field inside the siginfo_t argument that is passed to a SA_SIGINFO signal handler + // is a value indicating why the signal was sent. + // + // The following value can be placed in si_code for a SIGSYS signal: + // * SYS_SECCOMP (since Linux 3.5): Triggered by a seccomp(2) filter rule. + constexpr int32_t SYS_SECCOMP_SI_CODE{1}; + assert(signal_info->si_code == SYS_SECCOMP_SI_CODE); + + // The ucontext_t structure contains signal context information that was saved on the user-space + // stack by the kernel. + const ucontext_t* signal_context = static_cast<ucontext_t*>(void_signal_context); + assert(signal_context != nullptr); + + std::set_new_handler(std::terminate); + // Portability note: REG_RAX is Linux x86_64 specific. + const uint32_t syscall_number = static_cast<uint32_t>(signal_context->uc_mcontext.gregs[REG_RAX]); + const std::string syscall_name = GetLinuxSyscallName(syscall_number); + const std::string thread_name = !util::ThreadGetInternalName().empty() ? util::ThreadGetInternalName() : "*unnamed*"; + const std::string error_message = strprintf("ERROR: The syscall \"%s\" (syscall number %d) is not allowed by the syscall sandbox in thread \"%s\". Please report.", syscall_name, syscall_number, thread_name); + tfm::format(std::cerr, "%s\n", error_message); + LogPrintf("%s\n", error_message); + std::terminate(); +} + +// This function largely follows install_syscall_reporter from Kees Cook's seccomp guide: +// <https://outflux.net/teach-seccomp/step-3/syscall-reporter.c> +bool SetupSyscallSandboxDebugHandler() +{ + struct sigaction action = {}; + sigset_t mask; + sigemptyset(&mask); + sigaddset(&mask, SIGSYS); + action.sa_sigaction = &SyscallSandboxDebugSignalHandler; + action.sa_flags = SA_SIGINFO; + if (sigaction(SIGSYS, &action, nullptr) < 0) { + return false; + } + if (sigprocmask(SIG_UNBLOCK, &mask, nullptr)) { + return false; + } + return true; +} + +enum class SyscallSandboxAction { + KILL_PROCESS, + INVOKE_SIGNAL_HANDLER, +}; + +class SeccompPolicyBuilder +{ + std::set<uint32_t> allowed_syscalls; + +public: + SeccompPolicyBuilder() + { + // Allowed by default. + AllowAddressSpaceAccess(); + AllowEpoll(); + AllowEventFd(); + AllowFutex(); + AllowGeneralIo(); + AllowGetRandom(); + AllowGetSimpleId(); + AllowGetTime(); + AllowGlobalProcessEnvironment(); + AllowGlobalSystemStatus(); + AllowKernelInternalApi(); + AllowNetworkSocketInformation(); + AllowOperationOnExistingFileDescriptor(); + AllowPipe(); + AllowPrctl(); + AllowProcessStartOrDeath(); + AllowScheduling(); + AllowSignalHandling(); + AllowSleep(); + AllowUmask(); + } + + void AllowAddressSpaceAccess() + { + allowed_syscalls.insert(__NR_brk); // change data segment size + allowed_syscalls.insert(__NR_madvise); // give advice about use of memory + allowed_syscalls.insert(__NR_membarrier); // issue memory barriers on a set of threads + allowed_syscalls.insert(__NR_mlock); // lock memory + allowed_syscalls.insert(__NR_mmap); // map files or devices into memory + allowed_syscalls.insert(__NR_mprotect); // set protection on a region of memory + allowed_syscalls.insert(__NR_mremap); // remap a file in memory + allowed_syscalls.insert(__NR_munlock); // unlock memory + allowed_syscalls.insert(__NR_munmap); // unmap files or devices into memory + } + + void AllowEpoll() + { + allowed_syscalls.insert(__NR_epoll_create1); // open an epoll file descriptor + allowed_syscalls.insert(__NR_epoll_ctl); // control interface for an epoll file descriptor + allowed_syscalls.insert(__NR_epoll_pwait); // wait for an I/O event on an epoll file descriptor + allowed_syscalls.insert(__NR_epoll_wait); // wait for an I/O event on an epoll file descriptor + } + + void AllowEventFd() + { + allowed_syscalls.insert(__NR_eventfd2); // create a file descriptor for event notification + } + + void AllowFileSystem() + { + allowed_syscalls.insert(__NR_access); // check user's permissions for a file + allowed_syscalls.insert(__NR_chdir); // change working directory + allowed_syscalls.insert(__NR_chmod); // change permissions of a file + allowed_syscalls.insert(__NR_copy_file_range); // copy a range of data from one file to another + allowed_syscalls.insert(__NR_fallocate); // manipulate file space + allowed_syscalls.insert(__NR_fchmod); // change permissions of a file + allowed_syscalls.insert(__NR_fchown); // change ownership of a file + allowed_syscalls.insert(__NR_fdatasync); // synchronize a file's in-core state with storage device + allowed_syscalls.insert(__NR_flock); // apply or remove an advisory lock on an open file + allowed_syscalls.insert(__NR_fstat); // get file status + allowed_syscalls.insert(__NR_newfstatat); // get file status + allowed_syscalls.insert(__NR_fsync); // synchronize a file's in-core state with storage device + allowed_syscalls.insert(__NR_ftruncate); // truncate a file to a specified length + allowed_syscalls.insert(__NR_getcwd); // get current working directory + allowed_syscalls.insert(__NR_getdents); // get directory entries + allowed_syscalls.insert(__NR_getdents64); // get directory entries + allowed_syscalls.insert(__NR_lstat); // get file status + allowed_syscalls.insert(__NR_mkdir); // create a directory + allowed_syscalls.insert(__NR_open); // open and possibly create a file + allowed_syscalls.insert(__NR_openat); // open and possibly create a file + allowed_syscalls.insert(__NR_readlink); // read value of a symbolic link + allowed_syscalls.insert(__NR_rename); // change the name or location of a file + allowed_syscalls.insert(__NR_rmdir); // delete a directory + allowed_syscalls.insert(__NR_stat); // get file status + allowed_syscalls.insert(__NR_statfs); // get filesystem statistics + allowed_syscalls.insert(__NR_statx); // get file status (extended) + allowed_syscalls.insert(__NR_unlink); // delete a name and possibly the file it refers to + } + + void AllowFutex() + { + allowed_syscalls.insert(__NR_futex); // fast user-space locking + allowed_syscalls.insert(__NR_set_robust_list); // set list of robust futexes + } + + void AllowGeneralIo() + { + allowed_syscalls.insert(__NR_ioctl); // control device + allowed_syscalls.insert(__NR_lseek); // reposition read/write file offset + allowed_syscalls.insert(__NR_poll); // wait for some event on a file descriptor + allowed_syscalls.insert(__NR_ppoll); // wait for some event on a file descriptor + allowed_syscalls.insert(__NR_pread64); // read from a file descriptor at a given offset + allowed_syscalls.insert(__NR_pwrite64); // write to a file descriptor at a given offset + allowed_syscalls.insert(__NR_read); // read from a file descriptor + allowed_syscalls.insert(__NR_readv); // read data into multiple buffers + allowed_syscalls.insert(__NR_recvfrom); // receive a message from a socket + allowed_syscalls.insert(__NR_recvmsg); // receive a message from a socket + allowed_syscalls.insert(__NR_select); // synchronous I/O multiplexing + allowed_syscalls.insert(__NR_sendmmsg); // send multiple messages on a socket + allowed_syscalls.insert(__NR_sendmsg); // send a message on a socket + allowed_syscalls.insert(__NR_sendto); // send a message on a socket + allowed_syscalls.insert(__NR_write); // write to a file descriptor + allowed_syscalls.insert(__NR_writev); // write data into multiple buffers + } + + void AllowGetRandom() + { + allowed_syscalls.insert(__NR_getrandom); // obtain a series of random bytes + } + + void AllowGetSimpleId() + { + allowed_syscalls.insert(__NR_getegid); // get group identity + allowed_syscalls.insert(__NR_geteuid); // get user identity + allowed_syscalls.insert(__NR_getgid); // get group identity + allowed_syscalls.insert(__NR_getpgid); // get process group + allowed_syscalls.insert(__NR_getpid); // get process identification + allowed_syscalls.insert(__NR_getppid); // get process identification + allowed_syscalls.insert(__NR_getresgid); // get real, effective and saved group IDs + allowed_syscalls.insert(__NR_getresuid); // get real, effective and saved user IDs + allowed_syscalls.insert(__NR_getsid); // get session ID + allowed_syscalls.insert(__NR_gettid); // get thread identification + allowed_syscalls.insert(__NR_getuid); // get user identity + } + + void AllowGetTime() + { + allowed_syscalls.insert(__NR_clock_getres); // find the resolution (precision) of the specified clock + allowed_syscalls.insert(__NR_clock_gettime); // retrieve the time of the specified clock + } + + void AllowGlobalProcessEnvironment() + { + allowed_syscalls.insert(__NR_getrlimit); // get resource limits + allowed_syscalls.insert(__NR_getrusage); // get resource usage + allowed_syscalls.insert(__NR_prlimit64); // get/set resource limits + } + + void AllowGlobalSystemStatus() + { + allowed_syscalls.insert(__NR_sysinfo); // return system information + allowed_syscalls.insert(__NR_uname); // get name and information about current kernel + } + + void AllowKernelInternalApi() + { + allowed_syscalls.insert(__NR_restart_syscall); // restart a system call after interruption by a stop signal + } + + void AllowNetwork() + { + allowed_syscalls.insert(__NR_accept); // accept a connection on a socket + allowed_syscalls.insert(__NR_accept4); // accept a connection on a socket + allowed_syscalls.insert(__NR_bind); // bind a name to a socket + allowed_syscalls.insert(__NR_connect); // initiate a connection on a socket + allowed_syscalls.insert(__NR_listen); // listen for connections on a socket + allowed_syscalls.insert(__NR_setsockopt); // set options on sockets + allowed_syscalls.insert(__NR_socket); // create an endpoint for communication + allowed_syscalls.insert(__NR_socketpair); // create a pair of connected sockets + } + + void AllowNetworkSocketInformation() + { + allowed_syscalls.insert(__NR_getpeername); // get name of connected peer socket + allowed_syscalls.insert(__NR_getsockname); // get socket name + allowed_syscalls.insert(__NR_getsockopt); // get options on sockets + } + + void AllowOperationOnExistingFileDescriptor() + { + allowed_syscalls.insert(__NR_close); // close a file descriptor + allowed_syscalls.insert(__NR_dup); // duplicate a file descriptor + allowed_syscalls.insert(__NR_dup2); // duplicate a file descriptor + allowed_syscalls.insert(__NR_fcntl); // manipulate file descriptor + allowed_syscalls.insert(__NR_shutdown); // shut down part of a full-duplex connection + } + + void AllowPipe() + { + allowed_syscalls.insert(__NR_pipe); // create pipe + allowed_syscalls.insert(__NR_pipe2); // create pipe + } + + void AllowPrctl() + { + allowed_syscalls.insert(__NR_arch_prctl); // set architecture-specific thread state + allowed_syscalls.insert(__NR_prctl); // operations on a process + } + + void AllowProcessStartOrDeath() + { + allowed_syscalls.insert(__NR_clone); // create a child process + allowed_syscalls.insert(__NR_exit); // terminate the calling process + allowed_syscalls.insert(__NR_exit_group); // exit all threads in a process + allowed_syscalls.insert(__NR_fork); // create a child process + allowed_syscalls.insert(__NR_tgkill); // send a signal to a thread + allowed_syscalls.insert(__NR_wait4); // wait for process to change state, BSD style + } + + void AllowScheduling() + { + allowed_syscalls.insert(__NR_sched_getaffinity); // set a thread's CPU affinity mask + allowed_syscalls.insert(__NR_sched_getparam); // get scheduling parameters + allowed_syscalls.insert(__NR_sched_getscheduler); // get scheduling policy/parameters + allowed_syscalls.insert(__NR_sched_setscheduler); // set scheduling policy/parameters + allowed_syscalls.insert(__NR_sched_yield); // yield the processor + } + + void AllowSignalHandling() + { + allowed_syscalls.insert(__NR_rt_sigaction); // examine and change a signal action + allowed_syscalls.insert(__NR_rt_sigprocmask); // examine and change blocked signals + allowed_syscalls.insert(__NR_rt_sigreturn); // return from signal handler and cleanup stack frame + allowed_syscalls.insert(__NR_sigaltstack); // set and/or get signal stack context + } + + void AllowSleep() + { + allowed_syscalls.insert(__NR_clock_nanosleep); // high-resolution sleep with specifiable clock + allowed_syscalls.insert(__NR_nanosleep); // high-resolution sleep + } + + void AllowUmask() + { + allowed_syscalls.insert(__NR_umask); // set file mode creation mask + } + + // See Linux kernel developer Kees Cook's seccomp guide at <https://outflux.net/teach-seccomp/> + // for an accessible introduction to using seccomp. + // + // This function largely follows <https://outflux.net/teach-seccomp/step-3/seccomp-bpf.h>. + std::vector<sock_filter> BuildFilter(SyscallSandboxAction default_action) + { + std::vector<sock_filter> bpf_policy; + // See VALIDATE_ARCHITECTURE in seccomp-bpf.h referenced above. + bpf_policy.push_back(BPF_STMT(BPF_LD + BPF_W + BPF_ABS, offsetof(struct seccomp_data, arch))); + // Portability note: AUDIT_ARCH_X86_64 is Linux x86_64 specific. + bpf_policy.push_back(BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, AUDIT_ARCH_X86_64, 1, 0)); + bpf_policy.push_back(BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_KILL_PROCESS)); + // See EXAMINE_SYSCALL in seccomp-bpf.h referenced above. + bpf_policy.push_back(BPF_STMT(BPF_LD + BPF_W + BPF_ABS, offsetof(struct seccomp_data, nr))); + for (const uint32_t allowed_syscall : allowed_syscalls) { + // See ALLOW_SYSCALL in seccomp-bpf.h referenced above. + bpf_policy.push_back(BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, allowed_syscall, 0, 1)); + bpf_policy.push_back(BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_ALLOW)); + } + switch (default_action) { + case SyscallSandboxAction::KILL_PROCESS: + // Disallow syscall and kill the process. + // + // See KILL_PROCESS in seccomp-bpf.h referenced above. + // + // Note that we're using SECCOMP_RET_KILL_PROCESS (kill the process) instead + // of SECCOMP_RET_KILL_THREAD (kill the thread). The SECCOMP_RET_KILL_PROCESS + // action was introduced in Linux 4.14. + // + // SECCOMP_RET_KILL_PROCESS: Results in the entire process exiting immediately without + // executing the system call. + // + // SECCOMP_RET_KILL_PROCESS documentation: + // <https://www.kernel.org/doc/html/latest/userspace-api/seccomp_filter.html> + bpf_policy.push_back(BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_KILL_PROCESS)); + break; + case SyscallSandboxAction::INVOKE_SIGNAL_HANDLER: + // Disallow syscall and force a SIGSYS to trigger syscall debug reporter. + // + // SECCOMP_RET_TRAP: Results in the kernel sending a SIGSYS signal to the triggering + // task without executing the system call. + // + // SECCOMP_RET_TRAP documentation: + // <https://www.kernel.org/doc/html/latest/userspace-api/seccomp_filter.html> + bpf_policy.push_back(BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_TRAP)); + break; + } + return bpf_policy; + } +}; +} // namespace + +bool SetupSyscallSandbox(bool log_syscall_violation_before_terminating) +{ + assert(!g_syscall_sandbox_enabled && "SetupSyscallSandbox(...) should only be called once."); + g_syscall_sandbox_enabled = true; + g_syscall_sandbox_log_violation_before_terminating = log_syscall_violation_before_terminating; + if (log_syscall_violation_before_terminating) { + if (!SetupSyscallSandboxDebugHandler()) { + return false; + } + } + SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION); + return true; +} + +void TestDisallowedSandboxCall() +{ + // The getgroups syscall is assumed NOT to be allowed by the syscall sandbox policy. + std::array<gid_t, 1> groups; + [[maybe_unused]] int32_t ignored = getgroups(groups.size(), groups.data()); +} +#endif // defined(USE_SYSCALL_SANDBOX) + +void SetSyscallSandboxPolicy(SyscallSandboxPolicy syscall_policy) +{ +#if defined(USE_SYSCALL_SANDBOX) + if (!g_syscall_sandbox_enabled) { + return; + } + SeccompPolicyBuilder seccomp_policy_builder; + switch (syscall_policy) { + case SyscallSandboxPolicy::INITIALIZATION: // Thread: main thread (state: init) + // SyscallSandboxPolicy::INITIALIZATION is the first policy loaded. + // + // Subsequently loaded policies can reduce the abilities further, but + // abilities can never be regained. + // + // SyscallSandboxPolicy::INITIALIZATION must thus be a superset of all + // other policies. + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::INITIALIZATION_DNS_SEED: // Thread: dnsseed + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::INITIALIZATION_LOAD_BLOCKS: // Thread: loadblk + seccomp_policy_builder.AllowFileSystem(); + break; + case SyscallSandboxPolicy::INITIALIZATION_MAP_PORT: // Thread: mapport + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::MESSAGE_HANDLER: // Thread: msghand + seccomp_policy_builder.AllowFileSystem(); + break; + case SyscallSandboxPolicy::NET: // Thread: net + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::NET_ADD_CONNECTION: // Thread: addcon + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::NET_HTTP_SERVER: // Thread: http + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::NET_HTTP_SERVER_WORKER: // Thread: httpworker.<N> + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::NET_OPEN_CONNECTION: // Thread: opencon + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::SCHEDULER: // Thread: scheduler + seccomp_policy_builder.AllowFileSystem(); + break; + case SyscallSandboxPolicy::TOR_CONTROL: // Thread: torcontrol + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::TX_INDEX: // Thread: txindex + seccomp_policy_builder.AllowFileSystem(); + break; + case SyscallSandboxPolicy::VALIDATION_SCRIPT_CHECK: // Thread: scriptch.<N> + break; + case SyscallSandboxPolicy::SHUTOFF: // Thread: main thread (state: shutoff) + seccomp_policy_builder.AllowFileSystem(); + break; + } + + const SyscallSandboxAction default_action = g_syscall_sandbox_log_violation_before_terminating ? SyscallSandboxAction::INVOKE_SIGNAL_HANDLER : SyscallSandboxAction::KILL_PROCESS; + std::vector<sock_filter> filter = seccomp_policy_builder.BuildFilter(default_action); + const sock_fprog prog = { + .len = static_cast<uint16_t>(filter.size()), + .filter = filter.data(), + }; + // Do not allow abilities to be regained after being dropped. + // + // PR_SET_NO_NEW_PRIVS documentation: <https://www.kernel.org/doc/html/latest/userspace-api/no_new_privs.html> + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) { + throw std::runtime_error("Syscall sandbox enforcement failed: prctl(PR_SET_NO_NEW_PRIVS)"); + } + // Install seccomp-bpf syscall filter. + // + // PR_SET_SECCOMP documentation: <https://www.kernel.org/doc/html/latest/userspace-api/seccomp_filter.html> + if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &prog) != 0) { + throw std::runtime_error("Syscall sandbox enforcement failed: prctl(PR_SET_SECCOMP)"); + } + + const std::string thread_name = !util::ThreadGetInternalName().empty() ? util::ThreadGetInternalName() : "*unnamed*"; + LogPrint(BCLog::UTIL, "Syscall filter installed for thread \"%s\"\n", thread_name); +#endif // defined(USE_SYSCALL_SANDBOX) +} diff --git a/src/util/syscall_sandbox.h b/src/util/syscall_sandbox.h new file mode 100644 index 0000000000..0a0c964f94 --- /dev/null +++ b/src/util/syscall_sandbox.h @@ -0,0 +1,57 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_SYSCALL_SANDBOX_H +#define BITCOIN_UTIL_SYSCALL_SANDBOX_H + +enum class SyscallSandboxPolicy { + // 1. Initialization + INITIALIZATION, + INITIALIZATION_DNS_SEED, + INITIALIZATION_LOAD_BLOCKS, + INITIALIZATION_MAP_PORT, + + // 2. Steady state (non-initialization, non-shutdown) + MESSAGE_HANDLER, + NET, + NET_ADD_CONNECTION, + NET_HTTP_SERVER, + NET_HTTP_SERVER_WORKER, + NET_OPEN_CONNECTION, + SCHEDULER, + TOR_CONTROL, + TX_INDEX, + VALIDATION_SCRIPT_CHECK, + + // 3. Shutdown + SHUTOFF, +}; + +//! Force the current thread (and threads created from the current thread) into a restricted-service +//! operating mode where only a subset of all syscalls are available. +//! +//! Subsequent calls to this function can reduce the abilities further, but abilities can never be +//! regained. +//! +//! This function is a no-op unless SetupSyscallSandbox(...) has been called. +//! +//! SetupSyscallSandbox(...) is called during bitcoind initialization if Bitcoin Core was compiled +//! with seccomp-bpf support (--with-seccomp) *and* the parameter -sandbox=<mode> was passed to +//! bitcoind. +//! +//! This experimental feature is available under Linux x86_64 only. +void SetSyscallSandboxPolicy(SyscallSandboxPolicy syscall_policy); + +#if defined(USE_SYSCALL_SANDBOX) +//! Setup and enable the experimental syscall sandbox for the running process. +//! +//! SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION) is called as part of +//! SetupSyscallSandbox(...). +[[nodiscard]] bool SetupSyscallSandbox(bool log_syscall_violation_before_terminating); + +//! Invoke a disallowed syscall. Use for testing purposes. +void TestDisallowedSandboxCall(); +#endif // defined(USE_SYSCALL_SANDBOX) + +#endif // BITCOIN_UTIL_SYSCALL_SANDBOX_H diff --git a/src/util/system.cpp b/src/util/system.cpp index ee7a76be29..79c08816fa 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -6,6 +6,11 @@ #include <util/system.h> #ifdef ENABLE_EXTERNAL_SIGNER +#if defined(WIN32) && !defined(__kernel_entry) +// A workaround for boost 1.71 incompatibility with mingw-w64 compiler. +// For details see https://github.com/bitcoin/bitcoin/pull/22348. +#define __kernel_entry +#endif #include <boost/process.hpp> #endif // ENABLE_EXTERNAL_SIGNER @@ -153,16 +158,14 @@ std::streampos GetFileSize(const char* path, std::streamsize max) { /** * Interpret a string argument as a boolean. * - * The definition of atoi() requires that non-numeric string values like "foo", - * return 0. This means that if a user unintentionally supplies a non-integer - * argument here, the return value is always false. This means that -foo=false - * does what the user probably expects, but -foo=true is well defined but does - * not do what they probably expected. + * The definition of LocaleIndependentAtoi<int>() requires that non-numeric string values + * like "foo", return 0. This means that if a user unintentionally supplies a + * non-integer argument here, the return value is always false. This means that + * -foo=false does what the user probably expects, but -foo=true is well defined + * but does not do what they probably expected. * - * The return value of atoi() is undefined when given input not representable as - * an int. On most systems this means string value between "-2147483648" and - * "2147483647" are well defined (this method will return true). Setting - * -txindex=2147483648 on most systems, however, is probably undefined. + * The return value of LocaleIndependentAtoi<int>(...) is zero when given input not + * representable as an int. * * For a more extensive discussion of this topic (and a wide range of opinions * on the Right Way to change this code), see PR12713. @@ -171,7 +174,7 @@ static bool InterpretBool(const std::string& strValue) { if (strValue.empty()) return true; - return (atoi(strValue) != 0); + return (LocaleIndependentAtoi<int>(strValue) != 0); } static std::string SettingName(const std::string& arg) @@ -497,11 +500,11 @@ bool ArgsManager::InitSettings(std::string& error) std::vector<std::string> errors; if (!ReadSettingsFile(&errors)) { - error = strprintf("Failed loading settings file:\n- %s\n", Join(errors, "\n- ")); + error = strprintf("Failed loading settings file:\n%s\n", MakeUnorderedList(errors)); return false; } if (!WriteSettingsFile(&errors)) { - error = strprintf("Failed saving settings file:\n- %s\n", Join(errors, "\n- ")); + error = strprintf("Failed saving settings file:\n%s\n", MakeUnorderedList(errors)); return false; } return true; @@ -586,10 +589,10 @@ std::string ArgsManager::GetArg(const std::string& strArg, const std::string& st return value.isNull() ? strDefault : value.isFalse() ? "0" : value.isTrue() ? "1" : value.get_str(); } -int64_t ArgsManager::GetArg(const std::string& strArg, int64_t nDefault) const +int64_t ArgsManager::GetIntArg(const std::string& strArg, int64_t nDefault) const { const util::SettingsValue value = GetSetting(strArg); - return value.isNull() ? nDefault : value.isFalse() ? 0 : value.isTrue() ? 1 : value.isNum() ? value.get_int64() : atoi64(value.get_str()); + return value.isNull() ? nDefault : value.isFalse() ? 0 : value.isTrue() ? 1 : value.isNum() ? value.get_int64() : LocaleIndependentAtoi<int64_t>(value.get_str()); } bool ArgsManager::GetBoolArg(const std::string& strArg, bool fDefault) const @@ -899,6 +902,11 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) const std::string confPath = GetArg("-conf", BITCOIN_CONF_FILENAME); fsbridge::ifstream stream(GetConfigFile(confPath)); + // not ok to have a config file specified that cannot be opened + if (IsArgSet("-conf") && !stream.good()) { + error = strprintf("specified config file \"%s\" could not be opened.", confPath); + return false; + } // ok to not have a config file if (stream.good()) { if (!ReadConfigStream(stream, confPath, error, ignore_invalid_keys)) { @@ -1296,7 +1304,7 @@ void SetupEnvironment() #endif // On most POSIX systems (e.g. Linux, but not BSD) the environment's locale // may be invalid, in which case the "C.UTF-8" locale is used as fallback. -#if !defined(WIN32) && !defined(MAC_OSX) && !defined(__FreeBSD__) && !defined(__OpenBSD__) +#if !defined(WIN32) && !defined(MAC_OSX) && !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__NetBSD__) try { std::locale(""); // Raises a runtime error if current locale is invalid } catch (const std::runtime_error&) { diff --git a/src/util/system.h b/src/util/system.h index 3547bad585..2e217f6f90 100644 --- a/src/util/system.h +++ b/src/util/system.h @@ -205,6 +205,7 @@ protected: */ bool UseDefaultSection(const std::string& arg) const EXCLUSIVE_LOCKS_REQUIRED(cs_args); + public: /** * Get setting value. * @@ -219,7 +220,6 @@ protected: */ std::vector<util::SettingsValue> GetSettingsList(const std::string& arg) const; -public: ArgsManager(); ~ArgsManager(); @@ -327,7 +327,7 @@ public: * @param nDefault (e.g. 1) * @return command-line argument (0 if invalid number) or default value */ - int64_t GetArg(const std::string& strArg, int64_t nDefault) const; + int64_t GetIntArg(const std::string& strArg, int64_t nDefault) const; /** * Return boolean argument or default value diff --git a/src/util/translation.h b/src/util/translation.h index 99899ef3c2..62388b568c 100644 --- a/src/util/translation.h +++ b/src/util/translation.h @@ -28,6 +28,12 @@ struct bilingual_str { { return original.empty(); } + + void clear() + { + original.clear(); + translated.clear(); + } }; inline bilingual_str operator+(bilingual_str lhs, const bilingual_str& rhs) diff --git a/src/util/types.h b/src/util/types.h new file mode 100644 index 0000000000..0047b00026 --- /dev/null +++ b/src/util/types.h @@ -0,0 +1,11 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_TYPES_H +#define BITCOIN_UTIL_TYPES_H + +template <class> +inline constexpr bool ALWAYS_FALSE{false}; + +#endif // BITCOIN_UTIL_TYPES_H diff --git a/src/validation.cpp b/src/validation.cpp index fe4de0d218..14dcd2c24b 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -9,22 +9,25 @@ #include <chain.h> #include <chainparams.h> #include <checkqueue.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/merkle.h> #include <consensus/tx_check.h> #include <consensus/tx_verify.h> #include <consensus/validation.h> #include <cuckoocache.h> +#include <deploymentstatus.h> #include <flatfile.h> #include <hash.h> #include <index/blockfilterindex.h> -#include <index/txindex.h> #include <logging.h> #include <logging/timer.h> #include <node/blockstorage.h> #include <node/coinstats.h> #include <node/ui_interface.h> +#include <node/utxo_snapshot.h> #include <policy/policy.h> +#include <policy/rbf.h> #include <policy/settings.h> #include <pow.h> #include <primitives/block.h> @@ -47,6 +50,7 @@ #include <util/rbf.h> #include <util/strencodings.h> #include <util/system.h> +#include <util/trace.h> #include <util/translation.h> #include <validationinterface.h> #include <warnings.h> @@ -169,8 +173,6 @@ CBlockIndex* BlockManager::FindForkInGlobalIndex(const CChain& chain, const CBlo return chain.Genesis(); } -std::unique_ptr<CBlockTreeDB> pblocktree; - bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, const CCoinsViewCache& inputs, unsigned int flags, bool cacheSigStore, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, @@ -192,7 +194,7 @@ bool CheckFinalTx(const CBlockIndex* active_chain_tip, const CTransaction &tx, i // CheckFinalTx() uses active_chain_tip.Height()+1 to evaluate // nLockTime because when IsFinalTx() is called within - // CBlock::AcceptBlock(), the height of the block *being* + // AcceptBlock(), the height of the block *being* // evaluated is what is used. Thus if we want to know if a // transaction can be part of the *next* block, we need to call // IsFinalTx() with one more than active_chain_tip.Height(). @@ -328,23 +330,14 @@ static bool IsCurrentForFeeEstimation(CChainState& active_chainstate) EXCLUSIVE_ return true; } -/* Make mempool consistent after a reorg, by re-adding or recursively erasing - * disconnected block transactions from the mempool, and also removing any - * other transactions from the mempool that are no longer valid given the new - * tip/height. - * - * Note: we assume that disconnectpool only contains transactions that are NOT - * confirmed in the current chain nor already in the mempool (otherwise, - * in-mempool descendants of such transactions would be removed). - * - * Passing fAddToMempool=false will skip trying to add the transactions back, - * and instead just erase from the mempool as needed. - */ - -static void UpdateMempoolForReorg(CChainState& active_chainstate, CTxMemPool& mempool, DisconnectedBlockTransactions& disconnectpool, bool fAddToMempool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, mempool.cs) +void CChainState::MaybeUpdateMempoolForReorg( + DisconnectedBlockTransactions& disconnectpool, + bool fAddToMempool) { + if (!m_mempool) return; + AssertLockHeld(cs_main); - AssertLockHeld(mempool.cs); + AssertLockHeld(m_mempool->cs); std::vector<uint256> vHashUpdate; // disconnectpool's insertion_order index sorts the entries from // oldest to newest, but the oldest entry will be the last tx from the @@ -356,11 +349,13 @@ static void UpdateMempoolForReorg(CChainState& active_chainstate, CTxMemPool& me while (it != disconnectpool.queuedTx.get<insertion_order>().rend()) { // ignore validation errors in resurrected transactions if (!fAddToMempool || (*it)->IsCoinBase() || - AcceptToMemoryPool(active_chainstate, mempool, *it, true /* bypass_limits */).m_result_type != MempoolAcceptResult::ResultType::VALID) { + AcceptToMemoryPool( + *this, *m_mempool, *it, true /* bypass_limits */).m_result_type != + MempoolAcceptResult::ResultType::VALID) { // If the transaction doesn't make it in to the mempool, remove any // transactions that depend on it (which would now be orphans). - mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); - } else if (mempool.exists((*it)->GetHash())) { + m_mempool->removeRecursive(**it, MemPoolRemovalReason::REORG); + } else if (m_mempool->exists((*it)->GetHash())) { vHashUpdate.push_back((*it)->GetHash()); } ++it; @@ -371,12 +366,16 @@ static void UpdateMempoolForReorg(CChainState& active_chainstate, CTxMemPool& me // previously-confirmed transactions back to the mempool. // UpdateTransactionsFromBlock finds descendants of any transactions in // the disconnectpool that were added back and cleans up the mempool state. - mempool.UpdateTransactionsFromBlock(vHashUpdate); + m_mempool->UpdateTransactionsFromBlock(vHashUpdate); // We also need to remove any now-immature transactions - mempool.removeForReorg(active_chainstate, STANDARD_LOCKTIME_VERIFY_FLAGS); + m_mempool->removeForReorg(*this, STANDARD_LOCKTIME_VERIFY_FLAGS); // Re-limit mempool size, in case we added any transactions - LimitMempoolSize(mempool, active_chainstate.CoinsTip(), gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, std::chrono::hours{gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); + LimitMempoolSize( + *m_mempool, + this->CoinsTip(), + gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, + std::chrono::hours{gArgs.GetIntArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); } /** @@ -418,7 +417,7 @@ static bool CheckInputsFromMempoolAndCache(const CTransaction& tx, TxValidationS } // Call CheckInputScripts() to cache signature and script validity against current tip consensus rules. - return CheckInputScripts(tx, state, view, flags, /* cacheSigStore = */ true, /* cacheFullSciptStore = */ true, txdata); + return CheckInputScripts(tx, state, view, flags, /* cacheSigStore= */ true, /* cacheFullScriptStore= */ true, txdata); } namespace { @@ -427,10 +426,10 @@ class MemPoolAccept { public: explicit MemPoolAccept(CTxMemPool& mempool, CChainState& active_chainstate) : m_pool(mempool), m_view(&m_dummy), m_viewmempool(&active_chainstate.CoinsTip(), m_pool), m_active_chainstate(active_chainstate), - m_limit_ancestors(gArgs.GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT)), - m_limit_ancestor_size(gArgs.GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT)*1000), - m_limit_descendants(gArgs.GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT)), - m_limit_descendant_size(gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT)*1000) { + m_limit_ancestors(gArgs.GetIntArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT)), + m_limit_ancestor_size(gArgs.GetIntArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT)*1000), + m_limit_descendants(gArgs.GetIntArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT)), + m_limit_descendant_size(gArgs.GetIntArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT)*1000) { } // We put the arguments we're handed into a struct, so we can pass them @@ -475,11 +474,12 @@ private: std::unique_ptr<CTxMemPoolEntry> m_entry; std::list<CTransactionRef> m_replaced_transactions; - bool m_replacement_transaction; CAmount m_base_fees; CAmount m_modified_fees; - CAmount m_conflicting_fees; - size_t m_conflicting_size; + /** Total modified fees of all transactions being replaced. */ + CAmount m_conflicting_fees{0}; + /** Total virtual size of all transactions being replaced. */ + size_t m_conflicting_size{0}; const CTransactionRef& m_ptx; const uint256& m_hash; @@ -510,7 +510,7 @@ private: // Compare a package's feerate against minimum allowed. bool CheckFeeRate(size_t package_size, CAmount package_fee, TxValidationState& state) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs) { - CAmount mempoolRejectFee = m_pool.GetMinFee(gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFee(package_size); + CAmount mempoolRejectFee = m_pool.GetMinFee(gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFee(package_size); if (mempoolRejectFee > 0 && package_fee < mempoolRejectFee) { return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool min fee not met", strprintf("%d < %d", package_fee, mempoolRejectFee)); } @@ -555,7 +555,6 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) CTxMemPool::setEntries& allConflicting = ws.m_all_conflicting; CTxMemPool::setEntries& setAncestors = ws.m_ancestors; std::unique_ptr<CTxMemPoolEntry>& entry = ws.m_entry; - bool& fReplacementTransaction = ws.m_replacement_transaction; CAmount& nModifiedFees = ws.m_modified_fees; CAmount& nConflictingFees = ws.m_conflicting_fees; size_t& nConflictingSize = ws.m_conflicting_size; @@ -586,9 +585,13 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) if (!CheckFinalTx(m_active_chainstate.m_chain.Tip(), tx, STANDARD_LOCKTIME_VERIFY_FLAGS)) return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-final"); - // is it already in the memory pool? - if (m_pool.exists(hash)) { + if (m_pool.exists(GenTxid(true, tx.GetWitnessHash()))) { + // Exact transaction already exists in the mempool. return state.Invalid(TxValidationResult::TX_CONFLICT, "txn-already-in-mempool"); + } else if (m_pool.exists(GenTxid(false, tx.GetHash()))) { + // Transaction with the same non-witness data but different witness (same txid, different + // wtxid) already exists in the mempool. + return state.Invalid(TxValidationResult::TX_CONFLICT, "txn-same-nonwitness-data-in-mempool"); } // Check for conflicts with in-memory transactions @@ -602,14 +605,6 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) } if (!setConflicts.count(ptxConflicting->GetHash())) { - // Allow opt-out of transaction replacement by setting - // nSequence > MAX_BIP125_RBF_SEQUENCE (SEQUENCE_FINAL-2) on all inputs. - // - // SEQUENCE_FINAL-1 is picked to still allow use of nLockTime by - // non-replaceable transactions. All inputs rather than just one - // is for the sake of multi-party protocols, where we don't - // want a single party to be able to disable replacement. - // // Transactions that don't explicitly signal replaceability are // *not* replaceable with the current logic, even if one of their // unconfirmed ancestors signals replaceability. This diverges @@ -617,16 +612,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // Applications relying on first-seen mempool behavior should // check all unconfirmed ancestors; otherwise an opt-in ancestor // might be replaced, causing removal of this descendant. - bool fReplacementOptOut = true; - for (const CTxIn &_txin : ptxConflicting->vin) - { - if (_txin.nSequence <= MAX_BIP125_RBF_SEQUENCE) - { - fReplacementOptOut = false; - break; - } - } - if (fReplacementOptOut) { + if (!SignalsOptInRBF(*ptxConflicting)) { return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "txn-mempool-conflict"); } @@ -683,9 +669,8 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) } // Check for non-standard pay-to-script-hash in inputs - const auto& params = args.m_chainparams.GetConsensus(); - auto taproot_state = VersionBitsState(m_active_chainstate.m_chain.Tip(), params, Consensus::DEPLOYMENT_TAPROOT, versionbitscache); - if (fRequireStandard && !AreInputsStandard(tx, m_view, taproot_state == ThresholdState::ACTIVE)) { + const bool taproot_active = DeploymentActiveAfter(m_active_chainstate.m_chain.Tip(), args.m_chainparams.GetConsensus(), Consensus::DEPLOYMENT_TAPROOT); + if (fRequireStandard && !AreInputsStandard(tx, m_view, taproot_active)) { return state.Invalid(TxValidationResult::TX_INPUTS_NOT_STANDARD, "bad-txns-nonstandard-inputs"); } @@ -785,130 +770,44 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // that we have the set of all ancestors we can detect this // pathological case by making sure setConflicts and setAncestors don't // intersect. - for (CTxMemPool::txiter ancestorIt : setAncestors) - { - const uint256 &hashAncestor = ancestorIt->GetTx().GetHash(); - if (setConflicts.count(hashAncestor)) - { - return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-spends-conflicting-tx", - strprintf("%s spends conflicting transaction %s", - hash.ToString(), - hashAncestor.ToString())); - } + if (const auto err_string{EntriesAndTxidsDisjoint(setAncestors, setConflicts, hash)}) { + // We classify this as a consensus error because a transaction depending on something it + // conflicts with would be inconsistent. + return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-spends-conflicting-tx", *err_string); } - // Check if it's economically rational to mine this transaction rather - // than the ones it replaces. - nConflictingFees = 0; - nConflictingSize = 0; - uint64_t nConflictingCount = 0; - // If we don't hold the lock allConflicting might be incomplete; the - // subsequent RemoveStaged() and addUnchecked() calls don't guarantee - // mempool consistency for us. - fReplacementTransaction = setConflicts.size(); - if (fReplacementTransaction) - { + if (!setConflicts.empty()) { CFeeRate newFeeRate(nModifiedFees, nSize); - std::set<uint256> setConflictsParents; - const int maxDescendantsToVisit = 100; - for (const auto& mi : setIterConflicting) { - // Don't allow the replacement to reduce the feerate of the - // mempool. - // - // We usually don't want to accept replacements with lower - // feerates than what they replaced as that would lower the - // feerate of the next block. Requiring that the feerate always - // be increased is also an easy-to-reason about way to prevent - // DoS attacks via replacements. - // - // We only consider the feerates of transactions being directly - // replaced, not their indirect descendants. While that does - // mean high feerate children are ignored when deciding whether - // or not to replace, we do require the replacement to pay more - // overall fees too, mitigating most cases. - CFeeRate oldFeeRate(mi->GetModifiedFee(), mi->GetTxSize()); - if (newFeeRate <= oldFeeRate) - { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", - strprintf("rejecting replacement %s; new feerate %s <= old feerate %s", - hash.ToString(), - newFeeRate.ToString(), - oldFeeRate.ToString())); - } - - for (const CTxIn &txin : mi->GetTx().vin) - { - setConflictsParents.insert(txin.prevout.hash); - } - - nConflictingCount += mi->GetCountWithDescendants(); - } - // This potentially overestimates the number of actual descendants - // but we just want to be conservative to avoid doing too much - // work. - if (nConflictingCount <= maxDescendantsToVisit) { - // If not too many to replace, then calculate the set of - // transactions that would have to be evicted - for (CTxMemPool::txiter it : setIterConflicting) { - m_pool.CalculateDescendants(it, allConflicting); - } - for (CTxMemPool::txiter it : allConflicting) { - nConflictingFees += it->GetModifiedFee(); - nConflictingSize += it->GetTxSize(); - } - } else { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "too many potential replacements", - strprintf("rejecting replacement %s; too many potential replacements (%d > %d)\n", - hash.ToString(), - nConflictingCount, - maxDescendantsToVisit)); + // It's possible that the replacement pays more fees than its direct conflicts but not more + // than all conflicts (i.e. the direct conflicts have high-fee descendants). However, if the + // replacement doesn't pay more fees than its direct conflicts, then we can be sure it's not + // more economically rational to mine. Before we go digging through the mempool for all + // transactions that would need to be removed (direct conflicts and all descendants), check + // that the replacement transaction pays more than its direct conflicts. + if (const auto err_string{PaysMoreThanConflicts(setIterConflicting, newFeeRate, hash)}) { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", *err_string); } - for (unsigned int j = 0; j < tx.vin.size(); j++) - { - // We don't want to accept replacements that require low - // feerate junk to be mined first. Ideally we'd keep track of - // the ancestor feerates and make the decision based on that, - // but for now requiring all new inputs to be confirmed works. - // - // Note that if you relax this to make RBF a little more useful, - // this may break the CalculateMempoolAncestors RBF relaxation, - // above. See the comment above the first CalculateMempoolAncestors - // call for more info. - if (!setConflictsParents.count(tx.vin[j].prevout.hash)) - { - // Rather than check the UTXO set - potentially expensive - - // it's cheaper to just check if the new input refers to a - // tx that's in the mempool. - if (m_pool.exists(tx.vin[j].prevout.hash)) { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "replacement-adds-unconfirmed", - strprintf("replacement %s adds unconfirmed input, idx %d", - hash.ToString(), j)); - } - } + // Calculate all conflicting entries and enforce BIP125 Rule #5. + if (const auto err_string{GetEntriesForConflicts(tx, m_pool, setIterConflicting, allConflicting)}) { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, + "too many potential replacements", *err_string); } - - // The replacement must pay greater fees than the transactions it - // replaces - if we did the bandwidth used by those conflicting - // transactions would not be paid for. - if (nModifiedFees < nConflictingFees) - { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", - strprintf("rejecting replacement %s, less fees than conflicting txs; %s < %s", - hash.ToString(), FormatMoney(nModifiedFees), FormatMoney(nConflictingFees))); + // Enforce BIP125 Rule #2. + if (const auto err_string{HasNoNewUnconfirmed(tx, m_pool, setIterConflicting)}) { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, + "replacement-adds-unconfirmed", *err_string); } - // Finally in addition to paying more fees than the conflicts the - // new transaction must pay for its own bandwidth. - CAmount nDeltaFees = nModifiedFees - nConflictingFees; - if (nDeltaFees < ::incrementalRelayFee.GetFee(nSize)) - { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", - strprintf("rejecting replacement %s, not enough additional fees to relay; %s < %s", - hash.ToString(), - FormatMoney(nDeltaFees), - FormatMoney(::incrementalRelayFee.GetFee(nSize)))); + // Check if it's economically rational to mine this transaction rather than the ones it + // replaces and pays for its own relay fees. Enforce BIP125 Rules #3 and #4. + for (CTxMemPool::txiter it : allConflicting) { + nConflictingFees += it->GetModifiedFee(); + nConflictingSize += it->GetTxSize(); + } + if (const auto err_string{PaysForRBF(nConflictingFees, nModifiedFees, nSize, ::incrementalRelayFee, hash)}) { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", *err_string); } } return true; @@ -983,7 +882,6 @@ bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) const CAmount& nModifiedFees = ws.m_modified_fees; const CAmount& nConflictingFees = ws.m_conflicting_fees; const size_t& nConflictingSize = ws.m_conflicting_size; - const bool fReplacementTransaction = ws.m_replacement_transaction; std::unique_ptr<CTxMemPoolEntry>& entry = ws.m_entry; // Remove conflicting transactions from the mempool @@ -999,18 +897,17 @@ bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) m_pool.RemoveStaged(allConflicting, false, MemPoolRemovalReason::REPLACED); // This transaction should only count for fee estimation if: - // - it isn't a BIP 125 replacement transaction (may not be widely supported) // - it's not being re-added during a reorg which bypasses typical mempool fee limits // - the node is not behind // - the transaction is not dependent on any other transactions in the mempool - bool validForFeeEstimation = !fReplacementTransaction && !bypass_limits && IsCurrentForFeeEstimation(m_active_chainstate) && m_pool.HasNoInputsOf(tx); + bool validForFeeEstimation = !bypass_limits && IsCurrentForFeeEstimation(m_active_chainstate) && m_pool.HasNoInputsOf(tx); // Store transaction in memory m_pool.addUnchecked(*entry, setAncestors, validForFeeEstimation); // trim mempool and check if tx was trimmed if (!bypass_limits) { - LimitMempoolSize(m_pool, m_active_chainstate.CoinsTip(), gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, std::chrono::hours{gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); + LimitMempoolSize(m_pool, m_active_chainstate.CoinsTip(), gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, std::chrono::hours{gArgs.GetIntArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); if (!m_pool.exists(hash)) return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool full"); } @@ -1080,6 +977,19 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: m_viewmempool.PackageAddTransaction(ws.m_ptx); } + // Apply package mempool ancestor/descendant limits. Skip if there is only one transaction, + // because it's unnecessary. Also, CPFP carve out can increase the limit for individual + // transactions, but this exemption is not extended to packages in CheckPackageLimits(). + std::string err_string; + if (txns.size() > 1 && + !m_pool.CheckPackageLimits(txns, m_limit_ancestors, m_limit_ancestor_size, m_limit_descendants, + m_limit_descendant_size, err_string)) { + // All transactions must have individually passed mempool ancestor and descendant limits + // inside of PreChecks(), so this is separate from an individual transaction error. + package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package-mempool-limits", err_string); + return PackageMempoolAcceptResult(package_state, std::move(results)); + } + for (Workspace& ws : workspaces) { PrecomputedTransactionData txdata; if (!PolicyScriptChecks(args, ws, txdata)) { @@ -1124,7 +1034,7 @@ static MempoolAcceptResult AcceptToMemoryPoolWithTime(const CChainParams& chainp } // After we've (potentially) uncached entries, ensure our coins cache is still within its size limits BlockValidationState state_dummy; - active_chainstate.FlushStateToDisk(chainparams, state_dummy, FlushStateMode::PERIODIC); + active_chainstate.FlushStateToDisk(state_dummy, FlushStateMode::PERIODIC); return result; } @@ -1155,33 +1065,6 @@ PackageMempoolAcceptResult ProcessNewPackage(CChainState& active_chainstate, CTx return result; } -CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock) -{ - LOCK(cs_main); - - if (block_index) { - CBlock block; - if (ReadBlockFromDisk(block, block_index, consensusParams)) { - for (const auto& tx : block.vtx) { - if (tx->GetHash() == hash) { - hashBlock = block_index->GetBlockHash(); - return tx; - } - } - } - return nullptr; - } - if (mempool) { - CTransactionRef ptx = mempool->get(hash); - if (ptx) return ptx; - } - if (g_txindex) { - CTransactionRef tx; - if (g_txindex->FindTx(hash, hashBlock, tx)) return tx; - } - return nullptr; -} - CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams) { int halvings = nHeight / consensusParams.nSubsidyHalvingInterval; @@ -1208,9 +1091,15 @@ void CoinsViews::InitCache() m_cacheview = std::make_unique<CCoinsViewCache>(&m_catcherview); } -CChainState::CChainState(CTxMemPool& mempool, BlockManager& blockman, std::optional<uint256> from_snapshot_blockhash) +CChainState::CChainState( + CTxMemPool* mempool, + BlockManager& blockman, + ChainstateManager& chainman, + std::optional<uint256> from_snapshot_blockhash) : m_mempool(mempool), + m_params(::Params()), m_blockman(blockman), + m_chainman(chainman), m_from_snapshot_blockhash(from_snapshot_blockhash) {} void CChainState::InitCoinsDB( @@ -1380,7 +1269,7 @@ void InitScriptExecutionCache() { g_scriptExecutionCacheHasher.Write(nonce.begin(), 32); // nMaxCacheSize is unsigned. If -maxsigcachesize is set to zero, // setup_bytes creates the minimum possible cache (2 elements). - size_t nMaxCacheSize = std::min(std::max((int64_t)0, gArgs.GetArg("-maxsigcachesize", DEFAULT_MAX_SIG_CACHE_SIZE) / 2), MAX_MAX_SIG_CACHE_SIZE) * ((size_t) 1 << 20); + size_t nMaxCacheSize = std::min(std::max((int64_t)0, gArgs.GetIntArg("-maxsigcachesize", DEFAULT_MAX_SIG_CACHE_SIZE) / 2), MAX_MAX_SIG_CACHE_SIZE) * ((size_t) 1 << 20); size_t nElems = g_scriptExecutionCache.setup_bytes(nMaxCacheSize); LogPrintf("Using %zu MiB out of %zu/2 requested for script execution cache, able to store %zu elements\n", (nElems*sizeof(uint256)) >>20, (nMaxCacheSize*2)>>20, nElems); @@ -1605,23 +1494,6 @@ void StopScriptCheckWorkerThreads() scriptcheckqueue.StopWorkerThreads(); } -VersionBitsCache versionbitscache GUARDED_BY(cs_main); - -int32_t ComputeBlockVersion(const CBlockIndex* pindexPrev, const Consensus::Params& params) -{ - LOCK(cs_main); - int32_t nVersion = VERSIONBITS_TOP_BITS; - - for (int i = 0; i < (int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; i++) { - ThresholdState state = VersionBitsState(pindexPrev, params, static_cast<Consensus::DeploymentPos>(i), versionbitscache); - if (state == ThresholdState::LOCKED_IN || state == ThresholdState::STARTED) { - nVersion |= VersionBitsMask(params, static_cast<Consensus::DeploymentPos>(i)); - } - } - - return nVersion; -} - /** * Threshold condition checker that triggers when unknown versionbits are seen on the network. */ @@ -1643,24 +1515,14 @@ public: return pindex->nHeight >= params.MinBIP9WarningHeight && ((pindex->nVersion & VERSIONBITS_TOP_MASK) == VERSIONBITS_TOP_BITS) && ((pindex->nVersion >> bit) & 1) != 0 && - ((ComputeBlockVersion(pindex->pprev, params) >> bit) & 1) == 0; + ((g_versionbitscache.ComputeBlockVersion(pindex->pprev, params) >> bit) & 1) == 0; } }; static ThresholdConditionCache warningcache[VERSIONBITS_NUM_BITS] GUARDED_BY(cs_main); -// 0.13.0 was shipped with a segwit deployment defined for testnet, but not for -// mainnet. We no longer need to support disabling the segwit deployment -// except for testing purposes, due to limitations of the functional test -// environment. See test/functional/p2p-segwit.py. -static bool IsScriptWitnessEnabled(const Consensus::Params& params) +static unsigned int GetBlockScriptFlags(const CBlockIndex* pindex, const Consensus::Params& consensusparams) { - return params.SegwitHeight != std::numeric_limits<int>::max(); -} - -static unsigned int GetBlockScriptFlags(const CBlockIndex* pindex, const Consensus::Params& consensusparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - AssertLockHeld(cs_main); - unsigned int flags = SCRIPT_VERIFY_NONE; // BIP16 didn't become active until Apr 1 2012 (on mainnet, and @@ -1672,37 +1534,32 @@ static unsigned int GetBlockScriptFlags(const CBlockIndex* pindex, const Consens pindex->phashBlock == nullptr || // this is a new candidate block, eg from TestBlockValidity() *pindex->phashBlock != consensusparams.BIP16Exception) // this block isn't the historical exception { - flags |= SCRIPT_VERIFY_P2SH; - } - - // Enforce WITNESS rules whenever P2SH is in effect (and the segwit - // deployment is defined). - if (flags & SCRIPT_VERIFY_P2SH && IsScriptWitnessEnabled(consensusparams)) { - flags |= SCRIPT_VERIFY_WITNESS; + // Enforce WITNESS rules whenever P2SH is in effect + flags |= SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_WITNESS; } - // Start enforcing the DERSIG (BIP66) rule - if (pindex->nHeight >= consensusparams.BIP66Height) { + // Enforce the DERSIG (BIP66) rule + if (DeploymentActiveAt(*pindex, consensusparams, Consensus::DEPLOYMENT_DERSIG)) { flags |= SCRIPT_VERIFY_DERSIG; } - // Start enforcing CHECKLOCKTIMEVERIFY (BIP65) rule - if (pindex->nHeight >= consensusparams.BIP65Height) { + // Enforce CHECKLOCKTIMEVERIFY (BIP65) + if (DeploymentActiveAt(*pindex, consensusparams, Consensus::DEPLOYMENT_CLTV)) { flags |= SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY; } - // Start enforcing BIP112 (CHECKSEQUENCEVERIFY) - if (pindex->nHeight >= consensusparams.CSVHeight) { + // Enforce CHECKSEQUENCEVERIFY (BIP112) + if (DeploymentActiveAt(*pindex, consensusparams, Consensus::DEPLOYMENT_CSV)) { flags |= SCRIPT_VERIFY_CHECKSEQUENCEVERIFY; } - // Start enforcing Taproot using versionbits logic. - if (VersionBitsState(pindex->pprev, consensusparams, Consensus::DEPLOYMENT_TAPROOT, versionbitscache) == ThresholdState::ACTIVE) { + // Enforce Taproot (BIP340-BIP342) + if (DeploymentActiveAt(*pindex, consensusparams, Consensus::DEPLOYMENT_TAPROOT)) { flags |= SCRIPT_VERIFY_TAPROOT; } - // Start enforcing BIP147 NULLDUMMY (activated simultaneously with segwit) - if (IsWitnessEnabled(pindex->pprev, consensusparams)) { + // Enforce BIP147 NULLDUMMY (activated simultaneously with segwit) + if (DeploymentActiveAt(*pindex, consensusparams, Consensus::DEPLOYMENT_SEGWIT)) { flags |= SCRIPT_VERIFY_NULLDUMMY; } @@ -1716,7 +1573,6 @@ static int64_t nTimeForks = 0; static int64_t nTimeVerify = 0; static int64_t nTimeConnect = 0; static int64_t nTimeIndex = 0; -static int64_t nTimeCallbacks = 0; static int64_t nTimeTotal = 0; static int64_t nBlocksTotal = 0; @@ -1724,7 +1580,7 @@ static int64_t nBlocksTotal = 0; * Validity checks that depend on the UTXO set are also done; ConnectBlock() * can fail if those validity checks fail (among other reasons). */ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, CBlockIndex* pindex, - CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck) + CCoinsViewCache& view, bool fJustCheck) { AssertLockHeld(cs_main); assert(pindex); @@ -1744,7 +1600,7 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, // is enforced in ContextualCheckBlockHeader(); we wouldn't want to // re-enforce that rule here (at least until we make it impossible for // GetAdjustedTime() to go backward). - if (!CheckBlock(block, state, chainparams.GetConsensus(), !fJustCheck, !fJustCheck)) { + if (!CheckBlock(block, state, m_params.GetConsensus(), !fJustCheck, !fJustCheck)) { if (state.GetResult() == BlockValidationResult::BLOCK_MUTATED) { // We don't write down blocks to disk if they may have been // corrupted, so this should be impossible unless we're having hardware @@ -1762,7 +1618,7 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, // Special case for the genesis block, skipping connection of its transactions // (its coinbase is unspendable) - if (block.GetHash() == chainparams.GetConsensus().hashGenesisBlock) { + if (block.GetHash() == m_params.GetConsensus().hashGenesisBlock) { if (!fJustCheck) view.SetBestBlock(pindex->GetBlockHash()); return true; @@ -1794,7 +1650,7 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, // artificially set the default assumed verified block further back. // The test against nMinimumChainWork prevents the skipping when denied access to any chain at // least as good as the expected chain. - fScriptChecks = (GetBlockProofEquivalentTime(*pindexBestHeader, *pindex, *pindexBestHeader, chainparams.GetConsensus()) <= 60 * 60 * 24 * 7 * 2); + fScriptChecks = (GetBlockProofEquivalentTime(*pindexBestHeader, *pindex, *pindexBestHeader, m_params.GetConsensus()) <= 60 * 60 * 24 * 7 * 2); } } } @@ -1874,9 +1730,9 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, // be reset before it reaches block 1,983,702 and starts doing unnecessary // BIP30 checking again. assert(pindex->pprev); - CBlockIndex *pindexBIP34height = pindex->pprev->GetAncestor(chainparams.GetConsensus().BIP34Height); + CBlockIndex* pindexBIP34height = pindex->pprev->GetAncestor(m_params.GetConsensus().BIP34Height); //Only continue to enforce if we're below BIP34 activation height or the block hash at that height doesn't correspond. - fEnforceBIP30 = fEnforceBIP30 && (!pindexBIP34height || !(pindexBIP34height->GetBlockHash() == chainparams.GetConsensus().BIP34Hash)); + fEnforceBIP30 = fEnforceBIP30 && (!pindexBIP34height || !(pindexBIP34height->GetBlockHash() == m_params.GetConsensus().BIP34Hash)); // TODO: Remove BIP30 checking from block height 1,983,702 on, once we have a // consensus change that ensures coinbases at those heights can not @@ -1892,14 +1748,14 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, } } - // Start enforcing BIP68 (sequence locks) + // Enforce BIP68 (sequence locks) int nLockTimeFlags = 0; - if (pindex->nHeight >= chainparams.GetConsensus().CSVHeight) { + if (DeploymentActiveAt(*pindex, m_params.GetConsensus(), Consensus::DEPLOYMENT_CSV)) { nLockTimeFlags |= LOCKTIME_VERIFY_SEQUENCE; } // Get the script flags for this block - unsigned int flags = GetBlockScriptFlags(pindex, chainparams.GetConsensus()); + unsigned int flags = GetBlockScriptFlags(pindex, m_params.GetConsensus()); int64_t nTime2 = GetTimeMicros(); nTimeForks += nTime2 - nTime1; LogPrint(BCLog::BENCH, " - Fork checks: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime2 - nTime1), nTimeForks * MICRO, nTimeForks * MILLI / nBlocksTotal); @@ -1989,7 +1845,7 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, int64_t nTime3 = GetTimeMicros(); nTimeConnect += nTime3 - nTime2; LogPrint(BCLog::BENCH, " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs (%.2fms/blk)]\n", (unsigned)block.vtx.size(), MILLI * (nTime3 - nTime2), MILLI * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : MILLI * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * MICRO, nTimeConnect * MILLI / nBlocksTotal); - CAmount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, chainparams.GetConsensus()); + CAmount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, m_params.GetConsensus()); if (block.vtx[0]->GetValueOut() > blockReward) { LogPrintf("ERROR: ConnectBlock(): coinbase pays too much (actual=%d vs limit=%d)\n", block.vtx[0]->GetValueOut(), blockReward); return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-cb-amount"); @@ -2005,8 +1861,9 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, if (fJustCheck) return true; - if (!WriteUndoDataForBlock(blockundo, state, pindex, chainparams)) + if (!WriteUndoDataForBlock(blockundo, state, pindex, m_params)) { return false; + } if (!pindex->IsValid(BLOCK_VALID_SCRIPTS)) { pindex->RaiseValidity(BLOCK_VALID_SCRIPTS); @@ -2020,26 +1877,31 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, int64_t nTime5 = GetTimeMicros(); nTimeIndex += nTime5 - nTime4; LogPrint(BCLog::BENCH, " - Index writing: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime5 - nTime4), nTimeIndex * MICRO, nTimeIndex * MILLI / nBlocksTotal); - int64_t nTime6 = GetTimeMicros(); nTimeCallbacks += nTime6 - nTime5; - LogPrint(BCLog::BENCH, " - Callbacks: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime6 - nTime5), nTimeCallbacks * MICRO, nTimeCallbacks * MILLI / nBlocksTotal); + TRACE7(validation, block_connected, + block.GetHash().ToString().c_str(), + pindex->nHeight, + block.vtx.size(), + nInputs, + nSigOpsCost, + GetTimeMicros() - nTimeStart, // in microseconds (µs) + block.GetHash().data() + ); return true; } -CoinsCacheSizeState CChainState::GetCoinsCacheSizeState(const CTxMemPool* tx_pool) +CoinsCacheSizeState CChainState::GetCoinsCacheSizeState() { return this->GetCoinsCacheSizeState( - tx_pool, m_coinstip_cache_size_bytes, - gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); + gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); } CoinsCacheSizeState CChainState::GetCoinsCacheSizeState( - const CTxMemPool* tx_pool, size_t max_coins_cache_size_bytes, size_t max_mempool_size_bytes) { - const int64_t nMempoolUsage = tx_pool ? tx_pool->DynamicMemoryUsage() : 0; + const int64_t nMempoolUsage = m_mempool ? m_mempool->DynamicMemoryUsage() : 0; int64_t cacheSize = CoinsTip().DynamicMemoryUsage(); int64_t nTotalSpace = max_coins_cache_size_bytes + std::max<int64_t>(max_mempool_size_bytes - nMempoolUsage, 0); @@ -2059,7 +1921,6 @@ CoinsCacheSizeState CChainState::GetCoinsCacheSizeState( } bool CChainState::FlushStateToDisk( - const CChainParams& chainparams, BlockValidationState &state, FlushStateMode mode, int nManualPruneHeight) @@ -2079,7 +1940,7 @@ bool CChainState::FlushStateToDisk( bool fFlushForPrune = false; bool fDoFullFlush = false; - CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(&m_mempool); + CoinsCacheSizeState cache_state = GetCoinsCacheSizeState(); LOCK(cs_LastBlockFile); if (fPruneMode && (fCheckForPruning || nManualPruneHeight > 0) && !fReindex) { // make sure we don't prune above the blockfilterindexes bestblocks @@ -2096,13 +1957,13 @@ bool CChainState::FlushStateToDisk( } else { LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune", BCLog::BENCH); - m_blockman.FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight(), m_chain.Height(), last_prune, IsInitialBlockDownload()); + m_blockman.FindFilesToPrune(setFilesToPrune, m_params.PruneAfterHeight(), m_chain.Height(), last_prune, IsInitialBlockDownload()); fCheckForPruning = false; } if (!setFilesToPrune.empty()) { fFlushForPrune = true; if (!fHavePruned) { - pblocktree->WriteFlag("prunedblockfiles", true); + m_blockman.m_block_tree_db->WriteFlag("prunedblockfiles", true); fHavePruned = true; } } @@ -2154,7 +2015,7 @@ bool CChainState::FlushStateToDisk( vBlocks.push_back(*it); setDirtyBlockIndex.erase(it++); } - if (!pblocktree->WriteBatchSync(vFiles, nLastBlockFile, vBlocks)) { + if (!m_blockman.m_block_tree_db->WriteBatchSync(vFiles, nLastBlockFile, vBlocks)) { return AbortNode(state, "Failed to write to block index database"); } } @@ -2196,20 +2057,19 @@ bool CChainState::FlushStateToDisk( return true; } -void CChainState::ForceFlushStateToDisk() { +void CChainState::ForceFlushStateToDisk() +{ BlockValidationState state; - const CChainParams& chainparams = Params(); - if (!this->FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS)) { + if (!this->FlushStateToDisk(state, FlushStateMode::ALWAYS)) { LogPrintf("%s: failed to flush state (%s)\n", __func__, state.ToString()); } } -void CChainState::PruneAndFlush() { +void CChainState::PruneAndFlush() +{ BlockValidationState state; fCheckForPruning = true; - const CChainParams& chainparams = Params(); - - if (!this->FlushStateToDisk(chainparams, state, FlushStateMode::NONE)) { + if (!this->FlushStateToDisk(state, FlushStateMode::NONE)) { LogPrintf("%s: failed to flush state (%s)\n", __func__, state.ToString()); } } @@ -2231,12 +2091,46 @@ static void AppendWarning(bilingual_str& res, const bilingual_str& warn) res += warn; } -/** Check warning conditions and do some notifications on new chain tip set. */ -static void UpdateTip(CTxMemPool& mempool, const CBlockIndex* pindexNew, const CChainParams& chainParams, CChainState& active_chainstate) - EXCLUSIVE_LOCKS_REQUIRED(::cs_main) +static void UpdateTipLog( + const CCoinsViewCache& coins_tip, + const CBlockIndex* tip, + const CChainParams& params, + const std::string& func_name, + const std::string& prefix, + const std::string& warning_messages) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { + + AssertLockHeld(::cs_main); + LogPrintf("%s%s: new best=%s height=%d version=0x%08x log2_work=%f tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)%s\n", + prefix, func_name, + tip->GetBlockHash().ToString(), tip->nHeight, tip->nVersion, + log(tip->nChainWork.getdouble()) / log(2.0), (unsigned long)tip->nChainTx, + FormatISO8601DateTime(tip->GetBlockTime()), + GuessVerificationProgress(params.TxData(), tip), + coins_tip.DynamicMemoryUsage() * (1.0 / (1 << 20)), + coins_tip.GetCacheSize(), + !warning_messages.empty() ? strprintf(" warning='%s'", warning_messages) : ""); +} + +void CChainState::UpdateTip(const CBlockIndex* pindexNew) +{ + const auto& coins_tip = this->CoinsTip(); + + // The remainder of the function isn't relevant if we are not acting on + // the active chainstate, so return if need be. + if (this != &m_chainman.ActiveChainstate()) { + // Only log every so often so that we don't bury log messages at the tip. + constexpr int BACKGROUND_LOG_INTERVAL = 2000; + if (pindexNew->nHeight % BACKGROUND_LOG_INTERVAL == 0) { + UpdateTipLog(coins_tip, pindexNew, m_params, __func__, "[background validation] ", ""); + } + return; + } + // New best block - mempool.AddTransactionsUpdated(1); + if (m_mempool) { + m_mempool->AddTransactionsUpdated(1); + } { LOCK(g_best_block_mutex); @@ -2245,11 +2139,11 @@ static void UpdateTip(CTxMemPool& mempool, const CBlockIndex* pindexNew, const C } bilingual_str warning_messages; - if (!active_chainstate.IsInitialBlockDownload()) { + if (!this->IsInitialBlockDownload()) { const CBlockIndex* pindex = pindexNew; for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { WarningBitsConditionChecker checker(bit); - ThresholdState state = checker.GetStateFor(pindex, chainParams.GetConsensus(), warningcache[bit]); + ThresholdState state = checker.GetStateFor(pindex, m_params.GetConsensus(), warningcache[bit]); if (state == ThresholdState::ACTIVE || state == ThresholdState::LOCKED_IN) { const bilingual_str warning = strprintf(_("Unknown new rules activated (versionbit %i)"), bit); if (state == ThresholdState::ACTIVE) { @@ -2260,36 +2154,32 @@ static void UpdateTip(CTxMemPool& mempool, const CBlockIndex* pindexNew, const C } } } - LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%f tx=%lu date='%s' progress=%f cache=%.1fMiB(%utxo)%s\n", __func__, - pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, pindexNew->nVersion, - log(pindexNew->nChainWork.getdouble())/log(2.0), (unsigned long)pindexNew->nChainTx, - FormatISO8601DateTime(pindexNew->GetBlockTime()), - GuessVerificationProgress(chainParams.TxData(), pindexNew), active_chainstate.CoinsTip().DynamicMemoryUsage() * (1.0 / (1<<20)), active_chainstate.CoinsTip().GetCacheSize(), - !warning_messages.empty() ? strprintf(" warning='%s'", warning_messages.original) : ""); + UpdateTipLog(coins_tip, pindexNew, m_params, __func__, "", warning_messages.original); } /** Disconnect m_chain's tip. * After calling, the mempool will be in an inconsistent state, with * transactions from disconnected blocks being added to disconnectpool. You - * should make the mempool consistent again by calling UpdateMempoolForReorg. + * should make the mempool consistent again by calling MaybeUpdateMempoolForReorg. * with cs_main held. * * If disconnectpool is nullptr, then no disconnected transactions are added to * disconnectpool (note that the caller is responsible for mempool consistency * in any case). */ -bool CChainState::DisconnectTip(BlockValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions* disconnectpool) +bool CChainState::DisconnectTip(BlockValidationState& state, DisconnectedBlockTransactions* disconnectpool) { AssertLockHeld(cs_main); - AssertLockHeld(m_mempool.cs); + if (m_mempool) AssertLockHeld(m_mempool->cs); CBlockIndex *pindexDelete = m_chain.Tip(); assert(pindexDelete); // Read block from disk. std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); CBlock& block = *pblock; - if (!ReadBlockFromDisk(block, pindexDelete, chainparams.GetConsensus())) + if (!ReadBlockFromDisk(block, pindexDelete, m_params.GetConsensus())) { return error("DisconnectTip(): Failed to read block"); + } // Apply the block atomically to the chain state. int64_t nStart = GetTimeMicros(); { @@ -2302,10 +2192,11 @@ bool CChainState::DisconnectTip(BlockValidationState& state, const CChainParams& } LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * MILLI); // Write the chain state to disk, if necessary. - if (!FlushStateToDisk(chainparams, state, FlushStateMode::IF_NEEDED)) + if (!FlushStateToDisk(state, FlushStateMode::IF_NEEDED)) { return false; + } - if (disconnectpool) { + if (disconnectpool && m_mempool) { // Save transactions to re-add to mempool at end of reorg for (auto it = block.vtx.rbegin(); it != block.vtx.rend(); ++it) { disconnectpool->addTransaction(*it); @@ -2313,14 +2204,14 @@ bool CChainState::DisconnectTip(BlockValidationState& state, const CChainParams& while (disconnectpool->DynamicMemoryUsage() > MAX_DISCONNECTED_TX_POOL_SIZE * 1000) { // Drop the earliest entry, and remove its children from the mempool. auto it = disconnectpool->queuedTx.get<insertion_order>().begin(); - m_mempool.removeRecursive(**it, MemPoolRemovalReason::REORG); + m_mempool->removeRecursive(**it, MemPoolRemovalReason::REORG); disconnectpool->removeEntry(it); } } m_chain.SetTip(pindexDelete->pprev); - UpdateTip(m_mempool, pindexDelete->pprev, chainparams, *this); + UpdateTip(pindexDelete->pprev); // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: GetMainSignals().BlockDisconnected(pblock, pindexDelete); @@ -2379,10 +2270,10 @@ public: * * The block is added to connectTrace if connection succeeds. */ -bool CChainState::ConnectTip(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions &disconnectpool) +bool CChainState::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions& disconnectpool) { AssertLockHeld(cs_main); - AssertLockHeld(m_mempool.cs); + if (m_mempool) AssertLockHeld(m_mempool->cs); assert(pindexNew->pprev == m_chain.Tip()); // Read block from disk. @@ -2390,8 +2281,9 @@ bool CChainState::ConnectTip(BlockValidationState& state, const CChainParams& ch std::shared_ptr<const CBlock> pthisBlock; if (!pblock) { std::shared_ptr<CBlock> pblockNew = std::make_shared<CBlock>(); - if (!ReadBlockFromDisk(*pblockNew, pindexNew, chainparams.GetConsensus())) + if (!ReadBlockFromDisk(*pblockNew, pindexNew, m_params.GetConsensus())) { return AbortNode(state, "Failed to read block"); + } pthisBlock = pblockNew; } else { pthisBlock = pblock; @@ -2403,7 +2295,7 @@ bool CChainState::ConnectTip(BlockValidationState& state, const CChainParams& ch LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * MILLI, nTimeReadFromDisk * MICRO); { CCoinsViewCache view(&CoinsTip()); - bool rv = ConnectBlock(blockConnecting, state, pindexNew, view, chainparams); + bool rv = ConnectBlock(blockConnecting, state, pindexNew, view); GetMainSignals().BlockChecked(blockConnecting, state); if (!rv) { if (state.IsInvalid()) @@ -2419,16 +2311,19 @@ bool CChainState::ConnectTip(BlockValidationState& state, const CChainParams& ch int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3; LogPrint(BCLog::BENCH, " - Flush: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime4 - nTime3) * MILLI, nTimeFlush * MICRO, nTimeFlush * MILLI / nBlocksTotal); // Write the chain state to disk, if necessary. - if (!FlushStateToDisk(chainparams, state, FlushStateMode::IF_NEEDED)) + if (!FlushStateToDisk(state, FlushStateMode::IF_NEEDED)) { return false; + } int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; LogPrint(BCLog::BENCH, " - Writing chainstate: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime5 - nTime4) * MILLI, nTimeChainState * MICRO, nTimeChainState * MILLI / nBlocksTotal); // Remove conflicting transactions from the mempool.; - m_mempool.removeForBlock(blockConnecting.vtx, pindexNew->nHeight); - disconnectpool.removeForBlock(blockConnecting.vtx); + if (m_mempool) { + m_mempool->removeForBlock(blockConnecting.vtx, pindexNew->nHeight); + disconnectpool.removeForBlock(blockConnecting.vtx); + } // Update m_chain & related variables. m_chain.SetTip(pindexNew); - UpdateTip(m_mempool, pindexNew, chainparams, *this); + UpdateTip(pindexNew); int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; LogPrint(BCLog::BENCH, " - Connect postprocess: %.2fms [%.2fs (%.2fms/blk)]\n", (nTime6 - nTime5) * MILLI, nTimePostConnect * MICRO, nTimePostConnect * MILLI / nBlocksTotal); @@ -2515,10 +2410,10 @@ void CChainState::PruneBlockIndexCandidates() { * * @returns true unless a system error occurred */ -bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) +bool CChainState::ActivateBestChainStep(BlockValidationState& state, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) { AssertLockHeld(cs_main); - AssertLockHeld(m_mempool.cs); + if (m_mempool) AssertLockHeld(m_mempool->cs); const CBlockIndex* pindexOldTip = m_chain.Tip(); const CBlockIndex* pindexFork = m_chain.FindFork(pindexMostWork); @@ -2527,10 +2422,10 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChai bool fBlocksDisconnected = false; DisconnectedBlockTransactions disconnectpool; while (m_chain.Tip() && m_chain.Tip() != pindexFork) { - if (!DisconnectTip(state, chainparams, &disconnectpool)) { + if (!DisconnectTip(state, &disconnectpool)) { // This is likely a fatal error, but keep the mempool consistent, // just in case. Only remove from the mempool in this case. - UpdateMempoolForReorg(*this, m_mempool, disconnectpool, false); + MaybeUpdateMempoolForReorg(disconnectpool, false); // If we're unable to disconnect a block during normal operation, // then that is a failure of our local system -- we should abort @@ -2560,7 +2455,7 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChai // Connect new blocks. for (CBlockIndex* pindexConnect : reverse_iterate(vpindexToConnect)) { - if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr<const CBlock>(), connectTrace, disconnectpool)) { + if (!ConnectTip(state, pindexConnect, pindexConnect == pindexMostWork ? pblock : std::shared_ptr<const CBlock>(), connectTrace, disconnectpool)) { if (state.IsInvalid()) { // The block violates a consensus rule. if (state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { @@ -2574,7 +2469,7 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChai // A system error occurred (disk space, database error, ...). // Make the mempool consistent with the current tip, just in case // any observers try to use it before shutdown. - UpdateMempoolForReorg(*this, m_mempool, disconnectpool, false); + MaybeUpdateMempoolForReorg(disconnectpool, false); return false; } } else { @@ -2591,9 +2486,9 @@ bool CChainState::ActivateBestChainStep(BlockValidationState& state, const CChai if (fBlocksDisconnected) { // If any blocks were disconnected, disconnectpool may be non empty. Add // any disconnected transactions back to the mempool. - UpdateMempoolForReorg(*this, m_mempool, disconnectpool, true); + MaybeUpdateMempoolForReorg(disconnectpool, true); } - m_mempool.check(*this); + if (m_mempool) m_mempool->check(*this); CheckForkWarningConditions(); @@ -2637,7 +2532,8 @@ static void LimitValidationInterfaceQueue() LOCKS_EXCLUDED(cs_main) { } } -bool CChainState::ActivateBestChain(BlockValidationState &state, const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock) { +bool CChainState::ActivateBestChain(BlockValidationState& state, std::shared_ptr<const CBlock> pblock) +{ // Note that while we're often called here from ProcessNewBlock, this is // far from a guarantee. Things in the P2P/RPC will often end up calling // us in the middle of ProcessNewBlock - do not assume pblock is set @@ -2652,7 +2548,7 @@ bool CChainState::ActivateBestChain(BlockValidationState &state, const CChainPar CBlockIndex *pindexMostWork = nullptr; CBlockIndex *pindexNewTip = nullptr; - int nStopAtHeight = gArgs.GetArg("-stopatheight", DEFAULT_STOPATHEIGHT); + int nStopAtHeight = gArgs.GetIntArg("-stopatheight", DEFAULT_STOPATHEIGHT); do { // Block until the validation queue drains. This should largely // never happen in normal operation, however may happen during @@ -2664,7 +2560,8 @@ bool CChainState::ActivateBestChain(BlockValidationState &state, const CChainPar { LOCK(cs_main); - LOCK(m_mempool.cs); // Lock transaction pool for at least as long as it takes for connectTrace to be consumed + // Lock transaction pool for at least as long as it takes for connectTrace to be consumed + LOCK(MempoolMutex()); CBlockIndex* starting_tip = m_chain.Tip(); bool blocks_connected = false; do { @@ -2683,7 +2580,7 @@ bool CChainState::ActivateBestChain(BlockValidationState &state, const CChainPar bool fInvalidFound = false; std::shared_ptr<const CBlock> nullBlockPtr; - if (!ActivateBestChainStep(state, chainparams, pindexMostWork, pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : nullBlockPtr, fInvalidFound, connectTrace)) { + if (!ActivateBestChainStep(state, pindexMostWork, pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : nullBlockPtr, fInvalidFound, connectTrace)) { // A system error occurred return false; } @@ -2725,17 +2622,17 @@ bool CChainState::ActivateBestChain(BlockValidationState &state, const CChainPar // that the best block hash is non-null. if (ShutdownRequested()) break; } while (pindexNewTip != pindexMostWork); - CheckBlockIndex(chainparams.GetConsensus()); + CheckBlockIndex(); // Write changes periodically to disk, after relay. - if (!FlushStateToDisk(chainparams, state, FlushStateMode::PERIODIC)) { + if (!FlushStateToDisk(state, FlushStateMode::PERIODIC)) { return false; } return true; } -bool CChainState::PreciousBlock(BlockValidationState& state, const CChainParams& params, CBlockIndex *pindex) +bool CChainState::PreciousBlock(BlockValidationState& state, CBlockIndex* pindex) { { LOCK(cs_main); @@ -2761,10 +2658,10 @@ bool CChainState::PreciousBlock(BlockValidationState& state, const CChainParams& } } - return ActivateBestChain(state, params, std::shared_ptr<const CBlock>()); + return ActivateBestChain(state, std::shared_ptr<const CBlock>()); } -bool CChainState::InvalidateBlock(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) +bool CChainState::InvalidateBlock(BlockValidationState& state, CBlockIndex* pindex) { // Genesis block can't be invalidated assert(pindex); @@ -2814,7 +2711,9 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, const CChainParam LimitValidationInterfaceQueue(); LOCK(cs_main); - LOCK(m_mempool.cs); // Lock for as long as disconnectpool is in scope to make sure UpdateMempoolForReorg is called after DisconnectTip without unlocking in between + // Lock for as long as disconnectpool is in scope to make sure MaybeUpdateMempoolForReorg is + // called after DisconnectTip without unlocking in between + LOCK(MempoolMutex()); if (!m_chain.Contains(pindex)) break; pindex_was_in_chain = true; CBlockIndex *invalid_walk_tip = m_chain.Tip(); @@ -2822,13 +2721,13 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, const CChainParam // ActivateBestChain considers blocks already in m_chain // unconditionally valid already, so force disconnect away from it. DisconnectedBlockTransactions disconnectpool; - bool ret = DisconnectTip(state, chainparams, &disconnectpool); + bool ret = DisconnectTip(state, &disconnectpool); // DisconnectTip will add transactions to disconnectpool. // Adjust the mempool to be consistent with the new tip, adding // transactions back to the mempool if disconnecting was successful, // and we're not doing a very deep invalidation (in which case // keeping the mempool up to date is probably futile anyway). - UpdateMempoolForReorg(*this, m_mempool, disconnectpool, /* fAddToMempool = */ (++disconnected <= 10) && ret); + MaybeUpdateMempoolForReorg(disconnectpool, /* fAddToMempool = */ (++disconnected <= 10) && ret); if (!ret) return false; assert(invalid_walk_tip->pprev == m_chain.Tip()); @@ -2864,7 +2763,7 @@ bool CChainState::InvalidateBlock(BlockValidationState& state, const CChainParam to_mark_failed = invalid_walk_tip; } - CheckBlockIndex(chainparams.GetConsensus()); + CheckBlockIndex(); { LOCK(cs_main); @@ -2975,7 +2874,7 @@ CBlockIndex* BlockManager::AddToBlockIndex(const CBlockHeader& block) } /** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ -void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos, const Consensus::Params& consensusParams) +void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos) { pindexNew->nTx = block.vtx.size(); pindexNew->nChainTx = 0; @@ -2983,7 +2882,7 @@ void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pi pindexNew->nDataPos = pos.nPos; pindexNew->nUndoPos = 0; pindexNew->nStatus |= BLOCK_HAVE_DATA; - if (IsWitnessEnabled(pindexNew->pprev, consensusParams)) { + if (DeploymentActiveAt(*pindexNew, m_params.GetConsensus(), Consensus::DEPLOYMENT_SEGWIT)) { pindexNew->nStatus |= BLOCK_OPT_WITNESS; } pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); @@ -2999,10 +2898,7 @@ void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pi CBlockIndex *pindex = queue.front(); queue.pop_front(); pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; - { - LOCK(cs_nBlockSequenceId); - pindex->nSequenceId = nBlockSequenceId++; - } + pindex->nSequenceId = nBlockSequenceId++; if (m_chain.Tip() == nullptr || !setBlockIndexCandidates.value_comp()(pindex, m_chain.Tip())) { setBlockIndexCandidates.insert(pindex); } @@ -3104,17 +3000,11 @@ bool CheckBlock(const CBlock& block, BlockValidationState& state, const Consensu return true; } -bool IsWitnessEnabled(const CBlockIndex* pindexPrev, const Consensus::Params& params) -{ - int height = pindexPrev == nullptr ? 0 : pindexPrev->nHeight + 1; - return (height >= params.SegwitHeight); -} - void UpdateUncommittedBlockStructures(CBlock& block, const CBlockIndex* pindexPrev, const Consensus::Params& consensusParams) { int commitpos = GetWitnessCommitmentIndex(block); static const std::vector<unsigned char> nonce(32, 0x00); - if (commitpos != NO_WITNESS_COMMITMENT && IsWitnessEnabled(pindexPrev, consensusParams) && !block.vtx[0]->HasWitness()) { + if (commitpos != NO_WITNESS_COMMITMENT && DeploymentActiveAfter(pindexPrev, consensusParams, Consensus::DEPLOYMENT_SEGWIT) && !block.vtx[0]->HasWitness()) { CMutableTransaction tx(*block.vtx[0]); tx.vin[0].scriptWitness.stack.resize(1); tx.vin[0].scriptWitness.stack[0] = nonce; @@ -3127,25 +3017,23 @@ std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBloc std::vector<unsigned char> commitment; int commitpos = GetWitnessCommitmentIndex(block); std::vector<unsigned char> ret(32, 0x00); - if (consensusParams.SegwitHeight != std::numeric_limits<int>::max()) { - if (commitpos == NO_WITNESS_COMMITMENT) { - uint256 witnessroot = BlockWitnessMerkleRoot(block, nullptr); - CHash256().Write(witnessroot).Write(ret).Finalize(witnessroot); - CTxOut out; - out.nValue = 0; - out.scriptPubKey.resize(MINIMUM_WITNESS_COMMITMENT); - out.scriptPubKey[0] = OP_RETURN; - out.scriptPubKey[1] = 0x24; - out.scriptPubKey[2] = 0xaa; - out.scriptPubKey[3] = 0x21; - out.scriptPubKey[4] = 0xa9; - out.scriptPubKey[5] = 0xed; - memcpy(&out.scriptPubKey[6], witnessroot.begin(), 32); - commitment = std::vector<unsigned char>(out.scriptPubKey.begin(), out.scriptPubKey.end()); - CMutableTransaction tx(*block.vtx[0]); - tx.vout.push_back(out); - block.vtx[0] = MakeTransactionRef(std::move(tx)); - } + if (commitpos == NO_WITNESS_COMMITMENT) { + uint256 witnessroot = BlockWitnessMerkleRoot(block, nullptr); + CHash256().Write(witnessroot).Write(ret).Finalize(witnessroot); + CTxOut out; + out.nValue = 0; + out.scriptPubKey.resize(MINIMUM_WITNESS_COMMITMENT); + out.scriptPubKey[0] = OP_RETURN; + out.scriptPubKey[1] = 0x24; + out.scriptPubKey[2] = 0xaa; + out.scriptPubKey[3] = 0x21; + out.scriptPubKey[4] = 0xa9; + out.scriptPubKey[5] = 0xed; + memcpy(&out.scriptPubKey[6], witnessroot.begin(), 32); + commitment = std::vector<unsigned char>(out.scriptPubKey.begin(), out.scriptPubKey.end()); + CMutableTransaction tx(*block.vtx[0]); + tx.vout.push_back(out); + block.vtx[0] = MakeTransactionRef(std::move(tx)); } UpdateUncommittedBlockStructures(block, pindexPrev, consensusParams); return commitment; @@ -3205,13 +3093,13 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidatio if (block.GetBlockTime() > nAdjustedTime + MAX_FUTURE_BLOCK_TIME) return state.Invalid(BlockValidationResult::BLOCK_TIME_FUTURE, "time-too-new", "block timestamp too far in the future"); - // Reject outdated version blocks when 95% (75% on testnet) of the network has upgraded: - // check for version 2, 3 and 4 upgrades - if((block.nVersion < 2 && nHeight >= consensusParams.BIP34Height) || - (block.nVersion < 3 && nHeight >= consensusParams.BIP66Height) || - (block.nVersion < 4 && nHeight >= consensusParams.BIP65Height)) + // Reject blocks with outdated version + if ((block.nVersion < 2 && DeploymentActiveAfter(pindexPrev, consensusParams, Consensus::DEPLOYMENT_HEIGHTINCB)) || + (block.nVersion < 3 && DeploymentActiveAfter(pindexPrev, consensusParams, Consensus::DEPLOYMENT_DERSIG)) || + (block.nVersion < 4 && DeploymentActiveAfter(pindexPrev, consensusParams, Consensus::DEPLOYMENT_CLTV))) { return state.Invalid(BlockValidationResult::BLOCK_INVALID_HEADER, strprintf("bad-version(0x%08x)", block.nVersion), strprintf("rejected nVersion=0x%08x block", block.nVersion)); + } return true; } @@ -3226,9 +3114,9 @@ static bool ContextualCheckBlock(const CBlock& block, BlockValidationState& stat { const int nHeight = pindexPrev == nullptr ? 0 : pindexPrev->nHeight + 1; - // Start enforcing BIP113 (Median Time Past). + // Enforce BIP113 (Median Time Past). int nLockTimeFlags = 0; - if (nHeight >= consensusParams.CSVHeight) { + if (DeploymentActiveAfter(pindexPrev, consensusParams, Consensus::DEPLOYMENT_CSV)) { assert(pindexPrev != nullptr); nLockTimeFlags |= LOCKTIME_MEDIAN_TIME_PAST; } @@ -3245,7 +3133,7 @@ static bool ContextualCheckBlock(const CBlock& block, BlockValidationState& stat } // Enforce rule that the coinbase starts with serialized block height - if (nHeight >= consensusParams.BIP34Height) + if (DeploymentActiveAfter(pindexPrev, consensusParams, Consensus::DEPLOYMENT_HEIGHTINCB)) { CScript expect = CScript() << nHeight; if (block.vtx[0]->vin[0].scriptSig.size() < expect.size() || @@ -3263,7 +3151,7 @@ static bool ContextualCheckBlock(const CBlock& block, BlockValidationState& stat // {0xaa, 0x21, 0xa9, 0xed}, and the following 32 bytes are SHA256^2(witness root, witness reserved value). In case there are // multiple, the last one is used. bool fHaveWitness = false; - if (nHeight >= consensusParams.SegwitHeight) { + if (DeploymentActiveAfter(pindexPrev, consensusParams, Consensus::DEPLOYMENT_SEGWIT)) { int commitpos = GetWitnessCommitmentIndex(block); if (commitpos != NO_WITNESS_COMMITMENT) { bool malleated = false; @@ -3400,7 +3288,7 @@ bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& CBlockIndex *pindex = nullptr; // Use a temp pindex instead of ppindex to avoid a const_cast bool accepted = m_blockman.AcceptBlockHeader( header, state, chainparams, &pindex); - ActiveChainstate().CheckBlockIndex(chainparams.GetConsensus()); + ActiveChainstate().CheckBlockIndex(); if (!accepted) { return false; @@ -3419,7 +3307,7 @@ bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& } /** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ -bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) +bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) { const CBlock& block = *pblock; @@ -3429,8 +3317,8 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, Block CBlockIndex *pindexDummy = nullptr; CBlockIndex *&pindex = ppindex ? *ppindex : pindexDummy; - bool accepted_header = m_blockman.AcceptBlockHeader(block, state, chainparams, &pindex); - CheckBlockIndex(chainparams.GetConsensus()); + bool accepted_header = m_blockman.AcceptBlockHeader(block, state, m_params, &pindex); + CheckBlockIndex(); if (!accepted_header) return false; @@ -3467,8 +3355,8 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, Block if (pindex->nChainWork < nMinimumChainWork) return true; } - if (!CheckBlock(block, state, chainparams.GetConsensus()) || - !ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindex->pprev)) { + if (!CheckBlock(block, state, m_params.GetConsensus()) || + !ContextualCheckBlock(block, state, m_params.GetConsensus(), pindex->pprev)) { if (state.IsInvalid() && state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { pindex->nStatus |= BLOCK_FAILED_VALID; setDirtyBlockIndex.insert(pindex); @@ -3484,19 +3372,19 @@ bool CChainState::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, Block // Write block to history file if (fNewBlock) *fNewBlock = true; try { - FlatFilePos blockPos = SaveBlockToDisk(block, pindex->nHeight, m_chain, chainparams, dbp); + FlatFilePos blockPos = SaveBlockToDisk(block, pindex->nHeight, m_chain, m_params, dbp); if (blockPos.IsNull()) { state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); return false; } - ReceivedBlockTransactions(block, pindex, blockPos, chainparams.GetConsensus()); + ReceivedBlockTransactions(block, pindex, blockPos); } catch (const std::runtime_error& e) { return AbortNode(state, std::string("System error: ") + e.what()); } - FlushStateToDisk(chainparams, state, FlushStateMode::NONE); + FlushStateToDisk(state, FlushStateMode::NONE); - CheckBlockIndex(chainparams.GetConsensus()); + CheckBlockIndex(); return true; } @@ -3522,7 +3410,7 @@ bool ChainstateManager::ProcessNewBlock(const CChainParams& chainparams, const s bool ret = CheckBlock(*block, state, chainparams.GetConsensus()); if (ret) { // Store to disk - ret = ActiveChainstate().AcceptBlock(block, state, chainparams, &pindex, force_processing, nullptr, new_block); + ret = ActiveChainstate().AcceptBlock(block, state, &pindex, force_processing, nullptr, new_block); } if (!ret) { GetMainSignals().BlockChecked(*block, state); @@ -3533,8 +3421,9 @@ bool ChainstateManager::ProcessNewBlock(const CChainParams& chainparams, const s NotifyHeaderTip(ActiveChainstate()); BlockValidationState state; // Only used to report errors, not invalidity - ignore it - if (!ActiveChainstate().ActivateBestChain(state, chainparams, block)) + if (!ActiveChainstate().ActivateBestChain(state, block)) { return error("%s: ActivateBestChain failed (%s)", __func__, state.ToString()); + } return true; } @@ -3563,8 +3452,9 @@ bool TestBlockValidity(BlockValidationState& state, return error("%s: Consensus::CheckBlock: %s", __func__, state.ToString()); if (!ContextualCheckBlock(block, state, chainparams.GetConsensus(), pindexPrev)) return error("%s: Consensus::ContextualCheckBlock: %s", __func__, state.ToString()); - if (!chainstate.ConnectBlock(block, state, &indexDummy, viewNew, chainparams, true)) + if (!chainstate.ConnectBlock(block, state, &indexDummy, viewNew, true)) { return false; + } assert(state.IsValid()); return true; @@ -3635,9 +3525,8 @@ void BlockManager::FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nM void PruneBlockFilesManual(CChainState& active_chainstate, int nManualPruneHeight) { BlockValidationState state; - const CChainParams& chainparams = Params(); if (!active_chainstate.FlushStateToDisk( - chainparams, state, FlushStateMode::NONE, nManualPruneHeight)) { + state, FlushStateMode::NONE, nManualPruneHeight)) { LogPrintf("%s: failed to flush state (%s)\n", __func__, state.ToString()); } } @@ -3723,11 +3612,11 @@ CBlockIndex * BlockManager::InsertBlockIndex(const uint256& hash) bool BlockManager::LoadBlockIndex( const Consensus::Params& consensus_params, - CBlockTreeDB& blocktree, std::set<CBlockIndex*, CBlockIndexWorkComparator>& block_index_candidates) { - if (!blocktree.LoadBlockIndexGuts(consensus_params, [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); })) + if (!m_block_tree_db->LoadBlockIndexGuts(consensus_params, [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); })) { return false; + } // Calculate nChainWork std::vector<std::pair<int, CBlockIndex*> > vSortedByHeight; @@ -3762,7 +3651,9 @@ bool BlockManager::LoadBlockIndex( pindex->nStatus |= BLOCK_FAILED_CHILD; setDirtyBlockIndex.insert(pindex); } - if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && (pindex->HaveTxsDownloaded() || pindex->pprev == nullptr)) { + if (pindex->IsAssumedValid() || + (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && + (pindex->HaveTxsDownloaded() || pindex->pprev == nullptr))) { block_index_candidates.insert(pindex); } if (pindex->nStatus & BLOCK_FAILED_MASK && (!pindexBestInvalid || pindex->nChainWork > pindexBestInvalid->nChainWork)) @@ -3787,25 +3678,25 @@ void BlockManager::Unload() { m_block_index.clear(); } -bool CChainState::LoadBlockIndexDB(const CChainParams& chainparams) +bool BlockManager::LoadBlockIndexDB(std::set<CBlockIndex*, CBlockIndexWorkComparator>& setBlockIndexCandidates) { - if (!m_blockman.LoadBlockIndex( - chainparams.GetConsensus(), *pblocktree, + if (!LoadBlockIndex( + ::Params().GetConsensus(), setBlockIndexCandidates)) { return false; } // Load block file info - pblocktree->ReadLastBlockFile(nLastBlockFile); + m_block_tree_db->ReadLastBlockFile(nLastBlockFile); vinfoBlockFile.resize(nLastBlockFile + 1); LogPrintf("%s: last block file = %i\n", __func__, nLastBlockFile); for (int nFile = 0; nFile <= nLastBlockFile; nFile++) { - pblocktree->ReadBlockFileInfo(nFile, vinfoBlockFile[nFile]); + m_block_tree_db->ReadBlockFileInfo(nFile, vinfoBlockFile[nFile]); } LogPrintf("%s: last block file info: %s\n", __func__, vinfoBlockFile[nLastBlockFile].ToString()); for (int nFile = nLastBlockFile + 1; true; nFile++) { CBlockFileInfo info; - if (pblocktree->ReadBlockFileInfo(nFile, info)) { + if (m_block_tree_db->ReadBlockFileInfo(nFile, info)) { vinfoBlockFile.push_back(info); } else { break; @@ -3815,7 +3706,7 @@ bool CChainState::LoadBlockIndexDB(const CChainParams& chainparams) // Check presence of blk files LogPrintf("Checking all blk files are present...\n"); std::set<int> setBlkDataFiles; - for (const std::pair<const uint256, CBlockIndex*>& item : m_blockman.m_block_index) { + for (const std::pair<const uint256, CBlockIndex*>& item : m_block_index) { CBlockIndex* pindex = item.second; if (pindex->nStatus & BLOCK_HAVE_DATA) { setBlkDataFiles.insert(pindex->nFile); @@ -3830,13 +3721,13 @@ bool CChainState::LoadBlockIndexDB(const CChainParams& chainparams) } // Check whether we have ever pruned block & undo files - pblocktree->ReadFlag("prunedblockfiles", fHavePruned); + m_block_tree_db->ReadFlag("prunedblockfiles", fHavePruned); if (fHavePruned) LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n"); // Check whether we need to continue reindexing bool fReindexing = false; - pblocktree->ReadReindexing(fReindexing); + m_block_tree_db->ReadReindexing(fReindexing); if(fReindexing) fReindex = true; return true; @@ -3844,13 +3735,14 @@ bool CChainState::LoadBlockIndexDB(const CChainParams& chainparams) void CChainState::LoadMempool(const ArgsManager& args) { - if (args.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { - ::LoadMempool(m_mempool, *this); + if (!m_mempool) return; + if (args.GetBoolArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { + ::LoadMempool(*m_mempool, *this); } - m_mempool.SetIsLoaded(!ShutdownRequested()); + m_mempool->SetIsLoaded(!ShutdownRequested()); } -bool CChainState::LoadChainTip(const CChainParams& chainparams) +bool CChainState::LoadChainTip() { AssertLockHeld(cs_main); const CCoinsViewCache& coins_cache = CoinsTip(); @@ -3871,10 +3763,10 @@ bool CChainState::LoadChainTip(const CChainParams& chainparams) tip = m_chain.Tip(); LogPrintf("Loaded best chain: hashBestChain=%s height=%d date=%s progress=%f\n", - tip->GetBlockHash().ToString(), - m_chain.Height(), - FormatISO8601DateTime(tip->GetBlockTime()), - GuessVerificationProgress(chainparams.TxData(), tip)); + tip->GetBlockHash().ToString(), + m_chain.Height(), + FormatISO8601DateTime(tip->GetBlockTime()), + GuessVerificationProgress(m_params.TxData(), tip)); return true; } @@ -3985,8 +3877,9 @@ bool CVerifyDB::VerifyDB( CBlock block; if (!ReadBlockFromDisk(block, pindex, chainparams.GetConsensus())) return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); - if (!chainstate.ConnectBlock(block, state, pindex, coins, chainparams)) + if (!chainstate.ConnectBlock(block, state, pindex, coins)) { return error("VerifyDB(): *** found unconnectable block at %d, hash=%s (%s)", pindex->nHeight, pindex->GetBlockHash().ToString(), state.ToString()); + } if (ShutdownRequested()) return true; } } @@ -3998,11 +3891,11 @@ bool CVerifyDB::VerifyDB( } /** Apply the effects of a block on the utxo cache, ignoring that it may already have been applied. */ -bool CChainState::RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, const CChainParams& params) +bool CChainState::RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs) { // TODO: merge with ConnectBlock CBlock block; - if (!ReadBlockFromDisk(block, pindex, params.GetConsensus())) { + if (!ReadBlockFromDisk(block, pindex, m_params.GetConsensus())) { return error("ReplayBlock(): ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } @@ -4018,7 +3911,7 @@ bool CChainState::RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& i return true; } -bool CChainState::ReplayBlocks(const CChainParams& params) +bool CChainState::ReplayBlocks() { LOCK(cs_main); @@ -4054,7 +3947,7 @@ bool CChainState::ReplayBlocks(const CChainParams& params) while (pindexOld != pindexFork) { if (pindexOld->nHeight > 0) { // Never disconnect the genesis block. CBlock block; - if (!ReadBlockFromDisk(block, pindexOld, params.GetConsensus())) { + if (!ReadBlockFromDisk(block, pindexOld, m_params.GetConsensus())) { return error("RollbackBlock(): ReadBlockFromDisk() failed at %d, hash=%s", pindexOld->nHeight, pindexOld->GetBlockHash().ToString()); } LogPrintf("Rolling back %s (%i)\n", pindexOld->GetBlockHash().ToString(), pindexOld->nHeight); @@ -4076,7 +3969,7 @@ bool CChainState::ReplayBlocks(const CChainParams& params) const CBlockIndex* pindex = pindexNew->GetAncestor(nHeight); LogPrintf("Rolling forward %s (%i)\n", pindex->GetBlockHash().ToString(), nHeight); uiInterface.ShowProgress(_("Replaying blocks…").translated, (int) ((nHeight - nForkHeight) * 100.0 / (pindexNew->nHeight - nForkHeight)) , false); - if (!RollforwardBlock(pindex, cache, params)) return false; + if (!RollforwardBlock(pindex, cache)) return false; } cache.SetBestBlock(pindexNew->GetBlockHash()); @@ -4085,15 +3978,14 @@ bool CChainState::ReplayBlocks(const CChainParams& params) return true; } -bool CChainState::NeedsRedownload(const CChainParams& params) const +bool CChainState::NeedsRedownload() const { AssertLockHeld(cs_main); - // At and above params.SegwitHeight, segwit consensus rules must be validated + // At and above m_params.SegwitHeight, segwit consensus rules must be validated CBlockIndex* block{m_chain.Tip()}; - const int segwit_height{params.GetConsensus().SegwitHeight}; - while (block != nullptr && block->nHeight >= segwit_height) { + while (block != nullptr && DeploymentActiveAt(*block, m_params.GetConsensus(), Consensus::DEPLOYMENT_SEGWIT)) { if (!(block->nStatus & BLOCK_OPT_WITNESS)) { // block is insufficiently validated for a segwit client return true; @@ -4123,20 +4015,20 @@ void UnloadBlockIndex(CTxMemPool* mempool, ChainstateManager& chainman) nLastBlockFile = 0; setDirtyBlockIndex.clear(); setDirtyFileInfo.clear(); - versionbitscache.Clear(); + g_versionbitscache.Clear(); for (int b = 0; b < VERSIONBITS_NUM_BITS; b++) { warningcache[b].clear(); } fHavePruned = false; } -bool ChainstateManager::LoadBlockIndex(const CChainParams& chainparams) +bool ChainstateManager::LoadBlockIndex() { AssertLockHeld(cs_main); // Load block index from databases bool needs_init = fReindex; if (!fReindex) { - bool ret = ActiveChainstate().LoadBlockIndexDB(chainparams); + bool ret = m_blockman.LoadBlockIndexDB(ActiveChainstate().setBlockIndexCandidates); if (!ret) return false; needs_init = m_blockman.m_block_index.empty(); } @@ -4153,7 +4045,7 @@ bool ChainstateManager::LoadBlockIndex(const CChainParams& chainparams) return true; } -bool CChainState::LoadGenesisBlock(const CChainParams& chainparams) +bool CChainState::LoadGenesisBlock() { LOCK(cs_main); @@ -4161,16 +4053,16 @@ bool CChainState::LoadGenesisBlock(const CChainParams& chainparams) // m_blockman.m_block_index. Note that we can't use m_chain here, since it is // set based on the coins db, not the block index db, which is the only // thing loaded at this point. - if (m_blockman.m_block_index.count(chainparams.GenesisBlock().GetHash())) + if (m_blockman.m_block_index.count(m_params.GenesisBlock().GetHash())) return true; try { - const CBlock& block = chainparams.GenesisBlock(); - FlatFilePos blockPos = SaveBlockToDisk(block, 0, m_chain, chainparams, nullptr); + const CBlock& block = m_params.GenesisBlock(); + FlatFilePos blockPos = SaveBlockToDisk(block, 0, m_chain, m_params, nullptr); if (blockPos.IsNull()) return error("%s: writing genesis block to disk failed", __func__); CBlockIndex *pindex = m_blockman.AddToBlockIndex(block); - ReceivedBlockTransactions(block, pindex, blockPos, chainparams.GetConsensus()); + ReceivedBlockTransactions(block, pindex, blockPos); } catch (const std::runtime_error& e) { return error("%s: failed to write genesis block: %s", __func__, e.what()); } @@ -4178,7 +4070,7 @@ bool CChainState::LoadGenesisBlock(const CChainParams& chainparams) return true; } -void CChainState::LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp) +void CChainState::LoadExternalBlockFile(FILE* fileIn, FlatFilePos* dbp) { // Map of disk positions for blocks with unknown parent (only used for reindex) static std::multimap<uint256, FlatFilePos> mapBlocksUnknownParent; @@ -4199,11 +4091,12 @@ void CChainState::LoadExternalBlockFile(const CChainParams& chainparams, FILE* f try { // locate a header unsigned char buf[CMessageHeader::MESSAGE_START_SIZE]; - blkdat.FindByte(chainparams.MessageStart()[0]); + blkdat.FindByte(m_params.MessageStart()[0]); nRewind = blkdat.GetPos()+1; blkdat >> buf; - if (memcmp(buf, chainparams.MessageStart(), CMessageHeader::MESSAGE_START_SIZE)) + if (memcmp(buf, m_params.MessageStart(), CMessageHeader::MESSAGE_START_SIZE)) { continue; + } // read size blkdat >> nSize; if (nSize < 80 || nSize > MAX_BLOCK_SERIALIZED_SIZE) @@ -4227,7 +4120,7 @@ void CChainState::LoadExternalBlockFile(const CChainParams& chainparams, FILE* f { LOCK(cs_main); // detect out of order blocks, and store them for later - if (hash != chainparams.GetConsensus().hashGenesisBlock && !m_blockman.LookupBlockIndex(block.hashPrevBlock)) { + if (hash != m_params.GetConsensus().hashGenesisBlock && !m_blockman.LookupBlockIndex(block.hashPrevBlock)) { LogPrint(BCLog::REINDEX, "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(), block.hashPrevBlock.ToString()); if (dbp) @@ -4239,21 +4132,21 @@ void CChainState::LoadExternalBlockFile(const CChainParams& chainparams, FILE* f CBlockIndex* pindex = m_blockman.LookupBlockIndex(hash); if (!pindex || (pindex->nStatus & BLOCK_HAVE_DATA) == 0) { BlockValidationState state; - if (AcceptBlock(pblock, state, chainparams, nullptr, true, dbp, nullptr)) { + if (AcceptBlock(pblock, state, nullptr, true, dbp, nullptr)) { nLoaded++; } if (state.IsError()) { break; } - } else if (hash != chainparams.GetConsensus().hashGenesisBlock && pindex->nHeight % 1000 == 0) { - LogPrint(BCLog::REINDEX, "Block Import: already had block %s at height %d\n", hash.ToString(), pindex->nHeight); + } else if (hash != m_params.GetConsensus().hashGenesisBlock && pindex->nHeight % 1000 == 0) { + LogPrint(BCLog::REINDEX, "Block Import: already had block %s at height %d\n", hash.ToString(), pindex->nHeight); } } // Activate the genesis block so normal node progress can continue - if (hash == chainparams.GetConsensus().hashGenesisBlock) { + if (hash == m_params.GetConsensus().hashGenesisBlock) { BlockValidationState state; - if (!ActivateBestChain(state, chainparams, nullptr)) { + if (!ActivateBestChain(state, nullptr)) { break; } } @@ -4270,14 +4163,12 @@ void CChainState::LoadExternalBlockFile(const CChainParams& chainparams, FILE* f while (range.first != range.second) { std::multimap<uint256, FlatFilePos>::iterator it = range.first; std::shared_ptr<CBlock> pblockrecursive = std::make_shared<CBlock>(); - if (ReadBlockFromDisk(*pblockrecursive, it->second, chainparams.GetConsensus())) - { + if (ReadBlockFromDisk(*pblockrecursive, it->second, m_params.GetConsensus())) { LogPrint(BCLog::REINDEX, "%s: Processing out of order child %s of %s\n", __func__, pblockrecursive->GetHash().ToString(), head.ToString()); LOCK(cs_main); BlockValidationState dummy; - if (AcceptBlock(pblockrecursive, dummy, chainparams, nullptr, true, &it->second, nullptr)) - { + if (AcceptBlock(pblockrecursive, dummy, nullptr, true, &it->second, nullptr)) { nLoaded++; queue.push_back(pblockrecursive->GetHash()); } @@ -4297,7 +4188,7 @@ void CChainState::LoadExternalBlockFile(const CChainParams& chainparams, FILE* f LogPrintf("Loaded %i blocks from external file in %dms\n", nLoaded, GetTimeMillis() - nStart); } -void CChainState::CheckBlockIndex(const Consensus::Params& consensusParams) +void CChainState::CheckBlockIndex() { if (!fCheckBlockIndex) { return; @@ -4341,23 +4232,46 @@ void CChainState::CheckBlockIndex(const Consensus::Params& consensusParams) while (pindex != nullptr) { nNodes++; if (pindexFirstInvalid == nullptr && pindex->nStatus & BLOCK_FAILED_VALID) pindexFirstInvalid = pindex; - if (pindexFirstMissing == nullptr && !(pindex->nStatus & BLOCK_HAVE_DATA)) pindexFirstMissing = pindex; + // Assumed-valid index entries will not have data since we haven't downloaded the + // full block yet. + if (pindexFirstMissing == nullptr && !(pindex->nStatus & BLOCK_HAVE_DATA) && !pindex->IsAssumedValid()) { + pindexFirstMissing = pindex; + } if (pindexFirstNeverProcessed == nullptr && pindex->nTx == 0) pindexFirstNeverProcessed = pindex; if (pindex->pprev != nullptr && pindexFirstNotTreeValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TREE) pindexFirstNotTreeValid = pindex; - if (pindex->pprev != nullptr && pindexFirstNotTransactionsValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) pindexFirstNotTransactionsValid = pindex; - if (pindex->pprev != nullptr && pindexFirstNotChainValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_CHAIN) pindexFirstNotChainValid = pindex; - if (pindex->pprev != nullptr && pindexFirstNotScriptsValid == nullptr && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_SCRIPTS) pindexFirstNotScriptsValid = pindex; + + if (pindex->pprev != nullptr && !pindex->IsAssumedValid()) { + // Skip validity flag checks for BLOCK_ASSUMED_VALID index entries, since these + // *_VALID_MASK flags will not be present for index entries we are temporarily assuming + // valid. + if (pindexFirstNotTransactionsValid == nullptr && + (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) { + pindexFirstNotTransactionsValid = pindex; + } + + if (pindexFirstNotChainValid == nullptr && + (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_CHAIN) { + pindexFirstNotChainValid = pindex; + } + + if (pindexFirstNotScriptsValid == nullptr && + (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_SCRIPTS) { + pindexFirstNotScriptsValid = pindex; + } + } // Begin: actual consistency checks. if (pindex->pprev == nullptr) { // Genesis block checks. - assert(pindex->GetBlockHash() == consensusParams.hashGenesisBlock); // Genesis block's hash must match. + assert(pindex->GetBlockHash() == m_params.GetConsensus().hashGenesisBlock); // Genesis block's hash must match. assert(pindex == m_chain.Genesis()); // The current active chain's genesis block must be this block. } if (!pindex->HaveTxsDownloaded()) assert(pindex->nSequenceId <= 0); // nSequenceId can't be set positive for blocks that aren't linked (negative is used for preciousblock) // VALID_TRANSACTIONS is equivalent to nTx > 0 for all nodes (whether or not pruning has occurred). // HAVE_DATA is only equivalent to nTx > 0 (or VALID_TRANSACTIONS) if no pruning has occurred. - if (!fHavePruned) { + // Unless these indexes are assumed valid and pending block download on a + // background chainstate. + if (!fHavePruned && !pindex->IsAssumedValid()) { // If we've never pruned, then HAVE_DATA should be equivalent to nTx > 0 assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0)); assert(pindexFirstMissing == pindexFirstNeverProcessed); @@ -4366,7 +4280,16 @@ void CChainState::CheckBlockIndex(const Consensus::Params& consensusParams) if (pindex->nStatus & BLOCK_HAVE_DATA) assert(pindex->nTx > 0); } if (pindex->nStatus & BLOCK_HAVE_UNDO) assert(pindex->nStatus & BLOCK_HAVE_DATA); - assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // This is pruning-independent. + if (pindex->IsAssumedValid()) { + // Assumed-valid blocks should have some nTx value. + assert(pindex->nTx > 0); + // Assumed-valid blocks should connect to the main chain. + assert((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TREE); + } else { + // Otherwise there should only be an nTx value if we have + // actually seen a block's transactions. + assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // This is pruning-independent. + } // All parents having had data (at some point) is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to HaveTxsDownloaded(). assert((pindexFirstNeverProcessed == nullptr) == pindex->HaveTxsDownloaded()); assert((pindexFirstNotTransactionsValid == nullptr) == pindex->HaveTxsDownloaded()); @@ -4383,11 +4306,17 @@ void CChainState::CheckBlockIndex(const Consensus::Params& consensusParams) } if (!CBlockIndexWorkComparator()(pindex, m_chain.Tip()) && pindexFirstNeverProcessed == nullptr) { if (pindexFirstInvalid == nullptr) { + const bool is_active = this == &m_chainman.ActiveChainstate(); + // If this block sorts at least as good as the current tip and // is valid and we have all data for its parents, it must be in // setBlockIndexCandidates. m_chain.Tip() must also be there // even if some data has been pruned. - if (pindexFirstMissing == nullptr || pindex == m_chain.Tip()) { + // + // Don't perform this check for the background chainstate since + // its setBlockIndexCandidates shouldn't have some entries (i.e. those past the + // snapshot block) which do exist in the block index for the active chainstate. + if (is_active && (pindexFirstMissing == nullptr || pindex == m_chain.Tip())) { assert(setBlockIndexCandidates.count(pindex)); } // If some parent is missing, then it could be that this block was in @@ -4507,16 +4436,14 @@ bool CChainState::ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) this->ToString(), coinstip_size * (1.0 / 1024 / 1024)); BlockValidationState state; - const CChainParams& chainparams = Params(); - bool ret; if (coinstip_size > old_coinstip_size) { // Likely no need to flush if cache sizes have grown. - ret = FlushStateToDisk(chainparams, state, FlushStateMode::IF_NEEDED); + ret = FlushStateToDisk(state, FlushStateMode::IF_NEEDED); } else { // Otherwise, flush state to disk and deallocate the in-memory coins map. - ret = FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS); + ret = FlushStateToDisk(state, FlushStateMode::ALWAYS); CoinsTip().ReallocateCache(); } return ret; @@ -4527,7 +4454,7 @@ static const uint64_t MEMPOOL_DUMP_VERSION = 1; bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mockable_fopen_function) { const CChainParams& chainparams = Params(); - int64_t nExpiryTimeout = gArgs.GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60; + int64_t nExpiryTimeout = gArgs.GetIntArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60; FILE* filestr{mockable_fopen_function(gArgs.GetDataDirNet() / "mempool.dat", "rb")}; CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); if (file.IsNull()) { @@ -4714,7 +4641,8 @@ std::vector<CChainState*> ChainstateManager::GetAll() return out; } -CChainState& ChainstateManager::InitializeChainstate(CTxMemPool& mempool, const std::optional<uint256>& snapshot_blockhash) +CChainState& ChainstateManager::InitializeChainstate( + CTxMemPool* mempool, const std::optional<uint256>& snapshot_blockhash) { bool is_snapshot = snapshot_blockhash.has_value(); std::unique_ptr<CChainState>& to_modify = @@ -4723,7 +4651,7 @@ CChainState& ChainstateManager::InitializeChainstate(CTxMemPool& mempool, const if (to_modify) { throw std::logic_error("should not be overwriting a chainstate"); } - to_modify.reset(new CChainState(mempool, m_blockman, snapshot_blockhash)); + to_modify.reset(new CChainState(mempool, m_blockman, *this, snapshot_blockhash)); // Snapshot chainstates and initial IBD chaintates always become active. if (is_snapshot || (!is_snapshot && !m_active_chainstate)) { @@ -4792,8 +4720,9 @@ bool ChainstateManager::ActivateSnapshot( static_cast<size_t>(current_coinsdb_cache_size * IBD_CACHE_PERC)); } - auto snapshot_chainstate = WITH_LOCK(::cs_main, return std::make_unique<CChainState>( - this->ActiveChainstate().m_mempool, m_blockman, base_blockhash)); + auto snapshot_chainstate = WITH_LOCK(::cs_main, + return std::make_unique<CChainState>( + /* mempool */ nullptr, m_blockman, *this, base_blockhash)); { LOCK(::cs_main); @@ -4816,7 +4745,7 @@ bool ChainstateManager::ActivateSnapshot( LOCK(::cs_main); assert(!m_snapshot_chainstate); m_snapshot_chainstate.swap(snapshot_chainstate); - const bool chaintip_loaded = m_snapshot_chainstate->LoadChainTip(::Params()); + const bool chaintip_loaded = m_snapshot_chainstate->LoadChainTip(); assert(chaintip_loaded); m_active_chainstate = m_snapshot_chainstate.get(); @@ -4909,7 +4838,7 @@ bool ChainstateManager::PopulateAndValidateSnapshot( } const auto snapshot_cache_state = WITH_LOCK(::cs_main, - return snapshot_chainstate.GetCoinsCacheSizeState(&snapshot_chainstate.m_mempool)); + return snapshot_chainstate.GetCoinsCacheSizeState()); if (snapshot_cache_state >= CoinsCacheSizeState::CRITICAL) { @@ -4996,11 +4925,25 @@ bool ChainstateManager::PopulateAndValidateSnapshot( // Fake nChainTx so that GuessVerificationProgress reports accurately index->nChainTx = index->pprev ? index->pprev->nChainTx + index->nTx : 1; + // Mark unvalidated block index entries beneath the snapshot base block as assumed-valid. + if (!index->IsValid(BLOCK_VALID_SCRIPTS)) { + // This flag will be removed once the block is fully validated by a + // background chainstate. + index->nStatus |= BLOCK_ASSUMED_VALID; + } + // Fake BLOCK_OPT_WITNESS so that CChainState::NeedsRedownload() // won't ask to rewind the entire assumed-valid chain on startup. - if (index->pprev && ::IsWitnessEnabled(index->pprev, ::Params().GetConsensus())) { + if (index->pprev && DeploymentActiveAt(*index, ::Params().GetConsensus(), Consensus::DEPLOYMENT_SEGWIT)) { index->nStatus |= BLOCK_OPT_WITNESS; } + + setDirtyBlockIndex.insert(index); + // Changes to the block index will be flushed to disk after this call + // returns in `ActivateSnapshot()`, when `MaybeRebalanceCaches()` is + // called, since we've added a snapshot chainstate and therefore will + // have to downsize the IBD chainstate, which will result in a call to + // `FlushStateToDisk(ALWAYS)`. } assert(index); @@ -5025,22 +4968,6 @@ bool ChainstateManager::IsSnapshotActive() const return m_snapshot_chainstate && m_active_chainstate == m_snapshot_chainstate.get(); } -CChainState& ChainstateManager::ValidatedChainstate() const -{ - LOCK(::cs_main); - if (m_snapshot_chainstate && IsSnapshotValidated()) { - return *m_snapshot_chainstate.get(); - } - assert(m_ibd_chainstate); - return *m_ibd_chainstate.get(); -} - -bool ChainstateManager::IsBackgroundIBD(CChainState* chainstate) const -{ - LOCK(::cs_main); - return (m_snapshot_chainstate && chainstate == m_ibd_chainstate.get()); -} - void ChainstateManager::Unload() { for (CChainState* chainstate : this->GetAll()) { diff --git a/src/validation.h b/src/validation.h index 5720ba8071..b10050f931 100644 --- a/src/validation.h +++ b/src/validation.h @@ -10,22 +10,18 @@ #include <config/bitcoin-config.h> #endif -#include <amount.h> +#include <arith_uint256.h> #include <attributes.h> -#include <coins.h> -#include <consensus/validation.h> -#include <crypto/common.h> // for ReadLE64 +#include <chain.h> +#include <consensus/amount.h> #include <fs.h> -#include <node/utxo_snapshot.h> #include <policy/feerate.h> #include <policy/packages.h> -#include <protocol.h> // For CMessageHeader::MessageStartChars #include <script/script_error.h> #include <sync.h> -#include <txmempool.h> // For CTxMemPool::cs #include <txdb.h> -#include <versionbits.h> -#include <serialize.h> +#include <txmempool.h> // For CTxMemPool::cs +#include <uint256.h> #include <util/check.h> #include <util/hasher.h> #include <util/translation.h> @@ -42,19 +38,13 @@ #include <vector> class CChainState; -class BlockValidationState; -class CBlockIndex; class CBlockTreeDB; -class CBlockUndo; class CChainParams; struct CCheckpointData; -class CInv; -class CConnman; -class CScriptCheck; class CTxMemPool; class ChainstateManager; +class SnapshotMetadata; struct ChainTxData; - struct DisconnectedBlockTransactions; struct PrecomputedTransactionData; struct LockPoints; @@ -110,6 +100,7 @@ extern RecursiveMutex cs_main; typedef std::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; extern Mutex g_best_block_mutex; extern std::condition_variable g_best_block_cv; +/** Used to notify getblocktemplate RPC of new tips. */ extern uint256 g_best_block; /** Whether there are dedicated script-checking threads running. * False indicates all script checking is done on the main threadMessageHandler thread. @@ -141,19 +132,7 @@ void UnloadBlockIndex(CTxMemPool* mempool, ChainstateManager& chainman); void StartScriptCheckWorkerThreads(int threads_num); /** Stop all of the script checking worker threads */ void StopScriptCheckWorkerThreads(); -/** - * Return transaction from the block at block_index. - * If block_index is not provided, fall back to mempool. - * If mempool is not provided or the tx couldn't be found in mempool, fall back to g_txindex. - * - * @param[in] block_index The block to read from disk, or nullptr - * @param[in] mempool If block_index is not provided, look in the mempool, if provided - * @param[in] hash The txid - * @param[in] consensusParams The params - * @param[out] hashBlock The hash of block_index, if the tx was found via block_index - * @returns The tx if found, otherwise nullptr - */ -CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock); + CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams); bool AbortNode(BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage = bilingual_str{}); @@ -212,7 +191,8 @@ struct PackageMempoolAcceptResult /** * Map from wtxid to finished MempoolAcceptResults. The client is responsible * for keeping track of the transaction objects themselves. If a result is not - * present, it means validation was unfinished for that transaction. + * present, it means validation was unfinished for that transaction. If there + * was a package-wide error (see result in m_state), m_tx_results will be empty. */ std::map<const uint256, const MempoolAcceptResult> m_tx_results; @@ -240,7 +220,8 @@ MempoolAcceptResult AcceptToMemoryPool(CChainState& active_chainstate, CTxMemPoo * @param[in] txns Group of transactions which may be independent or contain * parent-child dependencies. The transactions must not conflict * with each other, i.e., must not spend the same inputs. If any -* dependencies exist, parents must appear before children. +* dependencies exist, parents must appear anywhere in the list +* before their children. * @returns a PackageMempoolAcceptResult which includes a MempoolAcceptResult for each transaction. * If a transaction fails, validation will exit early and some results may be missing. */ @@ -345,10 +326,6 @@ bool TestBlockValidity(BlockValidationState& state, bool fCheckPOW = true, bool fCheckMerkleRoot = true) EXCLUSIVE_LOCKS_REQUIRED(cs_main); -/** Check whether witness commitments are required for a block, and whether to enforce NULLDUMMY (BIP 147) rules. - * Note that transaction witness validation rules are always enforced when P2SH is enforced. */ -bool IsWitnessEnabled(const CBlockIndex* pindexPrev, const Consensus::Params& params); - /** Update uncommitted block structures (currently: only the witness reserved value). This is safe for submitted blocks. */ void UpdateUncommittedBlockStructures(CBlock& block, const CBlockIndex* pindexPrev, const Consensus::Params& consensusParams); @@ -451,6 +428,10 @@ public: */ std::multimap<CBlockIndex*, CBlockIndex*> m_blocks_unlinked; + std::unique_ptr<CBlockTreeDB> m_block_tree_db GUARDED_BY(::cs_main); + + bool LoadBlockIndexDB(std::set<CBlockIndex*, CBlockIndexWorkComparator>& setBlockIndexCandidates) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /** * Load the blocktree off disk and into memory. Populate certain metadata * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as peripheral @@ -461,7 +442,6 @@ public: */ bool LoadBlockIndex( const Consensus::Params& consensus_params, - CBlockTreeDB& blocktree, std::set<CBlockIndex*, CBlockIndexWorkComparator>& block_index_candidates) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -570,9 +550,8 @@ protected: * Every received block is assigned a unique and increasing identifier, so we * know which one to give priority in case of a fork. */ - RecursiveMutex cs_nBlockSequenceId; /** Blocks loaded from disk are assigned id 0, so start the counter at 1. */ - int32_t nBlockSequenceId = 1; + int32_t nBlockSequenceId GUARDED_BY(::cs_main) = 1; /** Decreasing counter (used by subsequent preciousblock calls). */ int32_t nBlockReverseSequenceId = -1; /** chainwork for the last block that preciousblock has been applied to. */ @@ -592,8 +571,11 @@ protected: */ mutable std::atomic<bool> m_cached_finished_ibd{false}; - //! mempool that is kept in sync with the chain - CTxMemPool& m_mempool; + //! Optional mempool that is kept in sync with the chain. + //! Only the active chainstate has a mempool. + CTxMemPool* m_mempool; + + const CChainParams& m_params; //! Manages the UTXO set, which is a reflection of the contents of `m_chain`. std::unique_ptr<CoinsViews> m_coins_views; @@ -603,7 +585,16 @@ public: //! CChainState instances. BlockManager& m_blockman; - explicit CChainState(CTxMemPool& mempool, BlockManager& blockman, std::optional<uint256> from_snapshot_blockhash = std::nullopt); + //! The chainstate manager that owns this chainstate. The reference is + //! necessary so that this instance can check whether it is the active + //! chainstate within deeply nested method calls. + ChainstateManager& m_chainman; + + explicit CChainState( + CTxMemPool* mempool, + BlockManager& blockman, + ChainstateManager& chainman, + std::optional<uint256> from_snapshot_blockhash = std::nullopt); /** * Initialize the CoinsViews UTXO set database management data structures. The in-memory @@ -639,9 +630,10 @@ public: const std::optional<uint256> m_from_snapshot_blockhash; /** - * The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself and all ancestors) and - * as good as our current tip or better. Entries may be failed, though, and pruning nodes may be - * missing the data for the block. + * The set of all CBlockIndex entries with either BLOCK_VALID_TRANSACTIONS (for + * itself and all ancestors) *or* BLOCK_ASSUMED_VALID (if using background + * chainstates) and as good as our current tip or better. Entries may be failed, + * though, and pruning nodes may be missing the data for the block. */ std::set<CBlockIndex*, CBlockIndexWorkComparator> setBlockIndexCandidates; @@ -680,7 +672,7 @@ public: EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** Import blocks from an external file */ - void LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, FlatFilePos* dbp = nullptr); + void LoadExternalBlockFile(FILE* fileIn, FlatFilePos* dbp = nullptr); /** * Update the on-disk chain state. @@ -694,8 +686,7 @@ public: * @returns true unless a system error occurred */ bool FlushStateToDisk( - const CChainParams& chainparams, - BlockValidationState &state, + BlockValidationState& state, FlushStateMode mode, int nManualPruneHeight = 0); @@ -723,41 +714,40 @@ public: */ bool ActivateBestChain( BlockValidationState& state, - const CChainParams& chainparams, std::shared_ptr<const CBlock> pblock = nullptr) LOCKS_EXCLUDED(cs_main); - bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Block (dis)connection on a given view: DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view); bool ConnectBlock(const CBlock& block, BlockValidationState& state, CBlockIndex* pindex, - CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck = false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + CCoinsViewCache& view, bool fJustCheck = false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Apply the effects of a block disconnection on the UTXO set. - bool DisconnectTip(BlockValidationState& state, const CChainParams& chainparams, DisconnectedBlockTransactions* disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool.cs); + bool DisconnectTip(BlockValidationState& state, DisconnectedBlockTransactions* disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool->cs); // Manual block validity manipulation: /** Mark a block as precious and reorganize. * * May not be called in a validationinterface callback. */ - bool PreciousBlock(BlockValidationState& state, const CChainParams& params, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); + bool PreciousBlock(BlockValidationState& state, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); /** Mark a block as invalid. */ - bool InvalidateBlock(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); + bool InvalidateBlock(BlockValidationState& state, CBlockIndex* pindex) LOCKS_EXCLUDED(cs_main); /** Remove invalidity status from a block and its descendants. */ void ResetBlockFailureFlags(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Replay blocks that aren't fully applied to the database. */ - bool ReplayBlocks(const CChainParams& params); + bool ReplayBlocks(); /** Whether the chain state needs to be redownloaded due to lack of witness data */ - [[nodiscard]] bool NeedsRedownload(const CChainParams& params) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); + [[nodiscard]] bool NeedsRedownload() const EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Ensures we have a genesis block in the block tree, possibly writing one to disk. */ - bool LoadGenesisBlock(const CChainParams& chainparams); + bool LoadGenesisBlock(); void PruneBlockIndexCandidates(); - void UnloadBlockIndex(); + void UnloadBlockIndex() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** Check whether we are doing an initial block download (synchronizing from disk or network) */ bool IsInitialBlockDownload() const; @@ -767,41 +757,64 @@ public: * * By default this only executes fully when using the Regtest chain; see: fCheckBlockIndex. */ - void CheckBlockIndex(const Consensus::Params& consensusParams); + void CheckBlockIndex(); /** Load the persisted mempool from disk */ void LoadMempool(const ArgsManager& args); /** Update the chain tip based on database information, i.e. CoinsTip()'s best block. */ - bool LoadChainTip(const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool LoadChainTip() EXCLUSIVE_LOCKS_REQUIRED(cs_main); //! Dictates whether we need to flush the cache to disk or not. //! //! @return the state of the size of the coins cache. - CoinsCacheSizeState GetCoinsCacheSizeState(const CTxMemPool* tx_pool) - EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + CoinsCacheSizeState GetCoinsCacheSizeState() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); CoinsCacheSizeState GetCoinsCacheSizeState( - const CTxMemPool* tx_pool, size_t max_coins_cache_size_bytes, size_t max_mempool_size_bytes) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); std::string ToString() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); private: - bool ActivateBestChainStep(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool.cs); - bool ConnectTip(BlockValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions& disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool.cs); + bool ActivateBestChainStep(BlockValidationState& state, CBlockIndex* pindexMostWork, const std::shared_ptr<const CBlock>& pblock, bool& fInvalidFound, ConnectTrace& connectTrace) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool->cs); + bool ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, const std::shared_ptr<const CBlock>& pblock, ConnectTrace& connectTrace, DisconnectedBlockTransactions& disconnectpool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool->cs); - void InvalidBlockFound(CBlockIndex *pindex, const BlockValidationState &state) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void InvalidBlockFound(CBlockIndex* pindex, const BlockValidationState& state) EXCLUSIVE_LOCKS_REQUIRED(cs_main); CBlockIndex* FindMostWorkChain() EXCLUSIVE_LOCKS_REQUIRED(cs_main); - void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos, const Consensus::Params& consensusParams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs, const CChainParams& params) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs) EXCLUSIVE_LOCKS_REQUIRED(cs_main); void CheckForkWarningConditions() EXCLUSIVE_LOCKS_REQUIRED(cs_main); void InvalidChainFound(CBlockIndex* pindexNew) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - bool LoadBlockIndexDB(const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + //! Indirection necessary to make lock annotations work with an optional mempool. + RecursiveMutex* MempoolMutex() const LOCK_RETURNED(m_mempool->cs) + { + return m_mempool ? &m_mempool->cs : nullptr; + } + + /** + * Make mempool consistent after a reorg, by re-adding or recursively erasing + * disconnected block transactions from the mempool, and also removing any + * other transactions from the mempool that are no longer valid given the new + * tip/height. + * + * Note: we assume that disconnectpool only contains transactions that are NOT + * confirmed in the current chain nor already in the mempool (otherwise, + * in-mempool descendants of such transactions would be removed). + * + * Passing fAddToMempool=false will skip trying to add the transactions back, + * and instead just erase from the mempool as needed. + */ + void MaybeUpdateMempoolForReorg( + DisconnectedBlockTransactions& disconnectpool, + bool fAddToMempool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_mempool->cs); + + /** Check warning conditions and do some notifications on new chain tip set. */ + void UpdateTip(const CBlockIndex* pindexNew) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); friend ChainstateManager; }; @@ -832,12 +845,6 @@ private: * *Background IBD chainstate*: an IBD chainstate for which the * IBD process is happening in the background while use of the * active (snapshot) chainstate allows the rest of the system to function. - * - * *Validated chainstate*: the most-work chainstate which has been validated - * locally via initial block download. This will be the snapshot chainstate - * if a snapshot was loaded and all blocks up to the snapshot starting point - * have been downloaded and validated (via background validation), otherwise - * it will be the IBD chainstate. */ class ChainstateManager { @@ -912,8 +919,10 @@ public: // constructor //! @param[in] snapshot_blockhash If given, signify that this chainstate //! is based on a snapshot. - CChainState& InitializeChainstate(CTxMemPool& mempool, const std::optional<uint256>& snapshot_blockhash = std::nullopt) - EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + CChainState& InitializeChainstate( + CTxMemPool* mempool, + const std::optional<uint256>& snapshot_blockhash = std::nullopt) + LIFETIMEBOUND EXCLUSIVE_LOCKS_REQUIRED(::cs_main); //! Get all chainstates currently being used. std::vector<CChainState*> GetAll(); @@ -954,19 +963,6 @@ public: //! Is there a snapshot in use and has it been fully validated? bool IsSnapshotValidated() const { return m_snapshot_validated; } - //! @returns true if this chainstate is being used to validate an active - //! snapshot in the background. - bool IsBackgroundIBD(CChainState* chainstate) const; - - //! Return the most-work chainstate that has been fully validated. - //! - //! During background validation of a snapshot, this is the IBD chain. After - //! background validation has completed, this is the snapshot chain. - CChainState& ValidatedChainstate() const; - - CChain& ValidatedChain() const { return ValidatedChainstate().m_chain; } - CBlockIndex* ValidatedTip() const { return ValidatedChain().Tip(); } - /** * Process an incoming block. This only returns after the best known valid * block is made active. Note that it does not, however, guarantee that the @@ -1002,7 +998,7 @@ public: bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, BlockValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr) LOCKS_EXCLUDED(cs_main); //! Load the block tree and coins database from disk, initializing state if we're running with -reindex - bool LoadBlockIndex(const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + bool LoadBlockIndex() EXCLUSIVE_LOCKS_REQUIRED(cs_main); //! Unload block index and chain data before shutdown. void Unload() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); @@ -1021,16 +1017,6 @@ public: } }; -/** Global variable that points to the active block tree (protected by cs_main) */ -extern std::unique_ptr<CBlockTreeDB> pblocktree; - -extern VersionBitsCache versionbitscache; - -/** - * Determine what nVersion a new block should use. - */ -int32_t ComputeBlockVersion(const CBlockIndex* pindexPrev, const Consensus::Params& params); - using FopenFn = std::function<FILE*(const fs::path&, const char*)>; /** Dump the mempool to disk. */ diff --git a/src/versionbits.cpp b/src/versionbits.cpp index df2ec4e056..94c3c9559f 100644 --- a/src/versionbits.cpp +++ b/src/versionbits.cpp @@ -190,29 +190,48 @@ public: } // namespace -ThresholdState VersionBitsState(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos, VersionBitsCache& cache) +ThresholdState VersionBitsCache::State(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos) { - return VersionBitsConditionChecker(pos).GetStateFor(pindexPrev, params, cache.caches[pos]); + LOCK(m_mutex); + return VersionBitsConditionChecker(pos).GetStateFor(pindexPrev, params, m_caches[pos]); } -BIP9Stats VersionBitsStatistics(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos) +BIP9Stats VersionBitsCache::Statistics(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos) { return VersionBitsConditionChecker(pos).GetStateStatisticsFor(pindexPrev, params); } -int VersionBitsStateSinceHeight(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos, VersionBitsCache& cache) +int VersionBitsCache::StateSinceHeight(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos) { - return VersionBitsConditionChecker(pos).GetStateSinceHeightFor(pindexPrev, params, cache.caches[pos]); + LOCK(m_mutex); + return VersionBitsConditionChecker(pos).GetStateSinceHeightFor(pindexPrev, params, m_caches[pos]); } -uint32_t VersionBitsMask(const Consensus::Params& params, Consensus::DeploymentPos pos) +uint32_t VersionBitsCache::Mask(const Consensus::Params& params, Consensus::DeploymentPos pos) { return VersionBitsConditionChecker(pos).Mask(params); } +int32_t VersionBitsCache::ComputeBlockVersion(const CBlockIndex* pindexPrev, const Consensus::Params& params) +{ + LOCK(m_mutex); + int32_t nVersion = VERSIONBITS_TOP_BITS; + + for (int i = 0; i < (int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; i++) { + Consensus::DeploymentPos pos = static_cast<Consensus::DeploymentPos>(i); + ThresholdState state = VersionBitsConditionChecker(pos).GetStateFor(pindexPrev, params, m_caches[pos]); + if (state == ThresholdState::LOCKED_IN || state == ThresholdState::STARTED) { + nVersion |= Mask(params, pos); + } + } + + return nVersion; +} + void VersionBitsCache::Clear() { + LOCK(m_mutex); for (unsigned int d = 0; d < Consensus::MAX_VERSION_BITS_DEPLOYMENTS; d++) { - caches[d].clear(); + m_caches[d].clear(); } } diff --git a/src/versionbits.h b/src/versionbits.h index 634a848ef5..0b2f4a0258 100644 --- a/src/versionbits.h +++ b/src/versionbits.h @@ -6,6 +6,8 @@ #define BITCOIN_VERSIONBITS_H #include <chain.h> +#include <sync.h> + #include <map> /** What block version to use for new blocks (pre versionbits) */ @@ -71,21 +73,31 @@ public: int GetStateSinceHeightFor(const CBlockIndex* pindexPrev, const Consensus::Params& params, ThresholdConditionCache& cache) const; }; -/** BIP 9 allows multiple softforks to be deployed in parallel. We cache per-period state for every one of them - * keyed by the bit position used to signal support. */ -struct VersionBitsCache +/** BIP 9 allows multiple softforks to be deployed in parallel. We cache + * per-period state for every one of them. */ +class VersionBitsCache { - ThresholdConditionCache caches[Consensus::MAX_VERSION_BITS_DEPLOYMENTS]; +private: + Mutex m_mutex; + ThresholdConditionCache m_caches[Consensus::MAX_VERSION_BITS_DEPLOYMENTS] GUARDED_BY(m_mutex); + +public: + /** Get the numerical statistics for a given deployment for the signalling period that includes the block after pindexPrev. */ + static BIP9Stats Statistics(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos); + + static uint32_t Mask(const Consensus::Params& params, Consensus::DeploymentPos pos); + + /** Get the BIP9 state for a given deployment for the block after pindexPrev. */ + ThresholdState State(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos); + + /** Get the block height at which the BIP9 deployment switched into the state for the block after pindexPrev. */ + int StateSinceHeight(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos); + + /** Determine what nVersion a new block should use + */ + int32_t ComputeBlockVersion(const CBlockIndex* pindexPrev, const Consensus::Params& params); void Clear(); }; -/** Get the BIP9 state for a given deployment at the current tip. */ -ThresholdState VersionBitsState(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos, VersionBitsCache& cache); -/** Get the numerical statistics for the BIP9 state for a given deployment at the current tip. */ -BIP9Stats VersionBitsStatistics(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos); -/** Get the block height at which the BIP9 deployment switched into the state for the block building on the current tip. */ -int VersionBitsStateSinceHeight(const CBlockIndex* pindexPrev, const Consensus::Params& params, Consensus::DeploymentPos pos, VersionBitsCache& cache); -uint32_t VersionBitsMask(const Consensus::Params& params, Consensus::DeploymentPos pos); - #endif // BITCOIN_VERSIONBITS_H diff --git a/src/versionbitsinfo.cpp b/src/versionbitsinfo.cpp deleted file mode 100644 index fa41bad46d..0000000000 --- a/src/versionbitsinfo.cpp +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) 2016-2020 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <versionbitsinfo.h> - -#include <consensus/params.h> - -const struct VBDeploymentInfo VersionBitsDeploymentInfo[Consensus::MAX_VERSION_BITS_DEPLOYMENTS] = { - { - /*.name =*/ "testdummy", - /*.gbt_force =*/ true, - }, - { - /*.name =*/ "taproot", - /*.gbt_force =*/ true, - }, -}; diff --git a/src/versionbitsinfo.h b/src/versionbitsinfo.h deleted file mode 100644 index a7822bc747..0000000000 --- a/src/versionbitsinfo.h +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) 2016-2018 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_VERSIONBITSINFO_H -#define BITCOIN_VERSIONBITSINFO_H - -struct VBDeploymentInfo { - /** Deployment name */ - const char *name; - /** Whether GBT clients can safely ignore this rule in simplified usage */ - bool gbt_force; -}; - -extern const struct VBDeploymentInfo VersionBitsDeploymentInfo[]; - -#endif // BITCOIN_VERSIONBITSINFO_H diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index 1dc23374e3..2eb4d3106c 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -375,7 +375,7 @@ void BerkeleyBatch::Flush() nMinutes = 1; if (env) { // env is nullptr for dummy databases (i.e. in tests). Don't actually flush if env is nullptr so we don't segfault - env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); + env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetIntArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0); } } diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index 85cbec76b7..edd81e590f 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -9,9 +9,14 @@ #include <policy/feerate.h> #include <policy/fees.h> #include <primitives/transaction.h> +#include <script/keyorigin.h> +#include <script/signingprovider.h> #include <script/standard.h> #include <optional> +#include <algorithm> +#include <map> +#include <set> const int DEFAULT_MIN_DEPTH = 0; const int DEFAULT_MAX_DEPTH = 9999999; @@ -53,6 +58,8 @@ public: int m_min_depth = DEFAULT_MIN_DEPTH; //! Maximum chain depth value for coin availability int m_max_depth = DEFAULT_MAX_DEPTH; + //! SigningProvider that has pubkeys and scripts to do spend size estimation for external inputs + FlatSigningProvider m_external_provider; CCoinControl(); @@ -66,11 +73,32 @@ public: return (setSelected.count(output) > 0); } + bool IsExternalSelected(const COutPoint& output) const + { + return (m_external_txouts.count(output) > 0); + } + + bool GetExternalOutput(const COutPoint& outpoint, CTxOut& txout) const + { + const auto ext_it = m_external_txouts.find(outpoint); + if (ext_it == m_external_txouts.end()) { + return false; + } + txout = ext_it->second; + return true; + } + void Select(const COutPoint& output) { setSelected.insert(output); } + void SelectExternal(const COutPoint& outpoint, const CTxOut& txout) + { + setSelected.insert(outpoint); + m_external_txouts.emplace(outpoint, txout); + } + void UnSelect(const COutPoint& output) { setSelected.erase(output); @@ -88,6 +116,7 @@ public: private: std::set<COutPoint> setSelected; + std::map<COutPoint, CTxOut> m_external_txouts; }; #endif // BITCOIN_WALLET_COINCONTROL_H diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 6d502e1df1..e1ca3fb379 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -4,10 +4,13 @@ #include <wallet/coinselection.h> +#include <consensus/amount.h> #include <policy/feerate.h> +#include <util/check.h> #include <util/system.h> #include <util/moneystr.h> +#include <numeric> #include <optional> // Descending order comparator @@ -168,6 +171,30 @@ bool SelectCoinsBnB(std::vector<OutputGroup>& utxo_pool, const CAmount& selectio return true; } +std::optional<std::pair<std::set<CInputCoin>, CAmount>> SelectCoinsSRD(const std::vector<OutputGroup>& utxo_pool, CAmount target_value) +{ + std::set<CInputCoin> out_set; + CAmount value_ret = 0; + + std::vector<size_t> indexes; + indexes.resize(utxo_pool.size()); + std::iota(indexes.begin(), indexes.end(), 0); + Shuffle(indexes.begin(), indexes.end(), FastRandomContext()); + + CAmount selected_eff_value = 0; + for (const size_t i : indexes) { + const OutputGroup& group = utxo_pool.at(i); + Assume(group.GetSelectionAmount() > 0); + selected_eff_value += group.GetSelectionAmount(); + value_ret += group.m_value; + util::insert(out_set, group.m_outputs); + if (selected_eff_value >= target_value) { + return std::make_pair(out_set, value_ret); + } + } + return std::nullopt; +} + static void ApproximateBestSubset(const std::vector<OutputGroup>& groups, const CAmount& nTotalLower, const CAmount& nTargetValue, std::vector<char>& vfBest, CAmount& nBest, int iterations = 1000) { @@ -195,7 +222,7 @@ static void ApproximateBestSubset(const std::vector<OutputGroup>& groups, const //the selection random. if (nPass == 0 ? insecure_rand.randbool() : !vfIncluded[i]) { - nTotal += groups[i].m_value; + nTotal += groups[i].GetSelectionAmount(); vfIncluded[i] = true; if (nTotal >= nTargetValue) { @@ -205,7 +232,7 @@ static void ApproximateBestSubset(const std::vector<OutputGroup>& groups, const nBest = nTotal; vfBest = vfIncluded; } - nTotal -= groups[i].m_value; + nTotal -= groups[i].GetSelectionAmount(); vfIncluded[i] = false; } } @@ -279,13 +306,13 @@ bool KnapsackSolver(const CAmount& nTargetValue, std::vector<OutputGroup>& group } if (LogAcceptCategory(BCLog::SELECTCOINS)) { - LogPrint(BCLog::SELECTCOINS, "SelectCoins() best subset: "); /* Continued */ + std::string log_message{"Coin selection best subset: "}; for (unsigned int i = 0; i < applicable_groups.size(); i++) { if (vfBest[i]) { - LogPrint(BCLog::SELECTCOINS, "%s ", FormatMoney(applicable_groups[i].m_value)); /* Continued */ + log_message += strprintf("%s ", FormatMoney(applicable_groups[i].m_value)); } } - LogPrint(BCLog::SELECTCOINS, "total %s\n", FormatMoney(nBest)); + LogPrint(BCLog::SELECTCOINS, "%stotal %s\n", log_message, FormatMoney(nBest)); } } @@ -341,3 +368,30 @@ CAmount OutputGroup::GetSelectionAmount() const { return m_subtract_fee_outputs ? m_value : effective_value; } + +CAmount GetSelectionWaste(const std::set<CInputCoin>& inputs, CAmount change_cost, CAmount target, bool use_effective_value) +{ + // This function should not be called with empty inputs as that would mean the selection failed + assert(!inputs.empty()); + + // Always consider the cost of spending an input now vs in the future. + CAmount waste = 0; + CAmount selected_effective_value = 0; + for (const CInputCoin& coin : inputs) { + waste += coin.m_fee - coin.m_long_term_fee; + selected_effective_value += use_effective_value ? coin.effective_value : coin.txout.nValue; + } + + if (change_cost) { + // Consider the cost of making change and spending it in the future + // If we aren't making change, the caller should've set change_cost to 0 + assert(change_cost > 0); + waste += change_cost; + } else { + // When we are not making change (change_cost == 0), consider the excess we are throwing away to fees + assert(selected_effective_value >= target); + waste += selected_effective_value - target; + } + + return waste; +} diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index 7a3fb82139..e7d467660f 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -5,11 +5,13 @@ #ifndef BITCOIN_WALLET_COINSELECTION_H #define BITCOIN_WALLET_COINSELECTION_H -#include <amount.h> +#include <consensus/amount.h> #include <policy/feerate.h> #include <primitives/transaction.h> #include <random.h> +#include <optional> + //! target minimum change amount static constexpr CAmount MIN_CHANGE{COIN / 100}; //! final minimum change amount after paying for fees @@ -35,6 +37,18 @@ public: m_input_bytes = input_bytes; } + CInputCoin(const COutPoint& outpoint_in, const CTxOut& txout_in) + { + outpoint = outpoint_in; + txout = txout_in; + effective_value = txout.nValue; + } + + CInputCoin(const COutPoint& outpoint_in, const CTxOut& txout_in, int input_bytes) : CInputCoin(outpoint_in, txout_in) + { + m_input_bytes = input_bytes; + } + COutPoint outpoint; CTxOut txout; CAmount effective_value; @@ -166,8 +180,34 @@ struct OutputGroup CAmount GetSelectionAmount() const; }; +/** Compute the waste for this result given the cost of change + * and the opportunity cost of spending these inputs now vs in the future. + * If change exists, waste = change_cost + inputs * (effective_feerate - long_term_feerate) + * If no change, waste = excess + inputs * (effective_feerate - long_term_feerate) + * where excess = selected_effective_value - target + * change_cost = effective_feerate * change_output_size + long_term_feerate * change_spend_size + * + * @param[in] inputs The selected inputs + * @param[in] change_cost The cost of creating change and spending it in the future. + * Only used if there is change, in which case it must be positive. + * Must be 0 if there is no change. + * @param[in] target The amount targeted by the coin selection algorithm. + * @param[in] use_effective_value Whether to use the input's effective value (when true) or the real value (when false). + * @return The waste + */ +[[nodiscard]] CAmount GetSelectionWaste(const std::set<CInputCoin>& inputs, CAmount change_cost, CAmount target, bool use_effective_value = true); + bool SelectCoinsBnB(std::vector<OutputGroup>& utxo_pool, const CAmount& selection_target, const CAmount& cost_of_change, std::set<CInputCoin>& out_set, CAmount& value_ret); +/** Select coins by Single Random Draw. OutputGroups are selected randomly from the eligible + * outputs until the target is satisfied + * + * @param[in] utxo_pool The positive effective value OutputGroups eligible for selection + * @param[in] target_value The target value to select for + * @returns If successful, a pair of set of outputs and total selected value, otherwise, std::nullopt + */ +std::optional<std::pair<std::set<CInputCoin>, CAmount>> SelectCoinsSRD(const std::vector<OutputGroup>& utxo_pool, CAmount target_value); + // Original coin selection algorithm as a fallback bool KnapsackSolver(const CAmount& nTargetValue, std::vector<OutputGroup>& groups, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet); diff --git a/src/wallet/context.h b/src/wallet/context.h index a83591154f..a382fb9021 100644 --- a/src/wallet/context.h +++ b/src/wallet/context.h @@ -5,11 +5,22 @@ #ifndef BITCOIN_WALLET_CONTEXT_H #define BITCOIN_WALLET_CONTEXT_H +#include <sync.h> + +#include <functional> +#include <list> +#include <memory> +#include <vector> + class ArgsManager; +class CWallet; namespace interfaces { class Chain; +class Wallet; } // namespace interfaces +using LoadWalletFn = std::function<void(std::unique_ptr<interfaces::Wallet> wallet)>; + //! WalletContext struct containing references to state shared between CWallet //! instances, like the reference to the chain interface, and the list of opened //! wallets. @@ -22,7 +33,10 @@ class Chain; //! behavior. struct WalletContext { interfaces::Chain* chain{nullptr}; - ArgsManager* args{nullptr}; + ArgsManager* args{nullptr}; // Currently a raw pointer because the memory is not managed by this struct + Mutex wallets_mutex; + std::vector<std::shared_ptr<CWallet>> wallets GUARDED_BY(wallets_mutex); + std::list<LoadWalletFn> wallet_load_fns GUARDED_BY(wallets_mutex); //! Declare default constructor and destructor that are not inline, so code //! instantiating the WalletContext struct doesn't need to #include class diff --git a/src/wallet/external_signer_scriptpubkeyman.h b/src/wallet/external_signer_scriptpubkeyman.h index 8eed947b7b..61df3d0015 100644 --- a/src/wallet/external_signer_scriptpubkeyman.h +++ b/src/wallet/external_signer_scriptpubkeyman.h @@ -15,8 +15,8 @@ class ExternalSignerScriptPubKeyMan : public DescriptorScriptPubKeyMan ExternalSignerScriptPubKeyMan(WalletStorage& storage, WalletDescriptor& descriptor) : DescriptorScriptPubKeyMan(storage, descriptor) {} - ExternalSignerScriptPubKeyMan(WalletStorage& storage, bool internal) - : DescriptorScriptPubKeyMan(storage, internal) + ExternalSignerScriptPubKeyMan(WalletStorage& storage) + : DescriptorScriptPubKeyMan(storage) {} /** Provide a descriptor at setup time diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 30fef50c3b..f2de68295e 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -12,6 +12,8 @@ #include <wallet/coincontrol.h> #include <wallet/feebumper.h> #include <wallet/fees.h> +#include <wallet/receive.h> +#include <wallet/spend.h> #include <wallet/wallet.h> //! Check whether transaction has descendant in wallet or mempool, or has been @@ -30,7 +32,7 @@ static feebumper::Result PreconditionChecks(const CWallet& wallet, const CWallet } } - if (wtx.GetDepthInMainChain() != 0) { + if (wallet.GetTxDepthInMainChain(wtx) != 0) { errors.push_back(Untranslated("Transaction has been mined, or is conflicted with a mined transaction")); return feebumper::Result::WALLET_ERROR; } @@ -48,7 +50,7 @@ static feebumper::Result PreconditionChecks(const CWallet& wallet, const CWallet // check that original tx consists entirely of our inputs // if not, we can't bump the fee, because the wallet has no way of knowing the value of the other inputs (thus the fee) isminefilter filter = wallet.GetLegacyScriptPubKeyMan() && wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) ? ISMINE_WATCH_ONLY : ISMINE_SPENDABLE; - if (!wallet.IsAllFromMe(*wtx.tx, filter)) { + if (!AllInputsMine(wallet, *wtx.tx, filter)) { errors.push_back(Untranslated("Transaction contains inputs that don't belong to this wallet")); return feebumper::Result::WALLET_ERROR; } @@ -81,7 +83,7 @@ static feebumper::Result CheckFeeRate(const CWallet& wallet, const CWalletTx& wt // Given old total fee and transaction size, calculate the old feeRate isminefilter filter = wallet.GetLegacyScriptPubKeyMan() && wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) ? ISMINE_WATCH_ONLY : ISMINE_SPENDABLE; - CAmount old_fee = wtx.GetDebit(filter) - wtx.tx->GetValueOut(); + CAmount old_fee = CachedTxGetDebit(wallet, wtx, filter) - wtx.tx->GetValueOut(); const int64_t txSize = GetVirtualTransactionSize(*(wtx.tx)); CFeeRate nOldFeeRate(old_fee, txSize); // Min total fee is old fee + relay fee @@ -174,7 +176,7 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo // Fill in recipients(and preserve a single change key if there is one) std::vector<CRecipient> recipients; for (const auto& output : wtx.tx->vout) { - if (!wallet.IsChange(output)) { + if (!OutputIsChange(wallet, output)) { CRecipient recipient = {output.scriptPubKey, output.nValue, false}; recipients.push_back(recipient); } else { @@ -185,7 +187,7 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo } isminefilter filter = wallet.GetLegacyScriptPubKeyMan() && wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) ? ISMINE_WATCH_ONLY : ISMINE_SPENDABLE; - old_fee = wtx.GetDebit(filter) - wtx.tx->GetValueOut(); + old_fee = CachedTxGetDebit(wallet, wtx, filter) - wtx.tx->GetValueOut(); if (coin_control.m_feerate) { // The user provided a feeRate argument. @@ -220,7 +222,7 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo int change_pos_in_out = -1; // No requested location for change bilingual_str fail_reason; FeeCalculation fee_calc_out; - if (!wallet.CreateTransaction(recipients, tx_new, fee_ret, change_pos_in_out, fail_reason, new_coin_control, fee_calc_out, false)) { + if (!CreateTransaction(wallet, recipients, tx_new, fee_ret, change_pos_in_out, fail_reason, new_coin_control, fee_calc_out, false)) { errors.push_back(Untranslated("Unable to create transaction.") + Untranslated(" ") + fail_reason); return Result::WALLET_ERROR; } diff --git a/src/wallet/fees.h b/src/wallet/fees.h index 434f211dc2..d6d625d9c1 100644 --- a/src/wallet/fees.h +++ b/src/wallet/fees.h @@ -6,7 +6,7 @@ #ifndef BITCOIN_WALLET_FEES_H #define BITCOIN_WALLET_FEES_H -#include <amount.h> +#include <consensus/amount.h> class CCoinControl; class CFeeRate; diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index eb0d6316c0..59a59f9794 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -5,6 +5,7 @@ #include <init.h> #include <interfaces/chain.h> +#include <interfaces/init.h> #include <interfaces/wallet.h> #include <net.h> #include <node/context.h> @@ -45,6 +46,7 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const argsman.AddArg("-addresstype", strprintf("What type of addresses to use (\"legacy\", \"p2sh-segwit\", or \"bech32\", default: \"%s\")", FormatOutputType(DEFAULT_ADDRESS_TYPE)), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); argsman.AddArg("-avoidpartialspends", strprintf("Group outputs by address, selecting many (possibly all) or none, instead of selecting on a per-output basis. Privacy is improved as addresses are mostly swept with fewer transactions and outputs are aggregated in clean change addresses. It may result in higher fees due to less optimal coin selection caused by this added limitation and possibly a larger-than-necessary number of inputs being used. Always enabled for wallets with \"avoid_reuse\" enabled, otherwise default: %u.", DEFAULT_AVOIDPARTIALSPENDS), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); argsman.AddArg("-changetype", "What type of change to use (\"legacy\", \"p2sh-segwit\", or \"bech32\"). Default is same as -addresstype, except when -addresstype=p2sh-segwit a native segwit output is used when sending to a native segwit address)", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); + argsman.AddArg("-consolidatefeerate=<amt>", strprintf("The maximum feerate (in %s/kvB) at which transaction building may use more inputs than strictly necessary so that the wallet's UTXO pool can be reduced (default: %s).", CURRENCY_UNIT, FormatMoney(DEFAULT_CONSOLIDATE_FEERATE)), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); argsman.AddArg("-disablewallet", "Do not load the wallet and disable wallet RPC calls", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); argsman.AddArg("-discardfee=<amt>", strprintf("The fee rate (in %s/kvB) that indicates your tolerance for discarding change by adding it to the fee (default: %s). " "Note: An output is discarded if it is dust at this rate, but we will always discard up to the dust relay fee and a discard fee above that is limited by the fee estimate for the longest target", @@ -60,7 +62,6 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MINFEE)), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); argsman.AddArg("-paytxfee=<amt>", strprintf("Fee rate (in %s/kvB) to add to transactions you send (default: %s)", CURRENCY_UNIT, FormatMoney(CFeeRate{DEFAULT_PAY_TX_FEE}.GetFeePerK())), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); - argsman.AddArg("-rescan", "Rescan the block chain for missing wallet transactions on startup", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); #ifdef ENABLE_EXTERNAL_SIGNER argsman.AddArg("-signer=<cmd>", "External signing tool, see doc/external-signer.md", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); #endif @@ -129,7 +130,7 @@ void WalletInit::Construct(NodeContext& node) const LogPrintf("Wallet disabled!\n"); return; } - auto wallet_client = interfaces::MakeWalletClient(*node.chain, args); + auto wallet_client = node.init->makeWalletClient(*node.chain); node.wallet_client = wallet_client.get(); node.chain_clients.emplace_back(std::move(wallet_client)); } diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index 5a832d020b..7d0cdb6934 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -4,7 +4,7 @@ #include <interfaces/wallet.h> -#include <amount.h> +#include <consensus/amount.h> #include <interfaces/chain.h> #include <interfaces/handler.h> #include <policy/fees.h> @@ -16,13 +16,16 @@ #include <uint256.h> #include <util/check.h> #include <util/system.h> +#include <util/translation.h> #include <util/ui_change_type.h> #include <wallet/context.h> #include <wallet/feebumper.h> #include <wallet/fees.h> #include <wallet/ismine.h> #include <wallet/load.h> +#include <wallet/receive.h> #include <wallet/rpcwallet.h> +#include <wallet/spend.h> #include <wallet/wallet.h> #include <memory> @@ -54,7 +57,7 @@ WalletTx MakeWalletTx(CWallet& wallet, const CWalletTx& wtx) result.tx = wtx.tx; result.txin_is_mine.reserve(wtx.tx->vin.size()); for (const auto& txin : wtx.tx->vin) { - result.txin_is_mine.emplace_back(wallet.IsMine(txin)); + result.txin_is_mine.emplace_back(InputIsMine(wallet, txin)); } result.txout_is_mine.reserve(wtx.tx->vout.size()); result.txout_address.reserve(wtx.tx->vout.size()); @@ -66,9 +69,9 @@ WalletTx MakeWalletTx(CWallet& wallet, const CWalletTx& wtx) wallet.IsMine(result.txout_address.back()) : ISMINE_NO); } - result.credit = wtx.GetCredit(ISMINE_ALL); - result.debit = wtx.GetDebit(ISMINE_ALL); - result.change = wtx.GetChange(); + result.credit = CachedTxGetCredit(wallet, wtx, ISMINE_ALL); + result.debit = CachedTxGetDebit(wallet, wtx, ISMINE_ALL); + result.change = CachedTxGetChange(wallet, wtx); result.time = wtx.GetTxTime(); result.value_map = wtx.mapValue; result.is_coinbase = wtx.IsCoinBase(); @@ -80,15 +83,15 @@ WalletTxStatus MakeWalletTxStatus(const CWallet& wallet, const CWalletTx& wtx) { WalletTxStatus result; result.block_height = wtx.m_confirm.block_height > 0 ? wtx.m_confirm.block_height : std::numeric_limits<int>::max(); - result.blocks_to_maturity = wtx.GetBlocksToMaturity(); - result.depth_in_main_chain = wtx.GetDepthInMainChain(); + result.blocks_to_maturity = wallet.GetTxBlocksToMaturity(wtx); + result.depth_in_main_chain = wallet.GetTxDepthInMainChain(wtx); result.time_received = wtx.nTimeReceived; result.lock_time = wtx.tx->nLockTime; result.is_final = wallet.chain().checkFinalTx(*wtx.tx); - result.is_trusted = wtx.IsTrusted(); + result.is_trusted = CachedTxIsTrusted(wallet, wtx); result.is_abandoned = wtx.isAbandoned(); result.is_coinbase = wtx.IsCoinBase(); - result.is_in_main_chain = wtx.IsInMainChain(); + result.is_in_main_chain = wallet.IsTxInMainChain(wtx); return result; } @@ -109,7 +112,7 @@ WalletTxOut MakeWalletTxOut(const CWallet& wallet, class WalletImpl : public Wallet { public: - explicit WalletImpl(const std::shared_ptr<CWallet>& wallet) : m_wallet(wallet) {} + explicit WalletImpl(WalletContext& context, const std::shared_ptr<CWallet>& wallet) : m_context(context), m_wallet(wallet) {} bool encryptWallet(const SecureString& wallet_passphrase) override { @@ -130,7 +133,7 @@ public: bool getNewDestination(const OutputType type, const std::string label, CTxDestination& dest) override { LOCK(m_wallet->cs_wallet); - std::string error; + bilingual_str error; return m_wallet->GetNewDestination(type, label, dest, error); } bool getPubKey(const CScript& script, const CKeyID& address, CPubKey& pub_key) override @@ -211,15 +214,17 @@ public: LOCK(m_wallet->cs_wallet); return m_wallet->DisplayAddress(dest); } - void lockCoin(const COutPoint& output) override + bool lockCoin(const COutPoint& output, const bool write_to_db) override { LOCK(m_wallet->cs_wallet); - return m_wallet->LockCoin(output); + std::unique_ptr<WalletBatch> batch = write_to_db ? std::make_unique<WalletBatch>(m_wallet->GetDatabase()) : nullptr; + return m_wallet->LockCoin(output, batch.get()); } - void unlockCoin(const COutPoint& output) override + bool unlockCoin(const COutPoint& output) override { LOCK(m_wallet->cs_wallet); - return m_wallet->UnlockCoin(output); + std::unique_ptr<WalletBatch> batch = std::make_unique<WalletBatch>(m_wallet->GetDatabase()); + return m_wallet->UnlockCoin(output, batch.get()); } bool isLockedCoin(const COutPoint& output) override { @@ -241,7 +246,7 @@ public: LOCK(m_wallet->cs_wallet); CTransactionRef tx; FeeCalculation fee_calc_out; - if (!m_wallet->CreateTransaction(recipients, tx, fee, change_pos, + if (!CreateTransaction(*m_wallet, recipients, tx, fee, change_pos, fail_reason, coin_control, fee_calc_out, sign)) { return {}; } @@ -357,7 +362,7 @@ public: } WalletBalances getBalances() override { - const auto bal = m_wallet->GetBalance(); + const auto bal = GetBalance(*m_wallet); WalletBalances result; result.balance = bal.m_mine_trusted; result.unconfirmed_balance = bal.m_mine_untrusted_pending; @@ -380,15 +385,15 @@ public: balances = getBalances(); return true; } - CAmount getBalance() override { return m_wallet->GetBalance().m_mine_trusted; } + CAmount getBalance() override { return GetBalance(*m_wallet).m_mine_trusted; } CAmount getAvailableBalance(const CCoinControl& coin_control) override { - return m_wallet->GetAvailableBalance(&coin_control); + return GetAvailableBalance(*m_wallet, &coin_control); } isminetype txinIsMine(const CTxIn& txin) override { LOCK(m_wallet->cs_wallet); - return m_wallet->IsMine(txin); + return InputIsMine(*m_wallet, txin); } isminetype txoutIsMine(const CTxOut& txout) override { @@ -403,13 +408,13 @@ public: CAmount getCredit(const CTxOut& txout, isminefilter filter) override { LOCK(m_wallet->cs_wallet); - return m_wallet->GetCredit(txout, filter); + return OutputGetCredit(*m_wallet, txout, filter); } CoinsList listCoins() override { LOCK(m_wallet->cs_wallet); CoinsList result; - for (const auto& entry : m_wallet->ListCoins()) { + for (const auto& entry : ListCoins(*m_wallet)) { auto& group = result[entry.first]; for (const auto& coin : entry.second) { group.emplace_back(COutPoint(coin.tx->GetHash(), coin.i), @@ -427,7 +432,7 @@ public: result.emplace_back(); auto it = m_wallet->mapWallet.find(output.hash); if (it != m_wallet->mapWallet.end()) { - int depth = it->second.GetDepthInMainChain(); + int depth = m_wallet->GetTxDepthInMainChain(it->second); if (depth >= 0) { result.back() = MakeWalletTxOut(*m_wallet, it->second, output.n, depth); } @@ -457,7 +462,7 @@ public: CAmount getDefaultMaxTxFee() override { return m_wallet->m_default_max_tx_fee; } void remove() override { - RemoveWallet(m_wallet, false /* load_on_start */); + RemoveWallet(m_context, m_wallet, false /* load_on_start */); } bool isLegacy() override { return m_wallet->IsLegacy(); } std::unique_ptr<Handler> handleUnload(UnloadFn fn) override @@ -475,13 +480,13 @@ public: std::unique_ptr<Handler> handleAddressBookChanged(AddressBookChangedFn fn) override { return MakeHandler(m_wallet->NotifyAddressBookChanged.connect( - [fn](CWallet*, const CTxDestination& address, const std::string& label, bool is_mine, - const std::string& purpose, ChangeType status) { fn(address, label, is_mine, purpose, status); })); + [fn](const CTxDestination& address, const std::string& label, bool is_mine, + const std::string& purpose, ChangeType status) { fn(address, label, is_mine, purpose, status); })); } std::unique_ptr<Handler> handleTransactionChanged(TransactionChangedFn fn) override { return MakeHandler(m_wallet->NotifyTransactionChanged.connect( - [fn](CWallet*, const uint256& txid, ChangeType status) { fn(txid, status); })); + [fn](const uint256& txid, ChangeType status) { fn(txid, status); })); } std::unique_ptr<Handler> handleWatchOnlyChanged(WatchOnlyChangedFn fn) override { @@ -493,6 +498,7 @@ public: } CWallet* wallet() override { return m_wallet.get(); } + WalletContext& m_context; std::shared_ptr<CWallet> m_wallet; }; @@ -504,7 +510,7 @@ public: m_context.chain = &chain; m_context.args = &args; } - ~WalletClientImpl() override { UnloadWallets(); } + ~WalletClientImpl() override { UnloadWallets(m_context); } //! ChainClient methods void registerRpcs() override @@ -518,11 +524,11 @@ public: m_rpc_handlers.emplace_back(m_context.chain->handleRpc(m_rpc_commands.back())); } } - bool verify() override { return VerifyWallets(*m_context.chain); } - bool load() override { return LoadWallets(*m_context.chain); } - void start(CScheduler& scheduler) override { return StartWallets(scheduler, *Assert(m_context.args)); } - void flush() override { return FlushWallets(); } - void stop() override { return StopWallets(); } + bool verify() override { return VerifyWallets(m_context); } + bool load() override { return LoadWallets(m_context); } + void start(CScheduler& scheduler) override { return StartWallets(m_context, scheduler); } + void flush() override { return FlushWallets(m_context); } + void stop() override { return StopWallets(m_context); } void setMockTime(int64_t time) override { return SetMockTime(time); } //! WalletClient methods @@ -534,14 +540,14 @@ public: options.require_create = true; options.create_flags = wallet_creation_flags; options.create_passphrase = passphrase; - return MakeWallet(CreateWallet(*m_context.chain, name, true /* load_on_start */, options, status, error, warnings)); + return MakeWallet(m_context, CreateWallet(m_context, name, true /* load_on_start */, options, status, error, warnings)); } std::unique_ptr<Wallet> loadWallet(const std::string& name, bilingual_str& error, std::vector<bilingual_str>& warnings) override { DatabaseOptions options; DatabaseStatus status; options.require_existing = true; - return MakeWallet(LoadWallet(*m_context.chain, name, true /* load_on_start */, options, status, error, warnings)); + return MakeWallet(m_context, LoadWallet(m_context, name, true /* load_on_start */, options, status, error, warnings)); } std::string getWalletDir() override { @@ -558,15 +564,16 @@ public: std::vector<std::unique_ptr<Wallet>> getWallets() override { std::vector<std::unique_ptr<Wallet>> wallets; - for (const auto& wallet : GetWallets()) { - wallets.emplace_back(MakeWallet(wallet)); + for (const auto& wallet : GetWallets(m_context)) { + wallets.emplace_back(MakeWallet(m_context, wallet)); } return wallets; } std::unique_ptr<Handler> handleLoadWallet(LoadWalletFn fn) override { - return HandleLoadWallet(std::move(fn)); + return HandleLoadWallet(m_context, std::move(fn)); } + WalletContext* context() override { return &m_context; } WalletContext m_context; const std::vector<std::string> m_wallet_filenames; @@ -577,7 +584,7 @@ public: } // namespace wallet namespace interfaces { -std::unique_ptr<Wallet> MakeWallet(const std::shared_ptr<CWallet>& wallet) { return wallet ? std::make_unique<wallet::WalletImpl>(wallet) : nullptr; } +std::unique_ptr<Wallet> MakeWallet(WalletContext& context, const std::shared_ptr<CWallet>& wallet) { return wallet ? std::make_unique<wallet::WalletImpl>(context, wallet) : nullptr; } std::unique_ptr<WalletClient> MakeWalletClient(Chain& chain, ArgsManager& args) { diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index dbf9fd46b6..1b841026b8 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -8,18 +8,24 @@ #include <fs.h> #include <interfaces/chain.h> #include <scheduler.h> +#include <util/check.h> #include <util/string.h> #include <util/system.h> #include <util/translation.h> +#include <wallet/context.h> +#include <wallet/spend.h> #include <wallet/wallet.h> #include <wallet/walletdb.h> #include <univalue.h> -bool VerifyWallets(interfaces::Chain& chain) +bool VerifyWallets(WalletContext& context) { - if (gArgs.IsArgSet("-walletdir")) { - fs::path wallet_dir = gArgs.GetArg("-walletdir", ""); + interfaces::Chain& chain = *context.chain; + ArgsManager& args = *Assert(context.args); + + if (args.IsArgSet("-walletdir")) { + fs::path wallet_dir = args.GetArg("-walletdir", ""); boost::system::error_code error; // The canonical path cleans the path, preventing >1 Berkeley environment instances for the same directory fs::path canonical_wallet_dir = fs::canonical(wallet_dir, error); @@ -34,7 +40,7 @@ bool VerifyWallets(interfaces::Chain& chain) chain.initError(strprintf(_("Specified -walletdir \"%s\" is a relative path"), wallet_dir.string())); return false; } - gArgs.ForceSetArg("-walletdir", canonical_wallet_dir.string()); + args.ForceSetArg("-walletdir", canonical_wallet_dir.string()); } LogPrintf("Using wallet directory %s\n", GetWalletDir().string()); @@ -43,25 +49,27 @@ bool VerifyWallets(interfaces::Chain& chain) // For backwards compatibility if an unnamed top level wallet exists in the // wallets directory, include it in the default list of wallets to load. - if (!gArgs.IsArgSet("wallet")) { + if (!args.IsArgSet("wallet")) { DatabaseOptions options; DatabaseStatus status; bilingual_str error_string; options.require_existing = true; options.verify = false; if (MakeWalletDatabase("", options, status, error_string)) { - gArgs.LockSettings([&](util::Settings& settings) { - util::SettingsValue wallets(util::SettingsValue::VARR); - wallets.push_back(""); // Default wallet name is "" - settings.rw_settings["wallet"] = wallets; - }); + util::SettingsValue wallets(util::SettingsValue::VARR); + wallets.push_back(""); // Default wallet name is "" + // Pass write=false because no need to write file and probably + // better not to. If unnamed wallet needs to be added next startup + // and the setting is empty, this code will just run again. + chain.updateRwSetting("wallet", wallets, /* write= */ false); } } // Keep track of each wallet absolute path to detect duplicates. std::set<fs::path> wallet_paths; - for (const auto& wallet_file : gArgs.GetArgs("-wallet")) { + for (const auto& wallet : chain.getSettingsList("wallet")) { + const auto& wallet_file = wallet.get_str(); const fs::path path = fsbridge::AbsPathJoin(GetWalletDir(), wallet_file); if (!wallet_paths.insert(path).second) { @@ -87,11 +95,13 @@ bool VerifyWallets(interfaces::Chain& chain) return true; } -bool LoadWallets(interfaces::Chain& chain) +bool LoadWallets(WalletContext& context) { + interfaces::Chain& chain = *context.chain; try { std::set<fs::path> wallet_paths; - for (const std::string& name : gArgs.GetArgs("-wallet")) { + for (const auto& wallet : chain.getSettingsList("wallet")) { + const auto& name = wallet.get_str(); if (!wallet_paths.insert(name).second) { continue; } @@ -106,13 +116,13 @@ bool LoadWallets(interfaces::Chain& chain) continue; } chain.initMessage(_("Loading wallet…").translated); - std::shared_ptr<CWallet> pwallet = database ? CWallet::Create(&chain, name, std::move(database), options.create_flags, error, warnings) : nullptr; + std::shared_ptr<CWallet> pwallet = database ? CWallet::Create(context, name, std::move(database), options.create_flags, error, warnings) : nullptr; if (!warnings.empty()) chain.initWarning(Join(warnings, Untranslated("\n"))); if (!pwallet) { chain.initError(error); return false; } - AddWallet(pwallet); + AddWallet(context, pwallet); } return true; } catch (const std::runtime_error& e) { @@ -121,41 +131,41 @@ bool LoadWallets(interfaces::Chain& chain) } } -void StartWallets(CScheduler& scheduler, const ArgsManager& args) +void StartWallets(WalletContext& context, CScheduler& scheduler) { - for (const std::shared_ptr<CWallet>& pwallet : GetWallets()) { + for (const std::shared_ptr<CWallet>& pwallet : GetWallets(context)) { pwallet->postInitProcess(); } // Schedule periodic wallet flushes and tx rebroadcasts - if (args.GetBoolArg("-flushwallet", DEFAULT_FLUSHWALLET)) { - scheduler.scheduleEvery(MaybeCompactWalletDB, std::chrono::milliseconds{500}); + if (context.args->GetBoolArg("-flushwallet", DEFAULT_FLUSHWALLET)) { + scheduler.scheduleEvery([&context] { MaybeCompactWalletDB(context); }, std::chrono::milliseconds{500}); } - scheduler.scheduleEvery(MaybeResendWalletTxs, std::chrono::milliseconds{1000}); + scheduler.scheduleEvery([&context] { MaybeResendWalletTxs(context); }, std::chrono::milliseconds{1000}); } -void FlushWallets() +void FlushWallets(WalletContext& context) { - for (const std::shared_ptr<CWallet>& pwallet : GetWallets()) { + for (const std::shared_ptr<CWallet>& pwallet : GetWallets(context)) { pwallet->Flush(); } } -void StopWallets() +void StopWallets(WalletContext& context) { - for (const std::shared_ptr<CWallet>& pwallet : GetWallets()) { + for (const std::shared_ptr<CWallet>& pwallet : GetWallets(context)) { pwallet->Close(); } } -void UnloadWallets() +void UnloadWallets(WalletContext& context) { - auto wallets = GetWallets(); + auto wallets = GetWallets(context); while (!wallets.empty()) { auto wallet = wallets.back(); wallets.pop_back(); std::vector<bilingual_str> warnings; - RemoveWallet(wallet, std::nullopt, warnings); + RemoveWallet(context, wallet, /* load_on_start= */ std::nullopt, warnings); UnloadWallet(std::move(wallet)); } } diff --git a/src/wallet/load.h b/src/wallet/load.h index 7910f0d6e1..e207bc2e09 100644 --- a/src/wallet/load.h +++ b/src/wallet/load.h @@ -11,27 +11,28 @@ class ArgsManager; class CScheduler; +struct WalletContext; namespace interfaces { class Chain; } // namespace interfaces //! Responsible for reading and validating the -wallet arguments and verifying the wallet database. -bool VerifyWallets(interfaces::Chain& chain); +bool VerifyWallets(WalletContext& context); //! Load wallet databases. -bool LoadWallets(interfaces::Chain& chain); +bool LoadWallets(WalletContext& context); //! Complete startup of wallets. -void StartWallets(CScheduler& scheduler, const ArgsManager& args); +void StartWallets(WalletContext& context, CScheduler& scheduler); //! Flush all wallets in preparation for shutdown. -void FlushWallets(); +void FlushWallets(WalletContext& context); //! Stop all wallets. Wallets will be flushed first. -void StopWallets(); +void StopWallets(WalletContext& context); //! Close all wallets. -void UnloadWallets(); +void UnloadWallets(WalletContext& context); #endif // BITCOIN_WALLET_LOAD_H diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index de81dbf324..2fb274b55f 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -2,32 +2,33 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <consensus/amount.h> #include <consensus/consensus.h> #include <wallet/receive.h> #include <wallet/transaction.h> #include <wallet/wallet.h> -isminetype CWallet::IsMine(const CTxIn &txin) const +isminetype InputIsMine(const CWallet& wallet, const CTxIn &txin) { - AssertLockHeld(cs_wallet); - std::map<uint256, CWalletTx>::const_iterator mi = mapWallet.find(txin.prevout.hash); - if (mi != mapWallet.end()) + AssertLockHeld(wallet.cs_wallet); + std::map<uint256, CWalletTx>::const_iterator mi = wallet.mapWallet.find(txin.prevout.hash); + if (mi != wallet.mapWallet.end()) { const CWalletTx& prev = (*mi).second; if (txin.prevout.n < prev.tx->vout.size()) - return IsMine(prev.tx->vout[txin.prevout.n]); + return wallet.IsMine(prev.tx->vout[txin.prevout.n]); } return ISMINE_NO; } -bool CWallet::IsAllFromMe(const CTransaction& tx, const isminefilter& filter) const +bool AllInputsMine(const CWallet& wallet, const CTransaction& tx, const isminefilter& filter) { - LOCK(cs_wallet); + LOCK(wallet.cs_wallet); for (const CTxIn& txin : tx.vin) { - auto mi = mapWallet.find(txin.prevout.hash); - if (mi == mapWallet.end()) + auto mi = wallet.mapWallet.find(txin.prevout.hash); + if (mi == wallet.mapWallet.end()) return false; // any unknown inputs can't be from us const CWalletTx& prev = (*mi).second; @@ -35,33 +36,33 @@ bool CWallet::IsAllFromMe(const CTransaction& tx, const isminefilter& filter) co if (txin.prevout.n >= prev.tx->vout.size()) return false; // invalid input! - if (!(IsMine(prev.tx->vout[txin.prevout.n]) & filter)) + if (!(wallet.IsMine(prev.tx->vout[txin.prevout.n]) & filter)) return false; } return true; } -CAmount CWallet::GetCredit(const CTxOut& txout, const isminefilter& filter) const +CAmount OutputGetCredit(const CWallet& wallet, const CTxOut& txout, const isminefilter& filter) { if (!MoneyRange(txout.nValue)) throw std::runtime_error(std::string(__func__) + ": value out of range"); - LOCK(cs_wallet); - return ((IsMine(txout) & filter) ? txout.nValue : 0); + LOCK(wallet.cs_wallet); + return ((wallet.IsMine(txout) & filter) ? txout.nValue : 0); } -CAmount CWallet::GetCredit(const CTransaction& tx, const isminefilter& filter) const +CAmount TxGetCredit(const CWallet& wallet, const CTransaction& tx, const isminefilter& filter) { CAmount nCredit = 0; for (const CTxOut& txout : tx.vout) { - nCredit += GetCredit(txout, filter); + nCredit += OutputGetCredit(wallet, txout, filter); if (!MoneyRange(nCredit)) throw std::runtime_error(std::string(__func__) + ": value out of range"); } return nCredit; } -bool CWallet::IsChange(const CScript& script) const +bool ScriptIsChange(const CWallet& wallet, const CScript& script) { // TODO: fix handling of 'change' outputs. The assumption is that any // payment to a script that is ours, but is not in the address book @@ -70,179 +71,177 @@ bool CWallet::IsChange(const CScript& script) const // a better way of identifying which outputs are 'the send' and which are // 'the change' will need to be implemented (maybe extend CWalletTx to remember // which output, if any, was change). - AssertLockHeld(cs_wallet); - if (IsMine(script)) + AssertLockHeld(wallet.cs_wallet); + if (wallet.IsMine(script)) { CTxDestination address; if (!ExtractDestination(script, address)) return true; - if (!FindAddressBookEntry(address)) { + if (!wallet.FindAddressBookEntry(address)) { return true; } } return false; } -bool CWallet::IsChange(const CTxOut& txout) const +bool OutputIsChange(const CWallet& wallet, const CTxOut& txout) { - return IsChange(txout.scriptPubKey); + return ScriptIsChange(wallet, txout.scriptPubKey); } -CAmount CWallet::GetChange(const CTxOut& txout) const +CAmount OutputGetChange(const CWallet& wallet, const CTxOut& txout) { - AssertLockHeld(cs_wallet); + AssertLockHeld(wallet.cs_wallet); if (!MoneyRange(txout.nValue)) throw std::runtime_error(std::string(__func__) + ": value out of range"); - return (IsChange(txout) ? txout.nValue : 0); + return (OutputIsChange(wallet, txout) ? txout.nValue : 0); } -CAmount CWallet::GetChange(const CTransaction& tx) const +CAmount TxGetChange(const CWallet& wallet, const CTransaction& tx) { - LOCK(cs_wallet); + LOCK(wallet.cs_wallet); CAmount nChange = 0; for (const CTxOut& txout : tx.vout) { - nChange += GetChange(txout); + nChange += OutputGetChange(wallet, txout); if (!MoneyRange(nChange)) throw std::runtime_error(std::string(__func__) + ": value out of range"); } return nChange; } -CAmount CWalletTx::GetCachableAmount(AmountType type, const isminefilter& filter, bool recalculate) const +static CAmount GetCachableAmount(const CWallet& wallet, const CWalletTx& wtx, CWalletTx::AmountType type, const isminefilter& filter, bool recalculate = false) { - auto& amount = m_amounts[type]; + auto& amount = wtx.m_amounts[type]; if (recalculate || !amount.m_cached[filter]) { - amount.Set(filter, type == DEBIT ? pwallet->GetDebit(*tx, filter) : pwallet->GetCredit(*tx, filter)); - m_is_cache_empty = false; + amount.Set(filter, type == CWalletTx::DEBIT ? wallet.GetDebit(*wtx.tx, filter) : TxGetCredit(wallet, *wtx.tx, filter)); + wtx.m_is_cache_empty = false; } return amount.m_value[filter]; } -CAmount CWalletTx::GetCredit(const isminefilter& filter) const +CAmount CachedTxGetCredit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter) { // Must wait until coinbase is safely deep enough in the chain before valuing it - if (IsImmatureCoinBase()) + if (wallet.IsTxImmatureCoinBase(wtx)) return 0; CAmount credit = 0; if (filter & ISMINE_SPENDABLE) { // GetBalance can assume transactions in mapWallet won't change - credit += GetCachableAmount(CREDIT, ISMINE_SPENDABLE); + credit += GetCachableAmount(wallet, wtx, CWalletTx::CREDIT, ISMINE_SPENDABLE); } if (filter & ISMINE_WATCH_ONLY) { - credit += GetCachableAmount(CREDIT, ISMINE_WATCH_ONLY); + credit += GetCachableAmount(wallet, wtx, CWalletTx::CREDIT, ISMINE_WATCH_ONLY); } return credit; } -CAmount CWalletTx::GetDebit(const isminefilter& filter) const +CAmount CachedTxGetDebit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter) { - if (tx->vin.empty()) + if (wtx.tx->vin.empty()) return 0; CAmount debit = 0; if (filter & ISMINE_SPENDABLE) { - debit += GetCachableAmount(DEBIT, ISMINE_SPENDABLE); + debit += GetCachableAmount(wallet, wtx, CWalletTx::DEBIT, ISMINE_SPENDABLE); } if (filter & ISMINE_WATCH_ONLY) { - debit += GetCachableAmount(DEBIT, ISMINE_WATCH_ONLY); + debit += GetCachableAmount(wallet, wtx, CWalletTx::DEBIT, ISMINE_WATCH_ONLY); } return debit; } -CAmount CWalletTx::GetChange() const +CAmount CachedTxGetChange(const CWallet& wallet, const CWalletTx& wtx) { - if (fChangeCached) - return nChangeCached; - nChangeCached = pwallet->GetChange(*tx); - fChangeCached = true; - return nChangeCached; + if (wtx.fChangeCached) + return wtx.nChangeCached; + wtx.nChangeCached = TxGetChange(wallet, *wtx.tx); + wtx.fChangeCached = true; + return wtx.nChangeCached; } -CAmount CWalletTx::GetImmatureCredit(bool fUseCache) const +CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx, bool fUseCache) { - if (IsImmatureCoinBase() && IsInMainChain()) { - return GetCachableAmount(IMMATURE_CREDIT, ISMINE_SPENDABLE, !fUseCache); + if (wallet.IsTxImmatureCoinBase(wtx) && wallet.IsTxInMainChain(wtx)) { + return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, ISMINE_SPENDABLE, !fUseCache); } return 0; } -CAmount CWalletTx::GetImmatureWatchOnlyCredit(const bool fUseCache) const +CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletTx& wtx, const bool fUseCache) { - if (IsImmatureCoinBase() && IsInMainChain()) { - return GetCachableAmount(IMMATURE_CREDIT, ISMINE_WATCH_ONLY, !fUseCache); + if (wallet.IsTxImmatureCoinBase(wtx) && wallet.IsTxInMainChain(wtx)) { + return GetCachableAmount(wallet, wtx, CWalletTx::IMMATURE_CREDIT, ISMINE_WATCH_ONLY, !fUseCache); } return 0; } -CAmount CWalletTx::GetAvailableCredit(bool fUseCache, const isminefilter& filter) const +CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, bool fUseCache, const isminefilter& filter) { - if (pwallet == nullptr) - return 0; - // Avoid caching ismine for NO or ALL cases (could remove this check and simplify in the future). bool allow_cache = (filter & ISMINE_ALL) && (filter & ISMINE_ALL) != ISMINE_ALL; // Must wait until coinbase is safely deep enough in the chain before valuing it - if (IsImmatureCoinBase()) + if (wallet.IsTxImmatureCoinBase(wtx)) return 0; - if (fUseCache && allow_cache && m_amounts[AVAILABLE_CREDIT].m_cached[filter]) { - return m_amounts[AVAILABLE_CREDIT].m_value[filter]; + if (fUseCache && allow_cache && wtx.m_amounts[CWalletTx::AVAILABLE_CREDIT].m_cached[filter]) { + return wtx.m_amounts[CWalletTx::AVAILABLE_CREDIT].m_value[filter]; } - bool allow_used_addresses = (filter & ISMINE_USED) || !pwallet->IsWalletFlagSet(WALLET_FLAG_AVOID_REUSE); + bool allow_used_addresses = (filter & ISMINE_USED) || !wallet.IsWalletFlagSet(WALLET_FLAG_AVOID_REUSE); CAmount nCredit = 0; - uint256 hashTx = GetHash(); - for (unsigned int i = 0; i < tx->vout.size(); i++) + uint256 hashTx = wtx.GetHash(); + for (unsigned int i = 0; i < wtx.tx->vout.size(); i++) { - if (!pwallet->IsSpent(hashTx, i) && (allow_used_addresses || !pwallet->IsSpentKey(hashTx, i))) { - const CTxOut &txout = tx->vout[i]; - nCredit += pwallet->GetCredit(txout, filter); + if (!wallet.IsSpent(hashTx, i) && (allow_used_addresses || !wallet.IsSpentKey(hashTx, i))) { + const CTxOut &txout = wtx.tx->vout[i]; + nCredit += OutputGetCredit(wallet, txout, filter); if (!MoneyRange(nCredit)) throw std::runtime_error(std::string(__func__) + " : value out of range"); } } if (allow_cache) { - m_amounts[AVAILABLE_CREDIT].Set(filter, nCredit); - m_is_cache_empty = false; + wtx.m_amounts[CWalletTx::AVAILABLE_CREDIT].Set(filter, nCredit); + wtx.m_is_cache_empty = false; } return nCredit; } -void CWalletTx::GetAmounts(std::list<COutputEntry>& listReceived, - std::list<COutputEntry>& listSent, CAmount& nFee, const isminefilter& filter) const +void CachedTxGetAmounts(const CWallet& wallet, const CWalletTx& wtx, + std::list<COutputEntry>& listReceived, + std::list<COutputEntry>& listSent, CAmount& nFee, const isminefilter& filter) { nFee = 0; listReceived.clear(); listSent.clear(); // Compute fee: - CAmount nDebit = GetDebit(filter); + CAmount nDebit = CachedTxGetDebit(wallet, wtx, filter); if (nDebit > 0) // debit>0 means we signed/sent this transaction { - CAmount nValueOut = tx->GetValueOut(); + CAmount nValueOut = wtx.tx->GetValueOut(); nFee = nDebit - nValueOut; } - LOCK(pwallet->cs_wallet); + LOCK(wallet.cs_wallet); // Sent/received. - for (unsigned int i = 0; i < tx->vout.size(); ++i) + for (unsigned int i = 0; i < wtx.tx->vout.size(); ++i) { - const CTxOut& txout = tx->vout[i]; - isminetype fIsMine = pwallet->IsMine(txout); + const CTxOut& txout = wtx.tx->vout[i]; + isminetype fIsMine = wallet.IsMine(txout); // Only need to handle txouts if AT LEAST one of these is true: // 1) they debit from us (sent) // 2) the output is to us (received) if (nDebit > 0) { // Don't report 'change' txouts - if (pwallet->IsChange(txout)) + if (OutputIsChange(wallet, txout)) continue; } else if (!(fIsMine & filter)) @@ -253,8 +252,8 @@ void CWalletTx::GetAmounts(std::list<COutputEntry>& listReceived, if (!ExtractDestination(txout.scriptPubKey, address) && !txout.scriptPubKey.IsUnspendable()) { - pwallet->WalletLogPrintf("CWalletTx::GetAmounts: Unknown transaction type found, txid %s\n", - this->GetHash().ToString()); + wallet.WalletLogPrintf("CWalletTx::GetAmounts: Unknown transaction type found, txid %s\n", + wtx.GetHash().ToString()); address = CNoDestination(); } @@ -271,16 +270,21 @@ void CWalletTx::GetAmounts(std::list<COutputEntry>& listReceived, } -bool CWallet::IsTrusted(const CWalletTx& wtx, std::set<uint256>& trusted_parents) const +bool CachedTxIsFromMe(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter) +{ + return (CachedTxGetDebit(wallet, wtx, filter) > 0); +} + +bool CachedTxIsTrusted(const CWallet& wallet, const CWalletTx& wtx, std::set<uint256>& trusted_parents) { - AssertLockHeld(cs_wallet); + AssertLockHeld(wallet.cs_wallet); // Quick answer in most cases - if (!chain().checkFinalTx(*wtx.tx)) return false; - int nDepth = wtx.GetDepthInMainChain(); + if (!wallet.chain().checkFinalTx(*wtx.tx)) return false; + int nDepth = wallet.GetTxDepthInMainChain(wtx); if (nDepth >= 1) return true; if (nDepth < 0) return false; // using wtx's cached debit - if (!m_spend_zero_conf_change || !wtx.IsFromMe(ISMINE_ALL)) return false; + if (!wallet.m_spend_zero_conf_change || !CachedTxIsFromMe(wallet, wtx, ISMINE_ALL)) return false; // Don't trust unconfirmed transactions from us unless they are in the mempool. if (!wtx.InMempool()) return false; @@ -289,41 +293,41 @@ bool CWallet::IsTrusted(const CWalletTx& wtx, std::set<uint256>& trusted_parents for (const CTxIn& txin : wtx.tx->vin) { // Transactions not sent by us: not trusted - const CWalletTx* parent = GetWalletTx(txin.prevout.hash); + const CWalletTx* parent = wallet.GetWalletTx(txin.prevout.hash); if (parent == nullptr) return false; const CTxOut& parentOut = parent->tx->vout[txin.prevout.n]; // Check that this specific input being spent is trusted - if (IsMine(parentOut) != ISMINE_SPENDABLE) return false; + if (wallet.IsMine(parentOut) != ISMINE_SPENDABLE) return false; // If we've already trusted this parent, continue if (trusted_parents.count(parent->GetHash())) continue; // Recurse to check that the parent is also trusted - if (!IsTrusted(*parent, trusted_parents)) return false; + if (!CachedTxIsTrusted(wallet, *parent, trusted_parents)) return false; trusted_parents.insert(parent->GetHash()); } return true; } -bool CWalletTx::IsTrusted() const +bool CachedTxIsTrusted(const CWallet& wallet, const CWalletTx& wtx) { std::set<uint256> trusted_parents; - LOCK(pwallet->cs_wallet); - return pwallet->IsTrusted(*this, trusted_parents); + LOCK(wallet.cs_wallet); + return CachedTxIsTrusted(wallet, wtx, trusted_parents); } -CWallet::Balance CWallet::GetBalance(const int min_depth, bool avoid_reuse) const +Balance GetBalance(const CWallet& wallet, const int min_depth, bool avoid_reuse) { Balance ret; isminefilter reuse_filter = avoid_reuse ? ISMINE_NO : ISMINE_USED; { - LOCK(cs_wallet); + LOCK(wallet.cs_wallet); std::set<uint256> trusted_parents; - for (const auto& entry : mapWallet) + for (const auto& entry : wallet.mapWallet) { const CWalletTx& wtx = entry.second; - const bool is_trusted{IsTrusted(wtx, trusted_parents)}; - const int tx_depth{wtx.GetDepthInMainChain()}; - const CAmount tx_credit_mine{wtx.GetAvailableCredit(/* fUseCache */ true, ISMINE_SPENDABLE | reuse_filter)}; - const CAmount tx_credit_watchonly{wtx.GetAvailableCredit(/* fUseCache */ true, ISMINE_WATCH_ONLY | reuse_filter)}; + const bool is_trusted{CachedTxIsTrusted(wallet, wtx, trusted_parents)}; + const int tx_depth{wallet.GetTxDepthInMainChain(wtx)}; + const CAmount tx_credit_mine{CachedTxGetAvailableCredit(wallet, wtx, /* fUseCache */ true, ISMINE_SPENDABLE | reuse_filter)}; + const CAmount tx_credit_watchonly{CachedTxGetAvailableCredit(wallet, wtx, /* fUseCache */ true, ISMINE_WATCH_ONLY | reuse_filter)}; if (is_trusted && tx_depth >= min_depth) { ret.m_mine_trusted += tx_credit_mine; ret.m_watchonly_trusted += tx_credit_watchonly; @@ -332,43 +336,43 @@ CWallet::Balance CWallet::GetBalance(const int min_depth, bool avoid_reuse) cons ret.m_mine_untrusted_pending += tx_credit_mine; ret.m_watchonly_untrusted_pending += tx_credit_watchonly; } - ret.m_mine_immature += wtx.GetImmatureCredit(); - ret.m_watchonly_immature += wtx.GetImmatureWatchOnlyCredit(); + ret.m_mine_immature += CachedTxGetImmatureCredit(wallet, wtx); + ret.m_watchonly_immature += CachedTxGetImmatureWatchOnlyCredit(wallet, wtx); } } return ret; } -std::map<CTxDestination, CAmount> CWallet::GetAddressBalances() const +std::map<CTxDestination, CAmount> GetAddressBalances(const CWallet& wallet) { std::map<CTxDestination, CAmount> balances; { - LOCK(cs_wallet); + LOCK(wallet.cs_wallet); std::set<uint256> trusted_parents; - for (const auto& walletEntry : mapWallet) + for (const auto& walletEntry : wallet.mapWallet) { const CWalletTx& wtx = walletEntry.second; - if (!IsTrusted(wtx, trusted_parents)) + if (!CachedTxIsTrusted(wallet, wtx, trusted_parents)) continue; - if (wtx.IsImmatureCoinBase()) + if (wallet.IsTxImmatureCoinBase(wtx)) continue; - int nDepth = wtx.GetDepthInMainChain(); - if (nDepth < (wtx.IsFromMe(ISMINE_ALL) ? 0 : 1)) + int nDepth = wallet.GetTxDepthInMainChain(wtx); + if (nDepth < (CachedTxIsFromMe(wallet, wtx, ISMINE_ALL) ? 0 : 1)) continue; for (unsigned int i = 0; i < wtx.tx->vout.size(); i++) { CTxDestination addr; - if (!IsMine(wtx.tx->vout[i])) + if (!wallet.IsMine(wtx.tx->vout[i])) continue; if(!ExtractDestination(wtx.tx->vout[i].scriptPubKey, addr)) continue; - CAmount n = IsSpent(walletEntry.first, i) ? 0 : wtx.tx->vout[i].nValue; + CAmount n = wallet.IsSpent(walletEntry.first, i) ? 0 : wtx.tx->vout[i].nValue; balances[addr] += n; } } @@ -377,13 +381,13 @@ std::map<CTxDestination, CAmount> CWallet::GetAddressBalances() const return balances; } -std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() const +std::set< std::set<CTxDestination> > GetAddressGroupings(const CWallet& wallet) { - AssertLockHeld(cs_wallet); + AssertLockHeld(wallet.cs_wallet); std::set< std::set<CTxDestination> > groupings; std::set<CTxDestination> grouping; - for (const auto& walletEntry : mapWallet) + for (const auto& walletEntry : wallet.mapWallet) { const CWalletTx& wtx = walletEntry.second; @@ -394,9 +398,9 @@ std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() const for (const CTxIn& txin : wtx.tx->vin) { CTxDestination address; - if(!IsMine(txin)) /* If this input isn't mine, ignore it */ + if(!InputIsMine(wallet, txin)) /* If this input isn't mine, ignore it */ continue; - if(!ExtractDestination(mapWallet.at(txin.prevout.hash).tx->vout[txin.prevout.n].scriptPubKey, address)) + if(!ExtractDestination(wallet.mapWallet.at(txin.prevout.hash).tx->vout[txin.prevout.n].scriptPubKey, address)) continue; grouping.insert(address); any_mine = true; @@ -406,7 +410,7 @@ std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() const if (any_mine) { for (const CTxOut& txout : wtx.tx->vout) - if (IsChange(txout)) + if (OutputIsChange(wallet, txout)) { CTxDestination txoutAddr; if(!ExtractDestination(txout.scriptPubKey, txoutAddr)) @@ -423,7 +427,7 @@ std::set< std::set<CTxDestination> > CWallet::GetAddressGroupings() const // group lone addrs by themselves for (const auto& txout : wtx.tx->vout) - if (IsMine(txout)) + if (wallet.IsMine(txout)) { CTxDestination address; if(!ExtractDestination(txout.scriptPubKey, address)) diff --git a/src/wallet/receive.h b/src/wallet/receive.h index 8eead32413..f659955fc6 100644 --- a/src/wallet/receive.h +++ b/src/wallet/receive.h @@ -5,16 +5,60 @@ #ifndef BITCOIN_WALLET_RECEIVE_H #define BITCOIN_WALLET_RECEIVE_H -#include <amount.h> +#include <consensus/amount.h> #include <wallet/ismine.h> #include <wallet/transaction.h> #include <wallet/wallet.h> +isminetype InputIsMine(const CWallet& wallet, const CTxIn& txin) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); + +/** Returns whether all of the inputs match the filter */ +bool AllInputsMine(const CWallet& wallet, const CTransaction& tx, const isminefilter& filter); + +CAmount OutputGetCredit(const CWallet& wallet, const CTxOut& txout, const isminefilter& filter); +CAmount TxGetCredit(const CWallet& wallet, const CTransaction& tx, const isminefilter& filter); + +bool ScriptIsChange(const CWallet& wallet, const CScript& script) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); +bool OutputIsChange(const CWallet& wallet, const CTxOut& txout) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); +CAmount OutputGetChange(const CWallet& wallet, const CTxOut& txout) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); +CAmount TxGetChange(const CWallet& wallet, const CTransaction& tx); + +CAmount CachedTxGetCredit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter); +//! filter decides which addresses will count towards the debit +CAmount CachedTxGetDebit(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter); +CAmount CachedTxGetChange(const CWallet& wallet, const CWalletTx& wtx); +CAmount CachedTxGetImmatureCredit(const CWallet& wallet, const CWalletTx& wtx, bool fUseCache = true); +CAmount CachedTxGetImmatureWatchOnlyCredit(const CWallet& wallet, const CWalletTx& wtx, const bool fUseCache = true); +// TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct +// annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The +// annotation "NO_THREAD_SAFETY_ANALYSIS" was temporarily added to avoid +// having to resolve the issue of member access into incomplete type CWallet. +CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, bool fUseCache = true, const isminefilter& filter = ISMINE_SPENDABLE) NO_THREAD_SAFETY_ANALYSIS; struct COutputEntry { CTxDestination destination; CAmount amount; int vout; }; +void CachedTxGetAmounts(const CWallet& wallet, const CWalletTx& wtx, + std::list<COutputEntry>& listReceived, + std::list<COutputEntry>& listSent, + CAmount& nFee, const isminefilter& filter); +bool CachedTxIsFromMe(const CWallet& wallet, const CWalletTx& wtx, const isminefilter& filter); +bool CachedTxIsTrusted(const CWallet& wallet, const CWalletTx& wtx, std::set<uint256>& trusted_parents) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); +bool CachedTxIsTrusted(const CWallet& wallet, const CWalletTx& wtx); + +struct Balance { + CAmount m_mine_trusted{0}; //!< Trusted, at depth=GetBalance.min_depth or more + CAmount m_mine_untrusted_pending{0}; //!< Untrusted, but in mempool (pending) + CAmount m_mine_immature{0}; //!< Immature coinbases in the main chain + CAmount m_watchonly_trusted{0}; + CAmount m_watchonly_untrusted_pending{0}; + CAmount m_watchonly_immature{0}; +}; +Balance GetBalance(const CWallet& wallet, int min_depth = 0, bool avoid_reuse = true); + +std::map<CTxDestination, CAmount> GetAddressBalances(const CWallet& wallet); +std::set<std::set<CTxDestination>> GetAddressGroupings(const CWallet& wallet) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); #endif // BITCOIN_WALLET_RECEIVE_H diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 35649ab02c..4d7fb2d38c 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chain.h> +#include <clientversion.h> #include <core_io.h> #include <interfaces/chain.h> #include <key_io.h> @@ -740,7 +741,7 @@ RPCHelpMan dumpwallet() // the user could have gotten from another RPC command prior to now wallet.BlockUntilSyncedToCurrentChain(); - LOCK2(wallet.cs_wallet, spk_man.cs_KeyStore); + LOCK(wallet.cs_wallet); EnsureWalletIsUnlocked(wallet); @@ -762,9 +763,16 @@ RPCHelpMan dumpwallet() throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open wallet dump file"); std::map<CKeyID, int64_t> mapKeyBirth; - const std::map<CKeyID, int64_t>& mapKeyPool = spk_man.GetAllReserveKeys(); wallet.GetKeyBirthTimes(mapKeyBirth); + int64_t block_time = 0; + CHECK_NONFATAL(wallet.chain().findBlock(wallet.GetLastBlockHash(), FoundBlock().time(block_time))); + + // Note: To avoid a lock order issue, access to cs_main must be locked before cs_KeyStore. + // So we do the two things in this function that lock cs_main first: GetKeyBirthTimes, and findBlock. + LOCK(spk_man.cs_KeyStore); + + const std::map<CKeyID, int64_t>& mapKeyPool = spk_man.GetAllReserveKeys(); std::set<CScriptID> scripts = spk_man.GetCScripts(); // sort time/key pairs @@ -776,11 +784,9 @@ RPCHelpMan dumpwallet() std::sort(vKeyBirth.begin(), vKeyBirth.end()); // produce output - file << strprintf("# Wallet dump created by Bitcoin %s\n", CLIENT_BUILD); + file << strprintf("# Wallet dump created by %s %s\n", PACKAGE_NAME, FormatFullVersion()); file << strprintf("# * Created on %s\n", FormatISO8601DateTime(GetTime())); file << strprintf("# * Best block at time of backup was %i (%s),\n", wallet.GetLastBlockHeight(), wallet.GetLastBlockHash().ToString()); - int64_t block_time = 0; - CHECK_NONFATAL(wallet.chain().findBlock(wallet.GetLastBlockHash(), FoundBlock().time(block_time))); file << strprintf("# mined on %s\n", FormatISO8601DateTime(block_time)); file << "\n"; @@ -1433,7 +1439,7 @@ RPCHelpMan importmulti() "and coins using this key may not appear in the wallet. This error could be " "caused by pruning or data corruption (see bitcoind log for details) and could " "be dealt with by downloading and rescanning the relevant blocks (see -reindex " - "and -rescan options).", + "option and rescanblockchain RPC).", GetImportTimestamp(request, now), scannedTime - TIMESTAMP_WINDOW - 1, TIMESTAMP_WINDOW))); response.push_back(std::move(result)); } @@ -1482,7 +1488,7 @@ static UniValue ProcessDescriptorImport(CWallet& wallet, const UniValue& data, c } else { warnings.push_back("Range not given, using default keypool range"); range_start = 0; - range_end = gArgs.GetArg("-keypool", DEFAULT_KEYPOOL_SIZE); + range_end = gArgs.GetIntArg("-keypool", DEFAULT_KEYPOOL_SIZE); } next_index = range_start; @@ -1566,9 +1572,8 @@ static UniValue ProcessDescriptorImport(CWallet& wallet, const UniValue& data, c // Check if the wallet already contains the descriptor auto existing_spk_manager = wallet.GetDescriptorScriptPubKeyMan(w_desc); if (existing_spk_manager) { - LOCK(existing_spk_manager->cs_desc_man); - if (range_start > existing_spk_manager->GetWalletDescriptor().range_start) { - throw JSONRPCError(RPC_INVALID_PARAMS, strprintf("range_start can only decrease; current range = [%d,%d]", existing_spk_manager->GetWalletDescriptor().range_start, existing_spk_manager->GetWalletDescriptor().range_end)); + if (!existing_spk_manager->CanUpdateToWalletDescriptor(w_desc, error)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, error); } } @@ -1585,16 +1590,16 @@ static UniValue ProcessDescriptorImport(CWallet& wallet, const UniValue& data, c } else { wallet.AddActiveScriptPubKeyMan(spk_manager->GetID(), *w_desc.descriptor->GetOutputType(), internal); } + } else { + if (w_desc.descriptor->GetOutputType()) { + wallet.DeactivateScriptPubKeyMan(spk_manager->GetID(), *w_desc.descriptor->GetOutputType(), internal); + } } result.pushKV("success", UniValue(true)); } catch (const UniValue& e) { result.pushKV("success", UniValue(false)); result.pushKV("error", e); - } catch (...) { - result.pushKV("success", UniValue(false)); - - result.pushKV("error", JSONRPCError(RPC_MISC_ERROR, "Missing required fields")); } if (warnings.size()) result.pushKV("warnings", warnings); return result; @@ -1739,7 +1744,7 @@ RPCHelpMan importdescriptors() "and coins using this desc may not appear in the wallet. This error could be " "caused by pruning or data corruption (see bitcoind log for details) and could " "be dealt with by downloading and rescanning the relevant blocks (see -reindex " - "and -rescan options).", + "option and rescanblockchain RPC).", GetImportTimestamp(request, now), scanned_time - TIMESTAMP_WINDOW - 1, TIMESTAMP_WINDOW))); response.push_back(std::move(result)); } @@ -1756,8 +1761,10 @@ RPCHelpMan listdescriptors() { return RPCHelpMan{ "listdescriptors", - "\nList descriptors imported into a descriptor-enabled wallet.", - {}, + "\nList descriptors imported into a descriptor-enabled wallet.\n", + { + {"private", RPCArg::Type::BOOL, RPCArg::Default{false}, "Show private descriptors."} + }, RPCResult{RPCResult::Type::OBJ, "", "", { {RPCResult::Type::STR, "wallet_name", "Name of wallet this operation was performed on"}, {RPCResult::Type::ARR, "descriptors", "Array of descriptor objects", @@ -1777,6 +1784,7 @@ RPCHelpMan listdescriptors() }}, RPCExamples{ HelpExampleCli("listdescriptors", "") + HelpExampleRpc("listdescriptors", "") + + HelpExampleCli("listdescriptors", "true") + HelpExampleRpc("listdescriptors", "true") }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { @@ -1787,7 +1795,10 @@ RPCHelpMan listdescriptors() throw JSONRPCError(RPC_WALLET_ERROR, "listdescriptors is not available for non-descriptor wallets"); } - EnsureWalletIsUnlocked(*wallet); + const bool priv = !request.params[0].isNull() && request.params[0].get_bool(); + if (priv) { + EnsureWalletIsUnlocked(*wallet); + } LOCK(wallet->cs_wallet); @@ -1802,8 +1813,9 @@ RPCHelpMan listdescriptors() LOCK(desc_spk_man->cs_desc_man); const auto& wallet_descriptor = desc_spk_man->GetWalletDescriptor(); std::string descriptor; - if (!desc_spk_man->GetDescriptorString(descriptor, false)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Can't get normalized descriptor string."); + + if (!desc_spk_man->GetDescriptorString(descriptor, priv)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Can't get descriptor string."); } spk.pushKV("desc", descriptor); spk.pushKV("timestamp", wallet_descriptor.creation_time); diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index bc5d771b6e..8b481bc29c 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> +#include <consensus/amount.h> #include <core_io.h> #include <interfaces/chain.h> #include <key_io.h> @@ -31,7 +31,9 @@ #include <wallet/context.h> #include <wallet/feebumper.h> #include <wallet/load.h> +#include <wallet/receive.h> #include <wallet/rpcwallet.h> +#include <wallet/spend.h> #include <wallet/wallet.h> #include <wallet/walletdb.h> #include <wallet/walletutil.h> @@ -41,6 +43,7 @@ #include <univalue.h> +#include <map> using interfaces::FoundBlock; @@ -96,14 +99,16 @@ bool GetWalletNameFromJSONRPCRequest(const JSONRPCRequest& request, std::string& std::shared_ptr<CWallet> GetWalletForJSONRPCRequest(const JSONRPCRequest& request) { CHECK_NONFATAL(request.mode == JSONRPCRequest::EXECUTE); + WalletContext& context = EnsureWalletContext(request.context); + std::string wallet_name; if (GetWalletNameFromJSONRPCRequest(request, wallet_name)) { - std::shared_ptr<CWallet> pwallet = GetWallet(wallet_name); + std::shared_ptr<CWallet> pwallet = GetWallet(context, wallet_name); if (!pwallet) throw JSONRPCError(RPC_WALLET_NOT_FOUND, "Requested wallet does not exist or is not loaded"); return pwallet; } - std::vector<std::shared_ptr<CWallet>> wallets = GetWallets(); + std::vector<std::shared_ptr<CWallet>> wallets = GetWallets(context); if (wallets.size() == 1) { return wallets[0]; } @@ -145,9 +150,10 @@ LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet, bool also_cr return *spk_man; } -static void WalletTxToJSON(interfaces::Chain& chain, const CWalletTx& wtx, UniValue& entry) +static void WalletTxToJSON(const CWallet& wallet, const CWalletTx& wtx, UniValue& entry) { - int confirms = wtx.GetDepthInMainChain(); + interfaces::Chain& chain = wallet.chain(); + int confirms = wallet.GetTxDepthInMainChain(wtx); entry.pushKV("confirmations", confirms); if (wtx.IsCoinBase()) entry.pushKV("generated", true); @@ -160,12 +166,12 @@ static void WalletTxToJSON(interfaces::Chain& chain, const CWalletTx& wtx, UniVa CHECK_NONFATAL(chain.findBlock(wtx.m_confirm.hashBlock, FoundBlock().time(block_time))); entry.pushKV("blocktime", block_time); } else { - entry.pushKV("trusted", wtx.IsTrusted()); + entry.pushKV("trusted", CachedTxIsTrusted(wallet, wtx)); } uint256 hash = wtx.GetHash(); entry.pushKV("txid", hash.GetHex()); UniValue conflicts(UniValue::VARR); - for (const uint256& conflict : wtx.GetConflicts()) + for (const uint256& conflict : wallet.GetTxConflicts(wtx)) conflicts.push_back(conflict.GetHex()); entry.pushKV("walletconflicts", conflicts); entry.pushKV("time", wtx.GetTxTime()); @@ -266,18 +272,19 @@ static RPCHelpMan getnewaddress() OutputType output_type = pwallet->m_default_address_type; if (!request.params[1].isNull()) { - if (!ParseOutputType(request.params[1].get_str(), output_type)) { + std::optional<OutputType> parsed = ParseOutputType(request.params[1].get_str()); + if (!parsed) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Unknown address type '%s'", request.params[1].get_str())); - } - if (output_type == OutputType::BECH32M && pwallet->GetLegacyScriptPubKeyMan()) { + } else if (parsed.value() == OutputType::BECH32M && pwallet->GetLegacyScriptPubKeyMan()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Legacy wallets cannot provide bech32m addresses"); } + output_type = parsed.value(); } CTxDestination dest; - std::string error; + bilingual_str error; if (!pwallet->GetNewDestination(output_type, label, dest, error)) { - throw JSONRPCError(RPC_WALLET_KEYPOOL_RAN_OUT, error); + throw JSONRPCError(RPC_WALLET_KEYPOOL_RAN_OUT, error.original); } return EncodeDestination(dest); @@ -313,18 +320,19 @@ static RPCHelpMan getrawchangeaddress() OutputType output_type = pwallet->m_default_change_type.value_or(pwallet->m_default_address_type); if (!request.params[0].isNull()) { - if (!ParseOutputType(request.params[0].get_str(), output_type)) { + std::optional<OutputType> parsed = ParseOutputType(request.params[0].get_str()); + if (!parsed) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Unknown address type '%s'", request.params[0].get_str())); - } - if (output_type == OutputType::BECH32M && pwallet->GetLegacyScriptPubKeyMan()) { + } else if (parsed.value() == OutputType::BECH32M && pwallet->GetLegacyScriptPubKeyMan()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Legacy wallets cannot provide bech32m addresses"); } + output_type = parsed.value(); } CTxDestination dest; - std::string error; + bilingual_str error; if (!pwallet->GetNewChangeDestination(output_type, dest, error)) { - throw JSONRPCError(RPC_WALLET_KEYPOOL_RAN_OUT, error); + throw JSONRPCError(RPC_WALLET_KEYPOOL_RAN_OUT, error.original); } return EncodeDestination(dest); }, @@ -419,7 +427,7 @@ UniValue SendMoney(CWallet& wallet, const CCoinControl &coin_control, std::vecto bilingual_str error; CTransactionRef tx; FeeCalculation fee_calc_out; - const bool fCreated = wallet.CreateTransaction(recipients, tx, nFeeRequired, nChangePosRet, error, coin_control, fee_calc_out, true); + const bool fCreated = CreateTransaction(wallet, recipients, tx, nFeeRequired, nChangePosRet, error, coin_control, fee_calc_out, true); if (!fCreated) { throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, error.original); } @@ -465,7 +473,7 @@ static RPCHelpMan sendtoaddress() RPCResult::Type::OBJ, "", "", { {RPCResult::Type::STR_HEX, "txid", "The transaction id."}, - {RPCResult::Type::STR, "fee reason", "The transaction fee reason."} + {RPCResult::Type::STR, "fee_reason", "The transaction fee reason."} }, }, }, @@ -572,8 +580,8 @@ static RPCHelpMan listaddressgroupings() LOCK(pwallet->cs_wallet); UniValue jsonGroupings(UniValue::VARR); - std::map<CTxDestination, CAmount> balances = pwallet->GetAddressBalances(); - for (const std::set<CTxDestination>& grouping : pwallet->GetAddressGroupings()) { + std::map<CTxDestination, CAmount> balances = GetAddressBalances(*pwallet); + for (const std::set<CTxDestination>& grouping : GetAddressGroupings(*pwallet)) { UniValue jsonGrouping(UniValue::VARR); for (const CTxDestination& address : grouping) { @@ -682,7 +690,7 @@ static CAmount GetReceived(const CWallet& wallet, const UniValue& params, bool b CAmount amount = 0; for (const std::pair<const uint256, CWalletTx>& wtx_pair : wallet.mapWallet) { const CWalletTx& wtx = wtx_pair.second; - if (wtx.IsCoinBase() || !wallet.chain().checkFinalTx(*wtx.tx) || wtx.GetDepthInMainChain() < min_depth) { + if (wtx.IsCoinBase() || !wallet.chain().checkFinalTx(*wtx.tx) || wallet.GetTxDepthInMainChain(wtx) < min_depth) { continue; } @@ -822,7 +830,7 @@ static RPCHelpMan getbalance() bool avoid_reuse = GetAvoidReuseFlag(*pwallet, request.params[3]); - const auto bal = pwallet->GetBalance(min_depth, avoid_reuse); + const auto bal = GetBalance(*pwallet, min_depth, avoid_reuse); return ValueFromAmount(bal.m_mine_trusted + (include_watchonly ? bal.m_watchonly_trusted : 0)); }, @@ -847,7 +855,7 @@ static RPCHelpMan getunconfirmedbalance() LOCK(pwallet->cs_wallet); - return ValueFromAmount(pwallet->GetBalance().m_mine_untrusted_pending); + return ValueFromAmount(GetBalance(*pwallet).m_mine_untrusted_pending); }, }; } @@ -892,7 +900,7 @@ static RPCHelpMan sendmany() { {RPCResult::Type::STR_HEX, "txid", "The transaction id for the send. Only 1 transaction is created regardless of\n" "the number of addresses."}, - {RPCResult::Type::STR, "fee reason", "The transaction fee reason."} + {RPCResult::Type::STR, "fee_reason", "The transaction fee reason."} }, }, }, @@ -1007,12 +1015,13 @@ static RPCHelpMan addmultisigaddress() OutputType output_type = pwallet->m_default_address_type; if (!request.params[3].isNull()) { - if (!ParseOutputType(request.params[3].get_str(), output_type)) { + std::optional<OutputType> parsed = ParseOutputType(request.params[3].get_str()); + if (!parsed) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Unknown address type '%s'", request.params[3].get_str())); - } - if (output_type == OutputType::BECH32M) { + } else if (parsed.value() == OutputType::BECH32M) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Bech32m multisig addresses cannot be created with legacy wallets"); } + output_type = parsed.value(); } // Construct using pay-to-script-hash: @@ -1080,7 +1089,7 @@ static UniValue ListReceived(const CWallet& wallet, const UniValue& params, bool continue; } - int nDepth = wtx.GetDepthInMainChain(); + int nDepth = wallet.GetTxDepthInMainChain(wtx); if (nDepth < nMinDepth) continue; @@ -1205,7 +1214,7 @@ static RPCHelpMan listreceivedbyaddress() { {RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::BOOL, "involvesWatchonly", "Only returns true if imported addresses were involved in transaction"}, + {RPCResult::Type::BOOL, "involvesWatchonly", /* optional */ true, "Only returns true if imported addresses were involved in transaction"}, {RPCResult::Type::STR, "address", "The receiving address"}, {RPCResult::Type::STR_AMOUNT, "amount", "The total amount in " + CURRENCY_UNIT + " received by the address"}, {RPCResult::Type::NUM, "confirmations", "The number of confirmations of the most recent transaction included"}, @@ -1253,7 +1262,7 @@ static RPCHelpMan listreceivedbylabel() { {RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::BOOL, "involvesWatchonly", "Only returns true if imported addresses were involved in transaction"}, + {RPCResult::Type::BOOL, "involvesWatchonly", /* optional */ true, "Only returns true if imported addresses were involved in transaction"}, {RPCResult::Type::STR_AMOUNT, "amount", "The total amount received by addresses with this label"}, {RPCResult::Type::NUM, "confirmations", "The number of confirmations of the most recent transaction included"}, {RPCResult::Type::STR, "label", "The label of the receiving address. The default label is \"\""}, @@ -1305,9 +1314,9 @@ static void ListTransactions(const CWallet& wallet, const CWalletTx& wtx, int nM std::list<COutputEntry> listReceived; std::list<COutputEntry> listSent; - wtx.GetAmounts(listReceived, listSent, nFee, filter_ismine); + CachedTxGetAmounts(wallet, wtx, listReceived, listSent, nFee, filter_ismine); - bool involvesWatchonly = wtx.IsFromMe(ISMINE_WATCH_ONLY); + bool involvesWatchonly = CachedTxIsFromMe(wallet, wtx, ISMINE_WATCH_ONLY); // Sent if (!filter_label) @@ -1328,14 +1337,14 @@ static void ListTransactions(const CWallet& wallet, const CWalletTx& wtx, int nM entry.pushKV("vout", s.vout); entry.pushKV("fee", ValueFromAmount(-nFee)); if (fLong) - WalletTxToJSON(wallet.chain(), wtx, entry); + WalletTxToJSON(wallet, wtx, entry); entry.pushKV("abandoned", wtx.isAbandoned()); ret.push_back(entry); } } // Received - if (listReceived.size() > 0 && wtx.GetDepthInMainChain() >= nMinDepth) { + if (listReceived.size() > 0 && wallet.GetTxDepthInMainChain(wtx) >= nMinDepth) { for (const COutputEntry& r : listReceived) { std::string label; @@ -1353,9 +1362,9 @@ static void ListTransactions(const CWallet& wallet, const CWalletTx& wtx, int nM MaybePushAddress(entry, r.destination); if (wtx.IsCoinBase()) { - if (wtx.GetDepthInMainChain() < 1) + if (wallet.GetTxDepthInMainChain(wtx) < 1) entry.pushKV("category", "orphan"); - else if (wtx.IsImmatureCoinBase()) + else if (wallet.IsTxImmatureCoinBase(wtx)) entry.pushKV("category", "immature"); else entry.pushKV("category", "generate"); @@ -1370,7 +1379,7 @@ static void ListTransactions(const CWallet& wallet, const CWalletTx& wtx, int nM } entry.pushKV("vout", r.vout); if (fLong) - WalletTxToJSON(wallet.chain(), wtx, entry); + WalletTxToJSON(wallet, wtx, entry); ret.push_back(entry); } } @@ -1380,20 +1389,24 @@ static const std::vector<RPCResult> TransactionDescriptionString() { return{{RPCResult::Type::NUM, "confirmations", "The number of confirmations for the transaction. Negative confirmations means the\n" "transaction conflicted that many blocks ago."}, - {RPCResult::Type::BOOL, "generated", "Only present if transaction only input is a coinbase one."}, - {RPCResult::Type::BOOL, "trusted", "Only present if we consider transaction to be trusted and so safe to spend from."}, - {RPCResult::Type::STR_HEX, "blockhash", "The block hash containing the transaction."}, - {RPCResult::Type::NUM, "blockheight", "The block height containing the transaction."}, - {RPCResult::Type::NUM, "blockindex", "The index of the transaction in the block that includes it."}, - {RPCResult::Type::NUM_TIME, "blocktime", "The block time expressed in " + UNIX_EPOCH_TIME + "."}, + {RPCResult::Type::BOOL, "generated", /* optional */ true, "Only present if transaction only input is a coinbase one."}, + {RPCResult::Type::BOOL, "trusted", /* optional */ true, "Only present if we consider transaction to be trusted and so safe to spend from."}, + {RPCResult::Type::STR_HEX, "blockhash", /* optional */ true, "The block hash containing the transaction."}, + {RPCResult::Type::NUM, "blockheight", /* optional */ true, "The block height containing the transaction."}, + {RPCResult::Type::NUM, "blockindex", /* optional */ true, "The index of the transaction in the block that includes it."}, + {RPCResult::Type::NUM_TIME, "blocktime", /* optional */ true, "The block time expressed in " + UNIX_EPOCH_TIME + "."}, {RPCResult::Type::STR_HEX, "txid", "The transaction id."}, {RPCResult::Type::ARR, "walletconflicts", "Conflicting transaction ids.", { {RPCResult::Type::STR_HEX, "txid", "The transaction id."}, }}, + {RPCResult::Type::STR_HEX, "replaced_by_txid", /* optional */ true, "The txid if this tx was replaced."}, + {RPCResult::Type::STR_HEX, "replaces_txid", /* optional */ true, "The txid if the tx replaces one."}, + {RPCResult::Type::STR, "comment", /* optional */ true, ""}, + {RPCResult::Type::STR, "to", /* optional */ true, "If a comment to is associated with the transaction."}, {RPCResult::Type::NUM_TIME, "time", "The transaction time expressed in " + UNIX_EPOCH_TIME + "."}, {RPCResult::Type::NUM_TIME, "timereceived", "The time received expressed in " + UNIX_EPOCH_TIME + "."}, - {RPCResult::Type::STR, "comment", "If a comment is associated with the transaction, only present if not empty."}, + {RPCResult::Type::STR, "comment", /* optional */ true, "If a comment is associated with the transaction, only present if not empty."}, {RPCResult::Type::STR, "bip125-replaceable", "(\"yes|no|unknown\") Whether this transaction could be replaced due to BIP125 (replace-by-fee);\n" "may be unknown for unconfirmed transactions not in the mempool"}}; } @@ -1415,7 +1428,7 @@ static RPCHelpMan listtransactions() { {RPCResult::Type::OBJ, "", "", Cat(Cat<std::vector<RPCResult>>( { - {RPCResult::Type::BOOL, "involvesWatchonly", "Only returns true if imported addresses were involved in transaction."}, + {RPCResult::Type::BOOL, "involvesWatchonly", /* optional */ true, "Only returns true if imported addresses were involved in transaction."}, {RPCResult::Type::STR, "address", "The bitcoin address of the transaction."}, {RPCResult::Type::STR, "category", "The transaction category.\n" "\"send\" Transactions sent.\n" @@ -1425,14 +1438,14 @@ static RPCHelpMan listtransactions() "\"orphan\" Orphaned coinbase transactions received."}, {RPCResult::Type::STR_AMOUNT, "amount", "The amount in " + CURRENCY_UNIT + ". This is negative for the 'send' category, and is positive\n" "for all other categories"}, - {RPCResult::Type::STR, "label", "A comment for the address/transaction, if any"}, + {RPCResult::Type::STR, "label", /* optional */ true, "A comment for the address/transaction, if any"}, {RPCResult::Type::NUM, "vout", "the vout value"}, - {RPCResult::Type::STR_AMOUNT, "fee", "The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the\n" + {RPCResult::Type::STR_AMOUNT, "fee", /* optional */ true, "The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the\n" "'send' category of transactions."}, }, TransactionDescriptionString()), { - {RPCResult::Type::BOOL, "abandoned", "'true' if the transaction has been abandoned (inputs are respendable). Only available for the \n" + {RPCResult::Type::BOOL, "abandoned", /* optional */ true, "'true' if the transaction has been abandoned (inputs are respendable). Only available for the \n" "'send' category of transactions."}, })}, } @@ -1529,7 +1542,7 @@ static RPCHelpMan listsinceblock() { {RPCResult::Type::OBJ, "", "", Cat(Cat<std::vector<RPCResult>>( { - {RPCResult::Type::BOOL, "involvesWatchonly", "Only returns true if imported addresses were involved in transaction."}, + {RPCResult::Type::BOOL, "involvesWatchonly", /* optional */ true, "Only returns true if imported addresses were involved in transaction."}, {RPCResult::Type::STR, "address", "The bitcoin address of the transaction."}, {RPCResult::Type::STR, "category", "The transaction category.\n" "\"send\" Transactions sent.\n" @@ -1540,18 +1553,17 @@ static RPCHelpMan listsinceblock() {RPCResult::Type::STR_AMOUNT, "amount", "The amount in " + CURRENCY_UNIT + ". This is negative for the 'send' category, and is positive\n" "for all other categories"}, {RPCResult::Type::NUM, "vout", "the vout value"}, - {RPCResult::Type::STR_AMOUNT, "fee", "The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the\n" + {RPCResult::Type::STR_AMOUNT, "fee", /* optional */ true, "The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the\n" "'send' category of transactions."}, }, TransactionDescriptionString()), { - {RPCResult::Type::BOOL, "abandoned", "'true' if the transaction has been abandoned (inputs are respendable). Only available for the \n" + {RPCResult::Type::BOOL, "abandoned", /* optional */ true, "'true' if the transaction has been abandoned (inputs are respendable). Only available for the \n" "'send' category of transactions."}, - {RPCResult::Type::STR, "label", "A comment for the address/transaction, if any"}, - {RPCResult::Type::STR, "to", "If a comment to is associated with the transaction."}, + {RPCResult::Type::STR, "label", /* optional */ true, "A comment for the address/transaction, if any"}, })}, }}, - {RPCResult::Type::ARR, "removed", "<structure is the same as \"transactions\" above, only present if include_removed=true>\n" + {RPCResult::Type::ARR, "removed", /* optional */ true, "<structure is the same as \"transactions\" above, only present if include_removed=true>\n" "Note: transactions that were re-added in the active chain will appear as-is in this array, and may thus have a positive confirmation count." , {{RPCResult::Type::ELISION, "", ""},}}, {RPCResult::Type::STR_HEX, "lastblock", "The hash of the block (target_confirmations-1) from the best block on the main chain, or the genesis hash if the referenced block does not exist yet. This is typically used to feed back into listsinceblock the next time you call it. So you would generally use a target_confirmations of say 6, so you will be continually re-notified of transactions until they've reached 6 confirmations plus any new ones"}, @@ -1610,7 +1622,7 @@ static RPCHelpMan listsinceblock() for (const std::pair<const uint256, CWalletTx>& pairWtx : wallet.mapWallet) { const CWalletTx& tx = pairWtx.second; - if (depth == -1 || abs(tx.GetDepthInMainChain()) < depth) { + if (depth == -1 || abs(wallet.GetTxDepthInMainChain(tx)) < depth) { ListTransactions(wallet, tx, 0, true, transactions, filter, nullptr /* filter_label */); } } @@ -1664,7 +1676,7 @@ static RPCHelpMan gettransaction() RPCResult::Type::OBJ, "", "", Cat(Cat<std::vector<RPCResult>>( { {RPCResult::Type::STR_AMOUNT, "amount", "The amount in " + CURRENCY_UNIT}, - {RPCResult::Type::STR_AMOUNT, "fee", "The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the\n" + {RPCResult::Type::STR_AMOUNT, "fee", /* optional */ true, "The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the\n" "'send' category of transactions."}, }, TransactionDescriptionString()), @@ -1673,8 +1685,8 @@ static RPCHelpMan gettransaction() { {RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::BOOL, "involvesWatchonly", "Only returns true if imported addresses were involved in transaction."}, - {RPCResult::Type::STR, "address", "The bitcoin address involved in the transaction."}, + {RPCResult::Type::BOOL, "involvesWatchonly", /* optional */ true, "Only returns true if imported addresses were involved in transaction."}, + {RPCResult::Type::STR, "address", /* optional */ true, "The bitcoin address involved in the transaction."}, {RPCResult::Type::STR, "category", "The transaction category.\n" "\"send\" Transactions sent.\n" "\"receive\" Non-coinbase transactions received.\n" @@ -1682,16 +1694,16 @@ static RPCHelpMan gettransaction() "\"immature\" Coinbase transactions received with 100 or fewer confirmations.\n" "\"orphan\" Orphaned coinbase transactions received."}, {RPCResult::Type::STR_AMOUNT, "amount", "The amount in " + CURRENCY_UNIT}, - {RPCResult::Type::STR, "label", "A comment for the address/transaction, if any"}, + {RPCResult::Type::STR, "label", /* optional */ true, "A comment for the address/transaction, if any"}, {RPCResult::Type::NUM, "vout", "the vout value"}, - {RPCResult::Type::STR_AMOUNT, "fee", "The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the \n" + {RPCResult::Type::STR_AMOUNT, "fee", /* optional */ true, "The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the \n" "'send' category of transactions."}, - {RPCResult::Type::BOOL, "abandoned", "'true' if the transaction has been abandoned (inputs are respendable). Only available for the \n" + {RPCResult::Type::BOOL, "abandoned", /* optional */ true, "'true' if the transaction has been abandoned (inputs are respendable). Only available for the \n" "'send' category of transactions."}, }}, }}, {RPCResult::Type::STR_HEX, "hex", "Raw data for transaction"}, - {RPCResult::Type::OBJ, "decoded", "Optional, the decoded transaction (only present when `verbose` is passed)", + {RPCResult::Type::OBJ, "decoded", /* optional */ true, "The decoded transaction (only present when `verbose` is passed)", { {RPCResult::Type::ELISION, "", "Equivalent to the RPC decoderawtransaction method, or the RPC getrawtransaction method when `verbose` is passed."}, }}, @@ -1731,16 +1743,16 @@ static RPCHelpMan gettransaction() } const CWalletTx& wtx = it->second; - CAmount nCredit = wtx.GetCredit(filter); - CAmount nDebit = wtx.GetDebit(filter); + CAmount nCredit = CachedTxGetCredit(*pwallet, wtx, filter); + CAmount nDebit = CachedTxGetDebit(*pwallet, wtx, filter); CAmount nNet = nCredit - nDebit; - CAmount nFee = (wtx.IsFromMe(filter) ? wtx.tx->GetValueOut() - nDebit : 0); + CAmount nFee = (CachedTxIsFromMe(*pwallet, wtx, filter) ? wtx.tx->GetValueOut() - nDebit : 0); entry.pushKV("amount", ValueFromAmount(nNet - nFee)); - if (wtx.IsFromMe(filter)) + if (CachedTxIsFromMe(*pwallet, wtx, filter)) entry.pushKV("fee", ValueFromAmount(nFee)); - WalletTxToJSON(pwallet->chain(), wtx, entry); + WalletTxToJSON(*pwallet, wtx, entry); UniValue details(UniValue::VARR); ListTransactions(*pwallet, wtx, 0, false, details, filter, nullptr /* filter_label */); @@ -1751,7 +1763,7 @@ static RPCHelpMan gettransaction() if (verbose) { UniValue decoded(UniValue::VOBJ); - TxToUniv(*wtx.tx, uint256(), pwallet->chain().rpcEnableDeprecated("addresses"), decoded, false); + TxToUniv(*wtx.tx, uint256(), decoded, false); entry.pushKV("decoded", decoded); } @@ -2129,8 +2141,9 @@ static RPCHelpMan lockunspent() "If no transaction outputs are specified when unlocking then all current locked transaction outputs are unlocked.\n" "A locked transaction output will not be chosen by automatic coin selection, when spending bitcoins.\n" "Manually selected coins are automatically unlocked.\n" - "Locks are stored in memory only. Nodes start with zero locked outputs, and the locked output list\n" - "is always cleared (by virtue of process exit) when a node stops or fails.\n" + "Locks are stored in memory only, unless persistent=true, in which case they will be written to the\n" + "wallet database and loaded on node start. Unwritten (persistent=false) locks are always cleared\n" + "(by virtue of process exit) when a node stops or fails. Unlocking will clear both persistent and not.\n" "Also see the listunspent call\n", { {"unlock", RPCArg::Type::BOOL, RPCArg::Optional::NO, "Whether to unlock (true) or lock (false) the specified transactions"}, @@ -2144,6 +2157,7 @@ static RPCHelpMan lockunspent() }, }, }, + {"persistent", RPCArg::Type::BOOL, RPCArg::Default{false}, "Whether to write/erase this lock in the wallet database, or keep the change in memory only. Ignored for unlocking."}, }, RPCResult{ RPCResult::Type::BOOL, "", "Whether the command was successful or not" @@ -2157,6 +2171,8 @@ static RPCHelpMan lockunspent() + HelpExampleCli("listlockunspent", "") + "\nUnlock the transaction again\n" + HelpExampleCli("lockunspent", "true \"[{\\\"txid\\\":\\\"a08e6907dbbd3d809776dbfc5d82e371b764ed838b5655e72f463568df1aadf0\\\",\\\"vout\\\":1}]\"") + + "\nLock the transaction persistently in the wallet database\n" + + HelpExampleCli("lockunspent", "false \"[{\\\"txid\\\":\\\"a08e6907dbbd3d809776dbfc5d82e371b764ed838b5655e72f463568df1aadf0\\\",\\\"vout\\\":1}]\" true") + "\nAs a JSON-RPC call\n" + HelpExampleRpc("lockunspent", "false, \"[{\\\"txid\\\":\\\"a08e6907dbbd3d809776dbfc5d82e371b764ed838b5655e72f463568df1aadf0\\\",\\\"vout\\\":1}]\"") }, @@ -2175,9 +2191,13 @@ static RPCHelpMan lockunspent() bool fUnlock = request.params[0].get_bool(); + const bool persistent{request.params[2].isNull() ? false : request.params[2].get_bool()}; + if (request.params[1].isNull()) { - if (fUnlock) - pwallet->UnlockAllCoins(); + if (fUnlock) { + if (!pwallet->UnlockAllCoins()) + throw JSONRPCError(RPC_WALLET_ERROR, "Unlocking coins failed"); + } return true; } @@ -2228,17 +2248,24 @@ static RPCHelpMan lockunspent() throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, expected locked output"); } - if (!fUnlock && is_locked) { + if (!fUnlock && is_locked && !persistent) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, output already locked"); } outputs.push_back(outpt); } + std::unique_ptr<WalletBatch> batch = nullptr; + // Unlock is always persistent + if (fUnlock || persistent) batch = std::make_unique<WalletBatch>(pwallet->GetDatabase()); + // Atomically set (un)locked status for the outputs. for (const COutPoint& outpt : outputs) { - if (fUnlock) pwallet->UnlockCoin(outpt); - else pwallet->LockCoin(outpt); + if (fUnlock) { + if (!pwallet->UnlockCoin(outpt, batch.get())) throw JSONRPCError(RPC_WALLET_ERROR, "Unlocking coin failed"); + } else { + if (!pwallet->LockCoin(outpt, batch.get())) throw JSONRPCError(RPC_WALLET_ERROR, "Locking coin failed"); + } } return true; @@ -2354,9 +2381,9 @@ static RPCHelpMan getbalances() {RPCResult::Type::STR_AMOUNT, "trusted", "trusted balance (outputs created by the wallet or confirmed outputs)"}, {RPCResult::Type::STR_AMOUNT, "untrusted_pending", "untrusted pending balance (outputs created by others that are in the mempool)"}, {RPCResult::Type::STR_AMOUNT, "immature", "balance from immature coinbase outputs"}, - {RPCResult::Type::STR_AMOUNT, "used", "(only present if avoid_reuse is set) balance from coins sent to addresses that were previously spent from (potentially privacy violating)"}, + {RPCResult::Type::STR_AMOUNT, "used", /* optional */ true, "(only present if avoid_reuse is set) balance from coins sent to addresses that were previously spent from (potentially privacy violating)"}, }}, - {RPCResult::Type::OBJ, "watchonly", "watchonly balances (not present if wallet does not watch anything)", + {RPCResult::Type::OBJ, "watchonly", /* optional */ true, "watchonly balances (not present if wallet does not watch anything)", { {RPCResult::Type::STR_AMOUNT, "trusted", "trusted balance (outputs created by the wallet or confirmed outputs)"}, {RPCResult::Type::STR_AMOUNT, "untrusted_pending", "untrusted pending balance (outputs created by others that are in the mempool)"}, @@ -2379,7 +2406,7 @@ static RPCHelpMan getbalances() LOCK(wallet.cs_wallet); - const auto bal = wallet.GetBalance(); + const auto bal = GetBalance(wallet); UniValue balances{UniValue::VOBJ}; { UniValue balances_mine{UniValue::VOBJ}; @@ -2389,7 +2416,7 @@ static RPCHelpMan getbalances() if (wallet.IsWalletFlagSet(WALLET_FLAG_AVOID_REUSE)) { // If the AVOID_REUSE flag is set, bal has been set to just the un-reused address balance. Get // the total balance, and then subtract bal to get the reused address balance. - const auto full_bal = wallet.GetBalance(0, false); + const auto full_bal = GetBalance(wallet, 0, false); balances_mine.pushKV("used", ValueFromAmount(full_bal.m_mine_trusted + full_bal.m_mine_untrusted_pending - bal.m_mine_trusted - bal.m_mine_untrusted_pending)); } balances.pushKV("mine", balances_mine); @@ -2423,9 +2450,9 @@ static RPCHelpMan getwalletinfo() {RPCResult::Type::STR_AMOUNT, "unconfirmed_balance", "DEPRECATED. Identical to getbalances().mine.untrusted_pending"}, {RPCResult::Type::STR_AMOUNT, "immature_balance", "DEPRECATED. Identical to getbalances().mine.immature"}, {RPCResult::Type::NUM, "txcount", "the total number of transactions in the wallet"}, - {RPCResult::Type::NUM_TIME, "keypoololdest", "the " + UNIX_EPOCH_TIME + " of the oldest pre-generated key in the key pool. Legacy wallets only."}, + {RPCResult::Type::NUM_TIME, "keypoololdest", /* optional */ true, "the " + UNIX_EPOCH_TIME + " of the oldest pre-generated key in the key pool. Legacy wallets only."}, {RPCResult::Type::NUM, "keypoolsize", "how many new keys are pre-generated (only counts external keys)"}, - {RPCResult::Type::NUM, "keypoolsize_hd_internal", "how many new keys are pre-generated for internal use (used for change outputs, only appears if the wallet is using this feature, otherwise external keys are used)"}, + {RPCResult::Type::NUM, "keypoolsize_hd_internal", /* optional */ true, "how many new keys are pre-generated for internal use (used for change outputs, only appears if the wallet is using this feature, otherwise external keys are used)"}, {RPCResult::Type::NUM_TIME, "unlocked_until", /* optional */ true, "the " + UNIX_EPOCH_TIME + " until which the wallet is unlocked for transfers, or 0 if the wallet is locked (only present for passphrase-encrypted wallets)"}, {RPCResult::Type::STR_AMOUNT, "paytxfee", "the transaction fee configuration, set in " + CURRENCY_UNIT + "/kvB"}, {RPCResult::Type::STR_HEX, "hdseedid", /* optional */ true, "the Hash160 of the HD seed (only present when HD is enabled)"}, @@ -2457,7 +2484,7 @@ static RPCHelpMan getwalletinfo() UniValue obj(UniValue::VOBJ); size_t kpExternalSize = pwallet->KeypoolCountExternalKeys(); - const auto bal = pwallet->GetBalance(); + const auto bal = GetBalance(*pwallet); int64_t kp_oldest = pwallet->GetOldestKeyPoolTime(); obj.pushKV("walletname", pwallet->GetName()); obj.pushKV("walletversion", pwallet->GetVersion()); @@ -2559,7 +2586,8 @@ static RPCHelpMan listwallets() { UniValue obj(UniValue::VARR); - for (const std::shared_ptr<CWallet>& wallet : GetWallets()) { + WalletContext& context = EnsureWalletContext(request.context); + for (const std::shared_ptr<CWallet>& wallet : GetWallets(context)) { LOCK(wallet->cs_wallet); obj.push_back(wallet->GetName()); } @@ -2569,12 +2597,43 @@ static RPCHelpMan listwallets() }; } +static std::tuple<std::shared_ptr<CWallet>, std::vector<bilingual_str>> LoadWalletHelper(WalletContext& context, UniValue load_on_start_param, const std::string wallet_name) +{ + DatabaseOptions options; + DatabaseStatus status; + options.require_existing = true; + bilingual_str error; + std::vector<bilingual_str> warnings; + std::optional<bool> load_on_start = load_on_start_param.isNull() ? std::nullopt : std::optional<bool>(load_on_start_param.get_bool()); + std::shared_ptr<CWallet> const wallet = LoadWallet(context, wallet_name, load_on_start, options, status, error, warnings); + + if (!wallet) { + // Map bad format to not found, since bad format is returned when the + // wallet directory exists, but doesn't contain a data file. + RPCErrorCode code = RPC_WALLET_ERROR; + switch (status) { + case DatabaseStatus::FAILED_NOT_FOUND: + case DatabaseStatus::FAILED_BAD_FORMAT: + code = RPC_WALLET_NOT_FOUND; + break; + case DatabaseStatus::FAILED_ALREADY_LOADED: + code = RPC_WALLET_ALREADY_LOADED; + break; + default: // RPC_WALLET_ERROR is returned for all other cases. + break; + } + throw JSONRPCError(code, error.original); + } + + return { wallet, warnings }; +} + static RPCHelpMan loadwallet() { return RPCHelpMan{"loadwallet", "\nLoads a wallet from a wallet file or directory." "\nNote that all wallet command-line options used when starting bitcoind will be" - "\napplied to the new wallet (eg -rescan, etc).\n", + "\napplied to the new wallet.\n", { {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The wallet directory or .dat file."}, {"load_on_startup", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED_NAMED_ARG, "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, @@ -2595,30 +2654,7 @@ static RPCHelpMan loadwallet() WalletContext& context = EnsureWalletContext(request.context); const std::string name(request.params[0].get_str()); - DatabaseOptions options; - DatabaseStatus status; - options.require_existing = true; - bilingual_str error; - std::vector<bilingual_str> warnings; - std::optional<bool> load_on_start = request.params[1].isNull() ? std::nullopt : std::optional<bool>(request.params[1].get_bool()); - std::shared_ptr<CWallet> const wallet = LoadWallet(*context.chain, name, load_on_start, options, status, error, warnings); - if (!wallet) { - // Map bad format to not found, since bad format is returned when the - // wallet directory exists, but doesn't contain a data file. - RPCErrorCode code = RPC_WALLET_ERROR; - switch (status) { - case DatabaseStatus::FAILED_NOT_FOUND: - case DatabaseStatus::FAILED_BAD_FORMAT: - code = RPC_WALLET_NOT_FOUND; - break; - case DatabaseStatus::FAILED_ALREADY_LOADED: - code = RPC_WALLET_ALREADY_LOADED; - break; - default: // RPC_WALLET_ERROR is returned for all other cases. - break; - } - throw JSONRPCError(code, error.original); - } + auto [wallet, warnings] = LoadWalletHelper(context, request.params[1], name); UniValue obj(UniValue::VOBJ); obj.pushKV("name", wallet->GetName()); @@ -2777,7 +2813,7 @@ static RPCHelpMan createwallet() options.create_passphrase = passphrase; bilingual_str error; std::optional<bool> load_on_start = request.params[6].isNull() ? std::nullopt : std::optional<bool>(request.params[6].get_bool()); - std::shared_ptr<CWallet> wallet = CreateWallet(*context.chain, request.params[0].get_str(), load_on_start, options, status, error, warnings); + std::shared_ptr<CWallet> wallet = CreateWallet(context, request.params[0].get_str(), load_on_start, options, status, error, warnings); if (!wallet) { RPCErrorCode code = status == DatabaseStatus::FAILED_ENCRYPT ? RPC_WALLET_ENCRYPTION_FAILED : RPC_WALLET_ERROR; throw JSONRPCError(code, error.original); @@ -2792,6 +2828,68 @@ static RPCHelpMan createwallet() }; } +static RPCHelpMan restorewallet() +{ + return RPCHelpMan{ + "restorewallet", + "\nRestore and loads a wallet from backup.\n", + { + {"wallet_name", RPCArg::Type::STR, RPCArg::Optional::NO, "The name that will be applied to the restored wallet"}, + {"backup_file", RPCArg::Type::STR, RPCArg::Optional::NO, "The backup file that will be used to restore the wallet."}, + {"load_on_startup", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED_NAMED_ARG, "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR, "name", "The wallet name if restored successfully."}, + {RPCResult::Type::STR, "warning", "Warning message if wallet was not loaded cleanly."}, + } + }, + RPCExamples{ + HelpExampleCli("restorewallet", "\"testwallet\" \"home\\backups\\backup-file.bak\"") + + HelpExampleRpc("restorewallet", "\"testwallet\" \"home\\backups\\backup-file.bak\"") + + HelpExampleCliNamed("restorewallet", {{"wallet_name", "testwallet"}, {"backup_file", "home\\backups\\backup-file.bak\""}, {"load_on_startup", true}}) + + HelpExampleRpcNamed("restorewallet", {{"wallet_name", "testwallet"}, {"backup_file", "home\\backups\\backup-file.bak\""}, {"load_on_startup", true}}) + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + + WalletContext& context = EnsureWalletContext(request.context); + + std::string backup_file = request.params[1].get_str(); + + if (!fs::exists(backup_file)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Backup file does not exist"); + } + + std::string wallet_name = request.params[0].get_str(); + + const fs::path wallet_path = fsbridge::AbsPathJoin(GetWalletDir(), wallet_name); + + if (fs::exists(wallet_path)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Wallet name already exists."); + } + + if (!TryCreateDirectories(wallet_path)) { + throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Failed to create database path '%s'. Database already exists.", wallet_path.string())); + } + + auto wallet_file = wallet_path / "wallet.dat"; + + fs::copy_file(backup_file, wallet_file, fs::copy_option::fail_if_exists); + + auto [wallet, warnings] = LoadWalletHelper(context, request.params[2], wallet_name); + + UniValue obj(UniValue::VOBJ); + obj.pushKV("name", wallet->GetName()); + obj.pushKV("warning", Join(warnings, Untranslated("\n")).original); + + return obj; + +}, + }; +} + static RPCHelpMan unloadwallet() { return RPCHelpMan{"unloadwallet", @@ -2819,7 +2917,8 @@ static RPCHelpMan unloadwallet() wallet_name = request.params[0].get_str(); } - std::shared_ptr<CWallet> wallet = GetWallet(wallet_name); + WalletContext& context = EnsureWalletContext(request.context); + std::shared_ptr<CWallet> wallet = GetWallet(context, wallet_name); if (!wallet) { throw JSONRPCError(RPC_WALLET_NOT_FOUND, "Requested wallet does not exist or is not loaded"); } @@ -2829,7 +2928,7 @@ static RPCHelpMan unloadwallet() // is destroyed (see CheckUniqueFileid). std::vector<bilingual_str> warnings; std::optional<bool> load_on_start = request.params[1].isNull() ? std::nullopt : std::optional<bool>(request.params[1].get_bool()); - if (!RemoveWallet(wallet, load_on_start, warnings)) { + if (!RemoveWallet(context, wallet, load_on_start, warnings)) { throw JSONRPCError(RPC_MISC_ERROR, "Requested wallet already unloaded"); } @@ -2875,17 +2974,20 @@ static RPCHelpMan listunspent() { {RPCResult::Type::STR_HEX, "txid", "the transaction id"}, {RPCResult::Type::NUM, "vout", "the vout value"}, - {RPCResult::Type::STR, "address", "the bitcoin address"}, - {RPCResult::Type::STR, "label", "The associated label, or \"\" for the default label"}, + {RPCResult::Type::STR, "address", /* optional */ true, "the bitcoin address"}, + {RPCResult::Type::STR, "label", /* optional */ true, "The associated label, or \"\" for the default label"}, {RPCResult::Type::STR, "scriptPubKey", "the script key"}, {RPCResult::Type::STR_AMOUNT, "amount", "the transaction output amount in " + CURRENCY_UNIT}, {RPCResult::Type::NUM, "confirmations", "The number of confirmations"}, - {RPCResult::Type::STR_HEX, "redeemScript", "The redeemScript if scriptPubKey is P2SH"}, - {RPCResult::Type::STR, "witnessScript", "witnessScript if the scriptPubKey is P2WSH or P2SH-P2WSH"}, + {RPCResult::Type::NUM, "ancestorcount", /* optional */ true, "The number of in-mempool ancestor transactions, including this one (if transaction is in the mempool)"}, + {RPCResult::Type::NUM, "ancestorsize", /* optional */ true, "The virtual transaction size of in-mempool ancestors, including this one (if transaction is in the mempool)"}, + {RPCResult::Type::STR_AMOUNT, "ancestorfees", /* optional */ true, "The total fees of in-mempool ancestors (including this one) with fee deltas used for mining priority in " + CURRENCY_ATOM + " (if transaction is in the mempool)"}, + {RPCResult::Type::STR_HEX, "redeemScript", /* optional */ true, "The redeemScript if scriptPubKey is P2SH"}, + {RPCResult::Type::STR, "witnessScript", /* optional */ true, "witnessScript if the scriptPubKey is P2WSH or P2SH-P2WSH"}, {RPCResult::Type::BOOL, "spendable", "Whether we have the private keys to spend this output"}, {RPCResult::Type::BOOL, "solvable", "Whether we know how to spend this output, ignoring the lack of keys"}, - {RPCResult::Type::BOOL, "reused", "(only present if avoid_reuse is set) Whether this output is reused/dirty (sent to an address that was previously spent from)"}, - {RPCResult::Type::STR, "desc", "(only when solvable) A descriptor for spending this output"}, + {RPCResult::Type::BOOL, "reused", /* optional */ true, "(only present if avoid_reuse is set) Whether this output is reused/dirty (sent to an address that was previously spent from)"}, + {RPCResult::Type::STR, "desc", /* optional */ true, "(only when solvable) A descriptor for spending this output"}, {RPCResult::Type::BOOL, "safe", "Whether this output is considered safe to spend. Unconfirmed transactions\n" "from outside keys and unconfirmed replacement transactions are considered unsafe\n" "and are not eligible for spending by fundrawtransaction and sendtoaddress."}, @@ -2981,7 +3083,7 @@ static RPCHelpMan listunspent() cctl.m_max_depth = nMaxDepth; cctl.m_include_unsafe_inputs = include_unsafe; LOCK(pwallet->cs_wallet); - pwallet->AvailableCoins(vecOutputs, &cctl, nMinimumAmount, nMaximumAmount, nMinimumSumAmount, nMaximumCount); + AvailableCoins(*pwallet, vecOutputs, &cctl, nMinimumAmount, nMaximumAmount, nMinimumSumAmount, nMaximumCount); } LOCK(pwallet->cs_wallet); @@ -3046,6 +3148,16 @@ static RPCHelpMan listunspent() entry.pushKV("scriptPubKey", HexStr(scriptPubKey)); entry.pushKV("amount", ValueFromAmount(out.tx->tx->vout[out.i].nValue)); entry.pushKV("confirmations", out.nDepth); + if (!out.nDepth) { + size_t ancestor_count, descendant_count, ancestor_size; + CAmount ancestor_fees; + pwallet->chain().getTransactionAncestry(out.tx->GetHash(), ancestor_count, descendant_count, &ancestor_size, &ancestor_fees); + if (ancestor_count) { + entry.pushKV("ancestorcount", uint64_t(ancestor_count)); + entry.pushKV("ancestorsize", uint64_t(ancestor_size)); + entry.pushKV("ancestorfees", uint64_t(ancestor_fees)); + } + } entry.pushKV("spendable", out.fSpendable); entry.pushKV("solvable", out.fSolvable); if (out.fSolvable) { @@ -3065,6 +3177,34 @@ static RPCHelpMan listunspent() }; } +// Only includes key documentation where the key is snake_case in all RPC methods. MixedCase keys can be added later. +static std::vector<RPCArg> FundTxDoc() +{ + return { + {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, + {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, std::string() + "The fee estimate mode, must be one of (case insensitive):\n" + " \"" + FeeModes("\"\n\"") + "\""}, + {"replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125 replaceable.\n" + "Allows this transaction to be replaced by a transaction with higher fees"}, + {"solving_data", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "Keys and scripts needed for producing a final transaction with a dummy signature.\n" + "Used for fee estimation during coin selection.", + { + {"pubkeys", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Public keys involved in this transaction.", + { + {"pubkey", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A public key"}, + }}, + {"scripts", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Scripts involved in this transaction.", + { + {"script", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A script"}, + }}, + {"descriptors", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Descriptors that provide solving data for this transaction.", + { + {"descriptor", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "A descriptor"}, + }}, + }}, + }; +} + void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, int& change_position, const UniValue& options, CCoinControl& coinControl, bool override_min_fee) { // Make sure the results are valid at least up to the most recent block @@ -3102,6 +3242,7 @@ void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, {"fee_rate", UniValueType()}, // will be checked by AmountFromValue() in SetFeeEstimateMode() {"feeRate", UniValueType()}, // will be checked by AmountFromValue() below {"psbt", UniValueType(UniValue::VBOOL)}, + {"solving_data", UniValueType(UniValue::VOBJ)}, {"subtractFeeFromOutputs", UniValueType(UniValue::VARR)}, {"subtract_fee_from_outputs", UniValueType(UniValue::VARR)}, {"replaceable", UniValueType(UniValue::VBOOL)}, @@ -3133,11 +3274,11 @@ void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, if (options.exists("changeAddress") || options.exists("change_address")) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot specify both change address and address type options"); } - OutputType out_type; - if (!ParseOutputType(options["change_type"].get_str(), out_type)) { + if (std::optional<OutputType> parsed = ParseOutputType(options["change_type"].get_str())) { + coinControl.m_change_type.emplace(parsed.value()); + } else { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Unknown change type '%s'", options["change_type"].get_str())); } - coinControl.m_change_type.emplace(out_type); } const UniValue include_watching_option = options.exists("include_watching") ? options["include_watching"] : options["includeWatching"]; @@ -3178,6 +3319,54 @@ void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, coinControl.fAllowWatchOnly = ParseIncludeWatchonly(NullUniValue, wallet); } + if (options.exists("solving_data")) { + const UniValue solving_data = options["solving_data"].get_obj(); + if (solving_data.exists("pubkeys")) { + for (const UniValue& pk_univ : solving_data["pubkeys"].get_array().getValues()) { + const std::string& pk_str = pk_univ.get_str(); + if (!IsHex(pk_str)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("'%s' is not hex", pk_str)); + } + const std::vector<unsigned char> data(ParseHex(pk_str)); + const CPubKey pubkey(data.begin(), data.end()); + if (!pubkey.IsFullyValid()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("'%s' is not a valid public key", pk_str)); + } + coinControl.m_external_provider.pubkeys.emplace(pubkey.GetID(), pubkey); + // Add witness script for pubkeys + const CScript wit_script = GetScriptForDestination(WitnessV0KeyHash(pubkey)); + coinControl.m_external_provider.scripts.emplace(CScriptID(wit_script), wit_script); + } + } + + if (solving_data.exists("scripts")) { + for (const UniValue& script_univ : solving_data["scripts"].get_array().getValues()) { + const std::string& script_str = script_univ.get_str(); + if (!IsHex(script_str)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("'%s' is not hex", script_str)); + } + std::vector<unsigned char> script_data(ParseHex(script_str)); + const CScript script(script_data.begin(), script_data.end()); + coinControl.m_external_provider.scripts.emplace(CScriptID(script), script); + } + } + + if (solving_data.exists("descriptors")) { + for (const UniValue& desc_univ : solving_data["descriptors"].get_array().getValues()) { + const std::string& desc_str = desc_univ.get_str(); + FlatSigningProvider desc_out; + std::string error; + std::vector<CScript> scripts_temp; + std::unique_ptr<Descriptor> desc = Parse(desc_str, desc_out, error, true); + if (!desc) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Unable to parse descriptor '%s': %s", desc_str, error)); + } + desc->Expand(0, desc_out, scripts_temp, desc_out); + coinControl.m_external_provider = Merge(coinControl.m_external_provider, desc_out); + } + } + } + if (tx.vout.size() == 0) throw JSONRPCError(RPC_INVALID_PARAMETER, "TX must have at least one output"); @@ -3195,9 +3384,22 @@ void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, setSubtractFeeFromOutputs.insert(pos); } + // Fetch specified UTXOs from the UTXO set to get the scriptPubKeys and values of the outputs being selected + // and to match with the given solving_data. Only used for non-wallet outputs. + std::map<COutPoint, Coin> coins; + for (const CTxIn& txin : tx.vin) { + coins[txin.prevout]; // Create empty map entry keyed by prevout. + } + wallet.chain().findCoins(coins); + for (const auto& coin : coins) { + if (!coin.second.out.IsNull()) { + coinControl.SelectExternal(coin.first, coin.second.out); + } + } + bilingual_str error; - if (!wallet.FundTransaction(tx, fee_out, change_position, error, lockUnspents, setSubtractFeeFromOutputs, coinControl)) { + if (!FundTransaction(wallet, tx, fee_out, change_position, error, lockUnspents, setSubtractFeeFromOutputs, coinControl)) { throw JSONRPCError(RPC_WALLET_ERROR, error.original); } } @@ -3210,8 +3412,9 @@ static RPCHelpMan fundrawtransaction() "No existing outputs will be modified unless \"subtractFeeFromOutputs\" is specified.\n" "Note that inputs which were signed may need to be resigned after completion since in/outputs have been added.\n" "The inputs added will not be signed, use signrawtransactionwithkey\n" - " or signrawtransactionwithwallet for that.\n" - "Note that all existing inputs must have their previous output transaction be in the wallet.\n" + "or signrawtransactionwithwallet for that.\n" + "All existing inputs must either have their previous output transaction be in the wallet\n" + "or be in the UTXO set. Solving data must be provided for non-wallet inputs.\n" "Note that all inputs selected must be of standard form and P2SH scripts must be\n" "in the wallet using importaddress or addmultisigaddress (to calculate fees).\n" "You can see whether this is the case by checking the \"solvable\" field in the listunspent output.\n" @@ -3219,6 +3422,7 @@ static RPCHelpMan fundrawtransaction() { {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of the raw transaction"}, {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "for backward compatibility: passing in a true instead of an object will result in {\"includeWatching\":true}", + Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::Default{true}, "For a transaction with existing inputs, automatically include more if they are not enough."}, {"include_unsafe", RPCArg::Type::BOOL, RPCArg::Default{false}, "Include inputs that are not safe to spend (unconfirmed transactions from outside keys and unconfirmed replacement transactions).\n" @@ -3241,12 +3445,8 @@ static RPCHelpMan fundrawtransaction() {"vout_index", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "The zero-based output index, before a change output is added."}, }, }, - {"replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125 replaceable.\n" - "Allows this transaction to be replaced by a transaction with higher fees"}, - {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, - {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, std::string() + "The fee estimate mode, must be one of (case insensitive):\n" - " \"" + FeeModes("\"\n\"") + "\""}, }, + FundTxDoc()), "options"}, {"iswitness", RPCArg::Type::BOOL, RPCArg::DefaultHint{"depends on heuristic tests"}, "Whether the transaction hex is a serialized witness transaction.\n" "If iswitness is not present, heuristic tests will be used in decoding.\n" @@ -3349,6 +3549,10 @@ RPCHelpMan signrawtransactionwithwallet() { {RPCResult::Type::STR_HEX, "txid", "The hash of the referenced, previous transaction"}, {RPCResult::Type::NUM, "vout", "The index of the output to spent and used as input"}, + {RPCResult::Type::ARR, "witness", "", + { + {RPCResult::Type::STR_HEX, "witness", ""}, + }}, {RPCResult::Type::STR_HEX, "scriptSig", "The hex-encoded signature script"}, {RPCResult::Type::NUM, "sequence", "Script sequence number"}, {RPCResult::Type::STR, "error", "Verification or signing error related to the input"}, @@ -3389,7 +3593,7 @@ RPCHelpMan signrawtransactionwithwallet() int nHashType = ParseSighashString(request.params[2]); // Script verification errors - std::map<int, std::string> input_errors; + std::map<int, bilingual_str> input_errors; bool complete = pwallet->SignTransaction(mtx, coins, nHashType, input_errors); UniValue result(UniValue::VOBJ); @@ -3684,7 +3888,6 @@ public: obj.pushKV("embedded", std::move(subobj)); } else if (which_type == TxoutType::MULTISIG) { // Also report some information on multisig scripts (which do not have a corresponding address). - // TODO: abstract out the common functionality between this logic and ExtractDestinations. obj.pushKV("sigsrequired", solutions_data[0][0]); UniValue pubkeys(UniValue::VARR); for (size_t i = 1; i < solutions_data.size() - 1; ++i) { @@ -3872,7 +4075,7 @@ RPCHelpMan getaddressinfo() DescriptorScriptPubKeyMan* desc_spk_man = dynamic_cast<DescriptorScriptPubKeyMan*>(pwallet->GetScriptPubKeyMan(scriptPubKey)); if (desc_spk_man) { std::string desc_str; - if (desc_spk_man->GetDescriptorString(desc_str, false)) { + if (desc_spk_man->GetDescriptorString(desc_str, /* priv */ false)) { ret.pushKV("parent_desc", desc_str); } } @@ -3882,7 +4085,7 @@ RPCHelpMan getaddressinfo() UniValue detail = DescribeWalletAddress(*pwallet, dest); ret.pushKVs(detail); - ret.pushKV("ischange", pwallet->IsChange(scriptPubKey)); + ret.pushKV("ischange", ScriptIsChange(*pwallet, scriptPubKey)); ScriptPubKeyMan* spk_man = pwallet->GetScriptPubKeyMan(scriptPubKey); if (spk_man) { @@ -4052,6 +4255,7 @@ static RPCHelpMan send() " \"" + FeeModes("\"\n\"") + "\""}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", + Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::Default{false}, "If inputs are specified, automatically include more if they are not enough."}, {"include_unsafe", RPCArg::Type::BOOL, RPCArg::Default{false}, "Include inputs that are not safe to spend (unconfirmed transactions from outside keys and unconfirmed replacement transactions).\n" @@ -4061,9 +4265,6 @@ static RPCHelpMan send() {"change_address", RPCArg::Type::STR_HEX, RPCArg::DefaultHint{"pool address"}, "The bitcoin address to receive the change"}, {"change_position", RPCArg::Type::NUM, RPCArg::DefaultHint{"random"}, "The index of the change output"}, {"change_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The output type to use. Only valid if change_address is not specified. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, - {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, - {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, std::string() + "The fee estimate mode, must be one of (case insensitive):\n" - " \"" + FeeModes("\"\n\"") + "\""}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, {"include_watching", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Also select inputs which are watch only.\n" "Only solvable inputs can be used. Watch-only destinations are solvable if the public key and/or output script was imported,\n" @@ -4086,18 +4287,17 @@ static RPCHelpMan send() {"vout_index", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "The zero-based output index, before a change output is added."}, }, }, - {"replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125 replaceable.\n" - "Allows this transaction to be replaced by a transaction with higher fees"}, }, + FundTxDoc()), "options"}, }, RPCResult{ RPCResult::Type::OBJ, "", "", { {RPCResult::Type::BOOL, "complete", "If the transaction has a complete set of signatures"}, - {RPCResult::Type::STR_HEX, "txid", "The transaction id for the send. Only 1 transaction is created regardless of the number of addresses."}, - {RPCResult::Type::STR_HEX, "hex", "If add_to_wallet is false, the hex-encoded raw transaction with signature(s)"}, - {RPCResult::Type::STR, "psbt", "If more signatures are needed, or if add_to_wallet is false, the base64-encoded (partially) signed transaction"} + {RPCResult::Type::STR_HEX, "txid", /* optional */ true, "The transaction id for the send. Only 1 transaction is created regardless of the number of addresses."}, + {RPCResult::Type::STR_HEX, "hex", /* optional */ true, "If add_to_wallet is false, the hex-encoded raw transaction with signature(s)"}, + {RPCResult::Type::STR, "psbt", /* optional */ true, "If more signatures are needed, or if add_to_wallet is false, the base64-encoded (partially) signed transaction"} } }, RPCExamples{"" @@ -4305,7 +4505,7 @@ static RPCHelpMan walletprocesspsbt() HELP_REQUIRING_PASSPHRASE, { {"psbt", RPCArg::Type::STR, RPCArg::Optional::NO, "The transaction base64 string"}, - {"sign", RPCArg::Type::BOOL, RPCArg::Default{true}, "Also sign the transaction when updating"}, + {"sign", RPCArg::Type::BOOL, RPCArg::Default{true}, "Also sign the transaction when updating (requires wallet to be unlocked)"}, {"sighashtype", RPCArg::Type::STR, RPCArg::Default{"DEFAULT"}, "The signature hash type to sign with if not specified by the PSBT. Must be one of\n" " \"DEFAULT\"\n" " \"ALL\"\n" @@ -4352,6 +4552,9 @@ static RPCHelpMan walletprocesspsbt() bool sign = request.params[1].isNull() ? true : request.params[1].get_bool(); bool bip32derivs = request.params[3].isNull() ? true : request.params[3].get_bool(); bool complete = true; + + if (sign) EnsureWalletIsUnlocked(*pwallet); + const TransactionError err{wallet.FillPSBT(psbtx, complete, nHashType, sign, bip32derivs)}; if (err != TransactionError::OK) { throw JSONRPCTransactionError(err); @@ -4372,7 +4575,9 @@ static RPCHelpMan walletcreatefundedpsbt() { return RPCHelpMan{"walletcreatefundedpsbt", "\nCreates and funds a transaction in the Partially Signed Transaction format.\n" - "Implements the Creator and Updater roles.\n", + "Implements the Creator and Updater roles.\n" + "All existing inputs must either have their previous output transaction be in the wallet\n" + "or be in the UTXO set. Solving data must be provided for non-wallet inputs.\n", { {"inputs", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "Leave empty to add inputs automatically. See add_inputs option.", { @@ -4404,6 +4609,7 @@ static RPCHelpMan walletcreatefundedpsbt() }, {"locktime", RPCArg::Type::NUM, RPCArg::Default{0}, "Raw locktime. Non-0 value also locktime-activates inputs"}, {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", + Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::Default{false}, "If inputs are specified, automatically include more if they are not enough."}, {"include_unsafe", RPCArg::Type::BOOL, RPCArg::Default{false}, "Include inputs that are not safe to spend (unconfirmed transactions from outside keys and unconfirmed replacement transactions).\n" @@ -4424,12 +4630,8 @@ static RPCHelpMan walletcreatefundedpsbt() {"vout_index", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "The zero-based output index, before a change output is added."}, }, }, - {"replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125 replaceable.\n" - "Allows this transaction to be replaced by a transaction with higher fees"}, - {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, - {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, std::string() + "The fee estimate mode, must be one of (case insensitive):\n" - " \"" + FeeModes("\"\n\"") + "\""}, }, + FundTxDoc()), "options"}, {"bip32derivs", RPCArg::Type::BOOL, RPCArg::Default{true}, "Include BIP 32 derivation paths for public keys if we know them"}, }, @@ -4570,10 +4772,11 @@ static RPCHelpMan upgradewallet() #ifdef ENABLE_EXTERNAL_SIGNER static RPCHelpMan walletdisplayaddress() { - return RPCHelpMan{"walletdisplayaddress", + return RPCHelpMan{ + "walletdisplayaddress", "Display address on an external signer for verification.", { - {"address", RPCArg::Type::STR, RPCArg::Optional::NO, /* default_val */ "", "bitcoin address to display"}, + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "bitcoin address to display"}, }, RPCResult{ RPCResult::Type::OBJ,"","", @@ -4636,6 +4839,7 @@ static const CRPCCommand commands[] = { "wallet", &bumpfee, }, { "wallet", &psbtbumpfee, }, { "wallet", &createwallet, }, + { "wallet", &restorewallet, }, { "wallet", &dumpprivkey, }, { "wallet", &dumpwallet, }, { "wallet", &encryptwallet, }, diff --git a/src/wallet/salvage.cpp b/src/wallet/salvage.cpp index ea045eb6d8..4151099c1f 100644 --- a/src/wallet/salvage.cpp +++ b/src/wallet/salvage.cpp @@ -45,7 +45,7 @@ bool RecoverDatabaseFile(const fs::path& file_path, bilingual_str& error, std::v // Call Salvage with fAggressive=true to // get as much data as possible. // Rewrite salvaged data to fresh wallet file - // Set -rescan so any missing transactions will be + // Rescan so any missing transactions will be // found. int64_t now = GetTime(); std::string newFilename = strprintf("%s.%d.bak", filename, now); diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 44c3912544..fdfb36bb0a 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -20,10 +20,10 @@ //! Value for the first BIP 32 hardened derivation. Can be used as a bit mask and as a value. See BIP 32 for more details. const uint32_t BIP32_HARDENED_KEY_LIMIT = 0x80000000; -bool LegacyScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) +bool LegacyScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestination& dest, bilingual_str& error) { if (LEGACY_OUTPUT_TYPES.count(type) == 0) { - error = _("Error: Legacy wallets only support the \"legacy\", \"p2sh-segwit\", and \"bech32\" address types").translated; + error = _("Error: Legacy wallets only support the \"legacy\", \"p2sh-segwit\", and \"bech32\" address types"); return false; } assert(type != OutputType::BECH32M); @@ -34,7 +34,7 @@ bool LegacyScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestinat // Generate a new key that is added to wallet CPubKey new_key; if (!GetKeyFromPool(new_key, type)) { - error = _("Error: Keypool ran out, please call keypoolrefill first").translated; + error = _("Error: Keypool ran out, please call keypoolrefill first"); return false; } LearnRelatedScripts(new_key, type); @@ -295,22 +295,22 @@ bool LegacyScriptPubKeyMan::Encrypt(const CKeyingMaterial& master_key, WalletBat return true; } -bool LegacyScriptPubKeyMan::GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool, std::string& error) +bool LegacyScriptPubKeyMan::GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool, bilingual_str& error) { if (LEGACY_OUTPUT_TYPES.count(type) == 0) { - error = _("Error: Legacy wallets only support the \"legacy\", \"p2sh-segwit\", and \"bech32\" address types").translated; + error = _("Error: Legacy wallets only support the \"legacy\", \"p2sh-segwit\", and \"bech32\" address types"); return false; } assert(type != OutputType::BECH32M); LOCK(cs_KeyStore); if (!CanGetAddresses(internal)) { - error = _("Error: Keypool ran out, please call keypoolrefill first").translated; + error = _("Error: Keypool ran out, please call keypoolrefill first"); return false; } if (!ReserveKeyFromKeyPool(index, keypool, internal)) { - error = _("Error: Keypool ran out, please call keypoolrefill first").translated; + error = _("Error: Keypool ran out, please call keypoolrefill first"); return false; } address = GetDestinationForKey(keypool.vchPubKey, type); @@ -331,7 +331,7 @@ bool LegacyScriptPubKeyMan::TopUpInactiveHDChain(const CKeyID seed_id, int64_t i CHDChain& chain = it->second; // Top up key pool - int64_t target_size = std::max(gArgs.GetArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 1); + int64_t target_size = std::max(gArgs.GetIntArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 1); // "size" of the keypools. Not really the size, actually the difference between index and the chain counter // Since chain counter is 1 based and index is 0 based, one of them needs to be offset by 1. @@ -592,7 +592,7 @@ bool LegacyScriptPubKeyMan::CanProvide(const CScript& script, SignatureData& sig } } -bool LegacyScriptPubKeyMan::SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors) const +bool LegacyScriptPubKeyMan::SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, bilingual_str>& input_errors) const { return ::SignTransaction(tx, this, coins, sighash, input_errors); } @@ -1259,7 +1259,7 @@ bool LegacyScriptPubKeyMan::TopUp(unsigned int kpSize) if (kpSize > 0) nTargetSize = kpSize; else - nTargetSize = std::max(gArgs.GetArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 0); + nTargetSize = std::max(gArgs.GetIntArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 0); // count amount of available keys (internal, external) // make sure the keypool of external and internal keys fits the user selected target (-keypool) @@ -1613,13 +1613,11 @@ std::set<CKeyID> LegacyScriptPubKeyMan::GetKeys() const return set_address; } -void LegacyScriptPubKeyMan::SetInternal(bool internal) {} - -bool DescriptorScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) +bool DescriptorScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestination& dest, bilingual_str& error) { // Returns true if this descriptor supports getting new addresses. Conditions where we may be unable to fetch them (e.g. locked) are caught later - if (!CanGetAddresses(m_internal)) { - error = "No addresses available"; + if (!CanGetAddresses()) { + error = _("No addresses available"); return false; } { @@ -1638,12 +1636,12 @@ bool DescriptorScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDest std::vector<CScript> scripts_temp; if (m_wallet_descriptor.range_end <= m_max_cached_index && !TopUp(1)) { // We can't generate anymore keys - error = "Error: Keypool ran out, please call keypoolrefill first"; + error = _("Error: Keypool ran out, please call keypoolrefill first"); return false; } if (!m_wallet_descriptor.descriptor->ExpandFromCache(m_wallet_descriptor.next_index, m_wallet_descriptor.cache, scripts_temp, out_keys)) { // We can't generate anymore keys - error = "Error: Keypool ran out, please call keypoolrefill first"; + error = _("Error: Keypool ran out, please call keypoolrefill first"); return false; } @@ -1723,7 +1721,7 @@ bool DescriptorScriptPubKeyMan::Encrypt(const CKeyingMaterial& master_key, Walle return true; } -bool DescriptorScriptPubKeyMan::GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool, std::string& error) +bool DescriptorScriptPubKeyMan::GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool, bilingual_str& error) { LOCK(cs_desc_man); bool result = GetNewDestination(type, address, error); @@ -1766,7 +1764,7 @@ bool DescriptorScriptPubKeyMan::TopUp(unsigned int size) if (size > 0) { target_size = size; } else { - target_size = std::max(gArgs.GetArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 1); + target_size = std::max(gArgs.GetIntArg("-keypool", DEFAULT_KEYPOOL_SIZE), (int64_t) 1); } // Calculate the new range_end @@ -1805,34 +1803,10 @@ bool DescriptorScriptPubKeyMan::TopUp(unsigned int size) } m_map_pubkeys[pubkey] = i; } - // Write the cache - for (const auto& parent_xpub_pair : temp_cache.GetCachedParentExtPubKeys()) { - CExtPubKey xpub; - if (m_wallet_descriptor.cache.GetCachedParentExtPubKey(parent_xpub_pair.first, xpub)) { - if (xpub != parent_xpub_pair.second) { - throw std::runtime_error(std::string(__func__) + ": New cached parent xpub does not match already cached parent xpub"); - } - continue; - } - if (!batch.WriteDescriptorParentCache(parent_xpub_pair.second, id, parent_xpub_pair.first)) { - throw std::runtime_error(std::string(__func__) + ": writing cache item failed"); - } - m_wallet_descriptor.cache.CacheParentExtPubKey(parent_xpub_pair.first, parent_xpub_pair.second); - } - for (const auto& derived_xpub_map_pair : temp_cache.GetCachedDerivedExtPubKeys()) { - for (const auto& derived_xpub_pair : derived_xpub_map_pair.second) { - CExtPubKey xpub; - if (m_wallet_descriptor.cache.GetCachedDerivedExtPubKey(derived_xpub_map_pair.first, derived_xpub_pair.first, xpub)) { - if (xpub != derived_xpub_pair.second) { - throw std::runtime_error(std::string(__func__) + ": New cached derived xpub does not match already cached derived xpub"); - } - continue; - } - if (!batch.WriteDescriptorDerivedCache(derived_xpub_pair.second, id, derived_xpub_map_pair.first, derived_xpub_pair.first)) { - throw std::runtime_error(std::string(__func__) + ": writing cache item failed"); - } - m_wallet_descriptor.cache.CacheDerivedExtPubKey(derived_xpub_map_pair.first, derived_xpub_pair.first, derived_xpub_pair.second); - } + // Merge and write the cache + DescriptorCache new_items = m_wallet_descriptor.cache.MergeAndDiff(temp_cache); + if (!batch.WriteDescriptorCacheItems(id, new_items)) { + throw std::runtime_error(std::string(__func__) + ": writing cache items failed"); } m_max_cached_index++; } @@ -1875,6 +1849,12 @@ bool DescriptorScriptPubKeyMan::AddDescriptorKeyWithDB(WalletBatch& batch, const AssertLockHeld(cs_desc_man); assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); + // Check if provided key already exists + if (m_map_keys.find(pubkey.GetID()) != m_map_keys.end() || + m_map_crypted_keys.find(pubkey.GetID()) != m_map_crypted_keys.end()) { + return true; + } + if (m_storage.HasEncryptionKeys()) { if (m_storage.IsLocked()) { return false; @@ -1894,7 +1874,7 @@ bool DescriptorScriptPubKeyMan::AddDescriptorKeyWithDB(WalletBatch& batch, const } } -bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_key, OutputType addr_type) +bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_key, OutputType addr_type, bool internal) { if (addr_type == OutputType::BECH32M) { // Don't allow setting up taproot descriptors yet @@ -1942,7 +1922,7 @@ bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_ desc_prefix += "/0'"; } - std::string internal_path = m_internal ? "/1" : "/0"; + std::string internal_path = internal ? "/1" : "/0"; std::string desc_str = desc_prefix + "/0'" + internal_path + desc_suffix; // Make the descriptor @@ -1997,13 +1977,6 @@ int64_t DescriptorScriptPubKeyMan::GetOldestKeyPoolTime() const return 0; } -size_t DescriptorScriptPubKeyMan::KeypoolCountExternalKeys() const -{ - if (m_internal) { - return 0; - } - return GetKeyPoolSize(); -} unsigned int DescriptorScriptPubKeyMan::GetKeyPoolSize() const { @@ -2073,7 +2046,7 @@ bool DescriptorScriptPubKeyMan::CanProvide(const CScript& script, SignatureData& return IsMine(script); } -bool DescriptorScriptPubKeyMan::SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors) const +bool DescriptorScriptPubKeyMan::SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, bilingual_str>& input_errors) const { std::unique_ptr<FlatSigningProvider> keys = std::make_unique<FlatSigningProvider>(); for (const auto& coin_pair : coins) { @@ -2205,11 +2178,6 @@ uint256 DescriptorScriptPubKeyMan::GetID() const return id; } -void DescriptorScriptPubKeyMan::SetInternal(bool internal) -{ - this->m_internal = internal; -} - void DescriptorScriptPubKeyMan::SetCache(const DescriptorCache& cache) { LOCK(cs_desc_man); @@ -2290,15 +2258,82 @@ const std::vector<CScript> DescriptorScriptPubKeyMan::GetScriptPubKeys() const return script_pub_keys; } -bool DescriptorScriptPubKeyMan::GetDescriptorString(std::string& out, bool priv) const +bool DescriptorScriptPubKeyMan::GetDescriptorString(std::string& out, const bool priv) const { LOCK(cs_desc_man); - if (m_storage.IsLocked()) { - return false; + + FlatSigningProvider provider; + provider.keys = GetKeys(); + + if (priv) { + // For the private version, always return the master key to avoid + // exposing child private keys. The risk implications of exposing child + // private keys together with the parent xpub may be non-obvious for users. + return m_wallet_descriptor.descriptor->ToPrivateString(provider, out); + } + + return m_wallet_descriptor.descriptor->ToNormalizedString(provider, out, &m_wallet_descriptor.cache); +} + +void DescriptorScriptPubKeyMan::UpgradeDescriptorCache() +{ + LOCK(cs_desc_man); + if (m_storage.IsLocked() || m_storage.IsWalletFlagSet(WALLET_FLAG_LAST_HARDENED_XPUB_CACHED)) { + return; } + // Skip if we have the last hardened xpub cache + if (m_wallet_descriptor.cache.GetCachedLastHardenedExtPubKeys().size() > 0) { + return; + } + + // Expand the descriptor FlatSigningProvider provider; provider.keys = GetKeys(); + FlatSigningProvider out_keys; + std::vector<CScript> scripts_temp; + DescriptorCache temp_cache; + if (!m_wallet_descriptor.descriptor->Expand(0, provider, scripts_temp, out_keys, &temp_cache)){ + throw std::runtime_error("Unable to expand descriptor"); + } + + // Cache the last hardened xpubs + DescriptorCache diff = m_wallet_descriptor.cache.MergeAndDiff(temp_cache); + if (!WalletBatch(m_storage.GetDatabase()).WriteDescriptorCacheItems(GetID(), diff)) { + throw std::runtime_error(std::string(__func__) + ": writing cache items failed"); + } +} - return m_wallet_descriptor.descriptor->ToNormalizedString(provider, out, priv); +void DescriptorScriptPubKeyMan::UpdateWalletDescriptor(WalletDescriptor& descriptor) +{ + LOCK(cs_desc_man); + std::string error; + if (!CanUpdateToWalletDescriptor(descriptor, error)) { + throw std::runtime_error(std::string(__func__) + ": " + error); + } + + m_map_pubkeys.clear(); + m_map_script_pub_keys.clear(); + m_max_cached_index = -1; + m_wallet_descriptor = descriptor; +} + +bool DescriptorScriptPubKeyMan::CanUpdateToWalletDescriptor(const WalletDescriptor& descriptor, std::string& error) +{ + LOCK(cs_desc_man); + if (!HasWalletDescriptor(descriptor)) { + error = "can only update matching descriptor"; + return false; + } + + if (descriptor.range_start > m_wallet_descriptor.range_start || + descriptor.range_end < m_wallet_descriptor.range_end) { + // Use inclusive range for error + error = strprintf("new range must include current range = [%d,%d]", + m_wallet_descriptor.range_start, + m_wallet_descriptor.range_end - 1); + return false; + } + + return true; } diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index b2ca354b0a..ef74638751 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -148,17 +148,6 @@ public: } }; -class KeyIDHasher -{ -public: - KeyIDHasher() {} - - size_t operator()(const CKeyID& id) const - { - return id.GetUint64(0); - } -}; - /* * A class implementing ScriptPubKeyMan manages some (or all) scriptPubKeys used in a wallet. * It contains the scripts and keys related to the scriptPubKeys it manages. @@ -174,14 +163,14 @@ protected: public: explicit ScriptPubKeyMan(WalletStorage& storage) : m_storage(storage) {} virtual ~ScriptPubKeyMan() {}; - virtual bool GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) { return false; } + virtual bool GetNewDestination(const OutputType type, CTxDestination& dest, bilingual_str& error) { return false; } virtual isminetype IsMine(const CScript& script) const { return ISMINE_NO; } //! Check that the given decryption key is valid for this ScriptPubKeyMan, i.e. it decrypts all of the keys handled by it. virtual bool CheckDecryptionKey(const CKeyingMaterial& master_key, bool accept_no_keys = false) { return false; } virtual bool Encrypt(const CKeyingMaterial& master_key, WalletBatch* batch) { return false; } - virtual bool GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool, std::string& error) { return false; } + virtual bool GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool, bilingual_str& error) { return false; } virtual void KeepDestination(int64_t index, const OutputType& type) {} virtual void ReturnDestination(int64_t index, bool internal, const CTxDestination& addr) {} @@ -207,7 +196,7 @@ public: virtual bool CanGetAddresses(bool internal = false) const { return false; } /** Upgrades the wallet to the specified version */ - virtual bool Upgrade(int prev_version, int new_version, bilingual_str& error) { return false; } + virtual bool Upgrade(int prev_version, int new_version, bilingual_str& error) { return true; } virtual bool HavePrivateKeys() const { return false; } @@ -216,7 +205,6 @@ public: virtual int64_t GetOldestKeyPoolTime() const { return GetTime(); } - virtual size_t KeypoolCountExternalKeys() const { return 0; } virtual unsigned int GetKeyPoolSize() const { return 0; } virtual int64_t GetTimeFirstKey() const { return 0; } @@ -231,7 +219,7 @@ public: virtual bool CanProvide(const CScript& script, SignatureData& sigdata) { return false; } /** Creates new signatures and adds them to the transaction. Returns whether all inputs were signed */ - virtual bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors) const { return false; } + virtual bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, bilingual_str>& input_errors) const { return false; } /** Sign a message with the given script */ virtual SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const { return SigningResult::SIGNING_FAILED; }; /** Adds script and derivation path information to a PSBT, and optionally signs it. */ @@ -239,8 +227,6 @@ public: virtual uint256 GetID() const { return uint256(); } - virtual void SetInternal(bool internal) {} - /** Prepends the wallet name in logging output to ease debugging in multi-wallet use cases */ template<typename... Params> void WalletLogPrintf(std::string fmt, Params... parameters) const { @@ -358,13 +344,13 @@ private: public: using ScriptPubKeyMan::ScriptPubKeyMan; - bool GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) override; + bool GetNewDestination(const OutputType type, CTxDestination& dest, bilingual_str& error) override; isminetype IsMine(const CScript& script) const override; bool CheckDecryptionKey(const CKeyingMaterial& master_key, bool accept_no_keys = false) override; bool Encrypt(const CKeyingMaterial& master_key, WalletBatch* batch) override; - bool GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool, std::string& error) override; + bool GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool, bilingual_str& error) override; void KeepDestination(int64_t index, const OutputType& type) override; void ReturnDestination(int64_t index, bool internal, const CTxDestination&) override; @@ -386,7 +372,7 @@ public: void RewriteDB() override; int64_t GetOldestKeyPoolTime() const override; - size_t KeypoolCountExternalKeys() const override; + size_t KeypoolCountExternalKeys() const; unsigned int GetKeyPoolSize() const override; int64_t GetTimeFirstKey() const override; @@ -399,14 +385,12 @@ public: bool CanProvide(const CScript& script, SignatureData& sigdata) override; - bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors) const override; + bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, bilingual_str>& input_errors) const override; SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const override; TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr) const override; uint256 GetID() const override; - void SetInternal(bool internal) override; - // Map from Key ID to key metadata. std::map<CKeyID, CKeyMetadata> mapKeyMetadata GUARDED_BY(cs_KeyStore); @@ -533,8 +517,6 @@ private: PubKeyMap m_map_pubkeys GUARDED_BY(cs_desc_man); int32_t m_max_cached_index = -1; - bool m_internal = false; - KeyMap m_map_keys GUARDED_BY(cs_desc_man); CryptedKeyMap m_map_crypted_keys GUARDED_BY(cs_desc_man); @@ -560,20 +542,19 @@ public: : ScriptPubKeyMan(storage), m_wallet_descriptor(descriptor) {} - DescriptorScriptPubKeyMan(WalletStorage& storage, bool internal) - : ScriptPubKeyMan(storage), - m_internal(internal) + DescriptorScriptPubKeyMan(WalletStorage& storage) + : ScriptPubKeyMan(storage) {} mutable RecursiveMutex cs_desc_man; - bool GetNewDestination(const OutputType type, CTxDestination& dest, std::string& error) override; + bool GetNewDestination(const OutputType type, CTxDestination& dest, bilingual_str& error) override; isminetype IsMine(const CScript& script) const override; bool CheckDecryptionKey(const CKeyingMaterial& master_key, bool accept_no_keys = false) override; bool Encrypt(const CKeyingMaterial& master_key, WalletBatch* batch) override; - bool GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool, std::string& error) override; + bool GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool, bilingual_str& error) override; void ReturnDestination(int64_t index, bool internal, const CTxDestination& addr) override; // Tops up the descriptor cache and m_map_script_pub_keys. The cache is stored in the wallet file @@ -587,7 +568,7 @@ public: bool IsHDEnabled() const override; //! Setup descriptors based on the given CExtkey - bool SetupDescriptorGeneration(const CExtKey& master_key, OutputType addr_type); + bool SetupDescriptorGeneration(const CExtKey& master_key, OutputType addr_type, bool internal); /** Provide a descriptor at setup time * Returns false if already setup or setup fails, true if setup is successful @@ -597,7 +578,6 @@ public: bool HavePrivateKeys() const override; int64_t GetOldestKeyPoolTime() const override; - size_t KeypoolCountExternalKeys() const override; unsigned int GetKeyPoolSize() const override; int64_t GetTimeFirstKey() const override; @@ -610,27 +590,29 @@ public: bool CanProvide(const CScript& script, SignatureData& sigdata) override; - bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors) const override; + bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, bilingual_str>& input_errors) const override; SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const override; TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr) const override; uint256 GetID() const override; - void SetInternal(bool internal) override; - void SetCache(const DescriptorCache& cache); bool AddKey(const CKeyID& key_id, const CKey& key); bool AddCryptedKey(const CKeyID& key_id, const CPubKey& pubkey, const std::vector<unsigned char>& crypted_key); bool HasWalletDescriptor(const WalletDescriptor& desc) const; + void UpdateWalletDescriptor(WalletDescriptor& descriptor); + bool CanUpdateToWalletDescriptor(const WalletDescriptor& descriptor, std::string& error); void AddDescriptorKey(const CKey& key, const CPubKey &pubkey); void WriteDescriptor(); const WalletDescriptor GetWalletDescriptor() const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); const std::vector<CScript> GetScriptPubKeys() const; - bool GetDescriptorString(std::string& out, bool priv) const; + bool GetDescriptorString(std::string& out, const bool priv) const; + + void UpgradeDescriptorCache(); }; #endif // BITCOIN_WALLET_SCRIPTPUBKEYMAN_H diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 6a8df437ae..5470177440 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -2,9 +2,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <consensus/amount.h> #include <consensus/validation.h> #include <interfaces/chain.h> #include <policy/policy.h> +#include <script/signingprovider.h> #include <util/check.h> #include <util/fees.h> #include <util/moneystr.h> @@ -21,26 +23,37 @@ using interfaces::FoundBlock; static constexpr size_t OUTPUT_GROUP_MAX_ENTRIES{100}; +int GetTxSpendSize(const CWallet& wallet, const CWalletTx& wtx, unsigned int out, bool use_max_sig) +{ + return CalculateMaximumSignedInputSize(wtx.tx->vout[out], &wallet, use_max_sig); +} + std::string COutput::ToString() const { return strprintf("COutput(%s, %d, %d) [%s]", tx->GetHash().ToString(), i, nDepth, FormatMoney(tx->tx->vout[i].nValue)); } -int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* wallet, bool use_max_sig) +int CalculateMaximumSignedInputSize(const CTxOut& txout, const SigningProvider* provider, bool use_max_sig) { CMutableTransaction txn; txn.vin.push_back(CTxIn(COutPoint())); - if (!wallet->DummySignInput(txn.vin[0], txout, use_max_sig)) { + if (!provider || !DummySignInput(*provider, txn.vin[0], txout, use_max_sig)) { return -1; } return GetVirtualTransactionInputSize(txn.vin[0]); } +int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* wallet, bool use_max_sig) +{ + const std::unique_ptr<SigningProvider> provider = wallet->GetSolvingProvider(txout.scriptPubKey); + return CalculateMaximumSignedInputSize(txout, provider.get(), use_max_sig); +} + // txouts needs to be in the order of tx.vin -TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, const std::vector<CTxOut>& txouts, bool use_max_sig) +TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, const std::vector<CTxOut>& txouts, const CCoinControl* coin_control) { CMutableTransaction txNew(tx); - if (!wallet->DummySignTx(txNew, txouts, use_max_sig)) { + if (!wallet->DummySignTx(txNew, txouts, coin_control)) { return TxSize{-1, -1}; } CTransaction ctx(txNew); @@ -49,48 +62,56 @@ TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *walle return TxSize{vsize, weight}; } -TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, bool use_max_sig) +TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, const CCoinControl* coin_control) { std::vector<CTxOut> txouts; + // Look up the inputs. The inputs are either in the wallet, or in coin_control. for (const CTxIn& input : tx.vin) { const auto mi = wallet->mapWallet.find(input.prevout.hash); // Can not estimate size without knowing the input details - if (mi == wallet->mapWallet.end()) { + if (mi != wallet->mapWallet.end()) { + assert(input.prevout.n < mi->second.tx->vout.size()); + txouts.emplace_back(mi->second.tx->vout.at(input.prevout.n)); + } else if (coin_control) { + CTxOut txout; + if (!coin_control->GetExternalOutput(input.prevout, txout)) { + return TxSize{-1, -1}; + } + txouts.emplace_back(txout); + } else { return TxSize{-1, -1}; } - assert(input.prevout.n < mi->second.tx->vout.size()); - txouts.emplace_back(mi->second.tx->vout[input.prevout.n]); } - return CalculateMaximumSignedTxSize(tx, wallet, txouts, use_max_sig); + return CalculateMaximumSignedTxSize(tx, wallet, txouts, coin_control); } -void CWallet::AvailableCoins(std::vector<COutput>& vCoins, const CCoinControl* coinControl, const CAmount& nMinimumAmount, const CAmount& nMaximumAmount, const CAmount& nMinimumSumAmount, const uint64_t nMaximumCount) const +void AvailableCoins(const CWallet& wallet, std::vector<COutput>& vCoins, const CCoinControl* coinControl, const CAmount& nMinimumAmount, const CAmount& nMaximumAmount, const CAmount& nMinimumSumAmount, const uint64_t nMaximumCount) { - AssertLockHeld(cs_wallet); + AssertLockHeld(wallet.cs_wallet); vCoins.clear(); CAmount nTotal = 0; // Either the WALLET_FLAG_AVOID_REUSE flag is not set (in which case we always allow), or we default to avoiding, and only in the case where // a coin control object is provided, and has the avoid address reuse flag set to false, do we allow already used addresses - bool allow_used_addresses = !IsWalletFlagSet(WALLET_FLAG_AVOID_REUSE) || (coinControl && !coinControl->m_avoid_address_reuse); + bool allow_used_addresses = !wallet.IsWalletFlagSet(WALLET_FLAG_AVOID_REUSE) || (coinControl && !coinControl->m_avoid_address_reuse); const int min_depth = {coinControl ? coinControl->m_min_depth : DEFAULT_MIN_DEPTH}; const int max_depth = {coinControl ? coinControl->m_max_depth : DEFAULT_MAX_DEPTH}; const bool only_safe = {coinControl ? !coinControl->m_include_unsafe_inputs : true}; std::set<uint256> trusted_parents; - for (const auto& entry : mapWallet) + for (const auto& entry : wallet.mapWallet) { const uint256& wtxid = entry.first; const CWalletTx& wtx = entry.second; - if (!chain().checkFinalTx(*wtx.tx)) { + if (!wallet.chain().checkFinalTx(*wtx.tx)) { continue; } - if (wtx.IsImmatureCoinBase()) + if (wallet.IsTxImmatureCoinBase(wtx)) continue; - int nDepth = wtx.GetDepthInMainChain(); + int nDepth = wallet.GetTxDepthInMainChain(wtx); if (nDepth < 0) continue; @@ -99,7 +120,7 @@ void CWallet::AvailableCoins(std::vector<COutput>& vCoins, const CCoinControl* c if (nDepth == 0 && !wtx.InMempool()) continue; - bool safeTx = IsTrusted(wtx, trusted_parents); + bool safeTx = CachedTxIsTrusted(wallet, wtx, trusted_parents); // We should not consider coins from transactions that are replacing // other transactions. @@ -152,28 +173,28 @@ void CWallet::AvailableCoins(std::vector<COutput>& vCoins, const CCoinControl* c if (coinControl && coinControl->HasSelected() && !coinControl->fAllowOtherInputs && !coinControl->IsSelected(COutPoint(entry.first, i))) continue; - if (IsLockedCoin(entry.first, i)) + if (wallet.IsLockedCoin(entry.first, i)) continue; - if (IsSpent(wtxid, i)) + if (wallet.IsSpent(wtxid, i)) continue; - isminetype mine = IsMine(wtx.tx->vout[i]); + isminetype mine = wallet.IsMine(wtx.tx->vout[i]); if (mine == ISMINE_NO) { continue; } - if (!allow_used_addresses && IsSpentKey(wtxid, i)) { + if (!allow_used_addresses && wallet.IsSpentKey(wtxid, i)) { continue; } - std::unique_ptr<SigningProvider> provider = GetSolvingProvider(wtx.tx->vout[i].scriptPubKey); + std::unique_ptr<SigningProvider> provider = wallet.GetSolvingProvider(wtx.tx->vout[i].scriptPubKey); bool solvable = provider ? IsSolvable(*provider, wtx.tx->vout[i].scriptPubKey) : false; bool spendable = ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || (((mine & ISMINE_WATCH_ONLY) != ISMINE_NO) && (coinControl && coinControl->fAllowWatchOnly && solvable)); - vCoins.push_back(COutput(&wtx, i, nDepth, spendable, solvable, safeTx, (coinControl && coinControl->fAllowWatchOnly))); + vCoins.push_back(COutput(wallet, wtx, i, nDepth, spendable, solvable, safeTx, (coinControl && coinControl->fAllowWatchOnly))); // Checks the sum amount of all UTXO's. if (nMinimumSumAmount != MAX_MONEY) { @@ -192,13 +213,13 @@ void CWallet::AvailableCoins(std::vector<COutput>& vCoins, const CCoinControl* c } } -CAmount CWallet::GetAvailableBalance(const CCoinControl* coinControl) const +CAmount GetAvailableBalance(const CWallet& wallet, const CCoinControl* coinControl) { - LOCK(cs_wallet); + LOCK(wallet.cs_wallet); CAmount balance = 0; std::vector<COutput> vCoins; - AvailableCoins(vCoins, coinControl); + AvailableCoins(wallet, vCoins, coinControl); for (const COutput& out : vCoins) { if (out.fSpendable) { balance += out.tx->tx->vout[out.i].nValue; @@ -207,16 +228,16 @@ CAmount CWallet::GetAvailableBalance(const CCoinControl* coinControl) const return balance; } -const CTxOut& CWallet::FindNonChangeParentOutput(const CTransaction& tx, int output) const +const CTxOut& FindNonChangeParentOutput(const CWallet& wallet, const CTransaction& tx, int output) { - AssertLockHeld(cs_wallet); + AssertLockHeld(wallet.cs_wallet); const CTransaction* ptx = &tx; int n = output; - while (IsChange(ptx->vout[n]) && ptx->vin.size() > 0) { + while (OutputIsChange(wallet, ptx->vout[n]) && ptx->vin.size() > 0) { const COutPoint& prevout = ptx->vin[0].prevout; - auto it = mapWallet.find(prevout.hash); - if (it == mapWallet.end() || it->second.tx->vout.size() <= prevout.n || - !IsMine(it->second.tx->vout[prevout.n])) { + auto it = wallet.mapWallet.find(prevout.hash); + if (it == wallet.mapWallet.end() || it->second.tx->vout.size() <= prevout.n || + !wallet.IsMine(it->second.tx->vout[prevout.n])) { break; } ptx = it->second.tx.get(); @@ -225,39 +246,39 @@ const CTxOut& CWallet::FindNonChangeParentOutput(const CTransaction& tx, int out return ptx->vout[n]; } -std::map<CTxDestination, std::vector<COutput>> CWallet::ListCoins() const +std::map<CTxDestination, std::vector<COutput>> ListCoins(const CWallet& wallet) { - AssertLockHeld(cs_wallet); + AssertLockHeld(wallet.cs_wallet); std::map<CTxDestination, std::vector<COutput>> result; std::vector<COutput> availableCoins; - AvailableCoins(availableCoins); + AvailableCoins(wallet, availableCoins); for (const COutput& coin : availableCoins) { CTxDestination address; - if ((coin.fSpendable || (IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && coin.fSolvable)) && - ExtractDestination(FindNonChangeParentOutput(*coin.tx->tx, coin.i).scriptPubKey, address)) { + if ((coin.fSpendable || (wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && coin.fSolvable)) && + ExtractDestination(FindNonChangeParentOutput(wallet, *coin.tx->tx, coin.i).scriptPubKey, address)) { result[address].emplace_back(std::move(coin)); } } std::vector<COutPoint> lockedCoins; - ListLockedCoins(lockedCoins); + wallet.ListLockedCoins(lockedCoins); // Include watch-only for LegacyScriptPubKeyMan wallets without private keys - const bool include_watch_only = GetLegacyScriptPubKeyMan() && IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); + const bool include_watch_only = wallet.GetLegacyScriptPubKeyMan() && wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); const isminetype is_mine_filter = include_watch_only ? ISMINE_WATCH_ONLY : ISMINE_SPENDABLE; for (const COutPoint& output : lockedCoins) { - auto it = mapWallet.find(output.hash); - if (it != mapWallet.end()) { - int depth = it->second.GetDepthInMainChain(); + auto it = wallet.mapWallet.find(output.hash); + if (it != wallet.mapWallet.end()) { + int depth = wallet.GetTxDepthInMainChain(it->second); if (depth >= 0 && output.n < it->second.tx->vout.size() && - IsMine(it->second.tx->vout[output.n]) == is_mine_filter + wallet.IsMine(it->second.tx->vout[output.n]) == is_mine_filter ) { CTxDestination address; - if (ExtractDestination(FindNonChangeParentOutput(*it->second.tx, output.n).scriptPubKey, address)) { + if (ExtractDestination(FindNonChangeParentOutput(wallet, *it->second.tx, output.n).scriptPubKey, address)) { result[address].emplace_back( - &it->second, output.n, depth, true /* spendable */, true /* solvable */, false /* safe */); + wallet, it->second, output.n, depth, true /* spendable */, true /* solvable */, false /* safe */); } } } @@ -266,7 +287,7 @@ std::map<CTxDestination, std::vector<COutput>> CWallet::ListCoins() const return result; } -std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outputs, const CoinSelectionParams& coin_sel_params, const CoinEligibilityFilter& filter, bool positive_only) const +std::vector<OutputGroup> GroupOutputs(const CWallet& wallet, const std::vector<COutput>& outputs, const CoinSelectionParams& coin_sel_params, const CoinEligibilityFilter& filter, bool positive_only) { std::vector<OutputGroup> groups_out; @@ -277,12 +298,12 @@ std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outpu if (!output.fSpendable) continue; size_t ancestors, descendants; - chain().getTransactionAncestry(output.tx->GetHash(), ancestors, descendants); + wallet.chain().getTransactionAncestry(output.tx->GetHash(), ancestors, descendants); CInputCoin input_coin = output.GetInputCoin(); // Make an OutputGroup containing just this output OutputGroup group{coin_sel_params}; - group.Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants, positive_only); + group.Insert(input_coin, output.nDepth, CachedTxIsFromMe(wallet, *output.tx, ISMINE_ALL), ancestors, descendants, positive_only); // Check the OutputGroup's eligibility. Only add the eligible ones. if (positive_only && group.GetSelectionAmount() <= 0) continue; @@ -303,7 +324,7 @@ std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outpu if (!output.fSpendable) continue; size_t ancestors, descendants; - chain().getTransactionAncestry(output.tx->GetHash(), ancestors, descendants); + wallet.chain().getTransactionAncestry(output.tx->GetHash(), ancestors, descendants); CInputCoin input_coin = output.GetInputCoin(); CScript spk = input_coin.txout.scriptPubKey; @@ -327,7 +348,7 @@ std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outpu } // Add the input_coin to group - group->Insert(input_coin, output.nDepth, output.tx->IsFromMe(ISMINE_ALL), ancestors, descendants, positive_only); + group->Insert(input_coin, output.nDepth, CachedTxIsFromMe(wallet, *output.tx, ISMINE_ALL), ancestors, descendants, positive_only); } // Now we go through the entire map and pull out the OutputGroups @@ -352,25 +373,61 @@ std::vector<OutputGroup> CWallet::GroupOutputs(const std::vector<COutput>& outpu return groups_out; } -bool CWallet::AttemptSelection(const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<COutput> coins, - std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CoinSelectionParams& coin_selection_params) const +bool AttemptSelection(const CWallet& wallet, const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<COutput> coins, + std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CoinSelectionParams& coin_selection_params) { setCoinsRet.clear(); nValueRet = 0; + // Vector of results for use with waste calculation + // In order: calculated waste, selected inputs, selected input value (sum of input values) + // TODO: Use a struct representing the selection result + std::vector<std::tuple<CAmount, std::set<CInputCoin>, CAmount>> results; // Note that unlike KnapsackSolver, we do not include the fee for creating a change output as BnB will not create a change output. - std::vector<OutputGroup> positive_groups = GroupOutputs(coins, coin_selection_params, eligibility_filter, true /* positive_only */); - if (SelectCoinsBnB(positive_groups, nTargetValue, coin_selection_params.m_cost_of_change, setCoinsRet, nValueRet)) { - return true; + std::vector<OutputGroup> positive_groups = GroupOutputs(wallet, coins, coin_selection_params, eligibility_filter, true /* positive_only */); + std::set<CInputCoin> bnb_coins; + CAmount bnb_value; + if (SelectCoinsBnB(positive_groups, nTargetValue, coin_selection_params.m_cost_of_change, bnb_coins, bnb_value)) { + const auto waste = GetSelectionWaste(bnb_coins, /* cost of change */ CAmount(0), nTargetValue, !coin_selection_params.m_subtract_fee_outputs); + results.emplace_back(std::make_tuple(waste, std::move(bnb_coins), bnb_value)); } + // The knapsack solver has some legacy behavior where it will spend dust outputs. We retain this behavior, so don't filter for positive only here. - std::vector<OutputGroup> all_groups = GroupOutputs(coins, coin_selection_params, eligibility_filter, false /* positive_only */); + std::vector<OutputGroup> all_groups = GroupOutputs(wallet, coins, coin_selection_params, eligibility_filter, false /* positive_only */); // While nTargetValue includes the transaction fees for non-input things, it does not include the fee for creating a change output. // So we need to include that for KnapsackSolver as well, as we are expecting to create a change output. - return KnapsackSolver(nTargetValue + coin_selection_params.m_change_fee, all_groups, setCoinsRet, nValueRet); + std::set<CInputCoin> knapsack_coins; + CAmount knapsack_value; + if (KnapsackSolver(nTargetValue + coin_selection_params.m_change_fee, all_groups, knapsack_coins, knapsack_value)) { + const auto waste = GetSelectionWaste(knapsack_coins, coin_selection_params.m_cost_of_change, nTargetValue + coin_selection_params.m_change_fee, !coin_selection_params.m_subtract_fee_outputs); + results.emplace_back(std::make_tuple(waste, std::move(knapsack_coins), knapsack_value)); + } + + // We include the minimum final change for SRD as we do want to avoid making really small change. + // KnapsackSolver does not need this because it includes MIN_CHANGE internally. + const CAmount srd_target = nTargetValue + coin_selection_params.m_change_fee + MIN_FINAL_CHANGE; + auto srd_result = SelectCoinsSRD(positive_groups, srd_target); + if (srd_result != std::nullopt) { + const auto waste = GetSelectionWaste(srd_result->first, coin_selection_params.m_cost_of_change, srd_target, !coin_selection_params.m_subtract_fee_outputs); + results.emplace_back(std::make_tuple(waste, std::move(srd_result->first), srd_result->second)); + } + + if (results.size() == 0) { + // No solution found + return false; + } + + // Choose the result with the least waste + // If the waste is the same, choose the one which spends more inputs. + const auto& best_result = std::min_element(results.begin(), results.end(), [](const auto& a, const auto& b) { + return std::get<0>(a) < std::get<0>(b) || (std::get<0>(a) == std::get<0>(b) && std::get<1>(a).size() > std::get<1>(b).size()); + }); + setCoinsRet = std::get<1>(*best_result); + nValueRet = std::get<2>(*best_result); + return true; } -bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CCoinControl& coin_control, CoinSelectionParams& coin_selection_params) const +bool SelectCoins(const CWallet& wallet, const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CCoinControl& coin_control, CoinSelectionParams& coin_selection_params) { std::vector<COutput> vCoins(vAvailableCoins); CAmount value_to_select = nTargetValue; @@ -394,32 +451,40 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm std::vector<COutPoint> vPresetInputs; coin_control.ListSelected(vPresetInputs); - for (const COutPoint& outpoint : vPresetInputs) - { - std::map<uint256, CWalletTx>::const_iterator it = mapWallet.find(outpoint.hash); - if (it != mapWallet.end()) - { + for (const COutPoint& outpoint : vPresetInputs) { + int input_bytes = -1; + CTxOut txout; + std::map<uint256, CWalletTx>::const_iterator it = wallet.mapWallet.find(outpoint.hash); + if (it != wallet.mapWallet.end()) { const CWalletTx& wtx = it->second; // Clearly invalid input, fail if (wtx.tx->vout.size() <= outpoint.n) { return false; } - // Just to calculate the marginal byte size - CInputCoin coin(wtx.tx, outpoint.n, wtx.GetSpendSize(outpoint.n, false)); - nValueFromPresetInputs += coin.txout.nValue; - if (coin.m_input_bytes <= 0) { - return false; // Not solvable, can't estimate size for fee - } - coin.effective_value = coin.txout.nValue - coin_selection_params.m_effective_feerate.GetFee(coin.m_input_bytes); - if (coin_selection_params.m_subtract_fee_outputs) { - value_to_select -= coin.txout.nValue; - } else { - value_to_select -= coin.effective_value; + input_bytes = GetTxSpendSize(wallet, wtx, outpoint.n, false); + txout = wtx.tx->vout.at(outpoint.n); + } + if (input_bytes == -1) { + // The input is external. We either did not find the tx in mapWallet, or we did but couldn't compute the input size with wallet data + if (!coin_control.GetExternalOutput(outpoint, txout)) { + // Not ours, and we don't have solving data. + return false; } - setPresetCoins.insert(coin); + input_bytes = CalculateMaximumSignedInputSize(txout, &coin_control.m_external_provider, /* use_max_sig */ true); + } + + CInputCoin coin(outpoint, txout, input_bytes); + nValueFromPresetInputs += coin.txout.nValue; + if (coin.m_input_bytes == -1) { + return false; // Not solvable, can't estimate size for fee + } + coin.effective_value = coin.txout.nValue - coin_selection_params.m_effective_feerate.GetFee(coin.m_input_bytes); + if (coin_selection_params.m_subtract_fee_outputs) { + value_to_select -= coin.txout.nValue; } else { - return false; // TODO: Allow non-wallet inputs + value_to_select -= coin.effective_value; } + setPresetCoins.insert(coin); } // remove preset inputs from vCoins so that Coin Selection doesn't pick them. @@ -433,7 +498,7 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm unsigned int limit_ancestor_count = 0; unsigned int limit_descendant_count = 0; - chain().getPackageLimits(limit_ancestor_count, limit_descendant_count); + wallet.chain().getPackageLimits(limit_ancestor_count, limit_descendant_count); const size_t max_ancestors = (size_t)std::max<int64_t>(1, limit_ancestor_count); const size_t max_descendants = (size_t)std::max<int64_t>(1, limit_descendant_count); const bool fRejectLongChains = gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS); @@ -456,32 +521,32 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm // If possible, fund the transaction with confirmed UTXOs only. Prefer at least six // confirmations on outputs received from other wallets and only spend confirmed change. - if (AttemptSelection(value_to_select, CoinEligibilityFilter(1, 6, 0), vCoins, setCoinsRet, nValueRet, coin_selection_params)) return true; - if (AttemptSelection(value_to_select, CoinEligibilityFilter(1, 1, 0), vCoins, setCoinsRet, nValueRet, coin_selection_params)) return true; + if (AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(1, 6, 0), vCoins, setCoinsRet, nValueRet, coin_selection_params)) return true; + if (AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(1, 1, 0), vCoins, setCoinsRet, nValueRet, coin_selection_params)) return true; // Fall back to using zero confirmation change (but with as few ancestors in the mempool as // possible) if we cannot fund the transaction otherwise. - if (m_spend_zero_conf_change) { - if (AttemptSelection(value_to_select, CoinEligibilityFilter(0, 1, 2), vCoins, setCoinsRet, nValueRet, coin_selection_params)) return true; - if (AttemptSelection(value_to_select, CoinEligibilityFilter(0, 1, std::min((size_t)4, max_ancestors/3), std::min((size_t)4, max_descendants/3)), + if (wallet.m_spend_zero_conf_change) { + if (AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(0, 1, 2), vCoins, setCoinsRet, nValueRet, coin_selection_params)) return true; + if (AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(0, 1, std::min((size_t)4, max_ancestors/3), std::min((size_t)4, max_descendants/3)), vCoins, setCoinsRet, nValueRet, coin_selection_params)) { return true; } - if (AttemptSelection(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors/2, max_descendants/2), + if (AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(0, 1, max_ancestors/2, max_descendants/2), vCoins, setCoinsRet, nValueRet, coin_selection_params)) { return true; } // If partial groups are allowed, relax the requirement of spending OutputGroups (groups // of UTXOs sent to the same address, which are obviously controlled by a single wallet) // in their entirety. - if (AttemptSelection(value_to_select, CoinEligibilityFilter(0, 1, max_ancestors-1, max_descendants-1, true /* include_partial_groups */), + if (AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(0, 1, max_ancestors-1, max_descendants-1, true /* include_partial_groups */), vCoins, setCoinsRet, nValueRet, coin_selection_params)) { return true; } // Try with unsafe inputs if they are allowed. This may spend unconfirmed outputs // received from other wallets. if (coin_control.m_include_unsafe_inputs - && AttemptSelection(value_to_select, + && AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(0 /* conf_mine */, 0 /* conf_theirs */, max_ancestors-1, max_descendants-1, true /* include_partial_groups */), vCoins, setCoinsRet, nValueRet, coin_selection_params)) { return true; @@ -489,7 +554,7 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm // Try with unlimited ancestors/descendants. The transaction will still need to meet // mempool ancestor/descendant policy to be accepted to mempool and broadcasted, but // OutputGroups use heuristics that may overestimate ancestor/descendant counts. - if (!fRejectLongChains && AttemptSelection(value_to_select, + if (!fRejectLongChains && AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(0, 1, std::numeric_limits<uint64_t>::max(), std::numeric_limits<uint64_t>::max(), true /* include_partial_groups */), vCoins, setCoinsRet, nValueRet, coin_selection_params)) { return true; @@ -568,7 +633,8 @@ static uint32_t GetLocktimeForNewTransaction(interfaces::Chain& chain, const uin return locktime; } -bool CWallet::CreateTransactionInternal( +static bool CreateTransactionInternal( + CWallet& wallet, const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CAmount& nFeeRet, @@ -576,19 +642,22 @@ bool CWallet::CreateTransactionInternal( bilingual_str& error, const CCoinControl& coin_control, FeeCalculation& fee_calc_out, - bool sign) + bool sign) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { - AssertLockHeld(cs_wallet); + AssertLockHeld(wallet.cs_wallet); CMutableTransaction txNew; // The resulting transaction that we make - txNew.nLockTime = GetLocktimeForNewTransaction(chain(), GetLastBlockHash(), GetLastBlockHeight()); + txNew.nLockTime = GetLocktimeForNewTransaction(wallet.chain(), wallet.GetLastBlockHash(), wallet.GetLastBlockHeight()); CoinSelectionParams coin_selection_params; // Parameters for coin selection, init with dummy coin_selection_params.m_avoid_partial_spends = coin_control.m_avoid_partial_spends; + // Set the long term feerate estimate to the wallet's consolidate feerate + coin_selection_params.m_long_term_feerate = wallet.m_consolidate_feerate; + CAmount recipients_sum = 0; - const OutputType change_type = TransactionChangeType(coin_control.m_change_type ? *coin_control.m_change_type : m_default_change_type, vecSend); - ReserveDestination reservedest(this, change_type); + const OutputType change_type = wallet.TransactionChangeType(coin_control.m_change_type ? *coin_control.m_change_type : wallet.m_default_change_type, vecSend); + ReserveDestination reservedest(&wallet, change_type); unsigned int outputs_to_subtract_fee_from = 0; // The number of outputs which we are subtracting the fee from for (const auto& recipient : vecSend) { recipients_sum += recipient.nAmount; @@ -618,9 +687,9 @@ bool CWallet::CreateTransactionInternal( // Reserve a new key pair from key pool. If it fails, provide a dummy // destination in case we don't need change. CTxDestination dest; - std::string dest_err; + bilingual_str dest_err; if (!reservedest.GetReservedDestination(dest, true, dest_err)) { - error = strprintf(_("Transaction needs a change address, but we can't generate it. %s"), dest_err); + error = _("Transaction needs a change address, but we can't generate it.") + Untranslated(" ") + dest_err; } scriptChange = GetScriptForDestination(dest); // A valid destination implies a change script (and @@ -632,7 +701,7 @@ bool CWallet::CreateTransactionInternal( coin_selection_params.change_output_size = GetSerializeSize(change_prototype_txout); // Get size of spending the change output - int change_spend_size = CalculateMaximumSignedInputSize(change_prototype_txout, this); + int change_spend_size = CalculateMaximumSignedInputSize(change_prototype_txout, &wallet); // If the wallet doesn't know how to sign change output, assume p2sh-p2wpkh // as lower-bound to allow BnB to do it's thing if (change_spend_size == -1) { @@ -642,28 +711,23 @@ bool CWallet::CreateTransactionInternal( } // Set discard feerate - coin_selection_params.m_discard_feerate = GetDiscardRate(*this); + coin_selection_params.m_discard_feerate = GetDiscardRate(wallet); // Get the fee rate to use effective values in coin selection FeeCalculation feeCalc; - coin_selection_params.m_effective_feerate = GetMinimumFeeRate(*this, coin_control, &feeCalc); + coin_selection_params.m_effective_feerate = GetMinimumFeeRate(wallet, coin_control, &feeCalc); // Do not, ever, assume that it's fine to change the fee rate if the user has explicitly // provided one if (coin_control.m_feerate && coin_selection_params.m_effective_feerate > *coin_control.m_feerate) { error = strprintf(_("Fee rate (%s) is lower than the minimum fee rate setting (%s)"), coin_control.m_feerate->ToString(FeeEstimateMode::SAT_VB), coin_selection_params.m_effective_feerate.ToString(FeeEstimateMode::SAT_VB)); return false; } - if (feeCalc.reason == FeeReason::FALLBACK && !m_allow_fallback_fee) { + if (feeCalc.reason == FeeReason::FALLBACK && !wallet.m_allow_fallback_fee) { // eventually allow a fallback fee error = _("Fee estimation failed. Fallbackfee is disabled. Wait a few blocks or enable -fallbackfee."); return false; } - // Get long term estimate - CCoinControl cc_temp; - cc_temp.m_confirm_target = chain().estimateMaxBlocks(); - coin_selection_params.m_long_term_feerate = GetMinimumFeeRate(*this, cc_temp, nullptr); - // Calculate the cost of change // Cost of change is the cost of creating the change output + cost of spending the change output in the future. // For creating the change output now, we use the effective feerate. @@ -685,7 +749,7 @@ bool CWallet::CreateTransactionInternal( coin_selection_params.tx_noinputs_size += ::GetSerializeSize(txout, PROTOCOL_VERSION); } - if (IsDust(txout, chain().relayDustFee())) + if (IsDust(txout, wallet.chain().relayDustFee())) { error = _("Transaction amount too small"); return false; @@ -699,12 +763,12 @@ bool CWallet::CreateTransactionInternal( // Get available coins std::vector<COutput> vAvailableCoins; - AvailableCoins(vAvailableCoins, &coin_control, 1, MAX_MONEY, MAX_MONEY, 0); + AvailableCoins(wallet, vAvailableCoins, &coin_control, 1, MAX_MONEY, MAX_MONEY, 0); // Choose coins to use CAmount inputs_sum = 0; std::set<CInputCoin> setCoins; - if (!SelectCoins(vAvailableCoins, /* nTargetValue */ selection_target, setCoins, inputs_sum, coin_control, coin_selection_params)) + if (!SelectCoins(wallet, vAvailableCoins, /* nTargetValue */ selection_target, setCoins, inputs_sum, coin_control, coin_selection_params)) { error = _("Insufficient funds"); return false; @@ -742,16 +806,16 @@ bool CWallet::CreateTransactionInternal( // to avoid conflicting with other possible uses of nSequence, // and in the spirit of "smallest possible change from prior // behavior." - const uint32_t nSequence = coin_control.m_signal_bip125_rbf.value_or(m_signal_rbf) ? MAX_BIP125_RBF_SEQUENCE : (CTxIn::SEQUENCE_FINAL - 1); + const uint32_t nSequence = coin_control.m_signal_bip125_rbf.value_or(wallet.m_signal_rbf) ? MAX_BIP125_RBF_SEQUENCE : (CTxIn::SEQUENCE_FINAL - 1); for (const auto& coin : selected_coins) { txNew.vin.push_back(CTxIn(coin.outpoint, CScript(), nSequence)); } // Calculate the transaction fee - TxSize tx_sizes = CalculateMaximumSignedTxSize(CTransaction(txNew), this, coin_control.fAllowWatchOnly); + TxSize tx_sizes = CalculateMaximumSignedTxSize(CTransaction(txNew), &wallet, &coin_control); int nBytes = tx_sizes.vsize; - if (nBytes < 0) { - error = _("Signing transaction failed"); + if (nBytes == -1) { + error = _("Missing solving data for estimating transaction size"); return false; } nFeeRet = coin_selection_params.m_effective_feerate.GetFee(nBytes); @@ -773,11 +837,15 @@ bool CWallet::CreateTransactionInternal( txNew.vout.erase(change_position); // Because we have dropped this change, the tx size and required fee will be different, so let's recalculate those - tx_sizes = CalculateMaximumSignedTxSize(CTransaction(txNew), this, coin_control.fAllowWatchOnly); + tx_sizes = CalculateMaximumSignedTxSize(CTransaction(txNew), &wallet, &coin_control); nBytes = tx_sizes.vsize; fee_needed = coin_selection_params.m_effective_feerate.GetFee(nBytes); } + // The only time that fee_needed should be less than the amount available for fees (in change_and_fee - change_amount) is when + // we are subtracting the fee from the outputs. If this occurs at any other time, it is a bug. + assert(coin_selection_params.m_subtract_fee_outputs || fee_needed <= change_and_fee - change_amount); + // Update nFeeRet in case fee_needed changed due to dropping the change output if (fee_needed <= change_and_fee - change_amount) { nFeeRet = change_and_fee - change_amount; @@ -806,7 +874,7 @@ bool CWallet::CreateTransactionInternal( } // Error if this output is reduced to be below dust - if (IsDust(txout, chain().relayDustFee())) { + if (IsDust(txout, wallet.chain().relayDustFee())) { if (txout.nValue < 0) { error = _("The transaction amount is too small to pay the fee"); } else { @@ -825,7 +893,7 @@ bool CWallet::CreateTransactionInternal( return false; } - if (sign && !SignTransaction(txNew)) { + if (sign && !wallet.SignTransaction(txNew)) { error = _("Signing transaction failed"); return false; } @@ -841,14 +909,14 @@ bool CWallet::CreateTransactionInternal( return false; } - if (nFeeRet > m_default_max_tx_fee) { + if (nFeeRet > wallet.m_default_max_tx_fee) { error = TransactionErrorString(TransactionError::MAX_FEE_EXCEEDED); return false; } if (gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS)) { // Lastly, ensure this tx will pass the mempool's chain limits - if (!chain().checkChainLimits(tx)) { + if (!wallet.chain().checkChainLimits(tx)) { error = _("Transaction has too long of a mempool chain"); return false; } @@ -859,7 +927,7 @@ bool CWallet::CreateTransactionInternal( reservedest.KeepDestination(); fee_calc_out = feeCalc; - WalletLogPrintf("Fee Calculation: Fee:%d Bytes:%u Tgt:%d (requested %d) Reason:\"%s\" Decay %.5f: Estimation: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out) Fail: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out)\n", + wallet.WalletLogPrintf("Fee Calculation: Fee:%d Bytes:%u Tgt:%d (requested %d) Reason:\"%s\" Decay %.5f: Estimation: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out) Fail: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out)\n", nFeeRet, nBytes, feeCalc.returnedTarget, feeCalc.desiredTarget, StringForFeeReason(feeCalc.reason), feeCalc.est.decay, feeCalc.est.pass.start, feeCalc.est.pass.end, (feeCalc.est.pass.totalConfirmed + feeCalc.est.pass.inMempool + feeCalc.est.pass.leftMempool) > 0.0 ? 100 * feeCalc.est.pass.withinTarget / (feeCalc.est.pass.totalConfirmed + feeCalc.est.pass.inMempool + feeCalc.est.pass.leftMempool) : 0.0, @@ -870,7 +938,8 @@ bool CWallet::CreateTransactionInternal( return true; } -bool CWallet::CreateTransaction( +bool CreateTransaction( + CWallet& wallet, const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CAmount& nFeeRet, @@ -890,23 +959,23 @@ bool CWallet::CreateTransaction( return false; } - LOCK(cs_wallet); + LOCK(wallet.cs_wallet); int nChangePosIn = nChangePosInOut; Assert(!tx); // tx is an out-param. TODO change the return type from bool to tx (or nullptr) - bool res = CreateTransactionInternal(vecSend, tx, nFeeRet, nChangePosInOut, error, coin_control, fee_calc_out, sign); + bool res = CreateTransactionInternal(wallet, vecSend, tx, nFeeRet, nChangePosInOut, error, coin_control, fee_calc_out, sign); // try with avoidpartialspends unless it's enabled already - if (res && nFeeRet > 0 /* 0 means non-functional fee rate estimation */ && m_max_aps_fee > -1 && !coin_control.m_avoid_partial_spends) { + if (res && nFeeRet > 0 /* 0 means non-functional fee rate estimation */ && wallet.m_max_aps_fee > -1 && !coin_control.m_avoid_partial_spends) { CCoinControl tmp_cc = coin_control; tmp_cc.m_avoid_partial_spends = true; CAmount nFeeRet2; CTransactionRef tx2; int nChangePosInOut2 = nChangePosIn; bilingual_str error2; // fired and forgotten; if an error occurs, we discard the results - if (CreateTransactionInternal(vecSend, tx2, nFeeRet2, nChangePosInOut2, error2, tmp_cc, fee_calc_out, sign)) { + if (CreateTransactionInternal(wallet, vecSend, tx2, nFeeRet2, nChangePosInOut2, error2, tmp_cc, fee_calc_out, sign)) { // if fee of this alternative one is within the range of the max fee, we use this one - const bool use_aps = nFeeRet2 <= nFeeRet + m_max_aps_fee; - WalletLogPrintf("Fee non-grouped = %lld, grouped = %lld, using %s\n", nFeeRet, nFeeRet2, use_aps ? "grouped" : "non-grouped"); + const bool use_aps = nFeeRet2 <= nFeeRet + wallet.m_max_aps_fee; + wallet.WalletLogPrintf("Fee non-grouped = %lld, grouped = %lld, using %s\n", nFeeRet, nFeeRet2, use_aps ? "grouped" : "non-grouped"); if (use_aps) { tx = tx2; nFeeRet = nFeeRet2; @@ -917,7 +986,7 @@ bool CWallet::CreateTransaction( return res; } -bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, CCoinControl coinControl) +bool FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, CCoinControl coinControl) { std::vector<CRecipient> vecSend; @@ -936,11 +1005,11 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nC // Acquire the locks to prevent races to the new locked unspents between the // CreateTransaction call and LockCoin calls (when lockUnspents is true). - LOCK(cs_wallet); + LOCK(wallet.cs_wallet); CTransactionRef tx_new; FeeCalculation fee_calc_out; - if (!CreateTransaction(vecSend, tx_new, nFeeRet, nChangePosInOut, error, coinControl, fee_calc_out, false)) { + if (!CreateTransaction(wallet, vecSend, tx_new, nFeeRet, nChangePosInOut, error, coinControl, fee_calc_out, false)) { return false; } @@ -961,7 +1030,7 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nC } if (lockUnspents) { - LockCoin(txin.prevout); + wallet.LockCoin(txin.prevout); } } diff --git a/src/wallet/spend.h b/src/wallet/spend.h index 03f9a7c2b5..7467dd9fa3 100644 --- a/src/wallet/spend.h +++ b/src/wallet/spend.h @@ -5,10 +5,14 @@ #ifndef BITCOIN_WALLET_SPEND_H #define BITCOIN_WALLET_SPEND_H +#include <consensus/amount.h> #include <wallet/coinselection.h> #include <wallet/transaction.h> #include <wallet/wallet.h> +/** Get the marginal bytes if spending the specified output from this transaction */ +int GetTxSpendSize(const CWallet& wallet, const CWalletTx& wtx, unsigned int out, bool use_max_sig = false); + class COutput { public: @@ -43,13 +47,13 @@ public: */ bool fSafe; - COutput(const CWalletTx *txIn, int iIn, int nDepthIn, bool fSpendableIn, bool fSolvableIn, bool fSafeIn, bool use_max_sig_in = false) + COutput(const CWallet& wallet, const CWalletTx& wtx, int iIn, int nDepthIn, bool fSpendableIn, bool fSolvableIn, bool fSafeIn, bool use_max_sig_in = false) { - tx = txIn; i = iIn; nDepth = nDepthIn; fSpendable = fSpendableIn; fSolvable = fSolvableIn; fSafe = fSafeIn; nInputBytes = -1; use_max_sig = use_max_sig_in; + tx = &wtx; i = iIn; nDepth = nDepthIn; fSpendable = fSpendableIn; fSolvable = fSolvableIn; fSafe = fSafeIn; nInputBytes = -1; use_max_sig = use_max_sig_in; // If known and signable by the given wallet, compute nInputBytes // Failure will keep this value -1 - if (fSpendable && tx) { - nInputBytes = tx->GetSpendSize(i, use_max_sig); + if (fSpendable) { + nInputBytes = GetTxSpendSize(wallet, wtx, i, use_max_sig); } } @@ -61,4 +65,77 @@ public: } }; +//Get the marginal bytes of spending the specified output +int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* pwallet, bool use_max_sig = false); +int CalculateMaximumSignedInputSize(const CTxOut& txout, const SigningProvider* pwallet, bool use_max_sig = false); + +struct TxSize { + int64_t vsize{-1}; + int64_t weight{-1}; +}; + +/** Calculate the size of the transaction assuming all signatures are max size +* Use DummySignatureCreator, which inserts 71 byte signatures everywhere. +* NOTE: this requires that all inputs must be in mapWallet (eg the tx should +* be AllInputsMine). */ +TxSize CalculateMaximumSignedTxSize(const CTransaction& tx, const CWallet* wallet, const std::vector<CTxOut>& txouts, const CCoinControl* coin_control = nullptr); +TxSize CalculateMaximumSignedTxSize(const CTransaction& tx, const CWallet* wallet, const CCoinControl* coin_control = nullptr) EXCLUSIVE_LOCKS_REQUIRED(wallet->cs_wallet); + +/** + * populate vCoins with vector of available COutputs. + */ +void AvailableCoins(const CWallet& wallet, std::vector<COutput>& vCoins, const CCoinControl* coinControl = nullptr, const CAmount& nMinimumAmount = 1, const CAmount& nMaximumAmount = MAX_MONEY, const CAmount& nMinimumSumAmount = MAX_MONEY, const uint64_t nMaximumCount = 0) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); + +CAmount GetAvailableBalance(const CWallet& wallet, const CCoinControl* coinControl = nullptr); + +/** + * Find non-change parent output. + */ +const CTxOut& FindNonChangeParentOutput(const CWallet& wallet, const CTransaction& tx, int output) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); + +/** + * Return list of available coins and locked coins grouped by non-change output address. + */ +std::map<CTxDestination, std::vector<COutput>> ListCoins(const CWallet& wallet) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); + +std::vector<OutputGroup> GroupOutputs(const CWallet& wallet, const std::vector<COutput>& outputs, const CoinSelectionParams& coin_sel_params, const CoinEligibilityFilter& filter, bool positive_only); + +/** + * Shuffle and select coins until nTargetValue is reached while avoiding + * small change; This method is stochastic for some inputs and upon + * completion the coin set and corresponding actual target value is + * assembled + * param@[in] coins Set of UTXOs to consider. These will be categorized into + * OutputGroups and filtered using eligibility_filter before + * selecting coins. + * param@[out] setCoinsRet Populated with the coins selected if successful. + * param@[out] nValueRet Used to return the total value of selected coins. + */ +bool AttemptSelection(const CWallet& wallet, const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<COutput> coins, + std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CoinSelectionParams& coin_selection_params); + +/** + * Select a set of coins such that nValueRet >= nTargetValue and at least + * all coins from coin_control are selected; never select unconfirmed coins if they are not ours + * param@[out] setCoinsRet Populated with inputs including pre-selected inputs from + * coin_control and Coin Selection if successful. + * param@[out] nValueRet Total value of selected coins including pre-selected ones + * from coin_control and Coin Selection if successful. + */ +bool SelectCoins(const CWallet& wallet, const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, + const CCoinControl& coin_control, CoinSelectionParams& coin_selection_params) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); + +/** + * Create a new transaction paying the recipients with a set of coins + * selected by SelectCoins(); Also create the change output, when needed + * @note passing nChangePosInOut as -1 will result in setting a random position + */ +bool CreateTransaction(CWallet& wallet, const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, const CCoinControl& coin_control, FeeCalculation& fee_calc_out, bool sign = true); + +/** + * Insert additional inputs into the transaction by + * calling CreateTransaction(); + */ +bool FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, CCoinControl); + #endif // BITCOIN_WALLET_SPEND_H diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index 2e60aca017..815d17967c 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -212,6 +212,10 @@ void SQLiteDatabase::Open() if (ret != SQLITE_OK) { throw std::runtime_error(strprintf("SQLiteDatabase: Failed to open database: %s\n", sqlite3_errstr(ret))); } + ret = sqlite3_extended_result_codes(m_db, 1); + if (ret != SQLITE_OK) { + throw std::runtime_error(strprintf("SQLiteDatabase: Failed to enable extended result codes: %s\n", sqlite3_errstr(ret))); + } } if (sqlite3_db_readonly(m_db, "main") != 0) { diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index c65ebad52f..f80c4637b8 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -2,13 +2,15 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> +#include <consensus/amount.h> #include <node/context.h> #include <primitives/transaction.h> #include <random.h> #include <test/util/setup_common.h> +#include <util/translation.h> #include <wallet/coincontrol.h> #include <wallet/coinselection.h> +#include <wallet/spend.h> #include <wallet/test/wallet_test_fixture.h> #include <wallet/wallet.h> @@ -38,7 +40,7 @@ CoinEligibilityFilter filter_standard_extra(6, 6, 0); CoinSelectionParams coin_selection_params(/* change_output_size= */ 0, /* change_spend_size= */ 0, /* effective_feerate= */ CFeeRate(0), /* long_term_feerate= */ CFeeRate(0), /* discard_feerate= */ CFeeRate(0), - /* tx_no_inputs_size= */ 0, /* avoid_partial= */ false); + /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); static void add_coin(const CAmount& nValue, int nInput, std::vector<CInputCoin>& set) { @@ -48,12 +50,16 @@ static void add_coin(const CAmount& nValue, int nInput, std::vector<CInputCoin>& set.emplace_back(MakeTransactionRef(tx), nInput); } -static void add_coin(const CAmount& nValue, int nInput, CoinSet& set) +static void add_coin(const CAmount& nValue, int nInput, CoinSet& set, CAmount fee = 0, CAmount long_term_fee = 0) { CMutableTransaction tx; tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; - set.emplace(MakeTransactionRef(tx), nInput); + CInputCoin coin(MakeTransactionRef(tx), nInput); + coin.effective_value = nValue - fee; + coin.m_fee = fee; + coin.m_long_term_fee = long_term_fee; + set.insert(coin); } static void add_coin(CWallet& wallet, const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = false, int nInput=0, bool spendable = false) @@ -66,7 +72,7 @@ static void add_coin(CWallet& wallet, const CAmount& nValue, int nAge = 6*24, bo tx.vout[nInput].nValue = nValue; if (spendable) { CTxDestination dest; - std::string error; + bilingual_str error; const bool destination_ok = wallet.GetNewDestination(OutputType::BECH32, "", dest, error); assert(destination_ok); tx.vout[nInput].scriptPubKey = GetScriptForDestination(dest); @@ -82,7 +88,7 @@ static void add_coin(CWallet& wallet, const CAmount& nValue, int nAge = 6*24, bo wtx->m_amounts[CWalletTx::DEBIT].Set(ISMINE_SPENDABLE, 1); wtx->m_is_cache_empty = false; } - COutput output(wtx, nInput, nAge, true /* spendable */, true /* solvable */, true /* safe */); + COutput output(wallet, *wtx, nInput, nAge, true /* spendable */, true /* solvable */, true /* safe */); vCoins.push_back(output); } static void add_coin(const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = false, int nInput=0, bool spendable = false) @@ -136,6 +142,13 @@ inline std::vector<OutputGroup>& GroupCoins(const std::vector<COutput>& coins) return static_groups; } +inline std::vector<OutputGroup>& KnapsackGroupOutputs(const CoinEligibilityFilter& filter) +{ + static std::vector<OutputGroup> static_groups; + static_groups = GroupOutputs(testWallet, vCoins, coin_selection_params, filter, /* positive_only */false); + return static_groups; +} + // Branch and bound coin selection tests BOOST_AUTO_TEST_CASE(bnb_search_test) { @@ -274,20 +287,20 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) CoinSelectionParams coin_selection_params_bnb(/* change_output_size= */ 0, /* change_spend_size= */ 0, /* effective_feerate= */ CFeeRate(3000), /* long_term_feerate= */ CFeeRate(1000), /* discard_feerate= */ CFeeRate(1000), - /* tx_no_inputs_size= */ 0, /* avoid_partial= */ false); + /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); CoinSet setCoinsRet; CAmount nValueRet; empty_wallet(); add_coin(1); vCoins.at(0).nInputBytes = 40; // Make sure that it has a negative effective value. The next check should assert if this somehow got through. Otherwise it will fail - BOOST_CHECK(!testWallet.AttemptSelection( 1 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params_bnb)); + BOOST_CHECK(!SelectCoinsBnB(GroupCoins(vCoins), 1 * CENT, coin_selection_params_bnb.m_cost_of_change, setCoinsRet, nValueRet)); // Test fees subtracted from output: empty_wallet(); add_coin(1 * CENT); vCoins.at(0).nInputBytes = 40; coin_selection_params_bnb.m_subtract_fee_outputs = true; - BOOST_CHECK(testWallet.AttemptSelection( 1 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params_bnb)); + BOOST_CHECK(SelectCoinsBnB(GroupCoins(vCoins), 1 * CENT, coin_selection_params_bnb.m_cost_of_change, setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * CENT); // Make sure that can use BnB when there are preset inputs @@ -304,7 +317,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) coin_control.fAllowOtherInputs = true; coin_control.Select(COutPoint(vCoins.at(0).tx->GetHash(), vCoins.at(0).i)); coin_selection_params_bnb.m_effective_feerate = CFeeRate(0); - BOOST_CHECK(wallet->SelectCoins(vCoins, 10 * CENT, setCoinsRet, nValueRet, coin_control, coin_selection_params_bnb)); + BOOST_CHECK(SelectCoins(*wallet, vCoins, 10 * CENT, setCoinsRet, nValueRet, coin_control, coin_selection_params_bnb)); } } @@ -322,24 +335,24 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) empty_wallet(); // with an empty wallet we can't even pay one cent - BOOST_CHECK(!testWallet.AttemptSelection( 1 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(!KnapsackSolver(1 * CENT, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); add_coin(1*CENT, 4); // add a new 1 cent coin // with a new 1 cent coin, we still can't find a mature 1 cent - BOOST_CHECK(!testWallet.AttemptSelection( 1 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(!KnapsackSolver(1 * CENT, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); // but we can find a new 1 cent - BOOST_CHECK( testWallet.AttemptSelection( 1 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(1 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * CENT); add_coin(2*CENT); // add a mature 2 cent coin // we can't make 3 cents of mature coins - BOOST_CHECK(!testWallet.AttemptSelection( 3 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(!KnapsackSolver(3 * CENT, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); // we can make 3 cents of new coins - BOOST_CHECK( testWallet.AttemptSelection( 3 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(3 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 3 * CENT); add_coin(5*CENT); // add a mature 5 cent coin, @@ -349,33 +362,33 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // now we have new: 1+10=11 (of which 10 was self-sent), and mature: 2+5+20=27. total = 38 // we can't make 38 cents only if we disallow new coins: - BOOST_CHECK(!testWallet.AttemptSelection(38 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(!KnapsackSolver(38 * CENT, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); // we can't even make 37 cents if we don't allow new coins even if they're from us - BOOST_CHECK(!testWallet.AttemptSelection(38 * CENT, filter_standard_extra, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(!KnapsackSolver(38 * CENT, KnapsackGroupOutputs(filter_standard_extra), setCoinsRet, nValueRet)); // but we can make 37 cents if we accept new coins from ourself - BOOST_CHECK( testWallet.AttemptSelection(37 * CENT, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(37 * CENT, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 37 * CENT); // and we can make 38 cents if we accept all new coins - BOOST_CHECK( testWallet.AttemptSelection(38 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(38 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 38 * CENT); // try making 34 cents from 1,2,5,10,20 - we can't do it exactly - BOOST_CHECK( testWallet.AttemptSelection(34 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(34 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 35 * CENT); // but 35 cents is closest BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // the best should be 20+10+5. it's incredibly unlikely the 1 or 2 got included (but possible) // when we try making 7 cents, the smaller coins (1,2,5) are enough. We should see just 2+5 - BOOST_CHECK( testWallet.AttemptSelection( 7 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(7 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 7 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); // when we try making 8 cents, the smaller coins (1,2,5) are exactly enough. - BOOST_CHECK( testWallet.AttemptSelection( 8 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(8 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK(nValueRet == 8 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // when we try making 9 cents, no subset of smaller coins is enough, and we get the next bigger coin (10) - BOOST_CHECK( testWallet.AttemptSelection( 9 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(9 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 10 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); @@ -389,30 +402,30 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(30*CENT); // now we have 6+7+8+20+30 = 71 cents total // check that we have 71 and not 72 - BOOST_CHECK( testWallet.AttemptSelection(71 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); - BOOST_CHECK(!testWallet.AttemptSelection(72 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(71 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(!KnapsackSolver(72 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); // now try making 16 cents. the best smaller coins can do is 6+7+8 = 21; not as good at the next biggest coin, 20 - BOOST_CHECK( testWallet.AttemptSelection(16 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 20 * CENT); // we should get 20 in one coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); add_coin( 5*CENT); // now we have 5+6+7+8+20+30 = 75 cents total // now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, better than the next biggest coin, 20 - BOOST_CHECK( testWallet.AttemptSelection(16 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 18 * CENT); // we should get 18 in 3 coins BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); add_coin( 18*CENT); // now we have 5+6+7+8+18+20+30 // and now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, the same as the next biggest coin, 18 - BOOST_CHECK( testWallet.AttemptSelection(16 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 18 * CENT); // we should get 18 in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); // because in the event of a tie, the biggest coin wins // now try making 11 cents. we should get 5+6 - BOOST_CHECK( testWallet.AttemptSelection(11 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(11 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 11 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); @@ -421,11 +434,11 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin( 2*COIN); add_coin( 3*COIN); add_coin( 4*COIN); // now we have 5+6+7+8+18+20+30+100+200+300+400 = 1094 cents - BOOST_CHECK( testWallet.AttemptSelection(95 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(95 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * COIN); // we should get 1 BTC in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); - BOOST_CHECK( testWallet.AttemptSelection(195 * CENT, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(195 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 2 * COIN); // we should get 2 BTC in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); @@ -440,14 +453,14 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // try making 1 * MIN_CHANGE from the 1.5 * MIN_CHANGE // we'll get change smaller than MIN_CHANGE whatever happens, so can expect MIN_CHANGE exactly - BOOST_CHECK( testWallet.AttemptSelection(MIN_CHANGE, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(MIN_CHANGE, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE); // but if we add a bigger coin, small change is avoided add_coin(1111*MIN_CHANGE); // try making 1 from 0.1 + 0.2 + 0.3 + 0.4 + 0.5 + 1111 = 1112.5 - BOOST_CHECK( testWallet.AttemptSelection(1 * MIN_CHANGE, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * MIN_CHANGE); // we should get the exact amount // if we add more small coins: @@ -455,7 +468,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(MIN_CHANGE * 7 / 10); // and try again to make 1.0 * MIN_CHANGE - BOOST_CHECK( testWallet.AttemptSelection(1 * MIN_CHANGE, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * MIN_CHANGE); // we should get the exact amount // run the 'mtgox' test (see https://blockexplorer.com/tx/29a3efd3ef04f9153d47a990bd7b048a4b2d213daaa5fb8ed670fb85f13bdbcf) @@ -464,7 +477,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) for (int j = 0; j < 20; j++) add_coin(50000 * COIN); - BOOST_CHECK( testWallet.AttemptSelection(500000 * COIN, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(500000 * COIN, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 500000 * COIN); // we should get the exact amount BOOST_CHECK_EQUAL(setCoinsRet.size(), 10U); // in ten coins @@ -477,7 +490,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(MIN_CHANGE * 6 / 10); add_coin(MIN_CHANGE * 7 / 10); add_coin(1111 * MIN_CHANGE); - BOOST_CHECK( testWallet.AttemptSelection(1 * MIN_CHANGE, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1111 * MIN_CHANGE); // we get the bigger coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); @@ -487,7 +500,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(MIN_CHANGE * 6 / 10); add_coin(MIN_CHANGE * 8 / 10); add_coin(1111 * MIN_CHANGE); - BOOST_CHECK( testWallet.AttemptSelection(MIN_CHANGE, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(MIN_CHANGE, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE); // we should get the exact amount BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); // in two coins 0.4+0.6 @@ -498,12 +511,12 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) add_coin(MIN_CHANGE * 100); // trying to make 100.01 from these three coins - BOOST_CHECK(testWallet.AttemptSelection(MIN_CHANGE * 10001 / 100, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(MIN_CHANGE * 10001 / 100, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE * 10105 / 100); // we should get all coins BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // but if we try to make 99.9, we should take the bigger of the two small coins to avoid small change - BOOST_CHECK(testWallet.AttemptSelection(MIN_CHANGE * 9990 / 100, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(MIN_CHANGE * 9990 / 100, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 101 * MIN_CHANGE); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); } @@ -517,7 +530,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // We only create the wallet once to save time, but we still run the coin selection RUN_TESTS times. for (int i = 0; i < RUN_TESTS; i++) { - BOOST_CHECK(testWallet.AttemptSelection(2000, filter_confirmed, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(2000, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); if (amt - 2000 < MIN_CHANGE) { // needs more than one input: @@ -602,7 +615,7 @@ BOOST_AUTO_TEST_CASE(ApproximateBestSubset) add_coin(1000 * COIN); add_coin(3 * COIN); - BOOST_CHECK(testWallet.AttemptSelection(1003 * COIN, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params)); + BOOST_CHECK(KnapsackSolver(1003 * COIN, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1003 * COIN); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); @@ -641,13 +654,95 @@ BOOST_AUTO_TEST_CASE(SelectCoins_test) CoinSelectionParams cs_params(/* change_output_size= */ 34, /* change_spend_size= */ 148, /* effective_feerate= */ CFeeRate(0), /* long_term_feerate= */ CFeeRate(0), /* discard_feerate= */ CFeeRate(0), - /* tx_no_inputs_size= */ 0, /* avoid_partial= */ false); + /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); CoinSet out_set; CAmount out_value = 0; CCoinControl cc; - BOOST_CHECK(testWallet.SelectCoins(vCoins, target, out_set, out_value, cc, cs_params)); + BOOST_CHECK(SelectCoins(testWallet, vCoins, target, out_set, out_value, cc, cs_params)); BOOST_CHECK_GE(out_value, target); } } +BOOST_AUTO_TEST_CASE(waste_test) +{ + CoinSet selection; + const CAmount fee{100}; + const CAmount change_cost{125}; + const CAmount fee_diff{40}; + const CAmount in_amt{3 * COIN}; + const CAmount target{2 * COIN}; + const CAmount excess{in_amt - fee * 2 - target}; + + // Waste with change is the change cost and difference between fee and long term fee + add_coin(1 * COIN, 1, selection, fee, fee - fee_diff); + add_coin(2 * COIN, 2, selection, fee, fee - fee_diff); + const CAmount waste1 = GetSelectionWaste(selection, change_cost, target); + BOOST_CHECK_EQUAL(fee_diff * 2 + change_cost, waste1); + selection.clear(); + + // Waste without change is the excess and difference between fee and long term fee + add_coin(1 * COIN, 1, selection, fee, fee - fee_diff); + add_coin(2 * COIN, 2, selection, fee, fee - fee_diff); + const CAmount waste_nochange1 = GetSelectionWaste(selection, 0, target); + BOOST_CHECK_EQUAL(fee_diff * 2 + excess, waste_nochange1); + selection.clear(); + + // Waste with change and fee == long term fee is just cost of change + add_coin(1 * COIN, 1, selection, fee, fee); + add_coin(2 * COIN, 2, selection, fee, fee); + BOOST_CHECK_EQUAL(change_cost, GetSelectionWaste(selection, change_cost, target)); + selection.clear(); + + // Waste without change and fee == long term fee is just the excess + add_coin(1 * COIN, 1, selection, fee, fee); + add_coin(2 * COIN, 2, selection, fee, fee); + BOOST_CHECK_EQUAL(excess, GetSelectionWaste(selection, 0, target)); + selection.clear(); + + // Waste will be greater when fee is greater, but long term fee is the same + add_coin(1 * COIN, 1, selection, fee * 2, fee - fee_diff); + add_coin(2 * COIN, 2, selection, fee * 2, fee - fee_diff); + const CAmount waste2 = GetSelectionWaste(selection, change_cost, target); + BOOST_CHECK_GT(waste2, waste1); + selection.clear(); + + // Waste with change is the change cost and difference between fee and long term fee + // With long term fee greater than fee, waste should be less than when long term fee is less than fee + add_coin(1 * COIN, 1, selection, fee, fee + fee_diff); + add_coin(2 * COIN, 2, selection, fee, fee + fee_diff); + const CAmount waste3 = GetSelectionWaste(selection, change_cost, target); + BOOST_CHECK_EQUAL(fee_diff * -2 + change_cost, waste3); + BOOST_CHECK_LT(waste3, waste1); + selection.clear(); + + // Waste without change is the excess and difference between fee and long term fee + // With long term fee greater than fee, waste should be less than when long term fee is less than fee + add_coin(1 * COIN, 1, selection, fee, fee + fee_diff); + add_coin(2 * COIN, 2, selection, fee, fee + fee_diff); + const CAmount waste_nochange2 = GetSelectionWaste(selection, 0, target); + BOOST_CHECK_EQUAL(fee_diff * -2 + excess, waste_nochange2); + BOOST_CHECK_LT(waste_nochange2, waste_nochange1); + selection.clear(); + + // No Waste when fee == long_term_fee, no change, and no excess + add_coin(1 * COIN, 1, selection, fee, fee); + add_coin(2 * COIN, 2, selection, fee, fee); + const CAmount exact_target{in_amt - fee * 2}; + BOOST_CHECK_EQUAL(0, GetSelectionWaste(selection, /* change_cost */ 0, exact_target)); + selection.clear(); + + // No Waste when (fee - long_term_fee) == (-cost_of_change), and no excess + const CAmount new_change_cost{fee_diff * 2}; + add_coin(1 * COIN, 1, selection, fee, fee + fee_diff); + add_coin(2 * COIN, 2, selection, fee, fee + fee_diff); + BOOST_CHECK_EQUAL(0, GetSelectionWaste(selection, new_change_cost, target)); + selection.clear(); + + // No Waste when (fee - long_term_fee) == (-excess), no change cost + const CAmount new_target{in_amt - fee * 2 - fee_diff * 2}; + add_coin(1 * COIN, 1, selection, fee, fee + fee_diff); + add_coin(2 * COIN, 2, selection, fee, fee + fee_diff); + BOOST_CHECK_EQUAL(0, GetSelectionWaste(selection, /* change cost */ 0, new_target)); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index 17f5264b45..16cb7e0baf 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -25,7 +25,11 @@ BOOST_AUTO_TEST_CASE(getwalletenv_file) std::string test_name = "test_name.dat"; const fs::path datadir = gArgs.GetDataDirNet(); fs::path file_path = datadir / test_name; +#if BOOST_VERSION >= 107700 + std::ofstream f(BOOST_FILESYSTEM_C_STR(file_path)); +#else std::ofstream f(file_path.BOOST_FILESYSTEM_C_STR); +#endif // BOOST_VERSION >= 107700 f.close(); std::string filename; diff --git a/src/wallet/test/init_test_fixture.cpp b/src/wallet/test/init_test_fixture.cpp index dd9354848d..53c972c46d 100644 --- a/src/wallet/test/init_test_fixture.cpp +++ b/src/wallet/test/init_test_fixture.cpp @@ -32,7 +32,11 @@ InitWalletDirTestingSetup::InitWalletDirTestingSetup(const std::string& chainNam fs::create_directories(m_walletdir_path_cases["default"]); fs::create_directories(m_walletdir_path_cases["custom"]); fs::create_directories(m_walletdir_path_cases["relative"]); +#if BOOST_VERSION >= 107700 + std::ofstream f(BOOST_FILESYSTEM_C_STR(m_walletdir_path_cases["file"])); +#else std::ofstream f(m_walletdir_path_cases["file"].BOOST_FILESYSTEM_C_STR); +#endif // BOOST_VERSION >= 107700 f.close(); } diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 1cefa386b7..8a97f7779d 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -22,12 +22,12 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) CDataStream s_prev_tx1(ParseHex("0200000000010158e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd7501000000171600145f275f436b09a8cc9a2eb2a2f528485c68a56323feffffff02d8231f1b0100000017a914aed962d6654f9a2b36608eb9d64d2b260db4f1118700c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88702483045022100a22edcc6e5bc511af4cc4ae0de0fcd75c7e04d8c1c3a8aa9d820ed4b967384ec02200642963597b9b1bc22c75e9f3e117284a962188bf5e8a74c895089046a20ad770121035509a48eb623e10aace8bfd0212fdb8a8e5af3c94b0b133b95e114cab89e4f7965000000"), SER_NETWORK, PROTOCOL_VERSION); CTransactionRef prev_tx1; s_prev_tx1 >> prev_tx1; - m_wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(prev_tx1->GetHash()), std::forward_as_tuple(&m_wallet, prev_tx1)); + m_wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(prev_tx1->GetHash()), std::forward_as_tuple(prev_tx1)); CDataStream s_prev_tx2(ParseHex("0200000001aad73931018bd25f84ae400b68848be09db706eac2ac18298babee71ab656f8b0000000048473044022058f6fc7c6a33e1b31548d481c826c015bd30135aad42cd67790dab66d2ad243b02204a1ced2604c6735b6393e5b41691dd78b00f0c5942fb9f751856faa938157dba01feffffff0280f0fa020000000017a9140fb9463421696b82c833af241c78c17ddbde493487d0f20a270100000017a91429ca74f8a08f81999428185c97b5d852e4063f618765000000"), SER_NETWORK, PROTOCOL_VERSION); CTransactionRef prev_tx2; s_prev_tx2 >> prev_tx2; - m_wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(prev_tx2->GetHash()), std::forward_as_tuple(&m_wallet, prev_tx2)); + m_wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(prev_tx2->GetHash()), std::forward_as_tuple(prev_tx2)); // Add scripts CScript rs1; diff --git a/src/wallet/test/spend_tests.cpp b/src/wallet/test/spend_tests.cpp new file mode 100644 index 0000000000..becef70729 --- /dev/null +++ b/src/wallet/test/spend_tests.cpp @@ -0,0 +1,63 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <consensus/amount.h> +#include <policy/fees.h> +#include <validation.h> +#include <wallet/coincontrol.h> +#include <wallet/spend.h> +#include <wallet/test/util.h> +#include <wallet/test/wallet_test_fixture.h> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(spend_tests, WalletTestingSetup) + +BOOST_FIXTURE_TEST_CASE(SubtractFee, TestChain100Setup) +{ + CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); + auto wallet = CreateSyncedWallet(*m_node.chain, m_node.chainman->ActiveChain(), coinbaseKey); + + // Check that a subtract-from-recipient transaction slightly less than the + // coinbase input amount does not create a change output (because it would + // be uneconomical to add and spend the output), and make sure it pays the + // leftover input amount which would have been change to the recipient + // instead of the miner. + auto check_tx = [&wallet](CAmount leftover_input_amount) { + CRecipient recipient{GetScriptForRawPubKey({}), 50 * COIN - leftover_input_amount, true /* subtract fee */}; + CTransactionRef tx; + CAmount fee; + int change_pos = -1; + bilingual_str error; + CCoinControl coin_control; + coin_control.m_feerate.emplace(10000); + coin_control.fOverrideFeeRate = true; + FeeCalculation fee_calc; + BOOST_CHECK(CreateTransaction(*wallet, {recipient}, tx, fee, change_pos, error, coin_control, fee_calc)); + BOOST_CHECK_EQUAL(tx->vout.size(), 1); + BOOST_CHECK_EQUAL(tx->vout[0].nValue, recipient.nAmount + leftover_input_amount - fee); + BOOST_CHECK_GT(fee, 0); + return fee; + }; + + // Send full input amount to recipient, check that only nonzero fee is + // subtracted (to_reduce == fee). + const CAmount fee{check_tx(0)}; + + // Send slightly less than full input amount to recipient, check leftover + // input amount is paid to recipient not the miner (to_reduce == fee - 123) + BOOST_CHECK_EQUAL(fee, check_tx(123)); + + // Send full input minus fee amount to recipient, check leftover input + // amount is paid to recipient not the miner (to_reduce == 0) + BOOST_CHECK_EQUAL(fee, check_tx(fee)); + + // Send full input minus more than the fee amount to recipient, check + // leftover input amount is paid to recipient not the miner (to_reduce == + // -123). This overpays the recipient instead of overpaying the miner more + // than double the necessary fee. + BOOST_CHECK_EQUAL(fee, check_tx(fee + 123)); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/test/util.cpp b/src/wallet/test/util.cpp new file mode 100644 index 0000000000..c3061b93c0 --- /dev/null +++ b/src/wallet/test/util.cpp @@ -0,0 +1,38 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <wallet/test/util.h> + +#include <chain.h> +#include <key.h> +#include <test/util/setup_common.h> +#include <wallet/wallet.h> +#include <wallet/walletdb.h> + +#include <boost/test/unit_test.hpp> + +#include <memory> + +std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cchain, const CKey& key) +{ + auto wallet = std::make_unique<CWallet>(&chain, "", CreateMockWalletDatabase()); + { + LOCK2(wallet->cs_wallet, ::cs_main); + wallet->SetLastBlockProcessed(cchain.Height(), cchain.Tip()->GetBlockHash()); + } + wallet->LoadWallet(); + { + auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); + LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); + spk_man->AddKeyPubKey(key, key.GetPubKey()); + } + WalletRescanReserver reserver(*wallet); + reserver.reserve(); + CWallet::ScanResult result = wallet->ScanForWalletTransactions(cchain.Genesis()->GetBlockHash(), 0 /* start_height */, {} /* max_height */, reserver, false /* update */); + BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); + BOOST_CHECK_EQUAL(result.last_scanned_block, cchain.Tip()->GetBlockHash()); + BOOST_CHECK_EQUAL(*result.last_scanned_height, cchain.Height()); + BOOST_CHECK(result.last_failed_block.IsNull()); + return wallet; +} diff --git a/src/wallet/test/util.h b/src/wallet/test/util.h new file mode 100644 index 0000000000..288c111571 --- /dev/null +++ b/src/wallet/test/util.h @@ -0,0 +1,19 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_TEST_UTIL_H +#define BITCOIN_WALLET_TEST_UTIL_H + +#include <memory> + +class CChain; +class CKey; +class CWallet; +namespace interfaces { +class Chain; +} // namespace interfaces + +std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cchain, const CKey& key); + +#endif // BITCOIN_WALLET_TEST_UTIL_H diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index a0070b8dd3..9938380369 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -20,6 +20,10 @@ #include <util/translation.h> #include <validation.h> #include <wallet/coincontrol.h> +#include <wallet/context.h> +#include <wallet/receive.h> +#include <wallet/spend.h> +#include <wallet/test/util.h> #include <wallet/test/wallet_test_fixture.h> #include <boost/test/unit_test.hpp> @@ -29,8 +33,6 @@ RPCHelpMan importmulti(); RPCHelpMan dumpwallet(); RPCHelpMan importwallet(); -extern RecursiveMutex cs_wallets; - // Ensure that fee levels defined in the wallet are at least as high // as the default levels for node policy. static_assert(DEFAULT_TRANSACTION_MINFEE >= DEFAULT_MIN_RELAY_TX_FEE, "wallet minimum fee is smaller than default relay fee"); @@ -38,15 +40,15 @@ static_assert(WALLET_INCREMENTAL_RELAY_FEE >= DEFAULT_INCREMENTAL_RELAY_FEE, "wa BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) -static std::shared_ptr<CWallet> TestLoadWallet(interfaces::Chain* chain) +static std::shared_ptr<CWallet> TestLoadWallet(WalletContext& context) { DatabaseOptions options; DatabaseStatus status; bilingual_str error; std::vector<bilingual_str> warnings; auto database = MakeWalletDatabase("", options, status, error); - auto wallet = CWallet::Create(chain, "", std::move(database), options.create_flags, error, warnings); - if (chain) { + auto wallet = CWallet::Create(context, "", std::move(database), options.create_flags, error, warnings); + if (context.chain) { wallet->postInitProcess(); } return wallet; @@ -68,7 +70,7 @@ static CMutableTransaction TestSimpleSpend(const CTransaction& from, uint32_t in keystore.AddKey(key); std::map<COutPoint, Coin> coins; coins[mtx.vin[0].prevout].out = from.vout[index]; - std::map<int, std::string> input_errors; + std::map<int, bilingual_str> input_errors; BOOST_CHECK(SignTransaction(mtx, &keystore, coins, SIGHASH_ALL, input_errors)); return mtx; } @@ -103,7 +105,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) BOOST_CHECK(result.last_failed_block.IsNull()); BOOST_CHECK(result.last_scanned_block.IsNull()); BOOST_CHECK(!result.last_scanned_height); - BOOST_CHECK_EQUAL(wallet.GetBalance().m_mine_immature, 0); + BOOST_CHECK_EQUAL(GetBalance(wallet).m_mine_immature, 0); } // Verify ScanForWalletTransactions picks up transactions in both the old @@ -122,7 +124,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) BOOST_CHECK(result.last_failed_block.IsNull()); BOOST_CHECK_EQUAL(result.last_scanned_block, newTip->GetBlockHash()); BOOST_CHECK_EQUAL(*result.last_scanned_height, newTip->nHeight); - BOOST_CHECK_EQUAL(wallet.GetBalance().m_mine_immature, 100 * COIN); + BOOST_CHECK_EQUAL(GetBalance(wallet).m_mine_immature, 100 * COIN); } // Prune the older block file. @@ -148,7 +150,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) BOOST_CHECK_EQUAL(result.last_failed_block, oldTip->GetBlockHash()); BOOST_CHECK_EQUAL(result.last_scanned_block, newTip->GetBlockHash()); BOOST_CHECK_EQUAL(*result.last_scanned_height, newTip->nHeight); - BOOST_CHECK_EQUAL(wallet.GetBalance().m_mine_immature, 50 * COIN); + BOOST_CHECK_EQUAL(GetBalance(wallet).m_mine_immature, 50 * COIN); } // Prune the remaining block file. @@ -173,7 +175,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) BOOST_CHECK_EQUAL(result.last_failed_block, newTip->GetBlockHash()); BOOST_CHECK(result.last_scanned_block.IsNull()); BOOST_CHECK(!result.last_scanned_height); - BOOST_CHECK_EQUAL(wallet.GetBalance().m_mine_immature, 0); + BOOST_CHECK_EQUAL(GetBalance(wallet).m_mine_immature, 0); } } @@ -199,7 +201,9 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); wallet->SetupLegacyScriptPubKeyMan(); WITH_LOCK(wallet->cs_wallet, wallet->SetLastBlockProcessed(newTip->nHeight, newTip->GetBlockHash())); - AddWallet(wallet); + WalletContext context; + context.args = &gArgs; + AddWallet(context, wallet); UniValue keys; keys.setArray(); UniValue key; @@ -217,6 +221,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) key.pushKV("internal", UniValue(true)); keys.push_back(key); JSONRPCRequest request; + request.context = &context; request.params.setArray(); request.params.push_back(keys); @@ -227,10 +232,10 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) "seconds of key creation, and could contain transactions pertaining to the key. As a result, " "transactions and coins using this key may not appear in the wallet. This error could be caused " "by pruning or data corruption (see bitcoind log for details) and could be dealt with by " - "downloading and rescanning the relevant blocks (see -reindex and -rescan " - "options).\"}},{\"success\":true}]", + "downloading and rescanning the relevant blocks (see -reindex option and rescanblockchain " + "RPC).\"}},{\"success\":true}]", 0, oldTip->GetBlockTimeMax(), TIMESTAMP_WINDOW)); - RemoveWallet(wallet, std::nullopt); + RemoveWallet(context, wallet, /* load_on_start= */ std::nullopt); } } @@ -257,6 +262,8 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // Import key into wallet and call dumpwallet to create backup file. { + WalletContext context; + context.args = &gArgs; std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); { auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); @@ -264,15 +271,16 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) spk_man->mapKeyMetadata[coinbaseKey.GetPubKey().GetID()].nCreateTime = KEY_TIME; spk_man->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey()); - AddWallet(wallet); + AddWallet(context, wallet); wallet->SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); } JSONRPCRequest request; + request.context = &context; request.params.setArray(); request.params.push_back(backup_file); ::dumpwallet().HandleRequest(request); - RemoveWallet(wallet, std::nullopt); + RemoveWallet(context, wallet, /* load_on_start= */ std::nullopt); } // Call importwallet RPC and verify all blocks with timestamps >= BLOCK_TIME @@ -282,13 +290,16 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) LOCK(wallet->cs_wallet); wallet->SetupLegacyScriptPubKeyMan(); + WalletContext context; + context.args = &gArgs; JSONRPCRequest request; + request.context = &context; request.params.setArray(); request.params.push_back(backup_file); - AddWallet(wallet); + AddWallet(context, wallet); wallet->SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); ::importwallet().HandleRequest(request); - RemoveWallet(wallet, std::nullopt); + RemoveWallet(context, wallet, /* load_on_start= */ std::nullopt); BOOST_CHECK_EQUAL(wallet->mapWallet.size(), 3U); BOOST_CHECK_EQUAL(m_coinbase_txns.size(), 103U); @@ -310,7 +321,7 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) { CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); - CWalletTx wtx(&wallet, m_coinbase_txns.back()); + CWalletTx wtx(m_coinbase_txns.back()); LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); wallet.SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); @@ -320,13 +331,13 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) // Call GetImmatureCredit() once before adding the key to the wallet to // cache the current immature credit amount, which is 0. - BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(), 0); + BOOST_CHECK_EQUAL(CachedTxGetImmatureCredit(wallet, wtx), 0); // Invalidate the cached value, add the key, and make sure a new immature // credit amount is calculated. wtx.MarkDirty(); BOOST_CHECK(spk_man->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey())); - BOOST_CHECK_EQUAL(wtx.GetImmatureCredit(), 50*COIN); + BOOST_CHECK_EQUAL(CachedTxGetImmatureCredit(wallet, wtx), 50*COIN); } static int64_t AddTx(ChainstateManager& chainman, CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64_t blockTime) @@ -480,20 +491,7 @@ public: ListCoinsTestingSetup() { CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); - wallet = std::make_unique<CWallet>(m_node.chain.get(), "", CreateMockWalletDatabase()); - { - LOCK2(wallet->cs_wallet, ::cs_main); - wallet->SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); - } - wallet->LoadWallet(); - AddKey(*wallet, coinbaseKey); - WalletRescanReserver reserver(*wallet); - reserver.reserve(); - CWallet::ScanResult result = wallet->ScanForWalletTransactions(m_node.chainman->ActiveChain().Genesis()->GetBlockHash(), 0 /* start_height */, {} /* max_height */, reserver, false /* update */); - BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); - BOOST_CHECK_EQUAL(result.last_scanned_block, m_node.chainman->ActiveChain().Tip()->GetBlockHash()); - BOOST_CHECK_EQUAL(*result.last_scanned_height, m_node.chainman->ActiveChain().Height()); - BOOST_CHECK(result.last_failed_block.IsNull()); + wallet = CreateSyncedWallet(*m_node.chain, m_node.chainman->ActiveChain(), coinbaseKey); } ~ListCoinsTestingSetup() @@ -510,7 +508,7 @@ public: CCoinControl dummy; FeeCalculation fee_calc_out; { - BOOST_CHECK(wallet->CreateTransaction({recipient}, tx, fee, changePos, error, dummy, fee_calc_out)); + BOOST_CHECK(CreateTransaction(*wallet, {recipient}, tx, fee, changePos, error, dummy, fee_calc_out)); } wallet->CommitTransaction(tx, {}, {}); CMutableTransaction blocktx; @@ -532,7 +530,7 @@ public: std::unique_ptr<CWallet> wallet; }; -BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) +BOOST_FIXTURE_TEST_CASE(ListCoinsTest, ListCoinsTestingSetup) { std::string coinbaseAddress = coinbaseKey.GetPubKey().GetID().ToString(); @@ -541,14 +539,14 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) std::map<CTxDestination, std::vector<COutput>> list; { LOCK(wallet->cs_wallet); - list = wallet->ListCoins(); + list = ListCoins(*wallet); } BOOST_CHECK_EQUAL(list.size(), 1U); BOOST_CHECK_EQUAL(std::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); BOOST_CHECK_EQUAL(list.begin()->second.size(), 1U); // Check initial balance from one mature coinbase transaction. - BOOST_CHECK_EQUAL(50 * COIN, wallet->GetAvailableBalance()); + BOOST_CHECK_EQUAL(50 * COIN, GetAvailableBalance(*wallet)); // Add a transaction creating a change address, and confirm ListCoins still // returns the coin associated with the change address underneath the @@ -557,7 +555,7 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) AddTx(CRecipient{GetScriptForRawPubKey({}), 1 * COIN, false /* subtract fee */}); { LOCK(wallet->cs_wallet); - list = wallet->ListCoins(); + list = ListCoins(*wallet); } BOOST_CHECK_EQUAL(list.size(), 1U); BOOST_CHECK_EQUAL(std::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); @@ -567,7 +565,7 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) { LOCK(wallet->cs_wallet); std::vector<COutput> available; - wallet->AvailableCoins(available); + AvailableCoins(*wallet, available); BOOST_CHECK_EQUAL(available.size(), 2U); } for (const auto& group : list) { @@ -579,14 +577,14 @@ BOOST_FIXTURE_TEST_CASE(ListCoins, ListCoinsTestingSetup) { LOCK(wallet->cs_wallet); std::vector<COutput> available; - wallet->AvailableCoins(available); + AvailableCoins(*wallet, available); BOOST_CHECK_EQUAL(available.size(), 0U); } // Confirm ListCoins still returns same result as before, despite coins // being locked. { LOCK(wallet->cs_wallet); - list = wallet->ListCoins(); + list = ListCoins(*wallet); } BOOST_CHECK_EQUAL(list.size(), 1U); BOOST_CHECK_EQUAL(std::get<PKHash>(list.begin()->first).ToString(), coinbaseAddress); @@ -601,7 +599,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); BOOST_CHECK(!wallet->TopUpKeyPool(1000)); CTxDestination dest; - std::string error; + bilingual_str error; BOOST_CHECK(!wallet->GetNewDestination(OutputType::BECH32, "", dest, error)); } @@ -691,7 +689,10 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) { gArgs.ForceSetArg("-unsafesqlitesync", "1"); // Create new wallet with known key and unload it. - auto wallet = TestLoadWallet(m_node.chain.get()); + WalletContext context; + context.args = &gArgs; + context.chain = m_node.chain.get(); + auto wallet = TestLoadWallet(context); CKey key; key.MakeNewKey(true); AddKey(*wallet, key); @@ -731,7 +732,7 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) // Reload wallet and make sure new transactions are detected despite events // being blocked - wallet = TestLoadWallet(m_node.chain.get()); + wallet = TestLoadWallet(context); BOOST_CHECK(rescan_completed); BOOST_CHECK_EQUAL(addtx_count, 2); { @@ -758,20 +759,20 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) // deadlock during the sync and simulates a new block notification happening // as soon as possible. addtx_count = 0; - auto handler = HandleLoadWallet([&](std::unique_ptr<interfaces::Wallet> wallet) EXCLUSIVE_LOCKS_REQUIRED(wallet->wallet()->cs_wallet, cs_wallets) { + auto handler = HandleLoadWallet(context, [&](std::unique_ptr<interfaces::Wallet> wallet) EXCLUSIVE_LOCKS_REQUIRED(wallet->wallet()->cs_wallet, context.wallets_mutex) { BOOST_CHECK(rescan_completed); m_coinbase_txns.push_back(CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); block_tx = TestSimpleSpend(*m_coinbase_txns[2], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); m_coinbase_txns.push_back(CreateAndProcessBlock({block_tx}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); mempool_tx = TestSimpleSpend(*m_coinbase_txns[3], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); BOOST_CHECK(m_node.chain->broadcastTransaction(MakeTransactionRef(mempool_tx), DEFAULT_TRANSACTION_MAXFEE, false, error)); - LEAVE_CRITICAL_SECTION(cs_wallets); + LEAVE_CRITICAL_SECTION(context.wallets_mutex); LEAVE_CRITICAL_SECTION(wallet->wallet()->cs_wallet); SyncWithValidationInterfaceQueue(); ENTER_CRITICAL_SECTION(wallet->wallet()->cs_wallet); - ENTER_CRITICAL_SECTION(cs_wallets); + ENTER_CRITICAL_SECTION(context.wallets_mutex); }); - wallet = TestLoadWallet(m_node.chain.get()); + wallet = TestLoadWallet(context); BOOST_CHECK_EQUAL(addtx_count, 4); { LOCK(wallet->cs_wallet); @@ -785,7 +786,9 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) BOOST_FIXTURE_TEST_CASE(CreateWalletWithoutChain, BasicTestingSetup) { - auto wallet = TestLoadWallet(nullptr); + WalletContext context; + context.args = &gArgs; + auto wallet = TestLoadWallet(context); BOOST_CHECK(wallet); UnloadWallet(std::move(wallet)); } @@ -793,7 +796,10 @@ BOOST_FIXTURE_TEST_CASE(CreateWalletWithoutChain, BasicTestingSetup) BOOST_FIXTURE_TEST_CASE(ZapSelectTx, TestChain100Setup) { gArgs.ForceSetArg("-unsafesqlitesync", "1"); - auto wallet = TestLoadWallet(m_node.chain.get()); + WalletContext context; + context.args = &gArgs; + context.chain = m_node.chain.get(); + auto wallet = TestLoadWallet(context); CKey key; key.MakeNewKey(true); AddKey(*wallet, key); diff --git a/src/wallet/transaction.h b/src/wallet/transaction.h index 131faefe0b..6fc1bd1eed 100644 --- a/src/wallet/transaction.h +++ b/src/wallet/transaction.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_WALLET_TRANSACTION_H #define BITCOIN_WALLET_TRANSACTION_H -#include <amount.h> +#include <consensus/amount.h> #include <primitives/transaction.h> #include <serialize.h> #include <wallet/ismine.h> @@ -17,30 +17,8 @@ #include <list> #include <vector> -struct COutputEntry; - typedef std::map<std::string, std::string> mapValue_t; -//Get the marginal bytes of spending the specified output -int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* pwallet, bool use_max_sig = false); - -static inline void ReadOrderPos(int64_t& nOrderPos, mapValue_t& mapValue) -{ - if (!mapValue.count("n")) - { - nOrderPos = -1; // TODO: calculate elsewhere - return; - } - nOrderPos = atoi64(mapValue["n"]); -} - -static inline void WriteOrderPos(const int64_t& nOrderPos, mapValue_t& mapValue) -{ - if (nOrderPos == -1) - return; - mapValue["n"] = ToString(nOrderPos); -} - /** Legacy class used for deserializing vtxPrev for backwards compatibility. * vtxPrev was removed in commit 93a18a3650292afbb441a47d1fa1b94aeb0164e3, * but old wallet.dat files may still contain vtxPrev vectors of CMerkleTxs. @@ -68,8 +46,6 @@ public: class CWalletTx { private: - const CWallet* const pwallet; - /** Constant used in hashBlock to indicate tx has been abandoned, only used at * serialization/deserialization to avoid ambiguity with conflicted. */ @@ -126,7 +102,6 @@ public: // memory only enum AmountType { DEBIT, CREDIT, IMMATURE_CREDIT, AVAILABLE_CREDIT, AMOUNTTYPE_ENUM_ELEMENTS }; - CAmount GetCachableAmount(AmountType type, const isminefilter& filter, bool recalculate = false) const; mutable CachableAmount m_amounts[AMOUNTTYPE_ENUM_ELEMENTS]; /** * This flag is true if all m_amounts caches are empty. This is particularly @@ -139,9 +114,8 @@ public: mutable bool fInMempool; mutable CAmount nChangeCached; - CWalletTx(const CWallet* wallet, CTransactionRef arg) - : pwallet(wallet), - tx(std::move(arg)) + CWalletTx(CTransactionRef arg) + : tx(std::move(arg)) { Init(); } @@ -199,7 +173,9 @@ public: mapValue_t mapValueCopy = mapValue; mapValueCopy["fromaccount"] = ""; - WriteOrderPos(nOrderPos, mapValueCopy); + if (nOrderPos != -1) { + mapValueCopy["n"] = ToString(nOrderPos); + } if (nTimeSmart) { mapValueCopy["timesmart"] = strprintf("%u", nTimeSmart); } @@ -239,8 +215,10 @@ public: setConfirmed(); } - ReadOrderPos(nOrderPos, mapValue); - nTimeSmart = mapValue.count("timesmart") ? (unsigned int)atoi64(mapValue["timesmart"]) : 0; + const auto it_op = mapValue.find("n"); + nOrderPos = (it_op != mapValue.end()) ? LocaleIndependentAtoi<int64_t>(it_op->second) : -1; + const auto it_ts = mapValue.find("timesmart"); + nTimeSmart = (it_ts != mapValue.end()) ? static_cast<unsigned int>(LocaleIndependentAtoi<int64_t>(it_ts->second)) : 0; mapValue.erase("fromaccount"); mapValue.erase("spent"); @@ -264,72 +242,13 @@ public: m_is_cache_empty = true; } - //! filter decides which addresses will count towards the debit - CAmount GetDebit(const isminefilter& filter) const; - CAmount GetCredit(const isminefilter& filter) const; - CAmount GetImmatureCredit(bool fUseCache = true) const; - // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct - // annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The - // annotation "NO_THREAD_SAFETY_ANALYSIS" was temporarily added to avoid - // having to resolve the issue of member access into incomplete type CWallet. - CAmount GetAvailableCredit(bool fUseCache = true, const isminefilter& filter = ISMINE_SPENDABLE) const NO_THREAD_SAFETY_ANALYSIS; - CAmount GetImmatureWatchOnlyCredit(const bool fUseCache = true) const; - CAmount GetChange() const; - - /** Get the marginal bytes if spending the specified output from this transaction */ - int GetSpendSize(unsigned int out, bool use_max_sig = false) const - { - return CalculateMaximumSignedInputSize(tx->vout[out], pwallet, use_max_sig); - } - - void GetAmounts(std::list<COutputEntry>& listReceived, - std::list<COutputEntry>& listSent, CAmount& nFee, const isminefilter& filter) const; - - bool IsFromMe(const isminefilter& filter) const - { - return (GetDebit(filter) > 0); - } - /** True if only scriptSigs are different */ bool IsEquivalentTo(const CWalletTx& tx) const; bool InMempool() const; - bool IsTrusted() const; int64_t GetTxTime() const; - /** Pass this transaction to node for mempool insertion and relay to peers if flag set to true */ - bool SubmitMemoryPoolAndRelay(std::string& err_string, bool relay); - - // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct - // annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The annotation - // "NO_THREAD_SAFETY_ANALYSIS" was temporarily added to avoid having to - // resolve the issue of member access into incomplete type CWallet. Note - // that we still have the runtime check "AssertLockHeld(pwallet->cs_wallet)" - // in place. - std::set<uint256> GetConflicts() const NO_THREAD_SAFETY_ANALYSIS; - - /** - * Return depth of transaction in blockchain: - * <0 : conflicts with a transaction this deep in the blockchain - * 0 : in memory pool, waiting to be included in a block - * >=1 : this many blocks deep in the main chain - */ - // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct - // annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The annotation - // "NO_THREAD_SAFETY_ANALYSIS" was temporarily added to avoid having to - // resolve the issue of member access into incomplete type CWallet. Note - // that we still have the runtime check "AssertLockHeld(pwallet->cs_wallet)" - // in place. - int GetDepthInMainChain() const NO_THREAD_SAFETY_ANALYSIS; - bool IsInMainChain() const { return GetDepthInMainChain() > 0; } - - /** - * @return number of blocks to maturity for this transaction: - * 0 : is not a coinbase transaction, or is a mature coinbase transaction - * >0 : is a coinbase transaction which matures in this many blocks - */ - int GetBlocksToMaturity() const; bool isAbandoned() const { return m_confirm.status == CWalletTx::ABANDONED; } void setAbandoned() { @@ -346,7 +265,6 @@ public: void setConfirmed() { m_confirm.status = CWalletTx::CONFIRMED; } const uint256& GetHash() const { return tx->GetHash(); } bool IsCoinBase() const { return tx->IsCoinBase(); } - bool IsImmatureCoinBase() const; // Disable copying of CWalletTx objects to prevent bugs where instances get // copied in and out of the mapWallet map, and fields are updated in the diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index c2586b60b8..abfe9d7dba 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -6,6 +6,7 @@ #include <wallet/wallet.h> #include <chain.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/validation.h> #include <external_signer.h> @@ -33,6 +34,7 @@ #include <util/string.h> #include <util/translation.h> #include <wallet/coincontrol.h> +#include <wallet/context.h> #include <wallet/fees.h> #include <wallet/external_signer_scriptpubkeyman.h> @@ -54,10 +56,6 @@ const std::map<uint64_t,std::string> WALLET_FLAG_CAVEATS{ }, }; -RecursiveMutex cs_wallets; -static std::vector<std::shared_ptr<CWallet>> vpwallets GUARDED_BY(cs_wallets); -static std::list<LoadWalletFn> g_load_wallet_fns GUARDED_BY(cs_wallets); - bool AddWalletSetting(interfaces::Chain& chain, const std::string& wallet_name) { util::SettingsValue setting_value = chain.getRwSetting("wallet"); @@ -94,19 +92,29 @@ static void UpdateWalletSetting(interfaces::Chain& chain, } } -bool AddWallet(const std::shared_ptr<CWallet>& wallet) +/** + * Refresh mempool status so the wallet is in an internally consistent state and + * immediately knows the transaction's status: Whether it can be considered + * trusted and is eligible to be abandoned ... + */ +static void RefreshMempoolStatus(CWalletTx& tx, interfaces::Chain& chain) +{ + tx.fInMempool = chain.isInMempool(tx.GetHash()); +} + +bool AddWallet(WalletContext& context, const std::shared_ptr<CWallet>& wallet) { - LOCK(cs_wallets); + LOCK(context.wallets_mutex); assert(wallet); - std::vector<std::shared_ptr<CWallet>>::const_iterator i = std::find(vpwallets.begin(), vpwallets.end(), wallet); - if (i != vpwallets.end()) return false; - vpwallets.push_back(wallet); + std::vector<std::shared_ptr<CWallet>>::const_iterator i = std::find(context.wallets.begin(), context.wallets.end(), wallet); + if (i != context.wallets.end()) return false; + context.wallets.push_back(wallet); wallet->ConnectScriptPubKeyManNotifiers(); wallet->NotifyCanGetAddressesChanged(); return true; } -bool RemoveWallet(const std::shared_ptr<CWallet>& wallet, std::optional<bool> load_on_start, std::vector<bilingual_str>& warnings) +bool RemoveWallet(WalletContext& context, const std::shared_ptr<CWallet>& wallet, std::optional<bool> load_on_start, std::vector<bilingual_str>& warnings) { assert(wallet); @@ -115,10 +123,10 @@ bool RemoveWallet(const std::shared_ptr<CWallet>& wallet, std::optional<bool> lo // Unregister with the validation interface which also drops shared ponters. wallet->m_chain_notifications_handler.reset(); - LOCK(cs_wallets); - std::vector<std::shared_ptr<CWallet>>::iterator i = std::find(vpwallets.begin(), vpwallets.end(), wallet); - if (i == vpwallets.end()) return false; - vpwallets.erase(i); + LOCK(context.wallets_mutex); + std::vector<std::shared_ptr<CWallet>>::iterator i = std::find(context.wallets.begin(), context.wallets.end(), wallet); + if (i == context.wallets.end()) return false; + context.wallets.erase(i); // Write the wallet setting UpdateWalletSetting(chain, name, load_on_start, warnings); @@ -126,32 +134,32 @@ bool RemoveWallet(const std::shared_ptr<CWallet>& wallet, std::optional<bool> lo return true; } -bool RemoveWallet(const std::shared_ptr<CWallet>& wallet, std::optional<bool> load_on_start) +bool RemoveWallet(WalletContext& context, const std::shared_ptr<CWallet>& wallet, std::optional<bool> load_on_start) { std::vector<bilingual_str> warnings; - return RemoveWallet(wallet, load_on_start, warnings); + return RemoveWallet(context, wallet, load_on_start, warnings); } -std::vector<std::shared_ptr<CWallet>> GetWallets() +std::vector<std::shared_ptr<CWallet>> GetWallets(WalletContext& context) { - LOCK(cs_wallets); - return vpwallets; + LOCK(context.wallets_mutex); + return context.wallets; } -std::shared_ptr<CWallet> GetWallet(const std::string& name) +std::shared_ptr<CWallet> GetWallet(WalletContext& context, const std::string& name) { - LOCK(cs_wallets); - for (const std::shared_ptr<CWallet>& wallet : vpwallets) { + LOCK(context.wallets_mutex); + for (const std::shared_ptr<CWallet>& wallet : context.wallets) { if (wallet->GetName() == name) return wallet; } return nullptr; } -std::unique_ptr<interfaces::Handler> HandleLoadWallet(LoadWalletFn load_wallet) +std::unique_ptr<interfaces::Handler> HandleLoadWallet(WalletContext& context, LoadWalletFn load_wallet) { - LOCK(cs_wallets); - auto it = g_load_wallet_fns.emplace(g_load_wallet_fns.end(), std::move(load_wallet)); - return interfaces::MakeHandler([it] { LOCK(cs_wallets); g_load_wallet_fns.erase(it); }); + LOCK(context.wallets_mutex); + auto it = context.wallet_load_fns.emplace(context.wallet_load_fns.end(), std::move(load_wallet)); + return interfaces::MakeHandler([&context, it] { LOCK(context.wallets_mutex); context.wallet_load_fns.erase(it); }); } static Mutex g_loading_wallet_mutex; @@ -203,7 +211,7 @@ void UnloadWallet(std::shared_ptr<CWallet>&& wallet) } namespace { -std::shared_ptr<CWallet> LoadWalletInternal(interfaces::Chain& chain, const std::string& name, std::optional<bool> load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings) +std::shared_ptr<CWallet> LoadWalletInternal(WalletContext& context, const std::string& name, std::optional<bool> load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings) { try { std::unique_ptr<WalletDatabase> database = MakeWalletDatabase(name, options, status, error); @@ -212,18 +220,18 @@ std::shared_ptr<CWallet> LoadWalletInternal(interfaces::Chain& chain, const std: return nullptr; } - chain.initMessage(_("Loading wallet…").translated); - std::shared_ptr<CWallet> wallet = CWallet::Create(&chain, name, std::move(database), options.create_flags, error, warnings); + context.chain->initMessage(_("Loading wallet…").translated); + std::shared_ptr<CWallet> wallet = CWallet::Create(context, name, std::move(database), options.create_flags, error, warnings); if (!wallet) { error = Untranslated("Wallet loading failed.") + Untranslated(" ") + error; status = DatabaseStatus::FAILED_LOAD; return nullptr; } - AddWallet(wallet); + AddWallet(context, wallet); wallet->postInitProcess(); // Write the wallet setting - UpdateWalletSetting(chain, name, load_on_start, warnings); + UpdateWalletSetting(*context.chain, name, load_on_start, warnings); return wallet; } catch (const std::runtime_error& e) { @@ -234,7 +242,7 @@ std::shared_ptr<CWallet> LoadWalletInternal(interfaces::Chain& chain, const std: } } // namespace -std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, std::optional<bool> load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings) +std::shared_ptr<CWallet> LoadWallet(WalletContext& context, const std::string& name, std::optional<bool> load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings) { auto result = WITH_LOCK(g_loading_wallet_mutex, return g_loading_wallet_set.insert(name)); if (!result.second) { @@ -242,12 +250,12 @@ std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& status = DatabaseStatus::FAILED_LOAD; return nullptr; } - auto wallet = LoadWalletInternal(chain, name, load_on_start, options, status, error, warnings); + auto wallet = LoadWalletInternal(context, name, load_on_start, options, status, error, warnings); WITH_LOCK(g_loading_wallet_mutex, g_loading_wallet_set.erase(result.first)); return wallet; } -std::shared_ptr<CWallet> CreateWallet(interfaces::Chain& chain, const std::string& name, std::optional<bool> load_on_start, DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings) +std::shared_ptr<CWallet> CreateWallet(WalletContext& context, const std::string& name, std::optional<bool> load_on_start, DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings) { uint64_t wallet_creation_flags = options.create_flags; const SecureString& passphrase = options.create_passphrase; @@ -292,8 +300,8 @@ std::shared_ptr<CWallet> CreateWallet(interfaces::Chain& chain, const std::strin } // Make the wallet - chain.initMessage(_("Loading wallet…").translated); - std::shared_ptr<CWallet> wallet = CWallet::Create(&chain, name, std::move(database), wallet_creation_flags, error, warnings); + context.chain->initMessage(_("Loading wallet…").translated); + std::shared_ptr<CWallet> wallet = CWallet::Create(context, name, std::move(database), wallet_creation_flags, error, warnings); if (!wallet) { error = Untranslated("Wallet creation failed.") + Untranslated(" ") + error; status = DatabaseStatus::FAILED_CREATE; @@ -335,11 +343,11 @@ std::shared_ptr<CWallet> CreateWallet(interfaces::Chain& chain, const std::strin wallet->Lock(); } } - AddWallet(wallet); + AddWallet(context, wallet); wallet->postInitProcess(); // Write the wallet settings - UpdateWalletSetting(chain, name, load_on_start, warnings); + UpdateWalletSetting(*context.chain, name, load_on_start, warnings); status = DatabaseStatus::SUCCESS; return wallet; @@ -374,6 +382,19 @@ void CWallet::UpgradeKeyMetadata() SetWalletFlag(WALLET_FLAG_KEY_ORIGIN_METADATA); } +void CWallet::UpgradeDescriptorCache() +{ + if (!IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS) || IsLocked() || IsWalletFlagSet(WALLET_FLAG_LAST_HARDENED_XPUB_CACHED)) { + return; + } + + for (ScriptPubKeyMan* spkm : GetAllScriptPubKeyMans()) { + DescriptorScriptPubKeyMan* desc_spkm = dynamic_cast<DescriptorScriptPubKeyMan*>(spkm); + desc_spkm->UpgradeDescriptorCache(); + } + SetWalletFlag(WALLET_FLAG_LAST_HARDENED_XPUB_CACHED); +} + bool CWallet::Unlock(const SecureString& strWalletPassphrase, bool accept_no_keys) { CCrypter crypter; @@ -390,6 +411,8 @@ bool CWallet::Unlock(const SecureString& strWalletPassphrase, bool accept_no_key if (Unlock(_vMasterKey, accept_no_keys)) { // Now that we've unlocked, upgrade the key metadata UpgradeKeyMetadata(); + // Now that we've unlocked, upgrade the descriptor cache + UpgradeDescriptorCache(); return true; } } @@ -559,7 +582,7 @@ bool CWallet::IsSpent(const uint256& hash, unsigned int n) const const uint256& wtxid = it->second; std::map<uint256, CWalletTx>::const_iterator mit = mapWallet.find(wtxid); if (mit != mapWallet.end()) { - int depth = mit->second.GetDepthInMainChain(); + int depth = GetTxDepthInMainChain(mit->second); if (depth > 0 || (depth == 0 && !mit->second.isAbandoned())) return true; // Spent } @@ -567,11 +590,16 @@ bool CWallet::IsSpent(const uint256& hash, unsigned int n) const return false; } -void CWallet::AddToSpends(const COutPoint& outpoint, const uint256& wtxid) +void CWallet::AddToSpends(const COutPoint& outpoint, const uint256& wtxid, WalletBatch* batch) { mapTxSpends.insert(std::make_pair(outpoint, wtxid)); - setLockedCoins.erase(outpoint); + if (batch) { + UnlockCoin(outpoint, batch); + } else { + WalletBatch temp_batch(GetDatabase()); + UnlockCoin(outpoint, &temp_batch); + } std::pair<TxSpends::iterator, TxSpends::iterator> range; range = mapTxSpends.equal_range(outpoint); @@ -579,7 +607,7 @@ void CWallet::AddToSpends(const COutPoint& outpoint, const uint256& wtxid) } -void CWallet::AddToSpends(const uint256& wtxid) +void CWallet::AddToSpends(const uint256& wtxid, WalletBatch* batch) { auto it = mapWallet.find(wtxid); assert(it != mapWallet.end()); @@ -588,7 +616,7 @@ void CWallet::AddToSpends(const uint256& wtxid) return; for (const CTxIn& txin : thisTx.tx->vin) - AddToSpends(txin.prevout, wtxid); + AddToSpends(txin.prevout, wtxid, batch); } bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) @@ -788,10 +816,7 @@ bool CWallet::MarkReplaced(const uint256& originalHash, const uint256& newHash) wtx.mapValue["replaced_by_txid"] = newHash.ToString(); // Refresh mempool status without waiting for transactionRemovedFromMempool - // notification so the wallet is in an internally consistent state and - // immediately knows the old transaction should not be considered trusted - // and is eligible to be abandoned - wtx.fInMempool = chain().isInMempool(originalHash); + RefreshMempoolStatus(wtx, chain()); WalletBatch batch(GetDatabase()); @@ -801,7 +826,7 @@ bool CWallet::MarkReplaced(const uint256& originalHash, const uint256& newHash) success = false; } - NotifyTransactionChanged(this, originalHash, CT_UPDATED); + NotifyTransactionChanged(originalHash, CT_UPDATED); return success; } @@ -860,7 +885,7 @@ bool CWallet::IsSpentKey(const uint256& hash, unsigned int n) const return false; } -CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const CWalletTx::Confirmation& confirm, const UpdateWalletTxFn& update_wtx, bool fFlushOnClose) +CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const CWalletTx::Confirmation& confirm, const UpdateWalletTxFn& update_wtx, bool fFlushOnClose, bool rescanning_old_block) { LOCK(cs_wallet); @@ -881,7 +906,7 @@ CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const CWalletTx::Confirmatio } // Inserts only if not already there, returns tx inserted or tx found - auto ret = mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(hash), std::forward_as_tuple(this, tx)); + auto ret = mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(hash), std::forward_as_tuple(tx)); CWalletTx& wtx = (*ret.first).second; bool fInsertedNew = ret.second; bool fUpdated = update_wtx && update_wtx(wtx, fInsertedNew); @@ -890,8 +915,8 @@ CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const CWalletTx::Confirmatio wtx.nTimeReceived = chain().getAdjustedTime(); wtx.nOrderPos = IncOrderPosNext(&batch); wtx.m_it_wtxOrdered = wtxOrdered.insert(std::make_pair(wtx.nOrderPos, &wtx)); - wtx.nTimeSmart = ComputeTimeSmart(wtx); - AddToSpends(hash); + wtx.nTimeSmart = ComputeTimeSmart(wtx, rescanning_old_block); + AddToSpends(hash, &batch); } if (!fInsertedNew) @@ -930,7 +955,7 @@ CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const CWalletTx::Confirmatio wtx.MarkDirty(); // Notify UI of new or updated transaction - NotifyTransactionChanged(this, hash, fInsertedNew ? CT_NEW : CT_UPDATED); + NotifyTransactionChanged(hash, fInsertedNew ? CT_NEW : CT_UPDATED); #if HAVE_SYSTEM // notify an external script when a wallet transaction comes in or is updated @@ -965,7 +990,7 @@ CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const CWalletTx::Confirmatio bool CWallet::LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx) { - const auto& ins = mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(hash), std::forward_as_tuple(this, nullptr)); + const auto& ins = mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(hash), std::forward_as_tuple(nullptr)); CWalletTx& wtx = ins.first->second; if (!fill_wtx(wtx, ins.second)) { return false; @@ -1007,7 +1032,7 @@ bool CWallet::LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx return true; } -bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::Confirmation confirm, bool fUpdate) +bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::Confirmation confirm, bool fUpdate, bool rescanning_old_block) { const CTransaction& tx = *ptx; { @@ -1045,7 +1070,7 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::Co // Block disconnection override an abandoned tx as unconfirmed // which means user may have to call abandontransaction again - return AddToWallet(MakeTransactionRef(tx), confirm, /* update_wtx= */ nullptr, /* fFlushOnClose= */ false); + return AddToWallet(MakeTransactionRef(tx), confirm, /* update_wtx= */ nullptr, /* fFlushOnClose= */ false, rescanning_old_block); } } return false; @@ -1055,7 +1080,7 @@ bool CWallet::TransactionCanBeAbandoned(const uint256& hashTx) const { LOCK(cs_wallet); const CWalletTx* wtx = GetWalletTx(hashTx); - return wtx && !wtx->isAbandoned() && wtx->GetDepthInMainChain() == 0 && !wtx->InMempool(); + return wtx && !wtx->isAbandoned() && GetTxDepthInMainChain(*wtx) == 0 && !wtx->InMempool(); } void CWallet::MarkInputsDirty(const CTransactionRef& tx) @@ -1081,7 +1106,7 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) auto it = mapWallet.find(hashTx); assert(it != mapWallet.end()); const CWalletTx& origtx = it->second; - if (origtx.GetDepthInMainChain() != 0 || origtx.InMempool()) { + if (GetTxDepthInMainChain(origtx) != 0 || origtx.InMempool()) { return false; } @@ -1094,7 +1119,7 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) auto it = mapWallet.find(now); assert(it != mapWallet.end()); CWalletTx& wtx = it->second; - int currentconfirm = wtx.GetDepthInMainChain(); + int currentconfirm = GetTxDepthInMainChain(wtx); // If the orig tx was not in block, none of its spends can be assert(currentconfirm <= 0); // if (currentconfirm < 0) {Tx and spends are already conflicted, no need to abandon} @@ -1104,7 +1129,7 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) wtx.setAbandoned(); wtx.MarkDirty(); batch.WriteTx(wtx); - NotifyTransactionChanged(this, wtx.GetHash(), CT_UPDATED); + NotifyTransactionChanged(wtx.GetHash(), CT_UPDATED); // Iterate over all its outputs, and mark transactions in the wallet that spend them abandoned too TxSpends::const_iterator iter = mapTxSpends.lower_bound(COutPoint(now, 0)); while (iter != mapTxSpends.end() && iter->first.hash == now) { @@ -1149,7 +1174,7 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c auto it = mapWallet.find(now); assert(it != mapWallet.end()); CWalletTx& wtx = it->second; - int currentconfirm = wtx.GetDepthInMainChain(); + int currentconfirm = GetTxDepthInMainChain(wtx); if (conflictconfirms < currentconfirm) { // Block is 'more conflicted' than current confirm; update. // Mark transaction as conflicted with this block. @@ -1174,9 +1199,9 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c } } -void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Confirmation confirm, bool update_tx) +void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Confirmation confirm, bool update_tx, bool rescanning_old_block) { - if (!AddToWalletIfInvolvingMe(ptx, confirm, update_tx)) + if (!AddToWalletIfInvolvingMe(ptx, confirm, update_tx, rescanning_old_block)) return; // Not one of ours // If a transaction changes 'conflicted' state, that changes the balance @@ -1191,7 +1216,7 @@ void CWallet::transactionAddedToMempool(const CTransactionRef& tx, uint64_t memp auto it = mapWallet.find(tx->GetHash()); if (it != mapWallet.end()) { - it->second.fInMempool = true; + RefreshMempoolStatus(it->second, chain()); } } @@ -1199,7 +1224,7 @@ void CWallet::transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRe LOCK(cs_wallet); auto it = mapWallet.find(tx->GetHash()); if (it != mapWallet.end()) { - it->second.fInMempool = false; + RefreshMempoolStatus(it->second, chain()); } // Handle transactions that were removed from the mempool because they // conflict with transactions in a newly connected block. @@ -1345,9 +1370,10 @@ CAmount CWallet::GetDebit(const CTransaction& tx, const isminefilter& filter) co bool CWallet::IsHDEnabled() const { // All Active ScriptPubKeyMans must be HD for this to be true - bool result = true; + bool result = false; for (const auto& spk_man : GetActiveScriptPubKeyMans()) { - result &= spk_man->IsHDEnabled(); + if (!spk_man->IsHDEnabled()) return false; + result = true; } return result; } @@ -1423,19 +1449,13 @@ bool CWallet::AddWalletFlags(uint64_t flags) // Helper for producing a max-sized low-S low-R signature (eg 71 bytes) // or a max-sized low-S signature (e.g. 72 bytes) if use_max_sig is true -bool CWallet::DummySignInput(CTxIn &tx_in, const CTxOut &txout, bool use_max_sig) const +bool DummySignInput(const SigningProvider& provider, CTxIn &tx_in, const CTxOut &txout, bool use_max_sig) { // Fill in dummy signatures for fee calculation. const CScript& scriptPubKey = txout.scriptPubKey; SignatureData sigdata; - std::unique_ptr<SigningProvider> provider = GetSolvingProvider(scriptPubKey); - if (!provider) { - // We don't know about this scriptpbuKey; - return false; - } - - if (!ProduceSignature(*provider, use_max_sig ? DUMMY_MAXIMUM_SIGNATURE_CREATOR : DUMMY_SIGNATURE_CREATOR, scriptPubKey, sigdata)) { + if (!ProduceSignature(provider, use_max_sig ? DUMMY_MAXIMUM_SIGNATURE_CREATOR : DUMMY_SIGNATURE_CREATOR, scriptPubKey, sigdata)) { return false; } UpdateInput(tx_in, sigdata); @@ -1443,14 +1463,21 @@ bool CWallet::DummySignInput(CTxIn &tx_in, const CTxOut &txout, bool use_max_sig } // Helper for producing a bunch of max-sized low-S low-R signatures (eg 71 bytes) -bool CWallet::DummySignTx(CMutableTransaction &txNew, const std::vector<CTxOut> &txouts, bool use_max_sig) const +bool CWallet::DummySignTx(CMutableTransaction &txNew, const std::vector<CTxOut> &txouts, const CCoinControl* coin_control) const { // Fill in dummy signatures for fee calculation. int nIn = 0; for (const auto& txout : txouts) { - if (!DummySignInput(txNew.vin[nIn], txout, use_max_sig)) { - return false; + CTxIn& txin = txNew.vin[nIn]; + // Use max sig if watch only inputs were used or if this particular input is an external input + // to ensure a sufficient fee is attained for the requested feerate. + const bool use_max_sig = coin_control && (coin_control->fAllowWatchOnly || coin_control->IsExternalSelected(txin.prevout)); + const std::unique_ptr<SigningProvider> provider = GetSolvingProvider(txout.scriptPubKey); + if (!provider || !DummySignInput(*provider, txin, txout, use_max_sig)) { + if (!coin_control || !DummySignInput(coin_control->m_external_provider, txin, txout, use_max_sig)) { + return false; + } } nIn++; @@ -1575,7 +1602,7 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc WalletLogPrintf("Rescan started from block %s...\n", start_block.ToString()); fAbortRescan = false; - ShowProgress(strprintf("%s " + _("Rescanning…").translated, GetDisplayName()), 0); // show rescan progress in GUI as dialog or on splashscreen, if -rescan on startup + ShowProgress(strprintf("%s " + _("Rescanning…").translated, GetDisplayName()), 0); // show rescan progress in GUI as dialog or on splashscreen, if rescan required on startup (e.g. due to corruption) uint256 tip_hash = WITH_LOCK(cs_wallet, return GetLastBlockHash()); uint256 end_hash = tip_hash; if (max_height) chain().findAncestorByHeight(tip_hash, *max_height, FoundBlock().hash(end_hash)); @@ -1618,7 +1645,7 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc break; } for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { - SyncTransaction(block.vtx[posInBlock], {CWalletTx::Status::CONFIRMED, block_height, block_hash, (int)posInBlock}, fUpdate); + SyncTransaction(block.vtx[posInBlock], {CWalletTx::Status::CONFIRMED, block_height, block_hash, (int)posInBlock}, fUpdate, /* rescanning_old_block */ true); } // scan succeeded, record block as most recent successfully scanned result.last_scanned_block = block_hash; @@ -1678,7 +1705,7 @@ void CWallet::ReacceptWalletTransactions() CWalletTx& wtx = item.second; assert(wtx.GetHash() == wtxid); - int nDepth = wtx.GetDepthInMainChain(); + int nDepth = GetTxDepthInMainChain(wtx); if (!wtx.IsCoinBase() && (nDepth == 0 && !wtx.isAbandoned())) { mapSorted.insert(std::make_pair(wtx.nOrderPos, &wtx)); @@ -1689,24 +1716,24 @@ void CWallet::ReacceptWalletTransactions() for (const std::pair<const int64_t, CWalletTx*>& item : mapSorted) { CWalletTx& wtx = *(item.second); std::string unused_err_string; - wtx.SubmitMemoryPoolAndRelay(unused_err_string, false); + SubmitTxMemoryPoolAndRelay(wtx, unused_err_string, false); } } -bool CWalletTx::SubmitMemoryPoolAndRelay(std::string& err_string, bool relay) +bool CWallet::SubmitTxMemoryPoolAndRelay(const CWalletTx& wtx, std::string& err_string, bool relay) const { // Can't relay if wallet is not broadcasting - if (!pwallet->GetBroadcastTransactions()) return false; + if (!GetBroadcastTransactions()) return false; // Don't relay abandoned transactions - if (isAbandoned()) return false; + if (wtx.isAbandoned()) return false; // Don't try to submit coinbase transactions. These would fail anyway but would // cause log spam. - if (IsCoinBase()) return false; + if (wtx.IsCoinBase()) return false; // Don't try to submit conflicted or confirmed transactions. - if (GetDepthInMainChain() != 0) return false; + if (GetTxDepthInMainChain(wtx) != 0) return false; // Submit transaction to mempool for relay - pwallet->WalletLogPrintf("Submitting wtx %s to mempool for relay\n", GetHash().ToString()); + WalletLogPrintf("Submitting wtx %s to mempool for relay\n", wtx.GetHash().ToString()); // We must set fInMempool here - while it will be re-set to true by the // entered-mempool callback, if we did not there would be a race where a // user could call sendmoney in a loop and hit spurious out of funds errors @@ -1716,18 +1743,17 @@ bool CWalletTx::SubmitMemoryPoolAndRelay(std::string& err_string, bool relay) // Irrespective of the failure reason, un-marking fInMempool // out-of-order is incorrect - it should be unmarked when // TransactionRemovedFromMempool fires. - bool ret = pwallet->chain().broadcastTransaction(tx, pwallet->m_default_max_tx_fee, relay, err_string); - fInMempool |= ret; + bool ret = chain().broadcastTransaction(wtx.tx, m_default_max_tx_fee, relay, err_string); + wtx.fInMempool |= ret; return ret; } -std::set<uint256> CWalletTx::GetConflicts() const +std::set<uint256> CWallet::GetTxConflicts(const CWalletTx& wtx) const { std::set<uint256> result; - if (pwallet != nullptr) { - uint256 myHash = GetHash(); - result = pwallet->GetConflicts(myHash); + uint256 myHash = wtx.GetHash(); + result = GetConflicts(myHash); result.erase(myHash); } return result; @@ -1765,11 +1791,11 @@ void CWallet::ResendWalletTransactions() for (std::pair<const uint256, CWalletTx>& item : mapWallet) { CWalletTx& wtx = item.second; // Attempt to rebroadcast all txes more than 5 minutes older than - // the last block. SubmitMemoryPoolAndRelay() will not rebroadcast + // the last block. SubmitTxMemoryPoolAndRelay() will not rebroadcast // any confirmed or conflicting txs. if (wtx.nTimeReceived > m_best_block_time - 5 * 60) continue; std::string unused_err_string; - if (wtx.SubmitMemoryPoolAndRelay(unused_err_string, true)) ++submitted_tx_count; + if (SubmitTxMemoryPoolAndRelay(wtx, unused_err_string, true)) ++submitted_tx_count; } } // cs_wallet @@ -1780,9 +1806,9 @@ void CWallet::ResendWalletTransactions() /** @} */ // end of mapWallet -void MaybeResendWalletTxs() +void MaybeResendWalletTxs(WalletContext& context) { - for (const std::shared_ptr<CWallet>& pwallet : GetWallets()) { + for (const std::shared_ptr<CWallet>& pwallet : GetWallets(context)) { pwallet->ResendWalletTransactions(); } } @@ -1807,11 +1833,11 @@ bool CWallet::SignTransaction(CMutableTransaction& tx) const const CWalletTx& wtx = mi->second; coins[input.prevout] = Coin(wtx.tx->vout[input.prevout.n], wtx.m_confirm.block_height, wtx.IsCoinBase()); } - std::map<int, std::string> input_errors; + std::map<int, bilingual_str> input_errors; return SignTransaction(tx, coins, SIGHASH_DEFAULT, input_errors); } -bool CWallet::SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors) const +bool CWallet::SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, bilingual_str>& input_errors) const { // Try to sign with all ScriptPubKeyMans for (ScriptPubKeyMan* spk_man : GetAllScriptPubKeyMans()) { @@ -1944,7 +1970,7 @@ void CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::ve for (const CTxIn& txin : tx->vin) { CWalletTx &coin = mapWallet.at(txin.prevout.hash); coin.MarkDirty(); - NotifyTransactionChanged(this, coin.GetHash(), CT_UPDATED); + NotifyTransactionChanged(coin.GetHash(), CT_UPDATED); } // Get the inserted-CWalletTx from mapWallet so that the @@ -1957,7 +1983,7 @@ void CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::ve } std::string err_string; - if (!wtx.SubmitMemoryPoolAndRelay(err_string, true)) { + if (!SubmitTxMemoryPoolAndRelay(wtx, err_string, true)) { WalletLogPrintf("CommitTransaction(): Transaction cannot be broadcast immediately, %s\n", err_string); // TODO: if we expect the failure to be long term or permanent, instead delete wtx from the wallet and return failure. } @@ -1983,10 +2009,7 @@ DBErrors CWallet::LoadWallet() assert(m_internal_spk_managers.empty()); } - if (nLoadWalletRet != DBErrors::LOAD_OK) - return nLoadWalletRet; - - return DBErrors::LOAD_OK; + return nLoadWalletRet; } DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256>& vHashOut) @@ -1999,7 +2022,7 @@ DBErrors CWallet::ZapSelectTx(std::vector<uint256>& vHashIn, std::vector<uint256 for (const auto& txin : it->second.tx->vin) mapTxSpends.erase(txin.prevout); mapWallet.erase(it); - NotifyTransactionChanged(this, hash, CT_DELETED); + NotifyTransactionChanged(hash, CT_DELETED); } if (nZapSelectTxRet == DBErrors::NEED_REWRITE) @@ -2033,8 +2056,8 @@ bool CWallet::SetAddressBookWithDB(WalletBatch& batch, const CTxDestination& add m_address_book[address].purpose = strPurpose; is_mine = IsMine(address) != ISMINE_NO; } - NotifyAddressBookChanged(this, address, strName, is_mine, - strPurpose, (fUpdated ? CT_UPDATED : CT_NEW) ); + NotifyAddressBookChanged(address, strName, is_mine, + strPurpose, (fUpdated ? CT_UPDATED : CT_NEW)); if (!strPurpose.empty() && !batch.WritePurpose(EncodeDestination(address), strPurpose)) return false; return batch.WriteName(EncodeDestination(address), strName); @@ -2069,7 +2092,7 @@ bool CWallet::DelAddressBook(const CTxDestination& address) is_mine = IsMine(address) != ISMINE_NO; } - NotifyAddressBookChanged(this, address, "", is_mine, "", CT_DELETED); + NotifyAddressBookChanged(address, "", is_mine, "", CT_DELETED); batch.ErasePurpose(EncodeDestination(address)); return batch.EraseName(EncodeDestination(address)); @@ -2079,9 +2102,14 @@ size_t CWallet::KeypoolCountExternalKeys() const { AssertLockHeld(cs_wallet); + auto legacy_spk_man = GetLegacyScriptPubKeyMan(); + if (legacy_spk_man) { + return legacy_spk_man->KeypoolCountExternalKeys(); + } + unsigned int count = 0; - for (auto spk_man : GetActiveScriptPubKeyMans()) { - count += spk_man->KeypoolCountExternalKeys(); + for (auto spk_man : m_external_spk_managers) { + count += spk_man.second->GetKeyPoolSize(); } return count; @@ -2108,7 +2136,7 @@ bool CWallet::TopUpKeyPool(unsigned int kpSize) return res; } -bool CWallet::GetNewDestination(const OutputType type, const std::string label, CTxDestination& dest, std::string& error) +bool CWallet::GetNewDestination(const OutputType type, const std::string label, CTxDestination& dest, bilingual_str& error) { LOCK(cs_wallet); error.clear(); @@ -2118,7 +2146,7 @@ bool CWallet::GetNewDestination(const OutputType type, const std::string label, spk_man->TopUp(); result = spk_man->GetNewDestination(type, dest, error); } else { - error = strprintf(_("Error: No %s addresses available."), FormatOutputType(type)).translated; + error = strprintf(_("Error: No %s addresses available."), FormatOutputType(type)); } if (result) { SetAddressBook(dest, label, "receive"); @@ -2127,7 +2155,7 @@ bool CWallet::GetNewDestination(const OutputType type, const std::string label, return result; } -bool CWallet::GetNewChangeDestination(const OutputType type, CTxDestination& dest, std::string& error) +bool CWallet::GetNewChangeDestination(const OutputType type, CTxDestination& dest, bilingual_str& error) { LOCK(cs_wallet); error.clear(); @@ -2167,7 +2195,7 @@ void CWallet::MarkDestinationsDirty(const std::set<CTxDestination>& destinations std::set<CTxDestination> CWallet::GetLabelAddresses(const std::string& label) const { - LOCK(cs_wallet); + AssertLockHeld(cs_wallet); std::set<CTxDestination> result; for (const std::pair<const CTxDestination, CAddressBookData>& item : m_address_book) { @@ -2180,11 +2208,11 @@ std::set<CTxDestination> CWallet::GetLabelAddresses(const std::string& label) co return result; } -bool ReserveDestination::GetReservedDestination(CTxDestination& dest, bool internal, std::string& error) +bool ReserveDestination::GetReservedDestination(CTxDestination& dest, bool internal, bilingual_str& error) { m_spk_man = pwallet->GetScriptPubKeyMan(type, internal); if (!m_spk_man) { - error = strprintf(_("Error: No %s addresses available."), FormatOutputType(type)).translated; + error = strprintf(_("Error: No %s addresses available."), FormatOutputType(type)); return false; } @@ -2236,22 +2264,36 @@ bool CWallet::DisplayAddress(const CTxDestination& dest) return signer_spk_man->DisplayAddress(scriptPubKey, signer); } -void CWallet::LockCoin(const COutPoint& output) +bool CWallet::LockCoin(const COutPoint& output, WalletBatch* batch) { AssertLockHeld(cs_wallet); setLockedCoins.insert(output); + if (batch) { + return batch->WriteLockedUTXO(output); + } + return true; } -void CWallet::UnlockCoin(const COutPoint& output) +bool CWallet::UnlockCoin(const COutPoint& output, WalletBatch* batch) { AssertLockHeld(cs_wallet); - setLockedCoins.erase(output); + bool was_locked = setLockedCoins.erase(output); + if (batch && was_locked) { + return batch->EraseLockedUTXO(output); + } + return true; } -void CWallet::UnlockAllCoins() +bool CWallet::UnlockAllCoins() { AssertLockHeld(cs_wallet); + bool success = true; + WalletBatch batch(GetDatabase()); + for (auto it = setLockedCoins.begin(); it != setLockedCoins.end(); ++it) { + success &= batch.EraseLockedUTXO(*it); + } setLockedCoins.clear(); + return success; } bool CWallet::IsLockedCoin(uint256 hash, unsigned int n) const @@ -2278,44 +2320,48 @@ void CWallet::GetKeyBirthTimes(std::map<CKeyID, int64_t>& mapKeyBirth) const { AssertLockHeld(cs_wallet); mapKeyBirth.clear(); - LegacyScriptPubKeyMan* spk_man = GetLegacyScriptPubKeyMan(); - assert(spk_man != nullptr); - LOCK(spk_man->cs_KeyStore); - - // get birth times for keys with metadata - for (const auto& entry : spk_man->mapKeyMetadata) { - if (entry.second.nCreateTime) { - mapKeyBirth[entry.first] = entry.second.nCreateTime; - } - } - // map in which we'll infer heights of other keys std::map<CKeyID, const CWalletTx::Confirmation*> mapKeyFirstBlock; CWalletTx::Confirmation max_confirm; max_confirm.block_height = GetLastBlockHeight() > 144 ? GetLastBlockHeight() - 144 : 0; // the tip can be reorganized; use a 144-block safety margin CHECK_NONFATAL(chain().findAncestorByHeight(GetLastBlockHash(), max_confirm.block_height, FoundBlock().hash(max_confirm.hashBlock))); - for (const CKeyID &keyid : spk_man->GetKeys()) { - if (mapKeyBirth.count(keyid) == 0) - mapKeyFirstBlock[keyid] = &max_confirm; - } - // if there are no such keys, we're done - if (mapKeyFirstBlock.empty()) - return; + { + LegacyScriptPubKeyMan* spk_man = GetLegacyScriptPubKeyMan(); + assert(spk_man != nullptr); + LOCK(spk_man->cs_KeyStore); + + // get birth times for keys with metadata + for (const auto& entry : spk_man->mapKeyMetadata) { + if (entry.second.nCreateTime) { + mapKeyBirth[entry.first] = entry.second.nCreateTime; + } + } - // find first block that affects those keys, if there are any left - for (const auto& entry : mapWallet) { - // iterate over all wallet transactions... - const CWalletTx &wtx = entry.second; - if (wtx.m_confirm.status == CWalletTx::CONFIRMED) { - // ... which are already in a block - for (const CTxOut &txout : wtx.tx->vout) { - // iterate over all their outputs - for (const auto &keyid : GetAffectedKeys(txout.scriptPubKey, *spk_man)) { - // ... and all their affected keys - auto rit = mapKeyFirstBlock.find(keyid); - if (rit != mapKeyFirstBlock.end() && wtx.m_confirm.block_height < rit->second->block_height) { - rit->second = &wtx.m_confirm; + // Prepare to infer birth heights for keys without metadata + for (const CKeyID &keyid : spk_man->GetKeys()) { + if (mapKeyBirth.count(keyid) == 0) + mapKeyFirstBlock[keyid] = &max_confirm; + } + + // if there are no such keys, we're done + if (mapKeyFirstBlock.empty()) + return; + + // find first block that affects those keys, if there are any left + for (const auto& entry : mapWallet) { + // iterate over all wallet transactions... + const CWalletTx &wtx = entry.second; + if (wtx.m_confirm.status == CWalletTx::CONFIRMED) { + // ... which are already in a block + for (const CTxOut &txout : wtx.tx->vout) { + // iterate over all their outputs + for (const auto &keyid : GetAffectedKeys(txout.scriptPubKey, *spk_man)) { + // ... and all their affected keys + auto rit = mapKeyFirstBlock.find(keyid); + if (rit != mapKeyFirstBlock.end() && wtx.m_confirm.block_height < rit->second->block_height) { + rit->second = &wtx.m_confirm; + } } } } @@ -2337,6 +2383,8 @@ void CWallet::GetKeyBirthTimes(std::map<CKeyID, int64_t>& mapKeyBirth) const { * - If sending a transaction, assign its timestamp to the current time. * - If receiving a transaction outside a block, assign its timestamp to the * current time. + * - If receiving a transaction during a rescanning process, assign all its + * (not already known) transactions' timestamps to the block time. * - If receiving a block with a future timestamp, assign all its (not already * known) transactions' timestamps to the current time. * - If receiving a block with a past timestamp, before the most recent known @@ -2351,38 +2399,43 @@ void CWallet::GetKeyBirthTimes(std::map<CKeyID, int64_t>& mapKeyBirth) const { * https://bitcointalk.org/?topic=54527, or * https://github.com/bitcoin/bitcoin/pull/1393. */ -unsigned int CWallet::ComputeTimeSmart(const CWalletTx& wtx) const +unsigned int CWallet::ComputeTimeSmart(const CWalletTx& wtx, bool rescanning_old_block) const { unsigned int nTimeSmart = wtx.nTimeReceived; if (!wtx.isUnconfirmed() && !wtx.isAbandoned()) { int64_t blocktime; - if (chain().findBlock(wtx.m_confirm.hashBlock, FoundBlock().time(blocktime))) { - int64_t latestNow = wtx.nTimeReceived; - int64_t latestEntry = 0; - - // Tolerate times up to the last timestamp in the wallet not more than 5 minutes into the future - int64_t latestTolerated = latestNow + 300; - const TxItems& txOrdered = wtxOrdered; - for (auto it = txOrdered.rbegin(); it != txOrdered.rend(); ++it) { - CWalletTx* const pwtx = it->second; - if (pwtx == &wtx) { - continue; - } - int64_t nSmartTime; - nSmartTime = pwtx->nTimeSmart; - if (!nSmartTime) { - nSmartTime = pwtx->nTimeReceived; - } - if (nSmartTime <= latestTolerated) { - latestEntry = nSmartTime; - if (nSmartTime > latestNow) { - latestNow = nSmartTime; + int64_t block_max_time; + if (chain().findBlock(wtx.m_confirm.hashBlock, FoundBlock().time(blocktime).maxTime(block_max_time))) { + if (rescanning_old_block) { + nTimeSmart = block_max_time; + } else { + int64_t latestNow = wtx.nTimeReceived; + int64_t latestEntry = 0; + + // Tolerate times up to the last timestamp in the wallet not more than 5 minutes into the future + int64_t latestTolerated = latestNow + 300; + const TxItems& txOrdered = wtxOrdered; + for (auto it = txOrdered.rbegin(); it != txOrdered.rend(); ++it) { + CWalletTx* const pwtx = it->second; + if (pwtx == &wtx) { + continue; + } + int64_t nSmartTime; + nSmartTime = pwtx->nTimeSmart; + if (!nSmartTime) { + nSmartTime = pwtx->nTimeReceived; + } + if (nSmartTime <= latestTolerated) { + latestEntry = nSmartTime; + if (nSmartTime > latestNow) { + latestNow = nSmartTime; + } + break; } - break; } - } - nTimeSmart = std::max(latestEntry, std::min(blocktime, latestNow)); + nTimeSmart = std::max(latestEntry, std::min(blocktime, latestNow)); + } } else { WalletLogPrintf("%s: found %s in block %s not in index\n", __func__, wtx.GetHash().ToString(), wtx.m_confirm.hashBlock.ToString()); } @@ -2478,14 +2531,17 @@ std::unique_ptr<WalletDatabase> MakeWalletDatabase(const std::string& name, cons return MakeDatabase(wallet_path, options, status, error_string); } -std::shared_ptr<CWallet> CWallet::Create(interfaces::Chain* chain, const std::string& name, std::unique_ptr<WalletDatabase> database, uint64_t wallet_creation_flags, bilingual_str& error, std::vector<bilingual_str>& warnings) +std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::string& name, std::unique_ptr<WalletDatabase> database, uint64_t wallet_creation_flags, bilingual_str& error, std::vector<bilingual_str>& warnings) { + interfaces::Chain* chain = context.chain; + ArgsManager& args = *Assert(context.args); const std::string& walletFile = database->Filename(); int64_t nStart = GetTimeMillis(); // TODO: Can't use std::make_shared because we need a custom deleter but // should be possible to use std::allocate_shared. std::shared_ptr<CWallet> walletInstance(new CWallet(chain, name, std::move(database)), ReleaseWallet); + bool rescan_required = false; DBErrors nLoadWalletRet = walletInstance->LoadWallet(); if (nLoadWalletRet != DBErrors::LOAD_OK) { if (nLoadWalletRet == DBErrors::CORRUPT) { @@ -2506,6 +2562,10 @@ std::shared_ptr<CWallet> CWallet::Create(interfaces::Chain* chain, const std::st { error = strprintf(_("Wallet needed to be rewritten: restart %s to complete"), PACKAGE_NAME); return nullptr; + } else if (nLoadWalletRet == DBErrors::NEED_RESCAN) { + warnings.push_back(strprintf(_("Error reading %s! Transaction data may be missing or incorrect." + " Rescanning wallet."), walletFile)); + rescan_required = true; } else { error = strprintf(_("Error loading %s"), walletFile); @@ -2561,111 +2621,124 @@ std::shared_ptr<CWallet> CWallet::Create(interfaces::Chain* chain, const std::st } } - if (!gArgs.GetArg("-addresstype", "").empty()) { - if (!ParseOutputType(gArgs.GetArg("-addresstype", ""), walletInstance->m_default_address_type)) { - error = strprintf(_("Unknown address type '%s'"), gArgs.GetArg("-addresstype", "")); + if (!args.GetArg("-addresstype", "").empty()) { + std::optional<OutputType> parsed = ParseOutputType(args.GetArg("-addresstype", "")); + if (!parsed) { + error = strprintf(_("Unknown address type '%s'"), args.GetArg("-addresstype", "")); return nullptr; } + walletInstance->m_default_address_type = parsed.value(); } - if (!gArgs.GetArg("-changetype", "").empty()) { - OutputType out_type; - if (!ParseOutputType(gArgs.GetArg("-changetype", ""), out_type)) { - error = strprintf(_("Unknown change type '%s'"), gArgs.GetArg("-changetype", "")); + if (!args.GetArg("-changetype", "").empty()) { + std::optional<OutputType> parsed = ParseOutputType(args.GetArg("-changetype", "")); + if (!parsed) { + error = strprintf(_("Unknown change type '%s'"), args.GetArg("-changetype", "")); return nullptr; } - walletInstance->m_default_change_type = out_type; + walletInstance->m_default_change_type = parsed.value(); } - if (gArgs.IsArgSet("-mintxfee")) { - CAmount n = 0; - if (!ParseMoney(gArgs.GetArg("-mintxfee", ""), n) || 0 == n) { - error = AmountErrMsg("mintxfee", gArgs.GetArg("-mintxfee", "")); + if (args.IsArgSet("-mintxfee")) { + std::optional<CAmount> min_tx_fee = ParseMoney(args.GetArg("-mintxfee", "")); + if (!min_tx_fee || min_tx_fee.value() == 0) { + error = AmountErrMsg("mintxfee", args.GetArg("-mintxfee", "")); return nullptr; - } - if (n > HIGH_TX_FEE_PER_KB) { + } else if (min_tx_fee.value() > HIGH_TX_FEE_PER_KB) { warnings.push_back(AmountHighWarn("-mintxfee") + Untranslated(" ") + _("This is the minimum transaction fee you pay on every transaction.")); } - walletInstance->m_min_fee = CFeeRate(n); + + walletInstance->m_min_fee = CFeeRate{min_tx_fee.value()}; } - if (gArgs.IsArgSet("-maxapsfee")) { - const std::string max_aps_fee{gArgs.GetArg("-maxapsfee", "")}; - CAmount n = 0; + if (args.IsArgSet("-maxapsfee")) { + const std::string max_aps_fee{args.GetArg("-maxapsfee", "")}; if (max_aps_fee == "-1") { - n = -1; - } else if (!ParseMoney(max_aps_fee, n)) { + walletInstance->m_max_aps_fee = -1; + } else if (std::optional<CAmount> max_fee = ParseMoney(max_aps_fee)) { + if (max_fee.value() > HIGH_APS_FEE) { + warnings.push_back(AmountHighWarn("-maxapsfee") + Untranslated(" ") + + _("This is the maximum transaction fee you pay (in addition to the normal fee) to prioritize partial spend avoidance over regular coin selection.")); + } + walletInstance->m_max_aps_fee = max_fee.value(); + } else { error = AmountErrMsg("maxapsfee", max_aps_fee); return nullptr; } - if (n > HIGH_APS_FEE) { - warnings.push_back(AmountHighWarn("-maxapsfee") + Untranslated(" ") + - _("This is the maximum transaction fee you pay (in addition to the normal fee) to prioritize partial spend avoidance over regular coin selection.")); - } - walletInstance->m_max_aps_fee = n; } - if (gArgs.IsArgSet("-fallbackfee")) { - CAmount nFeePerK = 0; - if (!ParseMoney(gArgs.GetArg("-fallbackfee", ""), nFeePerK)) { - error = strprintf(_("Invalid amount for -fallbackfee=<amount>: '%s'"), gArgs.GetArg("-fallbackfee", "")); + if (args.IsArgSet("-fallbackfee")) { + std::optional<CAmount> fallback_fee = ParseMoney(args.GetArg("-fallbackfee", "")); + if (!fallback_fee) { + error = strprintf(_("Invalid amount for -fallbackfee=<amount>: '%s'"), args.GetArg("-fallbackfee", "")); return nullptr; - } - if (nFeePerK > HIGH_TX_FEE_PER_KB) { + } else if (fallback_fee.value() > HIGH_TX_FEE_PER_KB) { warnings.push_back(AmountHighWarn("-fallbackfee") + Untranslated(" ") + _("This is the transaction fee you may pay when fee estimates are not available.")); } - walletInstance->m_fallback_fee = CFeeRate(nFeePerK); + walletInstance->m_fallback_fee = CFeeRate{fallback_fee.value()}; } + // Disable fallback fee in case value was set to 0, enable if non-null value walletInstance->m_allow_fallback_fee = walletInstance->m_fallback_fee.GetFeePerK() != 0; - if (gArgs.IsArgSet("-discardfee")) { - CAmount nFeePerK = 0; - if (!ParseMoney(gArgs.GetArg("-discardfee", ""), nFeePerK)) { - error = strprintf(_("Invalid amount for -discardfee=<amount>: '%s'"), gArgs.GetArg("-discardfee", "")); + if (args.IsArgSet("-discardfee")) { + std::optional<CAmount> discard_fee = ParseMoney(args.GetArg("-discardfee", "")); + if (!discard_fee) { + error = strprintf(_("Invalid amount for -discardfee=<amount>: '%s'"), args.GetArg("-discardfee", "")); return nullptr; - } - if (nFeePerK > HIGH_TX_FEE_PER_KB) { + } else if (discard_fee.value() > HIGH_TX_FEE_PER_KB) { warnings.push_back(AmountHighWarn("-discardfee") + Untranslated(" ") + _("This is the transaction fee you may discard if change is smaller than dust at this level")); } - walletInstance->m_discard_rate = CFeeRate(nFeePerK); + walletInstance->m_discard_rate = CFeeRate{discard_fee.value()}; } - if (gArgs.IsArgSet("-paytxfee")) { - CAmount nFeePerK = 0; - if (!ParseMoney(gArgs.GetArg("-paytxfee", ""), nFeePerK)) { - error = AmountErrMsg("paytxfee", gArgs.GetArg("-paytxfee", "")); + + if (args.IsArgSet("-paytxfee")) { + std::optional<CAmount> pay_tx_fee = ParseMoney(args.GetArg("-paytxfee", "")); + if (!pay_tx_fee) { + error = AmountErrMsg("paytxfee", args.GetArg("-paytxfee", "")); return nullptr; - } - if (nFeePerK > HIGH_TX_FEE_PER_KB) { + } else if (pay_tx_fee.value() > HIGH_TX_FEE_PER_KB) { warnings.push_back(AmountHighWarn("-paytxfee") + Untranslated(" ") + _("This is the transaction fee you will pay if you send a transaction.")); } - walletInstance->m_pay_tx_fee = CFeeRate(nFeePerK, 1000); + + walletInstance->m_pay_tx_fee = CFeeRate{pay_tx_fee.value(), 1000}; + if (chain && walletInstance->m_pay_tx_fee < chain->relayMinFee()) { error = strprintf(_("Invalid amount for -paytxfee=<amount>: '%s' (must be at least %s)"), - gArgs.GetArg("-paytxfee", ""), chain->relayMinFee().ToString()); + args.GetArg("-paytxfee", ""), chain->relayMinFee().ToString()); return nullptr; } } - if (gArgs.IsArgSet("-maxtxfee")) { - CAmount nMaxFee = 0; - if (!ParseMoney(gArgs.GetArg("-maxtxfee", ""), nMaxFee)) { - error = AmountErrMsg("maxtxfee", gArgs.GetArg("-maxtxfee", "")); + if (args.IsArgSet("-maxtxfee")) { + std::optional<CAmount> max_fee = ParseMoney(args.GetArg("-maxtxfee", "")); + if (!max_fee) { + error = AmountErrMsg("maxtxfee", args.GetArg("-maxtxfee", "")); return nullptr; - } - if (nMaxFee > HIGH_MAX_TX_FEE) { + } else if (max_fee.value() > HIGH_MAX_TX_FEE) { warnings.push_back(_("-maxtxfee is set very high! Fees this large could be paid on a single transaction.")); } - if (chain && CFeeRate(nMaxFee, 1000) < chain->relayMinFee()) { + + if (chain && CFeeRate{max_fee.value(), 1000} < chain->relayMinFee()) { error = strprintf(_("Invalid amount for -maxtxfee=<amount>: '%s' (must be at least the minrelay fee of %s to prevent stuck transactions)"), - gArgs.GetArg("-maxtxfee", ""), chain->relayMinFee().ToString()); + args.GetArg("-maxtxfee", ""), chain->relayMinFee().ToString()); + return nullptr; + } + + walletInstance->m_default_max_tx_fee = max_fee.value(); + } + + if (gArgs.IsArgSet("-consolidatefeerate")) { + if (std::optional<CAmount> consolidate_feerate = ParseMoney(gArgs.GetArg("-consolidatefeerate", ""))) { + walletInstance->m_consolidate_feerate = CFeeRate(*consolidate_feerate); + } else { + error = AmountErrMsg("consolidatefeerate", gArgs.GetArg("-consolidatefeerate", "")); return nullptr; } - walletInstance->m_default_max_tx_fee = nMaxFee; } if (chain && chain->relayMinFee().GetFeePerK() > HIGH_TX_FEE_PER_KB) { @@ -2673,9 +2746,9 @@ std::shared_ptr<CWallet> CWallet::Create(interfaces::Chain* chain, const std::st _("The wallet will avoid paying less than the minimum relay fee.")); } - walletInstance->m_confirm_target = gArgs.GetArg("-txconfirmtarget", DEFAULT_TX_CONFIRM_TARGET); - walletInstance->m_spend_zero_conf_change = gArgs.GetBoolArg("-spendzeroconfchange", DEFAULT_SPEND_ZEROCONF_CHANGE); - walletInstance->m_signal_rbf = gArgs.GetBoolArg("-walletrbf", DEFAULT_WALLET_RBF); + walletInstance->m_confirm_target = args.GetIntArg("-txconfirmtarget", DEFAULT_TX_CONFIRM_TARGET); + walletInstance->m_spend_zero_conf_change = args.GetBoolArg("-spendzeroconfchange", DEFAULT_SPEND_ZEROCONF_CHANGE); + walletInstance->m_signal_rbf = args.GetBoolArg("-walletrbf", DEFAULT_WALLET_RBF); walletInstance->WalletLogPrintf("Wallet completed loading in %15dms\n", GetTimeMillis() - nStart); @@ -2684,18 +2757,18 @@ std::shared_ptr<CWallet> CWallet::Create(interfaces::Chain* chain, const std::st LOCK(walletInstance->cs_wallet); - if (chain && !AttachChain(walletInstance, *chain, error, warnings)) { + if (chain && !AttachChain(walletInstance, *chain, rescan_required, error, warnings)) { return nullptr; } { - LOCK(cs_wallets); - for (auto& load_wallet : g_load_wallet_fns) { - load_wallet(interfaces::MakeWallet(walletInstance)); + LOCK(context.wallets_mutex); + for (auto& load_wallet : context.wallet_load_fns) { + load_wallet(interfaces::MakeWallet(context, walletInstance)); } } - walletInstance->SetBroadcastTransactions(gArgs.GetBoolArg("-walletbroadcast", DEFAULT_WALLETBROADCAST)); + walletInstance->SetBroadcastTransactions(args.GetBoolArg("-walletbroadcast", DEFAULT_WALLETBROADCAST)); { walletInstance->WalletLogPrintf("setKeyPool.size() = %u\n", walletInstance->GetKeyPoolSize()); @@ -2706,7 +2779,7 @@ std::shared_ptr<CWallet> CWallet::Create(interfaces::Chain* chain, const std::st return walletInstance; } -bool CWallet::AttachChain(const std::shared_ptr<CWallet>& walletInstance, interfaces::Chain& chain, bilingual_str& error, std::vector<bilingual_str>& warnings) +bool CWallet::AttachChain(const std::shared_ptr<CWallet>& walletInstance, interfaces::Chain& chain, const bool rescan_required, bilingual_str& error, std::vector<bilingual_str>& warnings) { LOCK(walletInstance->cs_wallet); // allow setting the chain if it hasn't been set already but prevent changing it @@ -2723,8 +2796,9 @@ bool CWallet::AttachChain(const std::shared_ptr<CWallet>& walletInstance, interf // interface. walletInstance->m_chain_notifications_handler = walletInstance->chain().handleNotifications(walletInstance); + // If rescan_required = true, rescan_height remains equal to 0 int rescan_height = 0; - if (!gArgs.GetBoolArg("-rescan", false)) + if (!rescan_required) { WalletBatch batch(walletInstance->GetDatabase()); CBlockLocator locator; @@ -2867,28 +2941,27 @@ CKeyPool::CKeyPool(const CPubKey& vchPubKeyIn, bool internalIn) m_pre_split = false; } -int CWalletTx::GetDepthInMainChain() const +int CWallet::GetTxDepthInMainChain(const CWalletTx& wtx) const { - assert(pwallet != nullptr); - AssertLockHeld(pwallet->cs_wallet); - if (isUnconfirmed() || isAbandoned()) return 0; + AssertLockHeld(cs_wallet); + if (wtx.isUnconfirmed() || wtx.isAbandoned()) return 0; - return (pwallet->GetLastBlockHeight() - m_confirm.block_height + 1) * (isConflicted() ? -1 : 1); + return (GetLastBlockHeight() - wtx.m_confirm.block_height + 1) * (wtx.isConflicted() ? -1 : 1); } -int CWalletTx::GetBlocksToMaturity() const +int CWallet::GetTxBlocksToMaturity(const CWalletTx& wtx) const { - if (!IsCoinBase()) + if (!wtx.IsCoinBase()) return 0; - int chain_depth = GetDepthInMainChain(); + int chain_depth = GetTxDepthInMainChain(wtx); assert(chain_depth >= 0); // coinbase tx should not be conflicted return std::max(0, (COINBASE_MATURITY+1) - chain_depth); } -bool CWalletTx::IsImmatureCoinBase() const +bool CWallet::IsTxImmatureCoinBase(const CWalletTx& wtx) const { // note GetBlocksToMaturity is 0 for non-coinbase tx - return GetBlocksToMaturity() > 0; + return GetTxBlocksToMaturity(wtx) > 0; } bool CWallet::IsCrypted() const @@ -2962,7 +3035,6 @@ ScriptPubKeyMan* CWallet::GetScriptPubKeyMan(const OutputType& type, bool intern const std::map<OutputType, ScriptPubKeyMan*>& spk_managers = internal ? m_internal_spk_managers : m_external_spk_managers; std::map<OutputType, ScriptPubKeyMan*>::const_iterator it = spk_managers.find(type); if (it == spk_managers.end()) { - WalletLogPrintf("%s scriptPubKey Manager for output type %d does not exist\n", internal ? "Internal" : "External", static_cast<int>(type)); return nullptr; } return it->second; @@ -3097,7 +3169,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans() // TODO: Setup taproot (bech32m) descriptors by default continue; } - auto spk_manager = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, internal)); + auto spk_manager = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this)); if (IsCrypted()) { if (IsLocked()) { throw std::runtime_error(std::string(__func__) + ": Wallet is locked, cannot setup new descriptors"); @@ -3106,7 +3178,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans() throw std::runtime_error(std::string(__func__) + ": Could not encrypt new descriptors"); } } - spk_manager->SetupDescriptorGeneration(master_key, t); + spk_manager->SetupDescriptorGeneration(master_key, t, internal); uint256 id = spk_manager->GetID(); m_spk_managers[id] = std::move(spk_manager); AddActiveScriptPubKeyMan(id, t, internal); @@ -3132,7 +3204,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans() continue; } OutputType t = *desc->GetOutputType(); - auto spk_manager = std::unique_ptr<ExternalSignerScriptPubKeyMan>(new ExternalSignerScriptPubKeyMan(*this, internal)); + auto spk_manager = std::unique_ptr<ExternalSignerScriptPubKeyMan>(new ExternalSignerScriptPubKeyMan(*this)); spk_manager->SetupDescriptor(std::move(desc)); uint256 id = spk_manager->GetID(); m_spk_managers[id] = std::move(spk_manager); @@ -3153,12 +3225,37 @@ void CWallet::AddActiveScriptPubKeyMan(uint256 id, OutputType type, bool interna void CWallet::LoadActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal) { + // Activating ScriptPubKeyManager for a given output and change type is incompatible with legacy wallets. + // Legacy wallets have only one ScriptPubKeyManager and it's active for all output and change types. + Assert(IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); + WalletLogPrintf("Setting spkMan to active: id = %s, type = %d, internal = %d\n", id.ToString(), static_cast<int>(type), static_cast<int>(internal)); auto& spk_mans = internal ? m_internal_spk_managers : m_external_spk_managers; + auto& spk_mans_other = internal ? m_external_spk_managers : m_internal_spk_managers; auto spk_man = m_spk_managers.at(id).get(); - spk_man->SetInternal(internal); spk_mans[type] = spk_man; + if (spk_mans_other[type] == spk_man) { + spk_mans_other.erase(type); + } + + NotifyCanGetAddressesChanged(); +} + +void CWallet::DeactivateScriptPubKeyMan(uint256 id, OutputType type, bool internal) +{ + auto spk_man = GetScriptPubKeyMan(type, internal); + if (spk_man != nullptr && spk_man->GetID() == id) { + WalletLogPrintf("Deactivate spkMan: id = %s, type = %d, internal = %d\n", id.ToString(), static_cast<int>(type), static_cast<int>(internal)); + WalletBatch batch(GetDatabase()); + if (!batch.EraseActiveScriptPubKeyMan(static_cast<uint8_t>(type), internal)) { + throw std::runtime_error(std::string(__func__) + ": erasing active ScriptPubKeyMan id failed"); + } + + auto& spk_mans = internal ? m_internal_spk_managers : m_external_spk_managers; + spk_mans.erase(type); + } + NotifyCanGetAddressesChanged(); } @@ -3186,50 +3283,33 @@ DescriptorScriptPubKeyMan* CWallet::GetDescriptorScriptPubKeyMan(const WalletDes ScriptPubKeyMan* CWallet::AddWalletDescriptor(WalletDescriptor& desc, const FlatSigningProvider& signing_provider, const std::string& label, bool internal) { + AssertLockHeld(cs_wallet); + if (!IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { WalletLogPrintf("Cannot add WalletDescriptor to a non-descriptor wallet\n"); return nullptr; } - LOCK(cs_wallet); - auto new_spk_man = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, desc)); - - // If we already have this descriptor, remove it from the maps but add the existing cache to desc - auto old_spk_man = GetDescriptorScriptPubKeyMan(desc); - if (old_spk_man) { + auto spk_man = GetDescriptorScriptPubKeyMan(desc); + if (spk_man) { WalletLogPrintf("Update existing descriptor: %s\n", desc.descriptor->ToString()); + spk_man->UpdateWalletDescriptor(desc); + } else { + auto new_spk_man = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, desc)); + spk_man = new_spk_man.get(); - { - LOCK(old_spk_man->cs_desc_man); - new_spk_man->SetCache(old_spk_man->GetWalletDescriptor().cache); - } - - // Remove from maps of active spkMans - auto old_spk_man_id = old_spk_man->GetID(); - for (bool internal : {false, true}) { - for (OutputType t : OUTPUT_TYPES) { - auto active_spk_man = GetScriptPubKeyMan(t, internal); - if (active_spk_man && active_spk_man->GetID() == old_spk_man_id) { - if (internal) { - m_internal_spk_managers.erase(t); - } else { - m_external_spk_managers.erase(t); - } - break; - } - } - } - m_spk_managers.erase(old_spk_man_id); + // Save the descriptor to memory + m_spk_managers[new_spk_man->GetID()] = std::move(new_spk_man); } // Add the private keys to the descriptor for (const auto& entry : signing_provider.keys) { const CKey& key = entry.second; - new_spk_man->AddDescriptorKey(key, key.GetPubKey()); + spk_man->AddDescriptorKey(key, key.GetPubKey()); } // Top up key pool, the manager will generate new scriptPubKeys internally - if (!new_spk_man->TopUp()) { + if (!spk_man->TopUp()) { WalletLogPrintf("Could not top up scriptPubKeys\n"); return nullptr; } @@ -3237,7 +3317,7 @@ ScriptPubKeyMan* CWallet::AddWalletDescriptor(WalletDescriptor& desc, const Flat // Apply the label if necessary // Note: we disable labels for ranged descriptors if (!desc.descriptor->IsRange()) { - auto script_pub_keys = new_spk_man->GetScriptPubKeys(); + auto script_pub_keys = spk_man->GetScriptPubKeys(); if (script_pub_keys.empty()) { WalletLogPrintf("Could not generate scriptPubKeys (cache is empty)\n"); return nullptr; @@ -3249,12 +3329,8 @@ ScriptPubKeyMan* CWallet::AddWalletDescriptor(WalletDescriptor& desc, const Flat } } - // Save the descriptor to memory - auto ret = new_spk_man.get(); - m_spk_managers[new_spk_man->GetID()] = std::move(new_spk_man); - // Save the descriptor to DB - ret->WriteDescriptor(); + spk_man->WriteDescriptor(); - return ret; + return spk_man; } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index b63938c5f1..767b24adbb 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -6,7 +6,7 @@ #ifndef BITCOIN_WALLET_WALLET_H #define BITCOIN_WALLET_WALLET_H -#include <amount.h> +#include <consensus/amount.h> #include <interfaces/chain.h> #include <interfaces/handler.h> #include <outputtype.h> @@ -21,9 +21,7 @@ #include <validationinterface.h> #include <wallet/coinselection.h> #include <wallet/crypter.h> -#include <wallet/receive.h> #include <wallet/scriptpubkeyman.h> -#include <wallet/spend.h> #include <wallet/transaction.h> #include <wallet/walletdb.h> #include <wallet/walletutil.h> @@ -42,6 +40,8 @@ #include <boost/signals2/signal.hpp> +struct WalletContext; + using LoadWalletFn = std::function<void(std::unique_ptr<interfaces::Wallet> wallet)>; struct bilingual_str; @@ -53,14 +53,14 @@ struct bilingual_str; //! by the shared pointer deleter. void UnloadWallet(std::shared_ptr<CWallet>&& wallet); -bool AddWallet(const std::shared_ptr<CWallet>& wallet); -bool RemoveWallet(const std::shared_ptr<CWallet>& wallet, std::optional<bool> load_on_start, std::vector<bilingual_str>& warnings); -bool RemoveWallet(const std::shared_ptr<CWallet>& wallet, std::optional<bool> load_on_start); -std::vector<std::shared_ptr<CWallet>> GetWallets(); -std::shared_ptr<CWallet> GetWallet(const std::string& name); -std::shared_ptr<CWallet> LoadWallet(interfaces::Chain& chain, const std::string& name, std::optional<bool> load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings); -std::shared_ptr<CWallet> CreateWallet(interfaces::Chain& chain, const std::string& name, std::optional<bool> load_on_start, DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings); -std::unique_ptr<interfaces::Handler> HandleLoadWallet(LoadWalletFn load_wallet); +bool AddWallet(WalletContext& context, const std::shared_ptr<CWallet>& wallet); +bool RemoveWallet(WalletContext& context, const std::shared_ptr<CWallet>& wallet, std::optional<bool> load_on_start, std::vector<bilingual_str>& warnings); +bool RemoveWallet(WalletContext& context, const std::shared_ptr<CWallet>& wallet, std::optional<bool> load_on_start); +std::vector<std::shared_ptr<CWallet>> GetWallets(WalletContext& context); +std::shared_ptr<CWallet> GetWallet(WalletContext& context, const std::string& name); +std::shared_ptr<CWallet> LoadWallet(WalletContext& context, const std::string& name, std::optional<bool> load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings); +std::shared_ptr<CWallet> CreateWallet(WalletContext& context, const std::string& name, std::optional<bool> load_on_start, DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings); +std::unique_ptr<interfaces::Handler> HandleLoadWallet(WalletContext& context, LoadWalletFn load_wallet); std::unique_ptr<WalletDatabase> MakeWalletDatabase(const std::string& name, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error); //! -paytxfee default @@ -71,6 +71,8 @@ static const CAmount DEFAULT_FALLBACK_FEE = 0; static const CAmount DEFAULT_DISCARD_FEE = 10000; //! -mintxfee default static const CAmount DEFAULT_TRANSACTION_MINFEE = 1000; +//! -consolidatefeerate default +static const CAmount DEFAULT_CONSOLIDATE_FEERATE{10000}; // 10 sat/vbyte /** * maximum fee increase allowed to do partial spend avoidance, even for nodes with this feature disabled by default * @@ -117,6 +119,7 @@ static constexpr uint64_t KNOWN_WALLET_FLAGS = WALLET_FLAG_AVOID_REUSE | WALLET_FLAG_BLANK_WALLET | WALLET_FLAG_KEY_ORIGIN_METADATA + | WALLET_FLAG_LAST_HARDENED_XPUB_CACHED | WALLET_FLAG_DISABLE_PRIVATE_KEYS | WALLET_FLAG_DESCRIPTORS | WALLET_FLAG_EXTERNAL_SIGNER; @@ -128,6 +131,7 @@ static const std::map<std::string,WalletFlags> WALLET_FLAG_MAP{ {"avoid_reuse", WALLET_FLAG_AVOID_REUSE}, {"blank", WALLET_FLAG_BLANK_WALLET}, {"key_origin_metadata", WALLET_FLAG_KEY_ORIGIN_METADATA}, + {"last_hardened_xpub_cached", WALLET_FLAG_LAST_HARDENED_XPUB_CACHED}, {"disable_private_keys", WALLET_FLAG_DISABLE_PRIVATE_KEYS}, {"descriptor_wallet", WALLET_FLAG_DESCRIPTORS}, {"external_signer", WALLET_FLAG_EXTERNAL_SIGNER} @@ -181,7 +185,7 @@ public: } //! Reserve an address - bool GetReservedDestination(CTxDestination& pubkey, bool internal, std::string& error); + bool GetReservedDestination(CTxDestination& pubkey, bool internal, bilingual_str& error); //! Return reserved address void ReturnDestination(); //! Keep the address. Do not return it's key to the keypool when this object goes out of scope @@ -252,8 +256,8 @@ private: */ typedef std::multimap<COutPoint, uint256> TxSpends; TxSpends mapTxSpends GUARDED_BY(cs_wallet); - void AddToSpends(const COutPoint& outpoint, const uint256& wtxid) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void AddToSpends(const uint256& wtxid) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void AddToSpends(const COutPoint& outpoint, const uint256& wtxid, WalletBatch* batch = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void AddToSpends(const uint256& wtxid, WalletBatch* batch = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** * Add a transaction to the wallet, or update it. pIndex and posInBlock should @@ -267,8 +271,11 @@ private: * abandoned is an indication that it is not safe to be considered abandoned. * Abandoned state should probably be more carefully tracked via different * posInBlock signals or by checking mempool presence when necessary. + * + * Should be called with rescanning_old_block set to true, if the transaction is + * not discovered in real time, but during a rescan of old blocks. */ - bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool fUpdate) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool fUpdate, bool rescanning_old_block) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** Mark a transaction (and its in-wallet descendants) as conflicting with a particular block. */ void MarkConflicted(const uint256& hashBlock, int conflicting_height, const uint256& hashTx); @@ -280,7 +287,7 @@ private: /* Used by TransactionAddedToMemorypool/BlockConnected/Disconnected/ScanForWalletTransactions. * Should be called with non-zero block_hash and posInBlock if this is for a transaction that is included in a block. */ - void SyncTransaction(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool update_tx = true) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void SyncTransaction(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool update_tx = true, bool rescanning_old_block = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** WalletFlags set on this wallet. */ std::atomic<uint64_t> m_wallet_flags{0}; @@ -325,14 +332,12 @@ private: // ScriptPubKeyMan::GetID. In many cases it will be the hash of an internal structure std::map<uint256, std::unique_ptr<ScriptPubKeyMan>> m_spk_managers; - bool CreateTransactionInternal(const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, const CCoinControl& coin_control, FeeCalculation& fee_calc_out, bool sign) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - /** * Catch wallet up to current chain, scanning new blocks, updating the best * block locator and m_last_block_processed, and registering for * notifications about new blocks and transactions. */ - static bool AttachChain(const std::shared_ptr<CWallet>& wallet, interfaces::Chain& chain, bilingual_str& error, std::vector<bilingual_str>& warnings); + static bool AttachChain(const std::shared_ptr<CWallet>& wallet, interfaces::Chain& chain, const bool rescan_required, bilingual_str& error, std::vector<bilingual_str>& warnings); public: /** @@ -347,17 +352,6 @@ public: return *m_database; } - /** - * Select a set of coins such that nValueRet >= nTargetValue and at least - * all coins from coin_control are selected; never select unconfirmed coins if they are not ours - * param@[out] setCoinsRet Populated with inputs including pre-selected inputs from - * coin_control and Coin Selection if successful. - * param@[out] nValueRet Total value of selected coins including pre-selected ones - * from coin_control and Coin Selection if successful. - */ - bool SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, - const CCoinControl& coin_control, CoinSelectionParams& coin_selection_params) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - /** Get a name for this wallet for logging/debugging purposes. */ const std::string& GetName() const { return m_name; } @@ -413,39 +407,40 @@ public: interfaces::Chain& chain() const { assert(m_chain); return *m_chain; } const CWalletTx* GetWalletTx(const uint256& hash) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool IsTrusted(const CWalletTx& wtx, std::set<uint256>& trusted_parents) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - - //! check whether we support the named feature - bool CanSupportFeature(enum WalletFeature wf) const override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); return IsFeatureSupported(nWalletVersion, wf); } - /** - * populate vCoins with vector of available COutputs. - */ - void AvailableCoins(std::vector<COutput>& vCoins, const CCoinControl* coinControl = nullptr, const CAmount& nMinimumAmount = 1, const CAmount& nMaximumAmount = MAX_MONEY, const CAmount& nMinimumSumAmount = MAX_MONEY, const uint64_t nMaximumCount = 0) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct + // annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The annotation + // "NO_THREAD_SAFETY_ANALYSIS" was temporarily added to avoid having to + // resolve the issue of member access into incomplete type CWallet. Note + // that we still have the runtime check "AssertLockHeld(pwallet->cs_wallet)" + // in place. + std::set<uint256> GetTxConflicts(const CWalletTx& wtx) const NO_THREAD_SAFETY_ANALYSIS; /** - * Return list of available coins and locked coins grouped by non-change output address. + * Return depth of transaction in blockchain: + * <0 : conflicts with a transaction this deep in the blockchain + * 0 : in memory pool, waiting to be included in a block + * >=1 : this many blocks deep in the main chain */ - std::map<CTxDestination, std::vector<COutput>> ListCoins() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + // TODO: Remove "NO_THREAD_SAFETY_ANALYSIS" and replace it with the correct + // annotation "EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet)". The annotation + // "NO_THREAD_SAFETY_ANALYSIS" was temporarily added to avoid having to + // resolve the issue of member access into incomplete type CWallet. Note + // that we still have the runtime check "AssertLockHeld(pwallet->cs_wallet)" + // in place. + int GetTxDepthInMainChain(const CWalletTx& wtx) const NO_THREAD_SAFETY_ANALYSIS; + bool IsTxInMainChain(const CWalletTx& wtx) const { return GetTxDepthInMainChain(wtx) > 0; } /** - * Find non-change parent output. + * @return number of blocks to maturity for this transaction: + * 0 : is not a coinbase transaction, or is a mature coinbase transaction + * >0 : is a coinbase transaction which matures in this many blocks */ - const CTxOut& FindNonChangeParentOutput(const CTransaction& tx, int output) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + int GetTxBlocksToMaturity(const CWalletTx& wtx) const; + bool IsTxImmatureCoinBase(const CWalletTx& wtx) const; - /** - * Shuffle and select coins until nTargetValue is reached while avoiding - * small change; This method is stochastic for some inputs and upon - * completion the coin set and corresponding actual target value is - * assembled - * param@[in] coins Set of UTXOs to consider. These will be categorized into - * OutputGroups and filtered using eligibility_filter before - * selecting coins. - * param@[out] setCoinsRet Populated with the coins selected if successful. - * param@[out] nValueRet Used to return the total value of selected coins. - */ - bool AttemptSelection(const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, std::vector<COutput> coins, - std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CoinSelectionParams& coin_selection_params) const; + //! check whether we support the named feature + bool CanSupportFeature(enum WalletFeature wf) const override EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); return IsFeatureSupported(nWalletVersion, wf); } bool IsSpent(const uint256& hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -453,15 +448,13 @@ public: bool IsSpentKey(const uint256& hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void SetSpentKeyState(WalletBatch& batch, const uint256& hash, unsigned int n, bool used, std::set<CTxDestination>& tx_destinations) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - std::vector<OutputGroup> GroupOutputs(const std::vector<COutput>& outputs, const CoinSelectionParams& coin_sel_params, const CoinEligibilityFilter& filter, bool positive_only) const; - /** Display address on an external signer. Returns false if external signer support is not compiled */ bool DisplayAddress(const CTxDestination& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool IsLockedCoin(uint256 hash, unsigned int n) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void LockCoin(const COutPoint& output) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void UnlockCoin(const COutPoint& output) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void UnlockAllCoins() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool LockCoin(const COutPoint& output, WalletBatch* batch = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool UnlockCoin(const COutPoint& output, WalletBatch* batch = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool UnlockAllCoins() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void ListLockedCoins(std::vector<COutPoint>& vOutpts) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /* @@ -476,6 +469,9 @@ public: //! Upgrade stored CKeyMetadata objects to store key origin info as KeyOriginInfo void UpgradeKeyMetadata() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + //! Upgrade DescriptorCaches + void UpgradeDescriptorCache() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool LoadMinVersion(int nVersion) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { AssertLockHeld(cs_wallet); nWalletVersion = nVersion; return true; } //! Adds a destination data tuple to the store, without saving it to disk @@ -491,7 +487,7 @@ public: bool EncryptWallet(const SecureString& strWalletPassphrase); void GetKeyBirthTimes(std::map<CKeyID, int64_t> &mapKeyBirth) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - unsigned int ComputeTimeSmart(const CWalletTx& wtx) const; + unsigned int ComputeTimeSmart(const CWalletTx& wtx, bool rescanning_old_block) const; /** * Increment the next transaction order id @@ -510,7 +506,7 @@ public: //! @return true if wtx is changed and needs to be saved to disk, otherwise false using UpdateWalletTxFn = std::function<bool(CWalletTx& wtx, bool new_tx)>; - CWalletTx* AddToWallet(CTransactionRef tx, const CWalletTx::Confirmation& confirm, const UpdateWalletTxFn& update_wtx=nullptr, bool fFlushOnClose=true); + CWalletTx* AddToWallet(CTransactionRef tx, const CWalletTx::Confirmation& confirm, const UpdateWalletTxFn& update_wtx=nullptr, bool fFlushOnClose=true, bool rescanning_old_block = false); bool LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void transactionAddedToMempool(const CTransactionRef& tx, uint64_t mempool_sequence) override; void blockConnected(const CBlock& block, int height) override; @@ -537,28 +533,13 @@ public: void transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason, uint64_t mempool_sequence) override; void ReacceptWalletTransactions() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void ResendWalletTransactions(); - struct Balance { - CAmount m_mine_trusted{0}; //!< Trusted, at depth=GetBalance.min_depth or more - CAmount m_mine_untrusted_pending{0}; //!< Untrusted, but in mempool (pending) - CAmount m_mine_immature{0}; //!< Immature coinbases in the main chain - CAmount m_watchonly_trusted{0}; - CAmount m_watchonly_untrusted_pending{0}; - CAmount m_watchonly_immature{0}; - }; - Balance GetBalance(int min_depth = 0, bool avoid_reuse = true) const; - CAmount GetAvailableBalance(const CCoinControl* coinControl = nullptr) const; OutputType TransactionChangeType(const std::optional<OutputType>& change_type, const std::vector<CRecipient>& vecSend) const; - /** - * Insert additional inputs into the transaction by - * calling CreateTransaction(); - */ - bool FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, CCoinControl); /** Fetch the inputs and sign with SIGHASH_ALL. */ bool SignTransaction(CMutableTransaction& tx) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** Sign the tx given the input coins and sighash. */ - bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, std::string>& input_errors) const; + bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, bilingual_str>& input_errors) const; SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const; /** @@ -582,12 +563,6 @@ public: size_t* n_signed = nullptr) const; /** - * Create a new transaction paying the recipients with a set of coins - * selected by SelectCoins(); Also create the change output, when needed - * @note passing nChangePosInOut as -1 will result in setting a random position - */ - bool CreateTransaction(const std::vector<CRecipient>& vecSend, CTransactionRef& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, const CCoinControl& coin_control, FeeCalculation& fee_calc_out, bool sign = true); - /** * Submit the transaction to the node's mempool and then relay to peers. * Should be called after CreateTransaction unless you want to abort * broadcasting the transaction. @@ -598,14 +573,16 @@ public: */ void CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm); - bool DummySignTx(CMutableTransaction &txNew, const std::set<CTxOut> &txouts, bool use_max_sig = false) const + /** Pass this transaction to node for mempool insertion and relay to peers if flag set to true */ + bool SubmitTxMemoryPoolAndRelay(const CWalletTx& wtx, std::string& err_string, bool relay) const; + + bool DummySignTx(CMutableTransaction &txNew, const std::set<CTxOut> &txouts, const CCoinControl* coin_control = nullptr) const { std::vector<CTxOut> v_txouts(txouts.size()); std::copy(txouts.begin(), txouts.end(), v_txouts.begin()); - return DummySignTx(txNew, v_txouts, use_max_sig); + return DummySignTx(txNew, v_txouts, coin_control); } - bool DummySignTx(CMutableTransaction &txNew, const std::vector<CTxOut> &txouts, bool use_max_sig = false) const; - bool DummySignInput(CTxIn &tx_in, const CTxOut &txout, bool use_max_sig = false) const; + bool DummySignTx(CMutableTransaction &txNew, const std::vector<CTxOut> &txouts, const CCoinControl* coin_control = nullptr) const; bool ImportScripts(const std::set<CScript> scripts, int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -631,6 +608,12 @@ public: * output itself, just drop it to fees. */ CFeeRate m_discard_rate{DEFAULT_DISCARD_FEE}; + /** When the actual feerate is less than the consolidate feerate, we will tend to make transactions which + * consolidate inputs. When the actual feerate is greater than the consolidate feerate, we will tend to make + * transactions which have the lowest fees. + */ + CFeeRate m_consolidate_feerate{DEFAULT_CONSOLIDATE_FEERATE}; + /** The maximum fee amount we're willing to pay to prioritize partial spend avoidance. */ CAmount m_max_aps_fee{DEFAULT_MAX_AVOIDPARTIALSPEND_FEE}; //!< note: this is absolute fee, not fee rate OutputType m_default_address_type{DEFAULT_ADDRESS_TYPE}; @@ -649,10 +632,7 @@ public: int64_t GetOldestKeyPoolTime() const; - std::set<std::set<CTxDestination>> GetAddressGroupings() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - std::map<CTxDestination, CAmount> GetAddressBalances() const; - - std::set<CTxDestination> GetLabelAddresses(const std::string& label) const; + std::set<CTxDestination> GetLabelAddresses(const std::string& label) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** * Marks all outputs in each one of the destinations dirty, so their cache is @@ -660,30 +640,21 @@ public: */ void MarkDestinationsDirty(const std::set<CTxDestination>& destinations) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool GetNewDestination(const OutputType type, const std::string label, CTxDestination& dest, std::string& error); - bool GetNewChangeDestination(const OutputType type, CTxDestination& dest, std::string& error); + bool GetNewDestination(const OutputType type, const std::string label, CTxDestination& dest, bilingual_str& error); + bool GetNewChangeDestination(const OutputType type, CTxDestination& dest, bilingual_str& error); isminetype IsMine(const CTxDestination& dest) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); isminetype IsMine(const CScript& script) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - isminetype IsMine(const CTxIn& txin) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** * Returns amount of debit if the input matches the * filter, otherwise returns 0 */ CAmount GetDebit(const CTxIn& txin, const isminefilter& filter) const; isminetype IsMine(const CTxOut& txout) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - CAmount GetCredit(const CTxOut& txout, const isminefilter& filter) const; - bool IsChange(const CTxOut& txout) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool IsChange(const CScript& script) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - CAmount GetChange(const CTxOut& txout) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool IsMine(const CTransaction& tx) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** should probably be renamed to IsRelevantToMe */ bool IsFromMe(const CTransaction& tx) const; CAmount GetDebit(const CTransaction& tx, const isminefilter& filter) const; - /** Returns whether all of the inputs match the filter */ - bool IsAllFromMe(const CTransaction& tx, const isminefilter& filter) const; - CAmount GetCredit(const CTransaction& tx, const isminefilter& filter) const; - CAmount GetChange(const CTransaction& tx) const; void chainStateFlushed(const CBlockLocator& loc) override; DBErrors LoadWallet(); @@ -724,19 +695,18 @@ public: /** * Address book entry changed. - * @note called with lock cs_wallet held. + * @note called without lock cs_wallet held. */ - boost::signals2::signal<void (CWallet *wallet, const CTxDestination - &address, const std::string &label, bool isMine, - const std::string &purpose, - ChangeType status)> NotifyAddressBookChanged; + boost::signals2::signal<void(const CTxDestination& address, + const std::string& label, bool isMine, + const std::string& purpose, ChangeType status)> + NotifyAddressBookChanged; /** * Wallet transaction added, removed or updated. * @note called with lock cs_wallet held. */ - boost::signals2::signal<void (CWallet *wallet, const uint256 &hashTx, - ChangeType status)> NotifyTransactionChanged; + boost::signals2::signal<void(const uint256& hashTx, ChangeType status)> NotifyTransactionChanged; /** Show progress e.g. for rescan */ boost::signals2::signal<void (const std::string &title, int nProgress)> ShowProgress; @@ -768,7 +738,7 @@ public: bool MarkReplaced(const uint256& originalHash, const uint256& newHash); /* Initializes the wallet, returns a new CWallet instance or a null pointer in case of an error */ - static std::shared_ptr<CWallet> Create(interfaces::Chain* chain, const std::string& name, std::unique_ptr<WalletDatabase> database, uint64_t wallet_creation_flags, bilingual_str& error, std::vector<bilingual_str>& warnings); + static std::shared_ptr<CWallet> Create(WalletContext& context, const std::string& name, std::unique_ptr<WalletDatabase> database, uint64_t wallet_creation_flags, bilingual_str& error, std::vector<bilingual_str>& warnings); /** * Wallet post-init setup @@ -895,6 +865,12 @@ public: //! @param[in] internal Whether this ScriptPubKeyMan provides change addresses void LoadActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal); + //! Remove specified ScriptPubKeyMan from set of active SPK managers. Writes the change to the wallet file. + //! @param[in] id The unique id for the ScriptPubKeyMan + //! @param[in] type The OutputType this ScriptPubKeyMan provides addresses for + //! @param[in] internal Whether this ScriptPubKeyMan provides change addresses + void DeactivateScriptPubKeyMan(uint256 id, OutputType type, bool internal); + //! Create new DescriptorScriptPubKeyMans and add them to the wallet void SetupDescriptorScriptPubKeyMans() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -902,14 +878,14 @@ public: DescriptorScriptPubKeyMan* GetDescriptorScriptPubKeyMan(const WalletDescriptor& desc) const; //! Add a descriptor to the wallet, return a ScriptPubKeyMan & associated output type - ScriptPubKeyMan* AddWalletDescriptor(WalletDescriptor& desc, const FlatSigningProvider& signing_provider, const std::string& label, bool internal); + ScriptPubKeyMan* AddWalletDescriptor(WalletDescriptor& desc, const FlatSigningProvider& signing_provider, const std::string& label, bool internal) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); }; /** * Called periodically by the schedule thread. Prompts individual wallets to resend * their transactions. Actual rebroadcast schedule is managed by the wallets themselves. */ -void MaybeResendWalletTxs(); +void MaybeResendWalletTxs(WalletContext& context); /** RAII object to check and reserve a wallet rescan */ class WalletRescanReserver @@ -945,22 +921,12 @@ public: } }; -struct TxSize { - int64_t vsize{-1}; - int64_t weight{-1}; -}; - -/** Calculate the size of the transaction assuming all signatures are max size -* Use DummySignatureCreator, which inserts 71 byte signatures everywhere. -* NOTE: this requires that all inputs must be in mapWallet (eg the tx should -* be IsAllFromMe). */ -TxSize CalculateMaximumSignedTxSize(const CTransaction& tx, const CWallet* wallet, bool use_max_sig = false) EXCLUSIVE_LOCKS_REQUIRED(wallet->cs_wallet); -TxSize CalculateMaximumSignedTxSize(const CTransaction& tx, const CWallet* wallet, const std::vector<CTxOut>& txouts, bool use_max_sig = false); - //! Add wallet name to persistent configuration so it will be loaded on startup. bool AddWalletSetting(interfaces::Chain& chain, const std::string& wallet_name); //! Remove wallet name from persistent configuration so it will not be loaded on startup. bool RemoveWalletSetting(interfaces::Chain& chain, const std::string& wallet_name); +bool DummySignInput(const SigningProvider& provider, CTxIn &tx_in, const CTxOut &txout, bool use_max_sig); + #endif // BITCOIN_WALLET_WALLET_H diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 24d5351945..8ff09a0878 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -40,6 +40,7 @@ const std::string FLAGS{"flags"}; const std::string HDCHAIN{"hdchain"}; const std::string KEYMETA{"keymeta"}; const std::string KEY{"key"}; +const std::string LOCKED_UTXO{"lockedutxo"}; const std::string MASTER_KEY{"mkey"}; const std::string MINVERSION{"minversion"}; const std::string NAME{"name"}; @@ -52,6 +53,7 @@ const std::string TX{"tx"}; const std::string VERSION{"version"}; const std::string WALLETDESCRIPTOR{"walletdescriptor"}; const std::string WALLETDESCRIPTORCACHE{"walletdescriptorcache"}; +const std::string WALLETDESCRIPTORLHCACHE{"walletdescriptorlhcache"}; const std::string WALLETDESCRIPTORCKEY{"walletdescriptorckey"}; const std::string WALLETDESCRIPTORKEY{"walletdescriptorkey"}; const std::string WATCHMETA{"watchmeta"}; @@ -209,6 +211,12 @@ bool WalletBatch::WriteActiveScriptPubKeyMan(uint8_t type, const uint256& id, bo return WriteIC(make_pair(key, type), id); } +bool WalletBatch::EraseActiveScriptPubKeyMan(uint8_t type, bool internal) +{ + const std::string key{internal ? DBKeys::ACTIVEINTERNALSPK : DBKeys::ACTIVEEXTERNALSPK}; + return EraseIC(make_pair(key, type)); +} + bool WalletBatch::WriteDescriptorKey(const uint256& desc_id, const CPubKey& pubkey, const CPrivKey& privkey) { // hash pubkey/privkey to accelerate wallet load @@ -248,6 +256,45 @@ bool WalletBatch::WriteDescriptorParentCache(const CExtPubKey& xpub, const uint2 return WriteIC(std::make_pair(std::make_pair(DBKeys::WALLETDESCRIPTORCACHE, desc_id), key_exp_index), ser_xpub); } +bool WalletBatch::WriteDescriptorLastHardenedCache(const CExtPubKey& xpub, const uint256& desc_id, uint32_t key_exp_index) +{ + std::vector<unsigned char> ser_xpub(BIP32_EXTKEY_SIZE); + xpub.Encode(ser_xpub.data()); + return WriteIC(std::make_pair(std::make_pair(DBKeys::WALLETDESCRIPTORLHCACHE, desc_id), key_exp_index), ser_xpub); +} + +bool WalletBatch::WriteDescriptorCacheItems(const uint256& desc_id, const DescriptorCache& cache) +{ + for (const auto& parent_xpub_pair : cache.GetCachedParentExtPubKeys()) { + if (!WriteDescriptorParentCache(parent_xpub_pair.second, desc_id, parent_xpub_pair.first)) { + return false; + } + } + for (const auto& derived_xpub_map_pair : cache.GetCachedDerivedExtPubKeys()) { + for (const auto& derived_xpub_pair : derived_xpub_map_pair.second) { + if (!WriteDescriptorDerivedCache(derived_xpub_pair.second, desc_id, derived_xpub_map_pair.first, derived_xpub_pair.first)) { + return false; + } + } + } + for (const auto& lh_xpub_pair : cache.GetCachedLastHardenedExtPubKeys()) { + if (!WriteDescriptorLastHardenedCache(lh_xpub_pair.second, desc_id, lh_xpub_pair.first)) { + return false; + } + } + return true; +} + +bool WalletBatch::WriteLockedUTXO(const COutPoint& output) +{ + return WriteIC(std::make_pair(DBKeys::LOCKED_UTXO, std::make_pair(output.hash, output.n)), uint8_t{'1'}); +} + +bool WalletBatch::EraseLockedUTXO(const COutPoint& output) +{ + return EraseIC(std::make_pair(DBKeys::LOCKED_UTXO, std::make_pair(output.hash, output.n))); +} + class CWalletScanState { public: unsigned int nKeys{0}; @@ -264,6 +311,7 @@ public: std::map<std::pair<uint256, CKeyID>, CKey> m_descriptor_keys; std::map<std::pair<uint256, CKeyID>, std::pair<CPubKey, std::vector<unsigned char>>> m_descriptor_crypt_keys; std::map<uint160, CHDChain> m_hd_chains; + bool tx_corrupt{false}; CWalletScanState() { } @@ -298,7 +346,13 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, // LoadToWallet call below creates a new CWalletTx that fill_wtx // callback fills with transaction metadata. auto fill_wtx = [&](CWalletTx& wtx, bool new_tx) { - assert(new_tx); + if(!new_tx) { + // There's some corruption here since the tx we just tried to load was already in the wallet. + // We don't consider this type of corruption critical, and can fix it by removing tx data and + // rescanning. + wss.tx_corrupt = true; + return false; + } ssValue >> wtx; if (wtx.GetHash() != hash) return false; @@ -602,6 +656,17 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, } else { wss.m_descriptor_caches[desc_id].CacheDerivedExtPubKey(key_exp_index, der_index, xpub); } + } else if (strType == DBKeys::WALLETDESCRIPTORLHCACHE) { + uint256 desc_id; + uint32_t key_exp_index; + ssKey >> desc_id; + ssKey >> key_exp_index; + + std::vector<unsigned char> ser_xpub(BIP32_EXTKEY_SIZE); + ssValue >> ser_xpub; + CExtPubKey xpub; + xpub.Decode(ser_xpub.data()); + wss.m_descriptor_caches[desc_id].CacheLastHardenedExtPubKey(key_exp_index, xpub); } else if (strType == DBKeys::WALLETDESCRIPTORKEY) { uint256 desc_id; CPubKey pubkey; @@ -654,6 +719,12 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, wss.m_descriptor_crypt_keys.insert(std::make_pair(std::make_pair(desc_id, pubkey.GetID()), std::make_pair(pubkey, privkey))); wss.fIsEncrypted = true; + } else if (strType == DBKeys::LOCKED_UTXO) { + uint256 hash; + uint32_t n; + ssKey >> hash; + ssKey >> n; + pwallet->LockCoin(COutPoint(hash, n)); } else if (strType != DBKeys::BESTBLOCK && strType != DBKeys::BESTBLOCK_NOMERKLE && strType != DBKeys::MINVERSION && strType != DBKeys::ACENTRY && strType != DBKeys::VERSION && strType != DBKeys::SETTINGS && @@ -691,6 +762,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) { CWalletScanState wss; bool fNoncriticalErrors = false; + bool rescan_required = false; DBErrors result = DBErrors::LOAD_OK; LOCK(pwallet->cs_wallet); @@ -754,12 +826,17 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) } else if (strType == DBKeys::FLAGS) { // reading the wallet flags can only fail if unknown flags are present result = DBErrors::TOO_NEW; + } else if (wss.tx_corrupt) { + pwallet->WalletLogPrintf("Error: Corrupt transaction found. This can be fixed by removing transactions from wallet and rescanning.\n"); + // Set tx_corrupt back to false so that the error is only printed once (per corrupt tx) + wss.tx_corrupt = false; + result = DBErrors::CORRUPT; } else { // Leave other errors alone, if we try to fix them we might make things worse. fNoncriticalErrors = true; // ... but do warn the user there is something wrong. if (strType == DBKeys::TX) // Rescan if there is a bad transaction record: - gArgs.SoftSetBoolArg("-rescan", true); + rescan_required = true; } } if (!strErr.empty()) @@ -795,8 +872,11 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) ((DescriptorScriptPubKeyMan*)spk_man)->AddCryptedKey(desc_key_pair.first.second, desc_key_pair.second.first, desc_key_pair.second.second); } - if (fNoncriticalErrors && result == DBErrors::LOAD_OK) + if (rescan_required && result == DBErrors::LOAD_OK) { + result = DBErrors::NEED_RESCAN; + } else if (fNoncriticalErrors && result == DBErrors::LOAD_OK) { result = DBErrors::NONCRITICAL_ERROR; + } // Any wallet corruption at all: skip any rewriting or // upgrading, we don't want to make it worse. @@ -843,6 +923,14 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) result = DBErrors::CORRUPT; } + // Upgrade all of the descriptor caches to cache the last hardened xpub + // This operation is not atomic, but if it fails, only new entries are added so it is backwards compatible + try { + pwallet->UpgradeDescriptorCache(); + } catch (...) { + result = DBErrors::CORRUPT; + } + // Set the inactive chain if (wss.m_hd_chains.size() > 0) { LegacyScriptPubKeyMan* legacy_spkm = pwallet->GetLegacyScriptPubKeyMan(); @@ -899,7 +987,7 @@ DBErrors WalletBatch::FindWalletTx(std::vector<uint256>& vTxHash, std::list<CWal uint256 hash; ssKey >> hash; vTxHash.push_back(hash); - vWtx.emplace_back(nullptr /* wallet */, nullptr /* tx */); + vWtx.emplace_back(nullptr /* tx */); ssValue >> vWtx.back(); } } @@ -949,14 +1037,14 @@ DBErrors WalletBatch::ZapSelectTx(std::vector<uint256>& vTxHashIn, std::vector<u return DBErrors::LOAD_OK; } -void MaybeCompactWalletDB() +void MaybeCompactWalletDB(WalletContext& context) { static std::atomic<bool> fOneThread(false); if (fOneThread.exchange(true)) { return; } - for (const std::shared_ptr<CWallet>& pwallet : GetWallets()) { + for (const std::shared_ptr<CWallet>& pwallet : GetWallets(context)) { WalletDatabase& dbh = pwallet->GetDatabase(); unsigned int nUpdateCounter = dbh.nUpdateCounter; diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index e7b2d7d780..9c752623b3 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -6,7 +6,6 @@ #ifndef BITCOIN_WALLET_WALLETDB_H #define BITCOIN_WALLET_WALLETDB_H -#include <amount.h> #include <script/sign.h> #include <wallet/db.h> #include <wallet/walletutil.h> @@ -31,6 +30,7 @@ static const bool DEFAULT_FLUSHWALLET = true; struct CBlockLocator; +struct WalletContext; class CKeyPool; class CMasterKey; class CScript; @@ -47,7 +47,8 @@ enum class DBErrors NONCRITICAL_ERROR, TOO_NEW, LOAD_FAIL, - NEED_REWRITE + NEED_REWRITE, + NEED_RESCAN }; namespace DBKeys { @@ -64,6 +65,7 @@ extern const std::string FLAGS; extern const std::string HDCHAIN; extern const std::string KEY; extern const std::string KEYMETA; +extern const std::string LOCKED_UTXO; extern const std::string MASTER_KEY; extern const std::string MINVERSION; extern const std::string NAME; @@ -246,6 +248,11 @@ public: bool WriteDescriptor(const uint256& desc_id, const WalletDescriptor& descriptor); bool WriteDescriptorDerivedCache(const CExtPubKey& xpub, const uint256& desc_id, uint32_t key_exp_index, uint32_t der_index); bool WriteDescriptorParentCache(const CExtPubKey& xpub, const uint256& desc_id, uint32_t key_exp_index); + bool WriteDescriptorLastHardenedCache(const CExtPubKey& xpub, const uint256& desc_id, uint32_t key_exp_index); + bool WriteDescriptorCacheItems(const uint256& desc_id, const DescriptorCache& cache); + + bool WriteLockedUTXO(const COutPoint& output); + bool EraseLockedUTXO(const COutPoint& output); /// Write destination data key,value tuple to database bool WriteDestData(const std::string &address, const std::string &key, const std::string &value); @@ -253,6 +260,7 @@ public: bool EraseDestData(const std::string &address, const std::string &key); bool WriteActiveScriptPubKeyMan(uint8_t type, const uint256& id, bool internal); + bool EraseActiveScriptPubKeyMan(uint8_t type, bool internal); DBErrors LoadWallet(CWallet* pwallet); DBErrors FindWalletTx(std::vector<uint256>& vTxHash, std::list<CWalletTx>& vWtx); @@ -276,7 +284,7 @@ private: }; //! Compacts BDB state so that wallet.dat is self-contained (if there are changes) -void MaybeCompactWalletDB(); +void MaybeCompactWalletDB(WalletContext& context); //! Callback for filtering key types to deserialize in ReadKeyValue using KeyFilterFn = std::function<bool(const std::string&)>; diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 50b6c9d29f..e3cb5cee5d 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -76,6 +76,10 @@ static std::shared_ptr<CWallet> MakeWallet(const std::string& name, const fs::pa } else if (load_wallet_ret == DBErrors::NEED_REWRITE) { tfm::format(std::cerr, "Wallet needed to be rewritten: restart %s to complete", PACKAGE_NAME); return nullptr; + } else if (load_wallet_ret == DBErrors::NEED_RESCAN) { + tfm::format(std::cerr, "Error reading %s! Some transaction data might be missing or" + " incorrect. Wallet requires a rescan.", + name); } else { tfm::format(std::cerr, "Error loading %s", name); return nullptr; diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index 0713f768c1..c75e1759bc 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -43,6 +43,9 @@ enum WalletFlags : uint64_t { // Indicates that the metadata has already been upgraded to contain key origins WALLET_FLAG_KEY_ORIGIN_METADATA = (1ULL << 1), + // Indicates that the descriptor cache has been upgraded to cache last hardened xpubs + WALLET_FLAG_LAST_HARDENED_XPUB_CACHED = (1ULL << 2), + // will enforce the rule that the wallet can't contain any private keys (only watch-only/pubkeys) WALLET_FLAG_DISABLE_PRIVATE_KEYS = (1ULL << 32), diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index 86f47d71f3..a53de34db4 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -47,7 +47,7 @@ CZMQNotificationInterface* CZMQNotificationInterface::Create() std::unique_ptr<CZMQAbstractNotifier> notifier = factory(); notifier->SetType(entry.first); notifier->SetAddress(address); - notifier->SetOutboundMessageHighWaterMark(static_cast<int>(gArgs.GetArg(arg + "hwm", CZMQAbstractNotifier::DEFAULT_ZMQ_SNDHWM))); + notifier->SetOutboundMessageHighWaterMark(static_cast<int>(gArgs.GetIntArg(arg + "hwm", CZMQAbstractNotifier::DEFAULT_ZMQ_SNDHWM))); notifiers.push_back(std::move(notifier)); } } diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index 6ae866cc07..56f4c98317 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -6,6 +6,7 @@ #include <chain.h> #include <chainparams.h> +#include <netbase.h> #include <node/blockstorage.h> #include <rpc/server.h> #include <streams.h> @@ -73,6 +74,20 @@ static int zmq_send_multipart(void *sock, const void* data, size_t size, ...) return 0; } +static bool IsZMQAddressIPV6(const std::string &zmq_address) +{ + const std::string tcp_prefix = "tcp://"; + const size_t tcp_index = zmq_address.rfind(tcp_prefix); + const size_t colon_index = zmq_address.rfind(":"); + if (tcp_index == 0 && colon_index != std::string::npos) { + const std::string ip = zmq_address.substr(tcp_prefix.length(), colon_index - tcp_prefix.length()); + CNetAddr addr; + LookupHost(ip, addr, false); + if (addr.IsIPv6()) return true; + } + return false; +} + bool CZMQAbstractPublishNotifier::Initialize(void *pcontext) { assert(!psocket); @@ -107,6 +122,15 @@ bool CZMQAbstractPublishNotifier::Initialize(void *pcontext) return false; } + // On some systems (e.g. OpenBSD) the ZMQ_IPV6 must not be enabled, if the address to bind isn't IPv6 + const int enable_ipv6 { IsZMQAddressIPV6(address) ? 1 : 0}; + rc = zmq_setsockopt(psocket, ZMQ_IPV6, &enable_ipv6, sizeof(enable_ipv6)); + if (rc != 0) { + zmqError("Failed to set ZMQ_IPV6"); + zmq_close(psocket); + return false; + } + rc = zmq_bind(psocket, address.c_str()); if (rc != 0) { diff --git a/test/README.md b/test/README.md index ab34ce42dc..acd68d8d8f 100644 --- a/test/README.md +++ b/test/README.md @@ -5,30 +5,41 @@ etc. This directory contains the following sets of tests: +- [fuzz](/test/fuzz) A runner to execute all fuzz targets from + [/src/test/fuzz](/src/test/fuzz). - [functional](/test/functional) which test the functionality of bitcoind and bitcoin-qt by interacting with them through the RPC and P2P interfaces. -- [util](/test/util) which tests the bitcoin utilities, currently only -bitcoin-tx. +- [util](/test/util) which tests the utilities (bitcoin-util, bitcoin-tx, ...). - [lint](/test/lint/) which perform various static analysis checks. -The util tests are run as part of `make check` target. The functional +The util tests are run as part of `make check` target. The fuzz tests, functional tests and lint scripts can be run as explained in the sections below. # Running tests locally Before tests can be run locally, Bitcoin Core must be built. See the [building instructions](/doc#building) for help. +## Fuzz tests + +See [/doc/fuzzing.md](/doc/fuzzing.md) ### Functional tests -#### Dependencies +#### Dependencies and prerequisites The ZMQ functional test requires a python ZMQ library. To install it: - on Unix, run `sudo apt-get install python3-zmq` - on mac OS, run `pip3 install pyzmq` + +On Windows the `PYTHONUTF8` environment variable must be set to 1: + +```cmd +set PYTHONUTF8=1 +``` + #### Running the tests Individual tests can be run by directly calling the test script, e.g.: @@ -84,6 +95,12 @@ Run all possible tests with test/functional/test_runner.py --extended ``` +In order to run backwards compatibility tests, download the previous node binaries: + +``` +test/get_previous_releases.py -b v0.20.1 v0.19.1 v0.18.1 v0.17.2 v0.16.3 v0.15.2 +``` + By default, up to 4 tests will be run in parallel by test_runner. To specify how many jobs to run, append `--jobs=n` @@ -251,7 +268,7 @@ For ways to generate more granular profiles, see the README in ### Util tests -Util tests can be run locally by running `test/util/bitcoin-util-test.py`. +Util tests can be run locally by running `test/util/test_runner.py`. Use the `-v` option for verbose output. ### Lint tests @@ -263,7 +280,6 @@ Use the `-v` option for verbose output. | [`lint-python.sh`](lint/lint-python.sh) | [flake8](https://gitlab.com/pycqa/flake8) | [3.8.3](https://github.com/bitcoin/bitcoin/pull/19348) | `pip3 install flake8==3.8.3` | [`lint-python.sh`](lint/lint-python.sh) | [mypy](https://github.com/python/mypy) | [0.781](https://github.com/bitcoin/bitcoin/pull/19348) | `pip3 install mypy==0.781` | [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck) | [0.7.2](https://github.com/bitcoin/bitcoin/pull/21749) | [details...](https://github.com/koalaman/shellcheck#installing) -| [`lint-shell.sh`](lint/lint-shell.sh) | [yq](https://github.com/kislyuk/yq) | default | `pip3 install yq` | [`lint-spelling.sh`](lint/lint-spelling.sh) | [codespell](https://github.com/codespell-project/codespell) | [2.0.0](https://github.com/bitcoin/bitcoin/pull/20817) | `pip3 install codespell==2.0.0` Please be aware that on Linux distributions all dependencies are usually available as packages, but could be outdated. diff --git a/test/config.ini.in b/test/config.ini.in index e3872181cd..8bcba1b39c 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -3,7 +3,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. # These environment variables are set by the build process and read by -# test/functional/test_runner.py and test/util/bitcoin-util-test.py +# test/*/test_runner.py and test/util/rpcauth-test.py [environment] PACKAGE_NAME=@PACKAGE_NAME@ @@ -24,3 +24,4 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py @ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true @ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true @ENABLE_EXTERNAL_SIGNER_TRUE@ENABLE_EXTERNAL_SIGNER=true +@ENABLE_SYSCALL_SANDBOX_TRUE@ENABLE_SYSCALL_SANDBOX=true diff --git a/test/functional/README.md b/test/functional/README.md index d830ba0334..926810cf03 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -188,5 +188,5 @@ perf report -i /path/to/datadir/send-big-msgs.perf.data.xxxx --stdio | c++filt | #### See also: - [Installing perf](https://askubuntu.com/q/50145) -- [Perf examples](http://www.brendangregg.com/perf.html) +- [Perf examples](https://www.brendangregg.com/perf.html) - [Hotspot](https://github.com/KDAB/hotspot): a GUI for perf output analysis diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py index fab921ef19..cde0399d8b 100644 --- a/test/functional/data/invalid_txs.py +++ b/test/functional/data/invalid_txs.py @@ -29,27 +29,32 @@ from test_framework.messages import ( CTxOut, MAX_MONEY, ) -from test_framework import script as sc from test_framework.blocktools import create_tx_with_script, MAX_BLOCK_SIGOPS from test_framework.script import ( CScript, + OP_0, + OP_2DIV, + OP_2MUL, + OP_AND, OP_CAT, - OP_SUBSTR, - OP_LEFT, - OP_RIGHT, + OP_CHECKSIG, + OP_DIV, OP_INVERT, - OP_AND, + OP_LEFT, + OP_LSHIFT, + OP_MOD, + OP_MUL, OP_OR, + OP_RIGHT, + OP_RSHIFT, + OP_SUBSTR, + OP_TRUE, OP_XOR, - OP_2MUL, - OP_2DIV, - OP_MUL, - OP_DIV, - OP_MOD, - OP_LSHIFT, - OP_RSHIFT ) -basic_p2sh = sc.CScript([sc.OP_HASH160, sc.hash160(sc.CScript([sc.OP_0])), sc.OP_EQUAL]) +from test_framework.script_util import ( + script_to_p2sh_script, +) +basic_p2sh = script_to_p2sh_script(CScript([OP_0])) class BadTxTemplate: @@ -116,7 +121,7 @@ class SizeTooSmall(BadTxTemplate): def get_tx(self): tx = CTransaction() tx.vin.append(self.valid_txin) - tx.vout.append(CTxOut(0, sc.CScript([sc.OP_TRUE]))) + tx.vout.append(CTxOut(0, CScript([OP_TRUE]))) tx.calc_sha256() return tx @@ -151,6 +156,19 @@ class DuplicateInput(BadTxTemplate): return tx +class PrevoutNullInput(BadTxTemplate): + reject_reason = 'bad-txns-prevout-null' + expect_disconnect = True + + def get_tx(self): + tx = CTransaction() + tx.vin.append(self.valid_txin) + tx.vin.append(CTxIn(COutPoint(hash=0, n=0xffffffff))) + tx.vout.append(CTxOut(1, basic_p2sh)) + tx.calc_sha256() + return tx + + class NonexistentInput(BadTxTemplate): reject_reason = None # Added as an orphan tx. expect_disconnect = False @@ -217,7 +235,7 @@ class TooManySigops(BadTxTemplate): expect_disconnect = False def get_tx(self): - lotsa_checksigs = sc.CScript([sc.OP_CHECKSIG] * (MAX_BLOCK_SIGOPS)) + lotsa_checksigs = CScript([OP_CHECKSIG] * (MAX_BLOCK_SIGOPS)) return create_tx_with_script( self.spend_tx, 0, script_pub_key=lotsa_checksigs, diff --git a/test/functional/example_test.py b/test/functional/example_test.py index a0eb213a78..d6fc2d580f 100755 --- a/test/functional/example_test.py +++ b/test/functional/example_test.py @@ -141,7 +141,7 @@ class ExampleTest(BitcoinTestFramework): peer_messaging = self.nodes[0].add_p2p_connection(BaseNode()) # Generating a block on one of the nodes will get us out of IBD - blocks = [int(self.nodes[0].generate(nblocks=1)[0], 16)] + blocks = [int(self.generate(self.nodes[0], nblocks=1)[0], 16)] self.sync_all(self.nodes[0:2]) # Notice above how we called an RPC by calling a method with the same diff --git a/test/functional/feature_abortnode.py b/test/functional/feature_abortnode.py index 8abfdef3a1..e3cb7725bd 100755 --- a/test/functional/feature_abortnode.py +++ b/test/functional/feature_abortnode.py @@ -26,7 +26,7 @@ class AbortNodeTest(BitcoinTestFramework): # We'll connect the nodes later def run_test(self): - self.nodes[0].generate(3) + self.generate(self.nodes[0], 3) datadir = get_datadir_path(self.options.tmpdir, 0) # Deleting the undo file will result in reorg failure @@ -34,10 +34,10 @@ class AbortNodeTest(BitcoinTestFramework): # Connecting to a node with a more work chain will trigger a reorg # attempt. - self.nodes[1].generate(3) + self.generate(self.nodes[1], 3) with self.nodes[0].assert_debug_log(["Failed to disconnect block"]): self.connect_nodes(0, 1) - self.nodes[1].generate(1) + self.generate(self.nodes[1], 1) # Check that node0 aborted self.log.info("Waiting for crash") diff --git a/test/functional/feature_addrman.py b/test/functional/feature_addrman.py new file mode 100755 index 0000000000..93d50c1369 --- /dev/null +++ b/test/functional/feature_addrman.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test addrman functionality""" + +import os +import re +import struct + +from test_framework.messages import ser_uint256, hash256 +from test_framework.p2p import MAGIC_BYTES +from test_framework.test_framework import BitcoinTestFramework +from test_framework.test_node import ErrorMatch +from test_framework.util import assert_equal + + +def serialize_addrman( + *, + format=1, + lowest_compatible=3, + net_magic="regtest", + bucket_key=1, + len_new=None, + len_tried=None, + mock_checksum=None, +): + new = [] + tried = [] + INCOMPATIBILITY_BASE = 32 + r = MAGIC_BYTES[net_magic] + r += struct.pack("B", format) + r += struct.pack("B", INCOMPATIBILITY_BASE + lowest_compatible) + r += ser_uint256(bucket_key) + r += struct.pack("i", len_new or len(new)) + r += struct.pack("i", len_tried or len(tried)) + ADDRMAN_NEW_BUCKET_COUNT = 1 << 10 + r += struct.pack("i", ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30)) + for _ in range(ADDRMAN_NEW_BUCKET_COUNT): + r += struct.pack("i", 0) + checksum = hash256(r) + r += mock_checksum or checksum + return r + + +def write_addrman(peers_dat, **kwargs): + with open(peers_dat, "wb") as f: + f.write(serialize_addrman(**kwargs)) + + +class AddrmanTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def run_test(self): + peers_dat = os.path.join(self.nodes[0].datadir, self.chain, "peers.dat") + init_error = lambda reason: ( + f"Error: Invalid or corrupt peers.dat \\({reason}\\). If you believe this " + f"is a bug, please report it to {self.config['environment']['PACKAGE_BUGREPORT']}. " + f'As a workaround, you can move the file \\("{re.escape(peers_dat)}"\\) out of the way \\(rename, ' + "move, or delete\\) to have a new one created on the next start." + ) + + self.log.info("Check that mocked addrman is valid") + self.stop_node(0) + write_addrman(peers_dat) + with self.nodes[0].assert_debug_log(["Loaded 0 addresses from peers.dat"]): + self.start_node(0, extra_args=["-checkaddrman=1"]) + assert_equal(self.nodes[0].getnodeaddresses(), []) + + self.log.info("Check that addrman from future cannot be read") + self.stop_node(0) + write_addrman(peers_dat, lowest_compatible=111) + self.nodes[0].assert_start_raises_init_error( + expected_msg=init_error( + "Unsupported format of addrman database: 1. It is compatible with " + "formats >=111, but the maximum supported by this version of " + f"{self.config['environment']['PACKAGE_NAME']} is 3.: (.+)" + ), + match=ErrorMatch.FULL_REGEX, + ) + + self.log.info("Check that corrupt addrman cannot be read (EOF)") + self.stop_node(0) + with open(peers_dat, "wb") as f: + f.write(serialize_addrman()[:-1]) + self.nodes[0].assert_start_raises_init_error( + expected_msg=init_error("CAutoFile::read: end of file.*"), + match=ErrorMatch.FULL_REGEX, + ) + + self.log.info("Check that corrupt addrman cannot be read (magic)") + self.stop_node(0) + write_addrman(peers_dat, net_magic="signet") + self.nodes[0].assert_start_raises_init_error( + expected_msg=init_error("Invalid network magic number"), + match=ErrorMatch.FULL_REGEX, + ) + + self.log.info("Check that corrupt addrman cannot be read (checksum)") + self.stop_node(0) + write_addrman(peers_dat, mock_checksum=b"ab" * 32) + self.nodes[0].assert_start_raises_init_error( + expected_msg=init_error("Checksum mismatch, data corrupted"), + match=ErrorMatch.FULL_REGEX, + ) + + self.log.info("Check that corrupt addrman cannot be read (len_tried)") + self.stop_node(0) + write_addrman(peers_dat, len_tried=-1) + self.nodes[0].assert_start_raises_init_error( + expected_msg=init_error("Corrupt AddrMan serialization: nTried=-1, should be in \\[0, 16384\\]:.*"), + match=ErrorMatch.FULL_REGEX, + ) + + self.log.info("Check that corrupt addrman cannot be read (len_new)") + self.stop_node(0) + write_addrman(peers_dat, len_new=-1) + self.nodes[0].assert_start_raises_init_error( + expected_msg=init_error("Corrupt AddrMan serialization: nNew=-1, should be in \\[0, 65536\\]:.*"), + match=ErrorMatch.FULL_REGEX, + ) + + self.log.info("Check that corrupt addrman cannot be read (failed check)") + self.stop_node(0) + write_addrman(peers_dat, bucket_key=0) + self.nodes[0].assert_start_raises_init_error( + expected_msg=init_error("Corrupt data. Consistency check failed with code -16: .*"), + match=ErrorMatch.FULL_REGEX, + ) + + self.log.info("Check that missing addrman is recreated") + self.stop_node(0) + os.remove(peers_dat) + with self.nodes[0].assert_debug_log([ + f'Creating peers.dat because the file was not found ("{peers_dat}")', + ]): + self.start_node(0) + assert_equal(self.nodes[0].getnodeaddresses(), []) + + +if __name__ == "__main__": + AddrmanTest().main() diff --git a/test/functional/feature_anchors.py b/test/functional/feature_anchors.py index 24bb02bc90..7be393a4ea 100755 --- a/test/functional/feature_anchors.py +++ b/test/functional/feature_anchors.py @@ -23,9 +23,7 @@ def check_node_connections(*, node, num_in, num_out): class AnchorsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - - def setup_network(self): - self.setup_nodes() + self.disable_autoconnect = False def run_test(self): node_anchors_path = os.path.join( diff --git a/test/functional/feature_asmap.py b/test/functional/feature_asmap.py index 5fcecb4882..debd87962f 100755 --- a/test/functional/feature_asmap.py +++ b/test/functional/feature_asmap.py @@ -14,9 +14,11 @@ Verify node behaviour and debug log when launching bitcoind in these cases: 4. `bitcoind -asmap/-asmap=` with no file specified, using the default asmap -5. `bitcoind -asmap` with no file specified and a missing default asmap file +5. `bitcoind -asmap` restart with an addrman containing new and tried entries -6. `bitcoind -asmap` with an empty (unparsable) default asmap file +6. `bitcoind -asmap` with no file specified and a missing default asmap file + +7. `bitcoind -asmap` with an empty (unparsable) default asmap file The tests are order-independent. @@ -31,12 +33,18 @@ ASMAP = '../../src/test/data/asmap.raw' # path to unit test skeleton asmap VERSION = 'fec61fa21a9f46f3b17bdcd660d7f4cd90b966aad3aec593c99b35f0aca15853' def expected_messages(filename): - return ['Opened asmap file "{}" (59 bytes) from disk'.format(filename), - 'Using asmap version {} for IP bucketing'.format(VERSION)] + return [f'Opened asmap file "{filename}" (59 bytes) from disk', + f'Using asmap version {VERSION} for IP bucketing'] class AsmapTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 + self.extra_args = [["-checkaddrman=1"]] # Do addrman checks on all operations. + + def fill_addrman(self, node_id): + """Add 1 tried address to the addrman, followed by 1 new address.""" + for addr, tried in [[0, True], [1, False]]: + self.nodes[node_id].addpeeraddress(address=f"101.{addr}.0.0", tried=tried, port=8333) def test_without_asmap_arg(self): self.log.info('Test bitcoind with no -asmap arg passed') @@ -50,7 +58,7 @@ class AsmapTest(BitcoinTestFramework): filename = os.path.join(self.datadir, 'my-map-file.map') shutil.copyfile(self.asmap_raw, filename) with self.node.assert_debug_log(expected_messages(filename)): - self.start_node(0, ['-asmap={}'.format(filename)]) + self.start_node(0, [f'-asmap={filename}']) os.remove(filename) def test_asmap_with_relative_path(self): @@ -60,22 +68,38 @@ class AsmapTest(BitcoinTestFramework): filename = os.path.join(self.datadir, name) shutil.copyfile(self.asmap_raw, filename) with self.node.assert_debug_log(expected_messages(filename)): - self.start_node(0, ['-asmap={}'.format(name)]) + self.start_node(0, [f'-asmap={name}']) os.remove(filename) def test_default_asmap(self): shutil.copyfile(self.asmap_raw, self.default_asmap) for arg in ['-asmap', '-asmap=']: - self.log.info('Test bitcoind {} (using default map file)'.format(arg)) + self.log.info(f'Test bitcoind {arg} (using default map file)') self.stop_node(0) with self.node.assert_debug_log(expected_messages(self.default_asmap)): self.start_node(0, [arg]) os.remove(self.default_asmap) + def test_asmap_interaction_with_addrman_containing_entries(self): + self.log.info("Test bitcoind -asmap restart with addrman containing new and tried entries") + self.stop_node(0) + shutil.copyfile(self.asmap_raw, self.default_asmap) + self.start_node(0, ["-asmap", "-checkaddrman=1"]) + self.fill_addrman(node_id=0) + self.restart_node(0, ["-asmap", "-checkaddrman=1"]) + with self.node.assert_debug_log( + expected_msgs=[ + "Addrman checks started: new 1, tried 1, total 2", + "Addrman checks completed successfully", + ] + ): + self.node.getnodeaddresses() # getnodeaddresses re-runs the addrman checks + os.remove(self.default_asmap) + def test_default_asmap_with_missing_file(self): self.log.info('Test bitcoind -asmap with missing default map file') self.stop_node(0) - msg = "Error: Could not find asmap file \"{}\"".format(self.default_asmap) + msg = f"Error: Could not find asmap file \"{self.default_asmap}\"" self.node.assert_start_raises_init_error(extra_args=['-asmap'], expected_msg=msg) def test_empty_asmap(self): @@ -83,7 +107,7 @@ class AsmapTest(BitcoinTestFramework): self.stop_node(0) with open(self.default_asmap, "w", encoding="utf-8") as f: f.write("") - msg = "Error: Could not parse asmap file \"{}\"".format(self.default_asmap) + msg = f"Error: Could not parse asmap file \"{self.default_asmap}\"" self.node.assert_start_raises_init_error(extra_args=['-asmap'], expected_msg=msg) os.remove(self.default_asmap) @@ -97,6 +121,7 @@ class AsmapTest(BitcoinTestFramework): self.test_asmap_with_absolute_path() self.test_asmap_with_relative_path() self.test_default_asmap() + self.test_asmap_interaction_with_addrman_containing_entries() self.test_default_asmap_with_missing_file() self.test_empty_asmap() diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py index c712f7141c..e65525a023 100755 --- a/test/functional/feature_backwards_compatibility.py +++ b/test/functional/feature_backwards_compatibility.py @@ -4,9 +4,8 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Backwards compatibility functional test -Test various backwards compatibility scenarios. Download the previous node binaries: - -test/get_previous_releases.py -b v0.19.1 v0.18.1 v0.17.2 v0.16.3 v0.15.2 +Test various backwards compatibility scenarios. Requires previous releases binaries, +see test/README.md. v0.15.2 is not required by this test, but it is used in wallet_upgradewallet.py. Due to a hardfork in regtest, it can't be used to sync nodes. @@ -65,7 +64,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): self.import_deterministic_coinbase_privkeys() def run_test(self): - self.nodes[0].generatetoaddress(COINBASE_MATURITY + 1, self.nodes[0].getnewaddress()) + self.generatetoaddress(self.nodes[0], COINBASE_MATURITY + 1, self.nodes[0].getnewaddress()) self.sync_blocks() @@ -93,7 +92,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): address = wallet.getnewaddress() self.nodes[0].sendtoaddress(address, 10) self.sync_mempools() - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks() # Create a conflicting transaction using RBF return_address = self.nodes[0].getnewaddress() @@ -101,7 +100,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): tx2_id = self.nodes[1].bumpfee(tx1_id)["txid"] # Confirm the transaction self.sync_mempools() - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks() # Create another conflicting transaction using RBF tx3_id = self.nodes[1].sendtoaddress(return_address, 1) @@ -367,7 +366,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): assert_equal(load_res['warning'], '') wallet = node_master.get_wallet_rpc("u1_v16") info = wallet.getaddressinfo(v16_addr) - descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + v16_pubkey + ")" + descriptor = f"wpkh([{info['hdmasterfingerprint']}{hdkeypath[1:]}]{v16_pubkey})" assert_equal(info["desc"], descsum_create(descriptor)) # Now copy that same wallet back to 0.16 to make sure no automatic upgrade breaks it @@ -390,7 +389,7 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): node_master.loadwallet("u1_v17") wallet = node_master.get_wallet_rpc("u1_v17") info = wallet.getaddressinfo(address) - descriptor = "wpkh([" + info["hdmasterfingerprint"] + hdkeypath[1:] + "]" + pubkey + ")" + descriptor = f"wpkh([{info['hdmasterfingerprint']}{hdkeypath[1:]}]{pubkey})" assert_equal(info["desc"], descsum_create(descriptor)) # Now copy that same wallet back to 0.17 to make sure no automatic upgrade breaks it diff --git a/test/functional/feature_bind_extra.py b/test/functional/feature_bind_extra.py new file mode 100755 index 0000000000..6802da8d48 --- /dev/null +++ b/test/functional/feature_bind_extra.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test starting bitcoind with -bind and/or -bind=...=onion and confirm +that bind happens on the expected ports. +""" + +import sys + +from test_framework.netutil import ( + addr_to_hex, + get_bind_addrs, +) +from test_framework.test_framework import ( + BitcoinTestFramework, + SkipTest, +) +from test_framework.util import ( + PORT_MIN, + PORT_RANGE, + assert_equal, + rpc_port, +) + +class BindExtraTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + # Avoid any -bind= on the command line. Force the framework to avoid + # adding -bind=127.0.0.1. + self.bind_to_localhost_only = False + self.num_nodes = 2 + + def setup_network(self): + # Override setup_network() because we want to put the result of + # p2p_port() in self.extra_args[], before the nodes are started. + # p2p_port() is not usable in set_test_params() because PortSeed.n is + # not set at that time. + + # Due to OS-specific network stats queries, we only run on Linux. + self.log.info("Checking for Linux") + if not sys.platform.startswith('linux'): + raise SkipTest("This test can only be run on Linux.") + + loopback_ipv4 = addr_to_hex("127.0.0.1") + + # Start custom ports after p2p and rpc ports. + port = PORT_MIN + 2 * PORT_RANGE + + # Array of tuples [command line arguments, expected bind addresses]. + self.expected = [] + + # Node0, no normal -bind=... with -bind=...=onion, thus only the tor target. + self.expected.append( + [ + [f"-bind=127.0.0.1:{port}=onion"], + [(loopback_ipv4, port)] + ], + ) + port += 1 + + # Node1, both -bind=... and -bind=...=onion. + self.expected.append( + [ + [f"-bind=127.0.0.1:{port}", f"-bind=127.0.0.1:{port + 1}=onion"], + [(loopback_ipv4, port), (loopback_ipv4, port + 1)] + ], + ) + port += 2 + + self.extra_args = list(map(lambda e: e[0], self.expected)) + self.add_nodes(self.num_nodes, self.extra_args) + # Don't start the nodes, as some of them would collide trying to bind on the same port. + + def run_test(self): + for i in range(len(self.expected)): + self.log.info(f"Starting node {i} with {self.expected[i][0]}") + self.start_node(i) + pid = self.nodes[i].process.pid + binds = set(get_bind_addrs(pid)) + # Remove IPv6 addresses because on some CI environments "::1" is not configured + # on the system (so our test_ipv6_local() would return False), but it is + # possible to bind on "::". This makes it unpredictable whether to expect + # that bitcoind has bound on "::1" (for RPC) and "::" (for P2P). + ipv6_addr_len_bytes = 32 + binds = set(filter(lambda e: len(e[0]) != ipv6_addr_len_bytes, binds)) + # Remove RPC ports. They are not relevant for this test. + binds = set(filter(lambda e: e[1] != rpc_port(i), binds)) + assert_equal(binds, set(self.expected[i][1])) + self.stop_node(i) + self.log.info(f"Stopped node {i}") + +if __name__ == '__main__': + BindExtraTest().main() diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index e44ce9b57d..99ac1b5884 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -24,7 +24,6 @@ from test_framework.util import ( assert_equal, assert_greater_than, assert_raises_rpc_error, - satoshi_round, softfork_active, ) from test_framework.script_util import DUMMY_P2WPKH_SCRIPT @@ -42,10 +41,13 @@ class BIP68Test(BitcoinTestFramework): self.num_nodes = 2 self.extra_args = [ [ + '-testactivationheight=csv@432', "-acceptnonstdtxn=1", - "-peertimeout=9999", # bump because mocktime might cause a disconnect otherwise ], - ["-acceptnonstdtxn=0"], + [ + '-testactivationheight=csv@432', + "-acceptnonstdtxn=0", + ], ] def skip_test_if_missing_module(self): @@ -55,7 +57,7 @@ class BIP68Test(BitcoinTestFramework): self.relayfee = self.nodes[0].getnetworkinfo()["relayfee"] # Generate some coins - self.nodes[0].generate(110) + self.generate(self.nodes[0], 110) self.log.info("Running test disable flag") self.test_disable_flag() @@ -91,7 +93,7 @@ class BIP68Test(BitcoinTestFramework): utxo = utxos[0] tx1 = CTransaction() - value = int(satoshi_round(utxo["amount"] - self.relayfee)*COIN) + value = int((utxo["amount"] - self.relayfee) * COIN) # Check that the disable flag disables relative locktime. # If sequence locks were used, this would require 1 block for the @@ -143,7 +145,7 @@ class BIP68Test(BitcoinTestFramework): for i in range(num_outputs): outputs[addresses[i]] = random.randint(1, 20)*0.01 self.nodes[0].sendmany("", outputs) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) utxos = self.nodes[0].listunspent() @@ -273,7 +275,7 @@ class BIP68Test(BitcoinTestFramework): cur_time = int(time.time()) for _ in range(10): self.nodes[0].setmocktime(cur_time + 600) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) cur_time += 600 assert tx2.hash in self.nodes[0].getrawmempool() @@ -288,7 +290,7 @@ class BIP68Test(BitcoinTestFramework): self.nodes[0].setmocktime(cur_time+600) # Save block template now to use for the reorg later tmpl = self.nodes[0].getblocktemplate(NORMAL_GBT_REQUEST_PARAMS) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) assert tx2.hash not in self.nodes[0].getrawmempool() # Now that tx2 is not in the mempool, a sequence locked spend should @@ -296,7 +298,7 @@ class BIP68Test(BitcoinTestFramework): tx3 = test_nonzero_locks(tx2, self.nodes[0], self.relayfee, use_height_lock=False) assert tx3.hash in self.nodes[0].getrawmempool() - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) assert tx3.hash not in self.nodes[0].getrawmempool() # One more test, this time using height locks @@ -349,7 +351,7 @@ class BIP68Test(BitcoinTestFramework): # Reset the chain and get rid of the mocktimed-blocks self.nodes[0].setmocktime(0) self.nodes[0].invalidateblock(self.nodes[0].getblockhash(cur_height+1)) - self.nodes[0].generate(10) + self.generate(self.nodes[0], 10) # Make sure that BIP68 isn't being used to validate blocks prior to # activation height. If more blocks are mined prior to this test @@ -403,9 +405,9 @@ class BIP68Test(BitcoinTestFramework): min_activation_height = 432 height = self.nodes[0].getblockcount() assert_greater_than(min_activation_height - height, 2) - self.nodes[0].generate(min_activation_height - height - 2) + self.generate(self.nodes[0], min_activation_height - height - 2) assert not softfork_active(self.nodes[0], 'csv') - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) assert softfork_active(self.nodes[0], 'csv') self.sync_blocks() diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index 389db73bc9..b06ea8542b 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -22,7 +22,7 @@ from test_framework.messages import ( CTransaction, CTxIn, CTxOut, - MAX_BLOCK_BASE_SIZE, + MAX_BLOCK_WEIGHT, uint256_from_compact, uint256_from_str, ) @@ -37,17 +37,17 @@ from test_framework.script import ( OP_CHECKSIGVERIFY, OP_ELSE, OP_ENDIF, - OP_EQUAL, OP_DROP, OP_FALSE, - OP_HASH160, OP_IF, OP_INVALIDOPCODE, OP_RETURN, OP_TRUE, SIGHASH_ALL, LegacySignatureHash, - hash160, +) +from test_framework.script_util import ( + script_to_p2sh_script, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal @@ -82,7 +82,10 @@ class FullBlockTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True - self.extra_args = [['-acceptnonstdtxn=1']] # This is a consensus block test, we don't care about tx policy + self.extra_args = [[ + '-acceptnonstdtxn=1', # This is a consensus block test, we don't care about tx policy + '-testactivationheight=bip34@2', + ]] def run_test(self): node = self.nodes[0] # convenience reference to the node @@ -307,33 +310,33 @@ class FullBlockTest(BitcoinTestFramework): b22 = self.next_block(22, spend=out[5]) self.send_blocks([b22], success=False, reject_reason='bad-txns-premature-spend-of-coinbase', reconnect=True) - # Create a block on either side of MAX_BLOCK_BASE_SIZE and make sure its accepted/rejected + # Create a block on either side of MAX_BLOCK_WEIGHT and make sure its accepted/rejected # genesis -> b1 (0) -> b2 (1) -> b5 (2) -> b6 (3) # \-> b12 (3) -> b13 (4) -> b15 (5) -> b23 (6) # \-> b24 (6) -> b25 (7) # \-> b3 (1) -> b4 (2) - self.log.info("Accept a block of size MAX_BLOCK_BASE_SIZE") + self.log.info("Accept a block of weight MAX_BLOCK_WEIGHT") self.move_tip(15) b23 = self.next_block(23, spend=out[6]) tx = CTransaction() - script_length = MAX_BLOCK_BASE_SIZE - len(b23.serialize()) - 69 + script_length = (MAX_BLOCK_WEIGHT - b23.get_weight() - 276) // 4 script_output = CScript([b'\x00' * script_length]) tx.vout.append(CTxOut(0, script_output)) tx.vin.append(CTxIn(COutPoint(b23.vtx[1].sha256, 0))) b23 = self.update_block(23, [tx]) - # Make sure the math above worked out to produce a max-sized block - assert_equal(len(b23.serialize()), MAX_BLOCK_BASE_SIZE) + # Make sure the math above worked out to produce a max-weighted block + assert_equal(b23.get_weight(), MAX_BLOCK_WEIGHT) self.send_blocks([b23], True) self.save_spendable_output() - self.log.info("Reject a block of size MAX_BLOCK_BASE_SIZE + 1") + self.log.info("Reject a block of weight MAX_BLOCK_WEIGHT + 4") self.move_tip(15) b24 = self.next_block(24, spend=out[6]) - script_length = MAX_BLOCK_BASE_SIZE - len(b24.serialize()) - 69 + script_length = (MAX_BLOCK_WEIGHT - b24.get_weight() - 276) // 4 script_output = CScript([b'\x00' * (script_length + 1)]) tx.vout = [CTxOut(0, script_output)] b24 = self.update_block(24, [tx]) - assert_equal(len(b24.serialize()), MAX_BLOCK_BASE_SIZE + 1) + assert_equal(b24.get_weight(), MAX_BLOCK_WEIGHT + 1 * 4) self.send_blocks([b24], success=False, reject_reason='bad-blk-length', reconnect=True) b25 = self.next_block(25, spend=out[7]) @@ -373,7 +376,9 @@ class FullBlockTest(BitcoinTestFramework): # b30 has a max-sized coinbase scriptSig. self.move_tip(23) b30 = self.next_block(30) - b30.vtx[0].vin[0].scriptSig = b'\x00' * 100 + b30.vtx[0].vin[0].scriptSig = bytes(b30.vtx[0].vin[0].scriptSig) # Convert CScript to raw bytes + b30.vtx[0].vin[0].scriptSig += b'\x00' * (100 - len(b30.vtx[0].vin[0].scriptSig)) # Fill with 0s + assert_equal(len(b30.vtx[0].vin[0].scriptSig), 100) b30.vtx[0].rehash() b30 = self.update_block(30, []) self.send_blocks([b30], True) @@ -469,8 +474,7 @@ class FullBlockTest(BitcoinTestFramework): # Build the redeem script, hash it, use hash to create the p2sh script redeem_script = CScript([self.coinbase_pubkey] + [OP_2DUP, OP_CHECKSIGVERIFY] * 5 + [OP_CHECKSIG]) - redeem_script_hash = hash160(redeem_script) - p2sh_script = CScript([OP_HASH160, redeem_script_hash, OP_EQUAL]) + p2sh_script = script_to_p2sh_script(redeem_script) # Create a transaction that spends one satoshi to the p2sh_script, the rest to OP_TRUE # This must be signed because it is spending a coinbase @@ -485,13 +489,13 @@ class FullBlockTest(BitcoinTestFramework): # Until block is full, add tx's with 1 satoshi to p2sh_script, the rest to OP_TRUE tx_new = None tx_last = tx - total_size = len(b39.serialize()) - while(total_size < MAX_BLOCK_BASE_SIZE): + total_weight = b39.get_weight() + while total_weight < MAX_BLOCK_WEIGHT: tx_new = self.create_tx(tx_last, 1, 1, p2sh_script) tx_new.vout.append(CTxOut(tx_last.vout[1].nValue - 1, CScript([OP_TRUE]))) tx_new.rehash() - total_size += len(tx_new.serialize()) - if total_size >= MAX_BLOCK_BASE_SIZE: + total_weight += tx_new.get_weight() + if total_weight >= MAX_BLOCK_WEIGHT: break b39.vtx.append(tx_new) # add tx to block tx_last = tx_new @@ -502,7 +506,7 @@ class FullBlockTest(BitcoinTestFramework): # Make sure we didn't accidentally make too big a block. Note that the # size of the block has non-determinism due to the ECDSA signature in # the first transaction. - while (len(b39.serialize()) >= MAX_BLOCK_BASE_SIZE): + while b39.get_weight() >= MAX_BLOCK_WEIGHT: del b39.vtx[-1] b39 = self.update_block(39, []) @@ -834,6 +838,7 @@ class FullBlockTest(BitcoinTestFramework): b61.vtx[0].rehash() b61 = self.update_block(61, []) assert_equal(duplicate_tx.serialize(), b61.vtx[0].serialize()) + # BIP30 is always checked on regtest, regardless of the BIP34 activation height self.send_blocks([b61], success=False, reject_reason='bad-txns-BIP30', reconnect=True) # Test BIP30 (allow duplicate if spent) @@ -892,7 +897,7 @@ class FullBlockTest(BitcoinTestFramework): self.send_blocks([b63], success=False, reject_reason='bad-txns-nonfinal', reconnect=True) # This checks that a block with a bloated VARINT between the block_header and the array of tx such that - # the block is > MAX_BLOCK_BASE_SIZE with the bloated varint, but <= MAX_BLOCK_BASE_SIZE without the bloated varint, + # the block is > MAX_BLOCK_WEIGHT with the bloated varint, but <= MAX_BLOCK_WEIGHT without the bloated varint, # does not cause a subsequent, identical block with canonical encoding to be rejected. The test does not # care whether the bloated block is accepted or rejected; it only cares that the second block is accepted. # @@ -917,12 +922,12 @@ class FullBlockTest(BitcoinTestFramework): tx = CTransaction() # use canonical serialization to calculate size - script_length = MAX_BLOCK_BASE_SIZE - len(b64a.normal_serialize()) - 69 + script_length = (MAX_BLOCK_WEIGHT - 4 * len(b64a.normal_serialize()) - 276) // 4 script_output = CScript([b'\x00' * script_length]) tx.vout.append(CTxOut(0, script_output)) tx.vin.append(CTxIn(COutPoint(b64a.vtx[1].sha256, 0))) b64a = self.update_block("64a", [tx]) - assert_equal(len(b64a.serialize()), MAX_BLOCK_BASE_SIZE + 8) + assert_equal(b64a.get_weight(), MAX_BLOCK_WEIGHT + 8 * 4) self.send_blocks([b64a], success=False, reject_reason='non-canonical ReadCompactSize()') # bitcoind doesn't disconnect us for sending a bloated block, but if we subsequently @@ -936,7 +941,7 @@ class FullBlockTest(BitcoinTestFramework): b64 = CBlock(b64a) b64.vtx = copy.deepcopy(b64a.vtx) assert_equal(b64.hash, b64a.hash) - assert_equal(len(b64.serialize()), MAX_BLOCK_BASE_SIZE) + assert_equal(b64.get_weight(), MAX_BLOCK_WEIGHT) self.blocks[64] = b64 b64 = self.update_block(64, []) self.send_blocks([b64], True) @@ -1270,12 +1275,12 @@ class FullBlockTest(BitcoinTestFramework): for i in range(89, LARGE_REORG_SIZE + 89): b = self.next_block(i, spend) tx = CTransaction() - script_length = MAX_BLOCK_BASE_SIZE - len(b.serialize()) - 69 + script_length = (MAX_BLOCK_WEIGHT - b.get_weight() - 276) // 4 script_output = CScript([b'\x00' * script_length]) tx.vout.append(CTxOut(0, script_output)) tx.vin.append(CTxIn(COutPoint(b.vtx[1].sha256, 0))) b = self.update_block(i, [tx]) - assert_equal(len(b.serialize()), MAX_BLOCK_BASE_SIZE) + assert_equal(b.get_weight(), MAX_BLOCK_WEIGHT) blocks.append(b) self.save_spendable_output() spend = self.get_spendable_output() diff --git a/test/functional/feature_blockfilterindex_prune.py b/test/functional/feature_blockfilterindex_prune.py index 28d8f2fbbc..b740f2cc27 100755 --- a/test/functional/feature_blockfilterindex_prune.py +++ b/test/functional/feature_blockfilterindex_prune.py @@ -25,9 +25,9 @@ class FeatureBlockfilterindexPruneTest(BitcoinTestFramework): self.sync_index(height=200) assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getbestblockhash())['filter']), 0) # Mine two batches of blocks to avoid hitting NODE_NETWORK_LIMITED_MIN_BLOCKS disconnection - self.nodes[0].generate(250) + self.generate(self.nodes[0], 250) self.sync_all() - self.nodes[0].generate(250) + self.generate(self.nodes[0], 250) self.sync_all() self.sync_index(height=700) @@ -46,7 +46,7 @@ class FeatureBlockfilterindexPruneTest(BitcoinTestFramework): self.log.info("make sure accessing the blockfilters throws an error") assert_raises_rpc_error(-1, "Index is not enabled for filtertype basic", self.nodes[0].getblockfilter, self.nodes[0].getblockhash(2)) - self.nodes[0].generate(1000) + self.generate(self.nodes[0], 1000) self.log.info("prune below the blockfilterindexes best block while blockfilters are disabled") pruneheight_new = self.nodes[0].pruneblockchain(1000) diff --git a/test/functional/feature_blocksdir.py b/test/functional/feature_blocksdir.py index 7bfad52c24..28e6d6cdf9 100755 --- a/test/functional/feature_blocksdir.py +++ b/test/functional/feature_blocksdir.py @@ -24,12 +24,12 @@ class BlocksdirTest(BitcoinTestFramework): initialize_datadir(self.options.tmpdir, 0, self.chain) self.log.info("Starting with nonexistent blocksdir ...") blocksdir_path = os.path.join(self.options.tmpdir, 'blocksdir') - self.nodes[0].assert_start_raises_init_error(["-blocksdir=" + blocksdir_path], 'Error: Specified blocks directory "{}" does not exist.'.format(blocksdir_path)) + self.nodes[0].assert_start_raises_init_error([f"-blocksdir={blocksdir_path}"], f'Error: Specified blocks directory "{blocksdir_path}" does not exist.') os.mkdir(blocksdir_path) self.log.info("Starting with existing blocksdir ...") - self.start_node(0, ["-blocksdir=" + blocksdir_path]) + self.start_node(0, [f"-blocksdir={blocksdir_path}"]) self.log.info("mining blocks..") - self.nodes[0].generatetoaddress(10, self.nodes[0].get_deterministic_priv_key().address) + self.generatetoaddress(self.nodes[0], 10, self.nodes[0].get_deterministic_priv_key().address) assert os.path.isfile(os.path.join(blocksdir_path, self.chain, "blocks", "blk00000.dat")) assert os.path.isdir(os.path.join(self.nodes[0].datadir, self.chain, "blocks", "index")) diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index 10d2072dba..3dc858f5d2 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -4,8 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP65 (CHECKLOCKTIMEVERIFY). -Test that the CHECKLOCKTIMEVERIFY soft-fork activates at (regtest) block height -1351. +Test that the CHECKLOCKTIMEVERIFY soft-fork activates. """ from test_framework.blocktools import ( @@ -31,8 +30,6 @@ from test_framework.wallet import ( MiniWalletMode, ) -CLTV_HEIGHT = 1351 - # Helper function to modify a transaction by # 1) prepending a given script to the scriptSig of vin 0 and @@ -63,9 +60,9 @@ def cltv_invalidate(tx, failure_reason): # +-------------------------------------------------+------------+--------------+ [[OP_CHECKLOCKTIMEVERIFY], None, None], [[OP_1NEGATE, OP_CHECKLOCKTIMEVERIFY, OP_DROP], None, None], - [[CScriptNum(1000), OP_CHECKLOCKTIMEVERIFY, OP_DROP], 0, 1296688602], # timestamp of genesis block - [[CScriptNum(1000), OP_CHECKLOCKTIMEVERIFY, OP_DROP], 0, 500], - [[CScriptNum(500), OP_CHECKLOCKTIMEVERIFY, OP_DROP], 0xffffffff, 500], + [[CScriptNum(100), OP_CHECKLOCKTIMEVERIFY, OP_DROP], 0, 1296688602], # timestamp of genesis block + [[CScriptNum(100), OP_CHECKLOCKTIMEVERIFY, OP_DROP], 0, 50], + [[CScriptNum(50), OP_CHECKLOCKTIMEVERIFY, OP_DROP], 0xffffffff, 50], ][failure_reason] cltv_modify_tx(tx, prepend_scriptsig=scheme[0], nsequence=scheme[1], nlocktime=scheme[2]) @@ -78,10 +75,14 @@ def cltv_validate(tx, height): cltv_modify_tx(tx, prepend_scriptsig=scheme[0], nsequence=scheme[1], nlocktime=scheme[2]) +CLTV_HEIGHT = 111 + + class BIP65Test(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.extra_args = [[ + f'-testactivationheight=cltv@{CLTV_HEIGHT}', '-whitelist=noban@127.0.0.1', '-par=1', # Use only one script thread to get the exact reject reason for testing '-acceptnonstdtxn=1', # cltv_invalidate is nonstandard @@ -104,8 +105,9 @@ class BIP65Test(BitcoinTestFramework): self.test_cltv_info(is_active=False) self.log.info("Mining %d blocks", CLTV_HEIGHT - 2) - wallet.generate(10) - self.nodes[0].generate(CLTV_HEIGHT - 2 - 10) + self.generate(wallet, 10) + self.generate(self.nodes[0], CLTV_HEIGHT - 2 - 10) + assert_equal(self.nodes[0].getblockcount(), CLTV_HEIGHT - 2) self.log.info("Test that invalid-according-to-CLTV transactions can still appear in a block") @@ -136,7 +138,7 @@ class BIP65Test(BitcoinTestFramework): block.nVersion = 3 block.solve() - with self.nodes[0].assert_debug_log(expected_msgs=['{}, bad-version(0x00000003)'.format(block.hash)]): + with self.nodes[0].assert_debug_log(expected_msgs=[f'{block.hash}, bad-version(0x00000003)']): peer.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) peer.sync_with_ping() @@ -174,8 +176,7 @@ class BIP65Test(BitcoinTestFramework): block.hashMerkleRoot = block.calc_merkle_root() block.solve() - with self.nodes[0].assert_debug_log(expected_msgs=['CheckInputScripts on {} failed with {}'.format( - block.vtx[-1].hash, expected_cltv_reject_reason)]): + with self.nodes[0].assert_debug_log(expected_msgs=[f'CheckInputScripts on {block.vtx[-1].hash} failed with {expected_cltv_reject_reason}']): peer.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) peer.sync_with_ping() diff --git a/test/functional/feature_coinstatsindex.py b/test/functional/feature_coinstatsindex.py index 5d8ec2a8da..c592d7bd69 100755 --- a/test/functional/feature_coinstatsindex.py +++ b/test/functional/feature_coinstatsindex.py @@ -32,7 +32,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, assert_raises_rpc_error, - try_rpc, ) class CoinStatsIndexTest(BitcoinTestFramework): @@ -68,21 +67,19 @@ class CoinStatsIndexTest(BitcoinTestFramework): index_hash_options = ['none', 'muhash'] # Generate a normal transaction and mine it - node.generate(COINBASE_MATURITY + 1) + self.generate(node, COINBASE_MATURITY + 1) address = self.nodes[0].get_deterministic_priv_key().address node.sendtoaddress(address=address, amount=10, subtractfeefromamount=True) - node.generate(1) + self.generate(node, 1) self.sync_blocks(timeout=120) self.log.info("Test that gettxoutsetinfo() output is consistent with or without coinstatsindex option") - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", node.gettxoutsetinfo)) res0 = node.gettxoutsetinfo('none') # The fields 'disk_size' and 'transactions' do not exist on the index del res0['disk_size'], res0['transactions'] - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) for hash_option in index_hash_options: res1 = index_node.gettxoutsetinfo(hash_option) # The fields 'block_info' and 'total_unspendable_amount' only exist on the index @@ -95,9 +92,8 @@ class CoinStatsIndexTest(BitcoinTestFramework): self.log.info("Test that gettxoutsetinfo() can get fetch data on specific heights with index") # Generate a new tip - node.generate(5) + self.generate(node, 5) - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) for hash_option in index_hash_options: # Fetch old stats by height res2 = index_node.gettxoutsetinfo(hash_option, 102) @@ -168,28 +164,27 @@ class CoinStatsIndexTest(BitcoinTestFramework): # Generate and send another tx with an OP_RETURN output (which is unspendable) tx2 = CTransaction() tx2.vin.append(CTxIn(COutPoint(int(tx1_txid, 16), n), b'')) - tx2.vout.append(CTxOut(int(20.99 * COIN), CScript([OP_RETURN] + [OP_FALSE]*30))) + tx2.vout.append(CTxOut(int(Decimal('20.99') * COIN), CScript([OP_RETURN] + [OP_FALSE]*30))) tx2_hex = self.nodes[0].signrawtransactionwithwallet(tx2.serialize().hex())['hex'] self.nodes[0].sendrawtransaction(tx2_hex) # Include both txs in a block - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) for hash_option in index_hash_options: # Check all amounts were registered correctly res6 = index_node.gettxoutsetinfo(hash_option, 108) - assert_equal(res6['total_unspendable_amount'], Decimal('70.98999999')) + assert_equal(res6['total_unspendable_amount'], Decimal('70.99000000')) assert_equal(res6['block_info'], { - 'unspendable': Decimal('20.98999999'), + 'unspendable': Decimal('20.99000000'), 'prevout_spent': 111, 'new_outputs_ex_coinbase': Decimal('89.99993620'), - 'coinbase': Decimal('50.01006381'), + 'coinbase': Decimal('50.01006380'), 'unspendables': { 'genesis_block': 0, 'bip30': 0, - 'scripts': Decimal('20.98999999'), + 'scripts': Decimal('20.99000000'), 'unclaimed_rewards': 0 } }) @@ -209,10 +204,9 @@ class CoinStatsIndexTest(BitcoinTestFramework): self.nodes[0].submitblock(block.serialize().hex()) self.sync_all() - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) for hash_option in index_hash_options: res7 = index_node.gettxoutsetinfo(hash_option, 109) - assert_equal(res7['total_unspendable_amount'], Decimal('80.98999999')) + assert_equal(res7['total_unspendable_amount'], Decimal('80.99000000')) assert_equal(res7['block_info'], { 'unspendable': 10, 'prevout_spent': 0, @@ -234,8 +228,7 @@ class CoinStatsIndexTest(BitcoinTestFramework): res9 = index_node.gettxoutsetinfo('muhash') assert_equal(res8, res9) - index_node.generate(1) - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) + self.generate(index_node, 1) res10 = index_node.gettxoutsetinfo('muhash') assert(res8['txouts'] < res10['txouts']) @@ -254,16 +247,14 @@ class CoinStatsIndexTest(BitcoinTestFramework): # Generate two block, let the index catch up, then invalidate the blocks index_node = self.nodes[1] - reorg_blocks = index_node.generatetoaddress(2, index_node.getnewaddress()) + reorg_blocks = self.generatetoaddress(index_node, 2, index_node.getnewaddress()) reorg_block = reorg_blocks[1] - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) res_invalid = index_node.gettxoutsetinfo('muhash') index_node.invalidateblock(reorg_blocks[0]) assert_equal(index_node.gettxoutsetinfo('muhash')['height'], 110) # Add two new blocks - block = index_node.generate(2)[1] - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) + block = self.generate(index_node, 2)[1] res = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=None, use_index=False) # Test that the result of the reorged block is not returned for its old block height @@ -279,15 +270,13 @@ class CoinStatsIndexTest(BitcoinTestFramework): # Add another block, so we don't depend on reconsiderblock remembering which # blocks were touched by invalidateblock - index_node.generate(1) + self.generate(index_node, 1) self.sync_all() # Ensure that removing and re-adding blocks yields consistent results block = index_node.getblockhash(99) index_node.invalidateblock(block) - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) index_node.reconsiderblock(block) - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", index_node.gettxoutsetinfo, 'muhash')) res3 = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=112) assert_equal(res2, res3) @@ -297,8 +286,7 @@ class CoinStatsIndexTest(BitcoinTestFramework): node.getblock(reorg_block) self.restart_node(0, ["-coinstatsindex"]) - self.wait_until(lambda: not try_rpc(-32603, "Unable to read UTXO set", node.gettxoutsetinfo, 'muhash')) - assert_raises_rpc_error(-32603, "Unable to read UTXO set", node.gettxoutsetinfo, 'muhash', reorg_block) + assert_raises_rpc_error(-32603, "Unable to get data because coinstatsindex is still syncing.", node.gettxoutsetinfo, 'muhash', reorg_block) def _test_index_rejects_hash_serialized(self): self.log.info("Test that the rpc raises if the legacy hash is passed with the index") diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index de9d0d2e80..3d9d8b7441 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -17,13 +17,14 @@ class ConfArgsTest(BitcoinTestFramework): self.num_nodes = 1 self.supports_cli = False self.wallet_names = [] + self.disable_autoconnect = False def test_config_file_parser(self): self.stop_node(0) inc_conf_file_path = os.path.join(self.nodes[0].datadir, 'include.conf') with open(os.path.join(self.nodes[0].datadir, 'bitcoin.conf'), 'a', encoding='utf-8') as conf: - conf.write('includeconf={}\n'.format(inc_conf_file_path)) + conf.write(f'includeconf={inc_conf_file_path}\n') self.nodes[0].assert_start_raises_init_error( expected_msg='Error: Error parsing command line arguments: Invalid parameter -dash_cli=1', @@ -42,13 +43,13 @@ class ConfArgsTest(BitcoinTestFramework): if self.is_wallet_compiled(): with open(inc_conf_file_path, 'w', encoding='utf8') as conf: conf.write("wallet=foo\n") - self.nodes[0].assert_start_raises_init_error(expected_msg='Error: Config setting for -wallet only applied on %s network when in [%s] section.' % (self.chain, self.chain)) + self.nodes[0].assert_start_raises_init_error(expected_msg=f'Error: Config setting for -wallet only applied on {self.chain} network when in [{self.chain}] section.') main_conf_file_path = os.path.join(self.options.tmpdir, 'node0', 'bitcoin_main.conf') - util.write_config(main_conf_file_path, n=0, chain='', extra_config='includeconf={}\n'.format(inc_conf_file_path)) + util.write_config(main_conf_file_path, n=0, chain='', extra_config=f'includeconf={inc_conf_file_path}\n') with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('acceptnonstdtxn=1\n') - self.nodes[0].assert_start_raises_init_error(extra_args=["-conf={}".format(main_conf_file_path)], expected_msg='Error: acceptnonstdtxn is not currently supported for main chain') + self.nodes[0].assert_start_raises_init_error(extra_args=[f"-conf={main_conf_file_path}"], expected_msg='Error: acceptnonstdtxn is not currently supported for main chain') with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('nono\n') @@ -68,14 +69,14 @@ class ConfArgsTest(BitcoinTestFramework): inc_conf_file2_path = os.path.join(self.nodes[0].datadir, 'include2.conf') with open(os.path.join(self.nodes[0].datadir, 'bitcoin.conf'), 'a', encoding='utf-8') as conf: - conf.write('includeconf={}\n'.format(inc_conf_file2_path)) + conf.write(f'includeconf={inc_conf_file2_path}\n') with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('testnot.datadir=1\n') with open(inc_conf_file2_path, 'w', encoding='utf-8') as conf: conf.write('[testnet]\n') self.restart_node(0) - self.nodes[0].stop_node(expected_stderr='Warning: ' + inc_conf_file_path + ':1 Section [testnot] is not recognized.' + os.linesep + inc_conf_file2_path + ':1 Section [testnet] is not recognized.') + self.nodes[0].stop_node(expected_stderr=f'Warning: {inc_conf_file_path}:1 Section [testnot] is not recognized.{os.linesep}{inc_conf_file2_path}:1 Section [testnet] is not recognized.') with open(inc_conf_file_path, 'w', encoding='utf-8') as conf: conf.write('') # clear @@ -104,8 +105,8 @@ class ConfArgsTest(BitcoinTestFramework): 'Command-line arg: rpcpassword=****', 'Command-line arg: rpcuser=****', 'Command-line arg: torpassword=****', - 'Config file arg: %s="1"' % self.chain, - 'Config file arg: [%s] server="1"' % self.chain, + f'Config file arg: {self.chain}="1"', + f'Config file arg: [{self.chain}] server="1"', ], unexpected_msgs=[ 'alice:f7efda5c189b999524f151318c0c86$d5b51b3beffbc0', @@ -158,15 +159,19 @@ class ConfArgsTest(BitcoinTestFramework): self.stop_node(0) # No peers.dat exists and -dnsseed=1 - # We expect the node will use DNS Seeds, but Regtest mode has 0 DNS seeds - # So after 60 seconds, the node should fallback to fixed seeds (this is a slow test) + # We expect the node will use DNS Seeds, but Regtest mode does not have + # any valid DNS seeds. So after 60 seconds, the node should fallback to + # fixed seeds assert not os.path.exists(os.path.join(default_data_dir, "peers.dat")) start = int(time.time()) - with self.nodes[0].assert_debug_log(expected_msgs=[ - "Loaded 0 addresses from peers.dat", - "0 addresses found from DNS seeds", - "opencon thread start", # Ensure ThreadOpenConnections::start time is properly set - ]): + with self.nodes[0].assert_debug_log( + expected_msgs=[ + "Loaded 0 addresses from peers.dat", + "0 addresses found from DNS seeds", + "opencon thread start", # Ensure ThreadOpenConnections::start time is properly set + ], + timeout=10, + ): self.start_node(0, extra_args=['-dnsseed=1', '-fixedseeds=1', f'-mocktime={start}']) with self.nodes[0].assert_debug_log(expected_msgs=[ "Adding fixed seeds as 60 seconds have passed and addrman is empty", @@ -204,11 +209,14 @@ class ConfArgsTest(BitcoinTestFramework): # We expect the node will allow 60 seconds prior to using fixed seeds assert not os.path.exists(os.path.join(default_data_dir, "peers.dat")) start = int(time.time()) - with self.nodes[0].assert_debug_log(expected_msgs=[ - "Loaded 0 addresses from peers.dat", - "DNS seeding disabled", - "opencon thread start", # Ensure ThreadOpenConnections::start time is properly set - ]): + with self.nodes[0].assert_debug_log( + expected_msgs=[ + "Loaded 0 addresses from peers.dat", + "DNS seeding disabled", + "opencon thread start", # Ensure ThreadOpenConnections::start time is properly set + ], + timeout=10, + ): self.start_node(0, extra_args=['-dnsseed=0', '-fixedseeds=1', '-addnode=fakenodeaddr', f'-mocktime={start}']) with self.nodes[0].assert_debug_log(expected_msgs=[ "Adding fixed seeds as 60 seconds have passed and addrman is empty", @@ -233,7 +241,7 @@ class ConfArgsTest(BitcoinTestFramework): # Check that using -datadir argument on non-existent directory fails self.nodes[0].datadir = new_data_dir - self.nodes[0].assert_start_raises_init_error(['-datadir=' + new_data_dir], 'Error: Specified data directory "' + new_data_dir + '" does not exist.') + self.nodes[0].assert_start_raises_init_error([f'-datadir={new_data_dir}'], f'Error: Specified data directory "{new_data_dir}" does not exist.') # Check that using non-existent datadir in conf file fails conf_file = os.path.join(default_data_dir, "bitcoin.conf") @@ -241,21 +249,25 @@ class ConfArgsTest(BitcoinTestFramework): # datadir needs to be set before [chain] section conf_file_contents = open(conf_file, encoding='utf8').read() with open(conf_file, 'w', encoding='utf8') as f: - f.write("datadir=" + new_data_dir + "\n") + f.write(f"datadir={new_data_dir}\n") f.write(conf_file_contents) - self.nodes[0].assert_start_raises_init_error(['-conf=' + conf_file], 'Error: Error reading configuration file: specified data directory "' + new_data_dir + '" does not exist.') + self.nodes[0].assert_start_raises_init_error([f'-conf={conf_file}'], f'Error: Error reading configuration file: specified data directory "{new_data_dir}" does not exist.') + + # Check that an explicitly specified config file that cannot be opened fails + none_existent_conf_file = os.path.join(default_data_dir, "none_existent_bitcoin.conf") + self.nodes[0].assert_start_raises_init_error(['-conf=' + none_existent_conf_file], 'Error: Error reading configuration file: specified config file "' + none_existent_conf_file + '" could not be opened.') # Create the directory and ensure the config file now works os.mkdir(new_data_dir) - self.start_node(0, ['-conf='+conf_file]) + self.start_node(0, [f'-conf={conf_file}']) self.stop_node(0) assert os.path.exists(os.path.join(new_data_dir, self.chain, 'blocks')) # Ensure command line argument overrides datadir in conf os.mkdir(new_data_dir_2) self.nodes[0].datadir = new_data_dir_2 - self.start_node(0, ['-datadir='+new_data_dir_2, '-conf='+conf_file]) + self.start_node(0, [f'-datadir={new_data_dir_2}', f'-conf={conf_file}']) assert os.path.exists(os.path.join(new_data_dir_2, self.chain, 'blocks')) diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index 5081867319..5255b13bd1 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -63,12 +63,12 @@ from test_framework.wallet import ( TESTING_TX_COUNT = 83 # Number of testing transactions: 1 BIP113 tx, 16 BIP68 txs, 66 BIP112 txs (see comments above) COINBASE_BLOCK_COUNT = TESTING_TX_COUNT # Number of coinbase blocks we need to generate as inputs for our txs BASE_RELATIVE_LOCKTIME = 10 -CSV_ACTIVATION_HEIGHT = 432 SEQ_DISABLE_FLAG = 1 << 31 SEQ_RANDOM_HIGH_BIT = 1 << 25 SEQ_TYPE_FLAG = 1 << 22 SEQ_RANDOM_LOW_BIT = 1 << 18 + def relative_locktime(sdf, srhb, stf, srlb): """Returns a locktime with certain bits set.""" @@ -83,16 +83,21 @@ def relative_locktime(sdf, srhb, stf, srlb): locktime |= SEQ_RANDOM_LOW_BIT return locktime + def all_rlt_txs(txs): return [tx['tx'] for tx in txs] +CSV_ACTIVATION_HEIGHT = 432 + + class BIP68_112_113Test(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True self.extra_args = [[ '-whitelist=noban@127.0.0.1', + f'-testactivationheight=csv@{CSV_ACTIVATION_HEIGHT}', '-par=1', # Use only one script thread to get the exact reject reason for testing ]] self.supports_cli = False @@ -143,13 +148,13 @@ class BIP68_112_113Test(BitcoinTestFramework): for i, (sdf, srhb, stf, srlb) in enumerate(product(*[[True, False]] * 4)): locktime = relative_locktime(sdf, srhb, stf, srlb) tx = self.create_self_transfer_from_utxo(bip112inputs[i]) - if (varyOP_CSV): # if varying OP_CSV, nSequence is fixed + if varyOP_CSV: # if varying OP_CSV, nSequence is fixed tx.vin[0].nSequence = BASE_RELATIVE_LOCKTIME + locktime_delta else: # vary nSequence instead, OP_CSV is fixed tx.vin[0].nSequence = locktime + locktime_delta tx.nVersion = txversion self.miniwallet.sign_tx(tx) - if (varyOP_CSV): + if varyOP_CSV: tx.vin[0].scriptSig = CScript([locktime, OP_CHECKSEQUENCEVERIFY, OP_DROP] + list(CScript(tx.vin[0].scriptSig))) else: tx.vin[0].scriptSig = CScript([BASE_RELATIVE_LOCKTIME, OP_CHECKSEQUENCEVERIFY, OP_DROP] + list(CScript(tx.vin[0].scriptSig))) @@ -189,7 +194,7 @@ class BIP68_112_113Test(BitcoinTestFramework): self.log.info("Generate blocks in the past for coinbase outputs.") long_past_time = int(time.time()) - 600 * 1000 # enough to build up to 1000 blocks 10 minutes apart without worrying about getting into the future self.nodes[0].setmocktime(long_past_time - 100) # enough so that the generated blocks will still all be before long_past_time - self.coinbase_blocks = self.miniwallet.generate(COINBASE_BLOCK_COUNT) # blocks generated for inputs + self.coinbase_blocks = self.generate(self.miniwallet, COINBASE_BLOCK_COUNT) # blocks generated for inputs self.nodes[0].setmocktime(0) # set time back to present so yielded blocks aren't in the future as we advance last_block_time self.tipheight = COINBASE_BLOCK_COUNT # height of the next block to build self.last_block_time = long_past_time @@ -197,7 +202,7 @@ class BIP68_112_113Test(BitcoinTestFramework): # Activation height is hardcoded # We advance to block height five below BIP112 activation for the following tests - test_blocks = self.generate_blocks(CSV_ACTIVATION_HEIGHT-5 - COINBASE_BLOCK_COUNT) + test_blocks = self.generate_blocks(CSV_ACTIVATION_HEIGHT - 5 - COINBASE_BLOCK_COUNT) self.send_blocks(test_blocks) assert not softfork_active(self.nodes[0], 'csv') @@ -235,7 +240,7 @@ class BIP68_112_113Test(BitcoinTestFramework): bip113input = self.send_generic_input_tx(self.coinbase_blocks) self.nodes[0].setmocktime(self.last_block_time + 600) - inputblockhash = self.nodes[0].generate(1)[0] # 1 block generated for inputs to be in chain at height 431 + inputblockhash = self.generate(self.nodes[0], 1)[0] # 1 block generated for inputs to be in chain at height 431 self.nodes[0].setmocktime(0) self.tip = int(inputblockhash, 16) self.tipheight += 1 @@ -247,7 +252,7 @@ class BIP68_112_113Test(BitcoinTestFramework): self.send_blocks(test_blocks) assert_equal(self.tipheight, CSV_ACTIVATION_HEIGHT - 2) - self.log.info("Height = {}, CSV not yet active (will activate for block {}, not {})".format(self.tipheight, CSV_ACTIVATION_HEIGHT, CSV_ACTIVATION_HEIGHT - 1)) + self.log.info(f"Height = {self.tipheight}, CSV not yet active (will activate for block {CSV_ACTIVATION_HEIGHT}, not {CSV_ACTIVATION_HEIGHT - 1})") assert not softfork_active(self.nodes[0], 'csv') # Test both version 1 and version 2 transactions for all tests @@ -482,5 +487,6 @@ class BIP68_112_113Test(BitcoinTestFramework): self.send_blocks([self.create_test_block(time_txs)]) self.nodes[0].invalidateblock(self.nodes[0].getbestblockhash()) + if __name__ == '__main__': BIP68_112_113Test().main() diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index c532300ce2..f0766ca7c2 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -41,7 +41,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, create_confirmed_utxos, - hex_str_to_bytes, ) @@ -103,7 +102,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): # perhaps we generated a test case that blew up our cache? # TODO: If this happens a lot, we should try to restart without -dbcrashratio # and make sure that recovery happens. - raise AssertionError("Unable to successfully restart node %d in allotted time", node_index) + raise AssertionError(f"Unable to successfully restart node {node_index} in allotted time") def submit_block_catch_error(self, node_index, block): """Try submitting a block to the given node. @@ -115,10 +114,10 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): self.nodes[node_index].submitblock(block) return True except (http.client.CannotSendRequest, http.client.RemoteDisconnected) as e: - self.log.debug("node %d submitblock raised exception: %s", node_index, e) + self.log.debug(f"node {node_index} submitblock raised exception: {e}") return False except OSError as e: - self.log.debug("node %d submitblock raised OSError exception: errno=%s", node_index, e.errno) + self.log.debug(f"node {node_index} submitblock raised OSError exception: errno={e.errno}") if e.errno in [errno.EPIPE, errno.ECONNREFUSED, errno.ECONNRESET]: # The node has likely crashed return False @@ -143,15 +142,15 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): # Deliver each block to each other node for i in range(3): nodei_utxo_hash = None - self.log.debug("Syncing blocks to node %d", i) + self.log.debug(f"Syncing blocks to node {i}") for (block_hash, block) in blocks: # Get the block from node3, and submit to node_i - self.log.debug("submitting block %s", block_hash) + self.log.debug(f"submitting block {block_hash}") if not self.submit_block_catch_error(i, block): # TODO: more carefully check that the crash is due to -dbcrashratio # (change the exit code perhaps, and check that here?) self.wait_for_node_exit(i, timeout=30) - self.log.debug("Restarting node %d after block hash %s", i, block_hash) + self.log.debug(f"Restarting node {i} after block hash {block_hash}") nodei_utxo_hash = self.restart_node(i, block_hash) assert nodei_utxo_hash is not None self.restart_counts[i] += 1 @@ -168,7 +167,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): # - we only update the utxo cache after a node restart, since flushing # the cache is a no-op at that point if nodei_utxo_hash is not None: - self.log.debug("Checking txoutsetinfo matches for node %d", i) + self.log.debug(f"Checking txoutsetinfo matches for node {i}") assert_equal(nodei_utxo_hash, node3_utxo_hash) def verify_utxo_hash(self): @@ -204,7 +203,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): continue for _ in range(3): - tx.vout.append(CTxOut(output_amount, hex_str_to_bytes(utxo['scriptPubKey']))) + tx.vout.append(CTxOut(output_amount, bytes.fromhex(utxo['scriptPubKey']))) # Sign and send the transaction to get into the mempool tx_signed_hex = node.signrawtransactionwithwallet(tx.serialize().hex())['hex'] @@ -218,15 +217,15 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): # Start by creating a lot of utxos on node3 initial_height = self.nodes[3].getblockcount() - utxo_list = create_confirmed_utxos(self.nodes[3].getnetworkinfo()['relayfee'], self.nodes[3], 5000) - self.log.info("Prepped %d utxo entries", len(utxo_list)) + utxo_list = create_confirmed_utxos(self, self.nodes[3].getnetworkinfo()['relayfee'], self.nodes[3], 5000) + self.log.info(f"Prepped {len(utxo_list)} utxo entries") # Sync these blocks with the other nodes block_hashes_to_sync = [] for height in range(initial_height + 1, self.nodes[3].getblockcount() + 1): block_hashes_to_sync.append(self.nodes[3].getblockhash(height)) - self.log.debug("Syncing %d blocks with other nodes", len(block_hashes_to_sync)) + self.log.debug(f"Syncing {len(block_hashes_to_sync)} blocks with other nodes") # Syncing the blocks could cause nodes to crash, so the test begins here. self.sync_node3blocks(block_hashes_to_sync) @@ -236,33 +235,34 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): # each time through the loop, generate a bunch of transactions, # and then either mine a single new block on the tip, or some-sized reorg. for i in range(40): - self.log.info("Iteration %d, generating 2500 transactions %s", i, self.restart_counts) + self.log.info(f"Iteration {i}, generating 2500 transactions {self.restart_counts}") # Generate a bunch of small-ish transactions self.generate_small_transactions(self.nodes[3], 2500, utxo_list) # Pick a random block between current tip, and starting tip current_height = self.nodes[3].getblockcount() random_height = random.randint(starting_tip_height, current_height) - self.log.debug("At height %d, considering height %d", current_height, random_height) + self.log.debug(f"At height {current_height}, considering height {random_height}") if random_height > starting_tip_height: # Randomly reorg from this point with some probability (1/4 for # tip, 1/5 for tip-1, ...) if random.random() < 1.0 / (current_height + 4 - random_height): - self.log.debug("Invalidating block at height %d", random_height) + self.log.debug(f"Invalidating block at height {random_height}") self.nodes[3].invalidateblock(self.nodes[3].getblockhash(random_height)) # Now generate new blocks until we pass the old tip height self.log.debug("Mining longer tip") block_hashes = [] while current_height + 1 > self.nodes[3].getblockcount(): - block_hashes.extend(self.nodes[3].generatetoaddress( + block_hashes.extend(self.generatetoaddress( + self.nodes[3], nblocks=min(10, current_height + 1 - self.nodes[3].getblockcount()), # new address to avoid mining a block that has just been invalidated address=self.nodes[3].getnewaddress(), )) - self.log.debug("Syncing %d new blocks...", len(block_hashes)) + self.log.debug(f"Syncing {len(block_hashes)} new blocks...") self.sync_node3blocks(block_hashes) utxo_list = self.nodes[3].listunspent() - self.log.debug("Node3 utxo count: %d", len(utxo_list)) + self.log.debug(f"Node3 utxo count: {len(utxo_list)}") # Check that the utxo hashes agree with node3 # Useful side effect: each utxo cache gets flushed here, so that we @@ -270,7 +270,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): self.verify_utxo_hash() # Check the test coverage - self.log.info("Restarted nodes: %s; crashes on restart: %d", self.restart_counts, self.crashed_on_restart) + self.log.info(f"Restarted nodes: {self.restart_counts}; crashes on restart: {self.crashed_on_restart}") # If no nodes were restarted, we didn't test anything. assert self.restart_counts != [0, 0, 0] @@ -281,7 +281,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): # Warn if any of the nodes escaped restart. for i in range(3): if self.restart_counts[i] == 0: - self.log.warning("Node %d never crashed during utxo flush!", i) + self.log.warning(f"Node {i} never crashed during utxo flush!") if __name__ == "__main__": diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index eb027c554a..28aff1f2f9 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -4,7 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP66 (DER SIG). -Test that the DERSIG soft-fork activates at (regtest) height 1251. +Test the DERSIG soft-fork activation on regtest. """ from test_framework.blocktools import ( @@ -23,8 +23,6 @@ from test_framework.wallet import ( MiniWalletMode, ) -DERSIG_HEIGHT = 1251 - # A canonical signature consists of: # <30> <total len> <02> <len R> <R> <02> <len S> <S> <hashtype> @@ -43,10 +41,14 @@ def unDERify(tx): tx.vin[0].scriptSig = CScript(newscript) +DERSIG_HEIGHT = 102 + + class BIP66Test(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.extra_args = [[ + f'-testactivationheight=dersig@{DERSIG_HEIGHT}', '-whitelist=noban@127.0.0.1', '-par=1', # Use only one script thread to get the exact log msg for testing ]] @@ -73,7 +75,7 @@ class BIP66Test(BitcoinTestFramework): self.test_dersig_info(is_active=False) self.log.info("Mining %d blocks", DERSIG_HEIGHT - 2) - self.coinbase_txids = [self.nodes[0].getblock(b)['tx'][0] for b in self.miniwallet.generate(DERSIG_HEIGHT - 2)] + self.coinbase_txids = [self.nodes[0].getblock(b)['tx'][0] for b in self.generate(self.miniwallet, DERSIG_HEIGHT - 2)] self.log.info("Test that a transaction with non-DER signature can still appear in a block") @@ -84,14 +86,15 @@ class BIP66Test(BitcoinTestFramework): tip = self.nodes[0].getbestblockhash() block_time = self.nodes[0].getblockheader(tip)['mediantime'] + 1 block = create_block(int(tip, 16), create_coinbase(DERSIG_HEIGHT - 1), block_time) - block.nVersion = 2 block.vtx.append(spendtx) block.hashMerkleRoot = block.calc_merkle_root() block.rehash() block.solve() + assert_equal(self.nodes[0].getblockcount(), DERSIG_HEIGHT - 2) self.test_dersig_info(is_active=False) # Not active as of current tip and next block does not need to obey rules peer.send_and_ping(msg_block(block)) + assert_equal(self.nodes[0].getblockcount(), DERSIG_HEIGHT - 1) self.test_dersig_info(is_active=True) # Not active as of current tip, but next block must obey rules assert_equal(self.nodes[0].getbestblockhash(), block.hash) @@ -103,13 +106,13 @@ class BIP66Test(BitcoinTestFramework): block.rehash() block.solve() - with self.nodes[0].assert_debug_log(expected_msgs=['{}, bad-version(0x00000002)'.format(block.hash)]): + with self.nodes[0].assert_debug_log(expected_msgs=[f'{block.hash}, bad-version(0x00000002)']): peer.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) peer.sync_with_ping() self.log.info("Test that transactions with non-DER signatures cannot appear in a block") - block.nVersion = 3 + block.nVersion = 4 spendtx = self.create_tx(self.coinbase_txids[1]) unDERify(spendtx) @@ -133,12 +136,12 @@ class BIP66Test(BitcoinTestFramework): block.rehash() block.solve() - with self.nodes[0].assert_debug_log(expected_msgs=['CheckInputScripts on {} failed with non-mandatory-script-verify-flag (Non-canonical DER signature)'.format(block.vtx[-1].hash)]): + with self.nodes[0].assert_debug_log(expected_msgs=[f'CheckInputScripts on {block.vtx[-1].hash} failed with non-mandatory-script-verify-flag (Non-canonical DER signature)']): peer.send_and_ping(msg_block(block)) assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip) peer.sync_with_ping() - self.log.info("Test that a version 3 block with a DERSIG-compliant transaction is accepted") + self.log.info("Test that a block with a DERSIG-compliant transaction is accepted") block.vtx[1] = self.create_tx(self.coinbase_txids[1]) block.hashMerkleRoot = block.calc_merkle_root() block.rehash() diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index 8ccdf87ff3..ac00db8ff0 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -4,6 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test fee estimation code.""" from decimal import Decimal +import os import random from test_framework.messages import ( @@ -18,10 +19,10 @@ from test_framework.script import ( OP_1, OP_2, OP_DROP, - OP_EQUAL, - OP_HASH160, OP_TRUE, - hash160, +) +from test_framework.script_util import ( + script_to_p2sh_script, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -37,8 +38,8 @@ from test_framework.util import ( # time signing. REDEEM_SCRIPT_1 = CScript([OP_1, OP_DROP]) REDEEM_SCRIPT_2 = CScript([OP_2, OP_DROP]) -P2SH_1 = CScript([OP_HASH160, hash160(REDEEM_SCRIPT_1), OP_EQUAL]) -P2SH_2 = CScript([OP_HASH160, hash160(REDEEM_SCRIPT_2), OP_EQUAL]) +P2SH_1 = script_to_p2sh_script(REDEEM_SCRIPT_1) +P2SH_2 = script_to_p2sh_script(REDEEM_SCRIPT_2) # Associated ScriptSig's to spend satisfy P2SH_1 and P2SH_2 SCRIPT_SIG = [CScript([OP_TRUE, REDEEM_SCRIPT_1]), CScript([OP_TRUE, REDEEM_SCRIPT_2])] @@ -72,7 +73,7 @@ def small_txpuzzle_randfee(from_node, conflist, unconflist, amount, min_fee, fee total_in += t["amount"] tx.vin.append(CTxIn(COutPoint(int(t["txid"], 16), t["vout"]), b"")) if total_in <= amount + fee: - raise RuntimeError("Insufficient funds: need %d, have %d" % (amount + fee, total_in)) + raise RuntimeError(f"Insufficient funds: need {amount + fee}, have {total_in}") tx.vout.append(CTxOut(int((total_in - amount - fee) * COIN), P2SH_1)) tx.vout.append(CTxOut(int(amount * COIN), P2SH_2)) # These transactions don't need to be signed, but we still have to insert @@ -124,8 +125,7 @@ def check_raw_estimates(node, fees_seen): assert_greater_than(feerate, 0) if feerate + delta < min(fees_seen) or feerate - delta > max(fees_seen): - raise AssertionError("Estimated fee (%f) out of range (%f,%f)" - % (feerate, min(fees_seen), max(fees_seen))) + raise AssertionError(f"Estimated fee ({feerate}) out of range ({min(fees_seen)},{max(fees_seen)})") def check_smart_estimates(node, fees_seen): """Call estimatesmartfee and verify that the estimates meet certain invariants.""" @@ -133,16 +133,18 @@ def check_smart_estimates(node, fees_seen): delta = 1.0e-6 # account for rounding error last_feerate = float(max(fees_seen)) all_smart_estimates = [node.estimatesmartfee(i) for i in range(1, 26)] + mempoolMinFee = node.getmempoolinfo()['mempoolminfee'] + minRelaytxFee = node.getmempoolinfo()['minrelaytxfee'] for i, e in enumerate(all_smart_estimates): # estimate is for i+1 feerate = float(e["feerate"]) assert_greater_than(feerate, 0) + assert_greater_than_or_equal(feerate, float(mempoolMinFee)) + assert_greater_than_or_equal(feerate, float(minRelaytxFee)) if feerate + delta < min(fees_seen) or feerate - delta > max(fees_seen): - raise AssertionError("Estimated fee (%f) out of range (%f,%f)" - % (feerate, min(fees_seen), max(fees_seen))) + raise AssertionError(f"Estimated fee ({feerate}) out of range ({min(fees_seen)},{max(fees_seen)})") if feerate - delta > last_feerate: - raise AssertionError("Estimated fee (%f) larger than last fee (%f) for lower number of confirms" - % (feerate, last_feerate)) + raise AssertionError(f"Estimated fee ({feerate}) larger than last fee ({last_feerate}) for lower number of confirms") last_feerate = feerate if i == 0: @@ -154,6 +156,21 @@ def check_estimates(node, fees_seen): check_raw_estimates(node, fees_seen) check_smart_estimates(node, fees_seen) + +def send_tx(node, utxo, feerate): + """Broadcast a 1in-1out transaction with a specific input and feerate (sat/vb).""" + overhead, op, scriptsig, nseq, value, spk = 10, 36, 5, 4, 8, 24 + tx_size = overhead + op + scriptsig + nseq + value + spk + fee = tx_size * feerate + + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(int(utxo["txid"], 16), utxo["vout"]), SCRIPT_SIG[utxo["vout"]])] + tx.vout = [CTxOut(int(utxo["amount"] * COIN) - fee, P2SH_1)] + txid = node.sendrawtransaction(tx.serialize().hex()) + + return txid + + class EstimateFeeTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 3 @@ -200,7 +217,7 @@ class EstimateFeeTest(BitcoinTestFramework): tx_kbytes = (len(txhex) // 2) / 1000.0 self.fees_per_kb.append(float(fee) / tx_kbytes) self.sync_mempools(wait=.1) - mined = mining_node.getblock(mining_node.generate(1)[0], True)["tx"] + mined = mining_node.getblock(self.generate(mining_node, 1)[0], True)["tx"] self.sync_blocks(wait=.1) # update which txouts are confirmed newmem = [] @@ -211,20 +228,16 @@ class EstimateFeeTest(BitcoinTestFramework): newmem.append(utx) self.memutxo = newmem - def run_test(self): - self.log.info("This test is time consuming, please be patient") - self.log.info("Splitting inputs so we can generate tx's") - - # Start node0 - self.start_node(0) + def initial_split(self, node): + """Split two coinbase UTxOs into many small coins""" self.txouts = [] self.txouts2 = [] # Split a coinbase into two transaction puzzle outputs - split_inputs(self.nodes[0], self.nodes[0].listunspent(0), self.txouts, True) + split_inputs(node, node.listunspent(0), self.txouts, True) # Mine - while len(self.nodes[0].getrawmempool()) > 0: - self.nodes[0].generate(1) + while len(node.getrawmempool()) > 0: + self.generate(node, 1) # Repeatedly split those 2 outputs, doubling twice for each rep # Use txouts to monitor the available utxo, since these won't be tracked in wallet @@ -232,27 +245,19 @@ class EstimateFeeTest(BitcoinTestFramework): while reps < 5: # Double txouts to txouts2 while len(self.txouts) > 0: - split_inputs(self.nodes[0], self.txouts, self.txouts2) - while len(self.nodes[0].getrawmempool()) > 0: - self.nodes[0].generate(1) + split_inputs(node, self.txouts, self.txouts2) + while len(node.getrawmempool()) > 0: + self.generate(node, 1) # Double txouts2 to txouts while len(self.txouts2) > 0: - split_inputs(self.nodes[0], self.txouts2, self.txouts) - while len(self.nodes[0].getrawmempool()) > 0: - self.nodes[0].generate(1) + split_inputs(node, self.txouts2, self.txouts) + while len(node.getrawmempool()) > 0: + self.generate(node, 1) reps += 1 - self.log.info("Finished splitting") - - # Now we can connect the other nodes, didn't want to connect them earlier - # so the estimates would not be affected by the splitting transactions - self.start_node(1) - self.start_node(2) - self.connect_nodes(1, 0) - self.connect_nodes(0, 2) - self.connect_nodes(2, 1) - - self.sync_all() + def sanity_check_estimates_range(self): + """Populate estimation buckets, assert estimates are in a sane range and + are strictly increasing as the target decreases.""" self.fees_per_kb = [] self.memutxo = [] self.confutxo = self.txouts # Start with the set of confirmed txouts after splitting @@ -272,12 +277,107 @@ class EstimateFeeTest(BitcoinTestFramework): # Finish by mining a normal-sized block: while len(self.nodes[1].getrawmempool()) > 0: - self.nodes[1].generate(1) + self.generate(self.nodes[1], 1) self.sync_blocks(self.nodes[0:3], wait=.1) self.log.info("Final estimates after emptying mempools") check_estimates(self.nodes[1], self.fees_per_kb) + def test_feerate_mempoolminfee(self): + high_val = 3*self.nodes[1].estimatesmartfee(1)['feerate'] + self.restart_node(1, extra_args=[f'-minrelaytxfee={high_val}']) + check_estimates(self.nodes[1], self.fees_per_kb) + self.restart_node(1) + + def sanity_check_rbf_estimates(self, utxos): + """During 5 blocks, broadcast low fee transactions. Only 10% of them get + confirmed and the remaining ones get RBF'd with a high fee transaction at + the next block. + The block policy estimator should return the high feerate. + """ + # The broadcaster and block producer + node = self.nodes[0] + miner = self.nodes[1] + # In sat/vb + low_feerate = 1 + high_feerate = 10 + # Cache the utxos of which to replace the spender after it failed to get + # confirmed + utxos_to_respend = [] + txids_to_replace = [] + + assert len(utxos) >= 250 + for _ in range(5): + # Broadcast 45 low fee transactions that will need to be RBF'd + for _ in range(45): + u = utxos.pop(0) + txid = send_tx(node, u, low_feerate) + utxos_to_respend.append(u) + txids_to_replace.append(txid) + # Broadcast 5 low fee transaction which don't need to + for _ in range(5): + send_tx(node, utxos.pop(0), low_feerate) + # Mine the transactions on another node + self.sync_mempools(wait=.1, nodes=[node, miner]) + for txid in txids_to_replace: + miner.prioritisetransaction(txid=txid, fee_delta=-COIN) + self.generate(miner, 1) + self.sync_blocks(wait=.1, nodes=[node, miner]) + # RBF the low-fee transactions + while True: + try: + u = utxos_to_respend.pop(0) + send_tx(node, u, high_feerate) + except IndexError: + break + + # Mine the last replacement txs + self.sync_mempools(wait=.1, nodes=[node, miner]) + self.generate(miner, 1) + self.sync_blocks(wait=.1, nodes=[node, miner]) + + # Only 10% of the transactions were really confirmed with a low feerate, + # the rest needed to be RBF'd. We must return the 90% conf rate feerate. + high_feerate_kvb = Decimal(high_feerate) / COIN * 10**3 + est_feerate = node.estimatesmartfee(2)["feerate"] + assert est_feerate == high_feerate_kvb + + def run_test(self): + self.log.info("This test is time consuming, please be patient") + self.log.info("Splitting inputs so we can generate tx's") + + # Split two coinbases into many small utxos + self.start_node(0) + self.initial_split(self.nodes[0]) + self.log.info("Finished splitting") + + # Now we can connect the other nodes, didn't want to connect them earlier + # so the estimates would not be affected by the splitting transactions + self.start_node(1) + self.start_node(2) + self.connect_nodes(1, 0) + self.connect_nodes(0, 2) + self.connect_nodes(2, 1) + self.sync_all() + + self.log.info("Testing estimates with single transactions.") + self.sanity_check_estimates_range() + + # check that the effective feerate is greater than or equal to the mempoolminfee even for high mempoolminfee + self.log.info("Test fee rate estimation after restarting node with high MempoolMinFee") + self.test_feerate_mempoolminfee() + + self.log.info("Restarting node with fresh estimation") + self.stop_node(0) + fee_dat = os.path.join(self.nodes[0].datadir, self.chain, "fee_estimates.dat") + os.remove(fee_dat) + self.start_node(0) + self.connect_nodes(0, 1) + self.connect_nodes(0, 2) + + self.log.info("Testing estimates with RBF.") + self.sanity_check_rbf_estimates(self.confutxo + self.memutxo) + self.log.info("Testing that fee estimation is disabled in blocksonly.") self.restart_node(0, ["-blocksonly"]) assert_raises_rpc_error(-32603, "Fee estimation disabled", diff --git a/test/functional/feature_filelock.py b/test/functional/feature_filelock.py index 2798d11b0a..e09107802b 100755 --- a/test/functional/feature_filelock.py +++ b/test/functional/feature_filelock.py @@ -22,11 +22,11 @@ class FilelockTest(BitcoinTestFramework): def run_test(self): datadir = os.path.join(self.nodes[0].datadir, self.chain) - self.log.info("Using datadir {}".format(datadir)) + self.log.info(f"Using datadir {datadir}") self.log.info("Check that we can't start a second bitcoind instance using the same datadir") - expected_msg = "Error: Cannot obtain a lock on data directory {0}. {1} is probably already running.".format(datadir, self.config['environment']['PACKAGE_NAME']) - self.nodes[1].assert_start_raises_init_error(extra_args=['-datadir={}'.format(self.nodes[0].datadir), '-noserver'], expected_msg=expected_msg) + expected_msg = f"Error: Cannot obtain a lock on data directory {datadir}. {self.config['environment']['PACKAGE_NAME']} is probably already running." + self.nodes[1].assert_start_raises_init_error(extra_args=[f'-datadir={self.nodes[0].datadir}', '-noserver'], expected_msg=expected_msg) if self.is_wallet_compiled(): def check_wallet_filelock(descriptors): @@ -38,7 +38,7 @@ class FilelockTest(BitcoinTestFramework): expected_msg = "Error: SQLiteDatabase: Unable to obtain an exclusive lock on the database, is it being used by another bitcoind?" else: expected_msg = "Error: Error initializing wallet database environment" - self.nodes[1].assert_start_raises_init_error(extra_args=['-walletdir={}'.format(wallet_dir), '-wallet=' + wallet_name, '-noserver'], expected_msg=expected_msg, match=ErrorMatch.PARTIAL_REGEX) + self.nodes[1].assert_start_raises_init_error(extra_args=[f'-walletdir={wallet_dir}', f'-wallet={wallet_name}', '-noserver'], expected_msg=expected_msg, match=ErrorMatch.PARTIAL_REGEX) if self.is_bdb_compiled(): check_wallet_filelock(False) diff --git a/test/functional/feature_help.py b/test/functional/feature_help.py index babe9bfc80..837e95c128 100755 --- a/test/functional/feature_help.py +++ b/test/functional/feature_help.py @@ -40,14 +40,14 @@ class HelpTest(BitcoinTestFramework): # Node should exit immediately and output help to stdout. output, _ = self.get_node_output(ret_code_expected=0) assert b'Options' in output - self.log.info("Help text received: {} (...)".format(output[0:60])) + self.log.info(f"Help text received: {output[0:60]} (...)") self.log.info("Start bitcoin with -version for version information") self.nodes[0].start(extra_args=['-version']) # Node should exit immediately and output version to stdout. output, _ = self.get_node_output(ret_code_expected=0) assert b'version' in output - self.log.info("Version text received: {} (...)".format(output[0:60])) + self.log.info(f"Version text received: {output[0:60]} (...)") # Test that arguments not in the help results in an error self.log.info("Start bitcoind with -fakearg to make sure it does not start") @@ -55,7 +55,7 @@ class HelpTest(BitcoinTestFramework): # Node should exit immediately and output an error to stderr _, output = self.get_node_output(ret_code_expected=1) assert b'Error parsing command line arguments' in output - self.log.info("Error message received: {} (...)".format(output[0:60])) + self.log.info(f"Error message received: {output[0:60]} (...)") if __name__ == '__main__': diff --git a/test/functional/feature_loadblock.py b/test/functional/feature_loadblock.py index 14f64d63a2..13e6a8d6d7 100755 --- a/test/functional/feature_loadblock.py +++ b/test/functional/feature_loadblock.py @@ -29,7 +29,7 @@ class LoadblockTest(BitcoinTestFramework): def run_test(self): self.nodes[1].setnetworkactive(state=False) - self.nodes[0].generate(COINBASE_MATURITY) + self.generate(self.nodes[0], COINBASE_MATURITY) # Parsing the url of our node to get settings for config file data_dir = self.nodes[0].datadir @@ -45,17 +45,17 @@ class LoadblockTest(BitcoinTestFramework): self.log.info("Create linearization config file") with open(cfg_file, "a", encoding="utf-8") as cfg: - cfg.write("datadir={}\n".format(data_dir)) - cfg.write("rpcuser={}\n".format(node_url.username)) - cfg.write("rpcpassword={}\n".format(node_url.password)) - cfg.write("port={}\n".format(node_url.port)) - cfg.write("host={}\n".format(node_url.hostname)) - cfg.write("output_file={}\n".format(bootstrap_file)) - cfg.write("max_height=100\n") - cfg.write("netmagic=fabfb5da\n") - cfg.write("input={}\n".format(blocks_dir)) - cfg.write("genesis={}\n".format(genesis_block)) - cfg.write("hashlist={}\n".format(hash_list.name)) + cfg.write(f"datadir={data_dir}\n") + cfg.write(f"rpcuser={node_url.username}\n") + cfg.write(f"rpcpassword={node_url.password}\n") + cfg.write(f"port={node_url.port}\n") + cfg.write(f"host={node_url.hostname}\n") + cfg.write(f"output_file={bootstrap_file}\n") + cfg.write(f"max_height=100\n") + cfg.write(f"netmagic=fabfb5da\n") + cfg.write(f"input={blocks_dir}\n") + cfg.write(f"genesis={genesis_block}\n") + cfg.write(f"hashlist={hash_list.name}\n") base_dir = self.config["environment"]["SRCDIR"] linearize_dir = os.path.join(base_dir, "contrib", "linearize") @@ -72,7 +72,7 @@ class LoadblockTest(BitcoinTestFramework): check=True) self.log.info("Restart second, unsynced node with bootstrap file") - self.restart_node(1, extra_args=["-loadblock=" + bootstrap_file]) + self.restart_node(1, extra_args=[f"-loadblock={bootstrap_file}"]) assert_equal(self.nodes[1].getblockcount(), 100) # start_node is blocking on all block files being imported assert_equal(self.nodes[1].getblockchaininfo()['blocks'], 100) diff --git a/test/functional/feature_logging.py b/test/functional/feature_logging.py index afcbcf099a..722219518a 100755 --- a/test/functional/feature_logging.py +++ b/test/functional/feature_logging.py @@ -29,7 +29,7 @@ class LoggingTest(BitcoinTestFramework): # test alternative log file name outside datadir tempname = os.path.join(self.options.tmpdir, "foo.log") - self.restart_node(0, ["-debuglogfile=%s" % tempname]) + self.restart_node(0, [f"-debuglogfile={tempname}"]) assert os.path.isfile(tempname) # check that invalid log (relative) will cause error @@ -37,26 +37,26 @@ class LoggingTest(BitcoinTestFramework): invalidname = os.path.join("foo", "foo.log") self.stop_node(0) exp_stderr = r"Error: Could not open debug log file \S+$" - self.nodes[0].assert_start_raises_init_error(["-debuglogfile=%s" % (invalidname)], exp_stderr, match=ErrorMatch.FULL_REGEX) + self.nodes[0].assert_start_raises_init_error([f"-debuglogfile={invalidname}"], exp_stderr, match=ErrorMatch.FULL_REGEX) assert not os.path.isfile(os.path.join(invdir, "foo.log")) # check that invalid log (relative) works after path exists self.stop_node(0) os.mkdir(invdir) - self.start_node(0, ["-debuglogfile=%s" % (invalidname)]) + self.start_node(0, [f"-debuglogfile={invalidname}"]) assert os.path.isfile(os.path.join(invdir, "foo.log")) # check that invalid log (absolute) will cause error self.stop_node(0) invdir = os.path.join(self.options.tmpdir, "foo") invalidname = os.path.join(invdir, "foo.log") - self.nodes[0].assert_start_raises_init_error(["-debuglogfile=%s" % invalidname], exp_stderr, match=ErrorMatch.FULL_REGEX) + self.nodes[0].assert_start_raises_init_error([f"-debuglogfile={invalidname}"], exp_stderr, match=ErrorMatch.FULL_REGEX) assert not os.path.isfile(os.path.join(invdir, "foo.log")) # check that invalid log (absolute) works after path exists self.stop_node(0) os.mkdir(invdir) - self.start_node(0, ["-debuglogfile=%s" % (invalidname)]) + self.start_node(0, [f"-debuglogfile={invalidname}"]) assert os.path.isfile(os.path.join(invdir, "foo.log")) # check that -nodebuglogfile disables logging @@ -67,7 +67,7 @@ class LoggingTest(BitcoinTestFramework): assert not os.path.isfile(default_log_path) # just sanity check no crash here - self.restart_node(0, ["-debuglogfile=%s" % os.devnull]) + self.restart_node(0, [f"-debuglogfile={os.devnull}"]) if __name__ == '__main__': diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index d0a94658ff..ac4d40638e 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -38,7 +38,6 @@ class MaxUploadTest(BitcoinTestFramework): self.extra_args = [[ "-maxuploadtarget=800", "-acceptnonstdtxn=1", - "-peertimeout=9999", # bump because mocktime might cause a disconnect otherwise ]] self.supports_cli = False @@ -56,7 +55,7 @@ class MaxUploadTest(BitcoinTestFramework): self.nodes[0].setmocktime(old_time) # Generate some old blocks - self.nodes[0].generate(130) + self.generate(self.nodes[0], 130) # p2p_conns[0] will only request old blocks # p2p_conns[1] will only request new blocks @@ -67,7 +66,7 @@ class MaxUploadTest(BitcoinTestFramework): p2p_conns.append(self.nodes[0].add_p2p_connection(TestP2PConn())) # Now mine a big block - mine_large_block(self.nodes[0], self.utxo_cache) + mine_large_block(self, self.nodes[0], self.utxo_cache) # Store the hash; we'll request this later big_old_block = self.nodes[0].getbestblockhash() @@ -78,7 +77,7 @@ class MaxUploadTest(BitcoinTestFramework): self.nodes[0].setmocktime(int(time.time()) - 2*60*60*24) # Mine one more block, so that the prior block looks old - mine_large_block(self.nodes[0], self.utxo_cache) + mine_large_block(self, self.nodes[0], self.utxo_cache) # We'll be requesting this new block too big_new_block = self.nodes[0].getbestblockhash() diff --git a/test/functional/feature_minchainwork.py b/test/functional/feature_minchainwork.py index 803feb509a..11cb4aa3cb 100755 --- a/test/functional/feature_minchainwork.py +++ b/test/functional/feature_minchainwork.py @@ -45,16 +45,16 @@ class MinimumChainWorkTest(BitcoinTestFramework): # Start building a chain on node0. node2 shouldn't be able to sync until node1's # minchainwork is exceeded starting_chain_work = REGTEST_WORK_PER_BLOCK # Genesis block's work - self.log.info("Testing relay across node %d (minChainWork = %d)", 1, self.node_min_work[1]) + self.log.info(f"Testing relay across node 1 (minChainWork = {self.node_min_work[1]})") starting_blockcount = self.nodes[2].getblockcount() num_blocks_to_generate = int((self.node_min_work[1] - starting_chain_work) / REGTEST_WORK_PER_BLOCK) - self.log.info("Generating %d blocks on node0", num_blocks_to_generate) - hashes = self.nodes[0].generatetoaddress(num_blocks_to_generate, + self.log.info(f"Generating {num_blocks_to_generate} blocks on node0") + hashes = self.generatetoaddress(self.nodes[0], num_blocks_to_generate, self.nodes[0].get_deterministic_priv_key().address) - self.log.info("Node0 current chain work: %s", self.nodes[0].getblockheader(hashes[-1])['chainwork']) + self.log.info(f"Node0 current chain work: {self.nodes[0].getblockheader(hashes[-1])['chainwork']}") # Sleep a few seconds and verify that node2 didn't get any new blocks # or headers. We sleep, rather than sync_blocks(node0, node1) because @@ -63,7 +63,7 @@ class MinimumChainWorkTest(BitcoinTestFramework): time.sleep(3) self.log.info("Verifying node 2 has no more blocks than before") - self.log.info("Blockcounts: %s", [n.getblockcount() for n in self.nodes]) + self.log.info(f"Blockcounts: {[n.getblockcount() for n in self.nodes]}") # Node2 shouldn't have any new headers yet, because node1 should not # have relayed anything. assert_equal(len(self.nodes[2].getchaintips()), 1) @@ -73,7 +73,7 @@ class MinimumChainWorkTest(BitcoinTestFramework): assert_equal(self.nodes[2].getblockcount(), starting_blockcount) self.log.info("Generating one more block") - self.nodes[0].generatetoaddress(1, self.nodes[0].get_deterministic_priv_key().address) + self.generatetoaddress(self.nodes[0], 1, self.nodes[0].get_deterministic_priv_key().address) self.log.info("Verifying nodes are all synced") @@ -84,7 +84,7 @@ class MinimumChainWorkTest(BitcoinTestFramework): # continue the test. self.sync_all() - self.log.info("Blockcounts: %s", [n.getblockcount() for n in self.nodes]) + self.log.info(f"Blockcounts: {[n.getblockcount() for n in self.nodes]}") if __name__ == '__main__': MinimumChainWorkTest().main() diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index 6fc8773ee3..2a507c75c4 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -20,13 +20,16 @@ FILE_CHARS_DISALLOWED = '/\\?%*:|"<>' if os.name == 'nt' else '/' UNCONFIRMED_HASH_STRING = 'unconfirmed' def notify_outputname(walletname, txid): - return txid if os.name == 'nt' else '{}_{}'.format(walletname, txid) + return txid if os.name == 'nt' else f'{walletname}_{txid}' class NotificationsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True + # The experimental syscall sandbox feature (-sandbox) is not compatible with -alertnotify, + # -blocknotify or -walletnotify (which all invoke execve). + self.disable_syscall_sandbox = True def setup_network(self): self.wallet = ''.join(chr(i) for i in range(FILE_CHAR_START, FILE_CHAR_END) if chr(i) not in FILE_CHARS_DISALLOWED) @@ -39,11 +42,10 @@ class NotificationsTest(BitcoinTestFramework): # -alertnotify and -blocknotify on node0, walletnotify on node1 self.extra_args = [[ - "-alertnotify=echo > {}".format(os.path.join(self.alertnotify_dir, '%s')), - "-blocknotify=echo > {}".format(os.path.join(self.blocknotify_dir, '%s')), + f"-alertnotify=echo > {os.path.join(self.alertnotify_dir, '%s')}", + f"-blocknotify=echo > {os.path.join(self.blocknotify_dir, '%s')}", ], [ - "-rescan", - "-walletnotify=echo %h_%b > {}".format(os.path.join(self.walletnotify_dir, notify_outputname('%w', '%s'))), + f"-walletnotify=echo %h_%b > {os.path.join(self.walletnotify_dir, notify_outputname('%w', '%s'))}", ]] self.wallet_names = [self.default_wallet_name, self.wallet] super().setup_network() @@ -54,12 +56,12 @@ class NotificationsTest(BitcoinTestFramework): seed = "cTdGmKFWpbvpKQ7ejrdzqYT2hhjyb3GPHnLAK7wdi5Em67YLwSm9" xpriv = "tprv8ZgxMBicQKsPfHCsTwkiM1KT56RXbGGTqvc2hgqzycpwbHqqpcajQeMRZoBD35kW4RtyCemu6j34Ku5DEspmgjKdt2qe4SvRch5Kk8B8A2v" desc_imports = [{ - "desc": descsum_create("wpkh(" + xpriv + "/0/*)"), + "desc": descsum_create(f"wpkh({xpriv}/0/*)"), "timestamp": 0, "active": True, "keypool": True, },{ - "desc": descsum_create("wpkh(" + xpriv + "/1/*)"), + "desc": descsum_create(f"wpkh({xpriv}/1/*)"), "timestamp": 0, "active": True, "keypool": True, @@ -76,7 +78,7 @@ class NotificationsTest(BitcoinTestFramework): self.log.info("test -blocknotify") block_count = 10 - blocks = self.nodes[1].generatetoaddress(block_count, self.nodes[1].getnewaddress() if self.is_wallet_compiled() else ADDRESS_BCRT1_UNSPENDABLE) + blocks = self.generatetoaddress(self.nodes[1], block_count, self.nodes[1].getnewaddress() if self.is_wallet_compiled() else ADDRESS_BCRT1_UNSPENDABLE) # wait at most 10 seconds for expected number of files before reading the content self.wait_until(lambda: len(os.listdir(self.blocknotify_dir)) == block_count, timeout=10) @@ -91,16 +93,15 @@ class NotificationsTest(BitcoinTestFramework): # directory content should equal the generated transaction hashes tx_details = list(map(lambda t: (t['txid'], t['blockheight'], t['blockhash']), self.nodes[1].listtransactions("*", block_count))) - self.stop_node(1) self.expect_wallet_notify(tx_details) self.log.info("test -walletnotify after rescan") - # restart node to rescan to force wallet notifications - self.start_node(1) - self.connect_nodes(0, 1) - + # rescan to force wallet notifications + self.nodes[1].rescanblockchain() self.wait_until(lambda: len(os.listdir(self.walletnotify_dir)) == block_count, timeout=10) + self.connect_nodes(0, 1) + # directory content should equal the generated transaction hashes tx_details = list(map(lambda t: (t['txid'], t['blockheight'], t['blockhash']), self.nodes[1].listtransactions("*", block_count))) self.expect_wallet_notify(tx_details) @@ -110,7 +111,7 @@ class NotificationsTest(BitcoinTestFramework): # triggered by node 1 self.log.info("test -walletnotify with conflicting transactions") self.nodes[0].rescanblockchain() - self.nodes[0].generatetoaddress(100, ADDRESS_BCRT1_UNSPENDABLE) + self.generatetoaddress(self.nodes[0], 100, ADDRESS_BCRT1_UNSPENDABLE) self.sync_blocks() # Generate transaction on node 0, sync mempools, and check for @@ -131,7 +132,7 @@ class NotificationsTest(BitcoinTestFramework): # Add bump1 transaction to new block, checking for a notification # and the correct number of confirmations. - blockhash1 = self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE)[0] + blockhash1 = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE)[0] blockheight1 = self.nodes[0].getblockcount() self.sync_blocks() self.expect_wallet_notify([(bump1, blockheight1, blockhash1)]) @@ -148,7 +149,7 @@ class NotificationsTest(BitcoinTestFramework): # about newly confirmed bump2 and newly conflicted tx2. self.disconnect_nodes(0, 1) bump2 = self.nodes[0].bumpfee(tx2)["txid"] - blockhash2 = self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE)[0] + blockhash2 = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE)[0] blockheight2 = self.nodes[0].getblockcount() assert_equal(self.nodes[0].gettransaction(bump2)["confirmations"], 1) assert_equal(tx2 in self.nodes[1].getrawmempool(), True) diff --git a/test/functional/feature_nulldummy.py b/test/functional/feature_nulldummy.py index f467626801..217a38050d 100755 --- a/test/functional/feature_nulldummy.py +++ b/test/functional/feature_nulldummy.py @@ -24,7 +24,10 @@ from test_framework.blocktools import ( from test_framework.messages import CTransaction from test_framework.script import CScript from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) NULLDUMMY_ERROR = "non-mandatory-script-verify-flag (Dummy CHECKMULTISIG argument must be zero)" @@ -49,8 +52,9 @@ class NULLDUMMYTest(BitcoinTestFramework): # This script tests NULLDUMMY activation, which is part of the 'segwit' deployment, so we go through # normal segwit activation here (and don't use the default always-on behaviour). self.extra_args = [[ - f'-segwitheight={COINBASE_MATURITY + 5}', + f'-testactivationheight=segwit@{COINBASE_MATURITY + 5}', '-addresstype=legacy', + '-par=1', # Use only one script thread to get the exact reject reason for testing ]] def skip_test_if_missing_module(self): @@ -70,11 +74,11 @@ class NULLDUMMYTest(BitcoinTestFramework): wmulti.importaddress(self.ms_address) wmulti.importaddress(self.wit_ms_address) - self.coinbase_blocks = self.nodes[0].generate(2) # block height = 2 + self.coinbase_blocks = self.generate(self.nodes[0], 2) # block height = 2 coinbase_txid = [] for i in self.coinbase_blocks: coinbase_txid.append(self.nodes[0].getblock(i)['tx'][0]) - self.nodes[0].generate(COINBASE_MATURITY) # block height = COINBASE_MATURITY + 2 + self.generate(self.nodes[0], COINBASE_MATURITY) # block height = COINBASE_MATURITY + 2 self.lastblockhash = self.nodes[0].getbestblockhash() self.lastblockheight = COINBASE_MATURITY + 2 self.lastblocktime = int(time.time()) + self.lastblockheight @@ -86,7 +90,7 @@ class NULLDUMMYTest(BitcoinTestFramework): txid2 = self.nodes[0].sendrawtransaction(test1txs[1].serialize_with_witness().hex(), 0) test1txs.append(create_transaction(self.nodes[0], coinbase_txid[1], self.wit_ms_address, amount=49)) txid3 = self.nodes[0].sendrawtransaction(test1txs[2].serialize_with_witness().hex(), 0) - self.block_submit(self.nodes[0], test1txs, False, True) + self.block_submit(self.nodes[0], test1txs, accept=True) self.log.info("Test 2: Non-NULLDUMMY base multisig transaction should not be accepted to mempool before activation") test2tx = create_transaction(self.nodes[0], txid2, self.ms_address, amount=47) @@ -94,28 +98,28 @@ class NULLDUMMYTest(BitcoinTestFramework): assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test2tx.serialize_with_witness().hex(), 0) self.log.info(f"Test 3: Non-NULLDUMMY base transactions should be accepted in a block before activation [{COINBASE_MATURITY + 4}]") - self.block_submit(self.nodes[0], [test2tx], False, True) + self.block_submit(self.nodes[0], [test2tx], accept=True) self.log.info("Test 4: Non-NULLDUMMY base multisig transaction is invalid after activation") test4tx = create_transaction(self.nodes[0], test2tx.hash, self.address, amount=46) test6txs = [CTransaction(test4tx)] trueDummy(test4tx) assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test4tx.serialize_with_witness().hex(), 0) - self.block_submit(self.nodes[0], [test4tx]) + self.block_submit(self.nodes[0], [test4tx], accept=False) self.log.info("Test 5: Non-NULLDUMMY P2WSH multisig transaction invalid after activation") test5tx = create_transaction(self.nodes[0], txid3, self.wit_address, amount=48) test6txs.append(CTransaction(test5tx)) test5tx.wit.vtxinwit[0].scriptWitness.stack[0] = b'\x01' assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test5tx.serialize_with_witness().hex(), 0) - self.block_submit(self.nodes[0], [test5tx], True) + self.block_submit(self.nodes[0], [test5tx], with_witness=True, accept=False) self.log.info(f"Test 6: NULLDUMMY compliant base/witness transactions should be accepted to mempool and in block after activation [{COINBASE_MATURITY + 5}]") for i in test6txs: self.nodes[0].sendrawtransaction(i.serialize_with_witness().hex(), 0) - self.block_submit(self.nodes[0], test6txs, True, True) + self.block_submit(self.nodes[0], test6txs, with_witness=True, accept=True) - def block_submit(self, node, txs, witness=False, accept=False): + def block_submit(self, node, txs, *, with_witness=False, accept): tmpl = node.getblocktemplate(NORMAL_GBT_REQUEST_PARAMS) assert_equal(tmpl['previousblockhash'], self.lastblockhash) assert_equal(tmpl['height'], self.lastblockheight + 1) @@ -124,11 +128,12 @@ class NULLDUMMYTest(BitcoinTestFramework): tx.rehash() block.vtx.append(tx) block.hashMerkleRoot = block.calc_merkle_root() - witness and add_witness_commitment(block) + if with_witness: + add_witness_commitment(block) block.rehash() block.solve() - assert_equal(None if accept else 'block-validation-failed', node.submitblock(block.serialize().hex())) - if (accept): + assert_equal(None if accept else NULLDUMMY_ERROR, node.submitblock(block.serialize().hex())) + if accept: assert_equal(node.getbestblockhash(), block.hash) self.lastblockhash = block.hash self.lastblocktime += 1 diff --git a/test/functional/feature_presegwit_node_upgrade.py b/test/functional/feature_presegwit_node_upgrade.py new file mode 100755 index 0000000000..aac42d4dbf --- /dev/null +++ b/test/functional/feature_presegwit_node_upgrade.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test a pre-segwit node upgrading to segwit consensus""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + softfork_active, +) +import os + + +class SegwitUpgradeTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + self.extra_args = [["-testactivationheight=segwit@10"]] + + def run_test(self): + """A pre-segwit node with insufficiently validated blocks needs to redownload blocks""" + + self.log.info("Testing upgrade behaviour for pre-segwit node to segwit rules") + node = self.nodes[0] + + # Node hasn't been used or connected yet + assert_equal(node.getblockcount(), 0) + + assert not softfork_active(node, "segwit") + + # Generate 8 blocks without witness data + self.generate(node, 8) + assert_equal(node.getblockcount(), 8) + + self.stop_node(0) + # Restarting the node (with segwit activation height set to 5) should result in a shutdown + # because the blockchain consists of 3 insufficiently validated blocks per segwit consensus rules. + node.assert_start_raises_init_error( + extra_args=["-testactivationheight=segwit@5"], + expected_msg=": Witness data for blocks after height 5 requires " + f"validation. Please restart with -reindex..{os.linesep}" + "Please restart with -reindex or -reindex-chainstate to recover.", + ) + + # As directed, the user restarts the node with -reindex + self.start_node(0, extra_args=["-reindex", "-testactivationheight=segwit@5"]) + + # With the segwit consensus rules, the node is able to validate only up to block 4 + assert_equal(node.getblockcount(), 4) + + # The upgraded node should now have segwit activated + assert softfork_active(node, "segwit") + + +if __name__ == '__main__': + SegwitUpgradeTest().main() diff --git a/test/functional/feature_proxy.py b/test/functional/feature_proxy.py index 162814815e..2fb5e328f5 100755 --- a/test/functional/feature_proxy.py +++ b/test/functional/feature_proxy.py @@ -97,14 +97,14 @@ class ProxyTest(BitcoinTestFramework): # Note: proxies are not used to connect to local nodes. This is because the proxy to # use is based on CService.GetNetwork(), which returns NET_UNROUTABLE for localhost. args = [ - ['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-proxyrandomize=1'], - ['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-onion=%s:%i' % (self.conf2.addr), - '-i2psam=%s:%i' % (self.i2p_sam), '-i2pacceptincoming=0', '-proxyrandomize=0'], - ['-listen', '-proxy=%s:%i' % (self.conf2.addr),'-proxyrandomize=1'], + ['-listen', f'-proxy={self.conf1.addr[0]}:{self.conf1.addr[1]}','-proxyrandomize=1'], + ['-listen', f'-proxy={self.conf1.addr[0]}:{self.conf1.addr[1]}',f'-onion={self.conf2.addr[0]}:{self.conf2.addr[1]}', + f'-i2psam={self.i2p_sam[0]}:{self.i2p_sam[1]}', '-i2pacceptincoming=0', '-proxyrandomize=0'], + ['-listen', f'-proxy={self.conf2.addr[0]}:{self.conf2.addr[1]}','-proxyrandomize=1'], [] ] if self.have_ipv6: - args[3] = ['-listen', '-proxy=[%s]:%i' % (self.conf3.addr),'-proxyrandomize=0', '-noonion'] + args[3] = ['-listen', f'-proxy=[{self.conf3.addr[0]}]:{self.conf3.addr[1]}','-proxyrandomize=0', '-noonion'] self.add_nodes(self.num_nodes, extra_args=args) self.start_nodes() @@ -116,7 +116,7 @@ class ProxyTest(BitcoinTestFramework): def node_test(self, node, proxies, auth, test_onion=True): rv = [] addr = "15.61.23.23:1234" - self.log.debug("Test: outgoing IPv4 connection through node for address {}".format(addr)) + self.log.debug(f"Test: outgoing IPv4 connection through node for address {addr}") node.addnode(addr, "onetry") cmd = proxies[0].queue.get() assert isinstance(cmd, Socks5Command) @@ -132,7 +132,7 @@ class ProxyTest(BitcoinTestFramework): if self.have_ipv6: addr = "[1233:3432:2434:2343:3234:2345:6546:4534]:5443" - self.log.debug("Test: outgoing IPv6 connection through node for address {}".format(addr)) + self.log.debug(f"Test: outgoing IPv6 connection through node for address {addr}") node.addnode(addr, "onetry") cmd = proxies[1].queue.get() assert isinstance(cmd, Socks5Command) @@ -148,7 +148,7 @@ class ProxyTest(BitcoinTestFramework): if test_onion: addr = "pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion:8333" - self.log.debug("Test: outgoing onion connection through node for address {}".format(addr)) + self.log.debug(f"Test: outgoing onion connection through node for address {addr}") node.addnode(addr, "onetry") cmd = proxies[2].queue.get() assert isinstance(cmd, Socks5Command) @@ -162,7 +162,7 @@ class ProxyTest(BitcoinTestFramework): self.network_test(node, addr, network=NET_ONION) addr = "node.noumenon:8333" - self.log.debug("Test: outgoing DNS name connection through node for address {}".format(addr)) + self.log.debug(f"Test: outgoing DNS name connection through node for address {addr}") node.addnode(addr, "onetry") cmd = proxies[3].queue.get() assert isinstance(cmd, Socks5Command) @@ -218,12 +218,12 @@ class ProxyTest(BitcoinTestFramework): n1 = networks_dict(self.nodes[1].getnetworkinfo()) assert_equal(NETWORKS, n1.keys()) for net in ['ipv4', 'ipv6']: - assert_equal(n1[net]['proxy'], '%s:%i' % (self.conf1.addr)) + assert_equal(n1[net]['proxy'], f'{self.conf1.addr[0]}:{self.conf1.addr[1]}') assert_equal(n1[net]['proxy_randomize_credentials'], False) - assert_equal(n1['onion']['proxy'], '%s:%i' % (self.conf2.addr)) + assert_equal(n1['onion']['proxy'], f'{self.conf2.addr[0]}:{self.conf2.addr[1]}') assert_equal(n1['onion']['proxy_randomize_credentials'], False) assert_equal(n1['onion']['reachable'], True) - assert_equal(n1['i2p']['proxy'], '%s:%i' % (self.i2p_sam)) + assert_equal(n1['i2p']['proxy'], f'{self.i2p_sam[0]}:{self.i2p_sam[1]}') assert_equal(n1['i2p']['proxy_randomize_credentials'], False) assert_equal(n1['i2p']['reachable'], True) @@ -234,7 +234,7 @@ class ProxyTest(BitcoinTestFramework): expected_proxy = '' expected_randomize = False else: - expected_proxy = '%s:%i' % (self.conf2.addr) + expected_proxy = f'{self.conf2.addr[0]}:{self.conf2.addr[1]}' expected_randomize = True assert_equal(n2[net]['proxy'], expected_proxy) assert_equal(n2[net]['proxy_randomize_credentials'], expected_randomize) @@ -248,7 +248,7 @@ class ProxyTest(BitcoinTestFramework): if net == NET_I2P: expected_proxy = '' else: - expected_proxy = '[%s]:%i' % (self.conf3.addr) + expected_proxy = f'[{self.conf3.addr[0]}]:{self.conf3.addr[1]}' assert_equal(n3[net]['proxy'], expected_proxy) assert_equal(n3[net]['proxy_randomize_credentials'], False) assert_equal(n3['onion']['reachable'], False) diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index cedb7b57ca..c2463d0bcc 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -118,19 +118,37 @@ class PruneTest(BitcoinTestFramework): def create_big_chain(self): # Start by creating some coinbases we can spend later - self.nodes[1].generate(200) + self.generate(self.nodes[1], 200) self.sync_blocks(self.nodes[0:2]) - self.nodes[0].generate(150) + self.generate(self.nodes[0], 150) # Then mine enough full blocks to create more than 550MiB of data mine_large_blocks(self.nodes[0], 645) self.sync_blocks(self.nodes[0:5]) + def test_invalid_command_line_options(self): + self.nodes[0].assert_start_raises_init_error( + expected_msg='Error: Prune cannot be configured with a negative value.', + extra_args=['-prune=-1'], + ) + self.nodes[0].assert_start_raises_init_error( + expected_msg='Error: Prune configured below the minimum of 550 MiB. Please use a higher number.', + extra_args=['-prune=549'], + ) + self.nodes[0].assert_start_raises_init_error( + expected_msg='Error: Prune mode is incompatible with -txindex.', + extra_args=['-prune=550', '-txindex'], + ) + self.nodes[0].assert_start_raises_init_error( + expected_msg='Error: Prune mode is incompatible with -coinstatsindex.', + extra_args=['-prune=550', '-coinstatsindex'], + ) + def test_height_min(self): assert os.path.isfile(os.path.join(self.prunedir, "blk00000.dat")), "blk00000.dat is missing, pruning too early" self.log.info("Success") - self.log.info("Though we're already using more than 550MiB, current usage: %d" % calc_usage(self.prunedir)) + self.log.info(f"Though we're already using more than 550MiB, current usage: {calc_usage(self.prunedir)}") self.log.info("Mining 25 more blocks should cause the first block file to be pruned") # Pruning doesn't run until we're allocating another chunk, 20 full blocks past the height cutoff will ensure this mine_large_blocks(self.nodes[0], 25) @@ -140,7 +158,7 @@ class PruneTest(BitcoinTestFramework): self.log.info("Success") usage = calc_usage(self.prunedir) - self.log.info("Usage should be below target: %d" % usage) + self.log.info(f"Usage should be below target: {usage}") assert_greater_than(550, usage) def create_chain_with_staleblocks(self): @@ -163,18 +181,18 @@ class PruneTest(BitcoinTestFramework): self.connect_nodes(0, 2) self.sync_blocks(self.nodes[0:3]) - self.log.info("Usage can be over target because of high stale rate: %d" % calc_usage(self.prunedir)) + self.log.info(f"Usage can be over target because of high stale rate: {calc_usage(self.prunedir)}") def reorg_test(self): # Node 1 will mine a 300 block chain starting 287 blocks back from Node 0 and Node 2's tip # This will cause Node 2 to do a reorg requiring 288 blocks of undo data to the reorg_test chain height = self.nodes[1].getblockcount() - self.log.info("Current block height: %d" % height) + self.log.info(f"Current block height: {height}") self.forkheight = height - 287 self.forkhash = self.nodes[1].getblockhash(self.forkheight) - self.log.info("Invalidating block %s at height %d" % (self.forkhash, self.forkheight)) + self.log.info(f"Invalidating block {self.forkhash} at height {self.forkheight}") self.nodes[1].invalidateblock(self.forkhash) # We've now switched to our previously mined-24 block fork on node 1, but that's not what we want @@ -186,22 +204,22 @@ class PruneTest(BitcoinTestFramework): curhash = self.nodes[1].getblockhash(self.forkheight - 1) assert self.nodes[1].getblockcount() == self.forkheight - 1 - self.log.info("New best height: %d" % self.nodes[1].getblockcount()) + self.log.info(f"New best height: {self.nodes[1].getblockcount()}") # Disconnect node1 and generate the new chain self.disconnect_nodes(0, 1) self.disconnect_nodes(1, 2) self.log.info("Generating new longer chain of 300 more blocks") - self.nodes[1].generate(300) + self.generate(self.nodes[1], 300) self.log.info("Reconnect nodes") self.connect_nodes(0, 1) self.connect_nodes(1, 2) self.sync_blocks(self.nodes[0:3], timeout=120) - self.log.info("Verify height on node 2: %d" % self.nodes[2].getblockcount()) - self.log.info("Usage possibly still high because of stale blocks in block files: %d" % calc_usage(self.prunedir)) + self.log.info(f"Verify height on node 2: {self.nodes[2].getblockcount()}") + self.log.info(f"Usage possibly still high because of stale blocks in block files: {calc_usage(self.prunedir)}") self.log.info("Mine 220 more large blocks so we have requisite history") @@ -209,7 +227,7 @@ class PruneTest(BitcoinTestFramework): self.sync_blocks(self.nodes[0:3], timeout=120) usage = calc_usage(self.prunedir) - self.log.info("Usage should be below target: %d" % usage) + self.log.info(f"Usage should be below target: {usage}") assert_greater_than(550, usage) def reorg_back(self): @@ -217,7 +235,7 @@ class PruneTest(BitcoinTestFramework): assert_raises_rpc_error(-1, "Block not available (pruned data)", self.nodes[2].getblock, self.forkhash) with self.nodes[2].assert_debug_log(expected_msgs=['block verification stopping at height', '(pruning, no data)']): self.nodes[2].verifychain(checklevel=4, nblocks=0) - self.log.info("Will need to redownload block %d" % self.forkheight) + self.log.info(f"Will need to redownload block {self.forkheight}") # Verify that we have enough history to reorg back to the fork point # Although this is more than 288 blocks, because this chain was written more recently @@ -241,11 +259,11 @@ class PruneTest(BitcoinTestFramework): # At this point node 2 is within 288 blocks of the fork point so it will preserve its ability to reorg if self.nodes[2].getblockcount() < self.mainchainheight: blocks_to_mine = first_reorg_height + 1 - self.mainchainheight - self.log.info("Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed: %d" % blocks_to_mine) + self.log.info(f"Rewind node 0 to prev main chain to mine longer chain to trigger redownload. Blocks needed: {blocks_to_mine}") self.nodes[0].invalidateblock(curchainhash) assert_equal(self.nodes[0].getblockcount(), self.mainchainheight) assert_equal(self.nodes[0].getbestblockhash(), self.mainchainhash2) - goalbesthash = self.nodes[0].generate(blocks_to_mine)[-1] + goalbesthash = self.generate(self.nodes[0], blocks_to_mine)[-1] goalbestheight = first_reorg_height + 1 self.log.info("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload") @@ -278,7 +296,7 @@ class PruneTest(BitcoinTestFramework): assert_equal(ret, node.getblockchaininfo()['pruneheight']) def has_block(index): - return os.path.isfile(os.path.join(self.nodes[node_number].datadir, self.chain, "blocks", "blk{:05}.dat".format(index))) + return os.path.isfile(os.path.join(self.nodes[node_number].datadir, self.chain, "blocks", f"blk{index:05}.dat")) # should not prune because chain tip of node 3 (995) < PruneAfterHeight (1000) assert_raises_rpc_error(-1, "Blockchain is too short for pruning", node.pruneblockchain, height(500)) @@ -288,7 +306,7 @@ class PruneTest(BitcoinTestFramework): assert_equal(block1_details["nTx"], len(block1_details["tx"])) # mine 6 blocks so we are at height 1001 (i.e., above PruneAfterHeight) - node.generate(6) + self.generate(node, 6) assert_equal(node.getblockchaininfo()["blocks"], 1001) # Pruned block should still know the number of transactions @@ -319,7 +337,7 @@ class PruneTest(BitcoinTestFramework): assert has_block(2), "blk00002.dat is still there, should be pruned by now" # advance the tip so blk00002.dat and blk00003.dat can be pruned (the last 288 blocks should now be in blk00004.dat) - node.generate(288) + self.generate(node, 288) prune(1000) assert not has_block(2), "blk00002.dat is still there, should be pruned by now" assert not has_block(3), "blk00003.dat is still there, should be pruned by now" @@ -453,6 +471,9 @@ class PruneTest(BitcoinTestFramework): self.log.info("Test wallet re-scan") self.wallet_test() + self.log.info("Test invalid pruning command line options") + self.test_invalid_command_line_options() + self.log.info("Done") if __name__ == '__main__': diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index e42f8045db..4eaaf46454 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -4,9 +4,9 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RBF code.""" +from copy import deepcopy from decimal import Decimal -from test_framework.blocktools import COINBASE_MATURITY from test_framework.messages import ( BIP125_SEQUENCE_NUMBER, COIN, @@ -17,54 +17,18 @@ from test_framework.messages import ( ) from test_framework.script import CScript, OP_DROP from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_raises_rpc_error, satoshi_round -from test_framework.script_util import DUMMY_P2WPKH_SCRIPT, DUMMY_2_P2WPKH_SCRIPT +from test_framework.util import ( + assert_equal, + assert_raises_rpc_error, +) +from test_framework.script_util import ( + DUMMY_P2WPKH_SCRIPT, + DUMMY_2_P2WPKH_SCRIPT, +) from test_framework.wallet import MiniWallet +from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE MAX_REPLACEMENT_LIMIT = 100 - - -def make_utxo(node, amount, confirmed=True, scriptPubKey=DUMMY_P2WPKH_SCRIPT): - """Create a txout with a given amount and scriptPubKey - - Mines coins as needed. - - confirmed - txouts created will be confirmed in the blockchain; - unconfirmed otherwise. - """ - fee = 1 * COIN - while node.getbalance() < satoshi_round((amount + fee) / COIN): - node.generate(COINBASE_MATURITY) - - new_addr = node.getnewaddress() - txid = node.sendtoaddress(new_addr, satoshi_round((amount + fee) / COIN)) - tx1 = node.getrawtransaction(txid, 1) - txid = int(txid, 16) - i, _ = next(filter(lambda vout: new_addr == vout[1]['scriptPubKey']['address'], enumerate(tx1['vout']))) - - tx2 = CTransaction() - tx2.vin = [CTxIn(COutPoint(txid, i))] - tx2.vout = [CTxOut(amount, scriptPubKey)] - tx2.rehash() - - signed_tx = node.signrawtransactionwithwallet(tx2.serialize().hex()) - - txid = node.sendrawtransaction(signed_tx['hex'], 0) - - # If requested, ensure txouts are confirmed. - if confirmed: - mempool_size = len(node.getrawmempool()) - while mempool_size > 0: - node.generate(1) - new_size = len(node.getrawmempool()) - # Error out if we have something stuck in the mempool, as this - # would likely be a bug. - assert new_size < mempool_size - mempool_size = new_size - - return COutPoint(int(txid, 16), 0) - - class ReplaceByFeeTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 @@ -80,14 +44,12 @@ class ReplaceByFeeTest(BitcoinTestFramework): ] self.supports_cli = False - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def run_test(self): - make_utxo(self.nodes[0], 1 * COIN) - - # Ensure nodes are synced - self.sync_all() + self.wallet = MiniWallet(self.nodes[0]) + # the pre-mined test framework chain contains coinbase outputs to the + # MiniWallet's default address ADDRESS_BCRT1_P2WSH_OP_TRUE in blocks + # 76-100 (see method BitcoinTestFramework._initialize_chain()) + self.wallet.rescan_utxos() self.log.info("Running test simple doublespend...") self.test_simple_doublespend() @@ -122,28 +84,45 @@ class ReplaceByFeeTest(BitcoinTestFramework): self.log.info("Running test no inherited signaling...") self.test_no_inherited_signaling() + self.log.info("Running test replacement relay fee...") + self.test_replacement_relay_fee() + self.log.info("Passed") + def make_utxo(self, node, amount, confirmed=True, scriptPubKey=DUMMY_P2WPKH_SCRIPT): + """Create a txout with a given amount and scriptPubKey + + confirmed - txouts created will be confirmed in the blockchain; + unconfirmed otherwise. + """ + txid, n = self.wallet.send_to(from_node=node, scriptPubKey=scriptPubKey, amount=amount) + + # If requested, ensure txouts are confirmed. + if confirmed: + mempool_size = len(node.getrawmempool()) + while mempool_size > 0: + self.generate(node, 1) + new_size = len(node.getrawmempool()) + # Error out if we have something stuck in the mempool, as this + # would likely be a bug. + assert new_size < mempool_size + mempool_size = new_size + + return COutPoint(int(txid, 16), n) + def test_simple_doublespend(self): """Simple doublespend""" - tx0_outpoint = make_utxo(self.nodes[0], int(1.1 * COIN)) - - # make_utxo may have generated a bunch of blocks, so we need to sync - # before we can spend the coins generated, or else the resulting - # transactions might not be accepted by our peers. - self.sync_all() + # we use MiniWallet to create a transaction template with inputs correctly set, + # and modify the output (amount, scriptPubKey) according to our needs + tx_template = self.wallet.create_self_transfer(from_node=self.nodes[0])['tx'] - tx1a = CTransaction() - tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0)] + tx1a = deepcopy(tx_template) tx1a.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] tx1a_hex = tx1a.serialize().hex() tx1a_txid = self.nodes[0].sendrawtransaction(tx1a_hex, 0) - self.sync_all() - # Should fail because we haven't changed the fee - tx1b = CTransaction() - tx1b.vin = [CTxIn(tx0_outpoint, nSequence=0)] + tx1b = deepcopy(tx_template) tx1b.vout = [CTxOut(1 * COIN, DUMMY_2_P2WPKH_SCRIPT)] tx1b_hex = tx1b.serialize().hex() @@ -151,9 +130,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx1b_hex, 0) # Extra 0.1 BTC fee - tx1b = CTransaction() - tx1b.vin = [CTxIn(tx0_outpoint, nSequence=0)] - tx1b.vout = [CTxOut(int(0.9 * COIN), DUMMY_P2WPKH_SCRIPT)] + tx1b.vout[0].nValue -= int(0.1 * COIN) tx1b_hex = tx1b.serialize().hex() # Works when enabled tx1b_txid = self.nodes[0].sendrawtransaction(tx1b_hex, 0) @@ -168,14 +145,14 @@ class ReplaceByFeeTest(BitcoinTestFramework): def test_doublespend_chain(self): """Doublespend of a long chain""" - initial_nValue = 50 * COIN - tx0_outpoint = make_utxo(self.nodes[0], initial_nValue) + initial_nValue = 5 * COIN + tx0_outpoint = self.make_utxo(self.nodes[0], initial_nValue) prevout = tx0_outpoint remaining_value = initial_nValue chain_txids = [] - while remaining_value > 10 * COIN: - remaining_value -= 1 * COIN + while remaining_value > 1 * COIN: + remaining_value -= int(0.1 * COIN) tx = CTransaction() tx.vin = [CTxIn(prevout, nSequence=0)] tx.vout = [CTxOut(remaining_value, CScript([1, OP_DROP] * 15 + [1]))] @@ -185,10 +162,10 @@ class ReplaceByFeeTest(BitcoinTestFramework): prevout = COutPoint(int(txid, 16), 0) # Whether the double-spend is allowed is evaluated by including all - # child fees - 40 BTC - so this attempt is rejected. + # child fees - 4 BTC - so this attempt is rejected. dbl_tx = CTransaction() dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] - dbl_tx.vout = [CTxOut(initial_nValue - 30 * COIN, DUMMY_P2WPKH_SCRIPT)] + dbl_tx.vout = [CTxOut(initial_nValue - 3 * COIN, DUMMY_P2WPKH_SCRIPT)] dbl_tx_hex = dbl_tx.serialize().hex() # This will raise an exception due to insufficient fee @@ -197,7 +174,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): # Accepted with sufficient fee dbl_tx = CTransaction() dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] - dbl_tx.vout = [CTxOut(1 * COIN, DUMMY_P2WPKH_SCRIPT)] + dbl_tx.vout = [CTxOut(int(0.1 * COIN), DUMMY_P2WPKH_SCRIPT)] dbl_tx_hex = dbl_tx.serialize().hex() self.nodes[0].sendrawtransaction(dbl_tx_hex, 0) @@ -208,10 +185,10 @@ class ReplaceByFeeTest(BitcoinTestFramework): def test_doublespend_tree(self): """Doublespend of a big tree of transactions""" - initial_nValue = 50 * COIN - tx0_outpoint = make_utxo(self.nodes[0], initial_nValue) + initial_nValue = 5 * COIN + tx0_outpoint = self.make_utxo(self.nodes[0], initial_nValue) - def branch(prevout, initial_value, max_txs, tree_width=5, fee=0.0001 * COIN, _total_txs=None): + def branch(prevout, initial_value, max_txs, tree_width=5, fee=0.00001 * COIN, _total_txs=None): if _total_txs is None: _total_txs = [0] if _total_txs[0] >= max_txs: @@ -242,7 +219,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): _total_txs=_total_txs): yield x - fee = int(0.0001 * COIN) + fee = int(0.00001 * COIN) n = MAX_REPLACEMENT_LIMIT tree_txs = list(branch(tx0_outpoint, initial_nValue, n, fee=fee)) assert_equal(len(tree_txs), n) @@ -255,10 +232,10 @@ class ReplaceByFeeTest(BitcoinTestFramework): # This will raise an exception due to insufficient fee assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, dbl_tx_hex, 0) - # 1 BTC fee is enough + # 0.1 BTC fee is enough dbl_tx = CTransaction() dbl_tx.vin = [CTxIn(tx0_outpoint, nSequence=0)] - dbl_tx.vout = [CTxOut(initial_nValue - fee * n - 1 * COIN, DUMMY_P2WPKH_SCRIPT)] + dbl_tx.vout = [CTxOut(initial_nValue - fee * n - int(0.1 * COIN), DUMMY_P2WPKH_SCRIPT)] dbl_tx_hex = dbl_tx.serialize().hex() self.nodes[0].sendrawtransaction(dbl_tx_hex, 0) @@ -271,8 +248,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): # Try again, but with more total transactions than the "max txs # double-spent at once" anti-DoS limit. for n in (MAX_REPLACEMENT_LIMIT + 1, MAX_REPLACEMENT_LIMIT * 2): - fee = int(0.0001 * COIN) - tx0_outpoint = make_utxo(self.nodes[0], initial_nValue) + fee = int(0.00001 * COIN) + tx0_outpoint = self.make_utxo(self.nodes[0], initial_nValue) tree_txs = list(branch(tx0_outpoint, initial_nValue, n, fee=fee)) assert_equal(len(tree_txs), n) @@ -289,7 +266,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): def test_replacement_feeperkb(self): """Replacement requires fee-per-KB to be higher""" - tx0_outpoint = make_utxo(self.nodes[0], int(1.1 * COIN)) + tx0_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) tx1a = CTransaction() tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0)] @@ -309,8 +286,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): def test_spends_of_conflicting_outputs(self): """Replacements that spend conflicting tx outputs are rejected""" - utxo1 = make_utxo(self.nodes[0], int(1.2 * COIN)) - utxo2 = make_utxo(self.nodes[0], 3 * COIN) + utxo1 = self.make_utxo(self.nodes[0], int(1.2 * COIN)) + utxo2 = self.make_utxo(self.nodes[0], 3 * COIN) tx1a = CTransaction() tx1a.vin = [CTxIn(utxo1, nSequence=0)] @@ -349,8 +326,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): def test_new_unconfirmed_inputs(self): """Replacements that add new unconfirmed inputs are rejected""" - confirmed_utxo = make_utxo(self.nodes[0], int(1.1 * COIN)) - unconfirmed_utxo = make_utxo(self.nodes[0], int(0.1 * COIN), False) + confirmed_utxo = self.make_utxo(self.nodes[0], int(1.1 * COIN)) + unconfirmed_utxo = self.make_utxo(self.nodes[0], int(0.1 * COIN), False) tx1 = CTransaction() tx1.vin = [CTxIn(confirmed_utxo)] @@ -373,7 +350,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): # Start by creating a single transaction with many outputs initial_nValue = 10 * COIN - utxo = make_utxo(self.nodes[0], initial_nValue) + utxo = self.make_utxo(self.nodes[0], initial_nValue) fee = int(0.0001 * COIN) split_value = int((initial_nValue - fee) / (MAX_REPLACEMENT_LIMIT + 1)) @@ -421,7 +398,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): def test_opt_in(self): """Replacing should only work if orig tx opted in""" - tx0_outpoint = make_utxo(self.nodes[0], int(1.1 * COIN)) + tx0_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) # Create a non-opting in transaction tx1a = CTransaction() @@ -442,7 +419,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): # This will raise an exception assert_raises_rpc_error(-26, "txn-mempool-conflict", self.nodes[0].sendrawtransaction, tx1b_hex, 0) - tx1_outpoint = make_utxo(self.nodes[0], int(1.1 * COIN)) + tx1_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) # Create a different non-opting in transaction tx2a = CTransaction() @@ -498,7 +475,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): # correctly used by replacement logic # 1. Check that feeperkb uses modified fees - tx0_outpoint = make_utxo(self.nodes[0], int(1.1 * COIN)) + tx0_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) tx1a = CTransaction() tx1a.vin = [CTxIn(tx0_outpoint, nSequence=0)] @@ -524,7 +501,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): assert tx1b_txid in self.nodes[0].getrawmempool() # 2. Check that absolute fee checks use modified fee. - tx1_outpoint = make_utxo(self.nodes[0], int(1.1 * COIN)) + tx1_outpoint = self.make_utxo(self.nodes[0], int(1.1 * COIN)) tx2a = CTransaction() tx2a.vin = [CTxIn(tx1_outpoint, nSequence=0)] @@ -551,9 +528,9 @@ class ReplaceByFeeTest(BitcoinTestFramework): assert tx2b_txid in self.nodes[0].getrawmempool() def test_rpc(self): - us0 = self.nodes[0].listunspent()[0] + us0 = self.wallet.get_utxo() ins = [us0] - outs = {self.nodes[0].getnewaddress(): Decimal(1.0000000)} + outs = {ADDRESS_BCRT1_UNSPENDABLE: Decimal(1.0000000)} rawtx0 = self.nodes[0].createrawtransaction(ins, outs, 0, True) rawtx1 = self.nodes[0].createrawtransaction(ins, outs, 0, False) json0 = self.nodes[0].decoderawtransaction(rawtx0) @@ -561,22 +538,22 @@ class ReplaceByFeeTest(BitcoinTestFramework): assert_equal(json0["vin"][0]["sequence"], 4294967293) assert_equal(json1["vin"][0]["sequence"], 4294967295) - rawtx2 = self.nodes[0].createrawtransaction([], outs) - frawtx2a = self.nodes[0].fundrawtransaction(rawtx2, {"replaceable": True}) - frawtx2b = self.nodes[0].fundrawtransaction(rawtx2, {"replaceable": False}) + if self.is_wallet_compiled(): + self.init_wallet(0) + rawtx2 = self.nodes[0].createrawtransaction([], outs) + frawtx2a = self.nodes[0].fundrawtransaction(rawtx2, {"replaceable": True}) + frawtx2b = self.nodes[0].fundrawtransaction(rawtx2, {"replaceable": False}) - json0 = self.nodes[0].decoderawtransaction(frawtx2a['hex']) - json1 = self.nodes[0].decoderawtransaction(frawtx2b['hex']) - assert_equal(json0["vin"][0]["sequence"], 4294967293) - assert_equal(json1["vin"][0]["sequence"], 4294967294) + json0 = self.nodes[0].decoderawtransaction(frawtx2a['hex']) + json1 = self.nodes[0].decoderawtransaction(frawtx2b['hex']) + assert_equal(json0["vin"][0]["sequence"], 4294967293) + assert_equal(json1["vin"][0]["sequence"], 4294967294) def test_no_inherited_signaling(self): - wallet = MiniWallet(self.nodes[0]) - wallet.scan_blocks(start=76, num=1) - confirmed_utxo = wallet.get_utxo() + confirmed_utxo = self.wallet.get_utxo() # Create an explicitly opt-in parent transaction - optin_parent_tx = wallet.send_self_transfer( + optin_parent_tx = self.wallet.send_self_transfer( from_node=self.nodes[0], utxo_to_spend=confirmed_utxo, sequence=BIP125_SEQUENCE_NUMBER, @@ -584,7 +561,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): ) assert_equal(True, self.nodes[0].getmempoolentry(optin_parent_tx['txid'])['bip125-replaceable']) - replacement_parent_tx = wallet.create_self_transfer( + replacement_parent_tx = self.wallet.create_self_transfer( from_node=self.nodes[0], utxo_to_spend=confirmed_utxo, sequence=BIP125_SEQUENCE_NUMBER, @@ -598,8 +575,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): assert_equal(res['allowed'], True) # Create an opt-out child tx spending the opt-in parent - parent_utxo = wallet.get_utxo(txid=optin_parent_tx['txid']) - optout_child_tx = wallet.send_self_transfer( + parent_utxo = self.wallet.get_utxo(txid=optin_parent_tx['txid']) + optout_child_tx = self.wallet.send_self_transfer( from_node=self.nodes[0], utxo_to_spend=parent_utxo, sequence=0xffffffff, @@ -609,7 +586,7 @@ class ReplaceByFeeTest(BitcoinTestFramework): # Reports true due to inheritance assert_equal(True, self.nodes[0].getmempoolentry(optout_child_tx['txid'])['bip125-replaceable']) - replacement_child_tx = wallet.create_self_transfer( + replacement_child_tx = self.wallet.create_self_transfer( from_node=self.nodes[0], utxo_to_spend=parent_utxo, sequence=0xffffffff, @@ -627,6 +604,24 @@ class ReplaceByFeeTest(BitcoinTestFramework): assert_equal(True, self.nodes[0].getmempoolentry(optin_parent_tx['txid'])['bip125-replaceable']) assert_raises_rpc_error(-26, 'txn-mempool-conflict', self.nodes[0].sendrawtransaction, replacement_child_tx["hex"], 0) + self.log.info('Check that the child tx can still be replaced (via a tx that also replaces the parent)') + replacement_parent_tx = self.wallet.send_self_transfer( + from_node=self.nodes[0], + utxo_to_spend=confirmed_utxo, + sequence=0xffffffff, + fee_rate=Decimal('0.03'), + ) + # Check that child is removed and update wallet utxo state + assert_raises_rpc_error(-5, 'Transaction not in mempool', self.nodes[0].getmempoolentry, optout_child_tx['txid']) + self.wallet.get_utxo(txid=optout_child_tx['txid']) + + def test_replacement_relay_fee(self): + tx = self.wallet.send_self_transfer(from_node=self.nodes[0])['tx'] + + # Higher fee, higher feerate, different txid, but the replacement does not provide a relay + # fee conforming to node's `incrementalrelayfee` policy of 1000 sat per KB. + tx.vout[0].nValue -= 1 + assert_raises_rpc_error(-26, "insufficient fee", self.nodes[0].sendrawtransaction, tx.serialize().hex()) if __name__ == '__main__': ReplaceByFeeTest().main() diff --git a/test/functional/feature_reindex.py b/test/functional/feature_reindex.py index 68585b7475..f0435b21b2 100755 --- a/test/functional/feature_reindex.py +++ b/test/functional/feature_reindex.py @@ -19,7 +19,7 @@ class ReindexTest(BitcoinTestFramework): self.num_nodes = 1 def reindex(self, justchainstate=False): - self.nodes[0].generatetoaddress(3, self.nodes[0].get_deterministic_priv_key().address) + self.generatetoaddress(self.nodes[0], 3, self.nodes[0].get_deterministic_priv_key().address) blockcount = self.nodes[0].getblockcount() self.stop_nodes() extra_args = [["-reindex-chainstate" if justchainstate else "-reindex"]] diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 42910904d7..2f9ab34e99 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -23,7 +23,6 @@ from test_framework.messages import ( CTransaction, CTxIn, CTxOut, - sha256, tx_from_hex, ) from test_framework.script import ( @@ -32,21 +31,22 @@ from test_framework.script import ( OP_1, OP_2, OP_CHECKMULTISIG, - OP_CHECKSIG, OP_DROP, - OP_DUP, - OP_EQUAL, - OP_EQUALVERIFY, - OP_HASH160, OP_TRUE, - hash160, +) +from test_framework.script_util import ( + key_to_p2pk_script, + key_to_p2pkh_script, + key_to_p2wpkh_script, + script_to_p2sh_script, + script_to_p2wsh_script, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, + assert_greater_than_or_equal, assert_is_hex_string, assert_raises_rpc_error, - hex_str_to_bytes, try_rpc, ) @@ -55,20 +55,24 @@ NODE_2 = 2 P2WPKH = 0 P2WSH = 1 + def getutxo(txid): utxo = {} utxo["vout"] = 0 utxo["txid"] = txid return utxo + def find_spendable_utxo(node, min_value): for utxo in node.listunspent(query_options={'minimumAmount': min_value}): if utxo['spendable']: return utxo - raise AssertionError("Unspent output equal or higher than %s not found" % min_value) + raise AssertionError(f"Unspent output equal or higher than {min_value} not found") + + +txs_mined = {} # txindex from txid to blockhash -txs_mined = {} # txindex from txid to blockhash class SegWitTest(BitcoinTestFramework): def set_test_params(self): @@ -79,18 +83,18 @@ class SegWitTest(BitcoinTestFramework): [ "-acceptnonstdtxn=1", "-rpcserialversion=0", - "-segwitheight=432", + "-testactivationheight=segwit@432", "-addresstype=legacy", ], [ "-acceptnonstdtxn=1", "-rpcserialversion=1", - "-segwitheight=432", + "-testactivationheight=segwit@432", "-addresstype=legacy", ], [ "-acceptnonstdtxn=1", - "-segwitheight=432", + "-testactivationheight=segwit@432", "-addresstype=legacy", ], ] @@ -106,13 +110,13 @@ class SegWitTest(BitcoinTestFramework): def success_mine(self, node, txid, sign, redeem_script=""): send_to_witness(1, node, getutxo(txid), self.pubkey[0], False, Decimal("49.998"), sign, redeem_script) - block = node.generate(1) + block = self.generate(node, 1) assert_equal(len(node.getblock(block[0])["tx"]), 2) self.sync_blocks() def skip_mine(self, node, txid, sign, redeem_script=""): send_to_witness(1, node, getutxo(txid), self.pubkey[0], False, Decimal("49.998"), sign, redeem_script) - block = node.generate(1) + block = self.generate(node, 1) assert_equal(len(node.getblock(block[0])["tx"]), 1) self.sync_blocks() @@ -120,27 +124,27 @@ class SegWitTest(BitcoinTestFramework): assert_raises_rpc_error(-26, error_msg, send_to_witness, use_p2wsh=1, node=node, utxo=getutxo(txid), pubkey=self.pubkey[0], encode_p2sh=False, amount=Decimal("49.998"), sign=sign, insert_redeem_script=redeem_script) def run_test(self): - self.nodes[0].generate(161) # block 161 + self.generate(self.nodes[0], 161) # block 161 self.log.info("Verify sigops are counted in GBT with pre-BIP141 rules before the fork") txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) tmpl = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert tmpl['sizelimit'] == 1000000 + assert_equal(tmpl['sizelimit'], 1000000) assert 'weightlimit' not in tmpl - assert tmpl['sigoplimit'] == 20000 - assert tmpl['transactions'][0]['hash'] == txid - assert tmpl['transactions'][0]['sigops'] == 2 + assert_equal(tmpl['sigoplimit'], 20000) + assert_equal(tmpl['transactions'][0]['hash'], txid) + assert_equal(tmpl['transactions'][0]['sigops'], 2) assert '!segwit' not in tmpl['rules'] - self.nodes[0].generate(1) # block 162 + self.generate(self.nodes[0], 1) # block 162 balance_presetup = self.nodes[0].getbalance() self.pubkey = [] - p2sh_ids = [] # p2sh_ids[NODE][TYPE] is an array of txids that spend to P2WPKH (TYPE=0) or P2WSH (TYPE=1) scripts to an address for NODE embedded in p2sh - wit_ids = [] # wit_ids[NODE][TYPE] is an array of txids that spend to P2WPKH (TYPE=0) or P2WSH (TYPE=1) scripts to an address for NODE via bare witness + p2sh_ids = [] # p2sh_ids[NODE][TYPE] is an array of txids that spend to P2WPKH (TYPE=0) or P2WSH (TYPE=1) scripts to an address for NODE embedded in p2sh + wit_ids = [] # wit_ids[NODE][TYPE] is an array of txids that spend to P2WPKH (TYPE=0) or P2WSH (TYPE=1) scripts to an address for NODE via bare witness for i in range(3): newaddress = self.nodes[i].getnewaddress() self.pubkey.append(self.nodes[i].getaddressinfo(newaddress)["pubkey"]) - multiscript = CScript([OP_1, hex_str_to_bytes(self.pubkey[-1]), OP_1, OP_CHECKMULTISIG]) + multiscript = CScript([OP_1, bytes.fromhex(self.pubkey[-1]), OP_1, OP_CHECKMULTISIG]) p2sh_ms_addr = self.nodes[i].addmultisigaddress(1, [self.pubkey[-1]], '', 'p2sh-segwit')['address'] bip173_ms_addr = self.nodes[i].addmultisigaddress(1, [self.pubkey[-1]], '', 'bech32')['address'] assert_equal(p2sh_ms_addr, script_to_p2sh_p2wsh(multiscript)) @@ -157,7 +161,7 @@ class SegWitTest(BitcoinTestFramework): wit_ids[n][v].append(send_to_witness(v, self.nodes[0], find_spendable_utxo(self.nodes[0], 50), self.pubkey[n], False, Decimal("49.999"))) p2sh_ids[n][v].append(send_to_witness(v, self.nodes[0], find_spendable_utxo(self.nodes[0], 50), self.pubkey[n], True, Decimal("49.999"))) - self.nodes[0].generate(1) # block 163 + self.generate(self.nodes[0], 1) # block 163 self.sync_blocks() # Make sure all nodes recognize the transactions as theirs @@ -165,7 +169,7 @@ class SegWitTest(BitcoinTestFramework): assert_equal(self.nodes[1].getbalance(), 20 * Decimal("49.999")) assert_equal(self.nodes[2].getbalance(), 20 * Decimal("49.999")) - self.nodes[0].generate(260) # block 423 + self.generate(self.nodes[0], 260) # block 423 self.sync_blocks() self.log.info("Verify witness txs are skipped for mining before the fork") @@ -178,11 +182,11 @@ class SegWitTest(BitcoinTestFramework): self.fail_accept(self.nodes[2], "mandatory-script-verify-flag-failed (Operation not valid with the current stack size)", p2sh_ids[NODE_2][P2WPKH][1], sign=False) self.fail_accept(self.nodes[2], "mandatory-script-verify-flag-failed (Operation not valid with the current stack size)", p2sh_ids[NODE_2][P2WSH][1], sign=False) - self.nodes[2].generate(4) # blocks 428-431 + self.generate(self.nodes[2], 4) # blocks 428-431 self.log.info("Verify previous witness txs skipped for mining can now be mined") assert_equal(len(self.nodes[2].getrawmempool()), 4) - blockhash = self.nodes[2].generate(1)[0] # block 432 (first block with new rules; 432 = 144 * 3) + blockhash = self.generate(self.nodes[2], 1)[0] # block 432 (first block with new rules; 432 = 144 * 3) self.sync_blocks() assert_equal(len(self.nodes[2].getrawmempool()), 0) segwit_tx_list = self.nodes[2].getblock(blockhash)["tx"] @@ -216,7 +220,7 @@ class SegWitTest(BitcoinTestFramework): witnesses = coinbase_tx["decoded"]["vin"][0]["txinwitness"] assert_equal(len(witnesses), 1) assert_is_hex_string(witnesses[0]) - assert_equal(witnesses[0], '00'*32) + assert_equal(witnesses[0], '00' * 32) self.log.info("Verify witness txs without witness data are invalid after the fork") self.fail_accept(self.nodes[2], 'non-mandatory-script-verify-flag (Witness program hash mismatch)', wit_ids[NODE_2][P2WPKH][2], sign=False) @@ -232,15 +236,17 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Verify sigops are counted in GBT with BIP141 rules after the fork") txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) + raw_tx = self.nodes[0].getrawtransaction(txid, True) tmpl = self.nodes[0].getblocktemplate({'rules': ['segwit']}) - assert tmpl['sizelimit'] >= 3999577 # actual maximum size is lower due to minimum mandatory non-witness data - assert tmpl['weightlimit'] == 4000000 - assert tmpl['sigoplimit'] == 80000 - assert tmpl['transactions'][0]['txid'] == txid - assert tmpl['transactions'][0]['sigops'] == 8 + assert_greater_than_or_equal(tmpl['sizelimit'], 3999577) # actual maximum size is lower due to minimum mandatory non-witness data + assert_equal(tmpl['weightlimit'], 4000000) + assert_equal(tmpl['sigoplimit'], 80000) + assert_equal(tmpl['transactions'][0]['txid'], txid) + expected_sigops = 9 if 'txinwitness' in raw_tx["vin"][0] else 8 + assert_equal(tmpl['transactions'][0]['sigops'], expected_sigops) assert '!segwit' in tmpl['rules'] - self.nodes[0].generate(1) # Mine a block to clear the gbt cache + self.generate(self.nodes[0], 1) # Mine a block to clear the gbt cache self.log.info("Non-segwit miners are able to use GBT response after activation.") # Create a 3-tx chain: tx1 (non-segwit input, paying to a segwit output) -> @@ -260,8 +266,8 @@ class SegWitTest(BitcoinTestFramework): assert_equal(int(self.nodes[0].getmempoolentry(txid1)["wtxid"], 16), tx1.calc_sha256(True)) # Check that weight and vsize are properly reported in mempool entry (txid1) - assert_equal(self.nodes[0].getmempoolentry(txid1)["vsize"], (self.nodes[0].getmempoolentry(txid1)["weight"] + 3) // 4) - assert_equal(self.nodes[0].getmempoolentry(txid1)["weight"], len(tx1.serialize_without_witness())*3 + len(tx1.serialize_with_witness())) + assert_equal(self.nodes[0].getmempoolentry(txid1)["vsize"], tx1.get_vsize()) + assert_equal(self.nodes[0].getmempoolentry(txid1)["weight"], tx1.get_weight()) # Now create tx2, which will spend from txid1. tx = CTransaction() @@ -276,8 +282,8 @@ class SegWitTest(BitcoinTestFramework): assert_equal(int(self.nodes[0].getmempoolentry(txid2)["wtxid"], 16), tx.calc_sha256(True)) # Check that weight and vsize are properly reported in mempool entry (txid2) - assert_equal(self.nodes[0].getmempoolentry(txid2)["vsize"], (self.nodes[0].getmempoolentry(txid2)["weight"] + 3) // 4) - assert_equal(self.nodes[0].getmempoolentry(txid2)["weight"], len(tx.serialize_without_witness())*3 + len(tx.serialize_with_witness())) + assert_equal(self.nodes[0].getmempoolentry(txid2)["vsize"], tx.get_vsize()) + assert_equal(self.nodes[0].getmempoolentry(txid2)["weight"], tx.get_weight()) # Now create tx3, which will spend from txid2 tx = CTransaction() @@ -299,11 +305,11 @@ class SegWitTest(BitcoinTestFramework): assert_equal(int(self.nodes[0].getmempoolentry(txid3)["wtxid"], 16), tx.calc_sha256(True)) # Check that weight and vsize are properly reported in mempool entry (txid3) - assert_equal(self.nodes[0].getmempoolentry(txid3)["vsize"], (self.nodes[0].getmempoolentry(txid3)["weight"] + 3) // 4) - assert_equal(self.nodes[0].getmempoolentry(txid3)["weight"], len(tx.serialize_without_witness())*3 + len(tx.serialize_with_witness())) + assert_equal(self.nodes[0].getmempoolentry(txid3)["vsize"], tx.get_vsize()) + assert_equal(self.nodes[0].getmempoolentry(txid3)["weight"], tx.get_weight()) # Mine a block to clear the gbt cache again. - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.log.info("Verify behaviour of importaddress and listunspent") @@ -352,12 +358,12 @@ class SegWitTest(BitcoinTestFramework): # Money sent to P2SH of multisig of this should only be seen after importaddress with the BASE58 P2SH address. multisig_without_privkey_address = self.nodes[0].addmultisigaddress(2, [pubkeys[3], pubkeys[4]])['address'] - script = CScript([OP_2, hex_str_to_bytes(pubkeys[3]), hex_str_to_bytes(pubkeys[4]), OP_2, OP_CHECKMULTISIG]) - solvable_after_importaddress.append(CScript([OP_HASH160, hash160(script), OP_EQUAL])) + script = CScript([OP_2, bytes.fromhex(pubkeys[3]), bytes.fromhex(pubkeys[4]), OP_2, OP_CHECKMULTISIG]) + solvable_after_importaddress.append(script_to_p2sh_script(script)) for i in compressed_spendable_address: v = self.nodes[0].getaddressinfo(i) - if (v['isscript']): + if v['isscript']: [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) # p2sh multisig with compressed keys should always be spendable spendable_anytime.extend([p2sh]) @@ -376,7 +382,7 @@ class SegWitTest(BitcoinTestFramework): for i in uncompressed_spendable_address: v = self.nodes[0].getaddressinfo(i) - if (v['isscript']): + if v['isscript']: [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) # p2sh multisig with uncompressed keys should always be spendable spendable_anytime.extend([p2sh]) @@ -395,7 +401,7 @@ class SegWitTest(BitcoinTestFramework): for i in compressed_solvable_address: v = self.nodes[0].getaddressinfo(i) - if (v['isscript']): + if v['isscript']: # Multisig without private is not seen after addmultisigaddress, but seen after importaddress [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) solvable_after_importaddress.extend([bare, p2sh, p2wsh, p2sh_p2wsh]) @@ -408,7 +414,7 @@ class SegWitTest(BitcoinTestFramework): for i in uncompressed_solvable_address: v = self.nodes[0].getaddressinfo(i) - if (v['isscript']): + if v['isscript']: [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) # Base uncompressed multisig without private is not seen after addmultisigaddress, but seen after importaddress solvable_after_importaddress.extend([bare, p2sh]) @@ -426,11 +432,11 @@ class SegWitTest(BitcoinTestFramework): op1 = CScript([OP_1]) op0 = CScript([OP_0]) # 2N7MGY19ti4KDMSzRfPAssP6Pxyuxoi6jLe is the P2SH(P2PKH) version of mjoE3sSrb8ByYEvgnC3Aox86u1CHnfJA4V - unsolvable_address_key = hex_str_to_bytes("02341AEC7587A51CDE5279E0630A531AEA2615A9F80B17E8D9376327BAEAA59E3D") - unsolvablep2pkh = CScript([OP_DUP, OP_HASH160, hash160(unsolvable_address_key), OP_EQUALVERIFY, OP_CHECKSIG]) - unsolvablep2wshp2pkh = CScript([OP_0, sha256(unsolvablep2pkh)]) - p2shop0 = CScript([OP_HASH160, hash160(op0), OP_EQUAL]) - p2wshop1 = CScript([OP_0, sha256(op1)]) + unsolvable_address_key = bytes.fromhex("02341AEC7587A51CDE5279E0630A531AEA2615A9F80B17E8D9376327BAEAA59E3D") + unsolvablep2pkh = key_to_p2pkh_script(unsolvable_address_key) + unsolvablep2wshp2pkh = script_to_p2wsh_script(unsolvablep2pkh) + p2shop0 = script_to_p2sh_script(op0) + p2wshop1 = script_to_p2wsh_script(op1) unsolvable_after_importaddress.append(unsolvablep2pkh) unsolvable_after_importaddress.append(unsolvablep2wshp2pkh) unsolvable_after_importaddress.append(op1) # OP_1 will be imported as script @@ -447,19 +453,19 @@ class SegWitTest(BitcoinTestFramework): importlist = [] for i in compressed_spendable_address + uncompressed_spendable_address + compressed_solvable_address + uncompressed_solvable_address: v = self.nodes[0].getaddressinfo(i) - if (v['isscript']): - bare = hex_str_to_bytes(v['hex']) + if v['isscript']: + bare = bytes.fromhex(v['hex']) importlist.append(bare.hex()) - importlist.append(CScript([OP_0, sha256(bare)]).hex()) + importlist.append(script_to_p2wsh_script(bare).hex()) else: - pubkey = hex_str_to_bytes(v['pubkey']) - p2pk = CScript([pubkey, OP_CHECKSIG]) - p2pkh = CScript([OP_DUP, OP_HASH160, hash160(pubkey), OP_EQUALVERIFY, OP_CHECKSIG]) + pubkey = bytes.fromhex(v['pubkey']) + p2pk = key_to_p2pk_script(pubkey) + p2pkh = key_to_p2pkh_script(pubkey) importlist.append(p2pk.hex()) importlist.append(p2pkh.hex()) - importlist.append(CScript([OP_0, hash160(pubkey)]).hex()) - importlist.append(CScript([OP_0, sha256(p2pk)]).hex()) - importlist.append(CScript([OP_0, sha256(p2pkh)]).hex()) + importlist.append(key_to_p2wpkh_script(pubkey).hex()) + importlist.append(script_to_p2wsh_script(p2pk).hex()) + importlist.append(script_to_p2wsh_script(p2pkh).hex()) importlist.append(unsolvablep2pkh.hex()) importlist.append(unsolvablep2wshp2pkh.hex()) @@ -510,7 +516,7 @@ class SegWitTest(BitcoinTestFramework): for i in compressed_spendable_address: v = self.nodes[0].getaddressinfo(i) - if (v['isscript']): + if v['isscript']: [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) premature_witaddress.append(script_to_p2sh(p2wsh)) else: @@ -520,7 +526,7 @@ class SegWitTest(BitcoinTestFramework): for i in uncompressed_spendable_address + uncompressed_solvable_address: v = self.nodes[0].getaddressinfo(i) - if (v['isscript']): + if v['isscript']: [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) # P2WSH and P2SH(P2WSH) multisig with uncompressed keys are never seen unseen_anytime.extend([p2wsh, p2sh_p2wsh]) @@ -531,7 +537,7 @@ class SegWitTest(BitcoinTestFramework): for i in compressed_solvable_address: v = self.nodes[0].getaddressinfo(i) - if (v['isscript']): + if v['isscript']: [bare, p2sh, p2wsh, p2sh_p2wsh] = self.p2sh_address_to_script(v) premature_witaddress.append(script_to_p2sh(p2wsh)) else: @@ -593,18 +599,18 @@ class SegWitTest(BitcoinTestFramework): tx.rehash() signresults = self.nodes[0].signrawtransactionwithwallet(tx.serialize_without_witness().hex())['hex'] txid = self.nodes[0].sendrawtransaction(hexstring=signresults, maxfeerate=0) - txs_mined[txid] = self.nodes[0].generate(1)[0] + txs_mined[txid] = self.generate(self.nodes[0], 1)[0] self.sync_blocks() watchcount = 0 spendcount = 0 for i in self.nodes[0].listunspent(): - if (i['txid'] == txid): + if i['txid'] == txid: watchcount += 1 if i['spendable']: spendcount += 1 - if (ismine == 2): + if ismine == 2: assert_equal(spendcount, len(script_list)) - elif (ismine == 1): + elif ismine == 1: assert_equal(watchcount, len(script_list)) assert_equal(spendcount, 0) else: @@ -612,24 +618,24 @@ class SegWitTest(BitcoinTestFramework): return txid def p2sh_address_to_script(self, v): - bare = CScript(hex_str_to_bytes(v['hex'])) - p2sh = CScript(hex_str_to_bytes(v['scriptPubKey'])) - p2wsh = CScript([OP_0, sha256(bare)]) - p2sh_p2wsh = CScript([OP_HASH160, hash160(p2wsh), OP_EQUAL]) - return([bare, p2sh, p2wsh, p2sh_p2wsh]) + bare = CScript(bytes.fromhex(v['hex'])) + p2sh = CScript(bytes.fromhex(v['scriptPubKey'])) + p2wsh = script_to_p2wsh_script(bare) + p2sh_p2wsh = script_to_p2sh_script(p2wsh) + return [bare, p2sh, p2wsh, p2sh_p2wsh] def p2pkh_address_to_script(self, v): - pubkey = hex_str_to_bytes(v['pubkey']) - p2wpkh = CScript([OP_0, hash160(pubkey)]) - p2sh_p2wpkh = CScript([OP_HASH160, hash160(p2wpkh), OP_EQUAL]) - p2pk = CScript([pubkey, OP_CHECKSIG]) - p2pkh = CScript(hex_str_to_bytes(v['scriptPubKey'])) - p2sh_p2pk = CScript([OP_HASH160, hash160(p2pk), OP_EQUAL]) - p2sh_p2pkh = CScript([OP_HASH160, hash160(p2pkh), OP_EQUAL]) - p2wsh_p2pk = CScript([OP_0, sha256(p2pk)]) - p2wsh_p2pkh = CScript([OP_0, sha256(p2pkh)]) - p2sh_p2wsh_p2pk = CScript([OP_HASH160, hash160(p2wsh_p2pk), OP_EQUAL]) - p2sh_p2wsh_p2pkh = CScript([OP_HASH160, hash160(p2wsh_p2pkh), OP_EQUAL]) + pubkey = bytes.fromhex(v['pubkey']) + p2wpkh = key_to_p2wpkh_script(pubkey) + p2sh_p2wpkh = script_to_p2sh_script(p2wpkh) + p2pk = key_to_p2pk_script(pubkey) + p2pkh = CScript(bytes.fromhex(v['scriptPubKey'])) + p2sh_p2pk = script_to_p2sh_script(p2pk) + p2sh_p2pkh = script_to_p2sh_script(p2pkh) + p2wsh_p2pk = script_to_p2wsh_script(p2pk) + p2wsh_p2pkh = script_to_p2wsh_script(p2pkh) + p2sh_p2wsh_p2pk = script_to_p2sh_script(p2wsh_p2pk) + p2sh_p2wsh_p2pkh = script_to_p2sh_script(p2wsh_p2pkh) return [p2wpkh, p2sh_p2wpkh, p2pk, p2pkh, p2sh_p2pk, p2sh_p2pkh, p2wsh_p2pk, p2wsh_p2pkh, p2sh_p2wsh_p2pk, p2sh_p2wsh_p2pkh] def create_and_mine_tx_from_txids(self, txids, success=True): @@ -643,7 +649,7 @@ class SegWitTest(BitcoinTestFramework): tx.rehash() signresults = self.nodes[0].signrawtransactionwithwallet(tx.serialize_without_witness().hex())['hex'] self.nodes[0].sendrawtransaction(hexstring=signresults, maxfeerate=0) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks() diff --git a/test/functional/feature_settings.py b/test/functional/feature_settings.py index 5a0236401d..26048d37f6 100755 --- a/test/functional/feature_settings.py +++ b/test/functional/feature_settings.py @@ -83,7 +83,7 @@ class SettingsTest(BitcoinTestFramework): with altsettings.open("w") as fp: fp.write('{"key": "value"}') with node.assert_debug_log(expected_msgs=['Setting file arg: key = "value"']): - self.start_node(0, extra_args=["-settings={}".format(altsettings)]) + self.start_node(0, extra_args=[f"-settings={altsettings}"]) self.stop_node(0) diff --git a/test/functional/feature_signet.py b/test/functional/feature_signet.py index 96c581dede..94138b0e6d 100755 --- a/test/functional/feature_signet.py +++ b/test/functional/feature_signet.py @@ -51,7 +51,7 @@ class SignetBasicTest(BitcoinTestFramework): assert_equal(mining_info['networkhashps'], Decimal('0')) assert_equal(mining_info['pooledtx'], 0) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.log.info("pregenerated signet blocks check") diff --git a/test/functional/feature_syscall_sandbox.py b/test/functional/feature_syscall_sandbox.py new file mode 100755 index 0000000000..caf7f1e7fc --- /dev/null +++ b/test/functional/feature_syscall_sandbox.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test bitcoind aborts if a disallowed syscall is used when compiled with the syscall sandbox.""" + +from test_framework.test_framework import BitcoinTestFramework, SkipTest + + +class SyscallSandboxTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + if not self.is_syscall_sandbox_compiled(): + raise SkipTest("bitcoind has not been built with syscall sandbox enabled.") + if self.options.nosandbox: + raise SkipTest("--nosandbox passed to test runner.") + + def run_test(self): + disallowed_syscall_terminated_bitcoind = False + expected_log_entry = 'ERROR: The syscall "getgroups" (syscall number 115) is not allowed by the syscall sandbox' + with self.nodes[0].assert_debug_log([expected_log_entry]): + self.log.info("Invoking disallowed syscall") + try: + self.nodes[0].invokedisallowedsyscall() + except ConnectionError: + disallowed_syscall_terminated_bitcoind = True + assert disallowed_syscall_terminated_bitcoind + self.nodes = [] + + +if __name__ == "__main__": + SyscallSandboxTest().main() diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index 99283b69b0..50a25ee1ef 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -57,7 +57,6 @@ from test_framework.script import ( OP_ENDIF, OP_EQUAL, OP_EQUALVERIFY, - OP_HASH160, OP_IF, OP_NOP, OP_NOT, @@ -76,12 +75,18 @@ from test_framework.script import ( is_op_success, taproot_construct, ) +from test_framework.script_util import ( + key_to_p2pk_script, + key_to_p2wpkh_script, + keyhash_to_p2pkh_script, + script_to_p2sh_script, + script_to_p2wsh_script, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_raises_rpc_error, assert_equal from test_framework.key import generate_privkey, compute_xonly_pubkey, sign_schnorr, tweak_add_privkey, ECKey from test_framework.address import ( hash160, - sha256, ) from collections import OrderedDict, namedtuple from io import BytesIO @@ -458,13 +463,13 @@ def make_spender(comment, *, tap=None, witv0=False, script=None, pkh=None, p2sh= # P2WPKH assert script is None pubkeyhash = hash160(pkh) - spk = CScript([OP_0, pubkeyhash]) - conf["scriptcode"] = CScript([OP_DUP, OP_HASH160, pubkeyhash, OP_EQUALVERIFY, OP_CHECKSIG]) + spk = key_to_p2wpkh_script(pkh) + conf["scriptcode"] = keyhash_to_p2pkh_script(pubkeyhash) conf["script_witv0"] = None conf["inputs"] = [getter("sign"), pkh] elif script is not None: # P2WSH - spk = CScript([OP_0, sha256(script)]) + spk = script_to_p2wsh_script(script) conf["scriptcode"] = script conf["script_witv0"] = script else: @@ -475,7 +480,7 @@ def make_spender(comment, *, tap=None, witv0=False, script=None, pkh=None, p2sh= # P2PKH assert script is None pubkeyhash = hash160(pkh) - spk = CScript([OP_DUP, OP_HASH160, pubkeyhash, OP_EQUALVERIFY, OP_CHECKSIG]) + spk = keyhash_to_p2pkh_script(pubkeyhash) conf["scriptcode"] = spk conf["inputs"] = [getter("sign"), pkh] elif script is not None: @@ -496,7 +501,7 @@ def make_spender(comment, *, tap=None, witv0=False, script=None, pkh=None, p2sh= if p2sh: # P2SH wrapper can be combined with anything else conf["script_p2sh"] = spk - spk = CScript([OP_HASH160, hash160(spk), OP_EQUAL]) + spk = script_to_p2sh_script(spk) conf = {**conf, **kwargs} @@ -518,9 +523,9 @@ def add_spender(spenders, *args, **kwargs): def random_checksig_style(pubkey): """Creates a random CHECKSIG* tapscript that would succeed with only the valid signature on witness stack.""" opcode = random.choice([OP_CHECKSIG, OP_CHECKSIGVERIFY, OP_CHECKSIGADD]) - if (opcode == OP_CHECKSIGVERIFY): + if opcode == OP_CHECKSIGVERIFY: ret = CScript([pubkey, opcode, OP_1]) - elif (opcode == OP_CHECKSIGADD): + elif opcode == OP_CHECKSIGADD: num = random.choice([0, 0x7fffffff, -0x7fffffff]) ret = CScript([num, pubkey, opcode, num + 1, OP_EQUAL]) else: @@ -1105,7 +1110,7 @@ def spenders_taproot_active(): for witv0 in [False, True]: for hashtype in VALID_SIGHASHES_ECDSA + [random.randrange(0x04, 0x80), random.randrange(0x84, 0x100)]: standard = (hashtype in VALID_SIGHASHES_ECDSA) and (compressed or not witv0) - add_spender(spenders, "legacy/pk-wrongkey", hashtype=hashtype, p2sh=p2sh, witv0=witv0, standard=standard, script=CScript([pubkey1, OP_CHECKSIG]), **SINGLE_SIG, key=eckey1, failure={"key": eckey2}, sigops_weight=4-3*witv0, **ERR_NO_SUCCESS) + add_spender(spenders, "legacy/pk-wrongkey", hashtype=hashtype, p2sh=p2sh, witv0=witv0, standard=standard, script=key_to_p2pk_script(pubkey1), **SINGLE_SIG, key=eckey1, failure={"key": eckey2}, sigops_weight=4-3*witv0, **ERR_NO_SUCCESS) add_spender(spenders, "legacy/pkh-sighashflip", hashtype=hashtype, p2sh=p2sh, witv0=witv0, standard=standard, pkh=pubkey1, key=eckey1, **SIGHASH_BITFLIP, sigops_weight=4-3*witv0, **ERR_NO_SUCCESS) # Verify that OP_CHECKSIGADD wasn't accidentally added to pre-taproot validation logic. @@ -1189,19 +1194,36 @@ def dump_json_test(tx, input_utxos, idx, success, failure): # Data type to keep track of UTXOs, where they were created, and how to spend them. UTXOData = namedtuple('UTXOData', 'outpoint,output,spender') + class TaprootTest(BitcoinTestFramework): def add_options(self, parser): parser.add_argument("--dumptests", dest="dump_tests", default=False, action="store_true", help="Dump generated test cases to directory set by TEST_DUMP_DIR environment variable") + parser.add_argument("--previous_release", dest="previous_release", default=False, action="store_true", + help="Use a previous release as taproot-inactive node") def skip_test_if_missing_module(self): self.skip_if_no_wallet() + if self.options.previous_release: + self.skip_if_no_previous_releases() def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True # Node 0 has Taproot inactive, Node 1 active. - self.extra_args = [["-par=1", "-vbparams=taproot:1:1"], ["-par=1"]] + self.extra_args = [["-par=1"], ["-par=1"]] + if self.options.previous_release: + self.wallet_names = [None, self.default_wallet_name] + else: + self.extra_args[0].append("-vbparams=taproot:1:1") + + def setup_nodes(self): + self.add_nodes(self.num_nodes, self.extra_args, versions=[ + 200100 if self.options.previous_release else None, + None, + ]) + self.start_nodes() + self.import_deterministic_coinbase_privkeys() def block_submit(self, node, txs, msg, err_msg, cb_pubkey=None, fees=0, sigops_weight=0, witness=False, accept=False): @@ -1223,7 +1245,7 @@ class TaprootTest(BitcoinTestFramework): block_response = node.submitblock(block.serialize().hex()) if err_msg is not None: assert block_response is not None and err_msg in block_response, "Missing error message '%s' from block response '%s': %s" % (err_msg, "(None)" if block_response is None else block_response, msg) - if (accept): + if accept: assert node.getbestblockhash() == block.hash, "Failed to accept: %s (response: %s)" % (msg, block_response) self.tip = block.sha256 self.lastblockhash = block.hash @@ -1440,7 +1462,7 @@ class TaprootTest(BitcoinTestFramework): def run_test(self): # Post-taproot activation tests go first (pre-taproot tests' blocks are invalid post-taproot). self.log.info("Post-activation tests...") - self.nodes[1].generate(COINBASE_MATURITY + 1) + self.generate(self.nodes[1], COINBASE_MATURITY + 1) self.test_spenders(self.nodes[1], spenders_taproot_active(), input_counts=[1, 2, 2, 2, 2, 3]) # Re-connect nodes in case they have been disconnected diff --git a/test/functional/feature_utxo_set_hash.py b/test/functional/feature_utxo_set_hash.py index afc0bdb8c5..b1b4703d37 100755 --- a/test/functional/feature_utxo_set_hash.py +++ b/test/functional/feature_utxo_set_hash.py @@ -31,13 +31,13 @@ class UTXOSetHashTest(BitcoinTestFramework): # Generate 100 blocks and remove the first since we plan to spend its # coinbase - block_hashes = wallet.generate(1) + node.generate(99) + block_hashes = self.generate(wallet, 1) + self.generate(node, 99) blocks = list(map(lambda block: from_hex(CBlock(), node.getblock(block, False)), block_hashes)) blocks.pop(0) # Create a spending transaction and mine a block which includes it txid = wallet.send_self_transfer(from_node=node)['txid'] - tx_block = node.generateblock(output=wallet.get_address(), transactions=[txid]) + tx_block = self.generateblock(node, output=wallet.get_address(), transactions=[txid]) blocks.append(from_hex(CBlock(), node.getblock(tx_block['hash'], False))) # Serialize the outputs that should be in the UTXO set and add them to diff --git a/test/functional/feature_versionbits_warning.py b/test/functional/feature_versionbits_warning.py index 1c9e237d78..d74ef5e088 100755 --- a/test/functional/feature_versionbits_warning.py +++ b/test/functional/feature_versionbits_warning.py @@ -21,20 +21,23 @@ VB_TOP_BITS = 0x20000000 VB_UNKNOWN_BIT = 27 # Choose a bit unassigned to any deployment VB_UNKNOWN_VERSION = VB_TOP_BITS | (1 << VB_UNKNOWN_BIT) -WARN_UNKNOWN_RULES_ACTIVE = "Unknown new rules activated (versionbit {})".format(VB_UNKNOWN_BIT) +WARN_UNKNOWN_RULES_ACTIVE = f"Unknown new rules activated (versionbit {VB_UNKNOWN_BIT})" VB_PATTERN = re.compile("Unknown new rules activated.*versionbit") class VersionBitsWarningTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 + # The experimental syscall sandbox feature (-sandbox) is not compatible with -alertnotify + # (which invokes execve). + self.disable_syscall_sandbox = True def setup_network(self): self.alert_filename = os.path.join(self.options.tmpdir, "alert.txt") # Open and close to create zero-length file with open(self.alert_filename, 'w', encoding='utf8'): pass - self.extra_args = [["-alertnotify=echo %s >> \"" + self.alert_filename + "\""]] + self.extra_args = [[f"-alertnotify=echo %s >> \"{self.alert_filename}\""]] self.setup_nodes() def send_blocks_with_version(self, peer, numblocks, version): @@ -65,12 +68,12 @@ class VersionBitsWarningTest(BitcoinTestFramework): node_deterministic_address = node.get_deterministic_priv_key().address # Mine one period worth of blocks - node.generatetoaddress(VB_PERIOD, node_deterministic_address) + self.generatetoaddress(node, VB_PERIOD, node_deterministic_address) self.log.info("Check that there is no warning if previous VB_BLOCKS have <VB_THRESHOLD blocks with unknown versionbits version.") # Build one period of blocks with < VB_THRESHOLD blocks signaling some unknown bit self.send_blocks_with_version(peer, VB_THRESHOLD - 1, VB_UNKNOWN_VERSION) - node.generatetoaddress(VB_PERIOD - VB_THRESHOLD + 1, node_deterministic_address) + self.generatetoaddress(node, VB_PERIOD - VB_THRESHOLD + 1, node_deterministic_address) # Check that we're not getting any versionbit-related errors in get*info() assert not VB_PATTERN.match(node.getmininginfo()["warnings"]) @@ -78,21 +81,21 @@ class VersionBitsWarningTest(BitcoinTestFramework): # Build one period of blocks with VB_THRESHOLD blocks signaling some unknown bit self.send_blocks_with_version(peer, VB_THRESHOLD, VB_UNKNOWN_VERSION) - node.generatetoaddress(VB_PERIOD - VB_THRESHOLD, node_deterministic_address) + self.generatetoaddress(node, VB_PERIOD - VB_THRESHOLD, node_deterministic_address) self.log.info("Check that there is a warning if previous VB_BLOCKS have >=VB_THRESHOLD blocks with unknown versionbits version.") # Mine a period worth of expected blocks so the generic block-version warning # is cleared. This will move the versionbit state to ACTIVE. - node.generatetoaddress(VB_PERIOD, node_deterministic_address) + self.generatetoaddress(node, VB_PERIOD, node_deterministic_address) # Stop-start the node. This is required because bitcoind will only warn once about unknown versions or unknown rules activating. self.restart_node(0) # Generating one block guarantees that we'll get out of IBD - node.generatetoaddress(1, node_deterministic_address) + self.generatetoaddress(node, 1, node_deterministic_address) self.wait_until(lambda: not node.getblockchaininfo()['initialblockdownload']) # Generating one more block will be enough to generate an error. - node.generatetoaddress(1, node_deterministic_address) + self.generatetoaddress(node, 1, node_deterministic_address) # Check that get*info() shows the versionbits unknown rules warning assert WARN_UNKNOWN_RULES_ACTIVE in node.getmininginfo()["warnings"] assert WARN_UNKNOWN_RULES_ACTIVE in node.getnetworkinfo()["warnings"] diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 22eec59600..c28186cde7 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -5,6 +5,7 @@ """Test bitcoin-cli""" from decimal import Decimal +import re from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework @@ -29,6 +30,41 @@ TOO_MANY_ARGS = 'error: too many arguments (maximum 2 for nblocks and maxtries)' WALLET_NOT_LOADED = 'Requested wallet does not exist or is not loaded' WALLET_NOT_SPECIFIED = 'Wallet file not specified' + +def cli_get_info_string_to_dict(cli_get_info_string): + """Helper method to convert human-readable -getinfo into a dictionary""" + cli_get_info = {} + lines = cli_get_info_string.splitlines() + line_idx = 0 + ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]') + while line_idx < len(lines): + # Remove ansi colour code + line = ansi_escape.sub('', lines[line_idx]) + if "Balances" in line: + # When "Balances" appears in a line, all of the following lines contain "balance: wallet" until an empty line + cli_get_info["Balances"] = {} + while line_idx < len(lines) and not (lines[line_idx + 1] == ''): + line_idx += 1 + balance, wallet = lines[line_idx].strip().split(" ") + # Remove right justification padding + wallet = wallet.strip() + if wallet == '""': + # Set default wallet("") to empty string + wallet = '' + cli_get_info["Balances"][wallet] = balance.strip() + elif ": " in line: + key, value = line.split(": ") + if key == 'Wallet' and value == '""': + # Set default wallet("") to empty string + value = '' + if key == "Proxies" and value == "n/a": + # Set N/A to empty string to represent no proxy + value = '' + cli_get_info[key.strip()] = value.strip() + line_idx += 1 + return cli_get_info + + class TestBitcoinCli(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -41,7 +77,7 @@ class TestBitcoinCli(BitcoinTestFramework): def run_test(self): """Main test logic""" - self.nodes[0].generate(BLOCKS) + self.generate(self.nodes[0], BLOCKS) self.log.info("Compare responses from getblockchaininfo RPC and `bitcoin-cli getblockchaininfo`") cli_response = self.nodes[0].cli.getblockchaininfo() @@ -51,12 +87,12 @@ class TestBitcoinCli(BitcoinTestFramework): user, password = get_auth_cookie(self.nodes[0].datadir, self.chain) self.log.info("Test -stdinrpcpass option") - assert_equal(BLOCKS, self.nodes[0].cli('-rpcuser={}'.format(user), '-stdinrpcpass', input=password).getblockcount()) - assert_raises_process_error(1, 'Incorrect rpcuser or rpcpassword', self.nodes[0].cli('-rpcuser={}'.format(user), '-stdinrpcpass', input='foo').echo) + assert_equal(BLOCKS, self.nodes[0].cli(f'-rpcuser={user}', '-stdinrpcpass', input=password).getblockcount()) + assert_raises_process_error(1, 'Incorrect rpcuser or rpcpassword', self.nodes[0].cli(f'-rpcuser={user}', '-stdinrpcpass', input='foo').echo) self.log.info("Test -stdin and -stdinrpcpass") - assert_equal(['foo', 'bar'], self.nodes[0].cli('-rpcuser={}'.format(user), '-stdin', '-stdinrpcpass', input=password + '\nfoo\nbar').echo()) - assert_raises_process_error(1, 'Incorrect rpcuser or rpcpassword', self.nodes[0].cli('-rpcuser={}'.format(user), '-stdin', '-stdinrpcpass', input='foo').echo) + assert_equal(['foo', 'bar'], self.nodes[0].cli(f'-rpcuser={user}', '-stdin', '-stdinrpcpass', input=f'{password}\nfoo\nbar').echo()) + assert_raises_process_error(1, 'Incorrect rpcuser or rpcpassword', self.nodes[0].cli(f'-rpcuser={user}', '-stdin', '-stdinrpcpass', input='foo').echo) self.log.info("Test connecting to a non-existing server") assert_raises_process_error(1, "Could not connect to the server", self.nodes[0].cli('-rpcport=1').echo) @@ -67,37 +103,50 @@ class TestBitcoinCli(BitcoinTestFramework): self.log.info("Test -getinfo with arguments fails") assert_raises_process_error(1, "-getinfo takes no arguments", self.nodes[0].cli('-getinfo').help) + self.log.info("Test -getinfo with -color=never does not return ANSI escape codes") + assert "\u001b[0m" not in self.nodes[0].cli('-getinfo', '-color=never').send_cli() + + self.log.info("Test -getinfo with -color=always returns ANSI escape codes") + assert "\u001b[0m" in self.nodes[0].cli('-getinfo', '-color=always').send_cli() + + self.log.info("Test -getinfo with invalid value for -color option") + assert_raises_process_error(1, "Invalid value for -color option. Valid values: always, auto, never.", self.nodes[0].cli('-getinfo', '-color=foo').send_cli) + self.log.info("Test -getinfo returns expected network and blockchain info") if self.is_wallet_compiled(): self.nodes[0].encryptwallet(password) - cli_get_info = self.nodes[0].cli('-getinfo').send_cli() + cli_get_info_string = self.nodes[0].cli('-getinfo').send_cli() + cli_get_info = cli_get_info_string_to_dict(cli_get_info_string) + network_info = self.nodes[0].getnetworkinfo() blockchain_info = self.nodes[0].getblockchaininfo() - assert_equal(cli_get_info['version'], network_info['version']) - assert_equal(cli_get_info['blocks'], blockchain_info['blocks']) - assert_equal(cli_get_info['headers'], blockchain_info['headers']) - assert_equal(cli_get_info['timeoffset'], network_info['timeoffset']) - assert_equal( - cli_get_info['connections'], - { - 'in': network_info['connections_in'], - 'out': network_info['connections_out'], - 'total': network_info['connections'] - } - ) - assert_equal(cli_get_info['proxy'], network_info['networks'][0]['proxy']) - assert_equal(cli_get_info['difficulty'], blockchain_info['difficulty']) - assert_equal(cli_get_info['chain'], blockchain_info['chain']) + assert_equal(int(cli_get_info['Version']), network_info['version']) + assert_equal(cli_get_info['Verification progress'], "%.4f%%" % (blockchain_info['verificationprogress'] * 100)) + assert_equal(int(cli_get_info['Blocks']), blockchain_info['blocks']) + assert_equal(int(cli_get_info['Headers']), blockchain_info['headers']) + assert_equal(int(cli_get_info['Time offset (s)']), network_info['timeoffset']) + expected_network_info = f"in {network_info['connections_in']}, out {network_info['connections_out']}, total {network_info['connections']}" + assert_equal(cli_get_info["Network"], expected_network_info) + assert_equal(cli_get_info['Proxies'], network_info['networks'][0]['proxy']) + assert_equal(Decimal(cli_get_info['Difficulty']), blockchain_info['difficulty']) + assert_equal(cli_get_info['Chain'], blockchain_info['chain']) + + self.log.info("Test -getinfo and bitcoin-cli return all proxies") + self.restart_node(0, extra_args=["-proxy=127.0.0.1:9050", "-i2psam=127.0.0.1:7656"]) + network_info = self.nodes[0].getnetworkinfo() + cli_get_info_string = self.nodes[0].cli('-getinfo').send_cli() + cli_get_info = cli_get_info_string_to_dict(cli_get_info_string) + assert_equal(cli_get_info["Proxies"], "127.0.0.1:9050 (ipv4, ipv6, onion), 127.0.0.1:7656 (i2p)") if self.is_wallet_compiled(): self.log.info("Test -getinfo and bitcoin-cli getwalletinfo return expected wallet info") - assert_equal(cli_get_info['balance'], BALANCE) - assert 'balances' not in cli_get_info.keys() + assert_equal(Decimal(cli_get_info['Balance']), BALANCE) + assert 'Balances' not in cli_get_info_string wallet_info = self.nodes[0].getwalletinfo() - assert_equal(cli_get_info['keypoolsize'], wallet_info['keypoolsize']) - assert_equal(cli_get_info['unlocked_until'], wallet_info['unlocked_until']) - assert_equal(cli_get_info['paytxfee'], wallet_info['paytxfee']) - assert_equal(cli_get_info['relayfee'], network_info['relayfee']) + assert_equal(int(cli_get_info['Keypool size']), wallet_info['keypoolsize']) + assert_equal(int(cli_get_info['Unlocked until']), wallet_info['unlocked_until']) + assert_equal(Decimal(cli_get_info['Transaction fee rate (-paytxfee) (BTC/kvB)']), wallet_info['paytxfee']) + assert_equal(Decimal(cli_get_info['Min tx relay fee rate (BTC/kvB)']), network_info['relayfee']) assert_equal(self.nodes[0].cli.getwalletinfo(), wallet_info) # Setup to test -getinfo, -generate, and -rpcwallet= with multiple wallets. @@ -108,56 +157,69 @@ class TestBitcoinCli(BitcoinTestFramework): w1 = self.nodes[0].get_wallet_rpc(wallets[0]) w2 = self.nodes[0].get_wallet_rpc(wallets[1]) w3 = self.nodes[0].get_wallet_rpc(wallets[2]) - rpcwallet2 = '-rpcwallet={}'.format(wallets[1]) - rpcwallet3 = '-rpcwallet={}'.format(wallets[2]) + rpcwallet2 = f'-rpcwallet={wallets[1]}' + rpcwallet3 = f'-rpcwallet={wallets[2]}' w1.walletpassphrase(password, self.rpc_timeout) w2.encryptwallet(password) w1.sendtoaddress(w2.getnewaddress(), amounts[1]) w1.sendtoaddress(w3.getnewaddress(), amounts[2]) # Mine a block to confirm; adds a block reward (50 BTC) to the default wallet. - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.log.info("Test -getinfo with multiple wallets and -rpcwallet returns specified wallet balance") for i in range(len(wallets)): - cli_get_info = self.nodes[0].cli('-getinfo', '-rpcwallet={}'.format(wallets[i])).send_cli() - assert 'balances' not in cli_get_info.keys() - assert_equal(cli_get_info['balance'], amounts[i]) + cli_get_info_string = self.nodes[0].cli('-getinfo', f'-rpcwallet={wallets[i]}').send_cli() + cli_get_info = cli_get_info_string_to_dict(cli_get_info_string) + assert 'Balances' not in cli_get_info_string + assert_equal(cli_get_info["Wallet"], wallets[i]) + assert_equal(Decimal(cli_get_info['Balance']), amounts[i]) self.log.info("Test -getinfo with multiple wallets and -rpcwallet=non-existing-wallet returns no balances") - cli_get_info_keys = self.nodes[0].cli('-getinfo', '-rpcwallet=does-not-exist').send_cli().keys() - assert 'balance' not in cli_get_info_keys - assert 'balances' not in cli_get_info_keys + cli_get_info_string = self.nodes[0].cli('-getinfo', '-rpcwallet=does-not-exist').send_cli() + assert 'Balance' not in cli_get_info_string + assert 'Balances' not in cli_get_info_string self.log.info("Test -getinfo with multiple wallets returns all loaded wallet names and balances") assert_equal(set(self.nodes[0].listwallets()), set(wallets)) - cli_get_info = self.nodes[0].cli('-getinfo').send_cli() - assert 'balance' not in cli_get_info.keys() - assert_equal(cli_get_info['balances'], {k: v for k, v in zip(wallets, amounts)}) + cli_get_info_string = self.nodes[0].cli('-getinfo').send_cli() + cli_get_info = cli_get_info_string_to_dict(cli_get_info_string) + assert 'Balance' not in cli_get_info + for k, v in zip(wallets, amounts): + assert_equal(Decimal(cli_get_info['Balances'][k]), v) # Unload the default wallet and re-verify. self.nodes[0].unloadwallet(wallets[0]) assert wallets[0] not in self.nodes[0].listwallets() - cli_get_info = self.nodes[0].cli('-getinfo').send_cli() - assert 'balance' not in cli_get_info.keys() - assert_equal(cli_get_info['balances'], {k: v for k, v in zip(wallets[1:], amounts[1:])}) + cli_get_info_string = self.nodes[0].cli('-getinfo').send_cli() + cli_get_info = cli_get_info_string_to_dict(cli_get_info_string) + assert 'Balance' not in cli_get_info + assert 'Balances' in cli_get_info_string + for k, v in zip(wallets[1:], amounts[1:]): + assert_equal(Decimal(cli_get_info['Balances'][k]), v) + assert wallets[0] not in cli_get_info self.log.info("Test -getinfo after unloading all wallets except a non-default one returns its balance") self.nodes[0].unloadwallet(wallets[2]) assert_equal(self.nodes[0].listwallets(), [wallets[1]]) - cli_get_info = self.nodes[0].cli('-getinfo').send_cli() - assert 'balances' not in cli_get_info.keys() - assert_equal(cli_get_info['balance'], amounts[1]) + cli_get_info_string = self.nodes[0].cli('-getinfo').send_cli() + cli_get_info = cli_get_info_string_to_dict(cli_get_info_string) + assert 'Balances' not in cli_get_info_string + assert_equal(cli_get_info['Wallet'], wallets[1]) + assert_equal(Decimal(cli_get_info['Balance']), amounts[1]) self.log.info("Test -getinfo with -rpcwallet=remaining-non-default-wallet returns only its balance") - cli_get_info = self.nodes[0].cli('-getinfo', rpcwallet2).send_cli() - assert 'balances' not in cli_get_info.keys() - assert_equal(cli_get_info['balance'], amounts[1]) + cli_get_info_string = self.nodes[0].cli('-getinfo', rpcwallet2).send_cli() + cli_get_info = cli_get_info_string_to_dict(cli_get_info_string) + assert 'Balances' not in cli_get_info_string + assert_equal(cli_get_info['Wallet'], wallets[1]) + assert_equal(Decimal(cli_get_info['Balance']), amounts[1]) self.log.info("Test -getinfo with -rpcwallet=unloaded wallet returns no balances") - cli_get_info_keys = self.nodes[0].cli('-getinfo', rpcwallet3).send_cli().keys() - assert 'balance' not in cli_get_info_keys - assert 'balances' not in cli_get_info_keys + cli_get_info_string = self.nodes[0].cli('-getinfo', rpcwallet3).send_cli() + cli_get_info_keys = cli_get_info_string_to_dict(cli_get_info_string) + assert 'Balance' not in cli_get_info_keys + assert 'Balances' not in cli_get_info_string # Test bitcoin-cli -generate. n1 = 3 @@ -236,12 +298,12 @@ class TestBitcoinCli(BitcoinTestFramework): assert_raises_rpc_error(-19, WALLET_NOT_SPECIFIED, self.nodes[0].cli('-generate', 1, 2, 3).echo) else: self.log.info("*** Wallet not compiled; cli getwalletinfo and -getinfo wallet tests skipped") - self.nodes[0].generate(25) # maintain block parity with the wallet_compiled conditional branch + self.generate(self.nodes[0], 25) # maintain block parity with the wallet_compiled conditional branch self.log.info("Test -version with node stopped") self.stop_node(0) cli_response = self.nodes[0].cli('-version').send_cli() - assert "{} RPC client version".format(self.config['environment']['PACKAGE_NAME']) in cli_response + assert f"{self.config['environment']['PACKAGE_NAME']} RPC client version" in cli_response self.log.info("Test -rpcwait option successfully waits for RPC connection") self.nodes[0].start() # start node without RPC connection diff --git a/test/functional/interface_http.py b/test/functional/interface_http.py index d007490f80..075224c011 100755 --- a/test/functional/interface_http.py +++ b/test/functional/interface_http.py @@ -24,8 +24,8 @@ class HTTPBasicsTest (BitcoinTestFramework): # lowlevel check for http persistent connection # ################################################# url = urllib.parse.urlparse(self.nodes[0].url) - authpair = url.username + ':' + url.password - headers = {"Authorization": "Basic " + str_to_b64str(authpair)} + authpair = f'{url.username}:{url.password}' + headers = {"Authorization": f"Basic {str_to_b64str(authpair)}"} conn = http.client.HTTPConnection(url.hostname, url.port) conn.connect() @@ -42,7 +42,7 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.close() #same should be if we add keep-alive because this should be the std. behaviour - headers = {"Authorization": "Basic " + str_to_b64str(authpair), "Connection": "keep-alive"} + headers = {"Authorization": f"Basic {str_to_b64str(authpair)}", "Connection": "keep-alive"} conn = http.client.HTTPConnection(url.hostname, url.port) conn.connect() @@ -59,7 +59,7 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.close() #now do the same with "Connection: close" - headers = {"Authorization": "Basic " + str_to_b64str(authpair), "Connection":"close"} + headers = {"Authorization": f"Basic {str_to_b64str(authpair)}", "Connection":"close"} conn = http.client.HTTPConnection(url.hostname, url.port) conn.connect() @@ -70,8 +70,8 @@ class HTTPBasicsTest (BitcoinTestFramework): #node1 (2nd node) is running with disabled keep-alive option urlNode1 = urllib.parse.urlparse(self.nodes[1].url) - authpair = urlNode1.username + ':' + urlNode1.password - headers = {"Authorization": "Basic " + str_to_b64str(authpair)} + authpair = f'{urlNode1.username}:{urlNode1.password}' + headers = {"Authorization": f"Basic {str_to_b64str(authpair)}"} conn = http.client.HTTPConnection(urlNode1.hostname, urlNode1.port) conn.connect() @@ -81,8 +81,8 @@ class HTTPBasicsTest (BitcoinTestFramework): #node2 (third node) is running with standard keep-alive parameters which means keep-alive is on urlNode2 = urllib.parse.urlparse(self.nodes[2].url) - authpair = urlNode2.username + ':' + urlNode2.password - headers = {"Authorization": "Basic " + str_to_b64str(authpair)} + authpair = f'{urlNode2.username}:{urlNode2.password}' + headers = {"Authorization": f"Basic {str_to_b64str(authpair)}"} conn = http.client.HTTPConnection(urlNode2.hostname, urlNode2.port) conn.connect() @@ -94,13 +94,13 @@ class HTTPBasicsTest (BitcoinTestFramework): # Check excessive request size conn = http.client.HTTPConnection(urlNode2.hostname, urlNode2.port) conn.connect() - conn.request('GET', '/' + ('x'*1000), '', headers) + conn.request('GET', f'/{"x"*1000}', '', headers) out1 = conn.getresponse() assert_equal(out1.status, http.client.NOT_FOUND) conn = http.client.HTTPConnection(urlNode2.hostname, urlNode2.port) conn.connect() - conn.request('GET', '/' + ('x'*10000), '', headers) + conn.request('GET', f'/{"x"*10000}', '', headers) out1 = conn.getresponse() assert_equal(out1.status, http.client.BAD_REQUEST) diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index e73ec90819..e0716fc54a 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the REST API.""" -import binascii from decimal import Decimal from enum import Enum from io import BytesIO @@ -19,7 +18,6 @@ from test_framework.util import ( assert_equal, assert_greater_than, assert_greater_than_or_equal, - hex_str_to_bytes, ) from test_framework.messages import BLOCK_HEADER_SIZE @@ -59,7 +57,7 @@ class RESTTest (BitcoinTestFramework): rest_uri += '.hex' conn = http.client.HTTPConnection(self.url.hostname, self.url.port) - self.log.debug('%s %s %s', http_method, rest_uri, body) + self.log.debug(f'{http_method} {rest_uri} {body}') if http_method == 'GET': conn.request('GET', rest_uri) elif http_method == 'POST': @@ -82,9 +80,9 @@ class RESTTest (BitcoinTestFramework): # Random address so node1's balance doesn't increase not_related_address = "2MxqoHEdNQTyYeX1mHcbrrpzgojbosTpCvJ" - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() - self.nodes[1].generatetoaddress(100, not_related_address) + self.generatetoaddress(self.nodes[1], 100, not_related_address) self.sync_all() assert_equal(self.nodes[0].getbalance(), 50) @@ -94,11 +92,11 @@ class RESTTest (BitcoinTestFramework): self.log.info("Test the /tx URI") - json_obj = self.test_rest_request("/tx/{}".format(txid)) + json_obj = self.test_rest_request(f"/tx/{txid}") assert_equal(json_obj['txid'], txid) # Check hex format response - hex_response = self.test_rest_request("/tx/{}".format(txid), req_type=ReqType.HEX, ret_type=RetType.OBJ) + hex_response = self.test_rest_request(f"/tx/{txid}", req_type=ReqType.HEX, ret_type=RetType.OBJ) assert_greater_than_or_equal(int(hex_response.getheader('content-length')), json_obj['size']*2) @@ -109,14 +107,14 @@ class RESTTest (BitcoinTestFramework): self.log.info("Query an unspent TXO using the /getutxos URI") - self.nodes[1].generatetoaddress(1, not_related_address) + self.generatetoaddress(self.nodes[1], 1, not_related_address) self.sync_all() bb_hash = self.nodes[0].getbestblockhash() assert_equal(self.nodes[1].getbalance(), Decimal("0.1")) # Check chainTip response - json_obj = self.test_rest_request("/getutxos/{}-{}".format(*spending)) + json_obj = self.test_rest_request(f"/getutxos/{spending[0]}-{spending[1]}") assert_equal(json_obj['chaintipHash'], bb_hash) # Make sure there is one utxo @@ -125,7 +123,7 @@ class RESTTest (BitcoinTestFramework): self.log.info("Query a spent TXO using the /getutxos URI") - json_obj = self.test_rest_request("/getutxos/{}-{}".format(*spent)) + json_obj = self.test_rest_request(f"/getutxos/{spent[0]}-{spent[1]}") # Check chainTip response assert_equal(json_obj['chaintipHash'], bb_hash) @@ -138,7 +136,7 @@ class RESTTest (BitcoinTestFramework): self.log.info("Query two TXOs using the /getutxos URI") - json_obj = self.test_rest_request("/getutxos/{}-{}/{}-{}".format(*(spending + spent))) + json_obj = self.test_rest_request(f"/getutxos/{spending[0]}-{spending[1]}/{spent[0]}-{spent[1]}") assert_equal(len(json_obj['utxos']), 1) assert_equal(json_obj['bitmap'], "10") @@ -147,7 +145,7 @@ class RESTTest (BitcoinTestFramework): bin_request = b'\x01\x02' for txid, n in [spending, spent]: - bin_request += hex_str_to_bytes(txid) + bin_request += bytes.fromhex(txid) bin_request += pack("i", n) bin_response = self.test_rest_request("/getutxos", http_method='POST', req_type=ReqType.BIN, body=bin_request, ret_type=RetType.BYTES) @@ -165,32 +163,32 @@ class RESTTest (BitcoinTestFramework): # do a tx and don't sync txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1) - json_obj = self.test_rest_request("/tx/{}".format(txid)) + json_obj = self.test_rest_request(f"/tx/{txid}") # get the spent output to later check for utxo (should be spent by then) spent = (json_obj['vin'][0]['txid'], json_obj['vin'][0]['vout']) # get n of 0.1 outpoint n, = filter_output_indices_by_value(json_obj['vout'], Decimal('0.1')) spending = (txid, n) - json_obj = self.test_rest_request("/getutxos/{}-{}".format(*spending)) + json_obj = self.test_rest_request(f"/getutxos/{spending[0]}-{spending[1]}") assert_equal(len(json_obj['utxos']), 0) - json_obj = self.test_rest_request("/getutxos/checkmempool/{}-{}".format(*spending)) + json_obj = self.test_rest_request(f"/getutxos/checkmempool/{spending[0]}-{spending[1]}") assert_equal(len(json_obj['utxos']), 1) - json_obj = self.test_rest_request("/getutxos/{}-{}".format(*spent)) + json_obj = self.test_rest_request(f"/getutxos/{spent[0]}-{spent[1]}") assert_equal(len(json_obj['utxos']), 1) - json_obj = self.test_rest_request("/getutxos/checkmempool/{}-{}".format(*spent)) + json_obj = self.test_rest_request(f"/getutxos/checkmempool/{spent[0]}-{spent[1]}") assert_equal(len(json_obj['utxos']), 0) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() - json_obj = self.test_rest_request("/getutxos/{}-{}".format(*spending)) + json_obj = self.test_rest_request(f"/getutxos/{spending[0]}-{spending[1]}") assert_equal(len(json_obj['utxos']), 1) - json_obj = self.test_rest_request("/getutxos/checkmempool/{}-{}".format(*spending)) + json_obj = self.test_rest_request(f"/getutxos/checkmempool/{spending[0]}-{spending[1]}") assert_equal(len(json_obj['utxos']), 1) # Do some invalid requests @@ -199,13 +197,13 @@ class RESTTest (BitcoinTestFramework): self.test_rest_request("/getutxos/checkmempool", http_method='POST', req_type=ReqType.JSON, status=400, ret_type=RetType.OBJ) # Test limits - long_uri = '/'.join(["{}-{}".format(txid, n_) for n_ in range(20)]) - self.test_rest_request("/getutxos/checkmempool/{}".format(long_uri), http_method='POST', status=400, ret_type=RetType.OBJ) + long_uri = '/'.join([f"{txid}-{n_}" for n_ in range(20)]) + self.test_rest_request(f"/getutxos/checkmempool/{long_uri}", http_method='POST', status=400, ret_type=RetType.OBJ) - long_uri = '/'.join(['{}-{}'.format(txid, n_) for n_ in range(15)]) - self.test_rest_request("/getutxos/checkmempool/{}".format(long_uri), http_method='POST', status=200) + long_uri = '/'.join([f'{txid}-{n_}' for n_ in range(15)]) + self.test_rest_request(f"/getutxos/checkmempool/{long_uri}", http_method='POST', status=200) - self.nodes[0].generate(1) # generate block to not affect upcoming tests + self.generate(self.nodes[0], 1) # generate block to not affect upcoming tests self.sync_all() self.log.info("Test the /block, /blockhashbyheight and /headers URIs") @@ -217,42 +215,42 @@ class RESTTest (BitcoinTestFramework): # Check result if block is not in the active chain self.nodes[0].invalidateblock(bb_hash) - assert_equal(self.test_rest_request('/headers/1/{}'.format(bb_hash)), []) - self.test_rest_request('/block/{}'.format(bb_hash)) + assert_equal(self.test_rest_request(f'/headers/1/{bb_hash}'), []) + self.test_rest_request(f'/block/{bb_hash}') self.nodes[0].reconsiderblock(bb_hash) # Check binary format - response = self.test_rest_request("/block/{}".format(bb_hash), req_type=ReqType.BIN, ret_type=RetType.OBJ) + response = self.test_rest_request(f"/block/{bb_hash}", req_type=ReqType.BIN, ret_type=RetType.OBJ) assert_greater_than(int(response.getheader('content-length')), BLOCK_HEADER_SIZE) response_bytes = response.read() # Compare with block header - response_header = self.test_rest_request("/headers/1/{}".format(bb_hash), req_type=ReqType.BIN, ret_type=RetType.OBJ) + response_header = self.test_rest_request(f"/headers/1/{bb_hash}", req_type=ReqType.BIN, ret_type=RetType.OBJ) assert_equal(int(response_header.getheader('content-length')), BLOCK_HEADER_SIZE) response_header_bytes = response_header.read() assert_equal(response_bytes[:BLOCK_HEADER_SIZE], response_header_bytes) # Check block hex format - response_hex = self.test_rest_request("/block/{}".format(bb_hash), req_type=ReqType.HEX, ret_type=RetType.OBJ) + response_hex = self.test_rest_request(f"/block/{bb_hash}", req_type=ReqType.HEX, ret_type=RetType.OBJ) assert_greater_than(int(response_hex.getheader('content-length')), BLOCK_HEADER_SIZE*2) response_hex_bytes = response_hex.read().strip(b'\n') - assert_equal(binascii.hexlify(response_bytes), response_hex_bytes) + assert_equal(response_bytes.hex().encode(), response_hex_bytes) # Compare with hex block header - response_header_hex = self.test_rest_request("/headers/1/{}".format(bb_hash), req_type=ReqType.HEX, ret_type=RetType.OBJ) + response_header_hex = self.test_rest_request(f"/headers/1/{bb_hash}", req_type=ReqType.HEX, ret_type=RetType.OBJ) assert_greater_than(int(response_header_hex.getheader('content-length')), BLOCK_HEADER_SIZE*2) response_header_hex_bytes = response_header_hex.read(BLOCK_HEADER_SIZE*2) - assert_equal(binascii.hexlify(response_bytes[:BLOCK_HEADER_SIZE]), response_header_hex_bytes) + assert_equal(response_bytes[:BLOCK_HEADER_SIZE].hex().encode(), response_header_hex_bytes) # Check json format - block_json_obj = self.test_rest_request("/block/{}".format(bb_hash)) + block_json_obj = self.test_rest_request(f"/block/{bb_hash}") assert_equal(block_json_obj['hash'], bb_hash) - assert_equal(self.test_rest_request("/blockhashbyheight/{}".format(block_json_obj['height']))['blockhash'], bb_hash) + assert_equal(self.test_rest_request(f"/blockhashbyheight/{block_json_obj['height']}")['blockhash'], bb_hash) # Check hex/bin format - resp_hex = self.test_rest_request("/blockhashbyheight/{}".format(block_json_obj['height']), req_type=ReqType.HEX, ret_type=RetType.OBJ) + resp_hex = self.test_rest_request(f"/blockhashbyheight/{block_json_obj['height']}", req_type=ReqType.HEX, ret_type=RetType.OBJ) assert_equal(resp_hex.read().decode('utf-8').rstrip(), bb_hash) - resp_bytes = self.test_rest_request("/blockhashbyheight/{}".format(block_json_obj['height']), req_type=ReqType.BIN, ret_type=RetType.BYTES) + resp_bytes = self.test_rest_request(f"/blockhashbyheight/{block_json_obj['height']}", req_type=ReqType.BIN, ret_type=RetType.BYTES) blockhash = resp_bytes[::-1].hex() assert_equal(blockhash, bb_hash) @@ -266,7 +264,7 @@ class RESTTest (BitcoinTestFramework): self.test_rest_request("/blockhashbyheight/", ret_type=RetType.OBJ, status=400) # Compare with json block header - json_obj = self.test_rest_request("/headers/1/{}".format(bb_hash)) + json_obj = self.test_rest_request(f"/headers/1/{bb_hash}") assert_equal(len(json_obj), 1) # ensure that there is one header in the json response assert_equal(json_obj[0]['hash'], bb_hash) # request/response hash should be the same @@ -276,9 +274,9 @@ class RESTTest (BitcoinTestFramework): assert_equal(json_obj[0][key], rpc_block_json[key]) # See if we can get 5 headers in one response - self.nodes[1].generate(5) + self.generate(self.nodes[1], 5) self.sync_all() - json_obj = self.test_rest_request("/headers/5/{}".format(bb_hash)) + json_obj = self.test_rest_request(f"/headers/5/{bb_hash}") assert_equal(len(json_obj), 5) # now we should have 5 header objects self.log.info("Test tx inclusion in the /mempool and /block URIs") @@ -304,17 +302,17 @@ class RESTTest (BitcoinTestFramework): assert_equal(json_obj[tx]['depends'], txs[i - 1:i]) # Now mine the transactions - newblockhash = self.nodes[1].generate(1) + newblockhash = self.generate(self.nodes[1], 1) self.sync_all() # Check if the 3 tx show up in the new block - json_obj = self.test_rest_request("/block/{}".format(newblockhash[0])) + json_obj = self.test_rest_request(f"/block/{newblockhash[0]}") non_coinbase_txs = {tx['txid'] for tx in json_obj['tx'] if 'coinbase' not in tx['vin'][0]} assert_equal(non_coinbase_txs, set(txs)) # Check the same but without tx details - json_obj = self.test_rest_request("/block/notxdetails/{}".format(newblockhash[0])) + json_obj = self.test_rest_request(f"/block/notxdetails/{newblockhash[0]}") for tx in txs: assert tx in json_obj['tx'] diff --git a/test/functional/interface_rpc.py b/test/functional/interface_rpc.py index 4d5666f414..89a7d29b24 100755 --- a/test/functional/interface_rpc.py +++ b/test/functional/interface_rpc.py @@ -16,7 +16,7 @@ def expect_http_status(expected_http_status, expected_rpc_code, fcn, *args): try: fcn(*args) - raise AssertionError("Expected RPC error %d, got none" % expected_rpc_code) + raise AssertionError(f"Expected RPC error {expected_rpc_code}, got none") except JSONRPCException as exc: assert_equal(exc.error["code"], expected_rpc_code) assert_equal(exc.http_status, expected_http_status) diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 15f352d68c..5a11a62ec4 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -24,6 +24,7 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, ) +from test_framework.netutil import test_ipv6_local from io import BytesIO from time import sleep @@ -82,8 +83,8 @@ class ZMQTestSetupBlock: raw transaction data. """ - def __init__(self, node): - self.block_hash = node.generate(1)[0] + def __init__(self, test_framework, node): + self.block_hash = test_framework.generate(node, 1)[0] coinbase = node.getblock(self.block_hash, 2)['tx'][0] self.tx_hash = coinbase['txid'] self.raw_tx = coinbase['hex'] @@ -119,6 +120,7 @@ class ZMQTest (BitcoinTestFramework): self.test_mempool_sync() self.test_reorg() self.test_multiple_interfaces() + self.test_ipv6() finally: # Destroy the ZMQ context. self.log.debug("Destroying ZMQ context") @@ -126,13 +128,15 @@ class ZMQTest (BitcoinTestFramework): # Restart node with the specified zmq notifications enabled, subscribe to # all of them and return the corresponding ZMQSubscriber objects. - def setup_zmq_test(self, services, *, recv_timeout=60, sync_blocks=True): + def setup_zmq_test(self, services, *, recv_timeout=60, sync_blocks=True, ipv6=False): subscribers = [] for topic, address in services: socket = self.ctx.socket(zmq.SUB) + if ipv6: + socket.setsockopt(zmq.IPV6, 1) subscribers.append(ZMQSubscriber(socket, topic.encode())) - self.restart_node(0, ["-zmqpub%s=%s" % (topic, address) for topic, address in services] + + self.restart_node(0, [f"-zmqpub{topic}={address}" for topic, address in services] + self.extra_args[0]) for i, sub in enumerate(subscribers): @@ -147,7 +151,7 @@ class ZMQTest (BitcoinTestFramework): for sub in subscribers: sub.socket.set(zmq.RCVTIMEO, 1000) while True: - test_block = ZMQTestSetupBlock(self.nodes[0]) + test_block = ZMQTestSetupBlock(self, self.nodes[0]) recv_failed = False for sub in subscribers: try: @@ -184,8 +188,8 @@ class ZMQTest (BitcoinTestFramework): rawtx = subs[3] num_blocks = 5 - self.log.info("Generate %(n)d blocks (and %(n)d coinbase txes)" % {"n": num_blocks}) - genhashes = self.nodes[0].generatetoaddress(num_blocks, ADDRESS_BCRT1_UNSPENDABLE) + self.log.info(f"Generate {num_blocks} blocks (and {num_blocks} coinbase txes)") + genhashes = self.generatetoaddress(self.nodes[0], num_blocks, ADDRESS_BCRT1_UNSPENDABLE) self.sync_all() @@ -226,7 +230,7 @@ class ZMQTest (BitcoinTestFramework): # Mining the block with this tx should result in second notification # after coinbase tx notification - self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE) + self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) hashtx.receive() txid = hashtx.receive() assert_equal(payment_txid, txid.hex()) @@ -257,14 +261,14 @@ class ZMQTest (BitcoinTestFramework): # Generate 1 block in nodes[0] with 1 mempool tx and receive all notifications payment_txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1.0) - disconnect_block = self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE)[0] + disconnect_block = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE)[0] disconnect_cb = self.nodes[0].getblock(disconnect_block)["tx"][0] assert_equal(self.nodes[0].getbestblockhash(), hashblock.receive().hex()) assert_equal(hashtx.receive().hex(), payment_txid) assert_equal(hashtx.receive().hex(), disconnect_cb) # Generate 2 blocks in nodes[1] to a different address to ensure split - connect_blocks = self.nodes[1].generatetoaddress(2, ADDRESS_BCRT1_P2WSH_OP_TRUE) + connect_blocks = self.generatetoaddress(self.nodes[1], 2, ADDRESS_BCRT1_P2WSH_OP_TRUE) # nodes[0] will reorg chain after connecting back nodes[1] self.connect_nodes(0, 1) @@ -308,13 +312,13 @@ class ZMQTest (BitcoinTestFramework): seq_num = 1 # Generate 1 block in nodes[0] and receive all notifications - dc_block = self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE)[0] + dc_block = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE)[0] # Note: We are not notified of any block transactions, coinbase or mined assert_equal((self.nodes[0].getbestblockhash(), "C", None), seq.receive_sequence()) # Generate 2 blocks in nodes[1] to a different address to ensure a chain split - self.nodes[1].generatetoaddress(2, ADDRESS_BCRT1_P2WSH_OP_TRUE) + self.generatetoaddress(self.nodes[1], 2, ADDRESS_BCRT1_P2WSH_OP_TRUE) # nodes[0] will reorg chain after connecting back nodes[1] self.connect_nodes(0, 1) @@ -349,7 +353,7 @@ class ZMQTest (BitcoinTestFramework): # though the mempool sequence number does go up by the number of transactions # removed from the mempool by the block mining it. mempool_size = len(self.nodes[0].getrawmempool()) - c_block = self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE)[0] + c_block = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE)[0] self.sync_all() # Make sure the number of mined transactions matches the number of txs out of mempool mempool_size_delta = mempool_size - len(self.nodes[0].getrawmempool()) @@ -389,7 +393,7 @@ class ZMQTest (BitcoinTestFramework): # Other things may happen but aren't wallet-deterministic so we don't test for them currently self.nodes[0].reconsiderblock(best_hash) - self.nodes[1].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE) + self.generatetoaddress(self.nodes[1], 1, ADDRESS_BCRT1_UNSPENDABLE) self.sync_all() self.log.info("Evict mempool transaction by block conflict") @@ -441,7 +445,7 @@ class ZMQTest (BitcoinTestFramework): # Last tx assert_equal((orig_txid_2, "A", mempool_seq), seq.receive_sequence()) mempool_seq += 1 - self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE) + self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) self.sync_all() # want to make sure we didn't break "consensus" for other tests def test_mempool_sync(self): @@ -493,7 +497,7 @@ class ZMQTest (BitcoinTestFramework): txids.append(self.nodes[0].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=0.1, replaceable=True)) self.nodes[0].bumpfee(txids[-1]) self.sync_all() - self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE) + self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) final_txid = self.nodes[0].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=0.1, replaceable=True) # 3) Consume ZMQ backlog until we get to "now" for the mempool snapshot @@ -504,7 +508,7 @@ class ZMQTest (BitcoinTestFramework): if mempool_sequence is not None: zmq_mem_seq = mempool_sequence if zmq_mem_seq > get_raw_seq: - raise Exception("We somehow jumped mempool sequence numbers! zmq_mem_seq: {} > get_raw_seq: {}".format(zmq_mem_seq, get_raw_seq)) + raise Exception(f"We somehow jumped mempool sequence numbers! zmq_mem_seq: {zmq_mem_seq} > get_raw_seq: {get_raw_seq}") # 4) Moving forward, we apply the delta to our local view # remaining txs(5) + 1 rbf(A+R) + 1 block connect + 1 final tx @@ -520,7 +524,7 @@ class ZMQTest (BitcoinTestFramework): assert mempool_sequence > expected_sequence r_gap += mempool_sequence - expected_sequence else: - raise Exception("WARNING: txhash has unexpected mempool sequence value: {} vs expected {}".format(mempool_sequence, expected_sequence)) + raise Exception(f"WARNING: txhash has unexpected mempool sequence value: {mempool_sequence} vs expected {expected_sequence}") if label == "A": assert hash_str not in mempool_view mempool_view.add(hash_str) @@ -549,7 +553,7 @@ class ZMQTest (BitcoinTestFramework): # 5) If you miss a zmq/mempool sequence number, go back to step (2) - self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE) + self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) def test_multiple_interfaces(self): # Set up two subscribers with different addresses @@ -562,11 +566,28 @@ class ZMQTest (BitcoinTestFramework): ], sync_blocks=False) # Generate 1 block in nodes[0] and receive all notifications - self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE) + self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) # Should receive the same block hash on both subscribers assert_equal(self.nodes[0].getbestblockhash(), subscribers[0].receive().hex()) assert_equal(self.nodes[0].getbestblockhash(), subscribers[1].receive().hex()) + def test_ipv6(self): + if not test_ipv6_local(): + self.log.info("Skipping IPv6 test, because IPv6 is not supported.") + return + self.log.info("Testing IPv6") + # Set up subscriber using IPv6 loopback address + subscribers = self.setup_zmq_test([ + ("hashblock", "tcp://[::1]:28332") + ], ipv6=True) + + # Generate 1 block in nodes[0] + self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) + + # Should receive the same block hash + assert_equal(self.nodes[0].getbestblockhash(), subscribers[0].receive().hex()) + + if __name__ == '__main__': ZMQTest().main() diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index 1705d957aa..2ee440bcb7 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -13,22 +13,24 @@ from test_framework.messages import ( BIP125_SEQUENCE_NUMBER, COIN, COutPoint, + CTxIn, CTxOut, - MAX_BLOCK_BASE_SIZE, + MAX_BLOCK_WEIGHT, MAX_MONEY, tx_from_hex, ) from test_framework.script import ( - hash160, CScript, OP_0, OP_2, OP_3, OP_CHECKMULTISIG, - OP_EQUAL, OP_HASH160, OP_RETURN, ) +from test_framework.script_util import ( + script_to_p2sh_script, +) from test_framework.util import ( assert_equal, assert_raises_rpc_error, @@ -76,7 +78,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): outputs=[{node.getnewaddress(): 0.3}, {node.getnewaddress(): 49}], ))['hex'] txid_in_block = node.sendrawtransaction(hexstring=raw_tx_in_block, maxfeerate=0) - node.generate(1) + self.generate(node, 1) self.mempool_size = 0 self.check_mempool_result( result_expected=[{'txid': txid_in_block, 'allowed': False, 'reject-reason': 'txn-already-known'}], @@ -170,7 +172,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): outputs=[{node.getnewaddress(): 0.1}] ))['hex'] txid_spend_both = node.sendrawtransaction(hexstring=raw_tx_spend_both, maxfeerate=0) - node.generate(1) + self.generate(node, 1) self.mempool_size = 0 # Now see if we can add the coins back to the utxo set by sending the exact txs again self.check_mempool_result( @@ -207,7 +209,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): self.log.info('A really large transaction') tx = tx_from_hex(raw_tx_reference) - tx.vin = [tx.vin[0]] * math.ceil(MAX_BLOCK_BASE_SIZE / len(tx.vin[0].serialize())) + tx.vin = [tx.vin[0]] * math.ceil(MAX_BLOCK_WEIGHT // 4 / len(tx.vin[0].serialize())) self.check_mempool_result( result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'bad-txns-oversize'}], rawtxs=[tx.serialize().hex()], @@ -247,6 +249,14 @@ class MempoolAcceptanceTest(BitcoinTestFramework): rawtxs=[tx.serialize().hex()], ) + self.log.info('A non-coinbase transaction with coinbase-like outpoint') + tx = tx_from_hex(raw_tx_reference) + tx.vin.append(CTxIn(COutPoint(hash=0, n=0xffffffff))) + self.check_mempool_result( + result_expected=[{'txid': tx.rehash(), 'allowed': False, 'reject-reason': 'bad-txns-prevout-null'}], + rawtxs=[tx.serialize().hex()], + ) + self.log.info('A coinbase transaction') # Pick the input of the first tx we signed, so it has to be a coinbase tx raw_tx_coinbase_spent = node.getrawtransaction(txid=node.decoderawtransaction(hexstring=raw_tx_in_block)['vin'][0]['txid']) @@ -291,7 +301,7 @@ class MempoolAcceptanceTest(BitcoinTestFramework): rawtxs=[tx.serialize().hex()], ) tx = tx_from_hex(raw_tx_reference) - output_p2sh_burn = CTxOut(nValue=540, scriptPubKey=CScript([OP_HASH160, hash160(b'burn'), OP_EQUAL])) + output_p2sh_burn = CTxOut(nValue=540, scriptPubKey=script_to_p2sh_script(b'burn')) num_scripts = 100000 // len(output_p2sh_burn.serialize()) # Use enough outputs to make the tx too large for our policy tx.vout = [output_p2sh_burn] * num_scripts self.check_mempool_result( diff --git a/test/functional/mempool_accept_wtxid.py b/test/functional/mempool_accept_wtxid.py new file mode 100755 index 0000000000..4767d6db22 --- /dev/null +++ b/test/functional/mempool_accept_wtxid.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test mempool acceptance in case of an already known transaction +with identical non-witness data but different witness. +""" + +from copy import deepcopy +from test_framework.messages import ( + COIN, + COutPoint, + CTransaction, + CTxIn, + CTxInWitness, + CTxOut, + sha256, +) +from test_framework.p2p import P2PTxInvStore +from test_framework.script import ( + CScript, + OP_0, + OP_ELSE, + OP_ENDIF, + OP_EQUAL, + OP_HASH160, + OP_IF, + OP_TRUE, + hash160, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, +) + +class MempoolWtxidTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def run_test(self): + node = self.nodes[0] + + self.log.info('Start with empty mempool and 101 blocks') + # The last 100 coinbase transactions are premature + blockhash = self.generate(node, 101)[0] + txid = node.getblock(blockhash=blockhash, verbosity=2)["tx"][0]["txid"] + assert_equal(node.getmempoolinfo()['size'], 0) + + self.log.info("Submit parent with multiple script branches to mempool") + hashlock = hash160(b'Preimage') + witness_script = CScript([OP_IF, OP_HASH160, hashlock, OP_EQUAL, OP_ELSE, OP_TRUE, OP_ENDIF]) + witness_program = sha256(witness_script) + script_pubkey = CScript([OP_0, witness_program]) + + parent = CTransaction() + parent.vin.append(CTxIn(COutPoint(int(txid, 16), 0), b"")) + parent.vout.append(CTxOut(int(9.99998 * COIN), script_pubkey)) + parent.rehash() + + privkeys = [node.get_deterministic_priv_key().key] + raw_parent = node.signrawtransactionwithkey(hexstring=parent.serialize().hex(), privkeys=privkeys)['hex'] + parent_txid = node.sendrawtransaction(hexstring=raw_parent, maxfeerate=0) + self.generate(node, 1) + + peer_wtxid_relay = node.add_p2p_connection(P2PTxInvStore()) + + # Create a new transaction with witness solving first branch + child_witness_script = CScript([OP_TRUE]) + child_witness_program = sha256(child_witness_script) + child_script_pubkey = CScript([OP_0, child_witness_program]) + + child_one = CTransaction() + child_one.vin.append(CTxIn(COutPoint(int(parent_txid, 16), 0), b"")) + child_one.vout.append(CTxOut(int(9.99996 * COIN), child_script_pubkey)) + child_one.wit.vtxinwit.append(CTxInWitness()) + child_one.wit.vtxinwit[0].scriptWitness.stack = [b'Preimage', b'\x01', witness_script] + child_one_wtxid = child_one.getwtxid() + child_one_txid = child_one.rehash() + + # Create another identical transaction with witness solving second branch + child_two = deepcopy(child_one) + child_two.wit.vtxinwit[0].scriptWitness.stack = [b'', witness_script] + child_two_wtxid = child_two.getwtxid() + child_two_txid = child_two.rehash() + + assert_equal(child_one_txid, child_two_txid) + assert child_one_wtxid != child_two_wtxid + + self.log.info("Submit child_one to the mempool") + txid_submitted = node.sendrawtransaction(child_one.serialize().hex()) + assert_equal(node.getmempoolentry(txid_submitted)['wtxid'], child_one_wtxid) + + peer_wtxid_relay.wait_for_broadcast([child_one_wtxid]) + assert_equal(node.getmempoolinfo()["unbroadcastcount"], 0) + + # testmempoolaccept reports the "already in mempool" error + assert_equal(node.testmempoolaccept([child_one.serialize().hex()]), [{ + "txid": child_one_txid, + "wtxid": child_one_wtxid, + "allowed": False, + "reject-reason": "txn-already-in-mempool" + }]) + assert_equal(node.testmempoolaccept([child_two.serialize().hex()])[0], { + "txid": child_two_txid, + "wtxid": child_two_wtxid, + "allowed": False, + "reject-reason": "txn-same-nonwitness-data-in-mempool" + }) + + # sendrawtransaction will not throw but quits early when the exact same transaction is already in mempool + node.sendrawtransaction(child_one.serialize().hex()) + + self.log.info("Connect another peer that hasn't seen child_one before") + peer_wtxid_relay_2 = node.add_p2p_connection(P2PTxInvStore()) + + self.log.info("Submit child_two to the mempool") + # sendrawtransaction will not throw but quits early when a transaction with the same non-witness data is already in mempool + node.sendrawtransaction(child_two.serialize().hex()) + + # The node should rebroadcast the transaction using the wtxid of the correct transaction + # (child_one, which is in its mempool). + peer_wtxid_relay_2.wait_for_broadcast([child_one_wtxid]) + assert_equal(node.getmempoolinfo()["unbroadcastcount"], 0) + +if __name__ == '__main__': + MempoolWtxidTest().main() diff --git a/test/functional/mempool_compatibility.py b/test/functional/mempool_compatibility.py index 6de6778909..a9f09b1cf8 100755 --- a/test/functional/mempool_compatibility.py +++ b/test/functional/mempool_compatibility.py @@ -7,10 +7,7 @@ NOTE: The test is designed to prevent cases when compatibility is broken accidentally. In case we need to break mempool compatibility we can continue to use the test by just bumping the version number. -Download node binaries: -test/get_previous_releases.py -b v0.19.1 v0.18.1 v0.17.2 v0.16.3 v0.15.2 - -Only v0.15.2 is required by this test. The rest is used in other backwards compatibility tests. +The previous release v0.15.2 is required by this test, see test/README.md. """ import os @@ -41,8 +38,8 @@ class MempoolCompatibilityTest(BitcoinTestFramework): old_node, new_node = self.nodes new_wallet = MiniWallet(new_node) - new_wallet.generate(1) - new_node.generate(COINBASE_MATURITY) + self.generate(new_wallet, 1) + self.generate(new_node, COINBASE_MATURITY) # Sync the nodes to ensure old_node has the block that contains the coinbase that new_wallet will spend. # Otherwise, because coinbases are only valid in a block and not as loose txns, if the nodes aren't synced # unbroadcasted_tx won't pass old_node's `MemPoolAccept::PreChecks`. @@ -68,8 +65,7 @@ class MempoolCompatibilityTest(BitcoinTestFramework): self.log.info("Add unbroadcasted tx to mempool on new node and shutdown") unbroadcasted_tx_hash = new_wallet.send_self_transfer(from_node=new_node)['txid'] assert unbroadcasted_tx_hash in new_node.getrawmempool() - mempool = new_node.getrawmempool(True) - assert mempool[unbroadcasted_tx_hash]['unbroadcast'] + assert new_node.getmempoolentry(unbroadcasted_tx_hash)['unbroadcast'] self.stop_node(1) self.log.info("Move mempool.dat from new to old node") diff --git a/test/functional/mempool_expiry.py b/test/functional/mempool_expiry.py index 7d1bfef333..942f79e8b0 100755 --- a/test/functional/mempool_expiry.py +++ b/test/functional/mempool_expiry.py @@ -36,8 +36,8 @@ class MempoolExpiryTest(BitcoinTestFramework): self.wallet = MiniWallet(node) # Add enough mature utxos to the wallet so that all txs spend confirmed coins. - self.wallet.generate(4) - node.generate(COINBASE_MATURITY) + self.generate(self.wallet, 4) + self.generate(node, COINBASE_MATURITY) # Send a parent transaction that will expire. parent_txid = self.wallet.send_self_transfer(from_node=node)['txid'] diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index 39035f7cb1..c82dbb3f3d 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -6,8 +6,11 @@ from decimal import Decimal +from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, create_confirmed_utxos, create_lots_of_big_transactions, gen_return_txouts +from test_framework.util import assert_equal, assert_greater_than, assert_raises_rpc_error, gen_return_txouts +from test_framework.wallet import MiniWallet + class MempoolLimitTest(BitcoinTestFramework): def set_test_params(self): @@ -20,55 +23,59 @@ class MempoolLimitTest(BitcoinTestFramework): ]] self.supports_cli = False - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + def send_large_txs(self, node, miniwallet, txouts, fee_rate, tx_batch_size): + for _ in range(tx_batch_size): + tx = miniwallet.create_self_transfer(from_node=node, fee_rate=fee_rate)['tx'] + for txout in txouts: + tx.vout.append(txout) + miniwallet.sendrawtransaction(from_node=node, tx_hex=tx.serialize().hex()) def run_test(self): txouts = gen_return_txouts() - relayfee = self.nodes[0].getnetworkinfo()['relayfee'] + node=self.nodes[0] + miniwallet = MiniWallet(node) + relayfee = node.getnetworkinfo()['relayfee'] + + self.log.info('Check that mempoolminfee is minrelaytxfee') + assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) + assert_equal(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) - self.log.info('Check that mempoolminfee is minrelytxfee') - assert_equal(self.nodes[0].getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) - assert_equal(self.nodes[0].getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) + tx_batch_size = 25 + num_of_batches = 3 + # Generate UTXOs to flood the mempool + # 1 to create a tx initially that will be evicted from the mempool later + # 3 batches of multiple transactions with a fee rate much higher than the previous UTXO + # And 1 more to verify that this tx does not get added to the mempool with a fee rate less than the mempoolminfee + self.generate(miniwallet, 1 + (num_of_batches * tx_batch_size) + 1) - txids = [] - utxos = create_confirmed_utxos(relayfee, self.nodes[0], 91) + # Mine 99 blocks so that the UTXOs are allowed to be spent + self.generate(node, COINBASE_MATURITY - 1) self.log.info('Create a mempool tx that will be evicted') - us0 = utxos.pop() - inputs = [{ "txid" : us0["txid"], "vout" : us0["vout"]}] - outputs = {self.nodes[0].getnewaddress() : 0.0001} - tx = self.nodes[0].createrawtransaction(inputs, outputs) - self.nodes[0].settxfee(relayfee) # specifically fund this tx with low fee - txF = self.nodes[0].fundrawtransaction(tx) - self.nodes[0].settxfee(0) # return to automatic fee selection - txFS = self.nodes[0].signrawtransactionwithwallet(txF['hex']) - txid = self.nodes[0].sendrawtransaction(txFS['hex']) - - relayfee = self.nodes[0].getnetworkinfo()['relayfee'] - base_fee = relayfee*100 - for i in range (3): - txids.append([]) - txids[i] = create_lots_of_big_transactions(self.nodes[0], txouts, utxos[30*i:30*i+30], 30, (i+1)*base_fee) + tx_to_be_evicted_id = miniwallet.send_self_transfer(from_node=node, fee_rate=relayfee)["txid"] + + # Increase the tx fee rate massively to give the subsequent transactions a higher priority in the mempool + base_fee = relayfee * 1000 + + self.log.info("Fill up the mempool with txs with higher fee rate") + for batch_of_txid in range(num_of_batches): + fee_rate=(batch_of_txid + 1) * base_fee + self.send_large_txs(node, miniwallet, txouts, fee_rate, tx_batch_size) self.log.info('The tx should be evicted by now') - assert txid not in self.nodes[0].getrawmempool() - txdata = self.nodes[0].gettransaction(txid) - assert txdata['confirmations'] == 0 #confirmation should still be 0 + # The number of transactions created should be greater than the ones present in the mempool + assert_greater_than(tx_batch_size * num_of_batches, len(node.getrawmempool())) + # Initial tx created should not be present in the mempool anymore as it had a lower fee rate + assert tx_to_be_evicted_id not in node.getrawmempool() - self.log.info('Check that mempoolminfee is larger than minrelytxfee') - assert_equal(self.nodes[0].getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) - assert_greater_than(self.nodes[0].getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) + self.log.info('Check that mempoolminfee is larger than minrelaytxfee') + assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000')) + assert_greater_than(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000')) + # Deliberately try to create a tx with a fee less than the minimum mempool fee to assert that it does not get added to the mempool self.log.info('Create a mempool tx that will not pass mempoolminfee') - us0 = utxos.pop() - inputs = [{ "txid" : us0["txid"], "vout" : us0["vout"]}] - outputs = {self.nodes[0].getnewaddress() : 0.0001} - tx = self.nodes[0].createrawtransaction(inputs, outputs) - # specifically fund this tx with a fee < mempoolminfee, >= than minrelaytxfee - txF = self.nodes[0].fundrawtransaction(tx, {'feeRate': relayfee}) - txFS = self.nodes[0].signrawtransactionwithwallet(txF['hex']) - assert_raises_rpc_error(-26, "mempool min fee not met", self.nodes[0].sendrawtransaction, txFS['hex']) + assert_raises_rpc_error(-26, "mempool min fee not met", miniwallet.send_self_transfer, from_node=node, fee_rate=relayfee, mempool_valid=False) + if __name__ == '__main__': MempoolLimitTest().main() diff --git a/test/functional/mempool_package_limits.py b/test/functional/mempool_package_limits.py new file mode 100755 index 0000000000..2217628858 --- /dev/null +++ b/test/functional/mempool_package_limits.py @@ -0,0 +1,545 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test logic for limiting mempool and package ancestors/descendants.""" + +from decimal import Decimal + +from test_framework.address import ADDRESS_BCRT1_P2WSH_OP_TRUE +from test_framework.test_framework import BitcoinTestFramework +from test_framework.messages import ( + COIN, + CTransaction, + CTxInWitness, + tx_from_hex, + WITNESS_SCALE_FACTOR, +) +from test_framework.script import ( + CScript, + OP_TRUE, +) +from test_framework.util import ( + assert_equal, +) +from test_framework.wallet import ( + bulk_transaction, + create_child_with_parents, + make_chain, + DEFAULT_FEE, +) + +class MempoolPackageLimitsTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def run_test(self): + self.log.info("Generate blocks to create UTXOs") + node = self.nodes[0] + self.privkeys = [node.get_deterministic_priv_key().key] + self.address = node.get_deterministic_priv_key().address + self.coins = [] + # The last 100 coinbase transactions are premature + for b in self.generatetoaddress(node, 200, self.address)[:100]: + coinbase = node.getblock(blockhash=b, verbosity=2)["tx"][0] + self.coins.append({ + "txid": coinbase["txid"], + "amount": coinbase["vout"][0]["value"], + "scriptPubKey": coinbase["vout"][0]["scriptPubKey"], + }) + + self.test_chain_limits() + self.test_desc_count_limits() + self.test_desc_count_limits_2() + self.test_anc_count_limits() + self.test_anc_count_limits_2() + self.test_anc_count_limits_bushy() + + # The node will accept our (nonstandard) extra large OP_RETURN outputs + self.restart_node(0, extra_args=["-acceptnonstdtxn=1"]) + self.test_anc_size_limits() + self.test_desc_size_limits() + + def test_chain_limits_helper(self, mempool_count, package_count): + node = self.nodes[0] + assert_equal(0, node.getmempoolinfo()["size"]) + first_coin = self.coins.pop() + spk = None + txid = first_coin["txid"] + chain_hex = [] + chain_txns = [] + value = first_coin["amount"] + + for i in range(mempool_count + package_count): + (tx, txhex, value, spk) = make_chain(node, self.address, self.privkeys, txid, value, 0, spk) + txid = tx.rehash() + if i < mempool_count: + node.sendrawtransaction(txhex) + assert_equal(node.getmempoolentry(txid)["ancestorcount"], i + 1) + else: + chain_hex.append(txhex) + chain_txns.append(tx) + testres_too_long = node.testmempoolaccept(rawtxs=chain_hex) + for txres in testres_too_long: + assert_equal(txres["package-error"], "package-mempool-limits") + + # Clear mempool and check that the package passes now + self.generate(node, 1) + assert all([res["allowed"] for res in node.testmempoolaccept(rawtxs=chain_hex)]) + + def test_chain_limits(self): + """Create chains from mempool and package transactions that are longer than 25, + but only if both in-mempool and in-package transactions are considered together. + This checks that both mempool and in-package transactions are taken into account when + calculating ancestors/descendant limits. + """ + self.log.info("Check that in-package ancestors count for mempool ancestor limits") + + # 24 transactions in the mempool and 2 in the package. The parent in the package has + # 24 in-mempool ancestors and 1 in-package descendant. The child has 0 direct parents + # in the mempool, but 25 in-mempool and in-package ancestors in total. + self.test_chain_limits_helper(24, 2) + # 2 transactions in the mempool and 24 in the package. + self.test_chain_limits_helper(2, 24) + # 13 transactions in the mempool and 13 in the package. + self.test_chain_limits_helper(13, 13) + + def test_desc_count_limits(self): + """Create an 'A' shaped package with 24 transactions in the mempool and 2 in the package: + M1 + ^ ^ + M2a M2b + . . + . . + . . + M12a ^ + ^ M13b + ^ ^ + Pa Pb + The top ancestor in the package exceeds descendant limits but only if the in-mempool and in-package + descendants are all considered together (24 including in-mempool descendants and 26 including both + package transactions). + """ + node = self.nodes[0] + assert_equal(0, node.getmempoolinfo()["size"]) + self.log.info("Check that in-mempool and in-package descendants are calculated properly in packages") + # Top parent in mempool, M1 + first_coin = self.coins.pop() + parent_value = (first_coin["amount"] - Decimal("0.0002")) / 2 # Deduct reasonable fee and make 2 outputs + inputs = [{"txid": first_coin["txid"], "vout": 0}] + outputs = [{self.address : parent_value}, {ADDRESS_BCRT1_P2WSH_OP_TRUE : parent_value}] + rawtx = node.createrawtransaction(inputs, outputs) + + parent_signed = node.signrawtransactionwithkey(hexstring=rawtx, privkeys=self.privkeys) + assert parent_signed["complete"] + parent_tx = tx_from_hex(parent_signed["hex"]) + parent_txid = parent_tx.rehash() + node.sendrawtransaction(parent_signed["hex"]) + + package_hex = [] + + # Chain A + spk = parent_tx.vout[0].scriptPubKey.hex() + value = parent_value + txid = parent_txid + for i in range(12): + (tx, txhex, value, spk) = make_chain(node, self.address, self.privkeys, txid, value, 0, spk) + txid = tx.rehash() + if i < 11: # M2a... M12a + node.sendrawtransaction(txhex) + else: # Pa + package_hex.append(txhex) + + # Chain B + value = parent_value - Decimal("0.0001") + rawtx_b = node.createrawtransaction([{"txid": parent_txid, "vout": 1}], {self.address : value}) + tx_child_b = tx_from_hex(rawtx_b) # M2b + tx_child_b.wit.vtxinwit = [CTxInWitness()] + tx_child_b.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] + tx_child_b_hex = tx_child_b.serialize().hex() + node.sendrawtransaction(tx_child_b_hex) + spk = tx_child_b.vout[0].scriptPubKey.hex() + txid = tx_child_b.rehash() + for i in range(12): + (tx, txhex, value, spk) = make_chain(node, self.address, self.privkeys, txid, value, 0, spk) + txid = tx.rehash() + if i < 11: # M3b... M13b + node.sendrawtransaction(txhex) + else: # Pb + package_hex.append(txhex) + + assert_equal(24, node.getmempoolinfo()["size"]) + assert_equal(2, len(package_hex)) + testres_too_long = node.testmempoolaccept(rawtxs=package_hex) + for txres in testres_too_long: + assert_equal(txres["package-error"], "package-mempool-limits") + + # Clear mempool and check that the package passes now + self.generate(node, 1) + assert all([res["allowed"] for res in node.testmempoolaccept(rawtxs=package_hex)]) + + def test_desc_count_limits_2(self): + """Create a Package with 24 transaction in mempool and 2 transaction in package: + M1 + ^ ^ + M2 ^ + . ^ + . ^ + . ^ + M24 ^ + ^ + P1 + ^ + P2 + P1 has M1 as a mempool ancestor, P2 has no in-mempool ancestors, but when + combined P2 has M1 as an ancestor and M1 exceeds descendant_limits(23 in-mempool + descendants + 2 in-package descendants, a total of 26 including itself). + """ + + node = self.nodes[0] + package_hex = [] + # M1 + first_coin_a = self.coins.pop() + parent_value = (first_coin_a["amount"] - DEFAULT_FEE) / 2 # Deduct reasonable fee and make 2 outputs + inputs = [{"txid": first_coin_a["txid"], "vout": 0}] + outputs = [{self.address : parent_value}, {ADDRESS_BCRT1_P2WSH_OP_TRUE : parent_value}] + rawtx = node.createrawtransaction(inputs, outputs) + + parent_signed = node.signrawtransactionwithkey(hexstring=rawtx, privkeys=self.privkeys) + assert parent_signed["complete"] + parent_tx = tx_from_hex(parent_signed["hex"]) + parent_txid = parent_tx.rehash() + node.sendrawtransaction(parent_signed["hex"]) + + # Chain M2...M24 + spk = parent_tx.vout[0].scriptPubKey.hex() + value = parent_value + txid = parent_txid + for i in range(23): # M2...M24 + (tx, txhex, value, spk) = make_chain(node, self.address, self.privkeys, txid, value, 0, spk) + txid = tx.rehash() + node.sendrawtransaction(txhex) + + # P1 + value_p1 = (parent_value - DEFAULT_FEE) + rawtx_p1 = node.createrawtransaction([{"txid": parent_txid, "vout": 1}], [{self.address : value_p1}]) + tx_child_p1 = tx_from_hex(rawtx_p1) + tx_child_p1.wit.vtxinwit = [CTxInWitness()] + tx_child_p1.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] + tx_child_p1_hex = tx_child_p1.serialize().hex() + txid_child_p1 = tx_child_p1.rehash() + package_hex.append(tx_child_p1_hex) + tx_child_p1_spk = tx_child_p1.vout[0].scriptPubKey.hex() + + # P2 + (_, tx_child_p2_hex, _, _) = make_chain(node, self.address, self.privkeys, txid_child_p1, value_p1, 0, tx_child_p1_spk) + package_hex.append(tx_child_p2_hex) + + assert_equal(24, node.getmempoolinfo()["size"]) + assert_equal(2, len(package_hex)) + testres = node.testmempoolaccept(rawtxs=package_hex) + assert_equal(len(testres), len(package_hex)) + for txres in testres: + assert_equal(txres["package-error"], "package-mempool-limits") + + # Clear mempool and check that the package passes now + node.generate(1) + assert all([res["allowed"] for res in node.testmempoolaccept(rawtxs=package_hex)]) + + def test_anc_count_limits(self): + """Create a 'V' shaped chain with 24 transactions in the mempool and 3 in the package: + M1a M1b + ^ ^ + M2a M2b + . . + . . + . . + M12a M12b + ^ ^ + Pa Pb + ^ ^ + Pc + The lowest descendant, Pc, exceeds ancestor limits, but only if the in-mempool + and in-package ancestors are all considered together. + """ + node = self.nodes[0] + assert_equal(0, node.getmempoolinfo()["size"]) + package_hex = [] + parents_tx = [] + values = [] + scripts = [] + + self.log.info("Check that in-mempool and in-package ancestors are calculated properly in packages") + + # Two chains of 13 transactions each + for _ in range(2): + spk = None + top_coin = self.coins.pop() + txid = top_coin["txid"] + value = top_coin["amount"] + for i in range(13): + (tx, txhex, value, spk) = make_chain(node, self.address, self.privkeys, txid, value, 0, spk) + txid = tx.rehash() + if i < 12: + node.sendrawtransaction(txhex) + else: # Save the 13th transaction for the package + package_hex.append(txhex) + parents_tx.append(tx) + scripts.append(spk) + values.append(value) + + # Child Pc + child_hex = create_child_with_parents(node, self.address, self.privkeys, parents_tx, values, scripts) + package_hex.append(child_hex) + + assert_equal(24, node.getmempoolinfo()["size"]) + assert_equal(3, len(package_hex)) + testres_too_long = node.testmempoolaccept(rawtxs=package_hex) + for txres in testres_too_long: + assert_equal(txres["package-error"], "package-mempool-limits") + + # Clear mempool and check that the package passes now + self.generate(node, 1) + assert all([res["allowed"] for res in node.testmempoolaccept(rawtxs=package_hex)]) + + def test_anc_count_limits_2(self): + """Create a 'Y' shaped chain with 24 transactions in the mempool and 2 in the package: + M1a M1b + ^ ^ + M2a M2b + . . + . . + . . + M12a M12b + ^ ^ + Pc + ^ + Pd + The lowest descendant, Pd, exceeds ancestor limits, but only if the in-mempool + and in-package ancestors are all considered together. + """ + node = self.nodes[0] + assert_equal(0, node.getmempoolinfo()["size"]) + parents_tx = [] + values = [] + scripts = [] + + self.log.info("Check that in-mempool and in-package ancestors are calculated properly in packages") + # Two chains of 12 transactions each + for _ in range(2): + spk = None + top_coin = self.coins.pop() + txid = top_coin["txid"] + value = top_coin["amount"] + for i in range(12): + (tx, txhex, value, spk) = make_chain(node, self.address, self.privkeys, txid, value, 0, spk) + txid = tx.rehash() + value -= Decimal("0.0001") + node.sendrawtransaction(txhex) + if i == 11: + # last 2 transactions will be the parents of Pc + parents_tx.append(tx) + values.append(value) + scripts.append(spk) + + # Child Pc + pc_hex = create_child_with_parents(node, self.address, self.privkeys, parents_tx, values, scripts) + pc_tx = tx_from_hex(pc_hex) + pc_value = sum(values) - Decimal("0.0002") + pc_spk = pc_tx.vout[0].scriptPubKey.hex() + + # Child Pd + (_, pd_hex, _, _) = make_chain(node, self.address, self.privkeys, pc_tx.rehash(), pc_value, 0, pc_spk) + + assert_equal(24, node.getmempoolinfo()["size"]) + testres_too_long = node.testmempoolaccept(rawtxs=[pc_hex, pd_hex]) + for txres in testres_too_long: + assert_equal(txres["package-error"], "package-mempool-limits") + + # Clear mempool and check that the package passes now + self.generate(node, 1) + assert all([res["allowed"] for res in node.testmempoolaccept(rawtxs=[pc_hex, pd_hex])]) + + def test_anc_count_limits_bushy(self): + """Create a tree with 20 transactions in the mempool and 6 in the package: + M1...M4 M5...M8 M9...M12 M13...M16 M17...M20 + ^ ^ ^ ^ ^ (each with 4 parents) + P0 P1 P2 P3 P4 + ^ ^ ^ ^ ^ (5 parents) + PC + Where M(4i+1)...M+(4i+4) are the parents of Pi and P0, P1, P2, P3, and P4 are the parents of PC. + P0... P4 individually only have 4 parents each, and PC has no in-mempool parents. But + combined, PC has 25 in-mempool and in-package parents. + """ + node = self.nodes[0] + assert_equal(0, node.getmempoolinfo()["size"]) + package_hex = [] + parent_txns = [] + parent_values = [] + scripts = [] + for _ in range(5): # Make package transactions P0 ... P4 + gp_tx = [] + gp_values = [] + gp_scripts = [] + for _ in range(4): # Make mempool transactions M(4i+1)...M(4i+4) + parent_coin = self.coins.pop() + value = parent_coin["amount"] + txid = parent_coin["txid"] + (tx, txhex, value, spk) = make_chain(node, self.address, self.privkeys, txid, value) + gp_tx.append(tx) + gp_values.append(value) + gp_scripts.append(spk) + node.sendrawtransaction(txhex) + # Package transaction Pi + pi_hex = create_child_with_parents(node, self.address, self.privkeys, gp_tx, gp_values, gp_scripts) + package_hex.append(pi_hex) + pi_tx = tx_from_hex(pi_hex) + parent_txns.append(pi_tx) + parent_values.append(Decimal(pi_tx.vout[0].nValue) / COIN) + scripts.append(pi_tx.vout[0].scriptPubKey.hex()) + # Package transaction PC + package_hex.append(create_child_with_parents(node, self.address, self.privkeys, parent_txns, parent_values, scripts)) + + assert_equal(20, node.getmempoolinfo()["size"]) + assert_equal(6, len(package_hex)) + testres = node.testmempoolaccept(rawtxs=package_hex) + for txres in testres: + assert_equal(txres["package-error"], "package-mempool-limits") + + # Clear mempool and check that the package passes now + self.generate(node, 1) + assert all([res["allowed"] for res in node.testmempoolaccept(rawtxs=package_hex)]) + + def test_anc_size_limits(self): + """Test Case with 2 independent transactions in the mempool and a parent + child in the + package, where the package parent is the child of both mempool transactions (30KvB each): + A B + ^ ^ + C + ^ + D + The lowest descendant, D, exceeds ancestor size limits, but only if the in-mempool + and in-package ancestors are all considered together. + """ + node = self.nodes[0] + assert_equal(0, node.getmempoolinfo()["size"]) + parents_tx = [] + values = [] + scripts = [] + target_weight = WITNESS_SCALE_FACTOR * 1000 * 30 # 30KvB + high_fee = Decimal("0.003") # 10 sats/vB + self.log.info("Check that in-mempool and in-package ancestor size limits are calculated properly in packages") + # Mempool transactions A and B + for _ in range(2): + spk = None + top_coin = self.coins.pop() + txid = top_coin["txid"] + value = top_coin["amount"] + (tx, _, _, _) = make_chain(node, self.address, self.privkeys, txid, value, 0, spk, high_fee) + bulked_tx = bulk_transaction(tx, node, target_weight, self.privkeys) + node.sendrawtransaction(bulked_tx.serialize().hex()) + parents_tx.append(bulked_tx) + values.append(Decimal(bulked_tx.vout[0].nValue) / COIN) + scripts.append(bulked_tx.vout[0].scriptPubKey.hex()) + + # Package transaction C + small_pc_hex = create_child_with_parents(node, self.address, self.privkeys, parents_tx, values, scripts, high_fee) + pc_tx = bulk_transaction(tx_from_hex(small_pc_hex), node, target_weight, self.privkeys) + pc_value = Decimal(pc_tx.vout[0].nValue) / COIN + pc_spk = pc_tx.vout[0].scriptPubKey.hex() + pc_hex = pc_tx.serialize().hex() + + # Package transaction D + (small_pd, _, val, spk) = make_chain(node, self.address, self.privkeys, pc_tx.rehash(), pc_value, 0, pc_spk, high_fee) + prevtxs = [{ + "txid": pc_tx.rehash(), + "vout": 0, + "scriptPubKey": spk, + "amount": val, + }] + pd_tx = bulk_transaction(small_pd, node, target_weight, self.privkeys, prevtxs) + pd_hex = pd_tx.serialize().hex() + + assert_equal(2, node.getmempoolinfo()["size"]) + testres_too_heavy = node.testmempoolaccept(rawtxs=[pc_hex, pd_hex]) + for txres in testres_too_heavy: + assert_equal(txres["package-error"], "package-mempool-limits") + + # Clear mempool and check that the package passes now + self.generate(node, 1) + assert all([res["allowed"] for res in node.testmempoolaccept(rawtxs=[pc_hex, pd_hex])]) + + def test_desc_size_limits(self): + """Create 3 mempool transactions and 2 package transactions (25KvB each): + Ma + ^ ^ + Mb Mc + ^ ^ + Pd Pe + The top ancestor in the package exceeds descendant size limits but only if the in-mempool + and in-package descendants are all considered together. + """ + node = self.nodes[0] + assert_equal(0, node.getmempoolinfo()["size"]) + target_weight = 21 * 1000 * WITNESS_SCALE_FACTOR + high_fee = Decimal("0.0021") # 10 sats/vB + self.log.info("Check that in-mempool and in-package descendant sizes are calculated properly in packages") + # Top parent in mempool, Ma + first_coin = self.coins.pop() + parent_value = (first_coin["amount"] - high_fee) / 2 # Deduct fee and make 2 outputs + inputs = [{"txid": first_coin["txid"], "vout": 0}] + outputs = [{self.address : parent_value}, {ADDRESS_BCRT1_P2WSH_OP_TRUE: parent_value}] + rawtx = node.createrawtransaction(inputs, outputs) + parent_tx = bulk_transaction(tx_from_hex(rawtx), node, target_weight, self.privkeys) + node.sendrawtransaction(parent_tx.serialize().hex()) + + package_hex = [] + for j in range(2): # Two legs (left and right) + # Mempool transaction (Mb and Mc) + mempool_tx = CTransaction() + spk = parent_tx.vout[j].scriptPubKey.hex() + value = Decimal(parent_tx.vout[j].nValue) / COIN + txid = parent_tx.rehash() + prevtxs = [{ + "txid": txid, + "vout": j, + "scriptPubKey": spk, + "amount": value, + }] + if j == 0: # normal key + (tx_small, _, _, _) = make_chain(node, self.address, self.privkeys, txid, value, j, spk, high_fee) + mempool_tx = bulk_transaction(tx_small, node, target_weight, self.privkeys, prevtxs) + else: # OP_TRUE + inputs = [{"txid": txid, "vout": 1}] + outputs = {self.address: value - high_fee} + small_tx = tx_from_hex(node.createrawtransaction(inputs, outputs)) + mempool_tx = bulk_transaction(small_tx, node, target_weight, None, prevtxs) + node.sendrawtransaction(mempool_tx.serialize().hex()) + + # Package transaction (Pd and Pe) + spk = mempool_tx.vout[0].scriptPubKey.hex() + value = Decimal(mempool_tx.vout[0].nValue) / COIN + txid = mempool_tx.rehash() + (tx_small, _, _, _) = make_chain(node, self.address, self.privkeys, txid, value, 0, spk, high_fee) + prevtxs = [{ + "txid": txid, + "vout": 0, + "scriptPubKey": spk, + "amount": value, + }] + package_tx = bulk_transaction(tx_small, node, target_weight, self.privkeys, prevtxs) + package_hex.append(package_tx.serialize().hex()) + + assert_equal(3, node.getmempoolinfo()["size"]) + assert_equal(2, len(package_hex)) + testres_too_heavy = node.testmempoolaccept(rawtxs=package_hex) + for txres in testres_too_heavy: + assert_equal(txres["package-error"], "package-mempool-limits") + + # Clear mempool and check that the package passes now + self.generate(node, 1) + assert all([res["allowed"] for res in node.testmempoolaccept(rawtxs=package_hex)]) + +if __name__ == "__main__": + MempoolPackageLimitsTest().main() diff --git a/test/functional/mempool_package_onemore.py b/test/functional/mempool_package_onemore.py index fcd8b061fa..69c21f32bc 100755 --- a/test/functional/mempool_package_onemore.py +++ b/test/functional/mempool_package_onemore.py @@ -30,7 +30,7 @@ class MempoolPackagesTest(BitcoinTestFramework): def run_test(self): # Mine some blocks and have them mature. - self.nodes[0].generate(COINBASE_MATURITY + 1) + self.generate(self.nodes[0], COINBASE_MATURITY + 1) utxo = self.nodes[0].listunspent(10) txid = utxo[0]['txid'] vout = utxo[0]['vout'] @@ -51,7 +51,7 @@ class MempoolPackagesTest(BitcoinTestFramework): (second_chain, second_chain_value) = chain_transaction(self.nodes[0], [utxo[1]['txid']], [utxo[1]['vout']], utxo[1]['amount'], fee, 1) # Check mempool has MAX_ANCESTORS + 1 transactions in it - assert_equal(len(self.nodes[0].getrawmempool(True)), MAX_ANCESTORS + 1) + assert_equal(len(self.nodes[0].getrawmempool()), MAX_ANCESTORS + 1) # Adding one more transaction on to the chain should fail. assert_raises_rpc_error(-26, "too-long-mempool-chain, too many unconfirmed ancestors [limit: 25]", chain_transaction, self.nodes[0], [txid], [0], value, fee, 1) @@ -74,7 +74,7 @@ class MempoolPackagesTest(BitcoinTestFramework): self.nodes[0].sendrawtransaction(signed_second_tx['hex']) # Finally, check that we added two transactions - assert_equal(len(self.nodes[0].getrawmempool(True)), MAX_ANCESTORS + 3) + assert_equal(len(self.nodes[0].getrawmempool()), MAX_ANCESTORS + 3) if __name__ == '__main__': MempoolPackagesTest().main() diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index 5fc3ec23ae..3943bba489 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -14,7 +14,6 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, chain_transaction, - satoshi_round, ) # default limits @@ -46,17 +45,22 @@ class MempoolPackagesTest(BitcoinTestFramework): def run_test(self): # Mine some blocks and have them mature. peer_inv_store = self.nodes[0].add_p2p_connection(P2PTxInvStore()) # keep track of invs - self.nodes[0].generate(COINBASE_MATURITY + 1) + self.generate(self.nodes[0], COINBASE_MATURITY + 1) utxo = self.nodes[0].listunspent(10) txid = utxo[0]['txid'] vout = utxo[0]['vout'] value = utxo[0]['amount'] + assert 'ancestorcount' not in utxo[0] + assert 'ancestorsize' not in utxo[0] + assert 'ancestorfees' not in utxo[0] fee = Decimal("0.0001") # MAX_ANCESTORS transactions off a confirmed tx should be fine chain = [] witness_chain = [] - for _ in range(MAX_ANCESTORS): + ancestor_vsize = 0 + ancestor_fees = Decimal(0) + for i in range(MAX_ANCESTORS): (txid, sent_value) = chain_transaction(self.nodes[0], [txid], [0], value, fee, 1) value = sent_value chain.append(txid) @@ -65,6 +69,15 @@ class MempoolPackagesTest(BitcoinTestFramework): witnesstx = self.nodes[0].decoderawtransaction(fulltx, True) witness_chain.append(witnesstx['hash']) + # Check that listunspent ancestor{count, size, fees} yield the correct results + wallet_unspent = self.nodes[0].listunspent(minconf=0) + this_unspent = next(utxo_info for utxo_info in wallet_unspent if utxo_info['txid'] == txid) + assert_equal(this_unspent['ancestorcount'], i + 1) + ancestor_vsize += self.nodes[0].getrawtransaction(txid=txid, verbose=True)['vsize'] + assert_equal(this_unspent['ancestorsize'], ancestor_vsize) + ancestor_fees -= self.nodes[0].gettransaction(txid=txid)['fee'] + assert_equal(this_unspent['ancestorfees'], ancestor_fees * COIN) + # Wait until mempool transactions have passed initial broadcast (sent inv and received getdata) # Otherwise, getrawmempool may be inconsistent with getmempoolentry if unbroadcast changes in between peer_inv_store.wait_for_broadcast(witness_chain) @@ -77,9 +90,9 @@ class MempoolPackagesTest(BitcoinTestFramework): descendant_fees = 0 descendant_vsize = 0 - ancestor_vsize = sum([mempool[tx]['vsize'] for tx in mempool]) + assert_equal(ancestor_vsize, sum([mempool[tx]['vsize'] for tx in mempool])) ancestor_count = MAX_ANCESTORS - ancestor_fees = sum([mempool[tx]['fee'] for tx in mempool]) + assert_equal(ancestor_fees, sum([mempool[tx]['fee'] for tx in mempool])) descendants = [] ancestors = list(chain) @@ -89,28 +102,28 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(entry, mempool[x]) # Check that the descendant calculations are correct - assert_equal(mempool[x]['descendantcount'], descendant_count) - descendant_fees += mempool[x]['fee'] - assert_equal(mempool[x]['modifiedfee'], mempool[x]['fee']) - assert_equal(mempool[x]['fees']['base'], mempool[x]['fee']) - assert_equal(mempool[x]['fees']['modified'], mempool[x]['modifiedfee']) - assert_equal(mempool[x]['descendantfees'], descendant_fees * COIN) - assert_equal(mempool[x]['fees']['descendant'], descendant_fees) - descendant_vsize += mempool[x]['vsize'] - assert_equal(mempool[x]['descendantsize'], descendant_vsize) + assert_equal(entry['descendantcount'], descendant_count) + descendant_fees += entry['fee'] + assert_equal(entry['modifiedfee'], entry['fee']) + assert_equal(entry['fees']['base'], entry['fee']) + assert_equal(entry['fees']['modified'], entry['modifiedfee']) + assert_equal(entry['descendantfees'], descendant_fees * COIN) + assert_equal(entry['fees']['descendant'], descendant_fees) + descendant_vsize += entry['vsize'] + assert_equal(entry['descendantsize'], descendant_vsize) descendant_count += 1 # Check that ancestor calculations are correct - assert_equal(mempool[x]['ancestorcount'], ancestor_count) - assert_equal(mempool[x]['ancestorfees'], ancestor_fees * COIN) - assert_equal(mempool[x]['ancestorsize'], ancestor_vsize) - ancestor_vsize -= mempool[x]['vsize'] - ancestor_fees -= mempool[x]['fee'] + assert_equal(entry['ancestorcount'], ancestor_count) + assert_equal(entry['ancestorfees'], ancestor_fees * COIN) + assert_equal(entry['ancestorsize'], ancestor_vsize) + ancestor_vsize -= entry['vsize'] + ancestor_fees -= entry['fee'] ancestor_count -= 1 # Check that parent/child list is correct - assert_equal(mempool[x]['spentby'], descendants[-1:]) - assert_equal(mempool[x]['depends'], ancestors[-2:-1]) + assert_equal(entry['spentby'], descendants[-1:]) + assert_equal(entry['depends'], ancestors[-2:-1]) # Check that getmempooldescendants is correct assert_equal(sorted(descendants), sorted(self.nodes[0].getmempooldescendants(x))) @@ -153,12 +166,12 @@ class MempoolPackagesTest(BitcoinTestFramework): # Check that ancestor modified fees includes fee deltas from # prioritisetransaction self.nodes[0].prioritisetransaction(txid=chain[0], fee_delta=1000) - mempool = self.nodes[0].getrawmempool(True) ancestor_fees = 0 for x in chain: - ancestor_fees += mempool[x]['fee'] - assert_equal(mempool[x]['fees']['ancestor'], ancestor_fees + Decimal('0.00001')) - assert_equal(mempool[x]['ancestorfees'], ancestor_fees * COIN + 1000) + entry = self.nodes[0].getmempoolentry(x) + ancestor_fees += entry['fee'] + assert_equal(entry['fees']['ancestor'], ancestor_fees + Decimal('0.00001')) + assert_equal(entry['ancestorfees'], ancestor_fees * COIN + 1000) # Undo the prioritisetransaction for later tests self.nodes[0].prioritisetransaction(txid=chain[0], fee_delta=-1000) @@ -166,20 +179,20 @@ class MempoolPackagesTest(BitcoinTestFramework): # Check that descendant modified fees includes fee deltas from # prioritisetransaction self.nodes[0].prioritisetransaction(txid=chain[-1], fee_delta=1000) - mempool = self.nodes[0].getrawmempool(True) descendant_fees = 0 for x in reversed(chain): - descendant_fees += mempool[x]['fee'] - assert_equal(mempool[x]['fees']['descendant'], descendant_fees + Decimal('0.00001')) - assert_equal(mempool[x]['descendantfees'], descendant_fees * COIN + 1000) + entry = self.nodes[0].getmempoolentry(x) + descendant_fees += entry['fee'] + assert_equal(entry['fees']['descendant'], descendant_fees + Decimal('0.00001')) + assert_equal(entry['descendantfees'], descendant_fees * COIN + 1000) # Adding one more transaction on to the chain should fail. assert_raises_rpc_error(-26, "too-long-mempool-chain", chain_transaction, self.nodes[0], [txid], [vout], value, fee, 1) # Check that prioritising a tx before it's added to the mempool works # First clear the mempool by mining a block. - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks() assert_equal(len(self.nodes[0].getrawmempool()), 0) # Prioritise a transaction that has been mined, then add it back to the @@ -190,16 +203,15 @@ class MempoolPackagesTest(BitcoinTestFramework): self.nodes[1].invalidateblock(self.nodes[1].getbestblockhash()) # Now check that the transaction is in the mempool, with the right modified fee - mempool = self.nodes[0].getrawmempool(True) - descendant_fees = 0 for x in reversed(chain): - descendant_fees += mempool[x]['fee'] + entry = self.nodes[0].getmempoolentry(x) + descendant_fees += entry['fee'] if (x == chain[-1]): - assert_equal(mempool[x]['modifiedfee'], mempool[x]['fee']+satoshi_round(0.00002)) - assert_equal(mempool[x]['fees']['modified'], mempool[x]['fee']+satoshi_round(0.00002)) - assert_equal(mempool[x]['descendantfees'], descendant_fees * COIN + 2000) - assert_equal(mempool[x]['fees']['descendant'], descendant_fees+satoshi_round(0.00002)) + assert_equal(entry['modifiedfee'], entry['fee'] + Decimal("0.00002")) + assert_equal(entry['fees']['modified'], entry['fee'] + Decimal("0.00002")) + assert_equal(entry['descendantfees'], descendant_fees * COIN + 2000) + assert_equal(entry['fees']['descendant'], descendant_fees + Decimal("0.00002")) # Check that node1's mempool is as expected (-> custom ancestor limit) mempool0 = self.nodes[0].getrawmempool(False) @@ -255,7 +267,7 @@ class MempoolPackagesTest(BitcoinTestFramework): # - txs from previous ancestor test (-> custom ancestor limit) # - parent tx for descendant test # - txs chained off parent tx (-> custom descendant limit) - self.wait_until(lambda: len(self.nodes[1].getrawmempool(False)) == + self.wait_until(lambda: len(self.nodes[1].getrawmempool()) == MAX_ANCESTORS_CUSTOM + 1 + MAX_DESCENDANTS_CUSTOM, timeout=10) mempool0 = self.nodes[0].getrawmempool(False) mempool1 = self.nodes[1].getrawmempool(False) @@ -271,7 +283,7 @@ class MempoolPackagesTest(BitcoinTestFramework): # Test reorg handling # First, the basics: - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks() self.nodes[1].invalidateblock(self.nodes[0].getbestblockhash()) self.nodes[1].reconsiderblock(self.nodes[0].getbestblockhash()) @@ -295,7 +307,7 @@ class MempoolPackagesTest(BitcoinTestFramework): value = utxo[0]['amount'] vout = utxo[0]['vout'] - send_value = satoshi_round((value - fee)/2) + send_value = (value - fee) / 2 inputs = [ {'txid' : txid, 'vout' : vout} ] outputs = {} for _ in range(2): @@ -318,7 +330,7 @@ class MempoolPackagesTest(BitcoinTestFramework): value = sent_value # Mine these in a block - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() # Now generate tx8, with a big fee diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index 752b925b92..71a132dca3 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -46,6 +46,7 @@ from test_framework.util import ( assert_greater_than_or_equal, assert_raises_rpc_error, ) +from test_framework.wallet import MiniWallet class MempoolPersistTest(BitcoinTestFramework): @@ -53,15 +54,26 @@ class MempoolPersistTest(BitcoinTestFramework): self.num_nodes = 3 self.extra_args = [[], ["-persistmempool=0"], []] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def run_test(self): + self.mini_wallet = MiniWallet(self.nodes[2]) + self.mini_wallet.rescan_utxos() + if self.is_sqlite_compiled(): + self.nodes[2].createwallet( + wallet_name="watch", + descriptors=True, + disable_private_keys=True, + load_on_startup=False, + ) + wallet_watch = self.nodes[2].get_wallet_rpc("watch") + assert_equal([{'success': True}], wallet_watch.importdescriptors([{'desc': self.mini_wallet.get_descriptor(), 'timestamp': 0}])) + self.log.debug("Send 5 transactions from node2 (to its own address)") tx_creation_time_lower = int(time.time()) for _ in range(5): - last_txid = self.nodes[2].sendtoaddress(self.nodes[2].getnewaddress(), Decimal("10")) - node2_balance = self.nodes[2].getbalance() + last_txid = self.mini_wallet.send_self_transfer(from_node=self.nodes[2])["txid"] + if self.is_sqlite_compiled(): + self.nodes[2].syncwithvalidationinterfacequeue() # Flush mempool to wallet + node2_balance = wallet_watch.getbalance() self.sync_all() tx_creation_time_higher = int(time.time()) @@ -82,16 +94,16 @@ class MempoolPersistTest(BitcoinTestFramework): assert_equal(total_fee_old, self.nodes[0].getmempoolinfo()['total_fee']) assert_equal(total_fee_old, sum(v['fees']['base'] for k, v in self.nodes[0].getrawmempool(verbose=True).items())) - tx_creation_time = self.nodes[0].getmempoolentry(txid=last_txid)['time'] + last_entry = self.nodes[0].getmempoolentry(txid=last_txid) + tx_creation_time = last_entry['time'] assert_greater_than_or_equal(tx_creation_time, tx_creation_time_lower) assert_greater_than_or_equal(tx_creation_time_higher, tx_creation_time) # disconnect nodes & make a txn that remains in the unbroadcast set. self.disconnect_nodes(0, 1) - assert(len(self.nodes[0].getpeerinfo()) == 0) - assert(len(self.nodes[0].p2ps) == 0) - self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), Decimal("12")) - self.connect_nodes(0, 2) + assert_equal(len(self.nodes[0].getpeerinfo()), 0) + assert_equal(len(self.nodes[0].p2ps), 0) + self.mini_wallet.send_self_transfer(from_node=self.nodes[0]) self.log.debug("Stop-start the nodes. Verify that node0 has the transactions in its mempool and node1 does not. Verify that node2 calculates its balance correctly after loading wallet transactions.") self.stop_nodes() @@ -111,17 +123,19 @@ class MempoolPersistTest(BitcoinTestFramework): fees = self.nodes[0].getmempoolentry(txid=last_txid)['fees'] assert_equal(fees['base'] + Decimal('0.00001000'), fees['modified']) - self.log.debug('Verify time is loaded correctly') - assert_equal(tx_creation_time, self.nodes[0].getmempoolentry(txid=last_txid)['time']) + self.log.debug('Verify all fields are loaded correctly') + assert_equal(last_entry, self.nodes[0].getmempoolentry(txid=last_txid)) # Verify accounting of mempool transactions after restart is correct - self.nodes[2].syncwithvalidationinterfacequeue() # Flush mempool to wallet - assert_equal(node2_balance, self.nodes[2].getbalance()) + if self.is_sqlite_compiled(): + self.nodes[2].loadwallet("watch") + wallet_watch = self.nodes[2].get_wallet_rpc("watch") + self.nodes[2].syncwithvalidationinterfacequeue() # Flush mempool to wallet + assert_equal(node2_balance, wallet_watch.getbalance()) - # start node0 with wallet disabled so wallet transactions don't get resubmitted self.log.debug("Stop-start node0 with -persistmempool=0. Verify that it doesn't load its mempool.dat file.") self.stop_nodes() - self.start_node(0, extra_args=["-persistmempool=0", "-disablewallet"]) + self.start_node(0, extra_args=["-persistmempool=0"]) assert self.nodes[0].getmempoolinfo()["loaded"] assert_equal(len(self.nodes[0].getrawmempool()), 0) @@ -141,7 +155,7 @@ class MempoolPersistTest(BitcoinTestFramework): self.log.debug("Stop nodes, make node1 use mempool.dat from node0. Verify it has 6 transactions") os.rename(mempooldat0, mempooldat1) self.stop_nodes() - self.start_node(1, extra_args=[]) + self.start_node(1, extra_args=["-persistmempool"]) assert self.nodes[1].getmempoolinfo()["loaded"] assert_equal(len(self.nodes[1].getrawmempool()), 6) @@ -160,22 +174,22 @@ class MempoolPersistTest(BitcoinTestFramework): self.start_node(0) # clear out mempool - node0.generate(1) + self.generate(node0, 1) # ensure node0 doesn't have any connections # make a transaction that will remain in the unbroadcast set - assert(len(node0.getpeerinfo()) == 0) - assert(len(node0.p2ps) == 0) - node0.sendtoaddress(self.nodes[1].getnewaddress(), Decimal("12")) + assert_equal(len(node0.getpeerinfo()), 0) + assert_equal(len(node0.p2ps), 0) + self.mini_wallet.send_self_transfer(from_node=node0) # shutdown, then startup with wallet disabled - self.stop_nodes() - self.start_node(0, extra_args=["-disablewallet"]) + self.restart_node(0, extra_args=["-disablewallet"]) # check that txn gets broadcast due to unbroadcast logic conn = node0.add_p2p_connection(P2PTxInvStore()) - node0.mockscheduler(16*60) # 15 min + 1 for buffer + node0.mockscheduler(16 * 60) # 15 min + 1 for buffer self.wait_until(lambda: len(conn.get_invs()) == 1) -if __name__ == '__main__': + +if __name__ == "__main__": MempoolPersistTest().main() diff --git a/test/functional/mempool_reorg.py b/test/functional/mempool_reorg.py index bcc6aa7bcc..260b41ef12 100755 --- a/test/functional/mempool_reorg.py +++ b/test/functional/mempool_reorg.py @@ -31,7 +31,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): self.log.info("Add 4 coinbase utxos to the miniwallet") # Block 76 contains the first spendable coinbase txs. first_block = 76 - wallet.scan_blocks(start=first_block, num=4) + wallet.rescan_utxos() # Three scenarios for re-orging coinbase spends in the memory pool: # 1. Direct coinbase spend : spend_1 @@ -65,7 +65,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=spend_2['hex']) wallet.sendrawtransaction(from_node=self.nodes[0], tx_hex=spend_3['hex']) self.log.info("Generate a block") - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.log.info("Check that time-locked transaction is still too immature to spend") assert_raises_rpc_error(-26, 'non-final', self.nodes[0].sendrawtransaction, timelock_tx) @@ -78,9 +78,9 @@ class MempoolCoinbaseTest(BitcoinTestFramework): self.log.info("Broadcast and mine spend_3_1") spend_3_1_id = self.nodes[0].sendrawtransaction(spend_3_1['hex']) self.log.info("Generate a block") - last_block = self.nodes[0].generate(1) + last_block = self.generate(self.nodes[0], 1) # Sync blocks, so that peer 1 gets the block before timelock_tx - # Otherwise, peer 1 would put the timelock_tx in recentRejects + # Otherwise, peer 1 would put the timelock_tx in m_recent_rejects self.sync_all() self.log.info("The time-locked transaction can now be spent") diff --git a/test/functional/mempool_resurrect.py b/test/functional/mempool_resurrect.py index 1b5ca7e15a..4fce07dad3 100755 --- a/test/functional/mempool_resurrect.py +++ b/test/functional/mempool_resurrect.py @@ -20,8 +20,8 @@ class MempoolCoinbaseTest(BitcoinTestFramework): wallet = MiniWallet(node) # Add enough mature utxos to the wallet so that all txs spend confirmed coins - wallet.generate(3) - node.generate(COINBASE_MATURITY) + self.generate(wallet, 3) + self.generate(node, COINBASE_MATURITY) # Spend block 1/2/3's coinbase transactions # Mine a block @@ -34,9 +34,9 @@ class MempoolCoinbaseTest(BitcoinTestFramework): # ... make sure all the transactions are confirmed again blocks = [] spends1_ids = [wallet.send_self_transfer(from_node=node)['txid'] for _ in range(3)] - blocks.extend(node.generate(1)) + blocks.extend(self.generate(node, 1)) spends2_ids = [wallet.send_self_transfer(from_node=node)['txid'] for _ in range(3)] - blocks.extend(node.generate(1)) + blocks.extend(self.generate(node, 1)) spends_ids = set(spends1_ids + spends2_ids) @@ -53,7 +53,7 @@ class MempoolCoinbaseTest(BitcoinTestFramework): assert_equal(set(node.getrawmempool()), spends_ids) # Generate another block, they should all get mined - blocks = node.generate(1) + blocks = self.generate(node, 1) # mempool should be empty, all txns confirmed assert_equal(set(node.getrawmempool()), set()) confirmed_txns = set(node.getblock(blocks[0])['tx']) diff --git a/test/functional/mempool_spend_coinbase.py b/test/functional/mempool_spend_coinbase.py index b900aa0b9c..4e1dd80ba7 100755 --- a/test/functional/mempool_spend_coinbase.py +++ b/test/functional/mempool_spend_coinbase.py @@ -28,14 +28,14 @@ class MempoolSpendCoinbaseTest(BitcoinTestFramework): chain_height = 198 self.nodes[0].invalidateblock(self.nodes[0].getblockhash(chain_height + 1)) assert_equal(chain_height, self.nodes[0].getblockcount()) + wallet.rescan_utxos() # Coinbase at height chain_height-100+1 ok in mempool, should # get mined. Coinbase at height chain_height-100+2 is # too immature to spend. - wallet.scan_blocks(start=chain_height - 100 + 1, num=1) - utxo_mature = wallet.get_utxo() - wallet.scan_blocks(start=chain_height - 100 + 2, num=1) - utxo_immature = wallet.get_utxo() + coinbase_txid = lambda h: self.nodes[0].getblock(self.nodes[0].getblockhash(h))['tx'][0] + utxo_mature = wallet.get_utxo(txid=coinbase_txid(chain_height - 100 + 1)) + utxo_immature = wallet.get_utxo(txid=coinbase_txid(chain_height - 100 + 2)) spend_mature_id = wallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_mature)["txid"] @@ -49,7 +49,7 @@ class MempoolSpendCoinbaseTest(BitcoinTestFramework): assert_equal(self.nodes[0].getrawmempool(), [spend_mature_id]) # mine a block, mature one should get confirmed - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) assert_equal(set(self.nodes[0].getrawmempool()), set()) # ... and now previously immature can be spent: diff --git a/test/functional/mempool_unbroadcast.py b/test/functional/mempool_unbroadcast.py index b475b65e68..4d6379fe86 100755 --- a/test/functional/mempool_unbroadcast.py +++ b/test/functional/mempool_unbroadcast.py @@ -32,7 +32,7 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): node = self.nodes[0] min_relay_fee = node.getnetworkinfo()["relayfee"] - utxos = create_confirmed_utxos(min_relay_fee, node, 10) + utxos = create_confirmed_utxos(self, min_relay_fee, node, 10) self.disconnect_nodes(0, 1) @@ -92,6 +92,10 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): self.disconnect_nodes(0, 1) node.disconnect_p2ps() + self.log.info("Rebroadcast transaction and ensure it is not added to unbroadcast set when already in mempool") + rpc_tx_hsh = node.sendrawtransaction(txFS["hex"]) + assert not node.getmempoolentry(rpc_tx_hsh)['unbroadcast'] + def test_txn_removal(self): self.log.info("Test that transactions removed from mempool are removed from unbroadcast set") node = self.nodes[0] @@ -105,7 +109,7 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): # a block removal_reason = "Removed {} from set of unbroadcast txns before confirmation that txn was sent out".format(txhsh) with node.assert_debug_log([removal_reason]): - node.generate(1) + self.generate(node, 1) if __name__ == "__main__": MempoolUnbroadcastTest().main() diff --git a/test/functional/mempool_updatefromblock.py b/test/functional/mempool_updatefromblock.py index 8baf974a0a..22f136d1a5 100755 --- a/test/functional/mempool_updatefromblock.py +++ b/test/functional/mempool_updatefromblock.py @@ -86,12 +86,12 @@ class MempoolUpdateFromBlockTest(BitcoinTestFramework): unsigned_raw_tx = self.nodes[0].createrawtransaction(inputs, outputs) signed_raw_tx = self.nodes[0].signrawtransactionwithwallet(unsigned_raw_tx) tx_id.append(self.nodes[0].sendrawtransaction(signed_raw_tx['hex'])) - tx_size.append(self.nodes[0].getrawmempool(True)[tx_id[-1]]['vsize']) + tx_size.append(self.nodes[0].getmempoolentry(tx_id[-1])['vsize']) if tx_count in n_tx_to_mine: # The created transactions are mined into blocks by batches. self.log.info('The batch of {} transactions has been accepted into the mempool.'.format(len(self.nodes[0].getrawmempool()))) - block_hash = self.nodes[0].generate(1)[0] + block_hash = self.generate(self.nodes[0], 1)[0] if not first_block_hash: first_block_hash = block_hash assert_equal(len(self.nodes[0].getrawmempool()), 0) @@ -109,10 +109,11 @@ class MempoolUpdateFromBlockTest(BitcoinTestFramework): self.log.info('Checking descendants/ancestors properties of all of the in-mempool transactions...') for k, tx in enumerate(tx_id): self.log.debug('Check transaction #{}.'.format(k)) - assert_equal(self.nodes[0].getrawmempool(True)[tx]['descendantcount'], size - k) - assert_equal(self.nodes[0].getrawmempool(True)[tx]['descendantsize'], sum(tx_size[k:size])) - assert_equal(self.nodes[0].getrawmempool(True)[tx]['ancestorcount'], k + 1) - assert_equal(self.nodes[0].getrawmempool(True)[tx]['ancestorsize'], sum(tx_size[0:(k + 1)])) + entry = self.nodes[0].getmempoolentry(tx) + assert_equal(entry['descendantcount'], size - k) + assert_equal(entry['descendantsize'], sum(tx_size[k:size])) + assert_equal(entry['ancestorcount'], k + 1) + assert_equal(entry['ancestorsize'], sum(tx_size[0:(k + 1)])) def run_test(self): # Use batch size limited by DEFAULT_ANCESTOR_LIMIT = 25 to not fire "too many unconfirmed parents" error. diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index ba467c1517..f141d201eb 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -13,6 +13,7 @@ from decimal import Decimal from test_framework.blocktools import ( create_coinbase, + get_witness_script, NORMAL_GBT_REQUEST_PARAMS, TIME_GENESIS_BLOCK, ) @@ -20,6 +21,7 @@ from test_framework.messages import ( CBlock, CBlockHeader, BLOCK_HEADER_SIZE, + ser_uint256, ) from test_framework.p2p import P2PDataStore from test_framework.test_framework import BitcoinTestFramework @@ -49,21 +51,24 @@ class MiningTest(BitcoinTestFramework): self.setup_clean_chain = True self.supports_cli = False + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + def mine_chain(self): self.log.info('Create some old blocks') for t in range(TIME_GENESIS_BLOCK, TIME_GENESIS_BLOCK + 200 * 600, 600): self.nodes[0].setmocktime(t) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) mining_info = self.nodes[0].getmininginfo() assert_equal(mining_info['blocks'], 200) assert_equal(mining_info['currentblocktx'], 0) assert_equal(mining_info['currentblockweight'], 4000) self.log.info('test blockversion') - self.restart_node(0, extra_args=['-mocktime={}'.format(t), '-blockversion=1337']) + self.restart_node(0, extra_args=[f'-mocktime={t}', '-blockversion=1337']) self.connect_nodes(0, 1) assert_equal(1337, self.nodes[0].getblocktemplate(NORMAL_GBT_REQUEST_PARAMS)['version']) - self.restart_node(0, extra_args=['-mocktime={}'.format(t)]) + self.restart_node(0, extra_args=[f'-mocktime={t}']) self.connect_nodes(0, 1) assert_equal(VERSIONBITS_TOP_BITS + (1 << VERSIONBITS_DEPLOYMENT_TESTDUMMY_BIT), self.nodes[0].getblocktemplate(NORMAL_GBT_REQUEST_PARAMS)['version']) self.restart_node(0) @@ -89,8 +94,22 @@ class MiningTest(BitcoinTestFramework): assert_equal(mining_info['networkhashps'], Decimal('0.003333333333333334')) assert_equal(mining_info['pooledtx'], 0) - # Mine a block to leave initial block download - node.generatetoaddress(1, node.get_deterministic_priv_key().address) + self.log.info("getblocktemplate: Test default witness commitment") + txid = int(node.sendtoaddress(node.getnewaddress(), 1), 16) + tmpl = node.getblocktemplate(NORMAL_GBT_REQUEST_PARAMS) + + # Check that default_witness_commitment is present. + assert 'default_witness_commitment' in tmpl + witness_commitment = tmpl['default_witness_commitment'] + + # Check that default_witness_commitment is correct. + witness_root = CBlock.get_merkle_root([ser_uint256(0), + ser_uint256(txid)]) + script = get_witness_script(witness_root, 0) + assert_equal(witness_commitment, script.hex()) + + # Mine a block to leave initial block download and clear the mempool + self.generatetoaddress(node, 1, node.get_deterministic_priv_key().address) tmpl = node.getblocktemplate(NORMAL_GBT_REQUEST_PARAMS) self.log.info("getblocktemplate: Test capability advertised") assert 'proposal' in tmpl['capabilities'] @@ -252,7 +271,7 @@ class MiningTest(BitcoinTestFramework): assert chain_tip(block.hash, status='active', branchlen=0) in node.getchaintips() # Building a few blocks should give the same results - node.generatetoaddress(10, node.get_deterministic_priv_key().address) + self.generatetoaddress(node, 10, node.get_deterministic_priv_key().address) assert_raises_rpc_error(-25, 'time-too-old', lambda: node.submitheader(hexdata=CBlockHeader(bad_block_time).serialize().hex())) assert_raises_rpc_error(-25, 'bad-prevblk', lambda: node.submitheader(hexdata=CBlockHeader(bad_block2).serialize().hex())) node.submitheader(hexdata=CBlockHeader(block).serialize().hex()) diff --git a/test/functional/mining_getblocktemplate_longpoll.py b/test/functional/mining_getblocktemplate_longpoll.py index 715b68e04c..0879fb9f2d 100755 --- a/test/functional/mining_getblocktemplate_longpoll.py +++ b/test/functional/mining_getblocktemplate_longpoll.py @@ -35,7 +35,7 @@ class GetBlockTemplateLPTest(BitcoinTestFramework): def run_test(self): self.log.info("Warning: this test will take about 70 seconds in the best case. Be patient.") self.log.info("Test that longpollid doesn't change between successive getblocktemplate() invocations if nothing else happens") - self.nodes[0].generate(10) + self.generate(self.nodes[0], 10) template = self.nodes[0].getblocktemplate({'rules': ['segwit']}) longpollid = template['longpollid'] template2 = self.nodes[0].getblocktemplate({'rules': ['segwit']}) @@ -48,9 +48,9 @@ class GetBlockTemplateLPTest(BitcoinTestFramework): thr.join(5) # wait 5 seconds or until thread exits assert thr.is_alive() - miniwallets = [ MiniWallet(node) for node in self.nodes ] + miniwallets = [MiniWallet(node) for node in self.nodes] self.log.info("Test that longpoll will terminate if another node generates a block") - miniwallets[1].generate(1) # generate a block on another node + self.generate(miniwallets[1], 1) # generate a block on another node # check that thread will exit now that new transaction entered mempool thr.join(5) # wait 5 seconds or until thread exits assert not thr.is_alive() @@ -58,12 +58,12 @@ class GetBlockTemplateLPTest(BitcoinTestFramework): self.log.info("Test that longpoll will terminate if we generate a block ourselves") thr = LongpollThread(self.nodes[0]) thr.start() - miniwallets[0].generate(1) # generate a block on own node + self.generate(miniwallets[0], 1) # generate a block on own node thr.join(5) # wait 5 seconds or until thread exits assert not thr.is_alive() # Add enough mature utxos to the wallets, so that all txs spend confirmed coins - self.nodes[0].generate(COINBASE_MATURITY) + self.generate(self.nodes[0], COINBASE_MATURITY) self.sync_blocks() self.log.info("Test that introducing a new transaction into the mempool will terminate the longpoll") diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index 1426fdaacb..da85ee54be 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -6,7 +6,7 @@ import time -from test_framework.messages import COIN, MAX_BLOCK_BASE_SIZE +from test_framework.messages import COIN, MAX_BLOCK_WEIGHT from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_rpc_error, create_confirmed_utxos, create_lots_of_big_transactions, gen_return_txouts @@ -48,7 +48,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): self.relayfee = self.nodes[0].getnetworkinfo()['relayfee'] utxo_count = 90 - utxos = create_confirmed_utxos(self.relayfee, self.nodes[0], utxo_count) + utxos = create_confirmed_utxos(self, self.relayfee, self.nodes[0], utxo_count) base_fee = self.relayfee*100 # our transactions are smaller than 100kb txids = [] @@ -61,21 +61,21 @@ class PrioritiseTransactionTest(BitcoinTestFramework): txids[i] = create_lots_of_big_transactions(self.nodes[0], self.txouts, utxos[start_range:end_range], end_range - start_range, (i+1)*base_fee) # Make sure that the size of each group of transactions exceeds - # MAX_BLOCK_BASE_SIZE -- otherwise the test needs to be revised to create - # more transactions. + # MAX_BLOCK_WEIGHT // 4 -- otherwise the test needs to be revised to + # create more transactions. mempool = self.nodes[0].getrawmempool(True) sizes = [0, 0, 0] for i in range(3): for j in txids[i]: assert j in mempool sizes[i] += mempool[j]['vsize'] - assert sizes[i] > MAX_BLOCK_BASE_SIZE # Fail => raise utxo_count + assert sizes[i] > MAX_BLOCK_WEIGHT // 4 # Fail => raise utxo_count # add a fee delta to something in the cheapest bucket and make sure it gets mined # also check that a different entry in the cheapest bucket is NOT mined self.nodes[0].prioritisetransaction(txid=txids[0][0], fee_delta=int(3*base_fee*COIN)) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) mempool = self.nodes[0].getrawmempool() self.log.info("Assert that prioritised transaction was mined") @@ -105,7 +105,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # the other high fee transactions. Keep mining until our mempool has # decreased by all the high fee size that we calculated above. while (self.nodes[0].getmempoolinfo()['bytes'] > sizes[0] + sizes[1]): - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) # High fee transaction should not have been mined, but other high fee rate # transactions should have been. diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index 1a414959b9..15b90fa61f 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -6,28 +6,34 @@ Test addr relay """ +import random +import time + from test_framework.messages import ( CAddress, - NODE_NETWORK, - NODE_WITNESS, msg_addr, - msg_getaddr + msg_getaddr, + msg_verack, ) -from test_framework.p2p import P2PInterface -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_equal, +from test_framework.p2p import ( + P2PInterface, + p2p_lock, + P2P_SERVICES, ) -import time +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal, assert_greater_than class AddrReceiver(P2PInterface): num_ipv4_received = 0 test_addr_contents = False + _tokens = 1 + send_getaddr = True - def __init__(self, test_addr_contents=False): + def __init__(self, test_addr_contents=False, send_getaddr=True): super().__init__() self.test_addr_contents = test_addr_contents + self.send_getaddr = send_getaddr def on_addr(self, message): for addr in message.addrs: @@ -40,9 +46,28 @@ class AddrReceiver(P2PInterface): raise AssertionError("Invalid addr.port of {} (8333-8342 expected)".format(addr.port)) assert addr.ip.startswith('123.123.123.') + def on_getaddr(self, message): + # When the node sends us a getaddr, it increments the addr relay tokens for the connection by 1000 + self._tokens += 1000 + + @property + def tokens(self): + with p2p_lock: + return self._tokens + + def increment_tokens(self, n): + # When we move mocktime forward, the node increments the addr relay tokens for its peers + with p2p_lock: + self._tokens += n + def addr_received(self): return self.num_ipv4_received != 0 + def on_version(self, message): + self.send_message(msg_verack()) + if (self.send_getaddr): + self.send_message(msg_getaddr()) + def getaddr_received(self): return self.message_count['getaddr'] > 0 @@ -53,19 +78,25 @@ class AddrTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 + self.extra_args = [["-whitelist=addr@127.0.0.1"]] def run_test(self): self.oversized_addr_test() self.relay_tests() + self.inbound_blackhole_tests() + + # This test populates the addrman, which can impact the node's behavior + # in subsequent tests self.getaddr_tests() self.blocksonly_mode_tests() + self.rate_limit_tests() def setup_addr_msg(self, num): addrs = [] for i in range(num): addr = CAddress() addr.time = self.mocktime + i - addr.nServices = NODE_NETWORK | NODE_WITNESS + addr.nServices = P2P_SERVICES addr.ip = f"123.123.123.{self.counter % 256}" addr.port = 8333 + i addrs.append(addr) @@ -75,6 +106,19 @@ class AddrTest(BitcoinTestFramework): msg.addrs = addrs return msg + def setup_rand_addr_msg(self, num): + addrs = [] + for i in range(num): + addr = CAddress() + addr.time = self.mocktime + i + addr.nServices = P2P_SERVICES + addr.ip = f"{random.randrange(128,169)}.{random.randrange(1,255)}.{random.randrange(1,255)}.{random.randrange(1,255)}" + addr.port = 8333 + addrs.append(addr) + msg = msg_addr() + msg.addrs = addrs + return msg + def send_addr_msg(self, source, msg, receivers): source.send_and_ping(msg) # pop m_next_addr_send timer @@ -124,7 +168,7 @@ class AddrTest(BitcoinTestFramework): self.nodes[0].disconnect_p2ps() self.log.info('Check relay of addresses received from outbound peers') - inbound_peer = self.nodes[0].add_p2p_connection(AddrReceiver(test_addr_contents=True)) + inbound_peer = self.nodes[0].add_p2p_connection(AddrReceiver(test_addr_contents=True, send_getaddr=False)) full_outbound_peer = self.nodes[0].add_outbound_p2p_connection(AddrReceiver(), p2p_idx=0, connection_type="outbound-full-relay") msg = self.setup_addr_msg(2) self.send_addr_msg(full_outbound_peer, msg, [inbound_peer]) @@ -135,6 +179,9 @@ class AddrTest(BitcoinTestFramework): # of the outbound peer which is often sent before the GETADDR response. assert_equal(inbound_peer.num_ipv4_received, 0) + # Send an empty ADDR message to initialize address relay on this connection. + inbound_peer.send_and_ping(msg_addr()) + self.log.info('Check that subsequent addr messages sent from an outbound peer are relayed') msg2 = self.setup_addr_msg(2) self.send_addr_msg(full_outbound_peer, msg2, [inbound_peer]) @@ -152,7 +199,64 @@ class AddrTest(BitcoinTestFramework): self.nodes[0].disconnect_p2ps() + def sum_addr_messages(self, msgs_dict): + return sum(bytes_received for (msg, bytes_received) in msgs_dict.items() if msg in ['addr', 'addrv2', 'getaddr']) + + def inbound_blackhole_tests(self): + self.log.info('Check that we only relay addresses to inbound peers who have previously sent us addr related messages') + + addr_source = self.nodes[0].add_p2p_connection(P2PInterface()) + receiver_peer = self.nodes[0].add_p2p_connection(AddrReceiver()) + blackhole_peer = self.nodes[0].add_p2p_connection(AddrReceiver(send_getaddr=False)) + initial_addrs_received = receiver_peer.num_ipv4_received + + peerinfo = self.nodes[0].getpeerinfo() + assert_equal(peerinfo[0]['addr_relay_enabled'], True) # addr_source + assert_equal(peerinfo[1]['addr_relay_enabled'], True) # receiver_peer + assert_equal(peerinfo[2]['addr_relay_enabled'], False) # blackhole_peer + + # addr_source sends 2 addresses to node0 + msg = self.setup_addr_msg(2) + addr_source.send_and_ping(msg) + self.mocktime += 30 * 60 + self.nodes[0].setmocktime(self.mocktime) + receiver_peer.sync_with_ping() + blackhole_peer.sync_with_ping() + + peerinfo = self.nodes[0].getpeerinfo() + + # Confirm node received addr-related messages from receiver peer + assert_greater_than(self.sum_addr_messages(peerinfo[1]['bytesrecv_per_msg']), 0) + # And that peer received addresses + assert_equal(receiver_peer.num_ipv4_received - initial_addrs_received, 2) + + # Confirm node has not received addr-related messages from blackhole peer + assert_equal(self.sum_addr_messages(peerinfo[2]['bytesrecv_per_msg']), 0) + # And that peer did not receive addresses + assert_equal(blackhole_peer.num_ipv4_received, 0) + + self.log.info("After blackhole peer sends addr message, it becomes eligible for addr gossip") + blackhole_peer.send_and_ping(msg_addr()) + + # Confirm node has now received addr-related messages from blackhole peer + assert_greater_than(self.sum_addr_messages(peerinfo[1]['bytesrecv_per_msg']), 0) + assert_equal(self.nodes[0].getpeerinfo()[2]['addr_relay_enabled'], True) + + msg = self.setup_addr_msg(2) + self.send_addr_msg(addr_source, msg, [receiver_peer, blackhole_peer]) + + # And that peer received addresses + assert_equal(blackhole_peer.num_ipv4_received, 2) + + self.nodes[0].disconnect_p2ps() + def getaddr_tests(self): + # In the previous tests, the node answered GETADDR requests with an + # empty addrman. Due to GETADDR response caching (see + # CConnman::GetAddresses), the node would continue to provide 0 addrs + # in response until enough time has passed or the node is restarted. + self.restart_node(0) + self.log.info('Test getaddr behavior') self.log.info('Check that we send a getaddr message upon connecting to an outbound-full-relay peer') full_outbound_peer = self.nodes[0].add_outbound_p2p_connection(AddrReceiver(), p2p_idx=0, connection_type="outbound-full-relay") @@ -165,7 +269,7 @@ class AddrTest(BitcoinTestFramework): assert_equal(block_relay_peer.getaddr_received(), False) self.log.info('Check that we answer getaddr messages only from inbound peers') - inbound_peer = self.nodes[0].add_p2p_connection(AddrReceiver()) + inbound_peer = self.nodes[0].add_p2p_connection(AddrReceiver(send_getaddr=False)) inbound_peer.sync_with_ping() # Add some addresses to addrman @@ -191,7 +295,7 @@ class AddrTest(BitcoinTestFramework): def blocksonly_mode_tests(self): self.log.info('Test addr relay in -blocksonly mode') - self.restart_node(0, ["-blocksonly"]) + self.restart_node(0, ["-blocksonly", "-whitelist=addr@127.0.0.1"]) self.mocktime = int(time.time()) self.log.info('Check that we send getaddr messages') @@ -207,6 +311,63 @@ class AddrTest(BitcoinTestFramework): self.nodes[0].disconnect_p2ps() + def send_addrs_and_test_rate_limiting(self, peer, no_relay, *, new_addrs, total_addrs): + """Send an addr message and check that the number of addresses processed and rate-limited is as expected""" + + peer.send_and_ping(self.setup_rand_addr_msg(new_addrs)) + + peerinfo = self.nodes[0].getpeerinfo()[0] + addrs_processed = peerinfo['addr_processed'] + addrs_rate_limited = peerinfo['addr_rate_limited'] + self.log.debug(f"addrs_processed = {addrs_processed}, addrs_rate_limited = {addrs_rate_limited}") + + if no_relay: + assert_equal(addrs_processed, 0) + assert_equal(addrs_rate_limited, 0) + else: + assert_equal(addrs_processed, min(total_addrs, peer.tokens)) + assert_equal(addrs_rate_limited, max(0, total_addrs - peer.tokens)) + + def rate_limit_tests(self): + self.mocktime = int(time.time()) + self.restart_node(0, []) + self.nodes[0].setmocktime(self.mocktime) + + for conn_type, no_relay in [("outbound-full-relay", False), ("block-relay-only", True), ("inbound", False)]: + self.log.info(f'Test rate limiting of addr processing for {conn_type} peers') + if conn_type == "inbound": + peer = self.nodes[0].add_p2p_connection(AddrReceiver()) + else: + peer = self.nodes[0].add_outbound_p2p_connection(AddrReceiver(), p2p_idx=0, connection_type=conn_type) + + # Send 600 addresses. For all but the block-relay-only peer this should result in addresses being processed. + self.send_addrs_and_test_rate_limiting(peer, no_relay, new_addrs=600, total_addrs=600) + + # Send 600 more addresses. For the outbound-full-relay peer (which we send a GETADDR, and thus will + # process up to 1001 incoming addresses), this means more addresses will be processed. + self.send_addrs_and_test_rate_limiting(peer, no_relay, new_addrs=600, total_addrs=1200) + + # Send 10 more. As we reached the processing limit for all nodes, no more addresses should be procesesd. + self.send_addrs_and_test_rate_limiting(peer, no_relay, new_addrs=10, total_addrs=1210) + + # Advance the time by 100 seconds, permitting the processing of 10 more addresses. + # Send 200 and verify that 10 are processed. + self.mocktime += 100 + self.nodes[0].setmocktime(self.mocktime) + peer.increment_tokens(10) + + self.send_addrs_and_test_rate_limiting(peer, no_relay, new_addrs=200, total_addrs=1410) + + # Advance the time by 1000 seconds, permitting the processing of 100 more addresses. + # Send 200 and verify that 100 are processed. + self.mocktime += 1000 + self.nodes[0].setmocktime(self.mocktime) + peer.increment_tokens(100) + + self.send_addrs_and_test_rate_limiting(peer, no_relay, new_addrs=200, total_addrs=1610) + + self.nodes[0].disconnect_p2ps() + if __name__ == '__main__': AddrTest().main() diff --git a/test/functional/p2p_addrfetch.py b/test/functional/p2p_addrfetch.py new file mode 100755 index 0000000000..25efd50040 --- /dev/null +++ b/test/functional/p2p_addrfetch.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test p2p addr-fetch connections +""" + +import time + +from test_framework.messages import ( + CAddress, + msg_addr, +) +from test_framework.p2p import ( + P2PInterface, + p2p_lock, + P2P_SERVICES, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + +ADDR = CAddress() +ADDR.time = int(time.time()) +ADDR.nServices = P2P_SERVICES +ADDR.ip = "192.0.0.8" +ADDR.port = 18444 + + +class P2PAddrFetch(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def assert_getpeerinfo(self, *, peer_ids): + num_peers = len(peer_ids) + info = self.nodes[0].getpeerinfo() + assert_equal(len(info), num_peers) + for n in range(0, num_peers): + assert_equal(info[n]['id'], peer_ids[n]) + assert_equal(info[n]['connection_type'], 'addr-fetch') + + def run_test(self): + node = self.nodes[0] + self.log.info("Connect to an addr-fetch peer") + peer_id = 0 + peer = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=peer_id, connection_type="addr-fetch") + self.assert_getpeerinfo(peer_ids=[peer_id]) + + self.log.info("Check that we send getaddr but don't try to sync headers with the addr-fetch peer") + peer.sync_send_with_ping() + with p2p_lock: + assert peer.message_count['getaddr'] == 1 + assert peer.message_count['getheaders'] == 0 + + self.log.info("Check that answering the getaddr with a single address does not lead to disconnect") + # This prevents disconnecting on self-announcements + msg = msg_addr() + msg.addrs = [ADDR] + peer.send_and_ping(msg) + self.assert_getpeerinfo(peer_ids=[peer_id]) + + self.log.info("Check that answering with larger addr messages leads to disconnect") + msg.addrs = [ADDR] * 2 + peer.send_message(msg) + peer.wait_for_disconnect(timeout=5) + + self.log.info("Check timeout for addr-fetch peer that does not send addrs") + peer_id = 1 + peer = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=peer_id, connection_type="addr-fetch") + + time_now = int(time.time()) + self.assert_getpeerinfo(peer_ids=[peer_id]) + + # Expect addr-fetch peer connection to be maintained up to 5 minutes. + node.setmocktime(time_now + 295) + self.assert_getpeerinfo(peer_ids=[peer_id]) + + # Expect addr-fetch peer connection to be disconnected after 5 minutes. + node.setmocktime(time_now + 301) + peer.wait_for_disconnect(timeout=5) + self.assert_getpeerinfo(peer_ids=[]) + + +if __name__ == '__main__': + P2PAddrFetch().main() diff --git a/test/functional/p2p_addrv2_relay.py b/test/functional/p2p_addrv2_relay.py index 23ce3e5d04..3833c58680 100755 --- a/test/functional/p2p_addrv2_relay.py +++ b/test/functional/p2p_addrv2_relay.py @@ -11,19 +11,27 @@ import time from test_framework.messages import ( CAddress, msg_addrv2, - NODE_NETWORK, - NODE_WITNESS, ) -from test_framework.p2p import P2PInterface +from test_framework.p2p import ( + P2PInterface, + P2P_SERVICES, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal +I2P_ADDR = "c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p" + ADDRS = [] for i in range(10): addr = CAddress() addr.time = int(time.time()) + i - addr.nServices = NODE_NETWORK | NODE_WITNESS - addr.ip = "123.123.123.{}".format(i % 256) + addr.nServices = P2P_SERVICES + # Add one I2P address at an arbitrary position. + if i == 5: + addr.net = addr.NET_I2P + addr.ip = I2P_ADDR + else: + addr.ip = f"123.123.123.{i % 256}" addr.port = 8333 + i ADDRS.append(addr) @@ -35,11 +43,10 @@ class AddrReceiver(P2PInterface): super().__init__(support_addrv2 = True) def on_addrv2(self, message): - for addr in message.addrs: - assert_equal(addr.nServices, 9) - assert addr.ip.startswith('123.123.123.') - assert (8333 <= addr.port < 8343) - self.addrv2_received_and_checked = True + expected_set = set((addr.ip, addr.port) for addr in ADDRS) + received_set = set((addr.ip, addr.port) for addr in message.addrs) + if expected_set == received_set: + self.addrv2_received_and_checked = True def wait_for_addrv2(self): self.wait_until(lambda: "addrv2" in self.last_message) @@ -49,6 +56,7 @@ class AddrTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 + self.extra_args = [["-whitelist=addr@127.0.0.1"]] def run_test(self): self.log.info('Create connection that sends addrv2 messages') @@ -64,15 +72,18 @@ class AddrTest(BitcoinTestFramework): addr_receiver = self.nodes[0].add_p2p_connection(AddrReceiver()) msg.addrs = ADDRS with self.nodes[0].assert_debug_log([ - 'Added 10 addresses from 127.0.0.1: 0 tried', - 'received: addrv2 (131 bytes) peer=0', - 'sending addrv2 (131 bytes) peer=1', + # The I2P address is not added to node's own addrman because it has no + # I2P reachability (thus 10 - 1 = 9). + 'Added 9 addresses from 127.0.0.1: 0 tried', + 'received: addrv2 (159 bytes) peer=0', + 'sending addrv2 (159 bytes) peer=1', ]): addr_source.send_and_ping(msg) self.nodes[0].setmocktime(int(time.time()) + 30 * 60) addr_receiver.wait_for_addrv2() assert addr_receiver.addrv2_received_and_checked + assert_equal(len(self.nodes[0].getnodeaddresses(count=0, network="i2p")), 0) if __name__ == '__main__': diff --git a/test/functional/p2p_blockfilters.py b/test/functional/p2p_blockfilters.py index 63fc2a98d4..3a4fcc4549 100755 --- a/test/functional/p2p_blockfilters.py +++ b/test/functional/p2p_blockfilters.py @@ -56,17 +56,17 @@ class CompactFiltersTest(BitcoinTestFramework): peer_1 = self.nodes[1].add_p2p_connection(FiltersClient()) # Nodes 0 & 1 share the same first 999 blocks in the chain. - self.nodes[0].generate(999) + self.generate(self.nodes[0], 999) self.sync_blocks(timeout=600) # Stale blocks by disconnecting nodes 0 & 1, mining, then reconnecting self.disconnect_nodes(0, 1) - stale_block_hash = self.nodes[0].generate(1)[0] + stale_block_hash = self.generate(self.nodes[0], 1)[0] self.nodes[0].syncwithvalidationinterfacequeue() assert_equal(self.nodes[0].getblockcount(), 1000) - self.nodes[1].generate(1001) + self.generate(self.nodes[1], 1001) assert_equal(self.nodes[1].getblockcount(), 2000) # Check that nodes have signalled NODE_COMPACT_FILTERS correctly. diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index 6409d4ea82..94ae758d46 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -6,8 +6,7 @@ import time -from test_framework.blocktools import COINBASE_MATURITY -from test_framework.messages import msg_tx +from test_framework.messages import msg_tx, msg_inv, CInv, MSG_WTX from test_framework.p2p import P2PInterface, P2PTxInvStore from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal @@ -16,15 +15,13 @@ from test_framework.wallet import MiniWallet class P2PBlocksOnly(BitcoinTestFramework): def set_test_params(self): - self.setup_clean_chain = True self.num_nodes = 1 self.extra_args = [["-blocksonly"]] def run_test(self): self.miniwallet = MiniWallet(self.nodes[0]) # Add enough mature utxos to the wallet, so that all txs spend confirmed coins - self.miniwallet.generate(2) - self.nodes[0].generate(COINBASE_MATURITY) + self.miniwallet.rescan_utxos() self.blocksonly_mode_tests() self.blocks_relay_conn_tests() @@ -36,12 +33,19 @@ class P2PBlocksOnly(BitcoinTestFramework): self.nodes[0].add_p2p_connection(P2PInterface()) tx, txid, wtxid, tx_hex = self.check_p2p_tx_violation() + self.log.info('Check that tx invs also violate the protocol') + self.nodes[0].add_p2p_connection(P2PInterface()) + with self.nodes[0].assert_debug_log(['transaction (0000000000000000000000000000000000000000000000000000000000001234) inv sent in violation of protocol, disconnecting peer']): + self.nodes[0].p2ps[0].send_message(msg_inv([CInv(t=MSG_WTX, h=0x1234)])) + self.nodes[0].p2ps[0].wait_for_disconnect() + del self.nodes[0].p2ps[0] + self.log.info('Check that txs from rpc are not rejected and relayed to other peers') tx_relay_peer = self.nodes[0].add_p2p_connection(P2PInterface()) assert_equal(self.nodes[0].getpeerinfo()[0]['relaytxes'], True) assert_equal(self.nodes[0].testmempoolaccept([tx_hex])[0]['allowed'], True) - with self.nodes[0].assert_debug_log(['received getdata for: wtx {} peer=1'.format(wtxid)]): + with self.nodes[0].assert_debug_log(['received getdata for: wtx {} peer'.format(wtxid)]): self.nodes[0].sendrawtransaction(tx_hex) tx_relay_peer.wait_for_tx(txid) assert_equal(self.nodes[0].getmempoolinfo()['size'], 1) @@ -73,7 +77,7 @@ class P2PBlocksOnly(BitcoinTestFramework): self.log.info("Relay-permission peer's transaction is accepted and relayed") self.nodes[0].disconnect_p2ps() - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) def blocks_relay_conn_tests(self): self.log.info('Tests with node in normal mode with block-relay-only connections') @@ -83,7 +87,7 @@ class P2PBlocksOnly(BitcoinTestFramework): # Ensure we disconnect if a block-relay-only connection sends us a transaction self.nodes[0].add_outbound_p2p_connection(P2PInterface(), p2p_idx=0, connection_type="block-relay-only") assert_equal(self.nodes[0].getpeerinfo()[0]['relaytxes'], False) - _, txid, _, tx_hex = self.check_p2p_tx_violation(index=2) + _, txid, _, tx_hex = self.check_p2p_tx_violation() self.log.info("Check that txs from RPC are not sent to blockrelay connection") conn = self.nodes[0].add_outbound_p2p_connection(P2PTxInvStore(), p2p_idx=1, connection_type="block-relay-only") @@ -96,11 +100,9 @@ class P2PBlocksOnly(BitcoinTestFramework): conn.sync_send_with_ping() assert(int(txid, 16) not in conn.get_invs()) - def check_p2p_tx_violation(self, index=1): + def check_p2p_tx_violation(self): self.log.info('Check that txs from P2P are rejected and result in disconnect') - input_txid = self.nodes[0].getblock(self.nodes[0].getblockhash(index), 2)['tx'][0]['txid'] - utxo_to_spend = self.miniwallet.get_utxo(txid=input_txid) - spendtx = self.miniwallet.create_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_to_spend) + spendtx = self.miniwallet.create_self_transfer(from_node=self.nodes[0]) with self.nodes[0].assert_debug_log(['transaction sent in violation of protocol peer=0']): self.nodes[0].p2ps[0].send_message(msg_tx(spendtx['tx'])) diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index b4e662de2e..3f01d552b2 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -165,7 +165,7 @@ class CompactBlocksTest(BitcoinTestFramework): block = self.build_block_on_tip(self.nodes[0]) self.segwit_node.send_and_ping(msg_no_witness_block(block)) assert int(self.nodes[0].getbestblockhash(), 16) == block.sha256 - self.nodes[0].generatetoaddress(COINBASE_MATURITY, self.nodes[0].getnewaddress(address_type="bech32")) + self.generatetoaddress(self.nodes[0], COINBASE_MATURITY, self.nodes[0].getnewaddress(address_type="bech32")) total_value = block.vtx[0].vout[0].nValue out_value = total_value // 10 @@ -212,7 +212,7 @@ class CompactBlocksTest(BitcoinTestFramework): def check_announcement_of_new_block(node, peer, predicate): peer.clear_block_announcement() - block_hash = int(node.generate(1)[0], 16) + block_hash = int(self.generate(node, 1)[0], 16) peer.wait_for_block_announcement(block_hash, timeout=30) assert peer.block_announced @@ -276,7 +276,7 @@ class CompactBlocksTest(BitcoinTestFramework): # This test actually causes bitcoind to (reasonably!) disconnect us, so do this last. def test_invalid_cmpctblock_message(self): - self.nodes[0].generate(COINBASE_MATURITY + 1) + self.generate(self.nodes[0], COINBASE_MATURITY + 1) block = self.build_block_on_tip(self.nodes[0]) cmpct_block = P2PHeaderAndShortIDs() @@ -294,7 +294,7 @@ class CompactBlocksTest(BitcoinTestFramework): version = test_node.cmpct_version node = self.nodes[0] # Generate a bunch of transactions. - node.generate(COINBASE_MATURITY + 1) + self.generate(node, COINBASE_MATURITY + 1) num_transactions = 25 address = node.getnewaddress() @@ -318,7 +318,7 @@ class CompactBlocksTest(BitcoinTestFramework): # Now mine a block, and look at the resulting compact block. test_node.clear_block_announcement() - block_hash = int(node.generate(1)[0], 16) + block_hash = int(self.generate(node, 1)[0], 16) # Store the raw block in our internal format. block = from_hex(CBlock(), node.getblock("%064x" % block_hash, False)) @@ -660,7 +660,7 @@ class CompactBlocksTest(BitcoinTestFramework): new_blocks = [] for _ in range(MAX_CMPCTBLOCK_DEPTH + 1): test_node.clear_block_announcement() - new_blocks.append(node.generate(1)[0]) + new_blocks.append(self.generate(node, 1)[0]) test_node.wait_until(test_node.received_block_announcement, timeout=30) test_node.clear_block_announcement() @@ -668,7 +668,7 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.wait_until(lambda: "cmpctblock" in test_node.last_message, timeout=30) test_node.clear_block_announcement() - node.generate(1) + self.generate(node, 1) test_node.wait_until(test_node.received_block_announcement, timeout=30) test_node.clear_block_announcement() with p2p_lock: @@ -844,7 +844,7 @@ class CompactBlocksTest(BitcoinTestFramework): def run_test(self): # Get the nodes out of IBD - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) # Setup the p2p connections self.segwit_node = self.nodes[0].add_p2p_connection(TestP2PConn(cmpct_version=2)) diff --git a/test/functional/p2p_compactblocks_blocksonly.py b/test/functional/p2p_compactblocks_blocksonly.py new file mode 100755 index 0000000000..5f01fa4dfe --- /dev/null +++ b/test/functional/p2p_compactblocks_blocksonly.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test that a node in blocksonly mode does not request compact blocks.""" + +from test_framework.messages import ( + MSG_BLOCK, + MSG_CMPCT_BLOCK, + MSG_WITNESS_FLAG, + CBlock, + CBlockHeader, + CInv, + from_hex, + msg_block, + msg_getdata, + msg_headers, + msg_sendcmpct, +) +from test_framework.p2p import P2PInterface +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + + +class P2PCompactBlocksBlocksOnly(BitcoinTestFramework): + def set_test_params(self): + self.extra_args = [["-blocksonly"], [], [], []] + self.num_nodes = 4 + + def setup_network(self): + self.setup_nodes() + # Start network with everyone disconnected + self.sync_all() + + def build_block_on_tip(self): + blockhash = self.generate(self.nodes[2], 1)[0] + block_hex = self.nodes[2].getblock(blockhash=blockhash, verbosity=0) + block = from_hex(CBlock(), block_hex) + block.rehash() + return block + + def run_test(self): + # Nodes will only request hb compact blocks mode when they're out of IBD + for node in self.nodes: + assert not node.getblockchaininfo()['initialblockdownload'] + + p2p_conn_blocksonly = self.nodes[0].add_p2p_connection(P2PInterface()) + p2p_conn_high_bw = self.nodes[1].add_p2p_connection(P2PInterface()) + p2p_conn_low_bw = self.nodes[3].add_p2p_connection(P2PInterface()) + for conn in [p2p_conn_blocksonly, p2p_conn_high_bw, p2p_conn_low_bw]: + assert_equal(conn.message_count['sendcmpct'], 2) + conn.send_and_ping(msg_sendcmpct(announce=False, version=2)) + + # Nodes: + # 0 -> blocksonly + # 1 -> high bandwidth + # 2 -> miner + # 3 -> low bandwidth + # + # Topology: + # p2p_conn_blocksonly ---> node0 + # p2p_conn_high_bw ---> node1 + # p2p_conn_low_bw ---> node3 + # node2 (no connections) + # + # node2 produces blocks that are passed to the rest of the nodes + # through the respective p2p connections. + + self.log.info("Test that -blocksonly nodes do not select peers for BIP152 high bandwidth mode") + + block0 = self.build_block_on_tip() + + # A -blocksonly node should not request BIP152 high bandwidth mode upon + # receiving a new valid block at the tip. + p2p_conn_blocksonly.send_and_ping(msg_block(block0)) + assert_equal(int(self.nodes[0].getbestblockhash(), 16), block0.sha256) + assert_equal(p2p_conn_blocksonly.message_count['sendcmpct'], 2) + assert_equal(p2p_conn_blocksonly.last_message['sendcmpct'].announce, False) + + # A normal node participating in transaction relay should request BIP152 + # high bandwidth mode upon receiving a new valid block at the tip. + p2p_conn_high_bw.send_and_ping(msg_block(block0)) + assert_equal(int(self.nodes[1].getbestblockhash(), 16), block0.sha256) + p2p_conn_high_bw.wait_until(lambda: p2p_conn_high_bw.message_count['sendcmpct'] == 3) + assert_equal(p2p_conn_high_bw.last_message['sendcmpct'].announce, True) + + # Don't send a block from the p2p_conn_low_bw so the low bandwidth node + # doesn't select it for BIP152 high bandwidth relay. + self.nodes[3].submitblock(block0.serialize().hex()) + + self.log.info("Test that -blocksonly nodes send getdata(BLOCK) instead" + " of getdata(CMPCT) in BIP152 low bandwidth mode") + + block1 = self.build_block_on_tip() + + p2p_conn_blocksonly.send_message(msg_headers(headers=[CBlockHeader(block1)])) + p2p_conn_blocksonly.sync_send_with_ping() + assert_equal(p2p_conn_blocksonly.last_message['getdata'].inv, [CInv(MSG_BLOCK | MSG_WITNESS_FLAG, block1.sha256)]) + + p2p_conn_high_bw.send_message(msg_headers(headers=[CBlockHeader(block1)])) + p2p_conn_high_bw.sync_send_with_ping() + assert_equal(p2p_conn_high_bw.last_message['getdata'].inv, [CInv(MSG_CMPCT_BLOCK, block1.sha256)]) + + self.log.info("Test that getdata(CMPCT) is still sent on BIP152 low bandwidth connections" + " when no -blocksonly nodes are involved") + + p2p_conn_low_bw.send_and_ping(msg_headers(headers=[CBlockHeader(block1)])) + p2p_conn_low_bw.sync_with_ping() + assert_equal(p2p_conn_low_bw.last_message['getdata'].inv, [CInv(MSG_CMPCT_BLOCK, block1.sha256)]) + + self.log.info("Test that -blocksonly nodes still serve compact blocks") + + def test_for_cmpctblock(block): + if 'cmpctblock' not in p2p_conn_blocksonly.last_message: + return False + return p2p_conn_blocksonly.last_message['cmpctblock'].header_and_shortids.header.rehash() == block.sha256 + + p2p_conn_blocksonly.send_message(msg_getdata([CInv(MSG_CMPCT_BLOCK, block0.sha256)])) + p2p_conn_blocksonly.wait_until(lambda: test_for_cmpctblock(block0)) + + # Request BIP152 high bandwidth mode from the -blocksonly node. + p2p_conn_blocksonly.send_and_ping(msg_sendcmpct(announce=True, version=2)) + + block2 = self.build_block_on_tip() + self.nodes[0].submitblock(block1.serialize().hex()) + self.nodes[0].submitblock(block2.serialize().hex()) + p2p_conn_blocksonly.wait_until(lambda: test_for_cmpctblock(block2)) + +if __name__ == '__main__': + P2PCompactBlocksBlocksOnly().main() diff --git a/test/functional/p2p_compactblocks_hb.py b/test/functional/p2p_compactblocks_hb.py index a3d30a6f04..72b3897b4f 100755 --- a/test/functional/p2p_compactblocks_hb.py +++ b/test/functional/p2p_compactblocks_hb.py @@ -30,7 +30,7 @@ class CompactBlocksConnectionTest(BitcoinTestFramework): def relay_block_through(self, peer): """Relay a new block through peer peer, and return HB status between 1 and [2,3,4,5].""" self.connect_nodes(peer, 0) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks() self.disconnect_nodes(peer, 0) status_to = [self.peer_info(1, i)['bip152_hb_to'] for i in range(2, 6)] @@ -44,7 +44,7 @@ class CompactBlocksConnectionTest(BitcoinTestFramework): # Connect everyone to node 0, and mine some blocks to get all nodes out of IBD. for i in range(1, 6): self.connect_nodes(i, 0) - self.nodes[0].generate(2) + self.generate(self.nodes[0], 2) self.sync_blocks() for i in range(1, 6): self.disconnect_nodes(i, 0) diff --git a/test/functional/p2p_dns_seeds.py b/test/functional/p2p_dns_seeds.py new file mode 100755 index 0000000000..e58ad8e0fc --- /dev/null +++ b/test/functional/p2p_dns_seeds.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test ThreadDNSAddressSeed logic for querying DNS seeds.""" + +import itertools + +from test_framework.p2p import P2PInterface +from test_framework.test_framework import BitcoinTestFramework + + +class P2PDNSSeeds(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + self.extra_args = [["-dnsseed=1"]] + + def run_test(self): + self.init_arg_tests() + self.existing_outbound_connections_test() + self.existing_block_relay_connections_test() + self.force_dns_test() + self.wait_time_tests() + + def init_arg_tests(self): + fakeaddr = "fakenodeaddr.fakedomain.invalid." + + self.log.info("Check that setting -connect disables -dnsseed by default") + self.nodes[0].stop_node() + with self.nodes[0].assert_debug_log(expected_msgs=["DNS seeding disabled"]): + self.start_node(0, [f"-connect={fakeaddr}"]) + + self.log.info("Check that running -connect and -dnsseed means DNS logic runs.") + with self.nodes[0].assert_debug_log(expected_msgs=["Loading addresses from DNS seed"], timeout=12): + self.restart_node(0, [f"-connect={fakeaddr}", "-dnsseed=1"]) + + self.log.info("Check that running -forcednsseed and -dnsseed=0 throws an error.") + self.nodes[0].stop_node() + self.nodes[0].assert_start_raises_init_error( + expected_msg="Error: Cannot set -forcednsseed to true when setting -dnsseed to false.", + extra_args=["-forcednsseed=1", "-dnsseed=0"], + ) + + self.log.info("Check that running -forcednsseed and -connect throws an error.") + # -connect soft sets -dnsseed to false, so throws the same error + self.nodes[0].stop_node() + self.nodes[0].assert_start_raises_init_error( + expected_msg="Error: Cannot set -forcednsseed to true when setting -dnsseed to false.", + extra_args=["-forcednsseed=1", f"-connect={fakeaddr}"], + ) + + # Restore default bitcoind settings + self.restart_node(0) + + def existing_outbound_connections_test(self): + # Make sure addrman is populated to enter the conditional where we + # delay and potentially skip DNS seeding. + self.nodes[0].addpeeraddress("192.0.0.8", 8333) + + self.log.info("Check that we *do not* query DNS seeds if we have 2 outbound connections") + + self.restart_node(0) + with self.nodes[0].assert_debug_log(expected_msgs=["P2P peers available. Skipped DNS seeding."], timeout=12): + for i in range(2): + self.nodes[0].add_outbound_p2p_connection(P2PInterface(), p2p_idx=i, connection_type="outbound-full-relay") + + def existing_block_relay_connections_test(self): + # Make sure addrman is populated to enter the conditional where we + # delay and potentially skip DNS seeding. No-op when run after + # existing_outbound_connections_test. + self.nodes[0].addpeeraddress("192.0.0.8", 8333) + + self.log.info("Check that we *do* query DNS seeds if we only have 2 block-relay-only connections") + + self.restart_node(0) + with self.nodes[0].assert_debug_log(expected_msgs=["Loading addresses from DNS seed"], timeout=12): + # This mimics the "anchors" logic where nodes are likely to + # reconnect to block-relay-only connections on startup. + # Since we do not participate in addr relay with these connections, + # we still want to query the DNS seeds. + for i in range(2): + self.nodes[0].add_outbound_p2p_connection(P2PInterface(), p2p_idx=i, connection_type="block-relay-only") + + def force_dns_test(self): + self.log.info("Check that we query DNS seeds if -forcednsseed param is set") + + with self.nodes[0].assert_debug_log(expected_msgs=["Loading addresses from DNS seed"], timeout=12): + # -dnsseed defaults to 1 in bitcoind, but 0 in the test framework, + # so pass it explicitly here + self.restart_node(0, ["-forcednsseed", "-dnsseed=1"]) + + # Restore default for subsequent tests + self.restart_node(0) + + def wait_time_tests(self): + self.log.info("Check the delay before querying DNS seeds") + + # Populate addrman with < 1000 addresses + for i in range(5): + a = f"192.0.0.{i}" + self.nodes[0].addpeeraddress(a, 8333) + + # The delay should be 11 seconds + with self.nodes[0].assert_debug_log(expected_msgs=["Waiting 11 seconds before querying DNS seeds.\n"]): + self.restart_node(0) + + # Populate addrman with > 1000 addresses + for i in itertools.count(): + first_octet = i % 2 + 1 + second_octet = i % 256 + third_octet = i % 100 + a = f"{first_octet}.{second_octet}.{third_octet}.1" + self.nodes[0].addpeeraddress(a, 8333) + if (i > 1000 and i % 100 == 0): + # The addrman size is non-deterministic because new addresses + # are sorted into buckets, potentially displacing existing + # addresses. Periodically check if we have met the desired + # threshold. + if len(self.nodes[0].getnodeaddresses(0)) > 1000: + break + + # The delay should be 5 mins + with self.nodes[0].assert_debug_log(expected_msgs=["Waiting 300 seconds before querying DNS seeds.\n"]): + self.restart_node(0) + + +if __name__ == '__main__': + P2PDNSSeeds().main() diff --git a/test/functional/p2p_eviction.py b/test/functional/p2p_eviction.py index 35bce7c69e..4ccc942164 100755 --- a/test/functional/p2p_eviction.py +++ b/test/functional/p2p_eviction.py @@ -53,7 +53,7 @@ class P2PEvict(BitcoinTestFramework): protected_peers = set() # peers that we expect to be protected from eviction current_peer = -1 node = self.nodes[0] - node.generatetoaddress(COINBASE_MATURITY + 1, node.get_deterministic_priv_key().address) + self.generatetoaddress(node, COINBASE_MATURITY + 1, node.get_deterministic_priv_key().address) self.log.info("Create 4 peers and protect them from eviction by sending us a block") for _ in range(4): diff --git a/test/functional/p2p_feefilter.py b/test/functional/p2p_feefilter.py index 0175b9f6c0..60adc2c7fa 100755 --- a/test/functional/p2p_feefilter.py +++ b/test/functional/p2p_feefilter.py @@ -81,8 +81,8 @@ class FeeFilterTest(BitcoinTestFramework): node0 = self.nodes[0] miniwallet = MiniWallet(node1) # Add enough mature utxos to the wallet, so that all txs spend confirmed coins - miniwallet.generate(5) - node1.generate(COINBASE_MATURITY) + self.generate(miniwallet, 5) + self.generate(node1, COINBASE_MATURITY) conn = self.nodes[0].add_p2p_connection(TestP2PConn()) diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index 359cfb9c34..0d8c298bea 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -8,6 +8,7 @@ Test BIP 37 from test_framework.messages import ( CInv, + COIN, MAX_BLOOM_FILTER_SIZE, MAX_BLOOM_HASH_FUNCS, MSG_BLOCK, @@ -28,11 +29,15 @@ from test_framework.p2p import ( ) from test_framework.script import MAX_SCRIPT_ELEMENT_SIZE from test_framework.test_framework import BitcoinTestFramework +from test_framework.wallet import ( + MiniWallet, + random_p2wpkh, +) class P2PBloomFilter(P2PInterface): # This is a P2SH watch-only wallet - watch_script_pubkey = 'a914ffffffffffffffffffffffffffffffffffffffff87' + watch_script_pubkey = bytes.fromhex('a914ffffffffffffffffffffffffffffffffffffffff87') # The initial filter (n=10, fp=0.000001) with just the above scriptPubKey added watch_filter_init = msg_filterload( data= @@ -93,8 +98,9 @@ class FilterTest(BitcoinTestFramework): '-whitelist=noban@127.0.0.1', # immediate tx relay ]] - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() + def generatetoscriptpubkey(self, scriptpubkey): + """Helper to generate a single block to the given scriptPubKey.""" + return self.generatetodescriptor(self.nodes[0], 1, f'raw({scriptpubkey.hex()})')[0] def test_size_limits(self, filter_peer): self.log.info('Check that too large filter is rejected') @@ -130,8 +136,7 @@ class FilterTest(BitcoinTestFramework): filter_peer = P2PBloomFilter() self.log.debug("Create a tx relevant to the peer before connecting") - filter_address = self.nodes[0].decodescript(filter_peer.watch_script_pubkey)['address'] - txid = self.nodes[0].sendtoaddress(filter_address, 90) + txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=filter_peer.watch_script_pubkey, amount=9 * COIN) self.log.debug("Send a mempool msg after connecting and check that the tx is received") self.nodes[0].add_p2p_connection(filter_peer) @@ -142,59 +147,57 @@ class FilterTest(BitcoinTestFramework): def test_frelay_false(self, filter_peer): self.log.info("Check that a node with fRelay set to false does not receive invs until the filter is set") filter_peer.tx_received = False - filter_address = self.nodes[0].decodescript(filter_peer.watch_script_pubkey)['address'] - self.nodes[0].sendtoaddress(filter_address, 90) + self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=filter_peer.watch_script_pubkey, amount=9 * COIN) # Sync to make sure the reason filter_peer doesn't receive the tx is not p2p delays filter_peer.sync_with_ping() assert not filter_peer.tx_received # Clear the mempool so that this transaction does not impact subsequent tests - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) def test_filter(self, filter_peer): # Set the bloomfilter using filterload filter_peer.send_and_ping(filter_peer.watch_filter_init) # If fRelay is not already True, sending filterload sets it to True assert self.nodes[0].getpeerinfo()[0]['relaytxes'] - filter_address = self.nodes[0].decodescript(filter_peer.watch_script_pubkey)['address'] self.log.info('Check that we receive merkleblock and tx if the filter matches a tx in a block') - block_hash = self.nodes[0].generatetoaddress(1, filter_address)[0] + block_hash = self.generatetoscriptpubkey(filter_peer.watch_script_pubkey) txid = self.nodes[0].getblock(block_hash)['tx'][0] filter_peer.wait_for_merkleblock(block_hash) filter_peer.wait_for_tx(txid) self.log.info('Check that we only receive a merkleblock if the filter does not match a tx in a block') filter_peer.tx_received = False - block_hash = self.nodes[0].generatetoaddress(1, self.nodes[0].getnewaddress())[0] + block_hash = self.generatetoscriptpubkey(random_p2wpkh()) filter_peer.wait_for_merkleblock(block_hash) assert not filter_peer.tx_received self.log.info('Check that we not receive a tx if the filter does not match a mempool tx') filter_peer.merkleblock_received = False filter_peer.tx_received = False - self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 90) + self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=random_p2wpkh(), amount=7 * COIN) filter_peer.sync_send_with_ping() assert not filter_peer.merkleblock_received assert not filter_peer.tx_received self.log.info('Check that we receive a tx if the filter matches a mempool tx') filter_peer.merkleblock_received = False - txid = self.nodes[0].sendtoaddress(filter_address, 90) + txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=filter_peer.watch_script_pubkey, amount=9 * COIN) filter_peer.wait_for_tx(txid) assert not filter_peer.merkleblock_received self.log.info('Check that after deleting filter all txs get relayed again') filter_peer.send_and_ping(msg_filterclear()) for _ in range(5): - txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 7) + txid, _ = self.wallet.send_to(from_node=self.nodes[0], scriptPubKey=random_p2wpkh(), amount=7 * COIN) filter_peer.wait_for_tx(txid) self.log.info('Check that request for filtered blocks is ignored if no filter is set') filter_peer.merkleblock_received = False filter_peer.tx_received = False with self.nodes[0].assert_debug_log(expected_msgs=['received getdata']): - block_hash = self.nodes[0].generatetoaddress(1, self.nodes[0].getnewaddress())[0] + block_hash = self.generatetoscriptpubkey(random_p2wpkh()) filter_peer.wait_for_inv([CInv(MSG_BLOCK, int(block_hash, 16))]) filter_peer.sync_with_ping() assert not filter_peer.merkleblock_received @@ -210,6 +213,9 @@ class FilterTest(BitcoinTestFramework): self.nodes[0].disconnect_p2ps() def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) + self.wallet.rescan_utxos() + filter_peer = self.nodes[0].add_p2p_connection(P2PBloomFilter()) self.log.info('Test filter size limits') self.test_size_limits(filter_peer) diff --git a/test/functional/p2p_fingerprint.py b/test/functional/p2p_fingerprint.py index 469d66a851..2962dc8085 100755 --- a/test/functional/p2p_fingerprint.py +++ b/test/functional/p2p_fingerprint.py @@ -69,7 +69,7 @@ class P2PFingerprintTest(BitcoinTestFramework): self.nodes[0].setmocktime(int(time.time()) - 60 * 24 * 60 * 60) # Generating a chain of 10 blocks - block_hashes = self.nodes[0].generatetoaddress(10, self.nodes[0].get_deterministic_priv_key().address) + block_hashes = self.generatetoaddress(self.nodes[0], 10, self.nodes[0].get_deterministic_priv_key().address) # Create longer chain starting 2 blocks before current tip height = len(block_hashes) - 2 @@ -98,7 +98,7 @@ class P2PFingerprintTest(BitcoinTestFramework): # Longest chain is extended so stale is much older than chain tip self.nodes[0].setmocktime(0) - block_hash = int(self.nodes[0].generatetoaddress(1, self.nodes[0].get_deterministic_priv_key().address)[-1], 16) + block_hash = int(self.generatetoaddress(self.nodes[0], 1, self.nodes[0].get_deterministic_priv_key().address)[-1], 16) assert_equal(self.nodes[0].getblockcount(), 14) node0.wait_for_block(block_hash, timeout=3) diff --git a/test/functional/p2p_i2p_ports.py b/test/functional/p2p_i2p_ports.py new file mode 100755 index 0000000000..13188b9305 --- /dev/null +++ b/test/functional/p2p_i2p_ports.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test ports handling for I2P hosts +""" + +import re + +from test_framework.test_framework import BitcoinTestFramework + + +class I2PPorts(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + # The test assumes that an I2P SAM proxy is not listening here. + self.extra_args = [["-i2psam=127.0.0.1:60000"]] + + def run_test(self): + node = self.nodes[0] + + self.log.info("Ensure we don't try to connect if port!=0") + addr = "zsxwyo6qcn3chqzwxnseusqgsnuw3maqnztkiypyfxtya4snkoka.b32.i2p:8333" + raised = False + try: + with node.assert_debug_log(expected_msgs=[f"Error connecting to {addr}"]): + node.addnode(node=addr, command="onetry") + except AssertionError as e: + raised = True + if not re.search(r"Expected messages .* does not partially match log", str(e)): + raise AssertionError(f"Assertion raised as expected, but with an unexpected message: {str(e)}") + if not raised: + raise AssertionError("Assertion should have been raised") + + self.log.info("Ensure we try to connect if port=0 and get an error due to missing I2P proxy") + addr = "h3r6bkn46qxftwja53pxiykntegfyfjqtnzbm6iv6r5mungmqgmq.b32.i2p:0" + with node.assert_debug_log(expected_msgs=[f"Error connecting to {addr}"]): + node.addnode(node=addr, command="onetry") + + +if __name__ == '__main__': + I2PPorts().main() diff --git a/test/functional/p2p_ibd_txrelay.py b/test/functional/p2p_ibd_txrelay.py index c3e758b021..c35053d9d4 100755 --- a/test/functional/p2p_ibd_txrelay.py +++ b/test/functional/p2p_ibd_txrelay.py @@ -29,7 +29,7 @@ class P2PIBDTxRelayTest(BitcoinTestFramework): self.wait_until(lambda: all(peer['minfeefilter'] == MAX_FEE_FILTER for peer in node.getpeerinfo())) # Come out of IBD by generating a block - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() self.log.info("Check that nodes reset minfilter after coming out of IBD") diff --git a/test/functional/p2p_invalid_block.py b/test/functional/p2p_invalid_block.py index 91666d0f08..875ab52db4 100755 --- a/test/functional/p2p_invalid_block.py +++ b/test/functional/p2p_invalid_block.py @@ -51,7 +51,7 @@ class InvalidBlockRequestTest(BitcoinTestFramework): peer.send_blocks_and_test([block1], node, success=True) self.log.info("Mature the block.") - node.generatetoaddress(100, node.get_deterministic_priv_key().address) + self.generatetoaddress(node, 100, node.get_deterministic_priv_key().address) best_block = node.getblock(node.getbestblockhash()) tip = int(node.getbestblockhash(), 16) diff --git a/test/functional/p2p_invalid_locator.py b/test/functional/p2p_invalid_locator.py index f884cf90ff..a586b48d4c 100755 --- a/test/functional/p2p_invalid_locator.py +++ b/test/functional/p2p_invalid_locator.py @@ -16,7 +16,7 @@ class InvalidLocatorTest(BitcoinTestFramework): def run_test(self): node = self.nodes[0] # convenience reference to the node - node.generatetoaddress(1, node.get_deterministic_priv_key().address) # Get node out of IBD + self.generatetoaddress(node, 1, node.get_deterministic_priv_key().address) # Get node out of IBD self.log.info('Test max locator size') block_count = node.getblockcount() diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 788a81d4af..f3b80abb59 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -28,7 +28,6 @@ from test_framework.p2p import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - hex_str_to_bytes, ) VALID_DATA_LIMIT = MAX_PROTOCOL_MESSAGE_LENGTH - 5 # Account for the 5-byte length prefix @@ -58,6 +57,7 @@ class InvalidMessagesTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 self.setup_clean_chain = True + self.extra_args = [["-whitelist=addr@127.0.0.1"]] def run_test(self): self.test_buffer() @@ -186,7 +186,7 @@ class InvalidMessagesTest(BitcoinTestFramework): [ 'received: addrv2 (1 bytes)', ], - hex_str_to_bytes('00')) + bytes.fromhex('00')) def test_addrv2_too_long_address(self): self.test_addrv2('too long address', @@ -195,7 +195,7 @@ class InvalidMessagesTest(BitcoinTestFramework): 'ProcessMessages(addrv2, 525 bytes): Exception', 'Address too long: 513 > 512', ], - hex_str_to_bytes( + bytes.fromhex( '01' + # number of entries '61bc6649' + # time, Fri Jan 9 02:54:25 UTC 2009 '00' + # service flags, COMPACTSIZE(NODE_NONE) @@ -212,7 +212,7 @@ class InvalidMessagesTest(BitcoinTestFramework): 'IP 9.9.9.9 mapped', 'Added 1 addresses', ], - hex_str_to_bytes( + bytes.fromhex( '02' + # number of entries # this should be ignored without impeding acceptance of subsequent ones now_hex + # time diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index 8783c244c3..0a3ae23f58 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -64,7 +64,7 @@ class InvalidTxRequestTest(BitcoinTestFramework): node.p2ps[0].send_blocks_and_test([block], node, success=True) self.log.info("Mature the block.") - self.nodes[0].generatetoaddress(100, self.nodes[0].get_deterministic_priv_key().address) + self.generatetoaddress(self.nodes[0], 100, self.nodes[0].get_deterministic_priv_key().address) # Iterate through a list of known invalid transaction types, ensuring each is # rejected. Some are consensus invalid and some just violate policy. @@ -141,9 +141,9 @@ class InvalidTxRequestTest(BitcoinTestFramework): tx_orphan_2_valid, # The valid transaction (with sufficient fee) ] } - # Transactions that do not end up in the mempool - # tx_orphan_no_fee, because it has too low fee (p2ps[0] is not disconnected for relaying that tx) - # tx_orphan_invalid, because it has negative fee (p2ps[1] is disconnected for relaying that tx) + # Transactions that do not end up in the mempool: + # tx_orphan_2_no_fee, because it has too low fee (p2ps[0] is not disconnected for relaying that tx) + # tx_orphan_2_invalid, because it has negative fee (p2ps[1] is disconnected for relaying that tx) self.wait_until(lambda: 1 == len(node.getpeerinfo()), timeout=12) # p2ps[1] is no longer connected assert_equal(expected_mempool, set(node.getrawmempool())) diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index f1538e8ac7..de58e07aad 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -133,7 +133,7 @@ class P2PLeakTest(BitcoinTestFramework): pre_wtxidrelay_peer.wait_until(lambda: pre_wtxidrelay_peer.version_received) # Mine a block and make sure that it's not sent to the connected peers - self.nodes[0].generate(nblocks=1) + self.generate(self.nodes[0], nblocks=1) # Give the node enough time to possibly leak out a message time.sleep(PEER_TIMEOUT + 2) diff --git a/test/functional/p2p_leak_tx.py b/test/functional/p2p_leak_tx.py index 9a4ceb86ae..9b80e1b877 100755 --- a/test/functional/p2p_leak_tx.py +++ b/test/functional/p2p_leak_tx.py @@ -27,8 +27,8 @@ class P2PLeakTxTest(BitcoinTestFramework): gen_node = self.nodes[0] # The block and tx generating node miniwallet = MiniWallet(gen_node) # Add enough mature utxos to the wallet, so that all txs spend confirmed coins - miniwallet.generate(1) - gen_node.generate(COINBASE_MATURITY) + self.generate(miniwallet, 1) + self.generate(gen_node, COINBASE_MATURITY) inbound_peer = self.nodes[0].add_p2p_connection(P2PNode()) # An "attacking" inbound peer diff --git a/test/functional/p2p_node_network_limited.py b/test/functional/p2p_node_network_limited.py index 8d95f155c8..e491fe7e07 100755 --- a/test/functional/p2p_node_network_limited.py +++ b/test/functional/p2p_node_network_limited.py @@ -59,7 +59,7 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): self.log.info("Mine enough blocks to reach the NODE_NETWORK_LIMITED range.") self.connect_nodes(0, 1) - blocks = self.nodes[1].generatetoaddress(292, self.nodes[1].get_deterministic_priv_key().address) + blocks = self.generatetoaddress(self.nodes[1], 292, self.nodes[1].get_deterministic_priv_key().address) self.sync_blocks([self.nodes[0], self.nodes[1]]) self.log.info("Make sure we can max retrieve block at tip-288.") @@ -101,7 +101,7 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): self.disconnect_all() # mine 10 blocks on node 0 (pruned node) - self.nodes[0].generatetoaddress(10, self.nodes[0].get_deterministic_priv_key().address) + self.generatetoaddress(self.nodes[0], 10, self.nodes[0].get_deterministic_priv_key().address) # connect node1 (non pruned) with node0 (pruned) and check if the can sync self.connect_nodes(0, 1) diff --git a/test/functional/p2p_permissions.py b/test/functional/p2p_permissions.py index 594a28d662..32f2ea14e1 100755 --- a/test/functional/p2p_permissions.py +++ b/test/functional/p2p_permissions.py @@ -93,7 +93,7 @@ class P2PPermissionsTests(BitcoinTestFramework): self.nodes[1].assert_start_raises_init_error(["-whitebind=noban@127.0.0.1/10"], "Cannot resolve -whitebind address", match=ErrorMatch.PARTIAL_REGEX) def check_tx_relay(self): - block_op_true = self.nodes[0].getblock(self.nodes[0].generatetoaddress(100, ADDRESS_BCRT1_P2WSH_OP_TRUE)[0]) + block_op_true = self.nodes[0].getblock(self.generatetoaddress(self.nodes[0], 100, ADDRESS_BCRT1_P2WSH_OP_TRUE)[0]) self.sync_all() self.log.debug("Create a connection from a forcerelay peer that rebroadcasts raw txs") @@ -130,7 +130,7 @@ class P2PPermissionsTests(BitcoinTestFramework): tx.vout[0].nValue += 1 txid = tx.rehash() # Send the transaction twice. The first time, it'll be rejected by ATMP because it conflicts - # with a mempool transaction. The second time, it'll be in the recentRejects filter. + # with a mempool transaction. The second time, it'll be in the m_recent_rejects filter. p2p_rebroadcast_wallet.send_txs_and_test( [tx], self.nodes[1], diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 95c7aec318..4e21d08e5c 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -4,16 +4,14 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test segwit transactions and blocks on P2P network.""" from decimal import Decimal -import math import random import struct import time -from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment, get_witness_script, WITNESS_COMMITMENT_HEADER +from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment, WITNESS_COMMITMENT_HEADER from test_framework.key import ECKey from test_framework.messages import ( BIP125_SEQUENCE_NUMBER, - CBlock, CBlockHeader, CInv, COutPoint, @@ -22,7 +20,7 @@ from test_framework.messages import ( CTxInWitness, CTxOut, CTxWitness, - MAX_BLOCK_BASE_SIZE, + MAX_BLOCK_WEIGHT, MSG_BLOCK, MSG_TX, MSG_WITNESS_FLAG, @@ -41,11 +39,11 @@ from test_framework.messages import ( ser_vector, sha256, tx_from_hex, - uint256_from_str, ) from test_framework.p2p import ( P2PInterface, p2p_lock, + P2P_SERVICES, ) from test_framework.script import ( CScript, @@ -60,12 +58,8 @@ from test_framework.script import ( OP_CHECKMULTISIG, OP_CHECKSIG, OP_DROP, - OP_DUP, OP_ELSE, OP_ENDIF, - OP_EQUAL, - OP_EQUALVERIFY, - OP_HASH160, OP_IF, OP_RETURN, OP_TRUE, @@ -77,18 +71,20 @@ from test_framework.script import ( LegacySignatureHash, hash160, ) +from test_framework.script_util import ( + key_to_p2pk_script, + key_to_p2wpkh_script, + keyhash_to_p2pkh_script, + script_to_p2sh_script, + script_to_p2wsh_script, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, softfork_active, - hex_str_to_bytes, assert_raises_rpc_error, ) -# The versionbit bit used to signal activation of SegWit -VB_WITNESS_BIT = 1 -VB_TOP_BITS = 0x20000000 - MAX_SIGOP_COST = 80000 SEGWIT_HEIGHT = 120 @@ -100,27 +96,13 @@ class UTXO(): self.n = n self.nValue = value -def get_p2pkh_script(pubkeyhash): - """Get the script associated with a P2PKH.""" - return CScript([CScriptOp(OP_DUP), CScriptOp(OP_HASH160), pubkeyhash, CScriptOp(OP_EQUALVERIFY), CScriptOp(OP_CHECKSIG)]) - def sign_p2pk_witness_input(script, tx_to, in_idx, hashtype, value, key): - """Add signature for a P2PK witness program.""" + """Add signature for a P2PK witness script.""" tx_hash = SegwitV0SignatureHash(script, tx_to, in_idx, hashtype, value) signature = key.sign_ecdsa(tx_hash) + chr(hashtype).encode('latin-1') tx_to.wit.vtxinwit[in_idx].scriptWitness.stack = [signature, script] tx_to.rehash() -def get_virtual_size(witness_block): - """Calculate the virtual size of a witness block. - - Virtual size is base + witness/4.""" - base_size = len(witness_block.serialize(with_witness=False)) - total_size = len(witness_block.serialize()) - # the "+3" is so we round up - vsize = int((3 * base_size + total_size + 3) / 4) - return vsize - def test_transaction_acceptance(node, p2p, tx, with_witness, accepted, reason=None): """Send a transaction to the node and check that it's accepted to the mempool @@ -209,33 +191,26 @@ class TestP2PConn(P2PInterface): class SegWitTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 3 + self.num_nodes = 2 # This test tests SegWit both pre and post-activation, so use the normal BIP9 activation. self.extra_args = [ - ["-acceptnonstdtxn=1", "-segwitheight={}".format(SEGWIT_HEIGHT), "-whitelist=noban@127.0.0.1"], - ["-acceptnonstdtxn=0", "-segwitheight={}".format(SEGWIT_HEIGHT)], - ["-acceptnonstdtxn=1", "-segwitheight=-1"], + ["-acceptnonstdtxn=1", f"-testactivationheight=segwit@{SEGWIT_HEIGHT}", "-whitelist=noban@127.0.0.1"], + ["-acceptnonstdtxn=0", f"-testactivationheight=segwit@{SEGWIT_HEIGHT}"], ] self.supports_cli = False def skip_test_if_missing_module(self): self.skip_if_no_wallet() - def setup_network(self): - self.setup_nodes() - self.connect_nodes(0, 1) - self.connect_nodes(0, 2) - self.sync_all() - # Helper functions - def build_next_block(self, version=4): + def build_next_block(self): """Build a block on top of node0's tip.""" tip = self.nodes[0].getbestblockhash() height = self.nodes[0].getblockcount() + 1 block_time = self.nodes[0].getblockheader(tip)["mediantime"] + 1 block = create_block(int(tip, 16), create_coinbase(height), block_time) - block.nVersion = version + block.nVersion = 4 block.rehash() return block @@ -247,14 +222,14 @@ class SegWitTest(BitcoinTestFramework): def run_test(self): # Setup the p2p connections - # self.test_node sets NODE_WITNESS|NODE_NETWORK - self.test_node = self.nodes[0].add_p2p_connection(TestP2PConn(), services=NODE_NETWORK | NODE_WITNESS) + # self.test_node sets P2P_SERVICES, i.e. NODE_WITNESS | NODE_NETWORK + self.test_node = self.nodes[0].add_p2p_connection(TestP2PConn(), services=P2P_SERVICES) # self.old_node sets only NODE_NETWORK self.old_node = self.nodes[0].add_p2p_connection(TestP2PConn(), services=NODE_NETWORK) # self.std_node is for testing node1 (fRequireStandard=true) - self.std_node = self.nodes[1].add_p2p_connection(TestP2PConn(), services=NODE_NETWORK | NODE_WITNESS) + self.std_node = self.nodes[1].add_p2p_connection(TestP2PConn(), services=P2P_SERVICES) # self.std_wtx_node is for testing node1 with wtxid relay - self.std_wtx_node = self.nodes[1].add_p2p_connection(TestP2PConn(wtxidrelay=True), services=NODE_NETWORK | NODE_WITNESS) + self.std_wtx_node = self.nodes[1].add_p2p_connection(TestP2PConn(wtxidrelay=True), services=P2P_SERVICES) assert self.test_node.nServices & NODE_WITNESS != 0 @@ -267,7 +242,6 @@ class SegWitTest(BitcoinTestFramework): self.test_non_witness_transaction() self.test_v0_outputs_arent_spendable() self.test_block_relay() - self.test_getblocktemplate_before_lockin() self.test_unnecessary_witness_before_segwit_activation() self.test_witness_tx_relay_before_segwit_activation() self.test_standardness_v0() @@ -284,7 +258,7 @@ class SegWitTest(BitcoinTestFramework): self.test_submit_block() self.test_extra_witness_data() self.test_max_witness_push_length() - self.test_max_witness_program_length() + self.test_max_witness_script_length() self.test_witness_input_length() self.test_block_relay() self.test_tx_relay_after_segwit_activation() @@ -295,7 +269,6 @@ class SegWitTest(BitcoinTestFramework): self.test_signature_version_1() self.test_non_standard_witness_blinding() self.test_non_standard_witness() - self.test_upgrade_after_activation() self.test_witness_sigops() self.test_superfluous_witness() self.test_wtxid_relay() @@ -323,12 +296,12 @@ class SegWitTest(BitcoinTestFramework): # Mine a block with an anyone-can-spend coinbase, # let it mature, then try to spend it. - block = self.build_next_block(version=1) + block = self.build_next_block() block.solve() self.test_node.send_and_ping(msg_no_witness_block(block)) # make sure the block was processed txid = block.vtx[0].sha256 - self.nodes[0].generate(99) # let the block mature + self.generate(self.nodes[0], 99) # let the block mature # Create a transaction that spends the coinbase tx = CTransaction() @@ -344,7 +317,7 @@ class SegWitTest(BitcoinTestFramework): assert tx.hash in self.nodes[0].getrawmempool() # Save this transaction for later self.utxo.append(UTXO(tx.sha256, 0, 49 * 100000000)) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) @subtest # type: ignore def test_unnecessary_witness_before_segwit_activation(self): @@ -361,8 +334,8 @@ class SegWitTest(BitcoinTestFramework): tx.rehash() assert tx.sha256 != tx.calc_sha256(with_witness=True) - # Construct a segwit-signaling block that includes the transaction. - block = self.build_next_block(version=(VB_TOP_BITS | (1 << VB_WITNESS_BIT))) + # Construct a block that includes the transaction. + block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) # Sending witness data before activation is not allowed (anti-spam # rule). @@ -389,7 +362,7 @@ class SegWitTest(BitcoinTestFramework): # test_node has set NODE_WITNESS, so all getdata requests should be for # witness blocks. # Test announcing a block via inv results in a getdata, and that - # announcing a version 4 or random VB block with a header results in a getdata + # announcing a block with a header results in a getdata block1 = self.build_next_block() block1.solve() @@ -397,19 +370,13 @@ class SegWitTest(BitcoinTestFramework): assert self.test_node.last_message["getdata"].inv[0].type == blocktype test_witness_block(self.nodes[0], self.test_node, block1, True) - block2 = self.build_next_block(version=4) + block2 = self.build_next_block() block2.solve() self.test_node.announce_block_and_wait_for_getdata(block2, use_header=True) assert self.test_node.last_message["getdata"].inv[0].type == blocktype test_witness_block(self.nodes[0], self.test_node, block2, True) - block3 = self.build_next_block(version=(VB_TOP_BITS | (1 << 15))) - block3.solve() - self.test_node.announce_block_and_wait_for_getdata(block3, use_header=True) - assert self.test_node.last_message["getdata"].inv[0].type == blocktype - test_witness_block(self.nodes[0], self.test_node, block3, True) - # Check that we can getdata for witness blocks or regular blocks, # and the right thing happens. if not self.segwit_active: @@ -428,7 +395,7 @@ class SegWitTest(BitcoinTestFramework): block = self.test_node.request_block(block_hash, 2) wit_block = self.test_node.request_block(block_hash, 2 | MSG_WITNESS_FLAG) assert_equal(block.serialize(), wit_block.serialize()) - assert_equal(block.serialize(), hex_str_to_bytes(rpc_block)) + assert_equal(block.serialize(), bytes.fromhex(rpc_block)) else: # After activation, witness blocks and non-witness blocks should # be different. Verify rpc getblock() returns witness blocks, while @@ -443,7 +410,7 @@ class SegWitTest(BitcoinTestFramework): rpc_block = self.nodes[0].getblock(block.hash, False) non_wit_block = self.test_node.request_block(block.sha256, 2) wit_block = self.test_node.request_block(block.sha256, 2 | MSG_WITNESS_FLAG) - assert_equal(wit_block.serialize(), hex_str_to_bytes(rpc_block)) + assert_equal(wit_block.serialize(), bytes.fromhex(rpc_block)) assert_equal(wit_block.serialize(False), non_wit_block.serialize()) assert_equal(wit_block.serialize(), block.serialize()) @@ -451,11 +418,10 @@ class SegWitTest(BitcoinTestFramework): rpc_details = self.nodes[0].getblock(block.hash, True) assert_equal(rpc_details["size"], len(block.serialize())) assert_equal(rpc_details["strippedsize"], len(block.serialize(False))) - weight = 3 * len(block.serialize(False)) + len(block.serialize()) - assert_equal(rpc_details["weight"], weight) + assert_equal(rpc_details["weight"], block.get_weight()) # Upgraded node should not ask for blocks from unupgraded - block4 = self.build_next_block(version=4) + block4 = self.build_next_block() block4.solve() self.old_node.getdataset = set() @@ -485,18 +451,10 @@ class SegWitTest(BitcoinTestFramework): witness, and so can't be spent before segwit activation (the point at which blocks are permitted to contain witnesses).""" - # node2 doesn't need to be connected for this test. - # (If it's connected, node0 may propagate an invalid block to it over - # compact blocks and the nodes would have inconsistent tips.) - self.disconnect_nodes(0, 2) - # Create two outputs, a p2wsh and p2sh-p2wsh - witness_program = CScript([OP_TRUE]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) - - p2sh_pubkey = hash160(script_pubkey) - p2sh_script_pubkey = CScript([OP_HASH160, p2sh_pubkey, OP_EQUAL]) + witness_script = CScript([OP_TRUE]) + script_pubkey = script_to_p2wsh_script(witness_script) + p2sh_script_pubkey = script_to_p2sh_script(script_pubkey) value = self.utxo[0].nValue // 3 @@ -550,38 +508,10 @@ class SegWitTest(BitcoinTestFramework): # TODO: support multiple acceptable reject reasons. test_witness_block(self.nodes[0], self.test_node, block, accepted=False, with_witness=False) - self.connect_nodes(0, 2) - self.utxo.pop(0) self.utxo.append(UTXO(txid, 2, value)) @subtest # type: ignore - def test_getblocktemplate_before_lockin(self): - txid = int(self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1), 16) - - for node in [self.nodes[0], self.nodes[2]]: - gbt_results = node.getblocktemplate({"rules": ["segwit"]}) - if node == self.nodes[2]: - # If this is a non-segwit node, we should not get a witness - # commitment. - assert 'default_witness_commitment' not in gbt_results - else: - # For segwit-aware nodes, check the witness - # commitment is correct. - assert 'default_witness_commitment' in gbt_results - witness_commitment = gbt_results['default_witness_commitment'] - - # Check that default_witness_commitment is present. - witness_root = CBlock.get_merkle_root([ser_uint256(0), - ser_uint256(txid)]) - script = get_witness_script(witness_root, 0) - assert_equal(witness_commitment, script.hex()) - - # Clear out the mempool - self.nodes[0].generate(1) - self.sync_blocks() - - @subtest # type: ignore def test_witness_tx_relay_before_segwit_activation(self): # Generate a transaction that doesn't require a witness, but send it @@ -617,7 +547,7 @@ class SegWitTest(BitcoinTestFramework): test_transaction_acceptance(self.nodes[0], self.test_node, tx, with_witness=False, accepted=True) # Cleanup: mine the first transaction and update utxo - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) assert_equal(len(self.nodes[0].getrawmempool()), 0) self.utxo.pop(0) @@ -630,12 +560,9 @@ class SegWitTest(BitcoinTestFramework): V0 segwit outputs and inputs are always standard. V0 segwit inputs may only be mined after activation, but not before.""" - witness_program = CScript([OP_TRUE]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) - - p2sh_pubkey = hash160(witness_program) - p2sh_script_pubkey = CScript([OP_HASH160, p2sh_pubkey, OP_EQUAL]) + witness_script = CScript([OP_TRUE]) + script_pubkey = script_to_p2wsh_script(witness_script) + p2sh_script_pubkey = script_to_p2sh_script(witness_script) # First prepare a p2sh output (so that spending it will pass standardness) p2sh_tx = CTransaction() @@ -645,13 +572,13 @@ class SegWitTest(BitcoinTestFramework): # Mine it on test_node to create the confirmed output. test_transaction_acceptance(self.nodes[0], self.test_node, p2sh_tx, with_witness=True, accepted=True) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks() # Now test standardness of v0 P2WSH outputs. # Start by creating a transaction with two outputs. tx = CTransaction() - tx.vin = [CTxIn(COutPoint(p2sh_tx.sha256, 0), CScript([witness_program]))] + tx.vin = [CTxIn(COutPoint(p2sh_tx.sha256, 0), CScript([witness_script]))] tx.vout = [CTxOut(p2sh_tx.vout[0].nValue - 10000, script_pubkey)] tx.vout.append(CTxOut(8000, script_pubkey)) # Might burn this later tx.vin[0].nSequence = BIP125_SEQUENCE_NUMBER # Just to have the option to bump this tx from the mempool @@ -662,13 +589,14 @@ class SegWitTest(BitcoinTestFramework): test_transaction_acceptance(self.nodes[1], self.std_node, tx, with_witness=True, accepted=True) # Now create something that looks like a P2PKH output. This won't be spendable. + witness_hash = sha256(witness_script) script_pubkey = CScript([OP_0, hash160(witness_hash)]) tx2 = CTransaction() # tx was accepted, so we spend the second output. tx2.vin = [CTxIn(COutPoint(tx.sha256, 1), b"")] tx2.vout = [CTxOut(7000, script_pubkey)] tx2.wit.vtxinwit.append(CTxInWitness()) - tx2.wit.vtxinwit[0].scriptWitness.stack = [witness_program] + tx2.wit.vtxinwit[0].scriptWitness.stack = [witness_script] tx2.rehash() test_transaction_acceptance(self.nodes[1], self.std_node, tx2, with_witness=True, accepted=True) @@ -681,7 +609,7 @@ class SegWitTest(BitcoinTestFramework): tx3.vin = [CTxIn(COutPoint(tx.sha256, 0), b"")] tx3.vout = [CTxOut(tx.vout[0].nValue - 1000, CScript([OP_TRUE, OP_DROP] * 15 + [OP_TRUE]))] tx3.wit.vtxinwit.append(CTxInWitness()) - tx3.wit.vtxinwit[0].scriptWitness.stack = [witness_program] + tx3.wit.vtxinwit[0].scriptWitness.stack = [witness_script] tx3.rehash() if not self.segwit_active: # Just check mempool acceptance, but don't add the transaction to the mempool, since witness is disallowed @@ -717,7 +645,7 @@ class SegWitTest(BitcoinTestFramework): ) test_transaction_acceptance(self.nodes[0], self.test_node, tx3, with_witness=True, accepted=True) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks() self.utxo.pop(0) self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) @@ -728,9 +656,9 @@ class SegWitTest(BitcoinTestFramework): """Mine enough blocks to activate segwit.""" assert not softfork_active(self.nodes[0], 'segwit') height = self.nodes[0].getblockcount() - self.nodes[0].generate(SEGWIT_HEIGHT - height - 2) + self.generate(self.nodes[0], SEGWIT_HEIGHT - height - 2) assert not softfork_active(self.nodes[0], 'segwit') - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) assert softfork_active(self.nodes[0], 'segwit') self.segwit_active = True @@ -739,11 +667,9 @@ class SegWitTest(BitcoinTestFramework): """Test P2SH wrapped witness programs.""" # Prepare the p2sh-wrapped witness output - witness_program = CScript([OP_DROP, OP_TRUE]) - witness_hash = sha256(witness_program) - p2wsh_pubkey = CScript([OP_0, witness_hash]) - p2sh_witness_hash = hash160(p2wsh_pubkey) - script_pubkey = CScript([OP_HASH160, p2sh_witness_hash, OP_EQUAL]) + witness_script = CScript([OP_DROP, OP_TRUE]) + p2wsh_pubkey = script_to_p2wsh_script(witness_script) + script_pubkey = script_to_p2sh_script(p2wsh_pubkey) script_sig = CScript([p2wsh_pubkey]) # a push of the redeem script # Fund the P2SH output @@ -786,7 +712,7 @@ class SegWitTest(BitcoinTestFramework): spend_tx.vin[0].scriptSig = script_sig spend_tx.rehash() spend_tx.wit.vtxinwit.append(CTxInWitness()) - spend_tx.wit.vtxinwit[0].scriptWitness.stack = [b'a', witness_program] + spend_tx.wit.vtxinwit[0].scriptWitness.stack = [b'a', witness_script] # Verify mempool acceptance test_transaction_acceptance(self.nodes[0], self.test_node, spend_tx, with_witness=True, accepted=True) @@ -835,19 +761,18 @@ class SegWitTest(BitcoinTestFramework): tx = CTransaction() tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) - # Let's construct a witness program - witness_program = CScript([OP_TRUE]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) + # Let's construct a witness script + witness_script = CScript([OP_TRUE]) + script_pubkey = script_to_p2wsh_script(witness_script) tx.vout.append(CTxOut(self.utxo[0].nValue - 1000, script_pubkey)) tx.rehash() # tx2 will spend tx1, and send back to a regular anyone-can-spend address tx2 = CTransaction() tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) - tx2.vout.append(CTxOut(tx.vout[0].nValue - 1000, witness_program)) + tx2.vout.append(CTxOut(tx.vout[0].nValue - 1000, witness_script)) tx2.wit.vtxinwit.append(CTxInWitness()) - tx2.wit.vtxinwit[0].scriptWitness.stack = [witness_program] + tx2.wit.vtxinwit[0].scriptWitness.stack = [witness_script] tx2.rehash() block_3 = self.build_next_block() @@ -882,7 +807,7 @@ class SegWitTest(BitcoinTestFramework): block_4 = self.build_next_block() tx3 = CTransaction() tx3.vin.append(CTxIn(COutPoint(tx2.sha256, 0), b"")) - tx3.vout.append(CTxOut(tx.vout[0].nValue - 1000, witness_program)) + tx3.vout.append(CTxOut(tx.vout[0].nValue - 1000, witness_script)) tx3.rehash() block_4.vtx.append(tx3) block_4.hashMerkleRoot = block_4.calc_merkle_root() @@ -904,7 +829,7 @@ class SegWitTest(BitcoinTestFramework): block.solve() block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.append(b'a' * 5000000) - assert get_virtual_size(block) > MAX_BLOCK_BASE_SIZE + assert block.get_weight() > MAX_BLOCK_WEIGHT # We can't send over the p2p network, because this is too big to relay # TODO: repeat this test with a block that can be relayed @@ -913,7 +838,7 @@ class SegWitTest(BitcoinTestFramework): assert self.nodes[0].getbestblockhash() != block.hash block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.pop() - assert get_virtual_size(block) < MAX_BLOCK_BASE_SIZE + assert block.get_weight() < MAX_BLOCK_WEIGHT assert_equal(None, self.nodes[0].submitblock(block.serialize().hex())) assert self.nodes[0].getbestblockhash() == block.hash @@ -944,15 +869,14 @@ class SegWitTest(BitcoinTestFramework): assert len(self.utxo) > 0 # Create a P2WSH transaction. - # The witness program will be a bunch of OP_2DROP's, followed by OP_TRUE. + # The witness script will be a bunch of OP_2DROP's, followed by OP_TRUE. # This should give us plenty of room to tweak the spending tx's # virtual size. NUM_DROPS = 200 # 201 max ops per script! NUM_OUTPUTS = 50 - witness_program = CScript([OP_2DROP] * NUM_DROPS + [OP_TRUE]) - witness_hash = uint256_from_str(sha256(witness_program)) - script_pubkey = CScript([OP_0, ser_uint256(witness_hash)]) + witness_script = CScript([OP_2DROP] * NUM_DROPS + [OP_TRUE]) + script_pubkey = script_to_p2wsh_script(witness_script) prevout = COutPoint(self.utxo[0].sha256, self.utxo[0].n) value = self.utxo[0].nValue @@ -972,15 +896,14 @@ class SegWitTest(BitcoinTestFramework): child_tx.vout = [CTxOut(value - 100000, CScript([OP_TRUE]))] for _ in range(NUM_OUTPUTS): child_tx.wit.vtxinwit.append(CTxInWitness()) - child_tx.wit.vtxinwit[-1].scriptWitness.stack = [b'a' * 195] * (2 * NUM_DROPS) + [witness_program] + child_tx.wit.vtxinwit[-1].scriptWitness.stack = [b'a' * 195] * (2 * NUM_DROPS) + [witness_script] child_tx.rehash() self.update_witness_block_with_transactions(block, [parent_tx, child_tx]) - vsize = get_virtual_size(block) - additional_bytes = (MAX_BLOCK_BASE_SIZE - vsize) * 4 + additional_bytes = MAX_BLOCK_WEIGHT - block.get_weight() i = 0 while additional_bytes > 0: - # Add some more bytes to each input until we hit MAX_BLOCK_BASE_SIZE+1 + # Add some more bytes to each input until we hit MAX_BLOCK_WEIGHT+1 extra_bytes = min(additional_bytes + 1, 55) block.vtx[-1].wit.vtxinwit[int(i / (2 * NUM_DROPS))].scriptWitness.stack[i % (2 * NUM_DROPS)] = b'a' * (195 + extra_bytes) additional_bytes -= extra_bytes @@ -989,8 +912,7 @@ class SegWitTest(BitcoinTestFramework): block.vtx[0].vout.pop() # Remove old commitment add_witness_commitment(block) block.solve() - vsize = get_virtual_size(block) - assert_equal(vsize, MAX_BLOCK_BASE_SIZE + 1) + assert_equal(block.get_weight(), MAX_BLOCK_WEIGHT + 1) # Make sure that our test case would exceed the old max-network-message # limit assert len(block.serialize()) > 2 * 1024 * 1024 @@ -1003,7 +925,7 @@ class SegWitTest(BitcoinTestFramework): block.vtx[0].vout.pop() add_witness_commitment(block) block.solve() - assert get_virtual_size(block) == MAX_BLOCK_BASE_SIZE + assert block.get_weight() == MAX_BLOCK_WEIGHT test_witness_block(self.nodes[0], self.test_node, block, accepted=True) @@ -1053,9 +975,8 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() - witness_program = CScript([OP_DROP, OP_TRUE]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) + witness_script = CScript([OP_DROP, OP_TRUE]) + script_pubkey = script_to_p2wsh_script(witness_script) # First try extra witness data on a tx that doesn't require a witness tx = CTransaction() @@ -1086,7 +1007,7 @@ class SegWitTest(BitcoinTestFramework): tx2.vin.append(CTxIn(COutPoint(tx.sha256, 1), b"")) # non-witness tx2.vout.append(CTxOut(tx.vout[0].nValue, CScript([OP_TRUE]))) tx2.wit.vtxinwit.extend([CTxInWitness(), CTxInWitness()]) - tx2.wit.vtxinwit[0].scriptWitness.stack = [CScript([CScriptNum(1)]), CScript([CScriptNum(1)]), witness_program] + tx2.wit.vtxinwit[0].scriptWitness.stack = [CScript([CScriptNum(1)]), CScript([CScriptNum(1)]), witness_script] tx2.wit.vtxinwit[1].scriptWitness.stack = [CScript([OP_TRUE])] block = self.build_next_block() @@ -1126,9 +1047,8 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() - witness_program = CScript([OP_DROP, OP_TRUE]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) + witness_script = CScript([OP_DROP, OP_TRUE]) + script_pubkey = script_to_p2wsh_script(witness_script) tx = CTransaction() tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) @@ -1140,7 +1060,7 @@ class SegWitTest(BitcoinTestFramework): tx2.vout.append(CTxOut(tx.vout[0].nValue - 1000, CScript([OP_TRUE]))) tx2.wit.vtxinwit.append(CTxInWitness()) # First try a 521-byte stack element - tx2.wit.vtxinwit[0].scriptWitness.stack = [b'a' * (MAX_SCRIPT_ELEMENT_SIZE + 1), witness_program] + tx2.wit.vtxinwit[0].scriptWitness.stack = [b'a' * (MAX_SCRIPT_ELEMENT_SIZE + 1), witness_script] tx2.rehash() self.update_witness_block_with_transactions(block, [tx, tx2]) @@ -1158,16 +1078,15 @@ class SegWitTest(BitcoinTestFramework): self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) @subtest # type: ignore - def test_max_witness_program_length(self): + def test_max_witness_script_length(self): """Test that witness outputs greater than 10kB can't be spent.""" - MAX_PROGRAM_LENGTH = 10000 + MAX_WITNESS_SCRIPT_LENGTH = 10000 - # This program is 19 max pushes (9937 bytes), then 64 more opcode-bytes. - long_witness_program = CScript([b'a' * MAX_SCRIPT_ELEMENT_SIZE] * 19 + [OP_DROP] * 63 + [OP_TRUE]) - assert len(long_witness_program) == MAX_PROGRAM_LENGTH + 1 - long_witness_hash = sha256(long_witness_program) - long_script_pubkey = CScript([OP_0, long_witness_hash]) + # This script is 19 max pushes (9937 bytes), then 64 more opcode-bytes. + long_witness_script = CScript([b'a' * MAX_SCRIPT_ELEMENT_SIZE] * 19 + [OP_DROP] * 63 + [OP_TRUE]) + assert len(long_witness_script) == MAX_WITNESS_SCRIPT_LENGTH + 1 + long_script_pubkey = script_to_p2wsh_script(long_witness_script) block = self.build_next_block() @@ -1180,23 +1099,22 @@ class SegWitTest(BitcoinTestFramework): tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) tx2.vout.append(CTxOut(tx.vout[0].nValue - 1000, CScript([OP_TRUE]))) tx2.wit.vtxinwit.append(CTxInWitness()) - tx2.wit.vtxinwit[0].scriptWitness.stack = [b'a'] * 44 + [long_witness_program] + tx2.wit.vtxinwit[0].scriptWitness.stack = [b'a'] * 44 + [long_witness_script] tx2.rehash() self.update_witness_block_with_transactions(block, [tx, tx2]) test_witness_block(self.nodes[0], self.test_node, block, accepted=False) - # Try again with one less byte in the witness program - witness_program = CScript([b'a' * MAX_SCRIPT_ELEMENT_SIZE] * 19 + [OP_DROP] * 62 + [OP_TRUE]) - assert len(witness_program) == MAX_PROGRAM_LENGTH - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) + # Try again with one less byte in the witness script + witness_script = CScript([b'a' * MAX_SCRIPT_ELEMENT_SIZE] * 19 + [OP_DROP] * 62 + [OP_TRUE]) + assert len(witness_script) == MAX_WITNESS_SCRIPT_LENGTH + script_pubkey = script_to_p2wsh_script(witness_script) tx.vout[0] = CTxOut(tx.vout[0].nValue, script_pubkey) tx.rehash() tx2.vin[0].prevout.hash = tx.sha256 - tx2.wit.vtxinwit[0].scriptWitness.stack = [b'a'] * 43 + [witness_program] + tx2.wit.vtxinwit[0].scriptWitness.stack = [b'a'] * 43 + [witness_script] tx2.rehash() block.vtx = [block.vtx[0]] self.update_witness_block_with_transactions(block, [tx, tx2]) @@ -1209,9 +1127,8 @@ class SegWitTest(BitcoinTestFramework): def test_witness_input_length(self): """Test that vin length must match vtxinwit length.""" - witness_program = CScript([OP_DROP, OP_TRUE]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) + witness_script = CScript([OP_DROP, OP_TRUE]) + script_pubkey = script_to_p2wsh_script(witness_script) # Create a transaction that splits our utxo into many outputs tx = CTransaction() @@ -1255,7 +1172,7 @@ class SegWitTest(BitcoinTestFramework): # First try using a too long vtxinwit for i in range(11): tx2.wit.vtxinwit.append(CTxInWitness()) - tx2.wit.vtxinwit[i].scriptWitness.stack = [b'a', witness_program] + tx2.wit.vtxinwit[i].scriptWitness.stack = [b'a', witness_script] block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx2]) @@ -1271,15 +1188,15 @@ class SegWitTest(BitcoinTestFramework): # Now make one of the intermediate witnesses be incorrect tx2.wit.vtxinwit.append(CTxInWitness()) - tx2.wit.vtxinwit[-1].scriptWitness.stack = [b'a', witness_program] - tx2.wit.vtxinwit[5].scriptWitness.stack = [witness_program] + tx2.wit.vtxinwit[-1].scriptWitness.stack = [b'a', witness_script] + tx2.wit.vtxinwit[5].scriptWitness.stack = [witness_script] block.vtx = [block.vtx[0]] self.update_witness_block_with_transactions(block, [tx2]) test_witness_block(self.nodes[0], self.test_node, block, accepted=False) # Fix the broken witness and the block should be accepted. - tx2.wit.vtxinwit[5].scriptWitness.stack = [b'a', witness_program] + tx2.wit.vtxinwit[5].scriptWitness.stack = [b'a', witness_script] block.vtx = [block.vtx[0]] self.update_witness_block_with_transactions(block, [tx2]) test_witness_block(self.nodes[0], self.test_node, block, accepted=True) @@ -1317,9 +1234,8 @@ class SegWitTest(BitcoinTestFramework): test_transaction_acceptance(self.nodes[0], self.test_node, tx, with_witness=False, accepted=True) # Now try to add extra witness data to a valid witness tx. - witness_program = CScript([OP_TRUE]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) + witness_script = CScript([OP_TRUE]) + script_pubkey = script_to_p2wsh_script(witness_script) tx2 = CTransaction() tx2.vin.append(CTxIn(COutPoint(tx_hash, 0), b"")) tx2.vout.append(CTxOut(tx.vout[0].nValue - 1000, script_pubkey)) @@ -1330,11 +1246,10 @@ class SegWitTest(BitcoinTestFramework): tx3.wit.vtxinwit.append(CTxInWitness()) # Add too-large for IsStandard witness and check that it does not enter reject filter - p2sh_program = CScript([OP_TRUE]) - p2sh_pubkey = hash160(p2sh_program) - witness_program2 = CScript([b'a' * 400000]) - tx3.vout.append(CTxOut(tx2.vout[0].nValue - 1000, CScript([OP_HASH160, p2sh_pubkey, OP_EQUAL]))) - tx3.wit.vtxinwit[0].scriptWitness.stack = [witness_program2] + p2sh_script = CScript([OP_TRUE]) + witness_script2 = CScript([b'a' * 400000]) + tx3.vout.append(CTxOut(tx2.vout[0].nValue - 1000, script_to_p2sh_script(p2sh_script))) + tx3.wit.vtxinwit[0].scriptWitness.stack = [witness_script2] tx3.rehash() # Node will not be blinded to the transaction, requesting it any number of times @@ -1348,14 +1263,14 @@ class SegWitTest(BitcoinTestFramework): # Remove witness stuffing, instead add extra witness push on stack tx3.vout[0] = CTxOut(tx2.vout[0].nValue - 1000, CScript([OP_TRUE, OP_DROP] * 15 + [OP_TRUE])) - tx3.wit.vtxinwit[0].scriptWitness.stack = [CScript([CScriptNum(1)]), witness_program] + tx3.wit.vtxinwit[0].scriptWitness.stack = [CScript([CScriptNum(1)]), witness_script] tx3.rehash() test_transaction_acceptance(self.nodes[0], self.test_node, tx2, with_witness=True, accepted=True) test_transaction_acceptance(self.nodes[0], self.test_node, tx3, with_witness=True, accepted=False) # Get rid of the extra witness, and verify acceptance. - tx3.wit.vtxinwit[0].scriptWitness.stack = [witness_program] + tx3.wit.vtxinwit[0].scriptWitness.stack = [witness_script] # Also check that old_node gets a tx announcement, even though this is # a witness transaction. self.old_node.wait_for_inv([CInv(MSG_TX, tx2.sha256)]) # wait until tx2 was inv'ed @@ -1367,16 +1282,15 @@ class SegWitTest(BitcoinTestFramework): raw_tx = self.nodes[0].getrawtransaction(tx3.hash, 1) assert_equal(int(raw_tx["hash"], 16), tx3.calc_sha256(True)) assert_equal(raw_tx["size"], len(tx3.serialize_with_witness())) - weight = len(tx3.serialize_with_witness()) + 3 * len(tx3.serialize_without_witness()) - vsize = math.ceil(weight / 4) + vsize = tx3.get_vsize() assert_equal(raw_tx["vsize"], vsize) - assert_equal(raw_tx["weight"], weight) + assert_equal(raw_tx["weight"], tx3.get_weight()) assert_equal(len(raw_tx["vin"][0]["txinwitness"]), 1) - assert_equal(raw_tx["vin"][0]["txinwitness"][0], witness_program.hex()) + assert_equal(raw_tx["vin"][0]["txinwitness"][0], witness_script.hex()) assert vsize != raw_tx["size"] # Cleanup: mine the transactions and update utxo for next test - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) assert_equal(len(self.nodes[0].getrawmempool()), 0) self.utxo.pop(0) @@ -1408,8 +1322,8 @@ class SegWitTest(BitcoinTestFramework): self.sync_blocks() temp_utxo = [] tx = CTransaction() - witness_program = CScript([OP_TRUE]) - witness_hash = sha256(witness_program) + witness_script = CScript([OP_TRUE]) + witness_hash = sha256(witness_script) assert_equal(len(self.nodes[1].getrawmempool()), 0) for version in list(range(OP_1, OP_16 + 1)) + [OP_0]: # First try to spend to a future version segwit script_pubkey. @@ -1426,7 +1340,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) temp_utxo.append(UTXO(tx.sha256, 0, tx.vout[0].nValue)) - self.nodes[0].generate(1) # Mine all the transactions + self.generate(self.nodes[0], 1) # Mine all the transactions self.sync_blocks() assert len(self.nodes[0].getrawmempool()) == 0 @@ -1437,7 +1351,7 @@ class SegWitTest(BitcoinTestFramework): tx2.vin = [CTxIn(COutPoint(tx.sha256, 0), b"")] tx2.vout = [CTxOut(tx.vout[0].nValue - 1000, script_pubkey)] tx2.wit.vtxinwit.append(CTxInWitness()) - tx2.wit.vtxinwit[0].scriptWitness.stack = [witness_program] + tx2.wit.vtxinwit[0].scriptWitness.stack = [witness_script] tx2.rehash() # Gets accepted to both policy-enforcing nodes and others. test_transaction_acceptance(self.nodes[0], self.test_node, tx2, with_witness=True, accepted=True) @@ -1452,7 +1366,7 @@ class SegWitTest(BitcoinTestFramework): tx3.vin.append(CTxIn(COutPoint(i.sha256, i.n), b"")) tx3.wit.vtxinwit.append(CTxInWitness()) total_value += i.nValue - tx3.wit.vtxinwit[-1].scriptWitness.stack = [witness_program] + tx3.wit.vtxinwit[-1].scriptWitness.stack = [witness_script] tx3.vout.append(CTxOut(total_value - 1000, script_pubkey)) tx3.rehash() @@ -1481,9 +1395,8 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() # Change the output of the block to be a witness output. - witness_program = CScript([OP_TRUE]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) + witness_script = CScript([OP_TRUE]) + script_pubkey = script_to_p2wsh_script(witness_script) block.vtx[0].vout[0].scriptPubKey = script_pubkey # This next line will rehash the coinbase and update the merkle # root, and solve. @@ -1492,20 +1405,20 @@ class SegWitTest(BitcoinTestFramework): spend_tx = CTransaction() spend_tx.vin = [CTxIn(COutPoint(block.vtx[0].sha256, 0), b"")] - spend_tx.vout = [CTxOut(block.vtx[0].vout[0].nValue, witness_program)] + spend_tx.vout = [CTxOut(block.vtx[0].vout[0].nValue, witness_script)] spend_tx.wit.vtxinwit.append(CTxInWitness()) - spend_tx.wit.vtxinwit[0].scriptWitness.stack = [witness_program] + spend_tx.wit.vtxinwit[0].scriptWitness.stack = [witness_script] spend_tx.rehash() # Now test a premature spend. - self.nodes[0].generate(98) + self.generate(self.nodes[0], 98) self.sync_blocks() block2 = self.build_next_block() self.update_witness_block_with_transactions(block2, [spend_tx]) test_witness_block(self.nodes[0], self.test_node, block2, accepted=False) # Advancing one more block should allow the spend. - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) block2 = self.build_next_block() self.update_witness_block_with_transactions(block2, [spend_tx]) test_witness_block(self.nodes[0], self.test_node, block2, accepted=True) @@ -1530,7 +1443,7 @@ class SegWitTest(BitcoinTestFramework): # Test 1: P2WPKH # First create a P2WPKH output that uses an uncompressed pubkey pubkeyhash = hash160(pubkey) - script_pkh = CScript([OP_0, pubkeyhash]) + script_pkh = key_to_p2wpkh_script(pubkey) tx = CTransaction() tx.vin.append(CTxIn(COutPoint(utxo.sha256, utxo.n), b"")) tx.vout.append(CTxOut(utxo.nValue - 1000, script_pkh)) @@ -1543,14 +1456,13 @@ class SegWitTest(BitcoinTestFramework): # Now try to spend it. Send it to a P2WSH output, which we'll # use in the next test. - witness_program = CScript([pubkey, CScriptOp(OP_CHECKSIG)]) - witness_hash = sha256(witness_program) - script_wsh = CScript([OP_0, witness_hash]) + witness_script = key_to_p2pk_script(pubkey) + script_wsh = script_to_p2wsh_script(witness_script) tx2 = CTransaction() tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) tx2.vout.append(CTxOut(tx.vout[0].nValue - 1000, script_wsh)) - script = get_p2pkh_script(pubkeyhash) + script = keyhash_to_p2pkh_script(pubkeyhash) sig_hash = SegwitV0SignatureHash(script, tx2, 0, SIGHASH_ALL, tx.vout[0].nValue) signature = key.sign_ecdsa(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL tx2.wit.vtxinwit.append(CTxInWitness()) @@ -1567,15 +1479,14 @@ class SegWitTest(BitcoinTestFramework): # Test 2: P2WSH # Try to spend the P2WSH output created in last test. # Send it to a P2SH(P2WSH) output, which we'll use in the next test. - p2sh_witness_hash = hash160(script_wsh) - script_p2sh = CScript([OP_HASH160, p2sh_witness_hash, OP_EQUAL]) + script_p2sh = script_to_p2sh_script(script_wsh) script_sig = CScript([script_wsh]) tx3 = CTransaction() tx3.vin.append(CTxIn(COutPoint(tx2.sha256, 0), b"")) tx3.vout.append(CTxOut(tx2.vout[0].nValue - 1000, script_p2sh)) tx3.wit.vtxinwit.append(CTxInWitness()) - sign_p2pk_witness_input(witness_program, tx3, 0, SIGHASH_ALL, tx2.vout[0].nValue, key) + sign_p2pk_witness_input(witness_script, tx3, 0, SIGHASH_ALL, tx2.vout[0].nValue, key) # Should fail policy test. test_transaction_acceptance(self.nodes[0], self.test_node, tx3, True, False, 'non-mandatory-script-verify-flag (Using non-compressed keys in segwit)') @@ -1587,12 +1498,12 @@ class SegWitTest(BitcoinTestFramework): # Test 3: P2SH(P2WSH) # Try to spend the P2SH output created in the last test. # Send it to a P2PKH output, which we'll use in the next test. - script_pubkey = get_p2pkh_script(pubkeyhash) + script_pubkey = keyhash_to_p2pkh_script(pubkeyhash) tx4 = CTransaction() tx4.vin.append(CTxIn(COutPoint(tx3.sha256, 0), script_sig)) tx4.vout.append(CTxOut(tx3.vout[0].nValue - 1000, script_pubkey)) tx4.wit.vtxinwit.append(CTxInWitness()) - sign_p2pk_witness_input(witness_program, tx4, 0, SIGHASH_ALL, tx3.vout[0].nValue, key) + sign_p2pk_witness_input(witness_script, tx4, 0, SIGHASH_ALL, tx3.vout[0].nValue, key) # Should fail policy test. test_transaction_acceptance(self.nodes[0], self.test_node, tx4, True, False, 'non-mandatory-script-verify-flag (Using non-compressed keys in segwit)') @@ -1623,9 +1534,8 @@ class SegWitTest(BitcoinTestFramework): key.generate() pubkey = key.get_pubkey().get_bytes() - witness_program = CScript([pubkey, CScriptOp(OP_CHECKSIG)]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) + witness_script = key_to_p2pk_script(pubkey) + script_pubkey = script_to_p2wsh_script(witness_script) # First create a witness output for use in the tests. tx = CTransaction() @@ -1652,18 +1562,18 @@ class SegWitTest(BitcoinTestFramework): tx.vout.append(CTxOut(prev_utxo.nValue - 1000, script_pubkey)) tx.wit.vtxinwit.append(CTxInWitness()) # Too-large input value - sign_p2pk_witness_input(witness_program, tx, 0, hashtype, prev_utxo.nValue + 1, key) + sign_p2pk_witness_input(witness_script, tx, 0, hashtype, prev_utxo.nValue + 1, key) self.update_witness_block_with_transactions(block, [tx]) test_witness_block(self.nodes[0], self.test_node, block, accepted=False) # Too-small input value - sign_p2pk_witness_input(witness_program, tx, 0, hashtype, prev_utxo.nValue - 1, key) + sign_p2pk_witness_input(witness_script, tx, 0, hashtype, prev_utxo.nValue - 1, key) block.vtx.pop() # remove last tx self.update_witness_block_with_transactions(block, [tx]) test_witness_block(self.nodes[0], self.test_node, block, accepted=False) # Now try correct value - sign_p2pk_witness_input(witness_program, tx, 0, hashtype, prev_utxo.nValue, key) + sign_p2pk_witness_input(witness_script, tx, 0, hashtype, prev_utxo.nValue, key) block.vtx.pop() self.update_witness_block_with_transactions(block, [tx]) test_witness_block(self.nodes[0], self.test_node, block, accepted=True) @@ -1684,7 +1594,7 @@ class SegWitTest(BitcoinTestFramework): for _ in range(NUM_SIGHASH_TESTS): tx.vout.append(CTxOut(split_value, script_pubkey)) tx.wit.vtxinwit.append(CTxInWitness()) - sign_p2pk_witness_input(witness_program, tx, 0, SIGHASH_ALL, prev_utxo.nValue, key) + sign_p2pk_witness_input(witness_script, tx, 0, SIGHASH_ALL, prev_utxo.nValue, key) for i in range(NUM_SIGHASH_TESTS): temp_utxos.append(UTXO(tx.sha256, i, split_value)) @@ -1719,7 +1629,7 @@ class SegWitTest(BitcoinTestFramework): if random.randint(0, 1): anyonecanpay = SIGHASH_ANYONECANPAY hashtype = random.randint(1, 3) | anyonecanpay - sign_p2pk_witness_input(witness_program, tx, i, hashtype, temp_utxos[i].nValue, key) + sign_p2pk_witness_input(witness_script, tx, i, hashtype, temp_utxos[i].nValue, key) if (hashtype == SIGHASH_SINGLE and i >= num_outputs): used_sighash_single_out_of_bounds = True tx.rehash() @@ -1730,7 +1640,7 @@ class SegWitTest(BitcoinTestFramework): block.vtx.append(tx) # Test the block periodically, if we're close to maxblocksize - if (get_virtual_size(block) > MAX_BLOCK_BASE_SIZE - 1000): + if block.get_weight() > MAX_BLOCK_WEIGHT - 4000: self.update_witness_block_with_transactions(block, []) test_witness_block(self.nodes[0], self.test_node, block, accepted=True) block = self.build_next_block() @@ -1744,17 +1654,17 @@ class SegWitTest(BitcoinTestFramework): # Now test witness version 0 P2PKH transactions pubkeyhash = hash160(pubkey) - script_pkh = CScript([OP_0, pubkeyhash]) + script_pkh = key_to_p2wpkh_script(pubkey) tx = CTransaction() tx.vin.append(CTxIn(COutPoint(temp_utxos[0].sha256, temp_utxos[0].n), b"")) tx.vout.append(CTxOut(temp_utxos[0].nValue, script_pkh)) tx.wit.vtxinwit.append(CTxInWitness()) - sign_p2pk_witness_input(witness_program, tx, 0, SIGHASH_ALL, temp_utxos[0].nValue, key) + sign_p2pk_witness_input(witness_script, tx, 0, SIGHASH_ALL, temp_utxos[0].nValue, key) tx2 = CTransaction() tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) tx2.vout.append(CTxOut(tx.vout[0].nValue, CScript([OP_TRUE]))) - script = get_p2pkh_script(pubkeyhash) + script = keyhash_to_p2pkh_script(pubkeyhash) sig_hash = SegwitV0SignatureHash(script, tx2, 0, SIGHASH_ALL, tx.vout[0].nValue) signature = key.sign_ecdsa(sig_hash) + b'\x01' # 0x1 is SIGHASH_ALL @@ -1789,7 +1699,7 @@ class SegWitTest(BitcoinTestFramework): # the signatures as we go. tx.vin.append(CTxIn(COutPoint(i.sha256, i.n), b"")) tx.wit.vtxinwit.append(CTxInWitness()) - sign_p2pk_witness_input(witness_program, tx, index, SIGHASH_ALL | SIGHASH_ANYONECANPAY, i.nValue, key) + sign_p2pk_witness_input(witness_script, tx, index, SIGHASH_ALL | SIGHASH_ANYONECANPAY, i.nValue, key) index += 1 block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx]) @@ -1806,8 +1716,7 @@ class SegWitTest(BitcoinTestFramework): # rules (an anyone-can-spend OP_TRUE would be rejected, if not wrapped # in P2SH). p2sh_program = CScript([OP_TRUE]) - p2sh_pubkey = hash160(p2sh_program) - script_pubkey = CScript([OP_HASH160, p2sh_pubkey, OP_EQUAL]) + script_pubkey = script_to_p2sh_script(p2sh_program) # Now check that unnecessary witnesses can't be used to blind a node # to a transaction, eg by violating standardness checks. @@ -1816,7 +1725,7 @@ class SegWitTest(BitcoinTestFramework): tx.vout.append(CTxOut(self.utxo[0].nValue - 1000, script_pubkey)) tx.rehash() test_transaction_acceptance(self.nodes[0], self.test_node, tx, False, True) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks() # We'll add an unnecessary witness to this transaction that would cause @@ -1845,7 +1754,7 @@ class SegWitTest(BitcoinTestFramework): test_transaction_acceptance(self.nodes[0], self.test_node, tx2, False, True) test_transaction_acceptance(self.nodes[0], self.test_node, tx3, False, True) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks() # Update our utxo list; we spent the first entry. @@ -1872,16 +1781,15 @@ class SegWitTest(BitcoinTestFramework): # For each script, generate a pair of P2WSH and P2SH-P2WSH output. outputvalue = (self.utxo[0].nValue - 1000) // (len(scripts) * 2) for i in scripts: - p2wsh = CScript([OP_0, sha256(i)]) - p2sh = hash160(p2wsh) + p2wsh = script_to_p2wsh_script(i) p2wsh_scripts.append(p2wsh) tx.vout.append(CTxOut(outputvalue, p2wsh)) - tx.vout.append(CTxOut(outputvalue, CScript([OP_HASH160, p2sh, OP_EQUAL]))) + tx.vout.append(CTxOut(outputvalue, script_to_p2sh_script(p2wsh))) tx.rehash() txid = tx.sha256 test_transaction_acceptance(self.nodes[0], self.test_node, tx, with_witness=False, accepted=True) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks() # Creating transactions for tests @@ -1890,13 +1798,13 @@ class SegWitTest(BitcoinTestFramework): for i in range(len(scripts)): p2wsh_tx = CTransaction() p2wsh_tx.vin.append(CTxIn(COutPoint(txid, i * 2))) - p2wsh_tx.vout.append(CTxOut(outputvalue - 5000, CScript([OP_0, hash160(hex_str_to_bytes(""))]))) + p2wsh_tx.vout.append(CTxOut(outputvalue - 5000, CScript([OP_0, hash160(b"")]))) p2wsh_tx.wit.vtxinwit.append(CTxInWitness()) p2wsh_tx.rehash() p2wsh_txs.append(p2wsh_tx) p2sh_tx = CTransaction() p2sh_tx.vin.append(CTxIn(COutPoint(txid, i * 2 + 1), CScript([p2wsh_scripts[i]]))) - p2sh_tx.vout.append(CTxOut(outputvalue - 5000, CScript([OP_0, hash160(hex_str_to_bytes(""))]))) + p2sh_tx.vout.append(CTxOut(outputvalue - 5000, CScript([OP_0, hash160(b"")]))) p2sh_tx.wit.vtxinwit.append(CTxInWitness()) p2sh_tx.rehash() p2sh_txs.append(p2sh_tx) @@ -1944,7 +1852,7 @@ class SegWitTest(BitcoinTestFramework): test_transaction_acceptance(self.nodes[1], self.std_node, p2sh_txs[3], True, False, 'bad-witness-nonstandard') test_transaction_acceptance(self.nodes[0], self.test_node, p2sh_txs[3], True, True) - self.nodes[0].generate(1) # Mine and clean up the mempool of non-standard node + self.generate(self.nodes[0], 1) # Mine and clean up the mempool of non-standard node # Valid but non-standard transactions in a block should be accepted by standard node self.sync_blocks() assert_equal(len(self.nodes[0].getrawmempool()), 0) @@ -1953,46 +1861,12 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) @subtest # type: ignore - def test_upgrade_after_activation(self): - """Test the behavior of starting up a segwit-aware node after the softfork has activated.""" - - # All nodes are caught up and node 2 is a pre-segwit node that will soon upgrade. - for n in range(2): - assert_equal(self.nodes[n].getblockcount(), self.nodes[2].getblockcount()) - assert softfork_active(self.nodes[n], "segwit") - assert SEGWIT_HEIGHT < self.nodes[2].getblockcount() - assert 'segwit' not in self.nodes[2].getblockchaininfo()['softforks'] - - # Restarting node 2 should result in a shutdown because the blockchain consists of - # insufficiently validated blocks per segwit consensus rules. - self.stop_node(2) - self.nodes[2].assert_start_raises_init_error( - extra_args=[f"-segwitheight={SEGWIT_HEIGHT}"], - expected_msg=f": Witness data for blocks after height {SEGWIT_HEIGHT} requires validation. Please restart with -reindex..\nPlease restart with -reindex or -reindex-chainstate to recover.", - ) - - # As directed, the user restarts the node with -reindex - self.start_node(2, extra_args=["-reindex", f"-segwitheight={SEGWIT_HEIGHT}"]) - - # With the segwit consensus rules, the node is able to validate only up to SEGWIT_HEIGHT - 1 - assert_equal(self.nodes[2].getblockcount(), SEGWIT_HEIGHT - 1) - self.connect_nodes(0, 2) - - # We reconnect more than 100 blocks, give it plenty of time - # sync_blocks() also verifies the best block hash is the same for all nodes - self.sync_blocks(timeout=240) - - # The upgraded node should now have segwit activated - assert softfork_active(self.nodes[2], "segwit") - - @subtest # type: ignore def test_witness_sigops(self): """Test sigop counting is correct inside witnesses.""" # Keep this under MAX_OPS_PER_SCRIPT (201) - witness_program = CScript([OP_TRUE, OP_IF, OP_TRUE, OP_ELSE] + [OP_CHECKMULTISIG] * 5 + [OP_CHECKSIG] * 193 + [OP_ENDIF]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) + witness_script = CScript([OP_TRUE, OP_IF, OP_TRUE, OP_ELSE] + [OP_CHECKMULTISIG] * 5 + [OP_CHECKSIG] * 193 + [OP_ENDIF]) + script_pubkey = script_to_p2wsh_script(witness_script) sigops_per_script = 20 * 5 + 193 * 1 # We'll produce 2 extra outputs, one with a program that would take us @@ -2007,15 +1881,13 @@ class SegWitTest(BitcoinTestFramework): # This script, when spent with the first # N(=MAX_SIGOP_COST//sigops_per_script) outputs of our transaction, # would push us just over the block sigop limit. - witness_program_toomany = CScript([OP_TRUE, OP_IF, OP_TRUE, OP_ELSE] + [OP_CHECKSIG] * (extra_sigops_available + 1) + [OP_ENDIF]) - witness_hash_toomany = sha256(witness_program_toomany) - script_pubkey_toomany = CScript([OP_0, witness_hash_toomany]) + witness_script_toomany = CScript([OP_TRUE, OP_IF, OP_TRUE, OP_ELSE] + [OP_CHECKSIG] * (extra_sigops_available + 1) + [OP_ENDIF]) + script_pubkey_toomany = script_to_p2wsh_script(witness_script_toomany) # If we spend this script instead, we would exactly reach our sigop # limit (for witness sigops). - witness_program_justright = CScript([OP_TRUE, OP_IF, OP_TRUE, OP_ELSE] + [OP_CHECKSIG] * (extra_sigops_available) + [OP_ENDIF]) - witness_hash_justright = sha256(witness_program_justright) - script_pubkey_justright = CScript([OP_0, witness_hash_justright]) + witness_script_justright = CScript([OP_TRUE, OP_IF, OP_TRUE, OP_ELSE] + [OP_CHECKSIG] * (extra_sigops_available) + [OP_ENDIF]) + script_pubkey_justright = script_to_p2wsh_script(witness_script_justright) # First split our available utxo into a bunch of outputs split_value = self.utxo[0].nValue // outputs @@ -2038,9 +1910,9 @@ class SegWitTest(BitcoinTestFramework): for i in range(outputs - 1): tx2.vin.append(CTxIn(COutPoint(tx.sha256, i), b"")) tx2.wit.vtxinwit.append(CTxInWitness()) - tx2.wit.vtxinwit[-1].scriptWitness.stack = [witness_program] + tx2.wit.vtxinwit[-1].scriptWitness.stack = [witness_script] total_value += tx.vout[i].nValue - tx2.wit.vtxinwit[-1].scriptWitness.stack = [witness_program_toomany] + tx2.wit.vtxinwit[-1].scriptWitness.stack = [witness_script_toomany] tx2.vout.append(CTxOut(total_value, CScript([OP_TRUE]))) tx2.rehash() @@ -2079,7 +1951,7 @@ class SegWitTest(BitcoinTestFramework): tx2.vout.pop() tx2.vin.append(CTxIn(COutPoint(tx.sha256, outputs - 1), b"")) tx2.wit.vtxinwit.append(CTxInWitness()) - tx2.wit.vtxinwit[-1].scriptWitness.stack = [witness_program_justright] + tx2.wit.vtxinwit[-1].scriptWitness.stack = [witness_script_justright] tx2.rehash() self.update_witness_block_with_transactions(block_5, [tx2]) test_witness_block(self.nodes[0], self.test_node, block_5, accepted=True) @@ -2118,7 +1990,7 @@ class SegWitTest(BitcoinTestFramework): return serialize_with_bogus_witness(self.tx) self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(address_type='bech32'), 5) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) unspent = next(u for u in self.nodes[0].listunspent() if u['spendable'] and u['address'].startswith('bcrt')) raw = self.nodes[0].createrawtransaction([{"txid": unspent['txid'], "vout": unspent['vout']}], {self.nodes[0].getnewaddress(): 1}) @@ -2137,8 +2009,8 @@ class SegWitTest(BitcoinTestFramework): @subtest # type: ignore def test_wtxid_relay(self): # Use brand new nodes to avoid contamination from earlier tests - self.wtx_node = self.nodes[0].add_p2p_connection(TestP2PConn(wtxidrelay=True), services=NODE_NETWORK | NODE_WITNESS) - self.tx_node = self.nodes[0].add_p2p_connection(TestP2PConn(wtxidrelay=False), services=NODE_NETWORK | NODE_WITNESS) + self.wtx_node = self.nodes[0].add_p2p_connection(TestP2PConn(wtxidrelay=True), services=P2P_SERVICES) + self.tx_node = self.nodes[0].add_p2p_connection(TestP2PConn(wtxidrelay=False), services=P2P_SERVICES) # Check wtxidrelay feature negotiation message through connecting a new peer def received_wtxidrelay(): @@ -2147,9 +2019,8 @@ class SegWitTest(BitcoinTestFramework): # Create a Segwit output from the latest UTXO # and announce it to the network - witness_program = CScript([OP_TRUE]) - witness_hash = sha256(witness_program) - script_pubkey = CScript([OP_0, witness_hash]) + witness_script = CScript([OP_TRUE]) + script_pubkey = script_to_p2wsh_script(witness_script) tx = CTransaction() tx.vin.append(CTxIn(COutPoint(self.utxo[0].sha256, self.utxo[0].n), b"")) @@ -2161,7 +2032,7 @@ class SegWitTest(BitcoinTestFramework): tx2.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) tx2.vout.append(CTxOut(tx.vout[0].nValue - 1000, script_pubkey)) tx2.wit.vtxinwit.append(CTxInWitness()) - tx2.wit.vtxinwit[0].scriptWitness.stack = [witness_program] + tx2.wit.vtxinwit[0].scriptWitness.stack = [witness_script] tx2.rehash() # Announce Segwit transaction with wtxid diff --git a/test/functional/p2p_sendheaders.py b/test/functional/p2p_sendheaders.py index 04e6ec4172..7bf1803780 100755 --- a/test/functional/p2p_sendheaders.py +++ b/test/functional/p2p_sendheaders.py @@ -205,7 +205,7 @@ class SendHeadersTest(BitcoinTestFramework): # Clear out block announcements from each p2p listener [x.clear_block_announcements() for x in self.nodes[0].p2ps] - self.nodes[0].generatetoaddress(count, self.nodes[0].get_deterministic_priv_key().address) + self.generatetoaddress(self.nodes[0], count, self.nodes[0].get_deterministic_priv_key().address) return int(self.nodes[0].getbestblockhash(), 16) def mine_reorg(self, length): @@ -216,7 +216,7 @@ class SendHeadersTest(BitcoinTestFramework): return the list of block hashes newly mined.""" # make sure all invalidated blocks are node0's - self.nodes[0].generatetoaddress(length, self.nodes[0].get_deterministic_priv_key().address) + self.generatetoaddress(self.nodes[0], length, self.nodes[0].get_deterministic_priv_key().address) self.sync_blocks(self.nodes, wait=0.1) for x in self.nodes[0].p2ps: x.wait_for_block_announcement(int(self.nodes[0].getbestblockhash(), 16)) @@ -225,7 +225,7 @@ class SendHeadersTest(BitcoinTestFramework): tip_height = self.nodes[1].getblockcount() hash_to_invalidate = self.nodes[1].getblockhash(tip_height - (length - 1)) self.nodes[1].invalidateblock(hash_to_invalidate) - all_hashes = self.nodes[1].generatetoaddress(length + 1, self.nodes[1].get_deterministic_priv_key().address) # Must be longer than the orig chain + all_hashes = self.generatetoaddress(self.nodes[1], length + 1, self.nodes[1].get_deterministic_priv_key().address) # Must be longer than the orig chain self.sync_blocks(self.nodes, wait=0.1) return [int(x, 16) for x in all_hashes] @@ -240,7 +240,7 @@ class SendHeadersTest(BitcoinTestFramework): self.test_nonnull_locators(test_node, inv_node) def test_null_locators(self, test_node, inv_node): - tip = self.nodes[0].getblockheader(self.nodes[0].generatetoaddress(1, self.nodes[0].get_deterministic_priv_key().address)[0]) + tip = self.nodes[0].getblockheader(self.generatetoaddress(self.nodes[0], 1, self.nodes[0].get_deterministic_priv_key().address)[0]) tip_hash = int(tip["hash"], 16) inv_node.check_last_inv_announcement(inv=[tip_hash]) diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index e7a05d8547..a9d5ed970a 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -77,7 +77,7 @@ class AcceptBlockTest(BitcoinTestFramework): min_work_node = self.nodes[1].add_p2p_connection(P2PInterface()) # 1. Have nodes mine a block (leave IBD) - [n.generatetoaddress(1, n.get_deterministic_priv_key().address) for n in self.nodes] + [self.generatetoaddress(n, 1, n.get_deterministic_priv_key().address) for n in self.nodes] tips = [int("0x" + n.getbestblockhash(), 0) for n in self.nodes] # 2. Send one block that builds on each tip. diff --git a/test/functional/rpc_addresses_deprecation.py b/test/functional/rpc_addresses_deprecation.py deleted file mode 100755 index ac430f5b39..0000000000 --- a/test/functional/rpc_addresses_deprecation.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test deprecation of reqSigs and addresses RPC fields.""" - -from test_framework.messages import ( - tx_from_hex, -) -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_equal, - hex_str_to_bytes -) - - -class AddressesDeprecationTest(BitcoinTestFramework): - def set_test_params(self): - self.num_nodes = 2 - self.extra_args = [[], ["-deprecatedrpc=addresses"]] - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - - def run_test(self): - self.test_addresses_deprecation() - - def test_addresses_deprecation(self): - node = self.nodes[0] - coin = node.listunspent().pop() - - inputs = [{'txid': coin['txid'], 'vout': coin['vout']}] - outputs = {node.getnewaddress(): 0.99} - raw = node.createrawtransaction(inputs, outputs) - signed = node.signrawtransactionwithwallet(raw)['hex'] - - # This transaction is derived from test/util/data/txcreatemultisig1.json - tx = tx_from_hex(signed) - tx.vout[0].scriptPubKey = hex_str_to_bytes("522102a5613bd857b7048924264d1e70e08fb2a7e6527d32b7ab1bb993ac59964ff39721021ac43c7ff740014c3b33737ede99c967e4764553d1b2b83db77c83b8715fa72d2102df2089105c77f266fa11a9d33f05c735234075f2e8780824c6b709415f9fb48553ae") - tx_signed = node.signrawtransactionwithwallet(tx.serialize().hex())['hex'] - txid = node.sendrawtransaction(hexstring=tx_signed, maxfeerate=0) - - self.log.info("Test RPCResult scriptPubKey no longer returns the fields addresses or reqSigs by default") - hash = node.generateblock(output=node.getnewaddress(), transactions=[txid])['hash'] - # Ensure both nodes have the newly generated block on disk. - self.sync_blocks() - script_pub_key = node.getblock(blockhash=hash, verbose=2)['tx'][-1]['vout'][0]['scriptPubKey'] - assert 'addresses' not in script_pub_key and 'reqSigs' not in script_pub_key - - self.log.info("Test RPCResult scriptPubKey returns the addresses field with -deprecatedrpc=addresses") - script_pub_key = self.nodes[1].getblock(blockhash=hash, verbose=2)['tx'][-1]['vout'][0]['scriptPubKey'] - assert_equal(script_pub_key['addresses'], ['mvKDK6D54HU8wQumJBLHY95eq5iHFqXSBz', 'mv3rHCQSwKp2BLSuMHD8uCS32LW5xiNAA5', 'mirrsyhAQYzo5CwVhcaYJKwUJu1WJRCRJe']) - assert_equal(script_pub_key['reqSigs'], 2) - - -if __name__ == "__main__": - AddressesDeprecationTest().main() diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index 90715cae26..c3c6ade684 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -6,13 +6,15 @@ Test the following RPCs: - getblockchaininfo + - getchaintxstats - gettxoutsetinfo - - getdifficulty - - getbestblockhash - - getblockhash - getblockheader - - getchaintxstats + - getdifficulty - getnetworkhashps + - waitforblockheight + - getblock + - getblockhash + - getbestblockhash - verifychain Tests correspond to code in rpc/blockchain.cpp. @@ -49,6 +51,12 @@ from test_framework.util import ( from test_framework.wallet import MiniWallet +HEIGHT = 200 # blocks mined +TIME_RANGE_STEP = 600 # ten-minute steps +TIME_RANGE_MTP = TIME_GENESIS_BLOCK + (HEIGHT - 6) * TIME_RANGE_STEP +TIME_RANGE_END = TIME_GENESIS_BLOCK + HEIGHT * TIME_RANGE_STEP + + class BlockchainTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True @@ -71,12 +79,11 @@ class BlockchainTest(BitcoinTestFramework): assert self.nodes[0].verifychain(4, 0) def mine_chain(self): - self.log.info('Create some old blocks') - for t in range(TIME_GENESIS_BLOCK, TIME_GENESIS_BLOCK + 200 * 600, 600): - # ten-minute steps from genesis block time + self.log.info(f"Generate {HEIGHT} blocks after the genesis block in ten-minute steps") + for t in range(TIME_GENESIS_BLOCK, TIME_RANGE_END, TIME_RANGE_STEP): self.nodes[0].setmocktime(t) - self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_P2WSH_OP_TRUE) - assert_equal(self.nodes[0].getblockchaininfo()['blocks'], 200) + self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_P2WSH_OP_TRUE) + assert_equal(self.nodes[0].getblockchaininfo()['blocks'], HEIGHT) def _test_getblockchaininfo(self): self.log.info("Test getblockchaininfo") @@ -93,11 +100,15 @@ class BlockchainTest(BitcoinTestFramework): 'pruned', 'size_on_disk', 'softforks', + 'time', 'verificationprogress', 'warnings', ] res = self.nodes[0].getblockchaininfo() + assert_equal(res['time'], TIME_RANGE_END - TIME_RANGE_STEP) + assert_equal(res['mediantime'], TIME_RANGE_MTP) + # result should have these additional pruning keys if manual pruning is enabled assert_equal(sorted(res.keys()), sorted(['pruneheight', 'automatic_pruning'] + keys)) @@ -116,7 +127,29 @@ class BlockchainTest(BitcoinTestFramework): # should have exact keys assert_equal(sorted(res.keys()), keys) - self.restart_node(0, ['-stopatheight=207', '-prune=550']) + self.stop_node(0) + self.nodes[0].assert_start_raises_init_error( + extra_args=['-testactivationheight=name@2'], + expected_msg='Error: Invalid name (name@2) for -testactivationheight=name@height.', + ) + self.nodes[0].assert_start_raises_init_error( + extra_args=['-testactivationheight=bip34@-2'], + expected_msg='Error: Invalid height value (bip34@-2) for -testactivationheight=name@height.', + ) + self.nodes[0].assert_start_raises_init_error( + extra_args=['-testactivationheight='], + expected_msg='Error: Invalid format () for -testactivationheight=name@height.', + ) + self.start_node(0, extra_args=[ + '-stopatheight=207', + '-prune=550', + '-testactivationheight=bip34@2', + '-testactivationheight=dersig@3', + '-testactivationheight=cltv@4', + '-testactivationheight=csv@5', + '-testactivationheight=segwit@6', + ]) + res = self.nodes[0].getblockchaininfo() # result should have these additional pruning keys if prune=550 assert_equal(sorted(res.keys()), sorted(['pruneheight', 'automatic_pruning', 'prune_target_size'] + keys)) @@ -129,11 +162,11 @@ class BlockchainTest(BitcoinTestFramework): assert_greater_than(res['size_on_disk'], 0) assert_equal(res['softforks'], { - 'bip34': {'type': 'buried', 'active': False, 'height': 500}, - 'bip66': {'type': 'buried', 'active': False, 'height': 1251}, - 'bip65': {'type': 'buried', 'active': False, 'height': 1351}, - 'csv': {'type': 'buried', 'active': False, 'height': 432}, - 'segwit': {'type': 'buried', 'active': True, 'height': 0}, + 'bip34': {'type': 'buried', 'active': True, 'height': 2}, + 'bip66': {'type': 'buried', 'active': True, 'height': 3}, + 'bip65': {'type': 'buried', 'active': True, 'height': 4}, + 'csv': {'type': 'buried', 'active': True, 'height': 5}, + 'segwit': {'type': 'buried', 'active': True, 'height': 6}, 'testdummy': { 'type': 'bip9', 'bip9': { @@ -145,8 +178,8 @@ class BlockchainTest(BitcoinTestFramework): 'statistics': { 'period': 144, 'threshold': 108, - 'elapsed': 57, - 'count': 57, + 'elapsed': HEIGHT - 143, + 'count': HEIGHT - 143, 'possible': True, }, 'min_activation_height': 0, @@ -183,33 +216,33 @@ class BlockchainTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "blockhash must be of length 64 (not 1, for '0')", self.nodes[0].getchaintxstats, blockhash='0') assert_raises_rpc_error(-8, "blockhash must be hexadecimal string (not 'ZZZ0000000000000000000000000000000000000000000000000000000000000')", self.nodes[0].getchaintxstats, blockhash='ZZZ0000000000000000000000000000000000000000000000000000000000000') assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getchaintxstats, blockhash='0000000000000000000000000000000000000000000000000000000000000000') - blockhash = self.nodes[0].getblockhash(200) + blockhash = self.nodes[0].getblockhash(HEIGHT) self.nodes[0].invalidateblock(blockhash) assert_raises_rpc_error(-8, "Block is not in main chain", self.nodes[0].getchaintxstats, blockhash=blockhash) self.nodes[0].reconsiderblock(blockhash) chaintxstats = self.nodes[0].getchaintxstats(nblocks=1) # 200 txs plus genesis tx - assert_equal(chaintxstats['txcount'], 201) + assert_equal(chaintxstats['txcount'], HEIGHT + 1) # tx rate should be 1 per 10 minutes, or 1/600 # we have to round because of binary math - assert_equal(round(chaintxstats['txrate'] * 600, 10), Decimal(1)) + assert_equal(round(chaintxstats['txrate'] * TIME_RANGE_STEP, 10), Decimal(1)) b1_hash = self.nodes[0].getblockhash(1) b1 = self.nodes[0].getblock(b1_hash) - b200_hash = self.nodes[0].getblockhash(200) + b200_hash = self.nodes[0].getblockhash(HEIGHT) b200 = self.nodes[0].getblock(b200_hash) time_diff = b200['mediantime'] - b1['mediantime'] chaintxstats = self.nodes[0].getchaintxstats() assert_equal(chaintxstats['time'], b200['time']) - assert_equal(chaintxstats['txcount'], 201) + assert_equal(chaintxstats['txcount'], HEIGHT + 1) assert_equal(chaintxstats['window_final_block_hash'], b200_hash) - assert_equal(chaintxstats['window_final_block_height'], 200) - assert_equal(chaintxstats['window_block_count'], 199) - assert_equal(chaintxstats['window_tx_count'], 199) + assert_equal(chaintxstats['window_final_block_height'], HEIGHT ) + assert_equal(chaintxstats['window_block_count'], HEIGHT - 1) + assert_equal(chaintxstats['window_tx_count'], HEIGHT - 1) assert_equal(chaintxstats['window_interval'], time_diff) - assert_equal(round(chaintxstats['txrate'] * time_diff, 10), Decimal(199)) + assert_equal(round(chaintxstats['txrate'] * time_diff, 10), Decimal(HEIGHT - 1)) chaintxstats = self.nodes[0].getchaintxstats(blockhash=b1_hash) assert_equal(chaintxstats['time'], b1['time']) @@ -226,18 +259,18 @@ class BlockchainTest(BitcoinTestFramework): res = node.gettxoutsetinfo() assert_equal(res['total_amount'], Decimal('8725.00000000')) - assert_equal(res['transactions'], 200) - assert_equal(res['height'], 200) - assert_equal(res['txouts'], 200) + assert_equal(res['transactions'], HEIGHT) + assert_equal(res['height'], HEIGHT) + assert_equal(res['txouts'], HEIGHT) assert_equal(res['bogosize'], 16800), - assert_equal(res['bestblock'], node.getblockhash(200)) + assert_equal(res['bestblock'], node.getblockhash(HEIGHT)) size = res['disk_size'] assert size > 6400 assert size < 64000 assert_equal(len(res['bestblock']), 64) assert_equal(len(res['hash_serialized_2']), 64) - self.log.info("Test that gettxoutsetinfo() works for blockchain with just the genesis block") + self.log.info("Test gettxoutsetinfo works for blockchain with just the genesis block") b1hash = node.getblockhash(1) node.invalidateblock(b1hash) @@ -250,7 +283,7 @@ class BlockchainTest(BitcoinTestFramework): assert_equal(res2['bestblock'], node.getblockhash(0)) assert_equal(len(res2['hash_serialized_2']), 64) - self.log.info("Test that gettxoutsetinfo() returns the same result after invalidate/reconsider block") + self.log.info("Test gettxoutsetinfo returns the same result after invalidate/reconsider block") node.reconsiderblock(b1hash) res3 = node.gettxoutsetinfo() @@ -259,7 +292,7 @@ class BlockchainTest(BitcoinTestFramework): del res['disk_size'], res3['disk_size'] assert_equal(res, res3) - self.log.info("Test hash_type option for gettxoutsetinfo()") + self.log.info("Test gettxoutsetinfo hash_type option") # Adding hash_type 'hash_serialized_2', which is the default, should # not change the result. res4 = node.gettxoutsetinfo(hash_type='hash_serialized_2') @@ -283,6 +316,7 @@ class BlockchainTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "foohash is not a valid hash_type", node.gettxoutsetinfo, "foohash") def _test_getblockheader(self): + self.log.info("Test getblockheader") node = self.nodes[0] assert_raises_rpc_error(-8, "hash must be of length 64 (not 8, for 'nonsense')", node.getblockheader, "nonsense") @@ -290,11 +324,11 @@ class BlockchainTest(BitcoinTestFramework): assert_raises_rpc_error(-5, "Block not found", node.getblockheader, "0cf7bb8b1697ea987f3b223ba7819250cae33efacb068d23dc24859824a77844") besthash = node.getbestblockhash() - secondbesthash = node.getblockhash(199) + secondbesthash = node.getblockhash(HEIGHT - 1) header = node.getblockheader(blockhash=besthash) assert_equal(header['hash'], besthash) - assert_equal(header['height'], 200) + assert_equal(header['height'], HEIGHT) assert_equal(header['confirmations'], 1) assert_equal(header['previousblockhash'], secondbesthash) assert_is_hex_string(header['chainwork']) @@ -304,7 +338,7 @@ class BlockchainTest(BitcoinTestFramework): assert_is_hash_string(header['merkleroot']) assert_is_hash_string(header['bits'], length=None) assert isinstance(header['time'], int) - assert isinstance(header['mediantime'], int) + assert_equal(header['mediantime'], TIME_RANGE_MTP) assert isinstance(header['nonce'], int) assert isinstance(header['version'], int) assert isinstance(int(header['versionHex'], 16), int) @@ -322,30 +356,33 @@ class BlockchainTest(BitcoinTestFramework): assert 'nextblockhash' not in node.getblockheader(node.getbestblockhash()) def _test_getdifficulty(self): + self.log.info("Test getdifficulty") difficulty = self.nodes[0].getdifficulty() # 1 hash in 2 should be valid, so difficulty should be 1/2**31 # binary => decimal => binary math is why we do this check assert abs(difficulty * 2**31 - 1) < 0.0001 def _test_getnetworkhashps(self): + self.log.info("Test getnetworkhashps") hashes_per_second = self.nodes[0].getnetworkhashps() # This should be 2 hashes every 10 minutes or 1/300 assert abs(hashes_per_second * 300 - 1) < 0.0001 def _test_stopatheight(self): - assert_equal(self.nodes[0].getblockcount(), 200) - self.nodes[0].generatetoaddress(6, ADDRESS_BCRT1_P2WSH_OP_TRUE) - assert_equal(self.nodes[0].getblockcount(), 206) + self.log.info("Test stopping at height") + assert_equal(self.nodes[0].getblockcount(), HEIGHT) + self.generatetoaddress(self.nodes[0], 6, ADDRESS_BCRT1_P2WSH_OP_TRUE) + assert_equal(self.nodes[0].getblockcount(), HEIGHT + 6) self.log.debug('Node should not stop at this height') assert_raises(subprocess.TimeoutExpired, lambda: self.nodes[0].process.wait(timeout=3)) try: - self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_P2WSH_OP_TRUE) + self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_P2WSH_OP_TRUE) except (ConnectionError, http.client.BadStatusLine): pass # The node already shut down before response self.log.debug('Node should stop at this height...') self.nodes[0].wait_until_stopped() self.start_node(0) - assert_equal(self.nodes[0].getblockcount(), 207) + assert_equal(self.nodes[0].getblockcount(), HEIGHT + 7) def _test_waitforblockheight(self): self.log.info("Test waitforblockheight") @@ -389,28 +426,28 @@ class BlockchainTest(BitcoinTestFramework): node = self.nodes[0] miniwallet = MiniWallet(node) - miniwallet.scan_blocks(num=5) + miniwallet.rescan_utxos() fee_per_byte = Decimal('0.00000010') fee_per_kb = 1000 * fee_per_byte miniwallet.send_self_transfer(fee_rate=fee_per_kb, from_node=node) - blockhash = node.generate(1)[0] + blockhash = self.generate(node, 1)[0] - self.log.info("Test that getblock with verbosity 1 doesn't include fee") + self.log.info("Test getblock with verbosity 1 doesn't include fee") block = node.getblock(blockhash, 1) assert 'fee' not in block['tx'][1] - self.log.info('Test that getblock with verbosity 2 includes expected fee') + self.log.info('Test getblock with verbosity 2 includes expected fee') block = node.getblock(blockhash, 2) tx = block['tx'][1] assert 'fee' in tx assert_equal(tx['fee'], tx['vsize'] * fee_per_byte) - self.log.info("Test that getblock with verbosity 2 still works with pruned Undo data") + self.log.info("Test getblock with verbosity 2 still works with pruned Undo data") datadir = get_datadir_path(self.options.tmpdir, 0) - self.log.info("Test that getblock with invalid verbosity type returns proper error message") + self.log.info("Test getblock with invalid verbosity type returns proper error message") assert_raises_rpc_error(-1, "JSON value is not an integer as expected", node.getblock, blockhash, "2") def move_block_file(old, new): diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index 816ec67492..696438ccfe 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -3,7 +3,6 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test multisig RPCs""" -import binascii import decimal import itertools import json @@ -46,7 +45,7 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): self.check_addmultisigaddress_errors() self.log.info('Generating blocks ...') - node0.generate(149) + self.generate(node0, 149) self.sync_all() self.moved = 0 @@ -66,9 +65,9 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): # decompress pk2 pk_obj = ECPubKey() - pk_obj.set(binascii.unhexlify(pk2)) + pk_obj.set(bytes.fromhex(pk2)) pk_obj.compressed = False - pk2 = binascii.hexlify(pk_obj.get_bytes()).decode() + pk2 = pk_obj.get_bytes().hex() node0.createwallet(wallet_name='wmulti0', disable_private_keys=True) wmulti0 = node0.get_wallet_rpc('wmulti0') @@ -117,7 +116,7 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): def checkbalances(self): node0, node1, node2 = self.nodes - node0.generate(COINBASE_MATURITY) + self.generate(node0, COINBASE_MATURITY) self.sync_all() bal0 = node0.getbalance() @@ -180,7 +179,7 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): value = tx["vout"][vout]["value"] prevtxs = [{"txid": txid, "vout": vout, "scriptPubKey": scriptPubKey, "redeemScript": mredeem, "amount": value}] - node0.generate(1) + self.generate(node0, 1) outval = value - decimal.Decimal("0.00001000") rawtx = node2.createrawtransaction([{"txid": txid, "vout": vout}], [{self.final: outval}]) @@ -216,7 +215,7 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): self.moved += outval tx = node0.sendrawtransaction(rawtx3["hex"], 0) - blk = node0.generate(1)[0] + blk = self.generate(node0, 1)[0] assert tx in node0.getblock(blk)["tx"] txinfo = node0.getrawtransaction(tx, True, blk) diff --git a/test/functional/rpc_decodescript.py b/test/functional/rpc_decodescript.py index f6643c7167..5b1514af6f 100755 --- a/test/functional/rpc_decodescript.py +++ b/test/functional/rpc_decodescript.py @@ -11,7 +11,6 @@ from test_framework.messages import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, - hex_str_to_bytes, ) @@ -86,7 +85,7 @@ class DecodeScriptTest(BitcoinTestFramework): rpc_result = self.nodes[0].decodescript(multisig_script) assert_equal('2 ' + public_key + ' ' + public_key + ' ' + public_key + ' 3 OP_CHECKMULTISIG', rpc_result['asm']) # multisig in P2WSH - multisig_script_hash = sha256(hex_str_to_bytes(multisig_script)).hex() + multisig_script_hash = sha256(bytes.fromhex(multisig_script)).hex() assert_equal('0 ' + multisig_script_hash, rpc_result['segwit']['asm']) # 4) P2SH scriptPubKey @@ -124,7 +123,7 @@ class DecodeScriptTest(BitcoinTestFramework): rpc_result = self.nodes[0].decodescript(cltv_script) assert_equal('OP_IF ' + public_key + ' OP_CHECKSIGVERIFY OP_ELSE 500000 OP_CHECKLOCKTIMEVERIFY OP_DROP OP_ENDIF ' + public_key + ' OP_CHECKSIG', rpc_result['asm']) # CLTV script in P2WSH - cltv_script_hash = sha256(hex_str_to_bytes(cltv_script)).hex() + cltv_script_hash = sha256(bytes.fromhex(cltv_script)).hex() assert_equal('0 ' + cltv_script_hash, rpc_result['segwit']['asm']) # 7) P2PK scriptPubKey @@ -209,23 +208,23 @@ class DecodeScriptTest(BitcoinTestFramework): signature_2_sighash_decoded = der_signature + '[NONE|ANYONECANPAY]' # 1) P2PK scriptSig - txSave.vin[0].scriptSig = hex_str_to_bytes(push_signature) + txSave.vin[0].scriptSig = bytes.fromhex(push_signature) rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal(signature_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # make sure that the sighash decodes come out correctly for a more complex / lesser used case. - txSave.vin[0].scriptSig = hex_str_to_bytes(push_signature_2) + txSave.vin[0].scriptSig = bytes.fromhex(push_signature_2) rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal(signature_2_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # 2) multisig scriptSig - txSave.vin[0].scriptSig = hex_str_to_bytes('00' + push_signature + push_signature_2) + txSave.vin[0].scriptSig = bytes.fromhex('00' + push_signature + push_signature_2) rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal('0 ' + signature_sighash_decoded + ' ' + signature_2_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # 3) test a scriptSig that contains more than push operations. # in fact, it contains an OP_RETURN with data specially crafted to cause improper decode if the code does not catch it. - txSave.vin[0].scriptSig = hex_str_to_bytes('6a143011020701010101010101020601010101010101') + txSave.vin[0].scriptSig = bytes.fromhex('6a143011020701010101010101020601010101010101') rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal('OP_RETURN 3011020701010101010101020601010101010101', rpc_result['vin'][0]['scriptSig']['asm']) diff --git a/test/functional/rpc_deprecated.py b/test/functional/rpc_deprecated.py index 1af79b9f7c..fdaed918a1 100755 --- a/test/functional/rpc_deprecated.py +++ b/test/functional/rpc_deprecated.py @@ -21,7 +21,7 @@ class DeprecatedRpcTest(BitcoinTestFramework): # In run_test: # self.log.info("Test generate RPC") # assert_raises_rpc_error(-32, 'The wallet generate rpc method is deprecated', self.nodes[0].rpc.generate, 1) - # self.nodes[1].generate(1) + # self.generate(self.nodes[1], 1) self.log.info("No tested deprecated RPC methods") diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py index 3efbdab013..89388df555 100755 --- a/test/functional/rpc_dumptxoutset.py +++ b/test/functional/rpc_dumptxoutset.py @@ -23,7 +23,7 @@ class DumptxoutsetTest(BitcoinTestFramework): node = self.nodes[0] mocktime = node.getblockheader(node.getblockhash(0))['time'] + 1 node.setmocktime(mocktime) - node.generate(COINBASE_MATURITY) + self.generate(node, COINBASE_MATURITY) FILENAME = 'txoutset.dat' out = node.dumptxoutset(FILENAME) diff --git a/test/functional/rpc_fundrawtransaction.py b/test/functional/rpc_fundrawtransaction.py index fa98c44152..b0e46c6ca7 100755 --- a/test/functional/rpc_fundrawtransaction.py +++ b/test/functional/rpc_fundrawtransaction.py @@ -8,6 +8,7 @@ from decimal import Decimal from itertools import product from test_framework.descriptors import descsum_create +from test_framework.key import ECKey from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_approx, @@ -19,6 +20,7 @@ from test_framework.util import ( count_bytes, find_vout_for_address, ) +from test_framework.wallet_util import bytes_to_wif def get_unspent(listunspent, amount): @@ -47,7 +49,40 @@ class RawTransactionsTest(BitcoinTestFramework): self.connect_nodes(0, 2) self.connect_nodes(0, 3) + def lock_outputs_type(self, wallet, outputtype): + """ + Only allow UTXOs of the given type + """ + if outputtype in ["legacy", "p2pkh", "pkh"]: + prefixes = ["pkh(", "sh(multi("] + elif outputtype in ["p2sh-segwit", "sh_wpkh"]: + prefixes = ["sh(wpkh(", "sh(wsh("] + elif outputtype in ["bech32", "wpkh"]: + prefixes = ["wpkh(", "wsh("] + else: + assert False, f"Unknown output type {outputtype}" + + to_lock = [] + for utxo in wallet.listunspent(): + if "desc" in utxo: + for prefix in prefixes: + if utxo["desc"].startswith(prefix): + to_lock.append({"txid": utxo["txid"], "vout": utxo["vout"]}) + wallet.lockunspent(False, to_lock) + + def unlock_utxos(self, wallet): + """ + Unlock all UTXOs except the watchonly one + """ + to_keep = [] + if self.watchonly_txid is not None and self.watchonly_vout is not None: + to_keep.append({"txid": self.watchonly_txid, "vout": self.watchonly_vout}) + wallet.lockunspent(True) + wallet.lockunspent(False, to_keep) + def run_test(self): + self.watchonly_txid = None + self.watchonly_vout = None self.log.info("Connect nodes, set fees, generate blocks, and sync") self.min_relay_tx_fee = self.nodes[0].getnetworkinfo()['relayfee'] # This test is not meant to test fee estimation and we'd like @@ -63,9 +98,9 @@ class RawTransactionsTest(BitcoinTestFramework): # = 2 bytes * minRelayTxFeePerByte self.fee_tolerance = 2 * self.min_relay_tx_fee / 1000 - self.nodes[2].generate(1) + self.generate(self.nodes[2], 1) self.sync_all() - self.nodes[0].generate(121) + self.generate(self.nodes[0], 121) self.sync_all() self.test_change_position() @@ -99,6 +134,8 @@ class RawTransactionsTest(BitcoinTestFramework): self.test_subtract_fee_with_presets() self.test_transaction_too_large() self.test_include_unsafe() + self.test_external_inputs() + self.test_22670() def test_change_position(self): """Ensure setting changePosition in fundraw with an exact match is handled properly.""" @@ -125,7 +162,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 1.0) self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 5.0) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() wwatch.unloadwallet() @@ -372,6 +409,7 @@ class RawTransactionsTest(BitcoinTestFramework): def test_fee_p2pkh(self): """Compare fee of a standard pubkeyhash transaction.""" self.log.info("Test fundrawtxn p2pkh fee") + self.lock_outputs_type(self.nodes[0], "p2pkh") inputs = [] outputs = {self.nodes[1].getnewaddress():1.1} rawtx = self.nodes[0].createrawtransaction(inputs, outputs) @@ -379,15 +417,18 @@ class RawTransactionsTest(BitcoinTestFramework): # Create same transaction over sendtoaddress. txId = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1.1) - signedFee = self.nodes[0].getrawmempool(True)[txId]['fee'] + signedFee = self.nodes[0].getmempoolentry(txId)['fee'] # Compare fee. feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) assert feeDelta >= 0 and feeDelta <= self.fee_tolerance + self.unlock_utxos(self.nodes[0]) + def test_fee_p2pkh_multi_out(self): """Compare fee of a standard pubkeyhash transaction with multiple outputs.""" self.log.info("Test fundrawtxn p2pkh fee with multiple outputs") + self.lock_outputs_type(self.nodes[0], "p2pkh") inputs = [] outputs = { self.nodes[1].getnewaddress():1.1, @@ -402,14 +443,17 @@ class RawTransactionsTest(BitcoinTestFramework): # Create same transaction over sendtoaddress. txId = self.nodes[0].sendmany("", outputs) - signedFee = self.nodes[0].getrawmempool(True)[txId]['fee'] + signedFee = self.nodes[0].getmempoolentry(txId)['fee'] # Compare fee. feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) assert feeDelta >= 0 and feeDelta <= self.fee_tolerance + self.unlock_utxos(self.nodes[0]) + def test_fee_p2sh(self): """Compare fee of a 2-of-2 multisig p2sh transaction.""" + self.lock_outputs_type(self.nodes[0], "p2pkh") # Create 2-of-2 addr. addr1 = self.nodes[1].getnewaddress() addr2 = self.nodes[1].getnewaddress() @@ -426,15 +470,18 @@ class RawTransactionsTest(BitcoinTestFramework): # Create same transaction over sendtoaddress. txId = self.nodes[0].sendtoaddress(mSigObj, 1.1) - signedFee = self.nodes[0].getrawmempool(True)[txId]['fee'] + signedFee = self.nodes[0].getmempoolentry(txId)['fee'] # Compare fee. feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) assert feeDelta >= 0 and feeDelta <= self.fee_tolerance + self.unlock_utxos(self.nodes[0]) + def test_fee_4of5(self): """Compare fee of a standard pubkeyhash transaction.""" self.log.info("Test fundrawtxn fee with 4-of-5 addresses") + self.lock_outputs_type(self.nodes[0], "p2pkh") # Create 4-of-5 addr. addr1 = self.nodes[1].getnewaddress() @@ -467,12 +514,14 @@ class RawTransactionsTest(BitcoinTestFramework): # Create same transaction over sendtoaddress. txId = self.nodes[0].sendtoaddress(mSigObj, 1.1) - signedFee = self.nodes[0].getrawmempool(True)[txId]['fee'] + signedFee = self.nodes[0].getmempoolentry(txId)['fee'] # Compare fee. feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) assert feeDelta >= 0 and feeDelta <= self.fee_tolerance + self.unlock_utxos(self.nodes[0]) + def test_spend_2of2(self): """Spend a 2-of-2 multisig transaction over fundraw.""" self.log.info("Test fundpsbt spending 2-of-2 multisig") @@ -499,7 +548,7 @@ class RawTransactionsTest(BitcoinTestFramework): # Send 1.2 BTC to msig addr. self.nodes[0].sendtoaddress(mSigObj, 1.2) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() oldBalance = self.nodes[1].getbalance() @@ -510,7 +559,7 @@ class RawTransactionsTest(BitcoinTestFramework): signed_psbt = w2.walletprocesspsbt(funded_psbt) final_psbt = w2.finalizepsbt(signed_psbt['psbt']) self.nodes[2].sendrawtransaction(final_psbt['hex']) - self.nodes[2].generate(1) + self.generate(self.nodes[2], 1) self.sync_all() # Make sure funds are received at node1. @@ -541,15 +590,18 @@ class RawTransactionsTest(BitcoinTestFramework): # Drain the keypool. self.nodes[1].getnewaddress() self.nodes[1].getrawchangeaddress() - inputs = [] - outputs = {self.nodes[0].getnewaddress():1.09999500} + + # Choose 2 inputs + inputs = self.nodes[1].listunspent()[0:2] + value = sum(inp["amount"] for inp in inputs) - Decimal("0.00000500") # Pay a 500 sat fee + outputs = {self.nodes[0].getnewaddress():value} rawtx = self.nodes[1].createrawtransaction(inputs, outputs) # fund a transaction that does not require a new key for the change output self.nodes[1].fundrawtransaction(rawtx) # fund a transaction that requires a new key for the change output # creating the key must be impossible because the wallet is locked - outputs = {self.nodes[0].getnewaddress():1.1} + outputs = {self.nodes[0].getnewaddress():value - Decimal("0.1")} rawtx = self.nodes[1].createrawtransaction(inputs, outputs) assert_raises_rpc_error(-4, "Transaction needs a change address, but we can't generate it.", self.nodes[1].fundrawtransaction, rawtx) @@ -571,7 +623,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[1].walletpassphrase("test", 600) signedTx = self.nodes[1].signrawtransactionwithwallet(fundedTx['hex']) self.nodes[1].sendrawtransaction(signedTx['hex']) - self.nodes[1].generate(1) + self.generate(self.nodes[1], 1) self.sync_all() # Make sure funds are received at node1. @@ -583,12 +635,12 @@ class RawTransactionsTest(BitcoinTestFramework): # Empty node1, send some small coins from node0 to node1. self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), self.nodes[1].getbalance(), "", "", True) - self.nodes[1].generate(1) + self.generate(self.nodes[1], 1) self.sync_all() for _ in range(20): self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.01) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() # Fund a tx with ~20 small inputs. @@ -599,7 +651,7 @@ class RawTransactionsTest(BitcoinTestFramework): # Create same transaction over sendtoaddress. txId = self.nodes[1].sendmany("", outputs) - signedFee = self.nodes[1].getrawmempool(True)[txId]['fee'] + signedFee = self.nodes[1].getmempoolentry(txId)['fee'] # Compare fee. feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) @@ -611,12 +663,12 @@ class RawTransactionsTest(BitcoinTestFramework): # Again, empty node1, send some small coins from node0 to node1. self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), self.nodes[1].getbalance(), "", "", True) - self.nodes[1].generate(1) + self.generate(self.nodes[1], 1) self.sync_all() for _ in range(20): self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.01) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() # Fund a tx with ~20 small inputs. @@ -628,7 +680,7 @@ class RawTransactionsTest(BitcoinTestFramework): fundedTx = self.nodes[1].fundrawtransaction(rawtx) fundedAndSignedTx = self.nodes[1].signrawtransactionwithwallet(fundedTx['hex']) self.nodes[1].sendrawtransaction(fundedAndSignedTx['hex']) - self.nodes[1].generate(1) + self.generate(self.nodes[1], 1) self.sync_all() assert_equal(oldBalance+Decimal('50.19000000'), self.nodes[0].getbalance()) #0.19+block reward @@ -706,7 +758,7 @@ class RawTransactionsTest(BitcoinTestFramework): signedtx = self.nodes[0].signrawtransactionwithwallet(signedtx["hex"]) assert signedtx["complete"] self.nodes[0].sendrawtransaction(signedtx["hex"]) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() wwatch.unloadwallet() @@ -932,8 +984,59 @@ class RawTransactionsTest(BitcoinTestFramework): for _ in range(1500): outputs[recipient.getnewaddress()] = 0.1 wallet.sendmany("", outputs) - self.nodes[0].generate(10) + self.generate(self.nodes[0], 10) assert_raises_rpc_error(-4, "Transaction too large", recipient.fundrawtransaction, rawtx) + self.nodes[0].unloadwallet("large") + + def test_external_inputs(self): + self.log.info("Test funding with external inputs") + + eckey = ECKey() + eckey.generate() + privkey = bytes_to_wif(eckey.get_bytes()) + + self.nodes[2].createwallet("extfund") + wallet = self.nodes[2].get_wallet_rpc("extfund") + + # Make a weird but signable script. sh(pkh()) descriptor accomplishes this + desc = descsum_create("sh(pkh({}))".format(privkey)) + if self.options.descriptors: + res = self.nodes[0].importdescriptors([{"desc": desc, "timestamp": "now"}]) + else: + res = self.nodes[0].importmulti([{"desc": desc, "timestamp": "now"}]) + assert res[0]["success"] + addr = self.nodes[0].deriveaddresses(desc)[0] + addr_info = self.nodes[0].getaddressinfo(addr) + + self.nodes[0].sendtoaddress(addr, 10) + self.nodes[0].sendtoaddress(wallet.getnewaddress(), 10) + self.generate(self.nodes[0], 6) + self.sync_all() + ext_utxo = self.nodes[0].listunspent(addresses=[addr])[0] + + # An external input without solving data should result in an error + raw_tx = wallet.createrawtransaction([ext_utxo], {self.nodes[0].getnewaddress(): 15}) + assert_raises_rpc_error(-4, "Insufficient funds", wallet.fundrawtransaction, raw_tx) + + # Error conditions + assert_raises_rpc_error(-5, "'not a pubkey' is not hex", wallet.fundrawtransaction, raw_tx, {"solving_data": {"pubkeys":["not a pubkey"]}}) + assert_raises_rpc_error(-5, "'01234567890a0b0c0d0e0f' is not a valid public key", wallet.fundrawtransaction, raw_tx, {"solving_data": {"pubkeys":["01234567890a0b0c0d0e0f"]}}) + assert_raises_rpc_error(-5, "'not a script' is not hex", wallet.fundrawtransaction, raw_tx, {"solving_data": {"scripts":["not a script"]}}) + assert_raises_rpc_error(-8, "Unable to parse descriptor 'not a descriptor'", wallet.fundrawtransaction, raw_tx, {"solving_data": {"descriptors":["not a descriptor"]}}) + + # But funding should work when the solving data is provided + funded_tx = wallet.fundrawtransaction(raw_tx, {"solving_data": {"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}}) + signed_tx = wallet.signrawtransactionwithwallet(funded_tx['hex']) + assert not signed_tx['complete'] + signed_tx = self.nodes[0].signrawtransactionwithwallet(signed_tx['hex']) + assert signed_tx['complete'] + + funded_tx = wallet.fundrawtransaction(raw_tx, {"solving_data": {"descriptors": [desc]}}) + signed_tx = wallet.signrawtransactionwithwallet(funded_tx['hex']) + assert not signed_tx['complete'] + signed_tx = self.nodes[0].signrawtransactionwithwallet(signed_tx['hex']) + assert signed_tx['complete'] + self.nodes[2].unloadwallet("extfund") def test_include_unsafe(self): self.log.info("Test fundrawtxn with unsafe inputs") @@ -943,32 +1046,89 @@ class RawTransactionsTest(BitcoinTestFramework): # We receive unconfirmed funds from external keys (unsafe outputs). addr = wallet.getnewaddress() - txid1 = self.nodes[2].sendtoaddress(addr, 6) - txid2 = self.nodes[2].sendtoaddress(addr, 4) - self.sync_all() - vout1 = find_vout_for_address(wallet, txid1, addr) - vout2 = find_vout_for_address(wallet, txid2, addr) + inputs = [] + for i in range(0, 2): + txid = self.nodes[2].sendtoaddress(addr, 5) + self.sync_mempools() + vout = find_vout_for_address(wallet, txid, addr) + inputs.append((txid, vout)) # Unsafe inputs are ignored by default. - rawtx = wallet.createrawtransaction([], [{self.nodes[2].getnewaddress(): 5}]) + rawtx = wallet.createrawtransaction([], [{self.nodes[2].getnewaddress(): 7.5}]) assert_raises_rpc_error(-4, "Insufficient funds", wallet.fundrawtransaction, rawtx) # But we can opt-in to use them for funding. fundedtx = wallet.fundrawtransaction(rawtx, {"include_unsafe": True}) tx_dec = wallet.decoderawtransaction(fundedtx['hex']) - assert any([txin['txid'] == txid1 and txin['vout'] == vout1 for txin in tx_dec['vin']]) + assert all((txin["txid"], txin["vout"]) in inputs for txin in tx_dec["vin"]) signedtx = wallet.signrawtransactionwithwallet(fundedtx['hex']) - wallet.sendrawtransaction(signedtx['hex']) + assert wallet.testmempoolaccept([signedtx['hex']])[0]["allowed"] # And we can also use them once they're confirmed. - self.nodes[0].generate(1) - rawtx = wallet.createrawtransaction([], [{self.nodes[2].getnewaddress(): 3}]) - fundedtx = wallet.fundrawtransaction(rawtx, {"include_unsafe": True}) + self.generate(self.nodes[0], 1) + fundedtx = wallet.fundrawtransaction(rawtx, {"include_unsafe": False}) tx_dec = wallet.decoderawtransaction(fundedtx['hex']) - assert any([txin['txid'] == txid2 and txin['vout'] == vout2 for txin in tx_dec['vin']]) + assert all((txin["txid"], txin["vout"]) in inputs for txin in tx_dec["vin"]) signedtx = wallet.signrawtransactionwithwallet(fundedtx['hex']) - wallet.sendrawtransaction(signedtx['hex']) - + assert wallet.testmempoolaccept([signedtx['hex']])[0]["allowed"] + self.nodes[0].unloadwallet("unsafe") + + def test_22670(self): + # In issue #22670, it was observed that ApproximateBestSubset may + # choose enough value to cover the target amount but not enough to cover the transaction fees. + # This leads to a transaction whose actual transaction feerate is lower than expected. + # However at normal feerates, the difference between the effective value and the real value + # that this bug is not detected because the transaction fee must be at least 0.01 BTC (the minimum change value). + # Otherwise the targeted minimum change value will be enough to cover the transaction fees that were not + # being accounted for. So the minimum relay fee is set to 0.1 BTC/kvB in this test. + self.log.info("Test issue 22670 ApproximateBestSubset bug") + # Make sure the default wallet will not be loaded when restarted with a high minrelaytxfee + self.nodes[0].unloadwallet(self.default_wallet_name, False) + feerate = Decimal("0.1") + self.restart_node(0, [f"-minrelaytxfee={feerate}", "-discardfee=0"]) # Set high minrelayfee, set discardfee to 0 for easier calculation + + self.nodes[0].loadwallet(self.default_wallet_name, True) + funds = self.nodes[0].get_wallet_rpc(self.default_wallet_name) + self.nodes[0].createwallet(wallet_name="tester") + tester = self.nodes[0].get_wallet_rpc("tester") + + # Because this test is specifically for ApproximateBestSubset, the target value must be greater + # than any single input available, and require more than 1 input. So we make 3 outputs + for i in range(0, 3): + funds.sendtoaddress(tester.getnewaddress(address_type="bech32"), 1) + self.generate(self.nodes[0], 1) + + # Create transactions in order to calculate fees for the target bounds that can trigger this bug + change_tx = tester.fundrawtransaction(tester.createrawtransaction([], [{funds.getnewaddress(): 1.5}])) + tx = tester.createrawtransaction([], [{funds.getnewaddress(): 2}]) + no_change_tx = tester.fundrawtransaction(tx, {"subtractFeeFromOutputs": [0]}) + + overhead_fees = feerate * len(tx) / 2 / 1000 + cost_of_change = change_tx["fee"] - no_change_tx["fee"] + fees = no_change_tx["fee"] + assert_greater_than(fees, 0.01) + + def do_fund_send(target): + create_tx = tester.createrawtransaction([], [{funds.getnewaddress(): target}]) + funded_tx = tester.fundrawtransaction(create_tx) + signed_tx = tester.signrawtransactionwithwallet(funded_tx["hex"]) + assert signed_tx["complete"] + decoded_tx = tester.decoderawtransaction(signed_tx["hex"]) + assert_equal(len(decoded_tx["vin"]), 3) + assert tester.testmempoolaccept([signed_tx["hex"]])[0]["allowed"] + + # We want to choose more value than is available in 2 inputs when considering the fee, + # but not enough to need 3 inputs when not considering the fee. + # So the target value must be at least 2.00000001 - fee. + lower_bound = Decimal("2.00000001") - fees + # The target value must be at most 2 - cost_of_change - not_input_fees - min_change (these are all + # included in the target before ApproximateBestSubset). + upper_bound = Decimal("2.0") - cost_of_change - overhead_fees - Decimal("0.01") + assert_greater_than_or_equal(upper_bound, lower_bound) + do_fund_send(lower_bound) + do_fund_send(upper_bound) + + self.restart_node(0) if __name__ == '__main__': RawTransactionsTest().main() diff --git a/test/functional/rpc_generateblock.py b/test/functional/rpc_generateblock.py index 7424416484..3c6b3fb125 100755 --- a/test/functional/rpc_generateblock.py +++ b/test/functional/rpc_generateblock.py @@ -24,13 +24,13 @@ class GenerateBlockTest(BitcoinTestFramework): self.log.info('Generate an empty block to address') address = node.getnewaddress() - hash = node.generateblock(output=address, transactions=[])['hash'] + hash = self.generateblock(node, output=address, transactions=[])['hash'] block = node.getblock(blockhash=hash, verbose=2) assert_equal(len(block['tx']), 1) assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], address) self.log.info('Generate an empty block to a descriptor') - hash = node.generateblock('addr(' + address + ')', [])['hash'] + hash = self.generateblock(node, 'addr(' + address + ')', [])['hash'] block = node.getblock(blockhash=hash, verbosity=2) assert_equal(len(block['tx']), 1) assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], address) @@ -38,7 +38,7 @@ class GenerateBlockTest(BitcoinTestFramework): self.log.info('Generate an empty block to a combo descriptor with compressed pubkey') combo_key = '0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798' combo_address = 'bcrt1qw508d6qejxtdg4y5r3zarvary0c5xw7kygt080' - hash = node.generateblock('combo(' + combo_key + ')', [])['hash'] + hash = self.generateblock(node, 'combo(' + combo_key + ')', [])['hash'] block = node.getblock(hash, 2) assert_equal(len(block['tx']), 1) assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], combo_address) @@ -46,13 +46,13 @@ class GenerateBlockTest(BitcoinTestFramework): self.log.info('Generate an empty block to a combo descriptor with uncompressed pubkey') combo_key = '0408ef68c46d20596cc3f6ddf7c8794f71913add807f1dc55949fa805d764d191c0b7ce6894c126fce0babc6663042f3dde9b0cf76467ea315514e5a6731149c67' combo_address = 'mkc9STceoCcjoXEXe6cm66iJbmjM6zR9B2' - hash = node.generateblock('combo(' + combo_key + ')', [])['hash'] + hash = self.generateblock(node, 'combo(' + combo_key + ')', [])['hash'] block = node.getblock(hash, 2) assert_equal(len(block['tx']), 1) assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], combo_address) # Generate 110 blocks to spend - node.generatetoaddress(110, address) + self.generatetoaddress(node, 110, address) # Generate some extra mempool transactions to verify they don't get mined for _ in range(10): @@ -60,7 +60,7 @@ class GenerateBlockTest(BitcoinTestFramework): self.log.info('Generate block with txid') txid = node.sendtoaddress(address, 1) - hash = node.generateblock(address, [txid])['hash'] + hash = self.generateblock(node, address, [txid])['hash'] block = node.getblock(hash, 1) assert_equal(len(block['tx']), 2) assert_equal(block['tx'][1], txid) @@ -69,7 +69,7 @@ class GenerateBlockTest(BitcoinTestFramework): utxos = node.listunspent(addresses=[address]) raw = node.createrawtransaction([{'txid':utxos[0]['txid'], 'vout':utxos[0]['vout']}],[{address:1}]) signed_raw = node.signrawtransactionwithwallet(raw)['hex'] - hash = node.generateblock(address, [signed_raw])['hash'] + hash = self.generateblock(node, address, [signed_raw])['hash'] block = node.getblock(hash, 1) assert_equal(len(block['tx']), 2) txid = block['tx'][1] @@ -81,26 +81,26 @@ class GenerateBlockTest(BitcoinTestFramework): txid1 = node.sendrawtransaction(signed_raw1) raw2 = node.createrawtransaction([{'txid':txid1, 'vout':0}],[{address:0.999}]) signed_raw2 = node.signrawtransactionwithwallet(raw2)['hex'] - assert_raises_rpc_error(-25, 'TestBlockValidity failed: bad-txns-inputs-missingorspent', node.generateblock, address, [signed_raw2, txid1]) + assert_raises_rpc_error(-25, 'TestBlockValidity failed: bad-txns-inputs-missingorspent', self.generateblock, node, address, [signed_raw2, txid1]) self.log.info('Fail to generate block with txid not in mempool') missing_txid = '0000000000000000000000000000000000000000000000000000000000000000' - assert_raises_rpc_error(-5, 'Transaction ' + missing_txid + ' not in mempool.', node.generateblock, address, [missing_txid]) + assert_raises_rpc_error(-5, 'Transaction ' + missing_txid + ' not in mempool.', self.generateblock, node, address, [missing_txid]) self.log.info('Fail to generate block with invalid raw tx') invalid_raw_tx = '0000' - assert_raises_rpc_error(-22, 'Transaction decode failed for ' + invalid_raw_tx, node.generateblock, address, [invalid_raw_tx]) + assert_raises_rpc_error(-22, 'Transaction decode failed for ' + invalid_raw_tx, self.generateblock, node, address, [invalid_raw_tx]) self.log.info('Fail to generate block with invalid address/descriptor') - assert_raises_rpc_error(-5, 'Invalid address or descriptor', node.generateblock, '1234', []) + assert_raises_rpc_error(-5, 'Invalid address or descriptor', self.generateblock, node, '1234', []) self.log.info('Fail to generate block with a ranged descriptor') ranged_descriptor = 'pkh(tpubD6NzVbkrYhZ4XgiXtGrdW5XDAPFCL9h7we1vwNCpn8tGbBcgfVYjXyhWo4E1xkh56hjod1RhGjxbaTLV3X4FyWuejifB9jusQ46QzG87VKp/0/*)' - assert_raises_rpc_error(-8, 'Ranged descriptor not accepted. Maybe pass through deriveaddresses first?', node.generateblock, ranged_descriptor, []) + assert_raises_rpc_error(-8, 'Ranged descriptor not accepted. Maybe pass through deriveaddresses first?', self.generateblock, node, ranged_descriptor, []) self.log.info('Fail to generate block with a descriptor missing a private key') child_descriptor = 'pkh(tpubD6NzVbkrYhZ4XgiXtGrdW5XDAPFCL9h7we1vwNCpn8tGbBcgfVYjXyhWo4E1xkh56hjod1RhGjxbaTLV3X4FyWuejifB9jusQ46QzG87VKp/0\'/0)' - assert_raises_rpc_error(-5, 'Cannot derive script without private keys', node.generateblock, child_descriptor, []) + assert_raises_rpc_error(-5, 'Cannot derive script without private keys', self.generateblock, node, child_descriptor, []) if __name__ == '__main__': GenerateBlockTest().main() diff --git a/test/functional/rpc_getblockfilter.py b/test/functional/rpc_getblockfilter.py index a99e50f29f..4d860d0f36 100755 --- a/test/functional/rpc_getblockfilter.py +++ b/test/functional/rpc_getblockfilter.py @@ -21,8 +21,8 @@ class GetBlockFilterTest(BitcoinTestFramework): # Create two chains by disconnecting nodes 0 & 1, mining, then reconnecting self.disconnect_nodes(0, 1) - self.nodes[0].generate(3) - self.nodes[1].generate(4) + self.generate(self.nodes[0], 3) + self.generate(self.nodes[1], 4) assert_equal(self.nodes[0].getblockcount(), 3) chain0_hashes = [self.nodes[0].getblockhash(block_height) for block_height in range(4)] diff --git a/test/functional/rpc_getblockstats.py b/test/functional/rpc_getblockstats.py index 4af518c870..456e2cb0ad 100755 --- a/test/functional/rpc_getblockstats.py +++ b/test/functional/rpc_getblockstats.py @@ -43,11 +43,11 @@ class GetblockstatsTest(BitcoinTestFramework): def generate_test_data(self, filename): mocktime = 1525107225 self.nodes[0].setmocktime(mocktime) - self.nodes[0].generate(COINBASE_MATURITY + 1) + self.generate(self.nodes[0], COINBASE_MATURITY + 1) address = self.nodes[0].get_deterministic_priv_key().address self.nodes[0].sendtoaddress(address=address, amount=10, subtractfeefromamount=True) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() self.nodes[0].sendtoaddress(address=address, amount=10, subtractfeefromamount=True) @@ -55,7 +55,7 @@ class GetblockstatsTest(BitcoinTestFramework): self.nodes[0].settxfee(amount=0.003) self.nodes[0].sendtoaddress(address=address, amount=1, subtractfeefromamount=True) self.sync_all() - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.expected_stats = self.get_stats() diff --git a/test/functional/rpc_getchaintips.py b/test/functional/rpc_getchaintips.py index 8dc8474374..ab0ee9142d 100755 --- a/test/functional/rpc_getchaintips.py +++ b/test/functional/rpc_getchaintips.py @@ -26,8 +26,8 @@ class GetChainTipsTest (BitcoinTestFramework): # Split the network and build two chains of different lengths. self.split_network() - self.nodes[0].generatetoaddress(10, self.nodes[0].get_deterministic_priv_key().address) - self.nodes[2].generatetoaddress(20, self.nodes[2].get_deterministic_priv_key().address) + self.generatetoaddress(self.nodes[0], 10, self.nodes[0].get_deterministic_priv_key().address) + self.generatetoaddress(self.nodes[2], 20, self.nodes[2].get_deterministic_priv_key().address) self.sync_all(self.nodes[:2]) self.sync_all(self.nodes[2:]) diff --git a/test/functional/rpc_invalidateblock.py b/test/functional/rpc_invalidateblock.py index 114bd5e7e9..a91ce85855 100755 --- a/test/functional/rpc_invalidateblock.py +++ b/test/functional/rpc_invalidateblock.py @@ -22,12 +22,12 @@ class InvalidateTest(BitcoinTestFramework): def run_test(self): self.log.info("Make sure we repopulate setBlockIndexCandidates after InvalidateBlock:") self.log.info("Mine 4 blocks on Node 0") - self.nodes[0].generatetoaddress(4, self.nodes[0].get_deterministic_priv_key().address) + self.generatetoaddress(self.nodes[0], 4, self.nodes[0].get_deterministic_priv_key().address) assert_equal(self.nodes[0].getblockcount(), 4) besthash_n0 = self.nodes[0].getbestblockhash() self.log.info("Mine competing 6 blocks on Node 1") - self.nodes[1].generatetoaddress(6, self.nodes[1].get_deterministic_priv_key().address) + self.generatetoaddress(self.nodes[1], 6, self.nodes[1].get_deterministic_priv_key().address) assert_equal(self.nodes[1].getblockcount(), 6) self.log.info("Connect nodes to force a reorg") @@ -53,14 +53,14 @@ class InvalidateTest(BitcoinTestFramework): self.nodes[2].invalidateblock(self.nodes[2].getblockhash(3)) assert_equal(self.nodes[2].getblockcount(), 2) self.log.info("..and then mine a block") - self.nodes[2].generatetoaddress(1, self.nodes[2].get_deterministic_priv_key().address) + self.generatetoaddress(self.nodes[2], 1, self.nodes[2].get_deterministic_priv_key().address) self.log.info("Verify all nodes are at the right height") self.wait_until(lambda: self.nodes[2].getblockcount() == 3, timeout=5) self.wait_until(lambda: self.nodes[0].getblockcount() == 4, timeout=5) self.wait_until(lambda: self.nodes[1].getblockcount() == 4, timeout=5) self.log.info("Verify that we reconsider all ancestors as well") - blocks = self.nodes[1].generatetodescriptor(10, ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR) + blocks = self.generatetodescriptor(self.nodes[1], 10, ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR) assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) # Invalidate the two blocks at the tip self.nodes[1].invalidateblock(blocks[-1]) @@ -72,7 +72,7 @@ class InvalidateTest(BitcoinTestFramework): assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) self.log.info("Verify that we reconsider all descendants") - blocks = self.nodes[1].generatetodescriptor(10, ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR) + blocks = self.generatetodescriptor(self.nodes[1], 10, ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR) assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) # Invalidate the two blocks at the tip self.nodes[1].invalidateblock(blocks[-2]) diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py index 52c8fa883d..ac2a7a309b 100755 --- a/test/functional/rpc_misc.py +++ b/test/functional/rpc_misc.py @@ -54,13 +54,27 @@ class RpcMiscTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "unknown mode foobar", node.getmemoryinfo, mode="foobar") - self.log.info("test logging") + self.log.info("test logging rpc and help") + + # Test logging RPC returns the expected number of logging categories. + assert_equal(len(node.logging()), 26) + + # Test toggling a logging category on/off/on with the logging RPC. assert_equal(node.logging()['qt'], True) node.logging(exclude=['qt']) assert_equal(node.logging()['qt'], False) node.logging(include=['qt']) assert_equal(node.logging()['qt'], True) + # Test logging RPC returns the logging categories in alphabetical order. + sorted_logging_categories = sorted(node.logging()) + assert_equal(list(node.logging()), sorted_logging_categories) + + # Test logging help returns the logging categories string in alphabetical order. + categories = ', '.join(sorted_logging_categories) + logging_help = self.nodes[0].help('logging') + assert f"valid logging categories are: {categories}" in logging_help + self.log.info("test echoipc (testing spawned process in multiprocess build)") assert_equal(node.echoipc("hello"), "hello") diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 6e5ef770d1..0f3bbce54c 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -12,11 +12,10 @@ from itertools import product import time from test_framework.blocktools import COINBASE_MATURITY -from test_framework.p2p import P2PInterface import test_framework.messages -from test_framework.messages import ( - NODE_NETWORK, - NODE_WITNESS, +from test_framework.p2p import ( + P2PInterface, + P2P_SERVICES, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -52,9 +51,9 @@ class NetTest(BitcoinTestFramework): def run_test(self): # We need miniwallet to make a transaction self.wallet = MiniWallet(self.nodes[0]) - self.wallet.generate(1) + self.generate(self.wallet, 1) # Get out of IBD for the minfeefilter and getpeerinfo tests. - self.nodes[0].generate(COINBASE_MATURITY + 1) + self.generate(self.nodes[0], COINBASE_MATURITY + 1) # By default, the test framework sets up an addnode connection from # node 1 --> node0. By connecting node0 --> node 1, we're left with @@ -81,7 +80,7 @@ class NetTest(BitcoinTestFramework): self.log.info("Test getpeerinfo") # Create a few getpeerinfo last_block/last_transaction values. self.wallet.send_self_transfer(from_node=self.nodes[0]) # Make a transaction so we can see it in the getpeerinfo results - self.nodes[1].generate(1) + self.generate(self.nodes[1], 1) self.sync_all() time_now = int(time.time()) peer_info = [x.getpeerinfo() for x in self.nodes] @@ -189,7 +188,6 @@ class NetTest(BitcoinTestFramework): def test_getnodeaddresses(self): self.log.info("Test getnodeaddresses") self.nodes[0].add_p2p_connection(P2PInterface()) - services = NODE_NETWORK | NODE_WITNESS # Add an IPv6 address to the address manager. ipv6_addr = "1233:3432:2434:2343:3234:2345:6546:4534" @@ -217,7 +215,7 @@ class NetTest(BitcoinTestFramework): assert_greater_than(10000, len(node_addresses)) for a in node_addresses: assert_greater_than(a["time"], 1527811200) # 1st June 2018 - assert_equal(a["services"], services) + assert_equal(a["services"], P2P_SERVICES) assert a["address"] in imported_addrs assert_equal(a["port"], 8333) assert_equal(a["network"], "ipv4") @@ -228,7 +226,7 @@ class NetTest(BitcoinTestFramework): assert_equal(res[0]["address"], ipv6_addr) assert_equal(res[0]["network"], "ipv6") assert_equal(res[0]["port"], 8333) - assert_equal(res[0]["services"], services) + assert_equal(res[0]["services"], P2P_SERVICES) # Test for the absence of onion and I2P addresses. for network in ["onion", "i2p"]: @@ -239,7 +237,16 @@ class NetTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "Network not recognized: Foo", self.nodes[0].getnodeaddresses, 1, "Foo") def test_addpeeraddress(self): + """RPC addpeeraddress sets the source address equal to the destination address. + If an address with the same /16 as an existing new entry is passed, it will be + placed in the same new bucket and have a 1/64 chance of the bucket positions + colliding (depending on the value of nKey in the addrman), in which case the + new address won't be added. The probability of collision can be reduced to + 1/2^16 = 1/65536 by using an address from a different /16. We avoid this here + by first testing adding a tried table entry before testing adding a new table one. + """ self.log.info("Test addpeeraddress") + self.restart_node(1, ["-checkaddrman=1"]) node = self.nodes[1] self.log.debug("Test that addpeerinfo is a hidden RPC") @@ -251,17 +258,25 @@ class NetTest(BitcoinTestFramework): assert_equal(node.addpeeraddress(address="", port=8333), {"success": False}) assert_equal(node.getnodeaddresses(count=0), []) - self.log.debug("Test that adding a valid address succeeds") - assert_equal(node.addpeeraddress(address="1.2.3.4", port=8333), {"success": True}) - addrs = node.getnodeaddresses(count=0) - assert_equal(len(addrs), 1) - assert_equal(addrs[0]["address"], "1.2.3.4") - assert_equal(addrs[0]["port"], 8333) - - self.log.debug("Test that adding the same address again when already present fails") - assert_equal(node.addpeeraddress(address="1.2.3.4", port=8333), {"success": False}) + self.log.debug("Test that adding a valid address to the tried table succeeds") + assert_equal(node.addpeeraddress(address="1.2.3.4", tried=True, port=8333), {"success": True}) + with node.assert_debug_log(expected_msgs=["Addrman checks started: new 0, tried 1, total 1"]): + addrs = node.getnodeaddresses(count=0) # getnodeaddresses re-runs the addrman checks + assert_equal(len(addrs), 1) + assert_equal(addrs[0]["address"], "1.2.3.4") + assert_equal(addrs[0]["port"], 8333) + + self.log.debug("Test that adding an already-present tried address to the new and tried tables fails") + for value in [True, False]: + assert_equal(node.addpeeraddress(address="1.2.3.4", tried=value, port=8333), {"success": False}) assert_equal(len(node.getnodeaddresses(count=0)), 1) + self.log.debug("Test that adding a second address, this time to the new table, succeeds") + assert_equal(node.addpeeraddress(address="2.0.0.0", port=8333), {"success": True}) + with node.assert_debug_log(expected_msgs=["Addrman checks started: new 1, tried 1, total 2"]): + addrs = node.getnodeaddresses(count=0) # getnodeaddresses re-runs the addrman checks + assert_equal(len(addrs), 2) + if __name__ == '__main__': NetTest().main() diff --git a/test/functional/rpc_packages.py b/test/functional/rpc_packages.py index 4b2ed20958..63533affd0 100755 --- a/test/functional/rpc_packages.py +++ b/test/functional/rpc_packages.py @@ -22,6 +22,11 @@ from test_framework.script import ( from test_framework.util import ( assert_equal, ) +from test_framework.wallet import ( + create_child_with_parents, + create_raw_chain, + make_chain, +) class RPCPackagesTest(BitcoinTestFramework): def set_test_params(self): @@ -46,7 +51,7 @@ class RPCPackagesTest(BitcoinTestFramework): self.address = node.get_deterministic_priv_key().address self.coins = [] # The last 100 coinbase transactions are premature - for b in node.generatetoaddress(200, self.address)[:100]: + for b in self.generatetoaddress(node, 200, self.address)[:100]: coinbase = node.getblock(blockhash=b, verbosity=2)["tx"][0] self.coins.append({ "txid": coinbase["txid"], @@ -78,26 +83,6 @@ class RPCPackagesTest(BitcoinTestFramework): self.test_conflicting() self.test_rbf() - def chain_transaction(self, parent_txid, parent_value, n=0, parent_locking_script=None): - """Build a transaction that spends parent_txid.vout[n] and produces one output with - amount = parent_value with a fee deducted. - Return tuple (CTransaction object, raw hex, nValue, scriptPubKey of the output created). - """ - node = self.nodes[0] - inputs = [{"txid": parent_txid, "vout": n}] - my_value = parent_value - Decimal("0.0001") - outputs = {self.address : my_value} - rawtx = node.createrawtransaction(inputs, outputs) - prevtxs = [{ - "txid": parent_txid, - "vout": n, - "scriptPubKey": parent_locking_script, - "amount": parent_value, - }] if parent_locking_script else None - signedtx = node.signrawtransactionwithkey(hexstring=rawtx, privkeys=self.privkeys, prevtxs=prevtxs) - assert signedtx["complete"] - tx = tx_from_hex(signedtx["hex"]) - return (tx, signedtx["hex"], my_value, tx.vout[0].scriptPubKey.hex()) def test_independent(self): self.log.info("Test multiple independent transactions in a package") @@ -148,20 +133,7 @@ class RPCPackagesTest(BitcoinTestFramework): def test_chain(self): node = self.nodes[0] first_coin = self.coins.pop() - - # Chain of 25 transactions - parent_locking_script = None - txid = first_coin["txid"] - chain_hex = [] - chain_txns = [] - value = first_coin["amount"] - - for _ in range(25): - (tx, txhex, value, parent_locking_script) = self.chain_transaction(txid, value, 0, parent_locking_script) - txid = tx.rehash() - chain_hex.append(txhex) - chain_txns.append(tx) - + (chain_hex, chain_txns) = create_raw_chain(node, first_coin, self.address, self.privkeys) self.log.info("Check that testmempoolaccept requires packages to be sorted by dependency") assert_equal(node.testmempoolaccept(rawtxs=chain_hex[::-1]), [{"txid": tx.rehash(), "wtxid": tx.getwtxid(), "package-error": "package-not-sorted"} for tx in chain_txns[::-1]]) @@ -179,7 +151,7 @@ class RPCPackagesTest(BitcoinTestFramework): assert_equal(testres_single, testres_multiple) # Clean up by clearing the mempool - node.generate(1) + self.generate(node, 1) def test_multiple_children(self): node = self.nodes[0] @@ -201,7 +173,7 @@ class RPCPackagesTest(BitcoinTestFramework): child_value = value - Decimal("0.0001") # Child A - (_, tx_child_a_hex, _, _) = self.chain_transaction(parent_txid, child_value, 0, parent_locking_script_a) + (_, tx_child_a_hex, _, _) = make_chain(node, self.address, self.privkeys, parent_txid, child_value, 0, parent_locking_script_a) assert not node.testmempoolaccept([tx_child_a_hex])[0]["allowed"] # Child B @@ -226,19 +198,6 @@ class RPCPackagesTest(BitcoinTestFramework): node.sendrawtransaction(rawtx) assert_equal(testres_single, testres_multiple_ab) - def create_child_with_parents(self, parents_tx, values, locking_scripts): - """Creates a transaction that spends the first output of each parent in parents_tx.""" - num_parents = len(parents_tx) - total_value = sum(values) - inputs = [{"txid": tx.rehash(), "vout": 0} for tx in parents_tx] - outputs = {self.address : total_value - num_parents * Decimal("0.0001")} - rawtx_child = self.nodes[0].createrawtransaction(inputs, outputs) - prevtxs = [] - for i in range(num_parents): - prevtxs.append({"txid": parents_tx[i].rehash(), "vout": 0, "scriptPubKey": locking_scripts[i], "amount": values[i]}) - signedtx_child = self.nodes[0].signrawtransactionwithkey(hexstring=rawtx_child, privkeys=self.privkeys, prevtxs=prevtxs) - assert signedtx_child["complete"] - return signedtx_child["hex"] def test_multiple_parents(self): node = self.nodes[0] @@ -253,12 +212,12 @@ class RPCPackagesTest(BitcoinTestFramework): for _ in range(num_parents): parent_coin = self.coins.pop() value = parent_coin["amount"] - (tx, txhex, value, parent_locking_script) = self.chain_transaction(parent_coin["txid"], value) + (tx, txhex, value, parent_locking_script) = make_chain(node, self.address, self.privkeys, parent_coin["txid"], value) package_hex.append(txhex) parents_tx.append(tx) values.append(value) parent_locking_scripts.append(parent_locking_script) - child_hex = self.create_child_with_parents(parents_tx, values, parent_locking_scripts) + child_hex = create_child_with_parents(node, self.address, self.privkeys, parents_tx, values, parent_locking_scripts) # Package accept should work with the parents in any order (as long as parents come before child) for _ in range(10): random.shuffle(package_hex) diff --git a/test/functional/rpc_preciousblock.py b/test/functional/rpc_preciousblock.py index 7d3a5cc9ce..3a00992ddc 100755 --- a/test/functional/rpc_preciousblock.py +++ b/test/functional/rpc_preciousblock.py @@ -43,18 +43,18 @@ class PreciousTest(BitcoinTestFramework): def run_test(self): self.log.info("Ensure submitblock can in principle reorg to a competing chain") gen_address = lambda i: self.nodes[i].get_deterministic_priv_key().address # A non-wallet address to mine to - self.nodes[0].generatetoaddress(1, gen_address(0)) + self.generatetoaddress(self.nodes[0], 1, gen_address(0)) assert_equal(self.nodes[0].getblockcount(), 1) - hashZ = self.nodes[1].generatetoaddress(2, gen_address(1))[-1] + hashZ = self.generatetoaddress(self.nodes[1], 2, gen_address(1))[-1] assert_equal(self.nodes[1].getblockcount(), 2) node_sync_via_rpc(self.nodes[0:3]) assert_equal(self.nodes[0].getbestblockhash(), hashZ) self.log.info("Mine blocks A-B-C on Node 0") - hashC = self.nodes[0].generatetoaddress(3, gen_address(0))[-1] + hashC = self.generatetoaddress(self.nodes[0], 3, gen_address(0))[-1] assert_equal(self.nodes[0].getblockcount(), 5) self.log.info("Mine competing blocks E-F-G on Node 1") - hashG = self.nodes[1].generatetoaddress(3, gen_address(1))[-1] + hashG = self.generatetoaddress(self.nodes[1], 3, gen_address(1))[-1] assert_equal(self.nodes[1].getblockcount(), 5) assert hashC != hashG self.log.info("Connect nodes and check no reorg occurs") @@ -83,7 +83,7 @@ class PreciousTest(BitcoinTestFramework): self.nodes[1].preciousblock(hashC) assert_equal(self.nodes[1].getbestblockhash(), hashC) self.log.info("Mine another block (E-F-G-)H on Node 0 and reorg Node 1") - self.nodes[0].generatetoaddress(1, gen_address(0)) + self.generatetoaddress(self.nodes[0], 1, gen_address(0)) assert_equal(self.nodes[0].getblockcount(), 6) self.sync_blocks(self.nodes[0:2]) hashH = self.nodes[0].getbestblockhash() @@ -92,7 +92,7 @@ class PreciousTest(BitcoinTestFramework): self.nodes[1].preciousblock(hashC) assert_equal(self.nodes[1].getbestblockhash(), hashH) self.log.info("Mine competing blocks I-J-K-L on Node 2") - self.nodes[2].generatetoaddress(4, gen_address(2)) + self.generatetoaddress(self.nodes[2], 4, gen_address(2)) assert_equal(self.nodes[2].getblockcount(), 6) hashL = self.nodes[2].getbestblockhash() self.log.info("Connect nodes and check no reorg occurs") diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index cf4d8a134c..b132ac3d31 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -8,6 +8,8 @@ from decimal import Decimal from itertools import product +from test_framework.descriptors import descsum_create +from test_framework.key import ECKey from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_approx, @@ -16,6 +18,7 @@ from test_framework.util import ( assert_raises_rpc_error, find_output, ) +from test_framework.wallet_util import bytes_to_wif import json import os @@ -57,7 +60,7 @@ class PSBTTest(BitcoinTestFramework): online_addr = w2.getnewaddress(address_type="p2sh-segwit") wonline.importaddress(offline_addr, "", False) mining_node.sendtoaddress(address=offline_addr, amount=1.0) - mining_node.generate(nblocks=1) + self.generate(mining_node, nblocks=1) self.sync_blocks([mining_node, online_node]) # Construct an unsigned PSBT on the online node (who doesn't know the output is Segwit, so will include a non-witness UTXO) @@ -72,7 +75,7 @@ class PSBTTest(BitcoinTestFramework): # Make sure we can mine the resulting transaction txid = mining_node.sendrawtransaction(mining_node.finalizepsbt(signed_psbt)["hex"]) - mining_node.generate(1) + self.generate(mining_node, 1) self.sync_blocks([mining_node, online_node]) assert_equal(online_node.gettxout(txid,0)["confirmations"], 1) @@ -108,6 +111,16 @@ class PSBTTest(BitcoinTestFramework): psbtx = self.nodes[1].walletprocesspsbt(psbtx1)['psbt'] assert_equal(psbtx1, psbtx) + # Node 0 should not be able to sign the transaction with the wallet is locked + self.nodes[0].encryptwallet("password") + assert_raises_rpc_error(-13, "Please enter the wallet passphrase with walletpassphrase first", self.nodes[0].walletprocesspsbt, psbtx) + + # Node 0 should be able to process without signing though + unsigned_tx = self.nodes[0].walletprocesspsbt(psbtx, False) + assert_equal(unsigned_tx['complete'], False) + + self.nodes[0].walletpassphrase(passphrase="password", timeout=1000000) + # Sign the transaction and send signed_tx = self.nodes[0].walletprocesspsbt(psbtx)['psbt'] final_tx = self.nodes[0].finalizepsbt(signed_tx)['hex'] @@ -148,7 +161,7 @@ class PSBTTest(BitcoinTestFramework): rawtx = self.nodes[0].fundrawtransaction(rawtx, {"changePosition":3}) signed_tx = self.nodes[0].signrawtransactionwithwallet(rawtx['hex'])['hex'] txid = self.nodes[0].sendrawtransaction(signed_tx) - self.nodes[0].generate(6) + self.generate(self.nodes[0], 6) self.sync_all() # Find the output pos @@ -307,7 +320,7 @@ class PSBTTest(BitcoinTestFramework): node2_addr = self.nodes[2].getnewaddress() txid1 = self.nodes[0].sendtoaddress(node1_addr, 13) txid2 = self.nodes[0].sendtoaddress(node2_addr, 13) - blockhash = self.nodes[0].generate(6)[0] + blockhash = self.generate(self.nodes[0], 6)[0] self.sync_all() vout1 = find_output(self.nodes[1], txid1, 13, blockhash=blockhash) vout2 = find_output(self.nodes[2], txid2, 13, blockhash=blockhash) @@ -335,7 +348,7 @@ class PSBTTest(BitcoinTestFramework): combined = self.nodes[0].combinepsbt([psbt1, psbt2]) finalized = self.nodes[0].finalizepsbt(combined)['hex'] self.nodes[0].sendrawtransaction(finalized) - self.nodes[0].generate(6) + self.generate(self.nodes[0], 6) self.sync_all() # Test additional args in walletcreatepsbt @@ -530,7 +543,7 @@ class PSBTTest(BitcoinTestFramework): addr4 = self.nodes[1].getnewaddress("", "p2sh-segwit") txid4 = self.nodes[0].sendtoaddress(addr4, 5) vout4 = find_output(self.nodes[0], txid4, 5) - self.nodes[0].generate(6) + self.generate(self.nodes[0], 6) self.sync_all() psbt2 = self.nodes[1].createpsbt([{"txid":txid4, "vout":vout4}], {self.nodes[0].getnewaddress():Decimal('4.999')}) psbt2 = self.nodes[1].walletprocesspsbt(psbt2)['psbt'] @@ -554,7 +567,7 @@ class PSBTTest(BitcoinTestFramework): addr = self.nodes[1].getnewaddress("", "p2sh-segwit") txid = self.nodes[0].sendtoaddress(addr, 7) addrinfo = self.nodes[1].getaddressinfo(addr) - blockhash = self.nodes[0].generate(6)[0] + blockhash = self.generate(self.nodes[0], 6)[0] self.sync_all() vout = find_output(self.nodes[0], txid, 7, blockhash=blockhash) psbt = self.nodes[1].createpsbt([{"txid":txid, "vout":vout}], {self.nodes[0].getnewaddress("", "p2sh-segwit"):Decimal('6.999')}) @@ -598,5 +611,43 @@ class PSBTTest(BitcoinTestFramework): assert_raises_rpc_error(-25, 'Inputs missing or spent', self.nodes[0].walletprocesspsbt, 'cHNidP8BAJoCAAAAAkvEW8NnDtdNtDpsmze+Ht2LH35IJcKv00jKAlUs21RrAwAAAAD/////S8Rbw2cO1020OmybN74e3Ysffkglwq/TSMoCVSzbVGsBAAAAAP7///8CwLYClQAAAAAWABSNJKzjaUb3uOxixsvh1GGE3fW7zQD5ApUAAAAAFgAUKNw0x8HRctAgmvoevm4u1SbN7XIAAAAAAAEAnQIAAAACczMa321tVHuN4GKWKRncycI22aX3uXgwSFUKM2orjRsBAAAAAP7///9zMxrfbW1Ue43gYpYpGdzJwjbZpfe5eDBIVQozaiuNGwAAAAAA/v///wIA+QKVAAAAABl2qRT9zXUVA8Ls5iVqynLHe5/vSe1XyYisQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAAAAAQEfQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAA==') + # Test that we can fund psbts with external inputs specified + eckey = ECKey() + eckey.generate() + privkey = bytes_to_wif(eckey.get_bytes()) + + # Make a weird but signable script. sh(pkh()) descriptor accomplishes this + desc = descsum_create("sh(pkh({}))".format(privkey)) + if self.options.descriptors: + res = self.nodes[0].importdescriptors([{"desc": desc, "timestamp": "now"}]) + else: + res = self.nodes[0].importmulti([{"desc": desc, "timestamp": "now"}]) + assert res[0]["success"] + addr = self.nodes[0].deriveaddresses(desc)[0] + addr_info = self.nodes[0].getaddressinfo(addr) + + self.nodes[0].sendtoaddress(addr, 10) + self.generate(self.nodes[0], 6) + self.sync_all() + ext_utxo = self.nodes[0].listunspent(addresses=[addr])[0] + + # An external input without solving data should result in an error + assert_raises_rpc_error(-4, "Insufficient funds", self.nodes[1].walletcreatefundedpsbt, [ext_utxo], {self.nodes[0].getnewaddress(): 10 + ext_utxo['amount']}, 0, {'add_inputs': True}) + + # But funding should work when the solving data is provided + psbt = self.nodes[1].walletcreatefundedpsbt([ext_utxo], {self.nodes[0].getnewaddress(): 15}, 0, {'add_inputs': True, "solving_data": {"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}}) + signed = self.nodes[1].walletprocesspsbt(psbt['psbt']) + assert not signed['complete'] + signed = self.nodes[0].walletprocesspsbt(signed['psbt']) + assert signed['complete'] + self.nodes[0].finalizepsbt(signed['psbt']) + + psbt = self.nodes[1].walletcreatefundedpsbt([ext_utxo], {self.nodes[0].getnewaddress(): 15}, 0, {'add_inputs': True, "solving_data":{"descriptors": [desc]}}) + signed = self.nodes[1].walletprocesspsbt(psbt['psbt']) + assert not signed['complete'] + signed = self.nodes[0].walletprocesspsbt(signed['psbt']) + assert signed['complete'] + self.nodes[0].finalizepsbt(signed['psbt']) + if __name__ == '__main__': PSBTTest().main() diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index db57368eae..fc812340fa 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -5,11 +5,11 @@ """Test the rawtransaction RPCs. Test the following RPCs: + - getrawtransaction - createrawtransaction - signrawtransactionwithwallet - sendrawtransaction - decoderawtransaction - - getrawtransaction """ from collections import OrderedDict @@ -28,6 +28,9 @@ from test_framework.util import ( ) +TXID = "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000" + + class multidict(dict): """Dictionary that allows duplicate keys. @@ -46,16 +49,20 @@ class multidict(dict): return self.x -# Create one-input, one-output, no-fee transaction: class RawTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 3 + self.num_nodes = 4 self.extra_args = [ ["-txindex"], ["-txindex"], ["-txindex"], + [], ] + # whitelist all peers to speed up tx relay / mempool sync + for args in self.extra_args: + args.append("-whitelist=noban@127.0.0.1") + self.supports_cli = False def skip_test_if_missing_module(self): @@ -66,23 +73,112 @@ class RawTransactionsTest(BitcoinTestFramework): self.connect_nodes(0, 2) def run_test(self): - self.log.info('prepare some coins for multiple *rawtransaction commands') - self.nodes[2].generate(1) + self.log.info("Prepare some coins for multiple *rawtransaction commands") + self.generate(self.nodes[2], 1) self.sync_all() - self.nodes[0].generate(COINBASE_MATURITY + 1) + self.generate(self.nodes[0], COINBASE_MATURITY + 1) self.sync_all() - self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(),1.5) - self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(),1.0) - self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(),5.0) + for amount in [1.5, 1.0, 5.0]: + self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), amount) + self.sync_all() + self.generate(self.nodes[0], 5) + self.sync_all() + + self.getrawtransaction_tests() + self.createrawtransaction_tests() + self.signrawtransactionwithwallet_tests() + self.sendrawtransaction_tests() + self.sendrawtransaction_testmempoolaccept_tests() + self.decoderawtransaction_tests() + self.transaction_version_number_tests() + if not self.options.descriptors: + self.raw_multisig_transaction_legacy_tests() + + def getrawtransaction_tests(self): + addr = self.nodes[1].getnewaddress() + txid = self.nodes[0].sendtoaddress(addr, 10) + self.generate(self.nodes[0], 1) self.sync_all() - self.nodes[0].generate(5) + vout = find_vout_for_address(self.nodes[1], txid, addr) + rawTx = self.nodes[1].createrawtransaction([{'txid': txid, 'vout': vout}], {self.nodes[1].getnewaddress(): 9.999}) + rawTxSigned = self.nodes[1].signrawtransactionwithwallet(rawTx) + txId = self.nodes[1].sendrawtransaction(rawTxSigned['hex']) + self.generate(self.nodes[0], 1) self.sync_all() - self.log.info('Test getrawtransaction on genesis block coinbase returns an error') + for n in [0, 3]: + self.log.info(f"Test getrawtransaction {'with' if n == 0 else 'without'} -txindex") + # 1. valid parameters - only supply txid + assert_equal(self.nodes[n].getrawtransaction(txId), rawTxSigned['hex']) + + # 2. valid parameters - supply txid and 0 for non-verbose + assert_equal(self.nodes[n].getrawtransaction(txId, 0), rawTxSigned['hex']) + + # 3. valid parameters - supply txid and False for non-verbose + assert_equal(self.nodes[n].getrawtransaction(txId, False), rawTxSigned['hex']) + + # 4. valid parameters - supply txid and 1 for verbose. + # We only check the "hex" field of the output so we don't need to update this test every time the output format changes. + assert_equal(self.nodes[n].getrawtransaction(txId, 1)["hex"], rawTxSigned['hex']) + + # 5. valid parameters - supply txid and True for non-verbose + assert_equal(self.nodes[n].getrawtransaction(txId, True)["hex"], rawTxSigned['hex']) + + # 6. invalid parameters - supply txid and invalid boolean values (strings) for verbose + for value in ["True", "False"]: + assert_raises_rpc_error(-1, "not a boolean", self.nodes[n].getrawtransaction, txid=txId, verbose=value) + + # 7. invalid parameters - supply txid and empty array + assert_raises_rpc_error(-1, "not a boolean", self.nodes[n].getrawtransaction, txId, []) + + # 8. invalid parameters - supply txid and empty dict + assert_raises_rpc_error(-1, "not a boolean", self.nodes[n].getrawtransaction, txId, {}) + + # Make a tx by sending, then generate 2 blocks; block1 has the tx in it + tx = self.nodes[2].sendtoaddress(self.nodes[1].getnewaddress(), 1) + block1, block2 = self.generate(self.nodes[2], 2) + self.sync_all() + for n in [0, 3]: + self.log.info(f"Test getrawtransaction {'with' if n == 0 else 'without'} -txindex, with blockhash") + # We should be able to get the raw transaction by providing the correct block + gottx = self.nodes[n].getrawtransaction(txid=tx, verbose=True, blockhash=block1) + assert_equal(gottx['txid'], tx) + assert_equal(gottx['in_active_chain'], True) + if n == 0: + self.log.info("Test getrawtransaction with -txindex, without blockhash: 'in_active_chain' should be absent") + gottx = self.nodes[n].getrawtransaction(txid=tx, verbose=True) + assert_equal(gottx['txid'], tx) + assert 'in_active_chain' not in gottx + else: + self.log.info("Test getrawtransaction without -txindex, without blockhash: expect the call to raise") + err_msg = ( + "No such mempool transaction. Use -txindex or provide a block hash to enable" + " blockchain transaction queries. Use gettransaction for wallet transactions." + ) + assert_raises_rpc_error(-5, err_msg, self.nodes[n].getrawtransaction, txid=tx, verbose=True) + # We should not get the tx if we provide an unrelated block + assert_raises_rpc_error(-5, "No such transaction found", self.nodes[n].getrawtransaction, txid=tx, blockhash=block2) + # An invalid block hash should raise the correct errors + assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[n].getrawtransaction, txid=tx, blockhash=True) + assert_raises_rpc_error(-8, "parameter 3 must be of length 64 (not 6, for 'foobar')", self.nodes[n].getrawtransaction, txid=tx, blockhash="foobar") + assert_raises_rpc_error(-8, "parameter 3 must be of length 64 (not 8, for 'abcd1234')", self.nodes[n].getrawtransaction, txid=tx, blockhash="abcd1234") + foo = "ZZZ0000000000000000000000000000000000000000000000000000000000000" + assert_raises_rpc_error(-8, f"parameter 3 must be hexadecimal string (not '{foo}')", self.nodes[n].getrawtransaction, txid=tx, blockhash=foo) + bar = "0000000000000000000000000000000000000000000000000000000000000000" + assert_raises_rpc_error(-5, "Block hash not found", self.nodes[n].getrawtransaction, txid=tx, blockhash=bar) + # Undo the blocks and verify that "in_active_chain" is false. + self.nodes[n].invalidateblock(block1) + gottx = self.nodes[n].getrawtransaction(txid=tx, verbose=True, blockhash=block1) + assert_equal(gottx['in_active_chain'], False) + self.nodes[n].reconsiderblock(block1) + assert_equal(self.nodes[n].getbestblockhash(), block2) + + self.log.info("Test getrawtransaction on genesis block coinbase returns an error") block = self.nodes[0].getblock(self.nodes[0].getblockhash(0)) assert_raises_rpc_error(-5, "The genesis block coinbase is not considered an ordinary transaction", self.nodes[0].getrawtransaction, block['merkleroot']) - self.log.info('Check parameter types and required parameters of createrawtransaction') + def createrawtransaction_tests(self): + self.log.info("Test createrawtransaction") # Test `createrawtransaction` required parameters assert_raises_rpc_error(-1, "createrawtransaction", self.nodes[0].createrawtransaction) assert_raises_rpc_error(-1, "createrawtransaction", self.nodes[0].createrawtransaction, []) @@ -91,16 +187,28 @@ class RawTransactionsTest(BitcoinTestFramework): assert_raises_rpc_error(-1, "createrawtransaction", self.nodes[0].createrawtransaction, [], {}, 0, False, 'foo') # Test `createrawtransaction` invalid `inputs` - txid = '1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000' assert_raises_rpc_error(-3, "Expected type array", self.nodes[0].createrawtransaction, 'foo', {}) assert_raises_rpc_error(-1, "JSON value is not an object as expected", self.nodes[0].createrawtransaction, ['foo'], {}) assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[0].createrawtransaction, [{}], {}) assert_raises_rpc_error(-8, "txid must be of length 64 (not 3, for 'foo')", self.nodes[0].createrawtransaction, [{'txid': 'foo'}], {}) - assert_raises_rpc_error(-8, "txid must be hexadecimal string (not 'ZZZ7bb8b1697ea987f3b223ba7819250cae33efacb068d23dc24859824a77844')", self.nodes[0].createrawtransaction, [{'txid': 'ZZZ7bb8b1697ea987f3b223ba7819250cae33efacb068d23dc24859824a77844'}], {}) - assert_raises_rpc_error(-8, "Invalid parameter, missing vout key", self.nodes[0].createrawtransaction, [{'txid': txid}], {}) - assert_raises_rpc_error(-8, "Invalid parameter, missing vout key", self.nodes[0].createrawtransaction, [{'txid': txid, 'vout': 'foo'}], {}) - assert_raises_rpc_error(-8, "Invalid parameter, vout cannot be negative", self.nodes[0].createrawtransaction, [{'txid': txid, 'vout': -1}], {}) - assert_raises_rpc_error(-8, "Invalid parameter, sequence number is out of range", self.nodes[0].createrawtransaction, [{'txid': txid, 'vout': 0, 'sequence': -1}], {}) + txid = "ZZZ7bb8b1697ea987f3b223ba7819250cae33efacb068d23dc24859824a77844" + assert_raises_rpc_error(-8, f"txid must be hexadecimal string (not '{txid}')", self.nodes[0].createrawtransaction, [{'txid': txid}], {}) + assert_raises_rpc_error(-8, "Invalid parameter, missing vout key", self.nodes[0].createrawtransaction, [{'txid': TXID}], {}) + assert_raises_rpc_error(-8, "Invalid parameter, missing vout key", self.nodes[0].createrawtransaction, [{'txid': TXID, 'vout': 'foo'}], {}) + assert_raises_rpc_error(-8, "Invalid parameter, vout cannot be negative", self.nodes[0].createrawtransaction, [{'txid': TXID, 'vout': -1}], {}) + # sequence number out of range + for invalid_seq in [-1, 4294967296]: + inputs = [{'txid': TXID, 'vout': 1, 'sequence': invalid_seq}] + outputs = {self.nodes[0].getnewaddress(): 1} + assert_raises_rpc_error(-8, 'Invalid parameter, sequence number is out of range', + self.nodes[0].createrawtransaction, inputs, outputs) + # with valid sequence number + for valid_seq in [1000, 4294967294]: + inputs = [{'txid': TXID, 'vout': 1, 'sequence': valid_seq}] + outputs = {self.nodes[0].getnewaddress(): 1} + rawtx = self.nodes[0].createrawtransaction(inputs, outputs) + decrawtx = self.nodes[0].decoderawtransaction(rawtx) + assert_equal(decrawtx['vin'][0]['sequence'], valid_seq) # Test `createrawtransaction` invalid `outputs` address = self.nodes[0].getnewaddress() @@ -127,53 +235,51 @@ class RawTransactionsTest(BitcoinTestFramework): # Test `createrawtransaction` invalid `replaceable` assert_raises_rpc_error(-3, "Expected type bool", self.nodes[0].createrawtransaction, [], {}, 0, 'foo') - self.log.info('Check that createrawtransaction accepts an array and object as outputs') + # Test that createrawtransaction accepts an array and object as outputs # One output - tx = tx_from_hex(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs={address: 99})) + tx = tx_from_hex(self.nodes[2].createrawtransaction(inputs=[{'txid': TXID, 'vout': 9}], outputs={address: 99})) assert_equal(len(tx.vout), 1) assert_equal( tx.serialize().hex(), - self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}]), + self.nodes[2].createrawtransaction(inputs=[{'txid': TXID, 'vout': 9}], outputs=[{address: 99}]), ) # Two outputs - tx = tx_from_hex(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=OrderedDict([(address, 99), (address2, 99)]))) + tx = tx_from_hex(self.nodes[2].createrawtransaction(inputs=[{'txid': TXID, 'vout': 9}], outputs=OrderedDict([(address, 99), (address2, 99)]))) assert_equal(len(tx.vout), 2) assert_equal( tx.serialize().hex(), - self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}, {address2: 99}]), + self.nodes[2].createrawtransaction(inputs=[{'txid': TXID, 'vout': 9}], outputs=[{address: 99}, {address2: 99}]), ) # Multiple mixed outputs - tx = tx_from_hex(self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=multidict([(address, 99), (address2, 99), ('data', '99')]))) + tx = tx_from_hex(self.nodes[2].createrawtransaction(inputs=[{'txid': TXID, 'vout': 9}], outputs=multidict([(address, 99), (address2, 99), ('data', '99')]))) assert_equal(len(tx.vout), 3) assert_equal( tx.serialize().hex(), - self.nodes[2].createrawtransaction(inputs=[{'txid': txid, 'vout': 9}], outputs=[{address: 99}, {address2: 99}, {'data': '99'}]), + self.nodes[2].createrawtransaction(inputs=[{'txid': TXID, 'vout': 9}], outputs=[{address: 99}, {address2: 99}, {'data': '99'}]), ) + def signrawtransactionwithwallet_tests(self): for type in ["bech32", "p2sh-segwit", "legacy"]: + self.log.info(f"Test signrawtransactionwithwallet with missing prevtx info ({type})") addr = self.nodes[0].getnewaddress("", type) addrinfo = self.nodes[0].getaddressinfo(addr) pubkey = addrinfo["scriptPubKey"] + inputs = [{'txid': TXID, 'vout': 3, 'sequence': 1000}] + outputs = {self.nodes[0].getnewaddress(): 1} + rawtx = self.nodes[0].createrawtransaction(inputs, outputs) - self.log.info('sendrawtransaction with missing prevtx info (%s)' %(type)) - - # Test `signrawtransactionwithwallet` invalid `prevtxs` - inputs = [ {'txid' : txid, 'vout' : 3, 'sequence' : 1000}] - outputs = { self.nodes[0].getnewaddress() : 1 } - rawtx = self.nodes[0].createrawtransaction(inputs, outputs) - - prevtx = dict(txid=txid, scriptPubKey=pubkey, vout=3, amount=1) + prevtx = dict(txid=TXID, scriptPubKey=pubkey, vout=3, amount=1) succ = self.nodes[0].signrawtransactionwithwallet(rawtx, [prevtx]) assert succ["complete"] + if type == "legacy": del prevtx["amount"] succ = self.nodes[0].signrawtransactionwithwallet(rawtx, [prevtx]) assert succ["complete"] - - if type != "legacy": + else: assert_raises_rpc_error(-3, "Missing amount", self.nodes[0].signrawtransactionwithwallet, rawtx, [ { - "txid": txid, + "txid": TXID, "scriptPubKey": pubkey, "vout": 3, } @@ -181,7 +287,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_raises_rpc_error(-3, "Missing vout", self.nodes[0].signrawtransactionwithwallet, rawtx, [ { - "txid": txid, + "txid": TXID, "scriptPubKey": pubkey, "amount": 1, } @@ -195,273 +301,23 @@ class RawTransactionsTest(BitcoinTestFramework): ]) assert_raises_rpc_error(-3, "Missing scriptPubKey", self.nodes[0].signrawtransactionwithwallet, rawtx, [ { - "txid": txid, + "txid": TXID, "vout": 3, "amount": 1 } ]) - ######################################### - # sendrawtransaction with missing input # - ######################################### - - self.log.info('sendrawtransaction with missing input') - inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1}] #won't exists - outputs = { self.nodes[0].getnewaddress() : 4.998 } - rawtx = self.nodes[2].createrawtransaction(inputs, outputs) - rawtx = self.nodes[2].signrawtransactionwithwallet(rawtx) - - # This will raise an exception since there are missing inputs + def sendrawtransaction_tests(self): + self.log.info("Test sendrawtransaction with missing input") + inputs = [{'txid': TXID, 'vout': 1}] # won't exist + outputs = {self.nodes[0].getnewaddress(): 4.998} + rawtx = self.nodes[2].createrawtransaction(inputs, outputs) + rawtx = self.nodes[2].signrawtransactionwithwallet(rawtx) assert_raises_rpc_error(-25, "bad-txns-inputs-missingorspent", self.nodes[2].sendrawtransaction, rawtx['hex']) - ##################################### - # getrawtransaction with block hash # - ##################################### - - # make a tx by sending then generate 2 blocks; block1 has the tx in it - tx = self.nodes[2].sendtoaddress(self.nodes[1].getnewaddress(), 1) - block1, block2 = self.nodes[2].generate(2) - self.sync_all() - # We should be able to get the raw transaction by providing the correct block - gottx = self.nodes[0].getrawtransaction(tx, True, block1) - assert_equal(gottx['txid'], tx) - assert_equal(gottx['in_active_chain'], True) - # We should not have the 'in_active_chain' flag when we don't provide a block - gottx = self.nodes[0].getrawtransaction(tx, True) - assert_equal(gottx['txid'], tx) - assert 'in_active_chain' not in gottx - # We should not get the tx if we provide an unrelated block - assert_raises_rpc_error(-5, "No such transaction found", self.nodes[0].getrawtransaction, tx, True, block2) - # An invalid block hash should raise the correct errors - assert_raises_rpc_error(-1, "JSON value is not a string as expected", self.nodes[0].getrawtransaction, tx, True, True) - assert_raises_rpc_error(-8, "parameter 3 must be of length 64 (not 6, for 'foobar')", self.nodes[0].getrawtransaction, tx, True, "foobar") - assert_raises_rpc_error(-8, "parameter 3 must be of length 64 (not 8, for 'abcd1234')", self.nodes[0].getrawtransaction, tx, True, "abcd1234") - assert_raises_rpc_error(-8, "parameter 3 must be hexadecimal string (not 'ZZZ0000000000000000000000000000000000000000000000000000000000000')", self.nodes[0].getrawtransaction, tx, True, "ZZZ0000000000000000000000000000000000000000000000000000000000000") - assert_raises_rpc_error(-5, "Block hash not found", self.nodes[0].getrawtransaction, tx, True, "0000000000000000000000000000000000000000000000000000000000000000") - # Undo the blocks and check in_active_chain - self.nodes[0].invalidateblock(block1) - gottx = self.nodes[0].getrawtransaction(txid=tx, verbose=True, blockhash=block1) - assert_equal(gottx['in_active_chain'], False) - self.nodes[0].reconsiderblock(block1) - assert_equal(self.nodes[0].getbestblockhash(), block2) - - if not self.options.descriptors: - # The traditional multisig workflow does not work with descriptor wallets so these are legacy only. - # The multisig workflow with descriptor wallets uses PSBTs and is tested elsewhere, no need to do them here. - ######################### - # RAW TX MULTISIG TESTS # - ######################### - # 2of2 test - addr1 = self.nodes[2].getnewaddress() - addr2 = self.nodes[2].getnewaddress() - - addr1Obj = self.nodes[2].getaddressinfo(addr1) - addr2Obj = self.nodes[2].getaddressinfo(addr2) - - # Tests for createmultisig and addmultisigaddress - assert_raises_rpc_error(-5, "Invalid public key", self.nodes[0].createmultisig, 1, ["01020304"]) - self.nodes[0].createmultisig(2, [addr1Obj['pubkey'], addr2Obj['pubkey']]) # createmultisig can only take public keys - assert_raises_rpc_error(-5, "Invalid public key", self.nodes[0].createmultisig, 2, [addr1Obj['pubkey'], addr1]) # addmultisigaddress can take both pubkeys and addresses so long as they are in the wallet, which is tested here. - - mSigObj = self.nodes[2].addmultisigaddress(2, [addr1Obj['pubkey'], addr1])['address'] - - #use balance deltas instead of absolute values - bal = self.nodes[2].getbalance() - - # send 1.2 BTC to msig adr - txId = self.nodes[0].sendtoaddress(mSigObj, 1.2) - self.sync_all() - self.nodes[0].generate(1) - self.sync_all() - assert_equal(self.nodes[2].getbalance(), bal+Decimal('1.20000000')) #node2 has both keys of the 2of2 ms addr., tx should affect the balance - - - # 2of3 test from different nodes - bal = self.nodes[2].getbalance() - addr1 = self.nodes[1].getnewaddress() - addr2 = self.nodes[2].getnewaddress() - addr3 = self.nodes[2].getnewaddress() - - addr1Obj = self.nodes[1].getaddressinfo(addr1) - addr2Obj = self.nodes[2].getaddressinfo(addr2) - addr3Obj = self.nodes[2].getaddressinfo(addr3) - - mSigObj = self.nodes[2].addmultisigaddress(2, [addr1Obj['pubkey'], addr2Obj['pubkey'], addr3Obj['pubkey']])['address'] - - txId = self.nodes[0].sendtoaddress(mSigObj, 2.2) - decTx = self.nodes[0].gettransaction(txId) - rawTx = self.nodes[0].decoderawtransaction(decTx['hex']) - self.sync_all() - self.nodes[0].generate(1) - self.sync_all() - - #THIS IS AN INCOMPLETE FEATURE - #NODE2 HAS TWO OF THREE KEY AND THE FUNDS SHOULD BE SPENDABLE AND COUNT AT BALANCE CALCULATION - assert_equal(self.nodes[2].getbalance(), bal) #for now, assume the funds of a 2of3 multisig tx are not marked as spendable - - txDetails = self.nodes[0].gettransaction(txId, True) - rawTx = self.nodes[0].decoderawtransaction(txDetails['hex']) - vout = next(o for o in rawTx['vout'] if o['value'] == Decimal('2.20000000')) - - bal = self.nodes[0].getbalance() - inputs = [{ "txid" : txId, "vout" : vout['n'], "scriptPubKey" : vout['scriptPubKey']['hex'], "amount" : vout['value']}] - outputs = { self.nodes[0].getnewaddress() : 2.19 } - rawTx = self.nodes[2].createrawtransaction(inputs, outputs) - rawTxPartialSigned = self.nodes[1].signrawtransactionwithwallet(rawTx, inputs) - assert_equal(rawTxPartialSigned['complete'], False) #node1 only has one key, can't comp. sign the tx - - rawTxSigned = self.nodes[2].signrawtransactionwithwallet(rawTx, inputs) - assert_equal(rawTxSigned['complete'], True) #node2 can sign the tx compl., own two of three keys - self.nodes[2].sendrawtransaction(rawTxSigned['hex']) - rawTx = self.nodes[0].decoderawtransaction(rawTxSigned['hex']) - self.sync_all() - self.nodes[0].generate(1) - self.sync_all() - assert_equal(self.nodes[0].getbalance(), bal+Decimal('50.00000000')+Decimal('2.19000000')) #block reward + tx - - # 2of2 test for combining transactions - bal = self.nodes[2].getbalance() - addr1 = self.nodes[1].getnewaddress() - addr2 = self.nodes[2].getnewaddress() - - addr1Obj = self.nodes[1].getaddressinfo(addr1) - addr2Obj = self.nodes[2].getaddressinfo(addr2) - - self.nodes[1].addmultisigaddress(2, [addr1Obj['pubkey'], addr2Obj['pubkey']])['address'] - mSigObj = self.nodes[2].addmultisigaddress(2, [addr1Obj['pubkey'], addr2Obj['pubkey']])['address'] - mSigObjValid = self.nodes[2].getaddressinfo(mSigObj) - - txId = self.nodes[0].sendtoaddress(mSigObj, 2.2) - decTx = self.nodes[0].gettransaction(txId) - rawTx2 = self.nodes[0].decoderawtransaction(decTx['hex']) - self.sync_all() - self.nodes[0].generate(1) - self.sync_all() - - assert_equal(self.nodes[2].getbalance(), bal) # the funds of a 2of2 multisig tx should not be marked as spendable - - txDetails = self.nodes[0].gettransaction(txId, True) - rawTx2 = self.nodes[0].decoderawtransaction(txDetails['hex']) - vout = next(o for o in rawTx2['vout'] if o['value'] == Decimal('2.20000000')) - - bal = self.nodes[0].getbalance() - inputs = [{ "txid" : txId, "vout" : vout['n'], "scriptPubKey" : vout['scriptPubKey']['hex'], "redeemScript" : mSigObjValid['hex'], "amount" : vout['value']}] - outputs = { self.nodes[0].getnewaddress() : 2.19 } - rawTx2 = self.nodes[2].createrawtransaction(inputs, outputs) - rawTxPartialSigned1 = self.nodes[1].signrawtransactionwithwallet(rawTx2, inputs) - self.log.debug(rawTxPartialSigned1) - assert_equal(rawTxPartialSigned1['complete'], False) #node1 only has one key, can't comp. sign the tx - - rawTxPartialSigned2 = self.nodes[2].signrawtransactionwithwallet(rawTx2, inputs) - self.log.debug(rawTxPartialSigned2) - assert_equal(rawTxPartialSigned2['complete'], False) #node2 only has one key, can't comp. sign the tx - rawTxComb = self.nodes[2].combinerawtransaction([rawTxPartialSigned1['hex'], rawTxPartialSigned2['hex']]) - self.log.debug(rawTxComb) - self.nodes[2].sendrawtransaction(rawTxComb) - rawTx2 = self.nodes[0].decoderawtransaction(rawTxComb) - self.sync_all() - self.nodes[0].generate(1) - self.sync_all() - assert_equal(self.nodes[0].getbalance(), bal+Decimal('50.00000000')+Decimal('2.19000000')) #block reward + tx - - # decoderawtransaction tests - # witness transaction - encrawtx = "010000000001010000000000000072c1a6a246ae63f74f931e8365e15a089c68d61900000000000000000000ffffffff0100e1f50500000000000102616100000000" - decrawtx = self.nodes[0].decoderawtransaction(encrawtx, True) # decode as witness transaction - assert_equal(decrawtx['vout'][0]['value'], Decimal('1.00000000')) - assert_raises_rpc_error(-22, 'TX decode failed', self.nodes[0].decoderawtransaction, encrawtx, False) # force decode as non-witness transaction - # non-witness transaction - encrawtx = "01000000010000000000000072c1a6a246ae63f74f931e8365e15a089c68d61900000000000000000000ffffffff0100e1f505000000000000000000" - decrawtx = self.nodes[0].decoderawtransaction(encrawtx, False) # decode as non-witness transaction - assert_equal(decrawtx['vout'][0]['value'], Decimal('1.00000000')) - # known ambiguous transaction in the chain (see https://github.com/bitcoin/bitcoin/issues/20579) - encrawtx = "020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff4b03c68708046ff8415c622f4254432e434f4d2ffabe6d6de1965d02c68f928e5b244ab1965115a36f56eb997633c7f690124bbf43644e23080000000ca3d3af6d005a65ff0200fd00000000ffffffff03f4c1fb4b0000000016001497cfc76442fe717f2a3f0cc9c175f7561b6619970000000000000000266a24aa21a9ed957d1036a80343e0d1b659497e1b48a38ebe876a056d45965fac4a85cda84e1900000000000000002952534b424c4f434b3a8e092581ab01986cbadc84f4b43f4fa4bb9e7a2e2a0caf9b7cf64d939028e22c0120000000000000000000000000000000000000000000000000000000000000000000000000" - decrawtx = self.nodes[0].decoderawtransaction(encrawtx) - decrawtx_wit = self.nodes[0].decoderawtransaction(encrawtx, True) - assert_raises_rpc_error(-22, 'TX decode failed', self.nodes[0].decoderawtransaction, encrawtx, False) # fails to decode as non-witness transaction - assert_equal(decrawtx, decrawtx_wit) # the witness interpretation should be chosen - assert_equal(decrawtx['vin'][0]['coinbase'], "03c68708046ff8415c622f4254432e434f4d2ffabe6d6de1965d02c68f928e5b244ab1965115a36f56eb997633c7f690124bbf43644e23080000000ca3d3af6d005a65ff0200fd00000000") - - # Basic signrawtransaction test - addr = self.nodes[1].getnewaddress() - txid = self.nodes[0].sendtoaddress(addr, 10) - self.nodes[0].generate(1) - self.sync_all() - vout = find_vout_for_address(self.nodes[1], txid, addr) - rawTx = self.nodes[1].createrawtransaction([{'txid': txid, 'vout': vout}], {self.nodes[1].getnewaddress(): 9.999}) - rawTxSigned = self.nodes[1].signrawtransactionwithwallet(rawTx) - txId = self.nodes[1].sendrawtransaction(rawTxSigned['hex']) - self.nodes[0].generate(1) - self.sync_all() - - # getrawtransaction tests - # 1. valid parameters - only supply txid - assert_equal(self.nodes[0].getrawtransaction(txId), rawTxSigned['hex']) - - # 2. valid parameters - supply txid and 0 for non-verbose - assert_equal(self.nodes[0].getrawtransaction(txId, 0), rawTxSigned['hex']) - - # 3. valid parameters - supply txid and False for non-verbose - assert_equal(self.nodes[0].getrawtransaction(txId, False), rawTxSigned['hex']) - - # 4. valid parameters - supply txid and 1 for verbose. - # We only check the "hex" field of the output so we don't need to update this test every time the output format changes. - assert_equal(self.nodes[0].getrawtransaction(txId, 1)["hex"], rawTxSigned['hex']) - - # 5. valid parameters - supply txid and True for non-verbose - assert_equal(self.nodes[0].getrawtransaction(txId, True)["hex"], rawTxSigned['hex']) - - # 6. invalid parameters - supply txid and string "Flase" - assert_raises_rpc_error(-1, "not a boolean", self.nodes[0].getrawtransaction, txId, "Flase") - - # 7. invalid parameters - supply txid and empty array - assert_raises_rpc_error(-1, "not a boolean", self.nodes[0].getrawtransaction, txId, []) - - # 8. invalid parameters - supply txid and empty dict - assert_raises_rpc_error(-1, "not a boolean", self.nodes[0].getrawtransaction, txId, {}) - - inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : 1000}] - outputs = { self.nodes[0].getnewaddress() : 1 } - rawtx = self.nodes[0].createrawtransaction(inputs, outputs) - decrawtx= self.nodes[0].decoderawtransaction(rawtx) - assert_equal(decrawtx['vin'][0]['sequence'], 1000) - - # 9. invalid parameters - sequence number out of range - inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : -1}] - outputs = { self.nodes[0].getnewaddress() : 1 } - assert_raises_rpc_error(-8, 'Invalid parameter, sequence number is out of range', self.nodes[0].createrawtransaction, inputs, outputs) - - # 10. invalid parameters - sequence number out of range - inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : 4294967296}] - outputs = { self.nodes[0].getnewaddress() : 1 } - assert_raises_rpc_error(-8, 'Invalid parameter, sequence number is out of range', self.nodes[0].createrawtransaction, inputs, outputs) - - inputs = [ {'txid' : "1d1d4e24ed99057e84c3f80fd8fbec79ed9e1acee37da269356ecea000000000", 'vout' : 1, 'sequence' : 4294967294}] - outputs = { self.nodes[0].getnewaddress() : 1 } - rawtx = self.nodes[0].createrawtransaction(inputs, outputs) - decrawtx= self.nodes[0].decoderawtransaction(rawtx) - assert_equal(decrawtx['vin'][0]['sequence'], 4294967294) - - #################################### - # TRANSACTION VERSION NUMBER TESTS # - #################################### - - # Test the minimum transaction version number that fits in a signed 32-bit integer. - # As transaction version is unsigned, this should convert to its unsigned equivalent. - tx = CTransaction() - tx.nVersion = -0x80000000 - rawtx = tx.serialize().hex() - decrawtx = self.nodes[0].decoderawtransaction(rawtx) - assert_equal(decrawtx['version'], 0x80000000) - - # Test the maximum transaction version number that fits in a signed 32-bit integer. - tx = CTransaction() - tx.nVersion = 0x7fffffff - rawtx = tx.serialize().hex() - decrawtx = self.nodes[0].decoderawtransaction(rawtx) - assert_equal(decrawtx['version'], 0x7fffffff) - - self.log.info('sendrawtransaction/testmempoolaccept with maxfeerate') + def sendrawtransaction_testmempoolaccept_tests(self): + self.log.info("Test sendrawtransaction/testmempoolaccept with maxfeerate") + fee_exceeds_max = "Fee exceeds maximum configured by user (e.g. -maxtxfee, maxfeerate)" # Test a transaction with a small fee. txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 1.0) @@ -469,9 +325,9 @@ class RawTransactionsTest(BitcoinTestFramework): vout = next(o for o in rawTx['vout'] if o['value'] == Decimal('1.00000000')) self.sync_all() - inputs = [{ "txid" : txId, "vout" : vout['n'] }] + inputs = [{"txid": txId, "vout": vout['n']}] # Fee 10,000 satoshis, (1 - (10000 sat * 0.00000001 BTC/sat)) = 0.9999 - outputs = { self.nodes[0].getnewaddress() : Decimal("0.99990000") } + outputs = {self.nodes[0].getnewaddress(): Decimal("0.99990000")} rawTx = self.nodes[2].createrawtransaction(inputs, outputs) rawTxSigned = self.nodes[2].signrawtransactionwithwallet(rawTx) assert_equal(rawTxSigned['complete'], True) @@ -481,7 +337,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(testres['allowed'], False) assert_equal(testres['reject-reason'], 'max-fee-exceeded') # and sendrawtransaction should throw - assert_raises_rpc_error(-25, 'Fee exceeds maximum configured by user (e.g. -maxtxfee, maxfeerate)', self.nodes[2].sendrawtransaction, rawTxSigned['hex'], 0.00001000) + assert_raises_rpc_error(-25, fee_exceeds_max, self.nodes[2].sendrawtransaction, rawTxSigned['hex'], 0.00001000) # and the following calls should both succeed testres = self.nodes[2].testmempoolaccept(rawtxs=[rawTxSigned['hex']])[0] assert_equal(testres['allowed'], True) @@ -493,9 +349,9 @@ class RawTransactionsTest(BitcoinTestFramework): vout = next(o for o in rawTx['vout'] if o['value'] == Decimal('1.00000000')) self.sync_all() - inputs = [{ "txid" : txId, "vout" : vout['n'] }] + inputs = [{"txid": txId, "vout": vout['n']}] # Fee 2,000,000 satoshis, (1 - (2000000 sat * 0.00000001 BTC/sat)) = 0.98 - outputs = { self.nodes[0].getnewaddress() : Decimal("0.98000000") } + outputs = {self.nodes[0].getnewaddress() : Decimal("0.98000000")} rawTx = self.nodes[2].createrawtransaction(inputs, outputs) rawTxSigned = self.nodes[2].signrawtransactionwithwallet(rawTx) assert_equal(rawTxSigned['complete'], True) @@ -505,12 +361,181 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal(testres['allowed'], False) assert_equal(testres['reject-reason'], 'max-fee-exceeded') # and sendrawtransaction should throw - assert_raises_rpc_error(-25, 'Fee exceeds maximum configured by user (e.g. -maxtxfee, maxfeerate)', self.nodes[2].sendrawtransaction, rawTxSigned['hex']) + assert_raises_rpc_error(-25, fee_exceeds_max, self.nodes[2].sendrawtransaction, rawTxSigned['hex']) # and the following calls should both succeed testres = self.nodes[2].testmempoolaccept(rawtxs=[rawTxSigned['hex']], maxfeerate='0.20000000')[0] assert_equal(testres['allowed'], True) self.nodes[2].sendrawtransaction(hexstring=rawTxSigned['hex'], maxfeerate='0.20000000') + self.log.info("Test sendrawtransaction/testmempoolaccept with tx already in the chain") + self.generate(self.nodes[2], 1) + self.sync_blocks() + for node in self.nodes: + testres = node.testmempoolaccept([rawTxSigned['hex']])[0] + assert_equal(testres['allowed'], False) + assert_equal(testres['reject-reason'], 'txn-already-known') + assert_raises_rpc_error(-27, 'Transaction already in block chain', node.sendrawtransaction, rawTxSigned['hex']) + + def decoderawtransaction_tests(self): + self.log.info("Test decoderawtransaction") + # witness transaction + encrawtx = "010000000001010000000000000072c1a6a246ae63f74f931e8365e15a089c68d61900000000000000000000ffffffff0100e1f50500000000000102616100000000" + decrawtx = self.nodes[0].decoderawtransaction(encrawtx, True) # decode as witness transaction + assert_equal(decrawtx['vout'][0]['value'], Decimal('1.00000000')) + assert_raises_rpc_error(-22, 'TX decode failed', self.nodes[0].decoderawtransaction, encrawtx, False) # force decode as non-witness transaction + # non-witness transaction + encrawtx = "01000000010000000000000072c1a6a246ae63f74f931e8365e15a089c68d61900000000000000000000ffffffff0100e1f505000000000000000000" + decrawtx = self.nodes[0].decoderawtransaction(encrawtx, False) # decode as non-witness transaction + assert_equal(decrawtx['vout'][0]['value'], Decimal('1.00000000')) + # known ambiguous transaction in the chain (see https://github.com/bitcoin/bitcoin/issues/20579) + coinbase = "03c68708046ff8415c622f4254432e434f4d2ffabe6d6de1965d02c68f928e5b244ab1965115a36f56eb997633c7f690124bbf43644e23080000000ca3d3af6d005a65ff0200fd00000000" + encrawtx = f"020000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff4b{coinbase}" \ + "ffffffff03f4c1fb4b0000000016001497cfc76442fe717f2a3f0cc9c175f7561b6619970000000000000000266a24aa21a9ed957d1036a80343e0d1b659497e1b48a38ebe876a056d45965fac4a85cda84e1900000000000000002952534b424c4f434b3a8e092581ab01986cbadc84f4b43f4fa4bb9e7a2e2a0caf9b7cf64d939028e22c0120000000000000000000000000000000000000000000000000000000000000000000000000" + decrawtx = self.nodes[0].decoderawtransaction(encrawtx) + decrawtx_wit = self.nodes[0].decoderawtransaction(encrawtx, True) + assert_raises_rpc_error(-22, 'TX decode failed', self.nodes[0].decoderawtransaction, encrawtx, False) # fails to decode as non-witness transaction + assert_equal(decrawtx, decrawtx_wit) # the witness interpretation should be chosen + assert_equal(decrawtx['vin'][0]['coinbase'], coinbase) + + def transaction_version_number_tests(self): + self.log.info("Test transaction version numbers") + + # Test the minimum transaction version number that fits in a signed 32-bit integer. + # As transaction version is unsigned, this should convert to its unsigned equivalent. + tx = CTransaction() + tx.nVersion = -0x80000000 + rawtx = tx.serialize().hex() + decrawtx = self.nodes[0].decoderawtransaction(rawtx) + assert_equal(decrawtx['version'], 0x80000000) + + # Test the maximum transaction version number that fits in a signed 32-bit integer. + tx = CTransaction() + tx.nVersion = 0x7fffffff + rawtx = tx.serialize().hex() + decrawtx = self.nodes[0].decoderawtransaction(rawtx) + assert_equal(decrawtx['version'], 0x7fffffff) + + def raw_multisig_transaction_legacy_tests(self): + self.log.info("Test raw multisig transactions (legacy)") + # The traditional multisig workflow does not work with descriptor wallets so these are legacy only. + # The multisig workflow with descriptor wallets uses PSBTs and is tested elsewhere, no need to do them here. + + # 2of2 test + addr1 = self.nodes[2].getnewaddress() + addr2 = self.nodes[2].getnewaddress() + + addr1Obj = self.nodes[2].getaddressinfo(addr1) + addr2Obj = self.nodes[2].getaddressinfo(addr2) + + # Tests for createmultisig and addmultisigaddress + assert_raises_rpc_error(-5, "Invalid public key", self.nodes[0].createmultisig, 1, ["01020304"]) + # createmultisig can only take public keys + self.nodes[0].createmultisig(2, [addr1Obj['pubkey'], addr2Obj['pubkey']]) + # addmultisigaddress can take both pubkeys and addresses so long as they are in the wallet, which is tested here + assert_raises_rpc_error(-5, "Invalid public key", self.nodes[0].createmultisig, 2, [addr1Obj['pubkey'], addr1]) + + mSigObj = self.nodes[2].addmultisigaddress(2, [addr1Obj['pubkey'], addr1])['address'] + + # use balance deltas instead of absolute values + bal = self.nodes[2].getbalance() + + # send 1.2 BTC to msig adr + txId = self.nodes[0].sendtoaddress(mSigObj, 1.2) + self.sync_all() + self.generate(self.nodes[0], 1) + self.sync_all() + # node2 has both keys of the 2of2 ms addr, tx should affect the balance + assert_equal(self.nodes[2].getbalance(), bal + Decimal('1.20000000')) + + + # 2of3 test from different nodes + bal = self.nodes[2].getbalance() + addr1 = self.nodes[1].getnewaddress() + addr2 = self.nodes[2].getnewaddress() + addr3 = self.nodes[2].getnewaddress() + + addr1Obj = self.nodes[1].getaddressinfo(addr1) + addr2Obj = self.nodes[2].getaddressinfo(addr2) + addr3Obj = self.nodes[2].getaddressinfo(addr3) + + mSigObj = self.nodes[2].addmultisigaddress(2, [addr1Obj['pubkey'], addr2Obj['pubkey'], addr3Obj['pubkey']])['address'] + + txId = self.nodes[0].sendtoaddress(mSigObj, 2.2) + decTx = self.nodes[0].gettransaction(txId) + rawTx = self.nodes[0].decoderawtransaction(decTx['hex']) + self.sync_all() + self.generate(self.nodes[0], 1) + self.sync_all() + + # THIS IS AN INCOMPLETE FEATURE + # NODE2 HAS TWO OF THREE KEYS AND THE FUNDS SHOULD BE SPENDABLE AND COUNT AT BALANCE CALCULATION + assert_equal(self.nodes[2].getbalance(), bal) # for now, assume the funds of a 2of3 multisig tx are not marked as spendable + + txDetails = self.nodes[0].gettransaction(txId, True) + rawTx = self.nodes[0].decoderawtransaction(txDetails['hex']) + vout = next(o for o in rawTx['vout'] if o['value'] == Decimal('2.20000000')) + + bal = self.nodes[0].getbalance() + inputs = [{"txid": txId, "vout": vout['n'], "scriptPubKey": vout['scriptPubKey']['hex'], "amount": vout['value']}] + outputs = {self.nodes[0].getnewaddress(): 2.19} + rawTx = self.nodes[2].createrawtransaction(inputs, outputs) + rawTxPartialSigned = self.nodes[1].signrawtransactionwithwallet(rawTx, inputs) + assert_equal(rawTxPartialSigned['complete'], False) # node1 only has one key, can't comp. sign the tx + + rawTxSigned = self.nodes[2].signrawtransactionwithwallet(rawTx, inputs) + assert_equal(rawTxSigned['complete'], True) # node2 can sign the tx compl., own two of three keys + self.nodes[2].sendrawtransaction(rawTxSigned['hex']) + rawTx = self.nodes[0].decoderawtransaction(rawTxSigned['hex']) + self.sync_all() + self.generate(self.nodes[0], 1) + self.sync_all() + assert_equal(self.nodes[0].getbalance(), bal + Decimal('50.00000000') + Decimal('2.19000000')) # block reward + tx + + # 2of2 test for combining transactions + bal = self.nodes[2].getbalance() + addr1 = self.nodes[1].getnewaddress() + addr2 = self.nodes[2].getnewaddress() + + addr1Obj = self.nodes[1].getaddressinfo(addr1) + addr2Obj = self.nodes[2].getaddressinfo(addr2) + + self.nodes[1].addmultisigaddress(2, [addr1Obj['pubkey'], addr2Obj['pubkey']])['address'] + mSigObj = self.nodes[2].addmultisigaddress(2, [addr1Obj['pubkey'], addr2Obj['pubkey']])['address'] + mSigObjValid = self.nodes[2].getaddressinfo(mSigObj) + + txId = self.nodes[0].sendtoaddress(mSigObj, 2.2) + decTx = self.nodes[0].gettransaction(txId) + rawTx2 = self.nodes[0].decoderawtransaction(decTx['hex']) + self.sync_all() + self.generate(self.nodes[0], 1) + self.sync_all() + + assert_equal(self.nodes[2].getbalance(), bal) # the funds of a 2of2 multisig tx should not be marked as spendable + + txDetails = self.nodes[0].gettransaction(txId, True) + rawTx2 = self.nodes[0].decoderawtransaction(txDetails['hex']) + vout = next(o for o in rawTx2['vout'] if o['value'] == Decimal('2.20000000')) + + bal = self.nodes[0].getbalance() + inputs = [{"txid": txId, "vout": vout['n'], "scriptPubKey": vout['scriptPubKey']['hex'], "redeemScript": mSigObjValid['hex'], "amount": vout['value']}] + outputs = {self.nodes[0].getnewaddress(): 2.19} + rawTx2 = self.nodes[2].createrawtransaction(inputs, outputs) + rawTxPartialSigned1 = self.nodes[1].signrawtransactionwithwallet(rawTx2, inputs) + self.log.debug(rawTxPartialSigned1) + assert_equal(rawTxPartialSigned1['complete'], False) # node1 only has one key, can't comp. sign the tx + + rawTxPartialSigned2 = self.nodes[2].signrawtransactionwithwallet(rawTx2, inputs) + self.log.debug(rawTxPartialSigned2) + assert_equal(rawTxPartialSigned2['complete'], False) # node2 only has one key, can't comp. sign the tx + rawTxComb = self.nodes[2].combinerawtransaction([rawTxPartialSigned1['hex'], rawTxPartialSigned2['hex']]) + self.log.debug(rawTxComb) + self.nodes[2].sendrawtransaction(rawTxComb) + rawTx2 = self.nodes[0].decoderawtransaction(rawTxComb) + self.sync_all() + self.generate(self.nodes[0], 1) + self.sync_all() + assert_equal(self.nodes[0].getbalance(), bal + Decimal('50.00000000') + Decimal('2.19000000')) # block reward + tx + if __name__ == '__main__': RawTransactionsTest().main() diff --git a/test/functional/rpc_scantxoutset.py b/test/functional/rpc_scantxoutset.py index 070f59d314..ec8205acd5 100755 --- a/test/functional/rpc_scantxoutset.py +++ b/test/functional/rpc_scantxoutset.py @@ -23,7 +23,7 @@ class ScantxoutsetTest(BitcoinTestFramework): def run_test(self): self.log.info("Mining blocks...") - self.nodes[0].generate(110) + self.generate(self.nodes[0], 110) addr_P2SH_SEGWIT = self.nodes[0].getnewaddress("", "p2sh-segwit") pubk1 = self.nodes[0].getaddressinfo(addr_P2SH_SEGWIT)['pubkey'] @@ -50,14 +50,14 @@ class ScantxoutsetTest(BitcoinTestFramework): self.nodes[0].sendtoaddress("mpQ8rokAhp1TAtJQR6F6TaUmjAWkAWYYBq", 16.384) # (m/1/1/1500) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.log.info("Stop node, remove wallet, mine again some blocks...") self.stop_node(0) shutil.rmtree(os.path.join(self.nodes[0].datadir, self.chain, 'wallets')) self.start_node(0, ['-nowallet']) self.import_deterministic_coinbase_privkeys() - self.nodes[0].generate(110) + self.generate(self.nodes[0], 110) scan = self.nodes[0].scantxoutset("start", []) info = self.nodes[0].gettxoutsetinfo() diff --git a/test/functional/rpc_signer.py b/test/functional/rpc_signer.py index 3188763f49..5c3722ef8f 100755 --- a/test/functional/rpc_signer.py +++ b/test/functional/rpc_signer.py @@ -27,6 +27,9 @@ class RPCSignerTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 + # The experimental syscall sandbox feature (-sandbox) is not compatible with -signer (which + # invokes execve). + self.disable_syscall_sandbox = True self.extra_args = [ [], @@ -53,8 +56,12 @@ class RPCSignerTest(BitcoinTestFramework): ) # Handle script missing: - assert_raises_rpc_error(-1, 'execve failed: No such file or directory', - self.nodes[3].enumeratesigners + assert_raises_rpc_error( + -1, + "CreateProcess failed: The system cannot find the file specified." + if platform.system() == "Windows" + else "execve failed: No such file or directory", + self.nodes[3].enumeratesigners, ) # Handle error thrown by script diff --git a/test/functional/rpc_signmessage.py b/test/functional/rpc_signmessagewithprivkey.py index 1c71732a61..95beba8730 100755 --- a/test/functional/rpc_signmessage.py +++ b/test/functional/rpc_signmessagewithprivkey.py @@ -2,7 +2,7 @@ # Copyright (c) 2016-2019 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test RPC commands for signing and verifying messages.""" +"""Test RPC commands for signing messages with private key.""" from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -10,14 +10,10 @@ from test_framework.util import ( assert_raises_rpc_error, ) -class SignMessagesTest(BitcoinTestFramework): +class SignMessagesWithPrivTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 - self.extra_args = [["-addresstype=legacy"]] - - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() def run_test(self): message = 'This is just a test message' @@ -30,33 +26,20 @@ class SignMessagesTest(BitcoinTestFramework): assert_equal(expected_signature, signature) assert self.nodes[0].verifymessage(address, signature, message) - self.log.info('test signing with an address with wallet') - address = self.nodes[0].getnewaddress() - signature = self.nodes[0].signmessage(address, message) - assert self.nodes[0].verifymessage(address, signature, message) - - self.log.info('test verifying with another address should not work') - other_address = self.nodes[0].getnewaddress() - other_signature = self.nodes[0].signmessage(other_address, message) - assert not self.nodes[0].verifymessage(other_address, signature, message) - assert not self.nodes[0].verifymessage(address, other_signature, message) - self.log.info('test parameter validity and error codes') - # signmessage(withprivkey) have two required parameters + # signmessagewithprivkey has two required parameters for num_params in [0, 1, 3, 4, 5]: param_list = ["dummy"]*num_params assert_raises_rpc_error(-1, "signmessagewithprivkey", self.nodes[0].signmessagewithprivkey, *param_list) - assert_raises_rpc_error(-1, "signmessage", self.nodes[0].signmessage, *param_list) # verifymessage has three required parameters for num_params in [0, 1, 2, 4, 5]: param_list = ["dummy"]*num_params assert_raises_rpc_error(-1, "verifymessage", self.nodes[0].verifymessage, *param_list) # invalid key or address provided assert_raises_rpc_error(-5, "Invalid private key", self.nodes[0].signmessagewithprivkey, "invalid_key", message) - assert_raises_rpc_error(-5, "Invalid address", self.nodes[0].signmessage, "invalid_addr", message) assert_raises_rpc_error(-5, "Invalid address", self.nodes[0].verifymessage, "invalid_addr", signature, message) # malformed signature provided - assert_raises_rpc_error(-3, "Malformed base64 encoding", self.nodes[0].verifymessage, self.nodes[0].getnewaddress(), "invalid_sig", message) + assert_raises_rpc_error(-3, "Malformed base64 encoding", self.nodes[0].verifymessage, 'mpLQjfK79b7CCV4VMJWEWAj5Mpx8Up5zxB', "invalid_sig", message) if __name__ == '__main__': - SignMessagesTest().main() + SignMessagesWithPrivTest().main() diff --git a/test/functional/rpc_signrawtransaction.py b/test/functional/rpc_signrawtransaction.py index ef5d08e7b9..c519d0c7d1 100755 --- a/test/functional/rpc_signrawtransaction.py +++ b/test/functional/rpc_signrawtransaction.py @@ -4,9 +4,10 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test transaction signing using the signrawtransaction* RPCs.""" -from test_framework.blocktools import COINBASE_MATURITY +from test_framework.blocktools import ( + COINBASE_MATURITY, +) from test_framework.address import ( - check_script, script_to_p2sh, script_to_p2wsh, ) @@ -16,23 +17,20 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, find_vout_for_address, - hex_str_to_bytes, ) from test_framework.messages import ( CTxInWitness, - sha256, tx_from_hex, ) from test_framework.script import ( CScript, - OP_0, OP_CHECKLOCKTIMEVERIFY, - OP_CHECKSIG, OP_CHECKSEQUENCEVERIFY, OP_DROP, OP_TRUE, ) from test_framework.script_util import ( + key_to_p2pk_script, key_to_p2pkh_script, script_to_p2sh_p2wsh_script, script_to_p2wsh_script, @@ -183,7 +181,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): def test_fully_signed_tx(self): self.log.info("Test signing a fully signed transaction does nothing") self.nodes[0].walletpassphrase("password", 9999) - self.nodes[0].generate(COINBASE_MATURITY + 1) + self.generate(self.nodes[0], COINBASE_MATURITY + 1) rawtx = self.nodes[0].createrawtransaction([], [{self.nodes[0].getnewaddress(): 10}]) fundedtx = self.nodes[0].fundrawtransaction(rawtx) signedtx = self.nodes[0].signrawtransactionwithwallet(fundedtx["hex"]) @@ -202,9 +200,9 @@ class SignRawTransactionsTest(BitcoinTestFramework): embedded_pubkey = eckey.get_pubkey().get_bytes().hex() p2sh_p2wsh_address = self.nodes[1].createmultisig(1, [embedded_pubkey], "p2sh-segwit") # send transaction to P2SH-P2WSH 1-of-1 multisig address - self.nodes[0].generate(COINBASE_MATURITY + 1) + self.generate(self.nodes[0], COINBASE_MATURITY + 1) self.nodes[0].sendtoaddress(p2sh_p2wsh_address["address"], 49.999) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() # Get the UTXO info from scantxoutset unspent_output = self.nodes[1].scantxoutset('start', [p2sh_p2wsh_address['descriptor']])['unspents'][0] @@ -231,15 +229,15 @@ class SignRawTransactionsTest(BitcoinTestFramework): embedded_pubkey = eckey.get_pubkey().get_bytes().hex() witness_script = { 'P2PKH': key_to_p2pkh_script(embedded_pubkey).hex(), - 'P2PK': CScript([hex_str_to_bytes(embedded_pubkey), OP_CHECKSIG]).hex() + 'P2PK': key_to_p2pk_script(embedded_pubkey).hex() }.get(tx_type, "Invalid tx_type") - redeem_script = CScript([OP_0, sha256(check_script(witness_script))]).hex() + redeem_script = script_to_p2wsh_script(witness_script).hex() addr = script_to_p2sh(redeem_script) script_pub_key = self.nodes[1].validateaddress(addr)['scriptPubKey'] # Fund that address txid = self.nodes[0].sendtoaddress(addr, 10) vout = find_vout_for_address(self.nodes[0], txid, addr) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) # Now create and sign a transaction spending that output on node[0], which doesn't know the scripts or keys spending_tx = self.nodes[0].createrawtransaction([{'txid': txid, 'vout': vout}], {self.nodes[1].getnewaddress(): Decimal("9.999")}) spending_tx_signed = self.nodes[0].signrawtransactionwithkey(spending_tx, [embedded_privkey], [{'txid': txid, 'vout': vout, 'scriptPubKey': script_pub_key, 'redeemScript': redeem_script, 'witnessScript': witness_script, 'amount': 10}]) @@ -273,7 +271,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): getcontext().prec = 8 # Make sure CSV is active - self.nodes[0].generate(500) + assert self.nodes[0].getblockchaininfo()['softforks']['csv']['active'] # Create a P2WSH script with CSV script = CScript([1, OP_CHECKSEQUENCEVERIFY, OP_DROP]) @@ -282,7 +280,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): # Fund that address and make the spend txid = self.nodes[0].sendtoaddress(address, 1) vout = find_vout_for_address(self.nodes[0], txid, address) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) utxo = self.nodes[0].listunspent()[0] amt = Decimal(1) + utxo["amount"] - Decimal(0.00001) tx = self.nodes[0].createrawtransaction( @@ -307,17 +305,17 @@ class SignRawTransactionsTest(BitcoinTestFramework): self.nodes[0].walletpassphrase("password", 9999) getcontext().prec = 8 - # Make sure CSV is active - self.nodes[0].generate(1500) + # Make sure CLTV is active + assert self.nodes[0].getblockchaininfo()['softforks']['bip65']['active'] # Create a P2WSH script with CLTV - script = CScript([1000, OP_CHECKLOCKTIMEVERIFY, OP_DROP]) + script = CScript([100, OP_CHECKLOCKTIMEVERIFY, OP_DROP]) address = script_to_p2wsh(script) # Fund that address and make the spend txid = self.nodes[0].sendtoaddress(address, 1) vout = find_vout_for_address(self.nodes[0], txid, address) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) utxo = self.nodes[0].listunspent()[0] amt = Decimal(1) + utxo["amount"] - Decimal(0.00001) tx = self.nodes[0].createrawtransaction( diff --git a/test/functional/rpc_txoutproof.py b/test/functional/rpc_txoutproof.py index 67af6b8f8e..a45694328a 100755 --- a/test/functional/rpc_txoutproof.py +++ b/test/functional/rpc_txoutproof.py @@ -29,8 +29,8 @@ class MerkleBlockTest(BitcoinTestFramework): def run_test(self): miniwallet = MiniWallet(self.nodes[0]) # Add enough mature utxos to the wallet, so that all txs spend confirmed coins - miniwallet.generate(5) - self.nodes[0].generate(COINBASE_MATURITY) + self.generate(miniwallet, 5) + self.generate(self.nodes[0], COINBASE_MATURITY) self.sync_all() chain_height = self.nodes[1].getblockcount() @@ -41,7 +41,7 @@ class MerkleBlockTest(BitcoinTestFramework): # This will raise an exception because the transaction is not yet in a block assert_raises_rpc_error(-5, "Transaction not yet in block", self.nodes[0].gettxoutproof, [txid1]) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) blockhash = self.nodes[0].getblockhash(chain_height + 1) self.sync_all() @@ -57,7 +57,7 @@ class MerkleBlockTest(BitcoinTestFramework): txin_spent = miniwallet.get_utxo() # Get the change from txid2 tx3 = miniwallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=txin_spent) txid3 = tx3['txid'] - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() txid_spent = txin_spent["txid"] diff --git a/test/functional/test-shell.md b/test/functional/test-shell.md index b8e899d675..78737509cb 100644 --- a/test/functional/test-shell.md +++ b/test/functional/test-shell.md @@ -94,7 +94,7 @@ rewards to a wallet address owned by the mining node. ``` >>> address = test.nodes[0].getnewaddress() ->>> test.nodes[0].generatetoaddress(101, address) +>>> test.self.generatetoaddress(nodes[0], 101, address) ['2b98dd0044aae6f1cca7f88a0acf366a4bfe053c7f7b00da3c0d115f03d67efb', ... ``` Since the two nodes are both initialized by default to establish an outbound diff --git a/test/functional/test_framework/address.py b/test/functional/test_framework/address.py index 360962b8da..fe733e9368 100644 --- a/test/functional/test_framework/address.py +++ b/test/functional/test_framework/address.py @@ -12,7 +12,7 @@ import unittest from .script import hash256, hash160, sha256, CScript, OP_0 from .segwit_addr import encode_segwit_address -from .util import assert_equal, hex_str_to_bytes +from .util import assert_equal ADDRESS_BCRT1_UNSPENDABLE = 'bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj' ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR = 'addr(bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj)#juyq9d97' @@ -33,7 +33,7 @@ def byte_to_base58(b, version): result = '' str = b.hex() str = chr(version).encode('latin-1').hex() + str - checksum = hash256(hex_str_to_bytes(str)).hex() + checksum = hash256(bytes.fromhex(str)).hex() str += checksum[:8] value = int('0x' + str, 0) while value > 0: @@ -100,7 +100,7 @@ def key_to_p2sh_p2wpkh(key, main=False): def program_to_witness(version, program, main=False): if (type(program) is str): - program = hex_str_to_bytes(program) + program = bytes.fromhex(program) assert 0 <= version <= 16 assert 2 <= len(program) <= 40 assert version > 0 or len(program) in [20, 32] @@ -121,14 +121,14 @@ def script_to_p2sh_p2wsh(script, main=False): def check_key(key): if (type(key) is str): - key = hex_str_to_bytes(key) # Assuming this is hex string + key = bytes.fromhex(key) # Assuming this is hex string if (type(key) is bytes and (len(key) == 33 or len(key) == 65)): return key assert False def check_script(script): if (type(script) is str): - script = hex_str_to_bytes(script) # Assuming this is hex string + script = bytes.fromhex(script) # Assuming this is hex string if (type(script) is bytes or type(script) is CScript): return script assert False diff --git a/test/functional/test_framework/authproxy.py b/test/functional/test_framework/authproxy.py index 81eb881234..c4ffd1fbf6 100644 --- a/test/functional/test_framework/authproxy.py +++ b/test/functional/test_framework/authproxy.py @@ -113,10 +113,8 @@ class AuthServiceProxy(): self.__conn.request(method, path, postdata, headers) return self._get_response() except OSError as e: - retry = ( - '[WinError 10053] An established connection was aborted by the software in your host machine' in str(e)) # Workaround for a bug on macOS. See https://bugs.python.org/issue33450 - retry = retry or ('[Errno 41] Protocol wrong type for socket' in str(e)) + retry = '[Errno 41] Protocol wrong type for socket' in str(e) if retry: self.__conn.close() self.__conn.request(method, path, postdata, headers) diff --git a/test/functional/test_framework/bdb.py b/test/functional/test_framework/bdb.py index 97b9c1d6d0..d623bcdf6e 100644 --- a/test/functional/test_framework/bdb.py +++ b/test/functional/test_framework/bdb.py @@ -24,7 +24,6 @@ transactions. `db_dump -da wallet.dat` is useful to see the data in a wallet.dat BDB file """ -import binascii import struct # Important constants @@ -96,7 +95,7 @@ def dump_meta_page(page): metadata['key_count'] = key_count metadata['record_count'] = record_count metadata['flags'] = flags - metadata['uid'] = binascii.hexlify(uid) + metadata['uid'] = uid.hex().encode() assert magic == BTREE_MAGIC, 'bdb magic does not match bdb btree magic' assert pg_type == BTREE_META, 'Metadata page is not a btree metadata page' @@ -110,8 +109,9 @@ def dump_meta_page(page): metadata['re_pad'] = re_pad metadata['root'] = root metadata['crypto_magic'] = crypto_magic - metadata['iv'] = binascii.hexlify(iv) - metadata['chksum'] = binascii.hexlify(chksum) + metadata['iv'] = iv.hex().encode() + metadata['chksum'] = chksum.hex().encode() + return metadata # Given the dict from dump_leaf_page, get the key-value pairs and put them into a dict diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index bc43438810..85e3c2a383 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Utilities for manipulating blocks and transactions.""" -from binascii import a2b_hex import struct import time import unittest @@ -24,9 +23,7 @@ from .messages import ( CTxInWitness, CTxOut, hash256, - hex_str_to_bytes, ser_uint256, - sha256, tx_from_hex, uint256_from_str, ) @@ -34,13 +31,15 @@ from .script import ( CScript, CScriptNum, CScriptOp, - OP_0, OP_1, OP_CHECKMULTISIG, - OP_CHECKSIG, OP_RETURN, OP_TRUE, - hash160, +) +from .script_util import ( + key_to_p2pk_script, + key_to_p2wpkh_script, + script_to_p2wsh_script, ) from .util import assert_equal @@ -58,6 +57,7 @@ COINBASE_MATURITY = 100 WITNESS_COMMITMENT_HEADER = b"\xaa\x21\xa9\xed" NORMAL_GBT_REQUEST_PARAMS = {"rules": ["segwit"]} +VERSIONBITS_LAST_OLD_BLOCK_VERSION = 4 def create_block(hashprev=None, coinbase=None, ntime=None, *, version=None, tmpl=None, txlist=None): @@ -65,11 +65,11 @@ def create_block(hashprev=None, coinbase=None, ntime=None, *, version=None, tmpl block = CBlock() if tmpl is None: tmpl = {} - block.nVersion = version or tmpl.get('version') or 1 + block.nVersion = version or tmpl.get('version') or VERSIONBITS_LAST_OLD_BLOCK_VERSION block.nTime = ntime or tmpl.get('curtime') or int(time.time() + 600) block.hashPrevBlock = hashprev or int(tmpl['previousblockhash'], 0x10) if tmpl and not tmpl.get('bits') is None: - block.nBits = struct.unpack('>I', a2b_hex(tmpl['bits']))[0] + block.nBits = struct.unpack('>I', bytes.fromhex(tmpl['bits']))[0] else: block.nBits = 0x207fffff # difficulty retargeting is disabled in REGTEST chainparams if coinbase is None: @@ -134,7 +134,7 @@ def create_coinbase(height, pubkey=None, extra_output_script=None, fees=0, nValu coinbaseoutput.nValue >>= halvings coinbaseoutput.nValue += fees if pubkey is not None: - coinbaseoutput.scriptPubKey = CScript([pubkey, OP_CHECKSIG]) + coinbaseoutput.scriptPubKey = key_to_p2pk_script(pubkey) else: coinbaseoutput.scriptPubKey = CScript([OP_TRUE]) coinbase.vout = [coinbaseoutput] @@ -206,13 +206,11 @@ def witness_script(use_p2wsh, pubkey): scriptPubKey.""" if not use_p2wsh: # P2WPKH instead - pubkeyhash = hash160(hex_str_to_bytes(pubkey)) - pkscript = CScript([OP_0, pubkeyhash]) + pkscript = key_to_p2wpkh_script(pubkey) else: # 1-of-1 multisig - witness_program = CScript([OP_1, hex_str_to_bytes(pubkey), OP_1, OP_CHECKMULTISIG]) - scripthash = sha256(witness_program) - pkscript = CScript([OP_0, scripthash]) + witness_script = CScript([OP_1, bytes.fromhex(pubkey), OP_1, OP_CHECKMULTISIG]) + pkscript = script_to_p2wsh_script(witness_script) return pkscript.hex() def create_witness_tx(node, use_p2wsh, utxo, pubkey, encode_p2sh, amount): @@ -220,7 +218,7 @@ def create_witness_tx(node, use_p2wsh, utxo, pubkey, encode_p2sh, amount): Optionally wrap the segwit output using P2SH.""" if use_p2wsh: - program = CScript([OP_1, hex_str_to_bytes(pubkey), OP_1, OP_CHECKMULTISIG]) + program = CScript([OP_1, bytes.fromhex(pubkey), OP_1, OP_CHECKMULTISIG]) addr = script_to_p2sh_p2wsh(program) if encode_p2sh else script_to_p2wsh(program) else: addr = key_to_p2sh_p2wpkh(pubkey) if encode_p2sh else key_to_p2wpkh(pubkey) @@ -243,7 +241,7 @@ def send_to_witness(use_p2wsh, node, utxo, pubkey, encode_p2sh, amount, sign=Tru else: if (insert_redeem_script): tx = tx_from_hex(tx_to_witness) - tx.vin[0].scriptSig += CScript([hex_str_to_bytes(insert_redeem_script)]) + tx.vin[0].scriptSig += CScript([bytes.fromhex(insert_redeem_script)]) tx_to_witness = tx.serialize().hex() return node.sendrawtransaction(tx_to_witness) diff --git a/test/functional/test_framework/coverage.py b/test/functional/test_framework/coverage.py index 7705dd3e4d..ad8cfe5c9a 100644 --- a/test/functional/test_framework/coverage.py +++ b/test/functional/test_framework/coverage.py @@ -10,6 +10,7 @@ testing. import os +from .authproxy import AuthServiceProxy REFERENCE_FILENAME = 'rpc_interface.txt' @@ -19,16 +20,17 @@ class AuthServiceProxyWrapper(): An object that wraps AuthServiceProxy to record specific RPC calls. """ - def __init__(self, auth_service_proxy_instance, coverage_logfile=None): + def __init__(self, auth_service_proxy_instance: AuthServiceProxy, rpc_url: str, coverage_logfile: str=None): """ Kwargs: - auth_service_proxy_instance (AuthServiceProxy): the instance - being wrapped. - coverage_logfile (str): if specified, write each service_name + auth_service_proxy_instance: the instance being wrapped. + rpc_url: url of the RPC instance being wrapped + coverage_logfile: if specified, write each service_name out to a file when called. """ self.auth_service_proxy_instance = auth_service_proxy_instance + self.rpc_url = rpc_url self.coverage_logfile = coverage_logfile def __getattr__(self, name): @@ -36,7 +38,7 @@ class AuthServiceProxyWrapper(): if not isinstance(return_val, type(self.auth_service_proxy_instance)): # If proxy getattr returned an unwrapped value, do the same here. return return_val - return AuthServiceProxyWrapper(return_val, self.coverage_logfile) + return AuthServiceProxyWrapper(return_val, self.rpc_url, self.coverage_logfile) def __call__(self, *args, **kwargs): """ @@ -57,6 +59,7 @@ class AuthServiceProxyWrapper(): def __truediv__(self, relative_uri): return AuthServiceProxyWrapper(self.auth_service_proxy_instance / relative_uri, + self.rpc_url, self.coverage_logfile) def get_request(self, *args, **kwargs): @@ -74,18 +77,18 @@ def get_filename(dirname, n_node): dirname, "coverage.pid%s.node%s.txt" % (pid, str(n_node))) -def write_all_rpc_commands(dirname, node): +def write_all_rpc_commands(dirname: str, node: AuthServiceProxy) -> bool: """ Write out a list of all RPC functions available in `bitcoin-cli` for coverage comparison. This will only happen once per coverage directory. Args: - dirname (str): temporary test dir - node (AuthServiceProxy): client + dirname: temporary test dir + node: client Returns: - bool. if the RPC interface file was written. + if the RPC interface file was written. """ filename = os.path.join(dirname, REFERENCE_FILENAME) diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 504c8c70d4..65d90f8448 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -18,7 +18,7 @@ ser_*, deser_*: functions that handle serialization/deserialization. Classes use __slots__ to ensure extraneous attributes aren't accidentally added by tests, compromising their intended effect. """ -from codecs import encode +from base64 import b32decode, b32encode import copy import hashlib from io import BytesIO @@ -29,10 +29,10 @@ import struct import time from test_framework.siphash import siphash256 -from test_framework.util import hex_str_to_bytes, assert_equal +from test_framework.util import assert_equal MAX_LOCATOR_SZ = 101 -MAX_BLOCK_BASE_SIZE = 1000000 +MAX_BLOCK_WEIGHT = 4000000 MAX_BLOOM_FILTER_SIZE = 36000 MAX_BLOOM_HASH_FUNCS = 50 @@ -196,7 +196,7 @@ def from_hex(obj, hex_string): Note that there is no complementary helper like e.g. `to_hex` for the inverse operation. To serialize a message object to a hex string, simply use obj.serialize().hex()""" - obj.deserialize(BytesIO(hex_str_to_bytes(hex_string))) + obj.deserialize(BytesIO(bytes.fromhex(hex_string))) return obj @@ -213,15 +213,20 @@ class CAddress: # see https://github.com/bitcoin/bips/blob/master/bip-0155.mediawiki NET_IPV4 = 1 + NET_I2P = 5 ADDRV2_NET_NAME = { - NET_IPV4: "IPv4" + NET_IPV4: "IPv4", + NET_I2P: "I2P" } ADDRV2_ADDRESS_LENGTH = { - NET_IPV4: 4 + NET_IPV4: 4, + NET_I2P: 32 } + I2P_PAD = "====" + def __init__(self): self.time = 0 self.nServices = 1 @@ -229,6 +234,9 @@ class CAddress: self.ip = "0.0.0.0" self.port = 0 + def __eq__(self, other): + return self.net == other.net and self.ip == other.ip and self.nServices == other.nServices and self.port == other.port and self.time == other.time + def deserialize(self, f, *, with_time=True): """Deserialize from addrv1 format (pre-BIP155)""" if with_time: @@ -261,24 +269,33 @@ class CAddress: self.nServices = deser_compact_size(f) self.net = struct.unpack("B", f.read(1))[0] - assert self.net == self.NET_IPV4 + assert self.net in (self.NET_IPV4, self.NET_I2P) address_length = deser_compact_size(f) assert address_length == self.ADDRV2_ADDRESS_LENGTH[self.net] - self.ip = socket.inet_ntoa(f.read(4)) + addr_bytes = f.read(address_length) + if self.net == self.NET_IPV4: + self.ip = socket.inet_ntoa(addr_bytes) + else: + self.ip = b32encode(addr_bytes)[0:-len(self.I2P_PAD)].decode("ascii").lower() + ".b32.i2p" self.port = struct.unpack(">H", f.read(2))[0] def serialize_v2(self): """Serialize in addrv2 format (BIP155)""" - assert self.net == self.NET_IPV4 + assert self.net in (self.NET_IPV4, self.NET_I2P) r = b"" r += struct.pack("<I", self.time) r += ser_compact_size(self.nServices) r += struct.pack("B", self.net) r += ser_compact_size(self.ADDRV2_ADDRESS_LENGTH[self.net]) - r += socket.inet_aton(self.ip) + if self.net == self.NET_IPV4: + r += socket.inet_aton(self.ip) + else: + sfx = ".b32.i2p" + assert self.ip.endswith(sfx) + r += b32decode(self.ip[0:-len(sfx)] + self.I2P_PAD, True) r += struct.pack(">H", self.port) return r @@ -590,12 +607,15 @@ class CTransaction: return False return True - # Calculate the virtual transaction size using witness and non-witness + # Calculate the transaction weight using witness and non-witness # serialization size (does NOT use sigops). - def get_vsize(self): + def get_weight(self): with_witness_size = len(self.serialize_with_witness()) without_witness_size = len(self.serialize_without_witness()) - return math.ceil(((WITNESS_SCALE_FACTOR - 1) * without_witness_size + with_witness_size) / WITNESS_SCALE_FACTOR) + return (WITNESS_SCALE_FACTOR - 1) * without_witness_size + with_witness_size + + def get_vsize(self): + return math.ceil(self.get_weight() / WITNESS_SCALE_FACTOR) def __repr__(self): return "CTransaction(nVersion=%i vin=%s vout=%s wit=%s nLockTime=%i)" \ @@ -621,7 +641,7 @@ class CBlockHeader: self.calc_sha256() def set_null(self): - self.nVersion = 1 + self.nVersion = 4 self.hashPrevBlock = 0 self.hashMerkleRoot = 0 self.nTime = 0 @@ -660,7 +680,7 @@ class CBlockHeader: r += struct.pack("<I", self.nBits) r += struct.pack("<I", self.nNonce) self.sha256 = uint256_from_str(hash256(r)) - self.hash = encode(hash256(r)[::-1], 'hex_codec').decode('ascii') + self.hash = hash256(r)[::-1].hex() def rehash(self): self.sha256 = None @@ -743,6 +763,13 @@ class CBlock(CBlockHeader): self.nNonce += 1 self.rehash() + # Calculate the block weight using witness and non-witness + # serialization size (does NOT use sigops). + def get_weight(self): + with_witness_size = len(self.serialize(with_witness=True)) + without_witness_size = len(self.serialize(with_witness=False)) + return (WITNESS_SCALE_FACTOR - 1) * without_witness_size + with_witness_size + def __repr__(self): return "CBlock(nVersion=%i hashPrevBlock=%064x hashMerkleRoot=%064x nTime=%s nBits=%08x nNonce=%08x vtx=%s)" \ % (self.nVersion, self.hashPrevBlock, self.hashMerkleRoot, diff --git a/test/functional/test_framework/netutil.py b/test/functional/test_framework/netutil.py index 5dc723c1d5..b5f78e0cf3 100644 --- a/test/functional/test_framework/netutil.py +++ b/test/functional/test_framework/netutil.py @@ -12,7 +12,6 @@ import socket import struct import array import os -from binascii import unhexlify # STATE_ESTABLISHED = '01' # STATE_SYN_SENT = '02' @@ -44,7 +43,7 @@ def _remove_empty(array): def _convert_ip_port(array): host,port = array.split(':') # convert host from mangled-per-four-bytes form as used by kernel - host = unhexlify(host) + host = bytes.fromhex(host) host_out = '' for x in range(0, len(host) // 4): (val,) = struct.unpack('=I', host[x*4:(x+1)*4]) diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index cc80b543cd..78c63b57a1 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -356,7 +356,7 @@ class P2PInterface(P2PConnection): return create_conn - def peer_accept_connection(self, *args, services=NODE_NETWORK | NODE_WITNESS, **kwargs): + def peer_accept_connection(self, *args, services=P2P_SERVICES, **kwargs): create_conn = super().peer_accept_connection(*args, **kwargs) self.peer_connect_send_version(services) @@ -438,6 +438,7 @@ class P2PInterface(P2PConnection): self.send_message(msg_sendaddrv2()) self.send_message(msg_verack()) self.nServices = message.nServices + self.send_message(msg_getaddr()) # Connection helper methods @@ -576,6 +577,8 @@ class NetworkThread(threading.Thread): NetworkThread.listeners = {} NetworkThread.protos = {} + if sys.platform == 'win32': + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) NetworkThread.network_event_loop = asyncio.new_event_loop() def run(self): diff --git a/test/functional/test_framework/script_util.py b/test/functional/test_framework/script_util.py index 457be6b0e6..82a9067dd2 100755 --- a/test/functional/test_framework/script_util.py +++ b/test/functional/test_framework/script_util.py @@ -3,8 +3,17 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Useful Script constants and utils.""" -from test_framework.script import CScript, hash160, sha256, OP_0, OP_DUP, OP_HASH160, OP_CHECKSIG, OP_EQUAL, OP_EQUALVERIFY -from test_framework.util import hex_str_to_bytes +from test_framework.script import ( + CScript, + OP_0, + OP_CHECKSIG, + OP_DUP, + OP_EQUAL, + OP_EQUALVERIFY, + OP_HASH160, + hash160, + sha256, +) # To prevent a "tx-size-small" policy rule error, a transaction has to have a # non-witness size of at least 82 bytes (MIN_STANDARD_TX_NONWITNESS_SIZE in @@ -26,58 +35,74 @@ from test_framework.util import hex_str_to_bytes DUMMY_P2WPKH_SCRIPT = CScript([b'a' * 21]) DUMMY_2_P2WPKH_SCRIPT = CScript([b'b' * 21]) -def keyhash_to_p2pkh_script(hash, main = False): + +def key_to_p2pk_script(key): + key = check_key(key) + return CScript([key, OP_CHECKSIG]) + + +def keyhash_to_p2pkh_script(hash): assert len(hash) == 20 return CScript([OP_DUP, OP_HASH160, hash, OP_EQUALVERIFY, OP_CHECKSIG]) -def scripthash_to_p2sh_script(hash, main = False): + +def scripthash_to_p2sh_script(hash): assert len(hash) == 20 return CScript([OP_HASH160, hash, OP_EQUAL]) -def key_to_p2pkh_script(key, main = False): + +def key_to_p2pkh_script(key): key = check_key(key) - return keyhash_to_p2pkh_script(hash160(key), main) + return keyhash_to_p2pkh_script(hash160(key)) + -def script_to_p2sh_script(script, main = False): +def script_to_p2sh_script(script): script = check_script(script) - return scripthash_to_p2sh_script(hash160(script), main) + return scripthash_to_p2sh_script(hash160(script)) -def key_to_p2sh_p2wpkh_script(key, main = False): + +def key_to_p2sh_p2wpkh_script(key): key = check_key(key) p2shscript = CScript([OP_0, hash160(key)]) - return script_to_p2sh_script(p2shscript, main) + return script_to_p2sh_script(p2shscript) + -def program_to_witness_script(version, program, main = False): +def program_to_witness_script(version, program): if isinstance(program, str): - program = hex_str_to_bytes(program) + program = bytes.fromhex(program) assert 0 <= version <= 16 assert 2 <= len(program) <= 40 assert version > 0 or len(program) in [20, 32] return CScript([version, program]) -def script_to_p2wsh_script(script, main = False): + +def script_to_p2wsh_script(script): script = check_script(script) - return program_to_witness_script(0, sha256(script), main) + return program_to_witness_script(0, sha256(script)) + -def key_to_p2wpkh_script(key, main = False): +def key_to_p2wpkh_script(key): key = check_key(key) - return program_to_witness_script(0, hash160(key), main) + return program_to_witness_script(0, hash160(key)) -def script_to_p2sh_p2wsh_script(script, main = False): + +def script_to_p2sh_p2wsh_script(script): script = check_script(script) p2shscript = CScript([OP_0, sha256(script)]) - return script_to_p2sh_script(p2shscript, main) + return script_to_p2sh_script(p2shscript) + def check_key(key): if isinstance(key, str): - key = hex_str_to_bytes(key) # Assuming this is hex string + key = bytes.fromhex(key) # Assuming this is hex string if isinstance(key, bytes) and (len(key) == 33 or len(key) == 65): return key assert False + def check_script(script): if isinstance(script, str): - script = hex_str_to_bytes(script) # Assuming this is hex string + script = bytes.fromhex(script) # Assuming this is hex string if isinstance(script, bytes) or isinstance(script, CScript): return script assert False diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 40360c54a0..d87d0cacfd 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -101,6 +101,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.supports_cli = True self.bind_to_localhost_only = True self.parse_args() + self.disable_syscall_sandbox = self.options.nosandbox self.default_wallet_name = "default_wallet" if self.options.descriptors else "" self.wallet_data_filename = "wallet.dat" # Optional list of wallet names that can be set in set_test_params to @@ -112,6 +113,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # By default the wallet is not required. Set to true by skip_if_no_wallet(). # When False, we ignore wallet_names regardless of what it is. self.requires_wallet = False + # Disable ThreadOpenConnections by default, so that adding entries to + # addrman will not result in automatic connections to them. + self.disable_autoconnect = True self.set_test_params() assert self.wallet_names is None or len(self.wallet_names) <= self.num_nodes if self.options.timeout_factor == 0 : @@ -156,6 +160,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): parser = argparse.ArgumentParser(usage="%(prog)s [options]") parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true", help="Leave bitcoinds and test.* datadir on exit or error") + parser.add_argument("--nosandbox", dest="nosandbox", default=False, action="store_true", + help="Don't use the syscall sandbox") parser.add_argument("--noshutdown", dest="noshutdown", default=False, action="store_true", help="Don't stop bitcoinds after the test execution") parser.add_argument("--cachedir", dest="cachedir", default=os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../../cache"), @@ -407,7 +413,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # To ensure that all nodes are out of IBD, the most recent block # must have a timestamp not too old (see IsInitialBlockDownload()). self.log.debug('Generate a block with current time') - block_hash = self.nodes[0].generate(1)[0] + block_hash = self.generate(self.nodes[0], 1)[0] block = self.nodes[0].getblock(blockhash=block_hash, verbosity=0) for n in self.nodes: n.submitblock(block) @@ -465,6 +471,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): extra_args = [[]] * num_nodes if versions is None: versions = [None] * num_nodes + if self.is_syscall_sandbox_compiled() and not self.disable_syscall_sandbox: + for i in range(len(extra_args)): + if versions[i] is None or versions[i] >= 219900: + extra_args[i] = extra_args[i] + ["-sandbox=log-and-abort"] if binary is None: binary = [get_bin_from_version(v, 'bitcoind', self.options.bitcoind) for v in versions] if binary_cli is None: @@ -557,18 +567,19 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.nodes[i].process.wait(timeout) def connect_nodes(self, a, b): - def connect_nodes_helper(from_connection, node_num): - ip_port = "127.0.0.1:" + str(p2p_port(node_num)) - from_connection.addnode(ip_port, "onetry") - # poll until version handshake complete to avoid race conditions - # with transaction relaying - # See comments in net_processing: - # * Must have a version message before anything else - # * Must have a verack message before anything else - wait_until_helper(lambda: all(peer['version'] != 0 for peer in from_connection.getpeerinfo())) - wait_until_helper(lambda: all(peer['bytesrecv_per_msg'].pop('verack', 0) == 24 for peer in from_connection.getpeerinfo())) - - connect_nodes_helper(self.nodes[a], b) + from_connection = self.nodes[a] + to_connection = self.nodes[b] + ip_port = "127.0.0.1:" + str(p2p_port(b)) + from_connection.addnode(ip_port, "onetry") + # poll until version handshake complete to avoid race conditions + # with transaction relaying + # See comments in net_processing: + # * Must have a version message before anything else + # * Must have a verack message before anything else + wait_until_helper(lambda: all(peer['version'] != 0 for peer in from_connection.getpeerinfo())) + wait_until_helper(lambda: all(peer['version'] != 0 for peer in to_connection.getpeerinfo())) + wait_until_helper(lambda: all(peer['bytesrecv_per_msg'].pop('verack', 0) == 24 for peer in from_connection.getpeerinfo())) + wait_until_helper(lambda: all(peer['bytesrecv_per_msg'].pop('verack', 0) == 24 for peer in to_connection.getpeerinfo())) def disconnect_nodes(self, a, b): def disconnect_nodes_helper(from_connection, node_num): @@ -616,6 +627,22 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.connect_nodes(1, 2) self.sync_all() + def generate(self, generator, *args, **kwargs): + blocks = generator.generate(*args, **kwargs) + return blocks + + def generateblock(self, generator, *args, **kwargs): + blocks = generator.generateblock(*args, **kwargs) + return blocks + + def generatetoaddress(self, generator, *args, **kwargs): + blocks = generator.generatetoaddress(*args, **kwargs) + return blocks + + def generatetodescriptor(self, generator, *args, **kwargs): + blocks = generator.generatetodescriptor(*args, **kwargs) + return blocks + def sync_blocks(self, nodes=None, wait=1, timeout=60): """ Wait until everybody has the same tip. @@ -711,7 +738,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): if not os.path.isdir(cache_node_dir): self.log.debug("Creating cache directory {}".format(cache_node_dir)) - initialize_datadir(self.options.cachedir, CACHE_NODE_ID, self.chain) + initialize_datadir(self.options.cachedir, CACHE_NODE_ID, self.chain, self.disable_autoconnect) self.nodes.append( TestNode( CACHE_NODE_ID, @@ -746,7 +773,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): gen_addresses = [k.address for k in TestNode.PRIV_KEYS][:3] + [ADDRESS_BCRT1_P2WSH_OP_TRUE] assert_equal(len(gen_addresses), 4) for i in range(8): - cache_node.generatetoaddress( + self.generatetoaddress( + cache_node, nblocks=25 if i != 7 else 24, address=gen_addresses[i % len(gen_addresses)], ) @@ -769,7 +797,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.log.debug("Copy cache directory {} to node {}".format(cache_node_dir, i)) to_dir = get_datadir_path(self.options.tmpdir, i) shutil.copytree(cache_node_dir, to_dir) - initialize_datadir(self.options.tmpdir, i, self.chain) # Overwrite port/rpcport in bitcoin.conf + initialize_datadir(self.options.tmpdir, i, self.chain, self.disable_autoconnect) # Overwrite port/rpcport in bitcoin.conf def _initialize_chain_clean(self): """Initialize empty blockchain for use by the test. @@ -777,7 +805,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): Create an empty blockchain and num_nodes wallets. Useful if a test case wants complete control over initialization.""" for i in range(self.num_nodes): - initialize_datadir(self.options.tmpdir, i, self.chain) + initialize_datadir(self.options.tmpdir, i, self.chain, self.disable_autoconnect) def skip_if_no_py3_zmq(self): """Attempt to import the zmq package and skip the test if the import fails.""" @@ -866,3 +894,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def is_bdb_compiled(self): """Checks whether the wallet module was compiled with BDB support.""" return self.config["components"].getboolean("USE_BDB") + + def is_syscall_sandbox_compiled(self): + """Checks whether the syscall sandbox was compiled.""" + return self.config["components"].getboolean("ENABLE_SYSCALL_SANDBOX") diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index ba52abc7dd..f9e2cfa2f5 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -258,7 +258,7 @@ class TestNode(): return self.rpc = rpc self.rpc_connected = True - self.url = self.rpc.url + self.url = self.rpc.rpc_url return except JSONRPCException as e: # Initialization phase # -28 RPC in warmup @@ -557,9 +557,8 @@ class TestNode(): return p2p_conn def add_outbound_p2p_connection(self, p2p_conn, *, p2p_idx, connection_type="outbound-full-relay", **kwargs): - """Add an outbound p2p connection from node. Either - full-relay("outbound-full-relay") or - block-relay-only("block-relay-only") connection. + """Add an outbound p2p connection from node. Must be an + "outbound-full-relay", "block-relay-only" or "addr-fetch" connection. This method adds the p2p connection to the self.p2ps list and returns the connection to the caller. diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 35dbfbba8d..9f5bca6884 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -5,7 +5,6 @@ """Helpful routines for regression testing.""" from base64 import b64encode -from binascii import unhexlify from decimal import Decimal, ROUND_DOWN from subprocess import CalledProcessError import hashlib @@ -35,13 +34,14 @@ def assert_approx(v, vexp, vspan=0.00001): raise AssertionError("%s > [%s..%s]" % (str(v), str(vexp - vspan), str(vexp + vspan))) -def assert_fee_amount(fee, tx_size, fee_per_kB): - """Assert the fee was in range""" - target_fee = round(tx_size * fee_per_kB / 1000, 8) +def assert_fee_amount(fee, tx_size, feerate_BTC_kvB): + """Assert the fee is in range.""" + feerate_BTC_vB = feerate_BTC_kvB / 1000 + target_fee = satoshi_round(tx_size * feerate_BTC_vB) if fee < target_fee: raise AssertionError("Fee of %s BTC too low! (Should be %s BTC)" % (str(fee), str(target_fee))) # allow the wallet's estimation to be at most 2 bytes off - if fee > (tx_size + 2) * fee_per_kB / 1000: + if fee > (tx_size + 2) * feerate_BTC_vB: raise AssertionError("Fee of %s BTC too high! (Should be %s BTC)" % (str(fee), str(target_fee))) @@ -214,10 +214,6 @@ def count_bytes(hex_string): return len(bytearray.fromhex(hex_string)) -def hex_str_to_bytes(hex_str): - return unhexlify(hex_str.encode('ascii')) - - def str_to_b64str(string): return b64encode(string.encode('utf-8')).decode('ascii') @@ -286,15 +282,15 @@ class PortSeed: n = None -def get_rpc_proxy(url, node_number, *, timeout=None, coveragedir=None): +def get_rpc_proxy(url: str, node_number: int, *, timeout: int=None, coveragedir: str=None) -> coverage.AuthServiceProxyWrapper: """ Args: - url (str): URL of the RPC server to call - node_number (int): the node number (or id) that this calls to + url: URL of the RPC server to call + node_number: the node number (or id) that this calls to Kwargs: - timeout (int): HTTP timeout in seconds - coveragedir (str): Directory + timeout: HTTP timeout in seconds + coveragedir: Directory Returns: AuthServiceProxy. convenience object for making RPC calls. @@ -305,11 +301,10 @@ def get_rpc_proxy(url, node_number, *, timeout=None, coveragedir=None): proxy_kwargs['timeout'] = int(timeout) proxy = AuthServiceProxy(url, **proxy_kwargs) - proxy.url = url # store URL on proxy for info coverage_logfile = coverage.get_filename(coveragedir, node_number) if coveragedir else None - return coverage.AuthServiceProxyWrapper(proxy, coverage_logfile) + return coverage.AuthServiceProxyWrapper(proxy, url, coverage_logfile) def p2p_port(n): @@ -338,17 +333,17 @@ def rpc_url(datadir, i, chain, rpchost): ################ -def initialize_datadir(dirname, n, chain): +def initialize_datadir(dirname, n, chain, disable_autoconnect=True): datadir = get_datadir_path(dirname, n) if not os.path.isdir(datadir): os.makedirs(datadir) - write_config(os.path.join(datadir, "bitcoin.conf"), n=n, chain=chain) + write_config(os.path.join(datadir, "bitcoin.conf"), n=n, chain=chain, disable_autoconnect=disable_autoconnect) os.makedirs(os.path.join(datadir, 'stderr'), exist_ok=True) os.makedirs(os.path.join(datadir, 'stdout'), exist_ok=True) return datadir -def write_config(config_path, *, n, chain, extra_config=""): +def write_config(config_path, *, n, chain, extra_config="", disable_autoconnect=True): # Translate chain subdirectory name to config name if chain == 'testnet3': chain_name_conf_arg = 'testnet' @@ -370,12 +365,19 @@ def write_config(config_path, *, n, chain, extra_config=""): f.write("dnsseed=0\n") f.write("fixedseeds=0\n") f.write("listenonion=0\n") + # Increase peertimeout to avoid disconnects while using mocktime. + # peertimeout is measured in wall clock time, so setting it to the + # duration of the longest test is sufficient. It can be overridden in + # tests. + f.write("peertimeout=999999\n") f.write("printtoconsole=0\n") f.write("upnp=0\n") f.write("natpmp=0\n") f.write("shrinkdebugfile=0\n") # To improve SQLite wallet performance so that the tests don't timeout, use -unsafesqlitesync f.write("unsafesqlitesync=1\n") + if disable_autoconnect: + f.write("connect=0\n") f.write(extra_config) @@ -449,10 +451,10 @@ def find_output(node, txid, amount, *, blockhash=None): # Helper to create at least "count" utxos # Pass in a fee that is sufficient for relay and mining new transactions. -def create_confirmed_utxos(fee, node, count): +def create_confirmed_utxos(test_framework, fee, node, count): to_generate = int(0.5 * count) + 101 while to_generate > 0: - node.generate(min(25, to_generate)) + test_framework.generate(node, min(25, to_generate)) to_generate -= 25 utxos = node.listunspent() iterations = count - len(utxos) @@ -473,7 +475,7 @@ def create_confirmed_utxos(fee, node, count): node.sendrawtransaction(signed_tx) while (node.getmempoolinfo()['size'] > 0): - node.generate(1) + test_framework.generate(node, 1) utxos = node.listunspent() assert len(utxos) >= count @@ -516,7 +518,7 @@ def gen_return_txouts(): from .messages import CTxOut txout = CTxOut() txout.nValue = 0 - txout.scriptPubKey = hex_str_to_bytes(script_pubkey) + txout.scriptPubKey = bytes.fromhex(script_pubkey) for _ in range(128): txouts.append(txout) return txouts @@ -545,7 +547,7 @@ def create_lots_of_big_transactions(node, txouts, utxos, num, fee): return txids -def mine_large_block(node, utxos=None): +def mine_large_block(test_framework, node, utxos=None): # generate a 66k transaction, # and 14 of them is close to the 1MB block limit num = 14 @@ -556,7 +558,7 @@ def mine_large_block(node, utxos=None): utxos.extend(node.listunspent()) fee = 100 * node.getnetworkinfo()["relayfee"] create_lots_of_big_transactions(node, txouts, utxos, num, fee=fee) - node.generate(1) + test_framework.generate(node, 1) def find_vout_for_address(node, txid, addr): diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index 47ec6b0be2..b0b578e544 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -4,10 +4,13 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """A limited-functionality wallet, which may replace a real wallet in tests""" +from copy import deepcopy from decimal import Decimal from enum import Enum +from random import choice from typing import Optional from test_framework.address import ADDRESS_BCRT1_P2WSH_OP_TRUE +from test_framework.descriptors import descsum_create from test_framework.key import ECKey from test_framework.messages import ( COIN, @@ -16,21 +19,25 @@ from test_framework.messages import ( CTxIn, CTxInWitness, CTxOut, + tx_from_hex, ) from test_framework.script import ( CScript, LegacySignatureHash, - OP_CHECKSIG, OP_TRUE, OP_NOP, SIGHASH_ALL, ) +from test_framework.script_util import ( + key_to_p2pk_script, + key_to_p2wpkh_script, +) from test_framework.util import ( assert_equal, - hex_str_to_bytes, - satoshi_round, + assert_greater_than_or_equal, ) +DEFAULT_FEE = Decimal("0.0001") class MiniWalletMode(Enum): """Determines the transaction type the MiniWallet is creating and spending. @@ -70,17 +77,18 @@ class MiniWallet: self._priv_key = ECKey() self._priv_key.set((1).to_bytes(32, 'big'), True) pub_key = self._priv_key.get_pubkey() - self._scriptPubKey = bytes(CScript([pub_key.get_bytes(), OP_CHECKSIG])) + self._scriptPubKey = key_to_p2pk_script(pub_key.get_bytes()) elif mode == MiniWalletMode.ADDRESS_OP_TRUE: self._address = ADDRESS_BCRT1_P2WSH_OP_TRUE - self._scriptPubKey = hex_str_to_bytes(self._test_node.validateaddress(self._address)['scriptPubKey']) + self._scriptPubKey = bytes.fromhex(self._test_node.validateaddress(self._address)['scriptPubKey']) - def scan_blocks(self, *, start=1, num): - """Scan the blocks for self._address outputs and add them to self._utxos""" - for i in range(start, start + num): - block = self._test_node.getblock(blockhash=self._test_node.getblockhash(i), verbosity=2) - for tx in block['tx']: - self.scan_tx(tx) + def rescan_utxos(self): + """Drop all utxos and rescan the utxo set""" + self._utxos = [] + res = self._test_node.scantxoutset(action="start", scanobjects=[self.get_descriptor()]) + assert_equal(True, res['success']) + for utxo in res['unspents']: + self._utxos.append({'txid': utxo['txid'], 'vout': utxo['vout'], 'value': utxo['amount']}) def scan_tx(self, tx): """Scan the tx for self._scriptPubKey outputs and add them to self._utxos""" @@ -105,12 +113,15 @@ class MiniWallet: def generate(self, num_blocks): """Generate blocks with coinbase outputs to the internal address, and append the outputs to the internal list""" - blocks = self._test_node.generatetodescriptor(num_blocks, f'raw({self._scriptPubKey.hex()})') + blocks = self._test_node.generatetodescriptor(num_blocks, self.get_descriptor()) for b in blocks: cb_tx = self._test_node.getblock(blockhash=b, verbosity=2)['tx'][0] self._utxos.append({'txid': cb_tx['txid'], 'vout': 0, 'value': cb_tx['vout'][0]['value']}) return blocks + def get_descriptor(self): + return descsum_create(f'raw({self._scriptPubKey.hex()})') + def get_address(self): return self._address @@ -138,6 +149,25 @@ class MiniWallet: self.sendrawtransaction(from_node=kwargs['from_node'], tx_hex=tx['hex']) return tx + def send_to(self, *, from_node, scriptPubKey, amount, fee=1000): + """ + Create and send a tx with an output to a given scriptPubKey/amount, + plus a change output to our internal address. To keep things simple, a + fixed fee given in Satoshi is used. + + Note that this method fails if there is no single internal utxo + available that can cover the cost for the amount and the fixed fee + (the utxo with the largest value is taken). + + Returns a tuple (txid, n) referring to the created external utxo outpoint. + """ + tx = self.create_self_transfer(from_node=from_node, fee_rate=0, mempool_valid=False)['tx'] + assert_greater_than_or_equal(tx.vout[0].nValue, amount + fee) + tx.vout[0].nValue -= (amount + fee) # change output -> MiniWallet + tx.vout.append(CTxOut(amount, scriptPubKey)) # arbitrary output -> to be returned + txid = self.sendrawtransaction(from_node=from_node, tx_hex=tx.serialize().hex()) + return txid, 1 + def create_self_transfer(self, *, fee_rate=Decimal("0.003"), from_node, utxo_to_spend=None, mempool_valid=True, locktime=0, sequence=0): """Create and return a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed.""" self._utxos = sorted(self._utxos, key=lambda k: k['value']) @@ -146,13 +176,12 @@ class MiniWallet: vsize = Decimal(96) # anyone-can-spend else: vsize = Decimal(168) # P2PK (73 bytes scriptSig + 35 bytes scriptPubKey + 60 bytes other) - send_value = satoshi_round(utxo_to_spend['value'] - fee_rate * (vsize / 1000)) - fee = utxo_to_spend['value'] - send_value + send_value = int(COIN * (utxo_to_spend['value'] - fee_rate * (vsize / 1000))) assert send_value > 0 tx = CTransaction() tx.vin = [CTxIn(COutPoint(int(utxo_to_spend['txid'], 16), utxo_to_spend['vout']), nSequence=sequence)] - tx.vout = [CTxOut(int(send_value * COIN), self._scriptPubKey)] + tx.vout = [CTxOut(send_value, self._scriptPubKey)] tx.nLockTime = locktime if not self._address: # raw script @@ -171,9 +200,91 @@ class MiniWallet: assert_equal(mempool_valid, tx_info['allowed']) if mempool_valid: assert_equal(tx_info['vsize'], vsize) - assert_equal(tx_info['fees']['base'], fee) + assert_equal(tx_info['fees']['base'], utxo_to_spend['value'] - Decimal(send_value) / COIN) return {'txid': tx_info['txid'], 'wtxid': tx_info['wtxid'], 'hex': tx_hex, 'tx': tx} def sendrawtransaction(self, *, from_node, tx_hex): - from_node.sendrawtransaction(tx_hex) + txid = from_node.sendrawtransaction(tx_hex) self.scan_tx(from_node.decoderawtransaction(tx_hex)) + return txid + + +def random_p2wpkh(): + """Generate a random P2WPKH scriptPubKey. Can be used when a random destination is needed, + but no compiled wallet is available (e.g. as replacement to the getnewaddress RPC).""" + key = ECKey() + key.generate() + return key_to_p2wpkh_script(key.get_pubkey().get_bytes()) + + +def make_chain(node, address, privkeys, parent_txid, parent_value, n=0, parent_locking_script=None, fee=DEFAULT_FEE): + """Build a transaction that spends parent_txid.vout[n] and produces one output with + amount = parent_value with a fee deducted. + Return tuple (CTransaction object, raw hex, nValue, scriptPubKey of the output created). + """ + inputs = [{"txid": parent_txid, "vout": n}] + my_value = parent_value - fee + outputs = {address : my_value} + rawtx = node.createrawtransaction(inputs, outputs) + prevtxs = [{ + "txid": parent_txid, + "vout": n, + "scriptPubKey": parent_locking_script, + "amount": parent_value, + }] if parent_locking_script else None + signedtx = node.signrawtransactionwithkey(hexstring=rawtx, privkeys=privkeys, prevtxs=prevtxs) + assert signedtx["complete"] + tx = tx_from_hex(signedtx["hex"]) + return (tx, signedtx["hex"], my_value, tx.vout[0].scriptPubKey.hex()) + +def create_child_with_parents(node, address, privkeys, parents_tx, values, locking_scripts, fee=DEFAULT_FEE): + """Creates a transaction that spends the first output of each parent in parents_tx.""" + num_parents = len(parents_tx) + total_value = sum(values) + inputs = [{"txid": tx.rehash(), "vout": 0} for tx in parents_tx] + outputs = {address : total_value - fee} + rawtx_child = node.createrawtransaction(inputs, outputs) + prevtxs = [] + for i in range(num_parents): + prevtxs.append({"txid": parents_tx[i].rehash(), "vout": 0, "scriptPubKey": locking_scripts[i], "amount": values[i]}) + signedtx_child = node.signrawtransactionwithkey(hexstring=rawtx_child, privkeys=privkeys, prevtxs=prevtxs) + assert signedtx_child["complete"] + return signedtx_child["hex"] + +def create_raw_chain(node, first_coin, address, privkeys, chain_length=25): + """Helper function: create a "chain" of chain_length transactions. The nth transaction in the + chain is a child of the n-1th transaction and parent of the n+1th transaction. + """ + parent_locking_script = None + txid = first_coin["txid"] + chain_hex = [] + chain_txns = [] + value = first_coin["amount"] + + for _ in range(chain_length): + (tx, txhex, value, parent_locking_script) = make_chain(node, address, privkeys, txid, value, 0, parent_locking_script) + txid = tx.rehash() + chain_hex.append(txhex) + chain_txns.append(tx) + + return (chain_hex, chain_txns) + +def bulk_transaction(tx, node, target_weight, privkeys, prevtxs=None): + """Pad a transaction with extra outputs until it reaches a target weight (or higher). + returns CTransaction object + """ + tx_heavy = deepcopy(tx) + assert_greater_than_or_equal(target_weight, tx_heavy.get_weight()) + while tx_heavy.get_weight() < target_weight: + random_spk = "6a4d0200" # OP_RETURN OP_PUSH2 512 bytes + for _ in range(512*2): + random_spk += choice("0123456789ABCDEF") + tx_heavy.vout.append(CTxOut(0, bytes.fromhex(random_spk))) + # Re-sign the transaction + if privkeys: + signed = node.signrawtransactionwithkey(tx_heavy.serialize().hex(), privkeys, prevtxs) + return tx_from_hex(signed["hex"]) + # OP_TRUE + tx_heavy.wit.vtxinwit = [CTxInWitness()] + tx_heavy.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] + return tx_heavy diff --git a/test/functional/test_framework/wallet_util.py b/test/functional/test_framework/wallet_util.py index b9c0fb6691..1ee55aa3b7 100755 --- a/test/functional/test_framework/wallet_util.py +++ b/test/functional/test_framework/wallet_util.py @@ -17,19 +17,16 @@ from test_framework.address import ( from test_framework.key import ECKey from test_framework.script import ( CScript, - OP_0, OP_2, OP_3, OP_CHECKMULTISIG, - OP_CHECKSIG, - OP_DUP, - OP_EQUAL, - OP_EQUALVERIFY, - OP_HASH160, - hash160, - sha256, ) -from test_framework.util import hex_str_to_bytes +from test_framework.script_util import ( + key_to_p2pkh_script, + key_to_p2wpkh_script, + script_to_p2sh_script, + script_to_p2wsh_script, +) Key = namedtuple('Key', ['privkey', 'pubkey', @@ -57,15 +54,14 @@ def get_key(node): Returns a named tuple of privkey, pubkey and all address and scripts.""" addr = node.getnewaddress() pubkey = node.getaddressinfo(addr)['pubkey'] - pkh = hash160(hex_str_to_bytes(pubkey)) return Key(privkey=node.dumpprivkey(addr), pubkey=pubkey, - p2pkh_script=CScript([OP_DUP, OP_HASH160, pkh, OP_EQUALVERIFY, OP_CHECKSIG]).hex(), + p2pkh_script=key_to_p2pkh_script(pubkey).hex(), p2pkh_addr=key_to_p2pkh(pubkey), - p2wpkh_script=CScript([OP_0, pkh]).hex(), + p2wpkh_script=key_to_p2wpkh_script(pubkey).hex(), p2wpkh_addr=key_to_p2wpkh(pubkey), - p2sh_p2wpkh_script=CScript([OP_HASH160, hash160(CScript([OP_0, pkh])), OP_EQUAL]).hex(), - p2sh_p2wpkh_redeem_script=CScript([OP_0, pkh]).hex(), + p2sh_p2wpkh_script=script_to_p2sh_script(key_to_p2wpkh_script(pubkey)).hex(), + p2sh_p2wpkh_redeem_script=key_to_p2wpkh_script(pubkey).hex(), p2sh_p2wpkh_addr=key_to_p2sh_p2wpkh(pubkey)) def get_generate_key(): @@ -76,15 +72,14 @@ def get_generate_key(): eckey.generate() privkey = bytes_to_wif(eckey.get_bytes()) pubkey = eckey.get_pubkey().get_bytes().hex() - pkh = hash160(hex_str_to_bytes(pubkey)) return Key(privkey=privkey, pubkey=pubkey, - p2pkh_script=CScript([OP_DUP, OP_HASH160, pkh, OP_EQUALVERIFY, OP_CHECKSIG]).hex(), + p2pkh_script=key_to_p2pkh_script(pubkey).hex(), p2pkh_addr=key_to_p2pkh(pubkey), - p2wpkh_script=CScript([OP_0, pkh]).hex(), + p2wpkh_script=key_to_p2wpkh_script(pubkey).hex(), p2wpkh_addr=key_to_p2wpkh(pubkey), - p2sh_p2wpkh_script=CScript([OP_HASH160, hash160(CScript([OP_0, pkh])), OP_EQUAL]).hex(), - p2sh_p2wpkh_redeem_script=CScript([OP_0, pkh]).hex(), + p2sh_p2wpkh_script=script_to_p2sh_script(key_to_p2wpkh_script(pubkey)).hex(), + p2sh_p2wpkh_redeem_script=key_to_p2wpkh_script(pubkey).hex(), p2sh_p2wpkh_addr=key_to_p2sh_p2wpkh(pubkey)) def get_multisig(node): @@ -97,16 +92,16 @@ def get_multisig(node): addr = node.getaddressinfo(node.getnewaddress()) addrs.append(addr['address']) pubkeys.append(addr['pubkey']) - script_code = CScript([OP_2] + [hex_str_to_bytes(pubkey) for pubkey in pubkeys] + [OP_3, OP_CHECKMULTISIG]) - witness_script = CScript([OP_0, sha256(script_code)]) + script_code = CScript([OP_2] + [bytes.fromhex(pubkey) for pubkey in pubkeys] + [OP_3, OP_CHECKMULTISIG]) + witness_script = script_to_p2wsh_script(script_code) return Multisig(privkeys=[node.dumpprivkey(addr) for addr in addrs], pubkeys=pubkeys, - p2sh_script=CScript([OP_HASH160, hash160(script_code), OP_EQUAL]).hex(), + p2sh_script=script_to_p2sh_script(script_code).hex(), p2sh_addr=script_to_p2sh(script_code), redeem_script=script_code.hex(), p2wsh_script=witness_script.hex(), p2wsh_addr=script_to_p2wsh(script_code), - p2sh_p2wsh_script=CScript([OP_HASH160, witness_script, OP_EQUAL]).hex(), + p2sh_p2wsh_script=script_to_p2sh_script(witness_script).hex(), p2sh_p2wsh_addr=script_to_p2sh_p2wsh(script_code)) def test_address(node, address, **kwargs): diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index ad1acd2e2f..c5f08b27f2 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -98,6 +98,7 @@ BASE_SCRIPTS = [ 'rpc_fundrawtransaction.py --legacy-wallet', 'rpc_fundrawtransaction.py --descriptors', 'p2p_compactblocks.py', + 'p2p_compactblocks_blocksonly.py', 'feature_segwit.py --legacy-wallet', # vv Tests less than 2m vv 'wallet_basic.py --legacy-wallet', @@ -109,8 +110,7 @@ BASE_SCRIPTS = [ 'p2p_tx_download.py', 'mempool_updatefromblock.py', 'wallet_dump.py --legacy-wallet', - 'wallet_listtransactions.py --legacy-wallet', - 'wallet_listtransactions.py --descriptors', + 'feature_taproot.py --previous_release', 'feature_taproot.py', 'rpc_signer.py', 'wallet_signer.py --descriptors', @@ -122,10 +122,9 @@ BASE_SCRIPTS = [ 'wallet_listreceivedby.py --legacy-wallet', 'wallet_listreceivedby.py --descriptors', 'wallet_abandonconflict.py --legacy-wallet', + 'p2p_dns_seeds.py', 'wallet_abandonconflict.py --descriptors', 'feature_csv_activation.py', - 'rpc_rawtransaction.py --legacy-wallet', - 'rpc_rawtransaction.py --descriptors', 'wallet_address_types.py --legacy-wallet', 'wallet_address_types.py --descriptors', 'feature_bip68_sequence.py', @@ -139,6 +138,7 @@ BASE_SCRIPTS = [ 'interface_zmq.py', 'rpc_invalid_address_message.py', 'interface_bitcoin_cli.py', + 'feature_bind_extra.py', 'mempool_resurrect.py', 'wallet_txn_doublespend.py --mineblock', 'tool_wallet.py --legacy-wallet', @@ -159,6 +159,8 @@ BASE_SCRIPTS = [ 'wallet_createwallet.py --legacy-wallet', 'wallet_createwallet.py --usecli', 'wallet_createwallet.py --descriptors', + 'wallet_listtransactions.py --legacy-wallet', + 'wallet_listtransactions.py --descriptors', 'wallet_watchonly.py --legacy-wallet', 'wallet_watchonly.py --usecli --legacy-wallet', 'wallet_reorgsrestore.py', @@ -169,9 +171,13 @@ BASE_SCRIPTS = [ 'rpc_users.py', 'rpc_whitelist.py', 'feature_proxy.py', + 'feature_syscall_sandbox.py', 'rpc_signrawtransaction.py --legacy-wallet', 'rpc_signrawtransaction.py --descriptors', + 'rpc_rawtransaction.py --legacy-wallet', + 'rpc_rawtransaction.py --descriptors', 'wallet_groups.py --legacy-wallet', + 'wallet_transactiontime_rescan.py', 'p2p_addrv2_relay.py', 'wallet_groups.py --descriptors', 'p2p_compactblocks_hb.py', @@ -184,6 +190,7 @@ BASE_SCRIPTS = [ 'p2p_addr_relay.py', 'p2p_getaddr_caching.py', 'p2p_getdata.py', + 'p2p_addrfetch.py', 'rpc_net.py', 'wallet_keypool.py --legacy-wallet', 'wallet_keypool.py --descriptors', @@ -214,13 +221,15 @@ BASE_SCRIPTS = [ 'rpc_createmultisig.py --legacy-wallet', 'rpc_createmultisig.py --descriptors', 'rpc_packages.py', + 'mempool_package_limits.py', 'feature_versionbits_warning.py', 'rpc_preciousblock.py', 'wallet_importprunedfunds.py --legacy-wallet', 'wallet_importprunedfunds.py --descriptors', 'p2p_leak_tx.py', 'p2p_eviction.py', - 'rpc_signmessage.py', + 'wallet_signmessagewithaddress.py', + 'rpc_signmessagewithprivkey.py', 'rpc_generateblock.py', 'rpc_generate.py', 'wallet_balance.py --legacy-wallet', @@ -276,9 +285,11 @@ BASE_SCRIPTS = [ 'p2p_blockfilters.py', 'p2p_message_capture.py', 'feature_includeconf.py', + 'feature_addrman.py', 'feature_asmap.py', 'mempool_unbroadcast.py', 'mempool_compatibility.py', + 'mempool_accept_wtxid.py', 'rpc_deriveaddresses.py', 'rpc_deriveaddresses.py --usecli', 'p2p_ping.py', @@ -291,10 +302,11 @@ BASE_SCRIPTS = [ 'p2p_permissions.py', 'feature_blocksdir.py', 'wallet_startup.py', + 'p2p_i2p_ports.py', 'feature_config_args.py', + 'feature_presegwit_node_upgrade.py', 'feature_settings.py', 'rpc_getdescriptorinfo.py', - 'rpc_addresses_deprecation.py', 'rpc_help.py', 'feature_help.py', 'feature_shutdown.py', @@ -391,8 +403,9 @@ def main(): for test in tests: script = test.split("/")[-1] script = script + ".py" if ".py" not in script else script - if script in ALL_SCRIPTS: - test_list.append(script) + matching_scripts = [s for s in ALL_SCRIPTS if s.startswith(script)] + if matching_scripts: + test_list.extend(matching_scripts) else: print("{}WARNING!{} Test '{}' not found in full test list.".format(BOLD[1], BOLD[0], test)) elif args.extended: diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 28103793df..4bf3927879 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -242,7 +242,7 @@ class ToolWalletTest(BitcoinTestFramework): """ self.start_node(0) self.log.info('Generating transaction to mutate wallet') - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.stop_node(0) self.log.info('Calling wallet tool info after generating a transaction, testing output') @@ -344,7 +344,7 @@ class ToolWalletTest(BitcoinTestFramework): non_exist_dump = os.path.join(self.nodes[0].datadir, "wallet.nodump") self.assert_raises_tool_error('Unknown wallet file format "notaformat" provided. Please provide one of "bdb" or "sqlite".', '-wallet=todump', '-format=notaformat', '-dumpfile={}'.format(wallet_dump), 'createfromdump') self.assert_raises_tool_error('Dump file {} does not exist.'.format(non_exist_dump), '-wallet=todump', '-dumpfile={}'.format(non_exist_dump), 'createfromdump') - wallet_path = os.path.join(self.nodes[0].datadir, 'regtest/wallets/todump2') + wallet_path = os.path.join(self.nodes[0].datadir, 'regtest', 'wallets', 'todump2') self.assert_raises_tool_error('Failed to create database path \'{}\'. Database already exists.'.format(wallet_path), '-wallet=todump2', '-dumpfile={}'.format(wallet_dump), 'createfromdump') self.assert_raises_tool_error("The -descriptors option can only be used with the 'create' command.", '-descriptors', '-wallet=todump2', '-dumpfile={}'.format(wallet_dump), 'createfromdump') diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index d24cc802a4..70ca5a2fe4 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -29,14 +29,14 @@ class AbandonConflictTest(BitcoinTestFramework): self.skip_if_no_wallet() def run_test(self): - self.nodes[1].generate(COINBASE_MATURITY) + self.generate(self.nodes[1], COINBASE_MATURITY) self.sync_blocks() balance = self.nodes[0].getbalance() txA = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) txB = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) txC = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) self.sync_mempools() - self.nodes[1].generate(1) + self.generate(self.nodes[1], 1) # Can not abandon non-wallet transaction assert_raises_rpc_error(-5, 'Invalid or non-wallet transaction id', lambda: self.nodes[0].abandontransaction(txid='ff' * 32)) @@ -149,6 +149,7 @@ class AbandonConflictTest(BitcoinTestFramework): assert_equal(newbalance, balance - Decimal("24.9996")) balance = newbalance + self.log.info("Test transactions conflicted by a double spend") # Create a double spend of AB1 by spending again from only A's 10 output # Mine double spend from node 1 inputs = [] @@ -158,11 +159,39 @@ class AbandonConflictTest(BitcoinTestFramework): tx = self.nodes[0].createrawtransaction(inputs, outputs) signed = self.nodes[0].signrawtransactionwithwallet(tx) self.nodes[1].sendrawtransaction(signed["hex"]) - self.nodes[1].generate(1) + self.generate(self.nodes[1], 1) self.connect_nodes(0, 1) self.sync_blocks() + tx_list = self.nodes[0].listtransactions() + + conflicted = [tx for tx in tx_list if tx["confirmations"] < 0] + assert_equal(4, len(conflicted)) + + wallet_conflicts = [tx for tx in conflicted if tx["walletconflicts"]] + assert_equal(2, len(wallet_conflicts)) + + double_spends = [tx for tx in tx_list if tx["walletconflicts"] and tx["confirmations"] > 0] + assert_equal(1, len(double_spends)) + double_spend = double_spends[0] + + # Test the properties of the conflicted transactions, i.e. with confirmations < 0. + for tx in conflicted: + assert_equal(tx["abandoned"], False) + assert_equal(tx["confirmations"], -1) + assert_equal(tx["trusted"], False) + + # Test the properties of the double-spend transaction, i.e. having wallet conflicts and confirmations > 0. + assert_equal(double_spend["abandoned"], False) + assert_equal(double_spend["confirmations"], 1) + assert "trusted" not in double_spend.keys() # "trusted" only returned if tx has 0 or negative confirmations. + + # Test the walletconflicts field of each. + for tx in wallet_conflicts: + assert_equal(double_spend["walletconflicts"], [tx["txid"]]) + assert_equal(tx["walletconflicts"], [double_spend["txid"]]) + # Verify that B and C's 10 BTC outputs are available for spending again because AB1 is now conflicted newbalance = self.nodes[0].getbalance() assert_equal(newbalance, balance + Decimal("20")) diff --git a/test/functional/wallet_address_types.py b/test/functional/wallet_address_types.py index 9b97d08424..bdee22e62b 100755 --- a/test/functional/wallet_address_types.py +++ b/test/functional/wallet_address_types.py @@ -221,7 +221,7 @@ class AddressTypeTest(BitcoinTestFramework): def run_test(self): # Mine 101 blocks on node5 to bring nodes out of IBD and make sure that # no coinbases are maturing for the nodes-under-test during the test - self.nodes[5].generate(COINBASE_MATURITY + 1) + self.generate(self.nodes[5], COINBASE_MATURITY + 1) self.sync_blocks() uncompressed_1 = "0496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858ee" @@ -306,7 +306,7 @@ class AddressTypeTest(BitcoinTestFramework): assert_equal(unconf_balances[to_node], to_send * 10 * (2 + n)) # node5 collects fee and block subsidy to keep accounting simple - self.nodes[5].generate(1) + self.generate(self.nodes[5], 1) self.sync_blocks() # Verify that the receiving wallet contains a UTXO with the expected address, and expected descriptor @@ -336,7 +336,7 @@ class AddressTypeTest(BitcoinTestFramework): # Fund node 4: self.nodes[5].sendtoaddress(self.nodes[4].getnewaddress(), Decimal("1")) - self.nodes[5].generate(1) + self.generate(self.nodes[5], 1) self.sync_blocks() assert_equal(self.nodes[4].getbalance(), 1) diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index c13d8de4b5..12357e2d63 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -79,7 +79,7 @@ class AvoidReuseTest(BitcoinTestFramework): self.test_persistence() self.test_immutable() - self.nodes[0].generate(110) + self.generate(self.nodes[0], 110) self.sync_all() self.test_change_remains_change(self.nodes[1]) reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) @@ -174,7 +174,7 @@ class AvoidReuseTest(BitcoinTestFramework): retaddr = self.nodes[0].getnewaddress() self.nodes[0].sendtoaddress(fundaddr, 10) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() # listunspent should show 1 single, unused 10 btc output @@ -185,7 +185,7 @@ class AvoidReuseTest(BitcoinTestFramework): assert("used" not in self.nodes[0].getbalances()["mine"]) self.nodes[1].sendtoaddress(retaddr, 5) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() # listunspent should show 1 single, unused 5 btc output @@ -194,7 +194,7 @@ class AvoidReuseTest(BitcoinTestFramework): assert_balances(self.nodes[1], mine={"used": 0, "trusted": 5}) self.nodes[0].sendtoaddress(fundaddr, 10) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() # listunspent should show 2 total outputs (5, 10 btc), one unused (5), one reused (10) @@ -228,7 +228,7 @@ class AvoidReuseTest(BitcoinTestFramework): retaddr = self.nodes[0].getnewaddress() self.nodes[0].sendtoaddress(fundaddr, 10) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() # listunspent should show 1 single, unused 10 btc output @@ -237,7 +237,7 @@ class AvoidReuseTest(BitcoinTestFramework): assert_balances(self.nodes[1], mine={"used": 0, "trusted": 10}) self.nodes[1].sendtoaddress(retaddr, 5) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() # listunspent should show 1 single, unused 5 btc output @@ -259,7 +259,7 @@ class AvoidReuseTest(BitcoinTestFramework): assert_equal(second_addr_type, "legacy") self.nodes[0].sendtoaddress(new_fundaddr, 10) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() # listunspent should show 2 total outputs (5, 10 btc), one unused (5), one reused (10) @@ -302,7 +302,7 @@ class AvoidReuseTest(BitcoinTestFramework): for _ in range(101): self.nodes[0].sendtoaddress(new_addr, 1) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() # send transaction that should not use all the available outputs @@ -334,7 +334,7 @@ class AvoidReuseTest(BitcoinTestFramework): for _ in range(101): self.nodes[0].sendtoaddress(new_addr, 1) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() # Sending a transaction that is smaller than each one of the @@ -363,7 +363,7 @@ class AvoidReuseTest(BitcoinTestFramework): for _ in range(202): self.nodes[0].sendtoaddress(new_addr, 1) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() # Sending a transaction that needs to use the full groups diff --git a/test/functional/wallet_backup.py b/test/functional/wallet_backup.py index 05a0ef0ea1..bc6d6206e5 100755 --- a/test/functional/wallet_backup.py +++ b/test/functional/wallet_backup.py @@ -88,7 +88,7 @@ class WalletBackupTest(BitcoinTestFramework): # Have the miner (node3) mine a block. # Must sync mempools before mining. self.sync_mempools() - self.nodes[3].generate(1) + self.generate(self.nodes[3], 1) self.sync_blocks() # As above, this mirrors the original bash test. @@ -111,6 +111,18 @@ class WalletBackupTest(BitcoinTestFramework): os.remove(os.path.join(self.nodes[1].datadir, self.chain, 'wallets', self.default_wallet_name, self.wallet_data_filename)) os.remove(os.path.join(self.nodes[2].datadir, self.chain, 'wallets', self.default_wallet_name, self.wallet_data_filename)) + def restore_nonexistent_wallet(self): + node = self.nodes[3] + nonexistent_wallet_file = os.path.join(self.nodes[0].datadir, 'nonexistent_wallet.bak') + wallet_name = "res0" + assert_raises_rpc_error(-8, "Backup file does not exist", node.restorewallet, wallet_name, nonexistent_wallet_file) + + def restore_wallet_existent_name(self): + node = self.nodes[3] + wallet_file = os.path.join(self.nodes[0].datadir, 'wallet.bak') + wallet_name = "res0" + assert_raises_rpc_error(-8, "Wallet name already exists.", node.restorewallet, wallet_name, wallet_file) + def init_three(self): self.init_wallet(0) self.init_wallet(1) @@ -118,13 +130,13 @@ class WalletBackupTest(BitcoinTestFramework): def run_test(self): self.log.info("Generating initial blockchain") - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks() - self.nodes[1].generate(1) + self.generate(self.nodes[1], 1) self.sync_blocks() - self.nodes[2].generate(1) + self.generate(self.nodes[2], 1) self.sync_blocks() - self.nodes[3].generate(COINBASE_MATURITY) + self.generate(self.nodes[3], COINBASE_MATURITY) self.sync_blocks() assert_equal(self.nodes[0].getbalance(), 50) @@ -153,7 +165,7 @@ class WalletBackupTest(BitcoinTestFramework): self.do_one_round() # Generate 101 more blocks, so any fees paid mature - self.nodes[3].generate(COINBASE_MATURITY + 1) + self.generate(self.nodes[3], COINBASE_MATURITY + 1) self.sync_all() balance0 = self.nodes[0].getbalance() @@ -169,26 +181,27 @@ class WalletBackupTest(BitcoinTestFramework): ## # Test restoring spender wallets from backups ## - self.log.info("Restoring using wallet.dat") - self.stop_three() - self.erase_three() + self.log.info("Restoring wallets on node 3 using backup files") - # Start node2 with no chain - shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain, 'blocks')) - shutil.rmtree(os.path.join(self.nodes[2].datadir, self.chain, 'chainstate')) + self.restore_nonexistent_wallet() - # Restore wallets from backup - shutil.copyfile(os.path.join(self.nodes[0].datadir, 'wallet.bak'), os.path.join(self.nodes[0].datadir, self.chain, 'wallets', self.default_wallet_name, self.wallet_data_filename)) - shutil.copyfile(os.path.join(self.nodes[1].datadir, 'wallet.bak'), os.path.join(self.nodes[1].datadir, self.chain, 'wallets', self.default_wallet_name, self.wallet_data_filename)) - shutil.copyfile(os.path.join(self.nodes[2].datadir, 'wallet.bak'), os.path.join(self.nodes[2].datadir, self.chain, 'wallets', self.default_wallet_name, self.wallet_data_filename)) + backup_file_0 = os.path.join(self.nodes[0].datadir, 'wallet.bak') + backup_file_1 = os.path.join(self.nodes[1].datadir, 'wallet.bak') + backup_file_2 = os.path.join(self.nodes[2].datadir, 'wallet.bak') - self.log.info("Re-starting nodes") - self.start_three() - self.sync_blocks() + self.nodes[3].restorewallet("res0", backup_file_0) + self.nodes[3].restorewallet("res1", backup_file_1) + self.nodes[3].restorewallet("res2", backup_file_2) + + res0_rpc = self.nodes[3].get_wallet_rpc("res0") + res1_rpc = self.nodes[3].get_wallet_rpc("res1") + res2_rpc = self.nodes[3].get_wallet_rpc("res2") + + assert_equal(res0_rpc.getbalance(), balance0) + assert_equal(res1_rpc.getbalance(), balance1) + assert_equal(res2_rpc.getbalance(), balance2) - assert_equal(self.nodes[0].getbalance(), balance0) - assert_equal(self.nodes[1].getbalance(), balance1) - assert_equal(self.nodes[2].getbalance(), balance2) + self.restore_wallet_existent_name() if not self.options.descriptors: self.log.info("Restoring using dumped wallet") diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index 204a866c55..2abac7bb92 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -70,10 +70,10 @@ class WalletTest(BitcoinTestFramework): assert 'watchonly' not in self.nodes[1].getbalances() self.log.info("Mining blocks ...") - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() - self.nodes[1].generate(1) - self.nodes[1].generatetoaddress(COINBASE_MATURITY + 1, ADDRESS_WATCHONLY) + self.generate(self.nodes[1], 1) + self.generatetoaddress(self.nodes[1], COINBASE_MATURITY + 1, ADDRESS_WATCHONLY) self.sync_all() if not self.options.descriptors: @@ -196,7 +196,7 @@ class WalletTest(BitcoinTestFramework): self.log.info("Test getbalance and getbalances.mine.untrusted_pending with conflicted unconfirmed inputs") test_balances(fee_node_1=Decimal('0.02')) - self.nodes[1].generatetoaddress(1, ADDRESS_WATCHONLY) + self.generatetoaddress(self.nodes[1], 1, ADDRESS_WATCHONLY) self.sync_all() # balances are correct after the transactions are confirmed @@ -210,7 +210,7 @@ class WalletTest(BitcoinTestFramework): # Send total balance away from node 1 txs = create_transactions(self.nodes[1], self.nodes[0].getnewaddress(), Decimal('29.97'), [Decimal('0.01')]) self.nodes[1].sendrawtransaction(txs[0]['hex']) - self.nodes[1].generatetoaddress(2, ADDRESS_WATCHONLY) + self.generatetoaddress(self.nodes[1], 2, ADDRESS_WATCHONLY) self.sync_all() # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago @@ -257,7 +257,7 @@ class WalletTest(BitcoinTestFramework): self.nodes[1].sendrawtransaction(hexstring=tx_replace, maxfeerate=0) # Now confirm tx_replace - block_reorg = self.nodes[1].generatetoaddress(1, ADDRESS_WATCHONLY)[0] + block_reorg = self.generatetoaddress(self.nodes[1], 1, ADDRESS_WATCHONLY)[0] self.sync_all() assert_equal(self.nodes[0].getbalance(minconf=0), total_amount) @@ -265,7 +265,7 @@ class WalletTest(BitcoinTestFramework): self.nodes[0].invalidateblock(block_reorg) self.nodes[1].invalidateblock(block_reorg) assert_equal(self.nodes[0].getbalance(minconf=0), 0) # wallet txs not in the mempool are untrusted - self.nodes[0].generatetoaddress(1, ADDRESS_WATCHONLY) + self.generatetoaddress(self.nodes[0], 1, ADDRESS_WATCHONLY) assert_equal(self.nodes[0].getbalance(minconf=0), 0) # wallet txs not in the mempool are untrusted # Now confirm tx_orig @@ -273,7 +273,7 @@ class WalletTest(BitcoinTestFramework): self.connect_nodes(0, 1) self.sync_blocks() self.nodes[1].sendrawtransaction(tx_orig) - self.nodes[1].generatetoaddress(1, ADDRESS_WATCHONLY) + self.generatetoaddress(self.nodes[1], 1, ADDRESS_WATCHONLY) self.sync_all() assert_equal(self.nodes[0].getbalance(minconf=0), total_amount + 1) # The reorg recovered our fee of 1 coin diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index b5afc3785e..599e506f98 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -13,6 +13,7 @@ from test_framework.util import ( assert_equal, assert_fee_amount, assert_raises_rpc_error, + find_vout_for_address, ) from test_framework.wallet_util import test_address @@ -59,14 +60,14 @@ class WalletTest(BitcoinTestFramework): self.log.info("Mining blocks...") - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) walletinfo = self.nodes[0].getwalletinfo() assert_equal(walletinfo['immature_balance'], 50) assert_equal(walletinfo['balance'], 0) self.sync_all(self.nodes[0:3]) - self.nodes[1].generate(COINBASE_MATURITY + 1) + self.generate(self.nodes[1], COINBASE_MATURITY + 1) self.sync_all(self.nodes[0:3]) assert_equal(self.nodes[0].getbalance(), 50) @@ -115,19 +116,55 @@ class WalletTest(BitcoinTestFramework): assert_equal(walletinfo['immature_balance'], 0) # Have node0 mine a block, thus it will collect its own fee. - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all(self.nodes[0:3]) # Exercise locking of unspent outputs unspent_0 = self.nodes[2].listunspent()[0] unspent_0 = {"txid": unspent_0["txid"], "vout": unspent_0["vout"]} + # Trying to unlock an output which isn't locked should error assert_raises_rpc_error(-8, "Invalid parameter, expected locked output", self.nodes[2].lockunspent, True, [unspent_0]) + + # Locking an already-locked output should error self.nodes[2].lockunspent(False, [unspent_0]) assert_raises_rpc_error(-8, "Invalid parameter, output already locked", self.nodes[2].lockunspent, False, [unspent_0]) + + # Restarting the node should clear the lock + self.restart_node(2) + self.nodes[2].lockunspent(False, [unspent_0]) + + # Unloading and reloating the wallet should clear the lock + assert_equal(self.nodes[0].listwallets(), [self.default_wallet_name]) + self.nodes[2].unloadwallet(self.default_wallet_name) + self.nodes[2].loadwallet(self.default_wallet_name) + assert_equal(len(self.nodes[2].listlockunspent()), 0) + + # Locking non-persistently, then re-locking persistently, is allowed + self.nodes[2].lockunspent(False, [unspent_0]) + self.nodes[2].lockunspent(False, [unspent_0], True) + + # Restarting the node with the lock written to the wallet should keep the lock + self.restart_node(2) + assert_raises_rpc_error(-8, "Invalid parameter, output already locked", self.nodes[2].lockunspent, False, [unspent_0]) + + # Unloading and reloading the wallet with a persistent lock should keep the lock + self.nodes[2].unloadwallet(self.default_wallet_name) + self.nodes[2].loadwallet(self.default_wallet_name) + assert_raises_rpc_error(-8, "Invalid parameter, output already locked", self.nodes[2].lockunspent, False, [unspent_0]) + + # Locked outputs should not be used, even if they are the only available funds assert_raises_rpc_error(-6, "Insufficient funds", self.nodes[2].sendtoaddress, self.nodes[2].getnewaddress(), 20) assert_equal([unspent_0], self.nodes[2].listlockunspent()) + + # Unlocking should remove the persistent lock self.nodes[2].lockunspent(True, [unspent_0]) + self.restart_node(2) assert_equal(len(self.nodes[2].listlockunspent()), 0) + + # Reconnect node 2 after restarts + self.connect_nodes(1, 2) + self.connect_nodes(0, 2) + assert_raises_rpc_error(-8, "txid must be of length 64 (not 34, for '0000000000000000000000000000000000')", self.nodes[2].lockunspent, False, [{"txid": "0000000000000000000000000000000000", "vout": 0}]) @@ -159,7 +196,7 @@ class WalletTest(BitcoinTestFramework): assert_equal(len(self.nodes[1].listlockunspent()), 0) # Have node1 generate 100 blocks (so node0 can recover the fee) - self.nodes[1].generate(COINBASE_MATURITY) + self.generate(self.nodes[1], COINBASE_MATURITY) self.sync_all(self.nodes[0:3]) # node0 should end up with 100 btc in block rewards plus fees, but @@ -188,7 +225,7 @@ class WalletTest(BitcoinTestFramework): self.nodes[1].sendrawtransaction(hexstring=txns_to_send[1]["hex"], maxfeerate=0) # Have node1 mine a block to confirm transactions: - self.nodes[1].generate(1) + self.generate(self.nodes[1], 1) self.sync_all(self.nodes[0:3]) assert_equal(self.nodes[0].getbalance(), 0) @@ -203,14 +240,14 @@ class WalletTest(BitcoinTestFramework): fee_per_byte = Decimal('0.001') / 1000 self.nodes[2].settxfee(fee_per_byte * 1000) txid = self.nodes[2].sendtoaddress(address, 10, "", "", False) - self.nodes[2].generate(1) + self.generate(self.nodes[2], 1) self.sync_all(self.nodes[0:3]) node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), Decimal('84'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) assert_equal(self.nodes[0].getbalance(), Decimal('10')) # Send 10 BTC with subtract fee from amount txid = self.nodes[2].sendtoaddress(address, 10, "", "", True) - self.nodes[2].generate(1) + self.generate(self.nodes[2], 1) self.sync_all(self.nodes[0:3]) node_2_bal -= Decimal('10') assert_equal(self.nodes[2].getbalance(), node_2_bal) @@ -220,7 +257,7 @@ class WalletTest(BitcoinTestFramework): # Sendmany 10 BTC txid = self.nodes[2].sendmany('', {address: 10}, 0, "", []) - self.nodes[2].generate(1) + self.generate(self.nodes[2], 1) self.sync_all(self.nodes[0:3]) node_0_bal += Decimal('10') node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), node_2_bal - Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) @@ -228,7 +265,7 @@ class WalletTest(BitcoinTestFramework): # Sendmany 10 BTC with subtract fee from amount txid = self.nodes[2].sendmany('', {address: 10}, 0, "", [address]) - self.nodes[2].generate(1) + self.generate(self.nodes[2], 1) self.sync_all(self.nodes[0:3]) node_2_bal -= Decimal('10') assert_equal(self.nodes[2].getbalance(), node_2_bal) @@ -241,7 +278,7 @@ class WalletTest(BitcoinTestFramework): # Test passing fee_rate as a string txid = self.nodes[2].sendmany(amounts={address: 10}, fee_rate=str(fee_rate_sat_vb)) - self.nodes[2].generate(1) + self.generate(self.nodes[2], 1) self.sync_all(self.nodes[0:3]) balance = self.nodes[2].getbalance() node_2_bal = self.check_fee_amount(balance, node_2_bal - Decimal('10'), explicit_fee_rate_btc_kvb, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) @@ -252,7 +289,7 @@ class WalletTest(BitcoinTestFramework): # Test passing fee_rate as an integer amount = Decimal("0.0001") txid = self.nodes[2].sendmany(amounts={address: amount}, fee_rate=fee_rate_sat_vb) - self.nodes[2].generate(1) + self.generate(self.nodes[2], 1) self.sync_all(self.nodes[0:3]) balance = self.nodes[2].getbalance() node_2_bal = self.check_fee_amount(balance, node_2_bal - amount, explicit_fee_rate_btc_kvb, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) @@ -314,7 +351,7 @@ class WalletTest(BitcoinTestFramework): self.nodes[1].sendrawtransaction(signed_raw_tx['hex']) self.sync_all() - self.nodes[1].generate(1) # mine a block + self.generate(self.nodes[1], 1) # mine a block self.sync_all() unspent_txs = self.nodes[0].listunspent() # zero value tx must be in listunspents output @@ -337,13 +374,13 @@ class WalletTest(BitcoinTestFramework): txid_not_broadcast = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 2) tx_obj_not_broadcast = self.nodes[0].gettransaction(txid_not_broadcast) - self.nodes[1].generate(1) # mine a block, tx should not be in there + self.generate(self.nodes[1], 1) # mine a block, tx should not be in there self.sync_all(self.nodes[0:3]) assert_equal(self.nodes[2].getbalance(), node_2_bal) # should not be changed because tx was not broadcasted # now broadcast from another node, mine a block, sync, and check the balance self.nodes[1].sendrawtransaction(tx_obj_not_broadcast['hex']) - self.nodes[1].generate(1) + self.generate(self.nodes[1], 1) self.sync_all(self.nodes[0:3]) node_2_bal += 2 tx_obj_not_broadcast = self.nodes[0].gettransaction(txid_not_broadcast) @@ -362,7 +399,7 @@ class WalletTest(BitcoinTestFramework): self.connect_nodes(0, 2) self.sync_blocks(self.nodes[0:3]) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks(self.nodes[0:3]) node_2_bal += 2 @@ -391,7 +428,7 @@ class WalletTest(BitcoinTestFramework): assert_raises_rpc_error(-3, "Invalid amount", self.nodes[0].sendtoaddress, self.nodes[2].getnewaddress(), "1f-4") # This will raise an exception since generate does not accept a string - assert_raises_rpc_error(-1, "not an integer", self.nodes[0].generate, "2") + assert_raises_rpc_error(-1, "not an integer", self.generate, self.nodes[0], "2") if not self.options.descriptors: @@ -427,7 +464,10 @@ class WalletTest(BitcoinTestFramework): # 1. Send some coins to generate new UTXO address_to_import = self.nodes[2].getnewaddress() txid = self.nodes[0].sendtoaddress(address_to_import, 1) - self.nodes[0].generate(1) + self.sync_mempools(self.nodes[0:3]) + vout = find_vout_for_address(self.nodes[2], txid, address_to_import) + self.nodes[2].lockunspent(False, [{"txid": txid, "vout": vout}]) + self.generate(self.nodes[0], 1) self.sync_all(self.nodes[0:3]) self.log.info("Test sendtoaddress with fee_rate param (explicit fee rate in sat/vB)") @@ -440,7 +480,7 @@ class WalletTest(BitcoinTestFramework): # Test passing fee_rate as an integer txid = self.nodes[2].sendtoaddress(address=address, amount=amount, fee_rate=fee_rate_sat_vb) tx_size = self.get_vsize(self.nodes[2].gettransaction(txid)['hex']) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all(self.nodes[0:3]) postbalance = self.nodes[2].getbalance() fee = prebalance - postbalance - Decimal(amount) @@ -453,7 +493,7 @@ class WalletTest(BitcoinTestFramework): # Test passing fee_rate as a string txid = self.nodes[2].sendtoaddress(address=address, amount=amount, fee_rate=str(fee_rate_sat_vb)) tx_size = self.get_vsize(self.nodes[2].gettransaction(txid)['hex']) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all(self.nodes[0:3]) postbalance = self.nodes[2].getbalance() fee = prebalance - postbalance - amount @@ -515,7 +555,7 @@ class WalletTest(BitcoinTestFramework): # Mine a block from node0 to an address from node1 coinbase_addr = self.nodes[1].getnewaddress() - block_hash = self.nodes[0].generatetoaddress(1, coinbase_addr)[0] + block_hash = self.generatetoaddress(self.nodes[0], 1, coinbase_addr)[0] coinbase_txid = self.nodes[0].getblock(block_hash)['tx'][0] self.sync_all(self.nodes[0:3]) @@ -524,7 +564,7 @@ class WalletTest(BitcoinTestFramework): # check if wallet or blockchain maintenance changes the balance self.sync_all(self.nodes[0:3]) - blocks = self.nodes[0].generate(2) + blocks = self.generate(self.nodes[0], 2) self.sync_all(self.nodes[0:3]) balance_nodes = [self.nodes[i].getbalance() for i in range(3)] block_count = self.nodes[0].getblockcount() @@ -542,23 +582,17 @@ class WalletTest(BitcoinTestFramework): assert label in self.nodes[0].listlabels() self.nodes[0].rpc.ensure_ascii = True # restore to default - # maintenance tests - maintenance = [ - '-rescan', - '-reindex', - ] + # -reindex tests chainlimit = 6 - for m in maintenance: - self.log.info("Test " + m) - self.stop_nodes() - # set lower ancestor limit for later - self.start_node(0, [m, "-limitancestorcount=" + str(chainlimit)]) - self.start_node(1, [m, "-limitancestorcount=" + str(chainlimit)]) - self.start_node(2, [m, "-limitancestorcount=" + str(chainlimit)]) - if m == '-reindex': - # reindex will leave rpc warm up "early"; Wait for it to finish - self.wait_until(lambda: [block_count] * 3 == [self.nodes[i].getblockcount() for i in range(3)]) - assert_equal(balance_nodes, [self.nodes[i].getbalance() for i in range(3)]) + self.log.info("Test -reindex") + self.stop_nodes() + # set lower ancestor limit for later + self.start_node(0, ['-reindex', "-limitancestorcount=" + str(chainlimit)]) + self.start_node(1, ['-reindex', "-limitancestorcount=" + str(chainlimit)]) + self.start_node(2, ['-reindex', "-limitancestorcount=" + str(chainlimit)]) + # reindex will leave rpc warm up "early"; Wait for it to finish + self.wait_until(lambda: [block_count] * 3 == [self.nodes[i].getblockcount() for i in range(3)]) + assert_equal(balance_nodes, [self.nodes[i].getbalance() for i in range(3)]) # Exercise listsinceblock with the last two blocks coinbase_tx_1 = self.nodes[0].listsinceblock(blocks[0]) @@ -572,13 +606,13 @@ class WalletTest(BitcoinTestFramework): # Get all non-zero utxos together chain_addrs = [self.nodes[0].getnewaddress(), self.nodes[0].getnewaddress()] singletxid = self.nodes[0].sendtoaddress(chain_addrs[0], self.nodes[0].getbalance(), "", "", True) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) node0_balance = self.nodes[0].getbalance() # Split into two chains rawtx = self.nodes[0].createrawtransaction([{"txid": singletxid, "vout": 0}], {chain_addrs[0]: node0_balance / 2 - Decimal('0.01'), chain_addrs[1]: node0_balance / 2 - Decimal('0.01')}) signedtx = self.nodes[0].signrawtransactionwithwallet(rawtx) singletxid = self.nodes[0].sendrawtransaction(hexstring=signedtx["hex"], maxfeerate=0) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) # Make a long chain of unconfirmed payments without hitting mempool limit # Each tx we make leaves only one output of change on a chain 1 longer @@ -600,7 +634,7 @@ class WalletTest(BitcoinTestFramework): total_txs = len(self.nodes[0].listtransactions("*", 99999)) # Try with walletrejectlongchains - # Double chain limit but require combining inputs, so we pass SelectCoinsMinConf + # Double chain limit but require combining inputs, so we pass AttemptSelection self.stop_node(0) extra_args = ["-walletrejectlongchains", "-limitancestorcount=" + str(2 * chainlimit)] self.start_node(0, extra_args=extra_args) @@ -629,7 +663,7 @@ class WalletTest(BitcoinTestFramework): assert not address_info["ischange"] # Test getaddressinfo 'ischange' field on change address. - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) destination = self.nodes[1].getnewaddress() txid = self.nodes[0].sendtoaddress(destination, 0.123) tx = self.nodes[0].decoderawtransaction(self.nodes[0].gettransaction(txid)['hex']) diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index c04986038d..a1676fffa5 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -59,7 +59,7 @@ class BumpFeeTest(BitcoinTestFramework): def clear_mempool(self): # Clear mempool between subtests. The subtests may only depend on chainstate (utxos) - self.nodes[1].generate(1) + self.generate(self.nodes[1], 1) self.sync_all() def run_test(self): @@ -72,12 +72,12 @@ class BumpFeeTest(BitcoinTestFramework): # fund rbf node with 10 coins of 0.001 btc (100,000 satoshis) self.log.info("Mining blocks...") - peer_node.generate(110) + self.generate(peer_node, 110) self.sync_all() for _ in range(25): peer_node.sendtoaddress(rbf_node_address, 0.001) self.sync_all() - peer_node.generate(1) + self.generate(peer_node, 1) self.sync_all() assert_equal(rbf_node.getbalance(), Decimal("0.025")) @@ -272,7 +272,7 @@ def test_small_output_with_feerate_succeeds(self, rbf_node, dest_address): self.log.info('Testing small output with feerate bump succeeds') # Make sure additional inputs exist - rbf_node.generatetoaddress(COINBASE_MATURITY + 1, rbf_node.getnewaddress()) + self.generatetoaddress(rbf_node, COINBASE_MATURITY + 1, rbf_node.getnewaddress()) rbfid = spend_one_input(rbf_node, dest_address) input_list = rbf_node.getrawtransaction(rbfid, 1)["vin"] assert_equal(len(input_list), 1) @@ -305,7 +305,7 @@ def test_small_output_with_feerate_succeeds(self, rbf_node, dest_address): if txin["txid"] == original_txin["txid"] and txin["vout"] == original_txin["vout"]] - rbf_node.generatetoaddress(1, rbf_node.getnewaddress()) + self.generatetoaddress(rbf_node, 1, rbf_node.getnewaddress()) assert_equal(rbf_node.gettransaction(rbfid)["confirmations"], 1) self.clear_mempool() @@ -433,7 +433,7 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): funding_address1 = watcher.getnewaddress(address_type='bech32') funding_address2 = watcher.getnewaddress(address_type='bech32') peer_node.sendmany("", {funding_address1: 0.001, funding_address2: 0.001}) - peer_node.generate(1) + self.generate(peer_node, 1) self.sync_all() # Create single-input PSBT for transaction to be bumped @@ -519,7 +519,7 @@ def test_unconfirmed_not_spendable(self, rbf_node, rbf_node_address): assert_equal([t for t in rbf_node.listunspent(minconf=0, include_unsafe=False) if t["txid"] == rbfid], []) # check that the main output from the rbf tx is spendable after confirmed - rbf_node.generate(1) + self.generate(rbf_node, 1) assert_equal( sum(1 for t in rbf_node.listunspent(minconf=0, include_unsafe=False) if t["txid"] == rbfid and t["address"] == rbf_node_address and t["spendable"]), 1) @@ -529,7 +529,7 @@ def test_unconfirmed_not_spendable(self, rbf_node, rbf_node_address): def test_bumpfee_metadata(self, rbf_node, dest_address): self.log.info('Test that bumped txn metadata persists to new txn record') assert(rbf_node.getbalance() < 49) - rbf_node.generatetoaddress(101, rbf_node.getnewaddress()) + self.generatetoaddress(rbf_node, 101, rbf_node.getnewaddress()) rbfid = rbf_node.sendtoaddress(dest_address, 49, "comment value", "to value") bumped_tx = rbf_node.bumpfee(rbfid) bumped_wtx = rbf_node.gettransaction(bumped_tx["txid"]) @@ -599,7 +599,7 @@ def submit_block_with_tx(node, tx): def test_no_more_inputs_fails(self, rbf_node, dest_address): self.log.info('Test that bumpfee fails when there are no available confirmed outputs') # feerate rbf requires confirmed outputs when change output doesn't exist or is insufficient - rbf_node.generatetoaddress(1, dest_address) + self.generatetoaddress(rbf_node, 1, dest_address) # spend all funds, no change output rbfid = rbf_node.sendtoaddress(rbf_node.getnewaddress(), rbf_node.getbalance(), "", "", True) assert_raises_rpc_error(-4, "Unable to create transaction. Insufficient funds", rbf_node.bumpfee, rbfid) diff --git a/test/functional/wallet_coinbase_category.py b/test/functional/wallet_coinbase_category.py index 7aa8b44ebd..3c7abd0800 100755 --- a/test/functional/wallet_coinbase_category.py +++ b/test/functional/wallet_coinbase_category.py @@ -33,7 +33,7 @@ class CoinbaseCategoryTest(BitcoinTestFramework): def run_test(self): # Generate one block to an address address = self.nodes[0].getnewaddress() - self.nodes[0].generatetoaddress(1, address) + self.generatetoaddress(self.nodes[0], 1, address) hash = self.nodes[0].getbestblockhash() txid = self.nodes[0].getblock(hash)["tx"][0] @@ -41,12 +41,12 @@ class CoinbaseCategoryTest(BitcoinTestFramework): self.assert_category("immature", address, txid, 0) # Mine another 99 blocks on top - self.nodes[0].generate(99) + self.generate(self.nodes[0], 99) # Coinbase transaction is still immature after 100 confirmations self.assert_category("immature", address, txid, 99) # Mine one more block - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) # Coinbase transaction is now matured, so category is "generate" self.assert_category("generate", address, txid, 100) diff --git a/test/functional/wallet_create_tx.py b/test/functional/wallet_create_tx.py index a39a3c8d9b..c8b92ef1bf 100755 --- a/test/functional/wallet_create_tx.py +++ b/test/functional/wallet_create_tx.py @@ -24,7 +24,7 @@ class CreateTxWalletTest(BitcoinTestFramework): def run_test(self): self.log.info('Create some old blocks') self.nodes[0].setmocktime(TIME_GENESIS_BLOCK) - self.nodes[0].generate(200) + self.generate(self.nodes[0], 200) self.nodes[0].setmocktime(0) self.test_anti_fee_sniping() @@ -38,7 +38,7 @@ class CreateTxWalletTest(BitcoinTestFramework): assert_equal(tx['locktime'], 0) self.log.info('Check that anti-fee-sniping is enabled when we mine a recent block') - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) tx = self.nodes[0].decoderawtransaction(self.nodes[0].gettransaction(txid)['hex']) assert 0 < tx['locktime'] <= 201 diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index 16a0a50b07..d806f8f6d2 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -24,7 +24,7 @@ class CreateWalletTest(BitcoinTestFramework): def run_test(self): node = self.nodes[0] - node.generate(1) # Leave IBD for sethdseed + self.generate(node, 1) # Leave IBD for sethdseed self.nodes[0].createwallet(wallet_name='w0') w0 = node.get_wallet_rpc('w0') diff --git a/test/functional/wallet_descriptor.py b/test/functional/wallet_descriptor.py index c6f5d334f8..17a4c79da3 100755 --- a/test/functional/wallet_descriptor.py +++ b/test/functional/wallet_descriptor.py @@ -84,7 +84,7 @@ class WalletDescriptorTest(BitcoinTestFramework): send_wrpc = self.nodes[0].get_wallet_rpc("desc1") # Generate some coins - send_wrpc.generatetoaddress(COINBASE_MATURITY + 1, send_wrpc.getnewaddress()) + self.generatetoaddress(send_wrpc, COINBASE_MATURITY + 1, send_wrpc.getnewaddress()) # Make transactions self.log.info("Test sending and receiving") diff --git a/test/functional/wallet_disable.py b/test/functional/wallet_disable.py index 78cf378642..d0043e9bbb 100755 --- a/test/functional/wallet_disable.py +++ b/test/functional/wallet_disable.py @@ -28,8 +28,8 @@ class DisableWalletTest (BitcoinTestFramework): # Checking mining to an address without a wallet. Generating to a valid address should succeed # but generating to an invalid address will fail. - self.nodes[0].generatetoaddress(1, 'mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ') - assert_raises_rpc_error(-5, "Invalid address", self.nodes[0].generatetoaddress, 1, '3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy') + self.generatetoaddress(self.nodes[0], 1, 'mneYUmWYsuk7kySiURxCi3AGxrAqZxLgPZ') + assert_raises_rpc_error(-5, "Invalid address", self.generatetoaddress, self.nodes[0], 1, '3J98t1WpEZ73CNmQviecrnyiWrnqRhWNLy') if __name__ == '__main__': DisableWalletTest ().main () diff --git a/test/functional/wallet_dump.py b/test/functional/wallet_dump.py index eb54da99f5..06460e17d2 100755 --- a/test/functional/wallet_dump.py +++ b/test/functional/wallet_dump.py @@ -134,7 +134,7 @@ class WalletDumpTest(BitcoinTestFramework): self.log.info('Mine a block one second before the wallet is dumped') dump_time = int(time.time()) self.nodes[0].setmocktime(dump_time - 1) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.nodes[0].setmocktime(dump_time) dump_time_str = '# * Created on {}Z'.format( datetime.datetime.fromtimestamp( @@ -209,6 +209,15 @@ class WalletDumpTest(BitcoinTestFramework): with self.nodes[0].assert_debug_log(['Flushing wallet.dat'], timeout=20): self.nodes[0].getnewaddress() + # Make sure that dumpwallet doesn't have a lock order issue when there is an unconfirmed tx and it is reloaded + # See https://github.com/bitcoin/bitcoin/issues/22489 + self.nodes[0].createwallet("w3") + w3 = self.nodes[0].get_wallet_rpc("w3") + w3.importprivkey(privkey=self.nodes[0].get_deterministic_priv_key().key, label="coinbase_import") + w3.sendtoaddress(w3.getnewaddress(), 10) + w3.unloadwallet() + self.nodes[0].loadwallet("w3") + w3.dumpwallet(os.path.join(self.nodes[0].datadir, "w3.dump")) if __name__ == '__main__': WalletDumpTest().main() diff --git a/test/functional/wallet_fallbackfee.py b/test/functional/wallet_fallbackfee.py index b28f3ecebc..674c37dc73 100755 --- a/test/functional/wallet_fallbackfee.py +++ b/test/functional/wallet_fallbackfee.py @@ -17,7 +17,7 @@ class WalletRBFTest(BitcoinTestFramework): self.skip_if_no_wallet() def run_test(self): - self.nodes[0].generate(COINBASE_MATURITY + 1) + self.generate(self.nodes[0], COINBASE_MATURITY + 1) # sending a transaction without fee estimations must be possible by default on regtest self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) diff --git a/test/functional/wallet_groups.py b/test/functional/wallet_groups.py index d9d135a986..802fed6e7d 100755 --- a/test/functional/wallet_groups.py +++ b/test/functional/wallet_groups.py @@ -34,7 +34,7 @@ class WalletGroupTest(BitcoinTestFramework): def run_test(self): self.log.info("Setting up") # Mine some coins - self.nodes[0].generate(COINBASE_MATURITY + 1) + self.generate(self.nodes[0], COINBASE_MATURITY + 1) # Get some addresses from the two nodes addr1 = [self.nodes[1].getnewaddress() for _ in range(3)] @@ -45,7 +45,7 @@ class WalletGroupTest(BitcoinTestFramework): [self.nodes[0].sendtoaddress(addr, 1.0) for addr in addrs] [self.nodes[0].sendtoaddress(addr, 0.5) for addr in addrs] - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() # For each node, send 0.2 coins back to 0; @@ -77,7 +77,7 @@ class WalletGroupTest(BitcoinTestFramework): self.log.info("Test avoiding partial spends if warranted, even if avoidpartialspends is disabled") self.sync_all() - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) # Nodes 1-2 now have confirmed UTXOs (letters denote destinations): # Node #1: Node #2: # - A 1.0 - D0 1.0 @@ -113,7 +113,7 @@ class WalletGroupTest(BitcoinTestFramework): addr_aps = self.nodes[3].getnewaddress() self.nodes[0].sendtoaddress(addr_aps, 1.0) self.nodes[0].sendtoaddress(addr_aps, 1.0) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() with self.nodes[3].assert_debug_log(['Fee non-grouped = 2820, grouped = 4160, using grouped']): txid4 = self.nodes[3].sendtoaddress(self.nodes[0].getnewaddress(), 0.1) @@ -125,7 +125,7 @@ class WalletGroupTest(BitcoinTestFramework): addr_aps2 = self.nodes[3].getnewaddress() [self.nodes[0].sendtoaddress(addr_aps2, 1.0) for _ in range(5)] - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() with self.nodes[3].assert_debug_log(['Fee non-grouped = 5520, grouped = 8240, using non-grouped']): txid5 = self.nodes[3].sendtoaddress(self.nodes[0].getnewaddress(), 2.95) @@ -139,7 +139,7 @@ class WalletGroupTest(BitcoinTestFramework): self.log.info("Test wallet option maxapsfee threshold from non-grouped to grouped") addr_aps3 = self.nodes[4].getnewaddress() [self.nodes[0].sendtoaddress(addr_aps3, 1.0) for _ in range(5)] - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() with self.nodes[4].assert_debug_log(['Fee non-grouped = 5520, grouped = 8240, using grouped']): txid6 = self.nodes[4].sendtoaddress(self.nodes[0].getnewaddress(), 2.95) @@ -151,7 +151,7 @@ class WalletGroupTest(BitcoinTestFramework): # Empty out node2's wallet self.nodes[2].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=self.nodes[2].getbalance(), subtractfeefromamount=True) self.sync_all() - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.log.info("Fill a wallet with 10,000 outputs corresponding to the same scriptPubKey") for _ in range(5): @@ -162,7 +162,7 @@ class WalletGroupTest(BitcoinTestFramework): funded_tx = self.nodes[0].fundrawtransaction(tx.serialize().hex()) signed_tx = self.nodes[0].signrawtransactionwithwallet(funded_tx['hex']) self.nodes[0].sendrawtransaction(signed_tx['hex']) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() # Check that we can create a transaction that only requires ~100 of our diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index d41a389197..974ce7f381 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -49,7 +49,7 @@ class WalletHDTest(BitcoinTestFramework): # Derive some HD addresses and remember the last # Also send funds to each add - self.nodes[0].generate(COINBASE_MATURITY + 1) + self.generate(self.nodes[0], COINBASE_MATURITY + 1) hd_add = None NUM_HD_ADDS = 10 for i in range(1, NUM_HD_ADDS + 1): @@ -61,9 +61,9 @@ class WalletHDTest(BitcoinTestFramework): assert_equal(hd_info["hdkeypath"], "m/0'/0'/" + str(i) + "'") assert_equal(hd_info["hdmasterfingerprint"], hd_fingerprint) self.nodes[0].sendtoaddress(hd_add, 1) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.nodes[0].sendtoaddress(non_hd_add, 1) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) # create an internal key (again) change_addr = self.nodes[1].getrawchangeaddress() @@ -103,7 +103,7 @@ class WalletHDTest(BitcoinTestFramework): self.sync_all() # Needs rescan - self.restart_node(1, extra_args=self.extra_args[1] + ['-rescan']) + self.nodes[1].rescanblockchain() assert_equal(self.nodes[1].getbalance(), NUM_HD_ADDS + 1) # Try a RPC based rescan @@ -179,7 +179,7 @@ class WalletHDTest(BitcoinTestFramework): assert_raises_rpc_error(-5, "Already have this key", self.nodes[1].sethdseed, False, self.nodes[1].dumpprivkey(self.nodes[1].getnewaddress())) self.log.info('Test sethdseed restoring with keys outside of the initial keypool') - self.nodes[0].generate(10) + self.generate(self.nodes[0], 10) # Restart node 1 with keypool of 3 and a different wallet self.nodes[1].createwallet(wallet_name='origin', blank=True) self.restart_node(1, extra_args=['-keypool=3', '-wallet=origin']) @@ -228,7 +228,7 @@ class WalletHDTest(BitcoinTestFramework): # The wallet that has set a new seed (restore_rpc) should not detect this transaction. txid = self.nodes[0].sendtoaddress(addr, 1) origin_rpc.sendrawtransaction(self.nodes[0].gettransaction(txid)['hex']) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks() origin_rpc.gettransaction(txid) assert_raises_rpc_error(-5, 'Invalid or non-wallet transaction id', restore_rpc.gettransaction, txid) @@ -239,7 +239,7 @@ class WalletHDTest(BitcoinTestFramework): # The previous transaction (out_of_kp_txid) should still not be detected as a rescan is required. txid = self.nodes[0].sendtoaddress(last_addr, 1) origin_rpc.sendrawtransaction(self.nodes[0].gettransaction(txid)['hex']) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks() origin_rpc.gettransaction(txid) restore_rpc.gettransaction(txid) diff --git a/test/functional/wallet_import_rescan.py b/test/functional/wallet_import_rescan.py index 59089456e9..cbe3e9bfdd 100755 --- a/test/functional/wallet_import_rescan.py +++ b/test/functional/wallet_import_rescan.py @@ -178,7 +178,7 @@ class ImportRescanTest(BitcoinTestFramework): variant.key = self.nodes[1].dumpprivkey(variant.address["address"]) variant.initial_amount = get_rand_amount() variant.initial_txid = self.nodes[0].sendtoaddress(variant.address["address"], variant.initial_amount) - self.nodes[0].generate(1) # Generate one block for each send + self.generate(self.nodes[0], 1) # Generate one block for each send variant.confirmation_height = self.nodes[0].getblockcount() variant.timestamp = self.nodes[0].getblockheader(self.nodes[0].getbestblockhash())["time"] self.sync_all() # Conclude sync before calling setmocktime to avoid timeouts @@ -189,7 +189,7 @@ class ImportRescanTest(BitcoinTestFramework): self.nodes, self.nodes[0].getblockheader(self.nodes[0].getbestblockhash())["time"] + TIMESTAMP_WINDOW + 1, ) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() # For each variation of wallet key import, invoke the import RPC and @@ -212,7 +212,7 @@ class ImportRescanTest(BitcoinTestFramework): for i, variant in enumerate(IMPORT_VARIANTS): variant.sent_amount = get_rand_amount() variant.sent_txid = self.nodes[0].sendtoaddress(variant.address["address"], variant.sent_amount) - self.nodes[0].generate(1) # Generate one block for each send + self.generate(self.nodes[0], 1) # Generate one block for each send variant.confirmation_height = self.nodes[0].getblockcount() assert_equal(self.nodes[0].getrawmempool(), []) diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index a2da16e5a3..d86c3737fe 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -74,12 +74,11 @@ class ImportDescriptorsTest(BitcoinTestFramework): assert_equal(wpriv.getwalletinfo()['keypoolsize'], 0) self.log.info('Mining coins') - w0.generatetoaddress(COINBASE_MATURITY + 1, w0.getnewaddress()) + self.generatetoaddress(w0, COINBASE_MATURITY + 1, w0.getnewaddress()) # RPC importdescriptors ----------------------------------------------- # # Test import fails if no descriptor present - key = get_generate_key() self.log.info("Import should fail if a descriptor is not provided") self.test_importdesc({"timestamp": "now"}, success=False, @@ -89,10 +88,10 @@ class ImportDescriptorsTest(BitcoinTestFramework): # # Test importing of a P2PKH descriptor key = get_generate_key() self.log.info("Should import a p2pkh descriptor") - self.test_importdesc({"desc": descsum_create("pkh(" + key.pubkey + ")"), - "timestamp": "now", - "label": "Descriptor import test"}, - success=True) + import_request = {"desc": descsum_create("pkh(" + key.pubkey + ")"), + "timestamp": "now", + "label": "Descriptor import test"} + self.test_importdesc(import_request, success=True) test_address(w1, key.p2pkh_addr, solvable=True, @@ -100,11 +99,15 @@ class ImportDescriptorsTest(BitcoinTestFramework): labels=["Descriptor import test"]) assert_equal(w1.getwalletinfo()['keypoolsize'], 0) + self.log.info("Test can import same descriptor with public key twice") + self.test_importdesc(import_request, success=True) + + self.log.info("Test can update descriptor label") + self.test_importdesc({**import_request, "label": "Updated label"}, success=True) + test_address(w1, key.p2pkh_addr, solvable=True, ismine=True, labels=["Updated label"]) + self.log.info("Internal addresses cannot have labels") - self.test_importdesc({"desc": descsum_create("pkh(" + key.pubkey + ")"), - "timestamp": "now", - "internal": True, - "label": "Descriptor import test"}, + self.test_importdesc({**import_request, "internal": True}, success=False, error_code=-8, error_message="Internal addresses should not have a label") @@ -252,6 +255,39 @@ class ImportDescriptorsTest(BitcoinTestFramework): self.test_importdesc({"desc": descsum_create(desc), "timestamp": "now", "range": [0, 1000001]}, success=False, error_code=-8, error_message='Range is too large') + self.log.info("Verify we can only extend descriptor's range") + range_request = {"desc": descsum_create(desc), "timestamp": "now", "range": [5, 10], 'active': True} + self.test_importdesc(range_request, wallet=wpriv, success=True) + assert_equal(wpriv.getwalletinfo()['keypoolsize'], 6) + self.test_importdesc({**range_request, "range": [0, 10]}, wallet=wpriv, success=True) + assert_equal(wpriv.getwalletinfo()['keypoolsize'], 11) + self.test_importdesc({**range_request, "range": [0, 20]}, wallet=wpriv, success=True) + assert_equal(wpriv.getwalletinfo()['keypoolsize'], 21) + # Can keep range the same + self.test_importdesc({**range_request, "range": [0, 20]}, wallet=wpriv, success=True) + assert_equal(wpriv.getwalletinfo()['keypoolsize'], 21) + + self.test_importdesc({**range_request, "range": [5, 10]}, wallet=wpriv, success=False, + error_code=-8, error_message='new range must include current range = [0,20]') + self.test_importdesc({**range_request, "range": [0, 10]}, wallet=wpriv, success=False, + error_code=-8, error_message='new range must include current range = [0,20]') + self.test_importdesc({**range_request, "range": [5, 20]}, wallet=wpriv, success=False, + error_code=-8, error_message='new range must include current range = [0,20]') + assert_equal(wpriv.getwalletinfo()['keypoolsize'], 21) + + self.log.info("Check we can change descriptor internal flag") + self.test_importdesc({**range_request, "range": [0, 20], "internal": True}, wallet=wpriv, success=True) + assert_equal(wpriv.getwalletinfo()['keypoolsize'], 0) + assert_raises_rpc_error(-4, 'This wallet has no available keys', wpriv.getnewaddress, '', 'p2sh-segwit') + assert_equal(wpriv.getwalletinfo()['keypoolsize_hd_internal'], 21) + wpriv.getrawchangeaddress('p2sh-segwit') + + self.test_importdesc({**range_request, "range": [0, 20], "internal": False}, wallet=wpriv, success=True) + assert_equal(wpriv.getwalletinfo()['keypoolsize'], 21) + wpriv.getnewaddress('', 'p2sh-segwit') + assert_equal(wpriv.getwalletinfo()['keypoolsize_hd_internal'], 0) + assert_raises_rpc_error(-4, 'This wallet has no available keys', wpriv.getrawchangeaddress, 'p2sh-segwit') + # Make sure ranged imports import keys in order w1 = self.nodes[1].get_wallet_rpc('w1') self.log.info('Key ranges should be imported in order') @@ -303,9 +339,21 @@ class ImportDescriptorsTest(BitcoinTestFramework): w1.keypoolrefill() assert_equal(w1.getwalletinfo()['keypoolsize'], 5 * 3) + self.log.info("Check we can change next_index") + # go back and forth with next_index + for i in [4, 0, 2, 1, 3]: + self.test_importdesc({'desc': descsum_create('wpkh([80002067/0h/0h]' + xpub + '/*)'), + 'active': True, + 'range': [0, 9], + 'next_index': i, + 'timestamp': 'now' + }, + success=True) + assert_equal(w1.getnewaddress('', 'bech32'), addresses[i]) + # Check active=False default self.log.info('Check imported descriptors are not active by default') - self.test_importdesc({'desc': descsum_create('pkh([12345678/0h/0h]' + xpub + '/*)'), + self.test_importdesc({'desc': descsum_create('pkh([12345678/1h]' + xpub + '/*)'), 'range' : [0, 2], 'timestamp': 'now', 'internal': True @@ -313,6 +361,32 @@ class ImportDescriptorsTest(BitcoinTestFramework): success=True) assert_raises_rpc_error(-4, 'This wallet has no available keys', w1.getrawchangeaddress, 'legacy') + self.log.info('Check can activate inactive descriptor') + self.test_importdesc({'desc': descsum_create('pkh([12345678]' + xpub + '/*)'), + 'range': [0, 5], + 'active': True, + 'timestamp': 'now', + 'internal': True + }, + success=True) + address = w1.getrawchangeaddress('legacy') + assert_equal(address, "mpA2Wh9dvZT7yfELq1UnrUmAoc5qCkMetg") + + self.log.info('Check can deactivate active descriptor') + self.test_importdesc({'desc': descsum_create('pkh([12345678]' + xpub + '/*)'), + 'range': [0, 5], + 'active': False, + 'timestamp': 'now', + 'internal': True + }, + success=True) + assert_raises_rpc_error(-4, 'This wallet has no available keys', w1.getrawchangeaddress, 'legacy') + + self.log.info('Verify activation state is persistent') + w1.unloadwallet() + self.nodes[1].loadwallet('w1') + assert_raises_rpc_error(-4, 'This wallet has no available keys', w1.getrawchangeaddress, 'legacy') + # # Test importing a descriptor containing a WIF private key wif_priv = "cTe1f5rdT8A8DFgVWTjyPwACsDPJM9ff4QngFxUixCSvvbg1x6sh" address = "2MuhcG52uHPknxDgmGPsV18jSHFBnnRgjPg" @@ -322,12 +396,16 @@ class ImportDescriptorsTest(BitcoinTestFramework): "timestamp": "now"}, success=True, wallet=wpriv) + + self.log.info('Test can import same descriptor with private key twice') + self.test_importdesc({"desc": descsum_create(desc), "timestamp": "now"}, success=True, wallet=wpriv) + test_address(wpriv, address, solvable=True, ismine=True) txid = w0.sendtoaddress(address, 49.99995540) - w0.generatetoaddress(6, w0.getnewaddress()) + self.generatetoaddress(w0, 6, w0.getnewaddress()) self.sync_blocks() tx = wpriv.createrawtransaction([{"txid": txid, "vout": 0}], {w0.getnewaddress(): 49.999}) signed_tx = wpriv.signrawtransactionwithwallet(tx) @@ -339,14 +417,25 @@ class ImportDescriptorsTest(BitcoinTestFramework): wmulti_priv = self.nodes[1].get_wallet_rpc("wmulti_priv") assert_equal(wmulti_priv.getwalletinfo()['keypoolsize'], 0) - self.test_importdesc({"desc":"wsh(multi(2,tprv8ZgxMBicQKsPevADjDCWsa6DfhkVXicu8NQUzfibwX2MexVwW4tCec5mXdCW8kJwkzBRRmAay1KZya4WsehVvjTGVW6JLqiqd8DdZ4xSg52/84h/0h/0h/*,tprv8ZgxMBicQKsPdSNWUhDiwTScDr6JfkZuLshTRwzvZGnMSnGikV6jxpmdDkC3YRc4T3GD6Nvg9uv6hQg73RVv1EiTXDZwxVbsLugVHU8B1aq/84h/0h/0h/*,tprv8ZgxMBicQKsPeonDt8Ka2mrQmHa61hQ5FQCsvWBTpSNzBFgM58cV2EuXNAHF14VawVpznnme3SuTbA62sGriwWyKifJmXntfNeK7zeqMCj1/84h/0h/0h/*))#m2sr93jn", + xprv1 = 'tprv8ZgxMBicQKsPevADjDCWsa6DfhkVXicu8NQUzfibwX2MexVwW4tCec5mXdCW8kJwkzBRRmAay1KZya4WsehVvjTGVW6JLqiqd8DdZ4xSg52' + acc_xpub1 = 'tpubDCJtdt5dgJpdhW4MtaVYDhG4T4tF6jcLR1PxL43q9pq1mxvXgMS9Mzw1HnXG15vxUGQJMMSqCQHMTy3F1eW5VkgVroWzchsPD5BUojrcWs8' # /84'/0'/0' + chg_xpub1 = 'tpubDCXqdwWZcszwqYJSnZp8eARkxGJfHAk23KDxbztV4BbschfaTfYLTcSkSJ3TN64dRqwa1rnFUScsYormKkGqNbbPwkorQimVevXjxzUV9Gf' # /84'/1'/0' + xprv2 = 'tprv8ZgxMBicQKsPdSNWUhDiwTScDr6JfkZuLshTRwzvZGnMSnGikV6jxpmdDkC3YRc4T3GD6Nvg9uv6hQg73RVv1EiTXDZwxVbsLugVHU8B1aq' + acc_xprv2 = 'tprv8gVCsmRAxVSxyUpsL13Y7ZEWBFPWbgS5E2MmFVNGuANrknvmmn2vWnmHvU8AwEFYzR2ji6EeZLSCLVacsYkvor3Pcb5JY5FGcevqTwYvdYx' + acc_xpub2 = 'tpubDDBF2BTR6s8drwrfDei8WxtckGuSm1cyoKxYY1QaKSBFbHBYQArWhHPA6eJrzZej6nfHGLSURYSLHr7GuYch8aY5n61tGqgn8b4cXrMuoPH' + chg_xpub2 = 'tpubDCYfZY2ceyHzYzMMVPt9MNeiqtQ2T7Uyp9QSFwYXh8Vi9iJFYXcuphJaGXfF3jUQJi5Y3GMNXvM11gaL4txzZgNGK22BFAwMXynnzv4z2Jh' + xprv3 = 'tprv8ZgxMBicQKsPeonDt8Ka2mrQmHa61hQ5FQCsvWBTpSNzBFgM58cV2EuXNAHF14VawVpznnme3SuTbA62sGriwWyKifJmXntfNeK7zeqMCj1' + acc_xpub3 = 'tpubDCsWoW1kuQB9kG5MXewHqkbjPtqPueRnXju7uM2NK7y3JYb2ajAZ9EiuZXNNuE4661RAfriBWhL8UsnAPpk8zrKKnZw1Ug7X4oHgMdZiU4E' + chg_xpub3 = 'tpubDC6UGqnsQStngYuGD4MKsMy7eD1Yg9NTJfPdvjdG2JE5oZ7EsSL3WHg4Gsw2pR5K39ZwJ46M1wZayhedVdQtMGaUhq5S23PH6fnENK3V1sb' + + self.test_importdesc({"desc":"wsh(multi(2," + xprv1 + "/84h/0h/0h/*," + xprv2 + "/84h/0h/0h/*," + xprv3 + "/84h/0h/0h/*))#m2sr93jn", "active": True, "range": 1000, "next_index": 0, "timestamp": "now"}, success=True, wallet=wmulti_priv) - self.test_importdesc({"desc":"wsh(multi(2,tprv8ZgxMBicQKsPevADjDCWsa6DfhkVXicu8NQUzfibwX2MexVwW4tCec5mXdCW8kJwkzBRRmAay1KZya4WsehVvjTGVW6JLqiqd8DdZ4xSg52/84h/1h/0h/*,tprv8ZgxMBicQKsPdSNWUhDiwTScDr6JfkZuLshTRwzvZGnMSnGikV6jxpmdDkC3YRc4T3GD6Nvg9uv6hQg73RVv1EiTXDZwxVbsLugVHU8B1aq/84h/1h/0h/*,tprv8ZgxMBicQKsPeonDt8Ka2mrQmHa61hQ5FQCsvWBTpSNzBFgM58cV2EuXNAHF14VawVpznnme3SuTbA62sGriwWyKifJmXntfNeK7zeqMCj1/84h/1h/0h/*))#q3sztvx5", + self.test_importdesc({"desc":"wsh(multi(2," + xprv1 + "/84h/1h/0h/*," + xprv2 + "/84h/1h/0h/*," + xprv3 + "/84h/1h/0h/*))#q3sztvx5", "active": True, "internal" : True, "range": 1000, @@ -362,26 +451,26 @@ class ImportDescriptorsTest(BitcoinTestFramework): assert_equal(change_addr, 'bcrt1qt9uhe3a9hnq7vajl7a094z4s3crm9ttf8zw3f5v9gr2nyd7e3lnsy44n8e') assert_equal(wmulti_priv.getwalletinfo()['keypoolsize'], 1000) txid = w0.sendtoaddress(addr, 10) - self.nodes[0].generate(6) + self.generate(self.nodes[0], 6) self.sync_all() send_txid = wmulti_priv.sendtoaddress(w0.getnewaddress(), 8) decoded = wmulti_priv.decoderawtransaction(wmulti_priv.gettransaction(send_txid)['hex']) assert_equal(len(decoded['vin'][0]['txinwitness']), 4) - self.nodes[0].generate(6) + self.generate(self.nodes[0], 6) self.sync_all() self.nodes[1].createwallet(wallet_name="wmulti_pub", disable_private_keys=True, blank=True, descriptors=True) wmulti_pub = self.nodes[1].get_wallet_rpc("wmulti_pub") assert_equal(wmulti_pub.getwalletinfo()['keypoolsize'], 0) - self.test_importdesc({"desc":"wsh(multi(2,[7b2d0242/84h/0h/0h]tpubDCJtdt5dgJpdhW4MtaVYDhG4T4tF6jcLR1PxL43q9pq1mxvXgMS9Mzw1HnXG15vxUGQJMMSqCQHMTy3F1eW5VkgVroWzchsPD5BUojrcWs8/*,[59b09cd6/84h/0h/0h]tpubDDBF2BTR6s8drwrfDei8WxtckGuSm1cyoKxYY1QaKSBFbHBYQArWhHPA6eJrzZej6nfHGLSURYSLHr7GuYch8aY5n61tGqgn8b4cXrMuoPH/*,[e81a0532/84h/0h/0h]tpubDCsWoW1kuQB9kG5MXewHqkbjPtqPueRnXju7uM2NK7y3JYb2ajAZ9EiuZXNNuE4661RAfriBWhL8UsnAPpk8zrKKnZw1Ug7X4oHgMdZiU4E/*))#tsry0s5e", + self.test_importdesc({"desc":"wsh(multi(2,[7b2d0242/84h/0h/0h]" + acc_xpub1 + "/*,[59b09cd6/84h/0h/0h]" + acc_xpub2 + "/*,[e81a0532/84h/0h/0h]" + acc_xpub3 +"/*))#tsry0s5e", "active": True, "range": 1000, "next_index": 0, "timestamp": "now"}, success=True, wallet=wmulti_pub) - self.test_importdesc({"desc":"wsh(multi(2,[7b2d0242/84h/1h/0h]tpubDCXqdwWZcszwqYJSnZp8eARkxGJfHAk23KDxbztV4BbschfaTfYLTcSkSJ3TN64dRqwa1rnFUScsYormKkGqNbbPwkorQimVevXjxzUV9Gf/*,[59b09cd6/84h/1h/0h]tpubDCYfZY2ceyHzYzMMVPt9MNeiqtQ2T7Uyp9QSFwYXh8Vi9iJFYXcuphJaGXfF3jUQJi5Y3GMNXvM11gaL4txzZgNGK22BFAwMXynnzv4z2Jh/*,[e81a0532/84h/1h/0h]tpubDC6UGqnsQStngYuGD4MKsMy7eD1Yg9NTJfPdvjdG2JE5oZ7EsSL3WHg4Gsw2pR5K39ZwJ46M1wZayhedVdQtMGaUhq5S23PH6fnENK3V1sb/*))#c08a2rzv", + self.test_importdesc({"desc":"wsh(multi(2,[7b2d0242/84h/1h/0h]" + chg_xpub1 + "/*,[59b09cd6/84h/1h/0h]" + chg_xpub2 + "/*,[e81a0532/84h/1h/0h]" + chg_xpub3 + "/*))#c08a2rzv", "active": True, "internal" : True, "range": 1000, @@ -396,9 +485,16 @@ class ImportDescriptorsTest(BitcoinTestFramework): change_addr = wmulti_pub.getrawchangeaddress('bech32') assert_equal(change_addr, 'bcrt1qt9uhe3a9hnq7vajl7a094z4s3crm9ttf8zw3f5v9gr2nyd7e3lnsy44n8e') assert_equal(wmulti_pub.getwalletinfo()['keypoolsize'], 999) + + # generate some utxos for next tests txid = w0.sendtoaddress(addr, 10) vout = find_vout_for_address(self.nodes[0], txid, addr) - self.nodes[0].generate(6) + + addr2 = wmulti_pub.getnewaddress('', 'bech32') + txid2 = w0.sendtoaddress(addr2, 10) + vout2 = find_vout_for_address(self.nodes[0], txid2, addr2) + + self.generate(self.nodes[0], 6) self.sync_all() assert_equal(wmulti_pub.getbalance(), wmulti_priv.getbalance()) @@ -411,14 +507,14 @@ class ImportDescriptorsTest(BitcoinTestFramework): wmulti_priv1 = self.nodes[1].get_wallet_rpc("wmulti_priv1") res = wmulti_priv1.importdescriptors([ { - "desc": descsum_create("wsh(multi(2,tprv8ZgxMBicQKsPevADjDCWsa6DfhkVXicu8NQUzfibwX2MexVwW4tCec5mXdCW8kJwkzBRRmAay1KZya4WsehVvjTGVW6JLqiqd8DdZ4xSg52/84h/0h/0h/*,[59b09cd6/84h/0h/0h]tpubDDBF2BTR6s8drwrfDei8WxtckGuSm1cyoKxYY1QaKSBFbHBYQArWhHPA6eJrzZej6nfHGLSURYSLHr7GuYch8aY5n61tGqgn8b4cXrMuoPH/*,[e81a0532/84h/0h/0h]tpubDCsWoW1kuQB9kG5MXewHqkbjPtqPueRnXju7uM2NK7y3JYb2ajAZ9EiuZXNNuE4661RAfriBWhL8UsnAPpk8zrKKnZw1Ug7X4oHgMdZiU4E/*))"), + "desc": descsum_create("wsh(multi(2," + xprv1 + "/84h/0h/0h/*,[59b09cd6/84h/0h/0h]" + acc_xpub2 + "/*,[e81a0532/84h/0h/0h]" + acc_xpub3 + "/*))"), "active": True, "range": 1000, "next_index": 0, "timestamp": "now" }, { - "desc": descsum_create("wsh(multi(2,tprv8ZgxMBicQKsPevADjDCWsa6DfhkVXicu8NQUzfibwX2MexVwW4tCec5mXdCW8kJwkzBRRmAay1KZya4WsehVvjTGVW6JLqiqd8DdZ4xSg52/84h/1h/0h/*,[59b09cd6/84h/1h/0h]tpubDCYfZY2ceyHzYzMMVPt9MNeiqtQ2T7Uyp9QSFwYXh8Vi9iJFYXcuphJaGXfF3jUQJi5Y3GMNXvM11gaL4txzZgNGK22BFAwMXynnzv4z2Jh/*,[e81a0532/84h/1h/0h]tpubDC6UGqnsQStngYuGD4MKsMy7eD1Yg9NTJfPdvjdG2JE5oZ7EsSL3WHg4Gsw2pR5K39ZwJ46M1wZayhedVdQtMGaUhq5S23PH6fnENK3V1sb/*))"), + "desc": descsum_create("wsh(multi(2," + xprv1 + "/84h/1h/0h/*,[59b09cd6/84h/1h/0h]" + chg_xpub2 + "/*,[e81a0532/84h/1h/0h]" + chg_xpub3 + "/*))"), "active": True, "internal" : True, "range": 1000, @@ -434,14 +530,14 @@ class ImportDescriptorsTest(BitcoinTestFramework): wmulti_priv2 = self.nodes[1].get_wallet_rpc('wmulti_priv2') res = wmulti_priv2.importdescriptors([ { - "desc": descsum_create("wsh(multi(2,[7b2d0242/84h/0h/0h]tpubDCJtdt5dgJpdhW4MtaVYDhG4T4tF6jcLR1PxL43q9pq1mxvXgMS9Mzw1HnXG15vxUGQJMMSqCQHMTy3F1eW5VkgVroWzchsPD5BUojrcWs8/*,tprv8ZgxMBicQKsPdSNWUhDiwTScDr6JfkZuLshTRwzvZGnMSnGikV6jxpmdDkC3YRc4T3GD6Nvg9uv6hQg73RVv1EiTXDZwxVbsLugVHU8B1aq/84h/0h/0h/*,[e81a0532/84h/0h/0h]tpubDCsWoW1kuQB9kG5MXewHqkbjPtqPueRnXju7uM2NK7y3JYb2ajAZ9EiuZXNNuE4661RAfriBWhL8UsnAPpk8zrKKnZw1Ug7X4oHgMdZiU4E/*))"), + "desc": descsum_create("wsh(multi(2,[7b2d0242/84h/0h/0h]" + acc_xpub1 + "/*," + xprv2 + "/84h/0h/0h/*,[e81a0532/84h/0h/0h]" + acc_xpub3 + "/*))"), "active": True, "range": 1000, "next_index": 0, "timestamp": "now" }, { - "desc": descsum_create("wsh(multi(2,[7b2d0242/84h/1h/0h]tpubDCXqdwWZcszwqYJSnZp8eARkxGJfHAk23KDxbztV4BbschfaTfYLTcSkSJ3TN64dRqwa1rnFUScsYormKkGqNbbPwkorQimVevXjxzUV9Gf/*,tprv8ZgxMBicQKsPdSNWUhDiwTScDr6JfkZuLshTRwzvZGnMSnGikV6jxpmdDkC3YRc4T3GD6Nvg9uv6hQg73RVv1EiTXDZwxVbsLugVHU8B1aq/84h/1h/0h/*,[e81a0532/84h/1h/0h]tpubDC6UGqnsQStngYuGD4MKsMy7eD1Yg9NTJfPdvjdG2JE5oZ7EsSL3WHg4Gsw2pR5K39ZwJ46M1wZayhedVdQtMGaUhq5S23PH6fnENK3V1sb/*))"), + "desc": descsum_create("wsh(multi(2,[7b2d0242/84h/1h/0h]" + chg_xpub1 + "/*," + xprv2 + "/84h/1h/0h/*,[e81a0532/84h/1h/0h]" + chg_xpub3 + "/*))"), "active": True, "internal" : True, "range": 1000, @@ -486,7 +582,7 @@ class ImportDescriptorsTest(BitcoinTestFramework): addr = wmulti_priv_big.getnewaddress() w0.sendtoaddress(addr, 10) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() # It is standard and would relay. txid = wmulti_priv_big.sendtoaddress(w0.getnewaddress(), 9.999) @@ -521,7 +617,7 @@ class ImportDescriptorsTest(BitcoinTestFramework): addr = multi_priv_big.getnewaddress("", "legacy") w0.sendtoaddress(addr, 10) - self.nodes[0].generate(6) + self.generate(self.nodes[0], 6) self.sync_all() # It is standard and would relay. txid = multi_priv_big.sendtoaddress(w0.getnewaddress(), 10, "", "", @@ -531,6 +627,33 @@ class ImportDescriptorsTest(BitcoinTestFramework): ) + self.log.info("Amending multisig with new private keys") + self.nodes[1].createwallet(wallet_name="wmulti_priv3", descriptors=True) + wmulti_priv3 = self.nodes[1].get_wallet_rpc("wmulti_priv3") + res = wmulti_priv3.importdescriptors([ + { + "desc": descsum_create("wsh(multi(2," + xprv1 + "/84h/0h/0h/*,[59b09cd6/84h/0h/0h]" + acc_xpub2 + "/*,[e81a0532/84h/0h/0h]" + acc_xpub3 + "/*))"), + "active": True, + "range": 1000, + "next_index": 0, + "timestamp": "now" + }]) + assert_equal(res[0]['success'], True) + res = wmulti_priv3.importdescriptors([ + { + "desc": descsum_create("wsh(multi(2," + xprv1 + "/84h/0h/0h/*,[59b09cd6/84h/0h/0h]" + acc_xprv2 + "/*,[e81a0532/84h/0h/0h]" + acc_xpub3 + "/*))"), + "active": True, + "range": 1000, + "next_index": 0, + "timestamp": "now" + }]) + assert_equal(res[0]['success'], True) + + rawtx = self.nodes[1].createrawtransaction([{'txid': txid2, 'vout': vout2}], {w0.getnewaddress(): 9.999}) + tx = wmulti_priv3.signrawtransactionwithwallet(rawtx) + assert_equal(tx['complete'], True) + self.nodes[1].sendrawtransaction(tx['hex']) + self.log.info("Combo descriptors cannot be active") self.test_importdesc({"desc": descsum_create("combo(tpubDCJtdt5dgJpdhW4MtaVYDhG4T4tF6jcLR1PxL43q9pq1mxvXgMS9Mzw1HnXG15vxUGQJMMSqCQHMTy3F1eW5VkgVroWzchsPD5BUojrcWs8/*)"), "active": True, diff --git a/test/functional/wallet_importmulti.py b/test/functional/wallet_importmulti.py index baeac655df..4e8907bc3d 100755 --- a/test/functional/wallet_importmulti.py +++ b/test/functional/wallet_importmulti.py @@ -62,8 +62,8 @@ class ImportMultiTest(BitcoinTestFramework): def run_test(self): self.log.info("Mining blocks...") - self.nodes[0].generate(1) - self.nodes[1].generate(1) + self.generate(self.nodes[0], 1) + self.generate(self.nodes[1], 1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] self.nodes[1].syncwithvalidationinterfacequeue() # Sync the timestamp to the wallet, so that importmulti works @@ -256,9 +256,9 @@ class ImportMultiTest(BitcoinTestFramework): # P2SH address multisig = get_multisig(self.nodes[0]) - self.nodes[1].generate(COINBASE_MATURITY) + self.generate(self.nodes[1], COINBASE_MATURITY) self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) - self.nodes[1].generate(1) + self.generate(self.nodes[1], 1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] self.nodes[1].syncwithvalidationinterfacequeue() @@ -277,9 +277,9 @@ class ImportMultiTest(BitcoinTestFramework): # P2SH + Redeem script multisig = get_multisig(self.nodes[0]) - self.nodes[1].generate(COINBASE_MATURITY) + self.generate(self.nodes[1], COINBASE_MATURITY) self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) - self.nodes[1].generate(1) + self.generate(self.nodes[1], 1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] self.nodes[1].syncwithvalidationinterfacequeue() @@ -298,9 +298,9 @@ class ImportMultiTest(BitcoinTestFramework): # P2SH + Redeem script + Private Keys + !Watchonly multisig = get_multisig(self.nodes[0]) - self.nodes[1].generate(COINBASE_MATURITY) + self.generate(self.nodes[1], COINBASE_MATURITY) self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) - self.nodes[1].generate(1) + self.generate(self.nodes[1], 1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] self.nodes[1].syncwithvalidationinterfacequeue() @@ -324,9 +324,9 @@ class ImportMultiTest(BitcoinTestFramework): # P2SH + Redeem script + Private Keys + Watchonly multisig = get_multisig(self.nodes[0]) - self.nodes[1].generate(COINBASE_MATURITY) + self.generate(self.nodes[1], COINBASE_MATURITY) self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) - self.nodes[1].generate(1) + self.generate(self.nodes[1], 1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] self.nodes[1].syncwithvalidationinterfacequeue() diff --git a/test/functional/wallet_importprunedfunds.py b/test/functional/wallet_importprunedfunds.py index ded0e64b1d..74c5100f40 100755 --- a/test/functional/wallet_importprunedfunds.py +++ b/test/functional/wallet_importprunedfunds.py @@ -25,7 +25,7 @@ class ImportPrunedFundsTest(BitcoinTestFramework): def run_test(self): self.log.info("Mining blocks...") - self.nodes[0].generate(COINBASE_MATURITY + 1) + self.generate(self.nodes[0], COINBASE_MATURITY + 1) self.sync_all() @@ -64,17 +64,17 @@ class ImportPrunedFundsTest(BitcoinTestFramework): # Send funds to self txnid1 = self.nodes[0].sendtoaddress(address1, 0.1) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) rawtxn1 = self.nodes[0].gettransaction(txnid1)['hex'] proof1 = self.nodes[0].gettxoutproof([txnid1]) txnid2 = self.nodes[0].sendtoaddress(address2, 0.05) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) rawtxn2 = self.nodes[0].gettransaction(txnid2)['hex'] proof2 = self.nodes[0].gettxoutproof([txnid2]) txnid3 = self.nodes[0].sendtoaddress(address3, 0.025) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) rawtxn3 = self.nodes[0].gettransaction(txnid3)['hex'] proof3 = self.nodes[0].gettxoutproof([txnid3]) diff --git a/test/functional/wallet_keypool.py b/test/functional/wallet_keypool.py index 28bfc9116f..c714993234 100755 --- a/test/functional/wallet_keypool.py +++ b/test/functional/wallet_keypool.py @@ -156,7 +156,7 @@ class KeyPoolTest(BitcoinTestFramework): w1.walletpassphrase('test', 100) res = w1.sendtoaddress(address=address, amount=0.00010000) - nodes[0].generate(1) + self.generate(nodes[0], 1) destination = addr.pop() # Using a fee rate (10 sat / byte) well above the minimum relay rate diff --git a/test/functional/wallet_keypool_topup.py b/test/functional/wallet_keypool_topup.py index 1ecf08b9ac..f730f82397 100755 --- a/test/functional/wallet_keypool_topup.py +++ b/test/functional/wallet_keypool_topup.py @@ -32,7 +32,7 @@ class KeypoolRestoreTest(BitcoinTestFramework): def run_test(self): wallet_path = os.path.join(self.nodes[1].datadir, self.chain, "wallets", self.default_wallet_name, self.wallet_data_filename) wallet_backup_path = os.path.join(self.nodes[1].datadir, "wallet.bak") - self.nodes[0].generate(COINBASE_MATURITY + 1) + self.generate(self.nodes[0], COINBASE_MATURITY + 1) self.log.info("Make backup of wallet") self.stop_node(1) @@ -63,9 +63,9 @@ class KeypoolRestoreTest(BitcoinTestFramework): self.log.info("Send funds to wallet") self.nodes[0].sendtoaddress(addr_oldpool, 10) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.nodes[0].sendtoaddress(addr_extpool, 5) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks() self.log.info("Restart node with wallet backup") diff --git a/test/functional/wallet_labels.py b/test/functional/wallet_labels.py index a571454acf..150f2b341e 100755 --- a/test/functional/wallet_labels.py +++ b/test/functional/wallet_labels.py @@ -32,8 +32,8 @@ class WalletLabelsTest(BitcoinTestFramework): # Note each time we call generate, all generated coins go into # the same address, so we call twice to get two addresses w/50 each - node.generatetoaddress(nblocks=1, address=node.getnewaddress(label='coinbase')) - node.generatetoaddress(nblocks=COINBASE_MATURITY + 1, address=node.getnewaddress(label='coinbase')) + self.generatetoaddress(node, nblocks=1, address=node.getnewaddress(label='coinbase')) + self.generatetoaddress(node, nblocks=COINBASE_MATURITY + 1, address=node.getnewaddress(label='coinbase')) assert_equal(node.getbalance(), 100) # there should be 2 address groups @@ -65,7 +65,7 @@ class WalletLabelsTest(BitcoinTestFramework): assert_equal(set([a[0] for a in address_groups[0]]), linked_addresses) assert_equal([a[1] for a in address_groups[0]], [0, 0]) - node.generate(1) + self.generate(node, 1) # we want to reset so that the "" label has what's expected. # otherwise we're off by exactly the fee amount as that's mined @@ -89,7 +89,7 @@ class WalletLabelsTest(BitcoinTestFramework): label.verify(node) # Check the amounts received. - node.generate(1) + self.generate(node, 1) for label in labels: assert_equal( node.getreceivedbyaddress(label.addresses[0]), amount_to_send) @@ -98,14 +98,14 @@ class WalletLabelsTest(BitcoinTestFramework): for i, label in enumerate(labels): to_label = labels[(i + 1) % len(labels)] node.sendtoaddress(to_label.addresses[0], amount_to_send) - node.generate(1) + self.generate(node, 1) for label in labels: address = node.getnewaddress(label.name) label.add_receive_address(address) label.verify(node) assert_equal(node.getreceivedbylabel(label.name), 2) label.verify(node) - node.generate(COINBASE_MATURITY + 1) + self.generate(node, COINBASE_MATURITY + 1) # Check that setlabel can assign a label to a new unused address. for label in labels: @@ -125,7 +125,7 @@ class WalletLabelsTest(BitcoinTestFramework): label.add_address(multisig_address) label.purpose[multisig_address] = "send" label.verify(node) - node.generate(COINBASE_MATURITY + 1) + self.generate(node, COINBASE_MATURITY + 1) # Check that setlabel can change the label of an address from a # different label. @@ -152,7 +152,7 @@ class WalletLabelsTest(BitcoinTestFramework): for l in BECH32_VALID: ad = BECH32_VALID[l] wallet_watch_only.importaddress(label=l, rescan=False, address=ad) - node.generatetoaddress(1, ad) + self.generatetoaddress(node, 1, ad) assert_equal(wallet_watch_only.getaddressesbylabel(label=l), {ad: {'purpose': 'receive'}}) assert_equal(wallet_watch_only.getreceivedbylabel(label=l), 0) for l in BECH32_INVALID: diff --git a/test/functional/wallet_listdescriptors.py b/test/functional/wallet_listdescriptors.py index c1444164ce..221f5262d9 100755 --- a/test/functional/wallet_listdescriptors.py +++ b/test/functional/wallet_listdescriptors.py @@ -30,9 +30,10 @@ class ListDescriptorsTest(BitcoinTestFramework): node = self.nodes[0] assert_raises_rpc_error(-18, 'No wallet is loaded.', node.listdescriptors) - self.log.info('Test that the command is not available for legacy wallets.') - node.createwallet(wallet_name='w1', descriptors=False) - assert_raises_rpc_error(-4, 'listdescriptors is not available for non-descriptor wallets', node.listdescriptors) + if self.is_bdb_compiled(): + self.log.info('Test that the command is not available for legacy wallets.') + node.createwallet(wallet_name='w1', descriptors=False) + assert_raises_rpc_error(-4, 'listdescriptors is not available for non-descriptor wallets', node.listdescriptors) self.log.info('Test the command for empty descriptors wallet.') node.createwallet(wallet_name='w2', blank=True, descriptors=True) @@ -71,6 +72,38 @@ class ListDescriptorsTest(BitcoinTestFramework): ], } assert_equal(expected, wallet.listdescriptors()) + assert_equal(expected, wallet.listdescriptors(False)) + + self.log.info('Test list private descriptors') + expected_private = { + 'wallet_name': 'w2', + 'descriptors': [ + {'desc': descsum_create('wpkh(' + xprv + hardened_path + '/0/*)'), + 'timestamp': 1296688602, + 'active': False, + 'range': [0, 0], + 'next': 0}, + ], + } + assert_equal(expected_private, wallet.listdescriptors(True)) + + self.log.info("Test listdescriptors with encrypted wallet") + wallet.encryptwallet("pass") + assert_equal(expected, wallet.listdescriptors()) + + self.log.info('Test list private descriptors with encrypted wallet') + assert_raises_rpc_error(-13, 'Please enter the wallet passphrase with walletpassphrase first.', wallet.listdescriptors, True) + wallet.walletpassphrase(passphrase="pass", timeout=1000000) + assert_equal(expected_private, wallet.listdescriptors(True)) + + self.log.info('Test list private descriptors with watch-only wallet') + node.createwallet(wallet_name='watch-only', descriptors=True, disable_private_keys=True) + watch_only_wallet = node.get_wallet_rpc('watch-only') + watch_only_wallet.importdescriptors([{ + 'desc': descsum_create('wpkh(' + xpub_acc + ')'), + 'timestamp': 1296688602, + }]) + assert_raises_rpc_error(-4, 'Can\'t get descriptor string', watch_only_wallet.listdescriptors, True) self.log.info('Test non-active non-range combo descriptor') node.createwallet(wallet_name='w4', blank=True, descriptors=True) diff --git a/test/functional/wallet_listreceivedby.py b/test/functional/wallet_listreceivedby.py index b0590b149a..975bf9a84b 100755 --- a/test/functional/wallet_listreceivedby.py +++ b/test/functional/wallet_listreceivedby.py @@ -24,7 +24,7 @@ class ReceivedByTest(BitcoinTestFramework): def run_test(self): # Generate block to get out of IBD - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks() # save the number of coinbase reward addresses so far @@ -43,7 +43,7 @@ class ReceivedByTest(BitcoinTestFramework): {}, True) # Bury Tx under 10 block so it will be returned by listreceivedbyaddress - self.nodes[1].generate(10) + self.generate(self.nodes[1], 10) self.sync_all() assert_array_result(self.nodes[1].listreceivedbyaddress(), {"address": addr}, @@ -78,7 +78,7 @@ class ReceivedByTest(BitcoinTestFramework): assert_equal(len(res), 2 + num_cb_reward_addresses) # Right now 2 entries other_addr = self.nodes[1].getnewaddress() txid2 = self.nodes[0].sendtoaddress(other_addr, 0.1) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() # Same test as above should still pass expected = {"address": addr, "label": "", "amount": Decimal("0.1"), "confirmations": 11, "txids": [txid, ]} @@ -115,7 +115,7 @@ class ReceivedByTest(BitcoinTestFramework): assert_equal(balance, Decimal("0.1")) # Bury Tx under 10 block so it will be returned by the default getreceivedbyaddress - self.nodes[1].generate(10) + self.generate(self.nodes[1], 10) self.sync_all() balance = self.nodes[1].getreceivedbyaddress(addr) assert_equal(balance, Decimal("0.1")) @@ -144,7 +144,7 @@ class ReceivedByTest(BitcoinTestFramework): balance = self.nodes[1].getreceivedbylabel(label) assert_equal(balance, balance_by_label) - self.nodes[1].generate(10) + self.generate(self.nodes[1], 10) self.sync_all() # listreceivedbylabel should return updated received list assert_array_result(self.nodes[1].listreceivedbylabel(), diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index 3899971bd7..bd3b29c81c 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -30,7 +30,7 @@ class ListSinceBlockTest(BitcoinTestFramework): # All nodes are in IBD from genesis, so they'll need the miner (node2) to be an outbound connection, or have # only one connection. (See fPreferredDownload in net_processing) self.connect_nodes(1, 2) - self.nodes[2].generate(COINBASE_MATURITY + 1) + self.generate(self.nodes[2], COINBASE_MATURITY + 1) self.sync_all() self.test_no_blockhash() @@ -44,7 +44,7 @@ class ListSinceBlockTest(BitcoinTestFramework): def test_no_blockhash(self): self.log.info("Test no blockhash") txid = self.nodes[2].sendtoaddress(self.nodes[0].getnewaddress(), 1) - blockhash, = self.nodes[2].generate(1) + blockhash, = self.generate(self.nodes[2], 1) blockheight = self.nodes[2].getblockheader(blockhash)['height'] self.sync_all() @@ -86,7 +86,7 @@ class ListSinceBlockTest(BitcoinTestFramework): a -8 invalid parameter error is thrown. ''' self.log.info("Test target_confirmations") - blockhash, = self.nodes[2].generate(1) + blockhash, = self.generate(self.nodes[2], 1) blockheight = self.nodes[2].getblockheader(blockhash)['height'] self.sync_all() @@ -136,8 +136,8 @@ class ListSinceBlockTest(BitcoinTestFramework): senttx = self.nodes[2].sendtoaddress(self.nodes[0].getnewaddress(), 1) # generate on both sides - nodes1_last_blockhash = self.nodes[1].generate(6)[-1] - nodes2_first_blockhash = self.nodes[2].generate(7)[0] + nodes1_last_blockhash = self.generate(self.nodes[1], 6)[-1] + nodes2_first_blockhash = self.generate(self.nodes[2], 7)[0] self.log.debug("nodes[1] last blockhash = {}".format(nodes1_last_blockhash)) self.log.debug("nodes[2] first blockhash = {}".format(nodes2_first_blockhash)) @@ -191,7 +191,7 @@ class ListSinceBlockTest(BitcoinTestFramework): privkey = bytes_to_wif(eckey.get_bytes()) address = key_to_p2wpkh(eckey.get_pubkey().get_bytes()) self.nodes[2].sendtoaddress(address, 10) - self.nodes[2].generate(6) + self.generate(self.nodes[2], 6) self.sync_all() self.nodes[2].importprivkey(privkey) utxos = self.nodes[2].listunspent() @@ -225,8 +225,8 @@ class ListSinceBlockTest(BitcoinTestFramework): self.nodes[2].createrawtransaction(utxo_dicts, recipient_dict2))['hex']) # generate on both sides - lastblockhash = self.nodes[1].generate(3)[2] - self.nodes[2].generate(4) + lastblockhash = self.generate(self.nodes[1], 3)[2] + self.generate(self.nodes[2], 4) self.join_network() @@ -297,7 +297,7 @@ class ListSinceBlockTest(BitcoinTestFramework): txid1 = self.nodes[1].sendrawtransaction(signedtx) # generate bb1-bb2 on right side - self.nodes[2].generate(2) + self.generate(self.nodes[2], 2) # send from nodes[2]; this will end up in bb3 txid2 = self.nodes[2].sendrawtransaction(signedtx) @@ -305,8 +305,8 @@ class ListSinceBlockTest(BitcoinTestFramework): assert_equal(txid1, txid2) # generate on both sides - lastblockhash = self.nodes[1].generate(3)[2] - self.nodes[2].generate(2) + lastblockhash = self.generate(self.nodes[1], 3)[2] + self.generate(self.nodes[2], 2) self.join_network() @@ -365,7 +365,7 @@ class ListSinceBlockTest(BitcoinTestFramework): assert_equal(original_found, True) assert_equal(double_found, True) - lastblockhash = spending_node.generate(1)[0] + lastblockhash = self.generate(spending_node, 1)[0] # check that neither transaction exists block_hash = spending_node.listsinceblock(lastblockhash) diff --git a/test/functional/wallet_listtransactions.py b/test/functional/wallet_listtransactions.py index 8b503f5971..a14bfe345c 100755 --- a/test/functional/wallet_listtransactions.py +++ b/test/functional/wallet_listtransactions.py @@ -18,14 +18,15 @@ from test_framework.util import ( class ListTransactionsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + # This test isn't testing txn relay/timing, so set whitelist on the + # peers for instant txn relay. This speeds up the test run time 2-3x. + self.extra_args = [["-whitelist=noban@127.0.0.1"]] * self.num_nodes def skip_test_if_missing_module(self): self.skip_if_no_wallet() def run_test(self): - self.nodes[0].generate(1) # Get out of IBD - self.sync_all() - # Simple send, 0 to 1: + self.log.info("Test simple send from node0 to node1") txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1) self.sync_all() assert_array_result(self.nodes[0].listtransactions(), @@ -34,8 +35,8 @@ class ListTransactionsTest(BitcoinTestFramework): assert_array_result(self.nodes[1].listtransactions(), {"txid": txid}, {"category": "receive", "amount": Decimal("0.1"), "confirmations": 0}) - # mine a block, confirmations should change: - blockhash = self.nodes[0].generate(1)[0] + self.log.info("Test confirmations change after mining a block") + blockhash = self.generate(self.nodes[0], 1)[0] blockheight = self.nodes[0].getblockheader(blockhash)['height'] self.sync_all() assert_array_result(self.nodes[0].listtransactions(), @@ -45,7 +46,7 @@ class ListTransactionsTest(BitcoinTestFramework): {"txid": txid}, {"category": "receive", "amount": Decimal("0.1"), "confirmations": 1, "blockhash": blockhash, "blockheight": blockheight}) - # send-to-self: + self.log.info("Test send-to-self on node0") txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 0.2) assert_array_result(self.nodes[0].listtransactions(), {"txid": txid, "category": "send"}, @@ -54,7 +55,7 @@ class ListTransactionsTest(BitcoinTestFramework): {"txid": txid, "category": "receive"}, {"amount": Decimal("0.2")}) - # sendmany from node1: twice to self, twice to node2: + self.log.info("Test sendmany from node1: twice to self, twice to node0") send_to = {self.nodes[0].getnewaddress(): 0.11, self.nodes[1].getnewaddress(): 0.22, self.nodes[0].getnewaddress(): 0.33, @@ -88,11 +89,12 @@ class ListTransactionsTest(BitcoinTestFramework): if not self.options.descriptors: # include_watchonly is a legacy wallet feature, so don't test it for descriptor wallets + self.log.info("Test 'include_watchonly' feature (legacy wallet)") pubkey = self.nodes[1].getaddressinfo(self.nodes[1].getnewaddress())['pubkey'] multisig = self.nodes[1].createmultisig(1, [pubkey]) self.nodes[0].importaddress(multisig["redeemScript"], "watchonly", False, True) txid = self.nodes[1].sendtoaddress(multisig["address"], 0.1) - self.nodes[1].generate(1) + self.generate(self.nodes[1], 1) self.sync_all() assert_equal(len(self.nodes[0].listtransactions(label="watchonly", include_watchonly=True)), 1) assert_equal(len(self.nodes[0].listtransactions(dummy="watchonly", include_watchonly=True)), 1) @@ -103,37 +105,38 @@ class ListTransactionsTest(BitcoinTestFramework): self.run_rbf_opt_in_test() - # Check that the opt-in-rbf flag works properly, for sent and received - # transactions. + def run_rbf_opt_in_test(self): - # Check whether a transaction signals opt-in RBF itself + """Test the opt-in-rbf flag for sent and received transactions.""" + def is_opt_in(node, txid): + """Check whether a transaction signals opt-in RBF itself.""" rawtx = node.getrawtransaction(txid, 1) for x in rawtx["vin"]: if x["sequence"] < 0xfffffffe: return True return False - # Find an unconfirmed output matching a certain txid def get_unconfirmed_utxo_entry(node, txid_to_match): + """Find an unconfirmed output matching a certain txid.""" utxo = node.listunspent(0, 0) for i in utxo: if i["txid"] == txid_to_match: return i return None - # 1. Chain a few transactions that don't opt-in. + self.log.info("Test txs w/o opt-in RBF (bip125-replaceable=no)") + # Chain a few transactions that don't opt in. txid_1 = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 1) assert not is_opt_in(self.nodes[0], txid_1) assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_1}, {"bip125-replaceable": "no"}) self.sync_mempools() assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_1}, {"bip125-replaceable": "no"}) - # Tx2 will build off txid_1, still not opting in to RBF. + # Tx2 will build off tx1, still not opting in to RBF. utxo_to_use = get_unconfirmed_utxo_entry(self.nodes[0], txid_1) assert_equal(utxo_to_use["safe"], True) utxo_to_use = get_unconfirmed_utxo_entry(self.nodes[1], txid_1) - utxo_to_use = get_unconfirmed_utxo_entry(self.nodes[1], txid_1) assert_equal(utxo_to_use["safe"], False) # Create tx2 using createrawtransaction @@ -149,6 +152,7 @@ class ListTransactionsTest(BitcoinTestFramework): self.sync_mempools() assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_2}, {"bip125-replaceable": "no"}) + self.log.info("Test txs with opt-in RBF (bip125-replaceable=yes)") # Tx3 will opt-in to RBF utxo_to_use = get_unconfirmed_utxo_entry(self.nodes[0], txid_2) inputs = [{"txid": txid_2, "vout": utxo_to_use["vout"]}] @@ -179,6 +183,7 @@ class ListTransactionsTest(BitcoinTestFramework): self.sync_mempools() assert_array_result(self.nodes[0].listtransactions(), {"txid": txid_4}, {"bip125-replaceable": "yes"}) + self.log.info("Test tx with unknown RBF state (bip125-replaceable=unknown)") # Replace tx3, and check that tx4 becomes unknown tx3_b = tx3_modified tx3_b.vout[0].nValue -= int(Decimal("0.004") * COIN) # bump the fee @@ -191,7 +196,7 @@ class ListTransactionsTest(BitcoinTestFramework): self.sync_mempools() assert_array_result(self.nodes[1].listtransactions(), {"txid": txid_4}, {"bip125-replaceable": "unknown"}) - # Check gettransaction as well: + self.log.info("Test bip125-replaceable status with gettransaction RPC") for n in self.nodes[0:2]: assert_equal(n.gettransaction(txid_1)["bip125-replaceable"], "no") assert_equal(n.gettransaction(txid_2)["bip125-replaceable"], "no") @@ -199,8 +204,8 @@ class ListTransactionsTest(BitcoinTestFramework): assert_equal(n.gettransaction(txid_3b)["bip125-replaceable"], "yes") assert_equal(n.gettransaction(txid_4)["bip125-replaceable"], "unknown") - # After mining a transaction, it's no longer BIP125-replaceable - self.nodes[0].generate(1) + self.log.info("Test mined transactions are no longer bip125-replaceable") + self.generate(self.nodes[0], 1) assert txid_3b not in self.nodes[0].getrawmempool() assert_equal(self.nodes[0].gettransaction(txid_3b)["bip125-replaceable"], "no") assert_equal(self.nodes[0].gettransaction(txid_4)["bip125-replaceable"], "unknown") diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index 00d2c9ffe4..d4768f5043 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -185,7 +185,7 @@ class MultiWalletTest(BitcoinTestFramework): self.nodes[0].createwallet("w5") assert_equal(set(node.listwallets()), {"w4", "w5"}) w5 = wallet("w5") - node.generatetoaddress(nblocks=1, address=w5.getnewaddress()) + self.generatetoaddress(node, nblocks=1, address=w5.getnewaddress()) # now if wallets/ exists again, but the rootdir is specified as the walletdir, w4 and w5 should still be loaded os.rename(wallet_dir2, wallet_dir()) @@ -217,7 +217,7 @@ class MultiWalletTest(BitcoinTestFramework): wallet_bad = wallet("bad") # check wallet names and balances - node.generatetoaddress(nblocks=1, address=wallets[0].getnewaddress()) + self.generatetoaddress(node, nblocks=1, address=wallets[0].getnewaddress()) for wallet_name, wallet in zip(wallet_names, wallets): info = wallet.getwalletinfo() assert_equal(info['immature_balance'], 50 if wallet is wallets[0] else 0) @@ -230,7 +230,7 @@ class MultiWalletTest(BitcoinTestFramework): assert_raises_rpc_error(-19, "Wallet file not specified", node.getwalletinfo) w1, w2, w3, w4, *_ = wallets - node.generatetoaddress(nblocks=COINBASE_MATURITY + 1, address=w1.getnewaddress()) + self.generatetoaddress(node, nblocks=COINBASE_MATURITY + 1, address=w1.getnewaddress()) assert_equal(w1.getbalance(), 100) assert_equal(w2.getbalance(), 0) assert_equal(w3.getbalance(), 0) @@ -239,7 +239,7 @@ class MultiWalletTest(BitcoinTestFramework): w1.sendtoaddress(w2.getnewaddress(), 1) w1.sendtoaddress(w3.getnewaddress(), 2) w1.sendtoaddress(w4.getnewaddress(), 3) - node.generatetoaddress(nblocks=1, address=w1.getnewaddress()) + self.generatetoaddress(node, nblocks=1, address=w1.getnewaddress()) assert_equal(w2.getbalance(), 1) assert_equal(w3.getbalance(), 2) assert_equal(w4.getbalance(), 3) diff --git a/test/functional/wallet_orphanedreward.py b/test/functional/wallet_orphanedreward.py index 097df2cf41..ff1d1bd49b 100755 --- a/test/functional/wallet_orphanedreward.py +++ b/test/functional/wallet_orphanedreward.py @@ -18,19 +18,19 @@ class OrphanedBlockRewardTest(BitcoinTestFramework): def run_test(self): # Generate some blocks and obtain some coins on node 0. We send # some balance to node 1, which will hold it as a single coin. - self.nodes[0].generate(150) + self.generate(self.nodes[0], 150) self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 10) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) # Get a block reward with node 1 and remember the block so we can orphan # it later. self.sync_blocks() - blk = self.nodes[1].generate(1)[0] + blk = self.generate(self.nodes[1], 1)[0] self.sync_blocks() # Let the block reward mature and send coins including both # the existing balance and the block reward. - self.nodes[0].generate(150) + self.generate(self.nodes[0], 150) self.sync_blocks() assert_equal(self.nodes[1].getbalance(), 10 + 25) txid = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 30) @@ -38,7 +38,7 @@ class OrphanedBlockRewardTest(BitcoinTestFramework): # Orphan the block reward and make sure that the original coins # from the wallet can still be spent. self.nodes[0].invalidateblock(blk) - self.nodes[0].generate(152) + self.generate(self.nodes[0], 152) self.sync_blocks() # Without the following abandontransaction call, the coins are # not considered available yet. diff --git a/test/functional/wallet_reorgsrestore.py b/test/functional/wallet_reorgsrestore.py index 9a5866a361..1f452f8337 100755 --- a/test/functional/wallet_reorgsrestore.py +++ b/test/functional/wallet_reorgsrestore.py @@ -32,7 +32,7 @@ class ReorgsRestoreTest(BitcoinTestFramework): def run_test(self): # Send a tx from which to conflict outputs later txid_conflict_from = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks() # Disconnect node1 from others to reorg its chain later @@ -43,7 +43,7 @@ class ReorgsRestoreTest(BitcoinTestFramework): # Send a tx to be unconfirmed later txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) tx = self.nodes[0].gettransaction(txid) - self.nodes[0].generate(4) + self.generate(self.nodes[0], 4) tx_before_reorg = self.nodes[0].gettransaction(txid) assert_equal(tx_before_reorg["confirmations"], 4) @@ -62,9 +62,9 @@ class ReorgsRestoreTest(BitcoinTestFramework): conflicting = self.nodes[0].signrawtransactionwithwallet(self.nodes[0].createrawtransaction(inputs, outputs_2)) conflicted_txid = self.nodes[0].sendrawtransaction(conflicted["hex"]) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) conflicting_txid = self.nodes[2].sendrawtransaction(conflicting["hex"]) - self.nodes[2].generate(9) + self.generate(self.nodes[2], 9) # Reconnect node0 and node2 and check that conflicted_txid is effectively conflicted self.connect_nodes(0, 2) @@ -78,11 +78,11 @@ class ReorgsRestoreTest(BitcoinTestFramework): self.restart_node(0) # The block chain re-orgs and the tx is included in a different block - self.nodes[1].generate(9) + self.generate(self.nodes[1], 9) self.nodes[1].sendrawtransaction(tx["hex"]) - self.nodes[1].generate(1) + self.generate(self.nodes[1], 1) self.nodes[1].sendrawtransaction(conflicted["hex"]) - self.nodes[1].generate(1) + self.generate(self.nodes[1], 1) # Node0 wallet file is loaded on longest sync'ed node1 self.stop_node(1) diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py index d24d1693af..c9daeabeb9 100755 --- a/test/functional/wallet_send.py +++ b/test/functional/wallet_send.py @@ -9,6 +9,7 @@ from itertools import product from test_framework.authproxy import JSONRPCException from test_framework.descriptors import descsum_create +from test_framework.key import ECKey from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -16,6 +17,7 @@ from test_framework.util import ( assert_greater_than, assert_raises_rpc_error, ) +from test_framework.wallet_util import bytes_to_wif class WalletSendTest(BitcoinTestFramework): def set_test_params(self): @@ -35,7 +37,7 @@ class WalletSendTest(BitcoinTestFramework): conf_target=None, estimate_mode=None, fee_rate=None, add_to_wallet=None, psbt=None, inputs=None, add_inputs=None, include_unsafe=None, change_address=None, change_position=None, change_type=None, include_watching=None, locktime=None, lock_unspents=None, replaceable=None, subtract_fee_from_outputs=None, - expect_error=None): + expect_error=None, solving_data=None): assert (amount is None) != (data is None) from_balance_before = from_wallet.getbalances()["mine"]["trusted"] @@ -94,6 +96,8 @@ class WalletSendTest(BitcoinTestFramework): options["replaceable"] = replaceable if subtract_fee_from_outputs is not None: options["subtract_fee_from_outputs"] = subtract_fee_from_outputs + if solving_data is not None: + options["solving_data"] = solving_data if len(options.keys()) == 0: options = None @@ -241,7 +245,7 @@ class WalletSendTest(BitcoinTestFramework): assert_equal(res, [{"success": True}, {"success": True}]) w0.sendtoaddress(a2_receive, 10) # fund w3 - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks() if not self.options.descriptors: @@ -260,7 +264,7 @@ class WalletSendTest(BitcoinTestFramework): assert_equal(res, [{"success": True}]) w0.sendtoaddress(a2_receive, 10) # fund w4 - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks() self.log.info("Send to address...") @@ -435,14 +439,14 @@ class WalletSendTest(BitcoinTestFramework): assert not res[0]["allowed"] assert_equal(res[0]["reject-reason"], "non-final") # It shouldn't be confirmed in the next block - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) assert_equal(self.nodes[0].gettransaction(txid)["confirmations"], 0) # The mempool should allow it now: res = self.nodes[0].testmempoolaccept([hex]) assert res[0]["allowed"] # Don't wait for wallet to add it to the mempool: res = self.nodes[0].sendrawtransaction(hex) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) assert_equal(self.nodes[0].gettransaction(txid)["confirmations"], 1) self.sync_all() @@ -476,6 +480,47 @@ class WalletSendTest(BitcoinTestFramework): res = self.test_send(from_wallet=w5, to_wallet=w0, amount=1, include_unsafe=True) assert res["complete"] + self.log.info("External outputs") + eckey = ECKey() + eckey.generate() + privkey = bytes_to_wif(eckey.get_bytes()) + + self.nodes[1].createwallet("extsend") + ext_wallet = self.nodes[1].get_wallet_rpc("extsend") + self.nodes[1].createwallet("extfund") + ext_fund = self.nodes[1].get_wallet_rpc("extfund") + + # Make a weird but signable script. sh(pkh()) descriptor accomplishes this + desc = descsum_create("sh(pkh({}))".format(privkey)) + if self.options.descriptors: + res = ext_fund.importdescriptors([{"desc": desc, "timestamp": "now"}]) + else: + res = ext_fund.importmulti([{"desc": desc, "timestamp": "now"}]) + assert res[0]["success"] + addr = self.nodes[0].deriveaddresses(desc)[0] + addr_info = ext_fund.getaddressinfo(addr) + + self.nodes[0].sendtoaddress(addr, 10) + self.nodes[0].sendtoaddress(ext_wallet.getnewaddress(), 10) + self.generate(self.nodes[0], 6) + self.sync_all() + ext_utxo = ext_fund.listunspent(addresses=[addr])[0] + + # An external input without solving data should result in an error + self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, expect_error=(-4, "Insufficient funds")) + + # But funding should work when the solving data is provided + res = self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, solving_data={"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}) + signed = ext_wallet.walletprocesspsbt(res["psbt"]) + signed = ext_fund.walletprocesspsbt(res["psbt"]) + assert signed["complete"] + self.nodes[0].finalizepsbt(signed["psbt"]) + + res = self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, solving_data={"descriptors": [desc]}) + signed = ext_wallet.walletprocesspsbt(res["psbt"]) + signed = ext_fund.walletprocesspsbt(res["psbt"]) + assert signed["complete"] + self.nodes[0].finalizepsbt(signed["psbt"]) if __name__ == '__main__': WalletSendTest().main() diff --git a/test/functional/wallet_signer.py b/test/functional/wallet_signer.py index afd4fd3691..c6c1cc8784 100755 --- a/test/functional/wallet_signer.py +++ b/test/functional/wallet_signer.py @@ -27,6 +27,9 @@ class WalletSignerTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + # The experimental syscall sandbox feature (-sandbox) is not compatible with -signer (which + # invokes execve). + self.disable_syscall_sandbox = True self.extra_args = [ [], @@ -108,7 +111,7 @@ class WalletSignerTest(BitcoinTestFramework): self.log.info('Prepare mock PSBT') self.nodes[0].sendtoaddress(address1, 1) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_all() # Load private key into wallet to generate a signed PSBT for the mock diff --git a/test/functional/wallet_signmessagewithaddress.py b/test/functional/wallet_signmessagewithaddress.py new file mode 100755 index 0000000000..bf6f95e3f1 --- /dev/null +++ b/test/functional/wallet_signmessagewithaddress.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test Wallet commands for signing and verifying messages.""" + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_raises_rpc_error, +) + +class SignMessagesWithAddressTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + self.extra_args = [["-addresstype=legacy"]] + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + message = 'This is just a test message' + + self.log.info('test signing with an address with wallet') + address = self.nodes[0].getnewaddress() + signature = self.nodes[0].signmessage(address, message) + assert self.nodes[0].verifymessage(address, signature, message) + + self.log.info('test verifying with another address should not work') + other_address = self.nodes[0].getnewaddress() + other_signature = self.nodes[0].signmessage(other_address, message) + assert not self.nodes[0].verifymessage(other_address, signature, message) + assert not self.nodes[0].verifymessage(address, other_signature, message) + + self.log.info('test parameter validity and error codes') + # signmessage has two required parameters + for num_params in [0, 1, 3, 4, 5]: + param_list = ["dummy"]*num_params + assert_raises_rpc_error(-1, "signmessage", self.nodes[0].signmessage, *param_list) + # invalid key or address provided + assert_raises_rpc_error(-5, "Invalid address", self.nodes[0].signmessage, "invalid_addr", message) + + +if __name__ == '__main__': + SignMessagesWithAddressTest().main() diff --git a/test/functional/wallet_taproot.py b/test/functional/wallet_taproot.py index 9eb204bf37..4f84dbd125 100755 --- a/test/functional/wallet_taproot.py +++ b/test/functional/wallet_taproot.py @@ -272,11 +272,11 @@ class WalletTaprootTest(BitcoinTestFramework): boring_balance = int(self.boring.getbalance() * 100000000) to_amnt = random.randrange(1000000, boring_balance) self.boring.sendtoaddress(address=addr_g, amount=Decimal(to_amnt) / 100000000, subtractfeefromamount=True) - self.nodes[0].generatetoaddress(1, self.boring.getnewaddress()) + self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress()) test_balance = int(self.rpc_online.getbalance() * 100000000) ret_amnt = random.randrange(100000, test_balance) res = self.rpc_online.sendtoaddress(address=self.boring.getnewaddress(), amount=Decimal(ret_amnt) / 100000000, subtractfeefromamount=True) - self.nodes[0].generatetoaddress(1, self.boring.getnewaddress()) + self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress()) assert(self.rpc_online.gettransaction(res)["confirmations"] > 0) def do_test_psbt(self, comment, pattern, privmap, treefn, keys_pay, keys_change): @@ -303,7 +303,7 @@ class WalletTaprootTest(BitcoinTestFramework): boring_balance = int(self.boring.getbalance() * 100000000) to_amnt = random.randrange(1000000, boring_balance) self.boring.sendtoaddress(address=addr_g, amount=Decimal(to_amnt) / 100000000, subtractfeefromamount=True) - self.nodes[0].generatetoaddress(1, self.boring.getnewaddress()) + self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress()) test_balance = int(self.psbt_online.getbalance() * 100000000) ret_amnt = random.randrange(100000, test_balance) psbt = self.psbt_online.walletcreatefundedpsbt([], [{self.boring.getnewaddress(): Decimal(ret_amnt) / 100000000}], None, {"subtractFeeFromOutputs":[0]})['psbt'] @@ -311,7 +311,7 @@ class WalletTaprootTest(BitcoinTestFramework): assert(res['complete']) rawtx = self.nodes[0].finalizepsbt(res['psbt'])['hex'] txid = self.nodes[0].sendrawtransaction(rawtx) - self.nodes[0].generatetoaddress(1, self.boring.getnewaddress()) + self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress()) assert(self.psbt_online.gettransaction(txid)['confirmations'] > 0) def do_test(self, comment, pattern, privmap, treefn, nkeys): @@ -343,7 +343,7 @@ class WalletTaprootTest(BitcoinTestFramework): self.log.info("Mining blocks...") gen_addr = self.boring.getnewaddress() - self.nodes[0].generatetoaddress(101, gen_addr) + self.generatetoaddress(self.nodes[0], 101, gen_addr) self.do_test( "tr(XPRV)", @@ -412,7 +412,7 @@ class WalletTaprootTest(BitcoinTestFramework): self.log.info("Sending everything back...") txid = self.rpc_online.sendtoaddress(address=self.boring.getnewaddress(), amount=self.rpc_online.getbalance(), subtractfeefromamount=True) - self.nodes[0].generatetoaddress(1, self.boring.getnewaddress()) + self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress()) assert(self.rpc_online.gettransaction(txid)["confirmations"] > 0) psbt = self.psbt_online.walletcreatefundedpsbt([], [{self.boring.getnewaddress(): self.psbt_online.getbalance()}], None, {"subtractFeeFromOutputs": [0]})['psbt'] @@ -420,7 +420,7 @@ class WalletTaprootTest(BitcoinTestFramework): assert(res['complete']) rawtx = self.nodes[0].finalizepsbt(res['psbt'])['hex'] txid = self.nodes[0].sendrawtransaction(rawtx) - self.nodes[0].generatetoaddress(1, self.boring.getnewaddress()) + self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress()) assert(self.psbt_online.gettransaction(txid)['confirmations'] > 0) if __name__ == '__main__': diff --git a/test/functional/wallet_transactiontime_rescan.py b/test/functional/wallet_transactiontime_rescan.py new file mode 100755 index 0000000000..78859e6131 --- /dev/null +++ b/test/functional/wallet_transactiontime_rescan.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +# Copyright (c) 2018-2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test transaction time during old block rescanning +""" + +import time + +from test_framework.blocktools import COINBASE_MATURITY +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal +) + + +class TransactionTimeRescanTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = False + self.num_nodes = 3 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + self.log.info('Prepare nodes and wallet') + + minernode = self.nodes[0] # node used to mine BTC and create transactions + usernode = self.nodes[1] # user node with correct time + restorenode = self.nodes[2] # node used to restore user wallet and check time determination in ComputeSmartTime (wallet.cpp) + + # time constant + cur_time = int(time.time()) + ten_days = 10 * 24 * 60 * 60 + + # synchronize nodes and time + self.sync_all() + minernode.setmocktime(cur_time) + usernode.setmocktime(cur_time) + restorenode.setmocktime(cur_time) + + # prepare miner wallet + minernode.createwallet(wallet_name='default') + miner_wallet = minernode.get_wallet_rpc('default') + m1 = miner_wallet.getnewaddress() + + # prepare the user wallet with 3 watch only addresses + wo1 = usernode.getnewaddress() + wo2 = usernode.getnewaddress() + wo3 = usernode.getnewaddress() + + usernode.createwallet(wallet_name='wo', disable_private_keys=True) + wo_wallet = usernode.get_wallet_rpc('wo') + + wo_wallet.importaddress(wo1) + wo_wallet.importaddress(wo2) + wo_wallet.importaddress(wo3) + + self.log.info('Start transactions') + + # check blockcount + assert_equal(minernode.getblockcount(), 200) + + # generate some btc to create transactions and check blockcount + initial_mine = COINBASE_MATURITY + 1 + minernode.generatetoaddress(initial_mine, m1) + assert_equal(minernode.getblockcount(), initial_mine + 200) + + # synchronize nodes and time + self.sync_all() + minernode.setmocktime(cur_time + ten_days) + usernode.setmocktime(cur_time + ten_days) + restorenode.setmocktime(cur_time + ten_days) + # send 10 btc to user's first watch-only address + self.log.info('Send 10 btc to user') + miner_wallet.sendtoaddress(wo1, 10) + + # generate blocks and check blockcount + minernode.generatetoaddress(COINBASE_MATURITY, m1) + assert_equal(minernode.getblockcount(), initial_mine + 300) + + # synchronize nodes and time + self.sync_all() + minernode.setmocktime(cur_time + ten_days + ten_days) + usernode.setmocktime(cur_time + ten_days + ten_days) + restorenode.setmocktime(cur_time + ten_days + ten_days) + # send 5 btc to our second watch-only address + self.log.info('Send 5 btc to user') + miner_wallet.sendtoaddress(wo2, 5) + + # generate blocks and check blockcount + minernode.generatetoaddress(COINBASE_MATURITY, m1) + assert_equal(minernode.getblockcount(), initial_mine + 400) + + # synchronize nodes and time + self.sync_all() + minernode.setmocktime(cur_time + ten_days + ten_days + ten_days) + usernode.setmocktime(cur_time + ten_days + ten_days + ten_days) + restorenode.setmocktime(cur_time + ten_days + ten_days + ten_days) + # send 1 btc to our third watch-only address + self.log.info('Send 1 btc to user') + miner_wallet.sendtoaddress(wo3, 1) + + # generate more blocks and check blockcount + minernode.generatetoaddress(COINBASE_MATURITY, m1) + assert_equal(minernode.getblockcount(), initial_mine + 500) + + self.log.info('Check user\'s final balance and transaction count') + assert_equal(wo_wallet.getbalance(), 16) + assert_equal(len(wo_wallet.listtransactions()), 3) + + self.log.info('Check transaction times') + for tx in wo_wallet.listtransactions(): + if tx['address'] == wo1: + assert_equal(tx['blocktime'], cur_time + ten_days) + assert_equal(tx['time'], cur_time + ten_days) + elif tx['address'] == wo2: + assert_equal(tx['blocktime'], cur_time + ten_days + ten_days) + assert_equal(tx['time'], cur_time + ten_days + ten_days) + elif tx['address'] == wo3: + assert_equal(tx['blocktime'], cur_time + ten_days + ten_days + ten_days) + assert_equal(tx['time'], cur_time + ten_days + ten_days + ten_days) + + # restore user wallet without rescan + self.log.info('Restore user wallet on another node without rescan') + restorenode.createwallet(wallet_name='wo', disable_private_keys=True) + restorewo_wallet = restorenode.get_wallet_rpc('wo') + + restorewo_wallet.importaddress(wo1, rescan=False) + restorewo_wallet.importaddress(wo2, rescan=False) + restorewo_wallet.importaddress(wo3, rescan=False) + + # check user has 0 balance and no transactions + assert_equal(restorewo_wallet.getbalance(), 0) + assert_equal(len(restorewo_wallet.listtransactions()), 0) + + # proceed to rescan, first with an incomplete one, then with a full rescan + self.log.info('Rescan last history part') + restorewo_wallet.rescanblockchain(initial_mine + 350) + self.log.info('Rescan all history') + restorewo_wallet.rescanblockchain() + + self.log.info('Check user\'s final balance and transaction count after restoration') + assert_equal(restorewo_wallet.getbalance(), 16) + assert_equal(len(restorewo_wallet.listtransactions()), 3) + + self.log.info('Check transaction times after restoration') + for tx in restorewo_wallet.listtransactions(): + if tx['address'] == wo1: + assert_equal(tx['blocktime'], cur_time + ten_days) + assert_equal(tx['time'], cur_time + ten_days) + elif tx['address'] == wo2: + assert_equal(tx['blocktime'], cur_time + ten_days + ten_days) + assert_equal(tx['time'], cur_time + ten_days + ten_days) + elif tx['address'] == wo3: + assert_equal(tx['blocktime'], cur_time + ten_days + ten_days + ten_days) + assert_equal(tx['time'], cur_time + ten_days + ten_days + ten_days) + + +if __name__ == '__main__': + TransactionTimeRescanTest().main() diff --git a/test/functional/wallet_txn_clone.py b/test/functional/wallet_txn_clone.py index 76b39201e3..7f178d7d46 100755 --- a/test/functional/wallet_txn_clone.py +++ b/test/functional/wallet_txn_clone.py @@ -7,6 +7,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, + find_vout_for_address ) from test_framework.messages import ( COIN, @@ -33,6 +34,13 @@ class TxnMallTest(BitcoinTestFramework): super().setup_network() self.disconnect_nodes(1, 2) + def spend_txid(self, txid, vout, outputs): + inputs = [{"txid": txid, "vout": vout}] + tx = self.nodes[0].createrawtransaction(inputs, outputs) + tx = self.nodes[0].fundrawtransaction(tx) + tx = self.nodes[0].signrawtransactionwithwallet(tx['hex']) + return self.nodes[0].sendrawtransaction(tx['hex']) + def run_test(self): if self.options.segwit: output_type = "p2sh-segwit" @@ -49,6 +57,7 @@ class TxnMallTest(BitcoinTestFramework): node0_address1 = self.nodes[0].getnewaddress(address_type=output_type) node0_txid1 = self.nodes[0].sendtoaddress(node0_address1, 1219) node0_tx1 = self.nodes[0].gettransaction(node0_txid1) + self.nodes[0].lockunspent(False, [{"txid":node0_txid1, "vout": find_vout_for_address(self.nodes[0], node0_txid1, node0_address1)}]) node0_address2 = self.nodes[0].getnewaddress(address_type=output_type) node0_txid2 = self.nodes[0].sendtoaddress(node0_address2, 29) @@ -61,8 +70,8 @@ class TxnMallTest(BitcoinTestFramework): node1_address = self.nodes[1].getnewaddress() # Send tx1, and another transaction tx2 that won't be cloned - txid1 = self.nodes[0].sendtoaddress(node1_address, 40) - txid2 = self.nodes[0].sendtoaddress(node1_address, 20) + txid1 = self.spend_txid(node0_txid1, find_vout_for_address(self.nodes[0], node0_txid1, node0_address1), {node1_address: 40}) + txid2 = self.spend_txid(node0_txid2, find_vout_for_address(self.nodes[0], node0_txid2, node0_address2), {node1_address: 20}) # Construct a clone of tx1, to be malleated rawtx1 = self.nodes[0].getrawtransaction(txid1, 1) @@ -84,7 +93,7 @@ class TxnMallTest(BitcoinTestFramework): # Have node0 mine a block, if requested: if (self.options.mine_block): - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks(self.nodes[0:2]) tx1 = self.nodes[0].gettransaction(txid1) @@ -114,13 +123,13 @@ class TxnMallTest(BitcoinTestFramework): return # ... mine a block... - self.nodes[2].generate(1) + self.generate(self.nodes[2], 1) # Reconnect the split network, and sync chain: self.connect_nodes(1, 2) self.nodes[2].sendrawtransaction(node0_tx2["hex"]) self.nodes[2].sendrawtransaction(tx2["hex"]) - self.nodes[2].generate(1) # Mine another block to make sure we sync + self.generate(self.nodes[2], 1) # Mine another block to make sure we sync self.sync_blocks() # Re-fetch transaction info: diff --git a/test/functional/wallet_txn_doublespend.py b/test/functional/wallet_txn_doublespend.py index 0cb7328948..150e4083b9 100755 --- a/test/functional/wallet_txn_doublespend.py +++ b/test/functional/wallet_txn_doublespend.py @@ -9,6 +9,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, find_output, + find_vout_for_address ) @@ -29,6 +30,13 @@ class TxnMallTest(BitcoinTestFramework): super().setup_network() self.disconnect_nodes(1, 2) + def spend_txid(self, txid, vout, outputs): + inputs = [{"txid": txid, "vout": vout}] + tx = self.nodes[0].createrawtransaction(inputs, outputs) + tx = self.nodes[0].fundrawtransaction(tx) + tx = self.nodes[0].signrawtransactionwithwallet(tx['hex']) + return self.nodes[0].sendrawtransaction(tx['hex']) + def run_test(self): # All nodes should start with 1,250 BTC: starting_balance = 1250 @@ -47,6 +55,7 @@ class TxnMallTest(BitcoinTestFramework): node0_address_foo = self.nodes[0].getnewaddress() fund_foo_txid = self.nodes[0].sendtoaddress(node0_address_foo, 1219) fund_foo_tx = self.nodes[0].gettransaction(fund_foo_txid) + self.nodes[0].lockunspent(False, [{"txid":fund_foo_txid, "vout": find_vout_for_address(self.nodes[0], fund_foo_txid, node0_address_foo)}]) node0_address_bar = self.nodes[0].getnewaddress() fund_bar_txid = self.nodes[0].sendtoaddress(node0_address_bar, 29) @@ -77,12 +86,12 @@ class TxnMallTest(BitcoinTestFramework): assert_equal(doublespend["complete"], True) # Create two spends using 1 50 BTC coin each - txid1 = self.nodes[0].sendtoaddress(node1_address, 40) - txid2 = self.nodes[0].sendtoaddress(node1_address, 20) + txid1 = self.spend_txid(fund_foo_txid, find_vout_for_address(self.nodes[0], fund_foo_txid, node0_address_foo), {node1_address: 40}) + txid2 = self.spend_txid(fund_bar_txid, find_vout_for_address(self.nodes[0], fund_bar_txid, node0_address_bar), {node1_address: 20}) # Have node0 mine a block: if (self.options.mine_block): - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) self.sync_blocks(self.nodes[0:2]) tx1 = self.nodes[0].gettransaction(txid1) @@ -111,11 +120,11 @@ class TxnMallTest(BitcoinTestFramework): self.nodes[2].sendrawtransaction(fund_bar_tx["hex"]) doublespend_txid = self.nodes[2].sendrawtransaction(doublespend["hex"]) # ... mine a block... - self.nodes[2].generate(1) + self.generate(self.nodes[2], 1) # Reconnect the split network, and sync chain: self.connect_nodes(1, 2) - self.nodes[2].generate(1) # Mine another block to make sure we sync + self.generate(self.nodes[2], 1) # Mine another block to make sure we sync self.sync_blocks() assert_equal(self.nodes[0].gettransaction(doublespend_txid)["confirmations"], 2) diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index e7dd7592b6..ed98db55c9 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -6,9 +6,8 @@ Test upgradewallet RPC. Download node binaries: -test/get_previous_releases.py -b v0.19.1 v0.18.1 v0.17.2 v0.16.3 v0.15.2 - -Only v0.15.2 and v0.16.3 are required by this test. The others are used in feature_backwards_compatibility.py +Requires previous releases binaries, see test/README.md. +Only v0.15.2 and v0.16.3 are required by this test. """ import os @@ -95,10 +94,11 @@ class UpgradeWalletTest(BitcoinTestFramework): def test_upgradewallet(self, wallet, previous_version, requested_version=None, expected_version=None): unchanged = expected_version == previous_version new_version = previous_version if unchanged else expected_version if expected_version else requested_version - assert_equal(wallet.getwalletinfo()["walletversion"], previous_version) + old_wallet_info = wallet.getwalletinfo() + assert_equal(old_wallet_info["walletversion"], previous_version) assert_equal(wallet.upgradewallet(requested_version), { - "wallet_name": "", + "wallet_name": old_wallet_info["walletname"], "previous_version": previous_version, "current_version": new_version, "result": "Already at latest version. Wallet version unchanged." if unchanged else "Wallet upgraded successfully from version {} to version {}.".format(previous_version, new_version), @@ -119,7 +119,7 @@ class UpgradeWalletTest(BitcoinTestFramework): assert_equal(wallet.getwalletinfo()["walletversion"], previous_version) def run_test(self): - self.nodes[0].generatetoaddress(COINBASE_MATURITY + 1, self.nodes[0].getnewaddress()) + self.generatetoaddress(self.nodes[0], COINBASE_MATURITY + 1, self.nodes[0].getnewaddress()) self.dumb_sync_blocks() # # Sanity check the test framework: res = self.nodes[0].getblockchaininfo() @@ -131,7 +131,7 @@ class UpgradeWalletTest(BitcoinTestFramework): # Send coins to old wallets for later conversion checks. v16_3_wallet = v16_3_node.get_wallet_rpc('wallet.dat') v16_3_address = v16_3_wallet.getnewaddress() - node_master.generatetoaddress(COINBASE_MATURITY + 1, v16_3_address) + self.generatetoaddress(node_master, COINBASE_MATURITY + 1, v16_3_address) self.dumb_sync_blocks() v16_3_balance = v16_3_wallet.getbalance() @@ -353,6 +353,11 @@ class UpgradeWalletTest(BitcoinTestFramework): v16_3_kvs = dump_bdb_kv(v16_3_wallet) assert b'\x0adefaultkey' not in v16_3_kvs + if self.is_sqlite_compiled(): + self.log.info("Checking that descriptor wallets do nothing, successfully") + self.nodes[0].createwallet(wallet_name="desc_upgrade", descriptors=True) + desc_wallet = self.nodes[0].get_wallet_rpc("desc_upgrade") + self.test_upgradewallet(desc_wallet, previous_version=169900, expected_version=169900) if __name__ == '__main__': UpgradeWalletTest().main() diff --git a/test/functional/wallet_watchonly.py b/test/functional/wallet_watchonly.py index 6743c4a49b..3a9800111b 100755 --- a/test/functional/wallet_watchonly.py +++ b/test/functional/wallet_watchonly.py @@ -37,11 +37,11 @@ class CreateWalletWatchonlyTest(BitcoinTestFramework): wo_wallet.importpubkey(pubkey=def_wallet.getaddressinfo(wo_change)['pubkey']) # generate some btc for testing - node.generatetoaddress(COINBASE_MATURITY + 1, a1) + self.generatetoaddress(node, COINBASE_MATURITY + 1, a1) # send 1 btc to our watch-only address txid = def_wallet.sendtoaddress(wo_addr, 1) - self.nodes[0].generate(1) + self.generate(self.nodes[0], 1) # getbalance self.log.info('include_watchonly should default to true for watch-only wallets') diff --git a/test/get_previous_releases.py b/test/get_previous_releases.py index b177fbb4b2..62fcad04b3 100755 --- a/test/get_previous_releases.py +++ b/test/get_previous_releases.py @@ -21,39 +21,47 @@ import hashlib SHA256_SUMS = { -"d40f18b4e43c6e6370ef7db9131f584fbb137276ec2e3dba67a4b267f81cb644": "bitcoin-0.15.2-aarch64-linux-gnu.tar.gz", -"54fb877a148a6ad189a1e1ab1ff8b11181e58ff2aaf430da55b3fd46ae549a6b": "bitcoin-0.15.2-arm-linux-gnueabihf.tar.gz", -"2b843506c3f1af0eeca5854a920264f9a829f02d0d50328005950ddcbe88874d": "bitcoin-0.15.2-i686-pc-linux-gnu.tar.gz", -"87e9340ff3d382d543b2b69112376077f0c8b4f7450d372e83b68f5a1e22b2df": "bitcoin-0.15.2-osx64.tar.gz", -"566be44190fd76daa01f13d428939dadfb8e3daacefc8fa17f433cad28f73bd5": "bitcoin-0.15.2-x86_64-linux-gnu.tar.gz", - -"0768c6c15caffbaca6524824c9563b42c24f70633c681c2744649158aa3fd484": "bitcoin-0.16.3-aarch64-linux-gnu.tar.gz", -"fb2818069854a6ad20ea03b28b55dbd35d8b1f7d453e90b83eace5d0098a2a87": "bitcoin-0.16.3-arm-linux-gnueabihf.tar.gz", -"75a537844313b0a84bdb61ffcdc5c4ce19a738f7ddf71007cd2edf664efd7c37": "bitcoin-0.16.3-i686-pc-linux-gnu.tar.gz", -"78c3bff3b619a19aed575961ea43cc9e142959218835cf51aede7f0b764fc25d": "bitcoin-0.16.3-osx64.tar.gz", -"5d422a9d544742bc0df12427383f9c2517433ce7b58cf672b9a9b17c2ef51e4f": "bitcoin-0.16.3-x86_64-linux-gnu.tar.gz", - -"5a6b35d1a348a402f2d2d6ab5aed653a1a1f13bc63aaaf51605e3501b0733b7a": "bitcoin-0.17.2-aarch64-linux-gnu.tar.gz", -"d1913a5d19c8e8da4a67d1bd5205d03c8614dfd2e02bba2fe3087476643a729e": "bitcoin-0.17.2-arm-linux-gnueabihf.tar.gz", -"d295fc93f39bbf0fd937b730a93184899a2eb6c3a6d53f3d857cbe77ef89b98c": "bitcoin-0.17.2-i686-pc-linux-gnu.tar.gz", -"a783ba20706dbfd5b47fbedf42165fce70fbbc7d78003305d964f6b3da14887f": "bitcoin-0.17.2-osx64.tar.gz", -"943f9362b9f11130177839116f48f809d83478b4c28591d486ee9a7e35179da6": "bitcoin-0.17.2-x86_64-linux-gnu.tar.gz", - -"88f343af72803b851c7da13874cc5525026b0b55e63e1b5e1298390c4688adc6": "bitcoin-0.18.1-aarch64-linux-gnu.tar.gz", -"cc7d483e4b20c5dabd4dcaf304965214cf4934bcc029ca99cbc9af00d3771a1f": "bitcoin-0.18.1-arm-linux-gnueabihf.tar.gz", -"989e847b3e95fc9fedc0b109cae1b4fa43348f2f712e187a118461876af9bd16": "bitcoin-0.18.1-i686-pc-linux-gnu.tar.gz", -"b7bbcee7a7540f711b171d6981f939ca8482005fde22689bc016596d80548bb1": "bitcoin-0.18.1-osx64.tar.gz", -"425ee5ec631ae8da71ebc1c3f5c0269c627cf459379b9b030f047107a28e3ef8": "bitcoin-0.18.1-riscv64-linux-gnu.tar.gz", -"600d1db5e751fa85903e935a01a74f5cc57e1e7473c15fd3e17ed21e202cfe5a": "bitcoin-0.18.1-x86_64-linux-gnu.tar.gz", - -"3a80431717842672df682bdb619e66523b59541483297772a7969413be3502ff": "bitcoin-0.19.1-aarch64-linux-gnu.tar.gz", -"657f28213823d240dd3324d14829702f9ad6f0710f8bdd1c379cb3c447197f48": "bitcoin-0.19.1-arm-linux-gnueabihf.tar.gz", -"10d1e53208aa7603022f4acc084a046299ab4ccf25fe01e81b3fb6f856772589": "bitcoin-0.19.1-i686-pc-linux-gnu.tar.gz", -"1ae1b87de26487075cd2fd22e0d4ead87d969bd55c44f2f1d873ecdc6147ebb3": "bitcoin-0.19.1-osx64.tar.gz", -"aa7a9563b48aa79252c8e7b6a41c07a5441bd9f14c5e4562cc72720ea6cb0ee5": "bitcoin-0.19.1-riscv64-linux-gnu.tar.gz", -"5fcac9416e486d4960e1a946145566350ca670f9aaba99de6542080851122e4c": "bitcoin-0.19.1-x86_64-linux-gnu.tar.gz" + "d40f18b4e43c6e6370ef7db9131f584fbb137276ec2e3dba67a4b267f81cb644": "bitcoin-0.15.2-aarch64-linux-gnu.tar.gz", + "54fb877a148a6ad189a1e1ab1ff8b11181e58ff2aaf430da55b3fd46ae549a6b": "bitcoin-0.15.2-arm-linux-gnueabihf.tar.gz", + "2b843506c3f1af0eeca5854a920264f9a829f02d0d50328005950ddcbe88874d": "bitcoin-0.15.2-i686-pc-linux-gnu.tar.gz", + "87e9340ff3d382d543b2b69112376077f0c8b4f7450d372e83b68f5a1e22b2df": "bitcoin-0.15.2-osx64.tar.gz", + "566be44190fd76daa01f13d428939dadfb8e3daacefc8fa17f433cad28f73bd5": "bitcoin-0.15.2-x86_64-linux-gnu.tar.gz", + # + "0768c6c15caffbaca6524824c9563b42c24f70633c681c2744649158aa3fd484": "bitcoin-0.16.3-aarch64-linux-gnu.tar.gz", + "fb2818069854a6ad20ea03b28b55dbd35d8b1f7d453e90b83eace5d0098a2a87": "bitcoin-0.16.3-arm-linux-gnueabihf.tar.gz", + "75a537844313b0a84bdb61ffcdc5c4ce19a738f7ddf71007cd2edf664efd7c37": "bitcoin-0.16.3-i686-pc-linux-gnu.tar.gz", + "78c3bff3b619a19aed575961ea43cc9e142959218835cf51aede7f0b764fc25d": "bitcoin-0.16.3-osx64.tar.gz", + "5d422a9d544742bc0df12427383f9c2517433ce7b58cf672b9a9b17c2ef51e4f": "bitcoin-0.16.3-x86_64-linux-gnu.tar.gz", + # + "5a6b35d1a348a402f2d2d6ab5aed653a1a1f13bc63aaaf51605e3501b0733b7a": "bitcoin-0.17.2-aarch64-linux-gnu.tar.gz", + "d1913a5d19c8e8da4a67d1bd5205d03c8614dfd2e02bba2fe3087476643a729e": "bitcoin-0.17.2-arm-linux-gnueabihf.tar.gz", + "d295fc93f39bbf0fd937b730a93184899a2eb6c3a6d53f3d857cbe77ef89b98c": "bitcoin-0.17.2-i686-pc-linux-gnu.tar.gz", + "a783ba20706dbfd5b47fbedf42165fce70fbbc7d78003305d964f6b3da14887f": "bitcoin-0.17.2-osx64.tar.gz", + "943f9362b9f11130177839116f48f809d83478b4c28591d486ee9a7e35179da6": "bitcoin-0.17.2-x86_64-linux-gnu.tar.gz", + # + "88f343af72803b851c7da13874cc5525026b0b55e63e1b5e1298390c4688adc6": "bitcoin-0.18.1-aarch64-linux-gnu.tar.gz", + "cc7d483e4b20c5dabd4dcaf304965214cf4934bcc029ca99cbc9af00d3771a1f": "bitcoin-0.18.1-arm-linux-gnueabihf.tar.gz", + "989e847b3e95fc9fedc0b109cae1b4fa43348f2f712e187a118461876af9bd16": "bitcoin-0.18.1-i686-pc-linux-gnu.tar.gz", + "b7bbcee7a7540f711b171d6981f939ca8482005fde22689bc016596d80548bb1": "bitcoin-0.18.1-osx64.tar.gz", + "425ee5ec631ae8da71ebc1c3f5c0269c627cf459379b9b030f047107a28e3ef8": "bitcoin-0.18.1-riscv64-linux-gnu.tar.gz", + "600d1db5e751fa85903e935a01a74f5cc57e1e7473c15fd3e17ed21e202cfe5a": "bitcoin-0.18.1-x86_64-linux-gnu.tar.gz", + # + "3a80431717842672df682bdb619e66523b59541483297772a7969413be3502ff": "bitcoin-0.19.1-aarch64-linux-gnu.tar.gz", + "657f28213823d240dd3324d14829702f9ad6f0710f8bdd1c379cb3c447197f48": "bitcoin-0.19.1-arm-linux-gnueabihf.tar.gz", + "10d1e53208aa7603022f4acc084a046299ab4ccf25fe01e81b3fb6f856772589": "bitcoin-0.19.1-i686-pc-linux-gnu.tar.gz", + "1ae1b87de26487075cd2fd22e0d4ead87d969bd55c44f2f1d873ecdc6147ebb3": "bitcoin-0.19.1-osx64.tar.gz", + "aa7a9563b48aa79252c8e7b6a41c07a5441bd9f14c5e4562cc72720ea6cb0ee5": "bitcoin-0.19.1-riscv64-linux-gnu.tar.gz", + "5fcac9416e486d4960e1a946145566350ca670f9aaba99de6542080851122e4c": "bitcoin-0.19.1-x86_64-linux-gnu.tar.gz", + # + "60c93e3462c303eb080be7cf623f1a7684b37fd47a018ad3848bc23e13c84e1c": "bitcoin-0.20.1-aarch64-linux-gnu.tar.gz", + "55b577e0fb306fb429d4be6c9316607753e8543e5946b542d75d876a2f08654c": "bitcoin-0.20.1-arm-linux-gnueabihf.tar.gz", + "b9024dde373ea7dad707363e07ec7e265383204127539ae0c234bff3a61da0d1": "bitcoin-0.20.1-osx64.tar.gz", + "c378d4e21109f09e8829f3591e015c66632dff2925a60b64d259be05a334c30b": "bitcoin-0.20.1-osx.dmg", + "fa71cb52ee5e0459cbf5248cdec72df27995840c796f58b304607a1ed4c165af": "bitcoin-0.20.1-riscv64-linux-gnu.tar.gz", + "376194f06596ecfa40331167c39bc70c355f960280bd2a645fdbf18f66527397": "bitcoin-0.20.1-x86_64-linux-gnu.tar.gz", } + @contextlib.contextmanager def pushd(new_dir) -> None: previous_dir = os.getcwd() @@ -104,7 +112,11 @@ def download_binary(tag, args) -> int: tarballHash = hasher.hexdigest() if tarballHash not in SHA256_SUMS or SHA256_SUMS[tarballHash] != tarball: - print("Checksum did not match") + if tarball in SHA256_SUMS.values(): + print("Checksum did not match") + return 1 + + print("Checksum for given version doesn't exist") return 1 print("Checksum matched") @@ -178,6 +190,7 @@ def check_host(args) -> int: 'aarch64-*-linux*': 'aarch64-linux-gnu', 'x86_64-*-linux*': 'x86_64-linux-gnu', 'x86_64-apple-darwin*': 'osx64', + 'aarch64-apple-darwin*': 'osx64', } args.platform = '' for pattern, target in platforms.items(): diff --git a/test/lint/README.md b/test/lint/README.md index 7e06308347..c4d76eac94 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -27,10 +27,10 @@ Usage: test/lint/git-subtree-check.sh [-r] DIR [COMMIT] To do a full check with `-r`, make sure that you have fetched the upstream repository branch in which the subtree is maintained: * for `src/secp256k1`: https://github.com/bitcoin-core/secp256k1.git (branch master) -* for `src/leveldb`: https://github.com/bitcoin-core/leveldb.git (branch bitcoin-fork) -* for `src/univalue`: https://github.com/bitcoin-core/univalue.git (branch master) +* for `src/leveldb`: https://github.com/bitcoin-core/leveldb-subtree.git (branch bitcoin-fork) +* for `src/univalue`: https://github.com/bitcoin-core/univalue-subtree.git (branch master) * for `src/crypto/ctaes`: https://github.com/bitcoin-core/ctaes.git (branch master) -* for `src/crc32c`: https://github.com/google/crc32c.git (branch master) +* for `src/crc32c`: https://github.com/bitcoin-core/crc32c-subtree.git (branch bitcoin-fork) To do so, add the upstream repository as remote: diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh index f8f24bb1ff..8e74f41bb6 100755 --- a/test/lint/lint-circular-dependencies.sh +++ b/test/lint/lint-circular-dependencies.sh @@ -10,12 +10,12 @@ export LC_ALL=C EXPECTED_CIRCULAR_DEPENDENCIES=( "chainparamsbase -> util/system -> chainparamsbase" - "index/txindex -> validation -> index/txindex" "node/blockstorage -> validation -> node/blockstorage" "index/blockfilterindex -> node/blockstorage -> validation -> index/blockfilterindex" "index/base -> validation -> index/blockfilterindex -> index/base" "index/coinstatsindex -> node/coinstats -> index/coinstatsindex" "policy/fees -> txmempool -> policy/fees" + "policy/rbf -> txmempool -> validation -> policy/rbf" "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel" "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel" "qt/sendcoinsdialog -> qt/walletmodel -> qt/sendcoinsdialog" @@ -24,10 +24,6 @@ EXPECTED_CIRCULAR_DEPENDENCIES=( "wallet/fees -> wallet/wallet -> wallet/fees" "wallet/wallet -> wallet/walletdb -> wallet/wallet" "node/coinstats -> validation -> node/coinstats" - # Temporary circular dependencies that allow wallet.h/wallet.cpp to be - # split up in a MOVEONLY commit. These are removed in #21206. - "wallet/receive -> wallet/wallet -> wallet/receive" - "wallet/spend -> wallet/wallet -> wallet/spend" ) EXIT_CODE=0 diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index 737d35a397..712c4f0d5b 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -37,25 +37,17 @@ export LC_ALL=C # See https://doc.qt.io/qt-5/qcoreapplication.html#locale-settings and # https://stackoverflow.com/a/34878283 for more details. +# TODO: Reduce KNOWN_VIOLATIONS by replacing uses of locale dependent stoul/strtol with locale +# independent ToIntegral<T>(...). +# TODO: Reduce KNOWN_VIOLATIONS by replacing uses of locale dependent snprintf with strprintf. KNOWN_VIOLATIONS=( "src/bitcoin-tx.cpp.*stoul" - "src/bitcoin-tx.cpp.*trim_right" - "src/dbwrapper.cpp.*stoul" "src/dbwrapper.cpp:.*vsnprintf" - "src/httprpc.cpp.*trim" - "src/node/blockstorage.cpp:.*atoi" - "src/qt/rpcconsole.cpp:.*atoi" "src/rest.cpp:.*strtol" "src/test/dbwrapper_tests.cpp:.*snprintf" "src/test/fuzz/locale.cpp" - "src/test/fuzz/parse_numbers.cpp:.*atoi" - "src/torcontrol.cpp:.*atoi" + "src/test/fuzz/string.cpp" "src/torcontrol.cpp:.*strtol" - "src/util/strencodings.cpp:.*atoi" - "src/util/strencodings.cpp:.*strtol" - "src/util/strencodings.cpp:.*strtoul" - "src/util/strencodings.h:.*atoi" - "src/util/system.cpp:.*atoi" ) REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(crypto/ctaes/|leveldb/|secp256k1/|tinyformat.h|univalue/)" diff --git a/test/lint/lint-logs.sh b/test/lint/lint-logs.sh index 2fbb4a38e7..d6c53e8ff3 100755 --- a/test/lint/lint-logs.sh +++ b/test/lint/lint-logs.sh @@ -7,7 +7,7 @@ # Check that all logs are terminated with '\n' # # Some logs are continued over multiple lines. They should be explicitly -# commented with \* Continued *\ +# commented with /* Continued */ # # There are some instances of LogPrintf() in comments. Those can be # ignored diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh index 51815963f6..c448fa6f9a 100755 --- a/test/lint/lint-python.sh +++ b/test/lint/lint-python.sh @@ -102,7 +102,7 @@ if ! PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; e EXIT_CODE=1 fi -if ! mypy --ignore-missing-imports $(git ls-files "test/functional/*.py" "contrib/devtools/*.py"); then +if ! mypy --ignore-missing-imports --show-error-codes $(git ls-files "test/functional/*.py" "contrib/devtools/*.py"); then EXIT_CODE=1 fi diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh index 4dbf5ed28e..73ac583d84 100755 --- a/test/lint/lint-shell.sh +++ b/test/lint/lint-shell.sh @@ -14,10 +14,6 @@ disabled=( SC2086 # Double quote to prevent globbing and word splitting. SC2162 # read without -r will mangle backslashes. ) -disabled_gitian=( - SC2094 # Make sure not to read and write the same file in the same pipeline. - SC2129 # Consider using { cmd1; cmd2; } >> file instead of individual redirects. -) EXIT_CODE=0 @@ -33,22 +29,4 @@ if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE" $SOURCED_FILES $(git ls-files -- '*.sh' | EXIT_CODE=1 fi -if ! command -v yq > /dev/null; then - echo "Skipping Gitian descriptor scripts checking since yq is not installed." - exit $EXIT_CODE -fi - -EXCLUDE_GITIAN=${EXCLUDE}",$(IFS=','; echo "${disabled_gitian[*]}")" -for descriptor in $(git ls-files -- 'contrib/gitian-descriptors/*.yml') -do - script=$(basename "$descriptor") - # Use #!/bin/bash as gitian-builder/bin/gbuild does to complete a script. - echo "#!/bin/bash" > $script - yq -r .script "$descriptor" >> $script - if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE_GITIAN" $script; then - EXIT_CODE=1 - fi - rm $script -done - exit $EXIT_CODE diff --git a/test/lint/lint-spelling.sh b/test/lint/lint-spelling.sh index fbdf3c59c1..111091b7f8 100755 --- a/test/lint/lint-spelling.sh +++ b/test/lint/lint-spelling.sh @@ -15,6 +15,6 @@ if ! command -v codespell > /dev/null; then fi IGNORE_WORDS_FILE=test/lint/lint-spelling.ignore-words.txt -if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} $(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/qt/locale/" ":(exclude)src/qt/*.qrc" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)contrib/gitian-keys/keys.txt"); then +if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} $(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/qt/locale/" ":(exclude)src/qt/*.qrc" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)contrib/builder-keys/keys.txt" ":(exclude)contrib/guix/patches"); then echo "^ Warning: codespell identified likely spelling errors. Any false positives? Add them to the list of ignored words in ${IGNORE_WORDS_FILE}" fi diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 2850cfcea5..b52e105a33 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -89,6 +89,7 @@ implicit-signed-integer-truncation:leveldb/ implicit-signed-integer-truncation:miner.cpp implicit-signed-integer-truncation:net.cpp implicit-signed-integer-truncation:net_processing.cpp +implicit-signed-integer-truncation:netaddress.cpp implicit-signed-integer-truncation:streams.h implicit-signed-integer-truncation:test/arith_uint256_tests.cpp implicit-signed-integer-truncation:test/skiplist_tests.cpp diff --git a/test/util/bitcoin-util-test.py b/test/util/test_runner.py index 7b1cc2b031..aa8fd6eee5 100755 --- a/test/util/bitcoin-util-test.py +++ b/test/util/test_runner.py @@ -10,7 +10,6 @@ Runs automatically during `make check`. Can also be run manually.""" import argparse -import binascii import configparser import difflib import json @@ -167,7 +166,7 @@ def parse_output(a, fmt): if fmt == 'json': # json: compare parsed data return json.loads(a) elif fmt == 'hex': # hex: parse and compare binary data - return binascii.a2b_hex(a.strip()) + return bytes.fromhex(a.strip()) else: raise NotImplementedError("Don't know how to compare %s" % fmt) |