diff options
591 files changed, 17529 insertions, 4967 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index f7962e951c..2cebee4744 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -51,10 +51,6 @@ global_task_template: &GLOBAL_TASK_TEMPLATE << : *BASE_TEMPLATE << : *MAIN_TEMPLATE -depends_sdk_cache_template: &DEPENDS_SDK_CACHE_TEMPLATE - depends_sdk_cache: - folder: "depends/sdk-sources" - compute_credits_template: &CREDITS_TEMPLATE # https://cirrus-ci.org/pricing/#compute-credits # Only use credits for pull requests to the main repo @@ -88,9 +84,9 @@ task: CI_VCPKG_TAG: '2021.05.12' VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads' VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' - QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.12/5.12.11/single/qt-everywhere-src-5.12.11.zip' - QT_LOCAL_PATH: 'C:\qt-everywhere-src-5.12.11.zip' - QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.12.11' + QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.2/single/qt-everywhere-src-5.15.2.zip' + QT_LOCAL_PATH: 'C:\qt-everywhere-src-5.15.2.zip' + QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.2' QTBASEDIR: 'C:\Qt_static' x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"' IgnoreWarnIntDirInTempDetected: 'true' @@ -119,7 +115,7 @@ task: - cd %QT_SOURCE_DIR% - mkdir build - cd build - - ..\configure -release -silent -opensource -confirm-license -opengl desktop -no-shared -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -no-libjpeg -nomake examples -nomake tests -nomake tools -no-dbus -no-libudev -no-icu -no-gtk -no-opengles3 -no-angle -no-sql-sqlite -no-sql-odbc -no-sqlite -no-libudev -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcanvas3d -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquickcontrols -skip qtquickcontrols2 -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-sql -no-feature-sqlmodel -prefix %QTBASEDIR% + - ..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml -prefix %QTBASEDIR% - jom - jom install vcpkg_tools_cache: @@ -164,7 +160,7 @@ task: - netsh int ipv4 set dynamicport tcp start=1025 num=64511 - netsh int ipv6 set dynamicport tcp start=1025 num=64511 # Exclude feature_dbcrash for now due to timeout - - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=4000 --jobs=4 --timeout-factor=8 --failfast --extended --exclude feature_dbcrash + - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=4000 --jobs=4 --timeout-factor=8 --extended --exclude feature_dbcrash task: name: 'ARM [unit tests, no functional tests] [bullseye]' @@ -208,10 +204,10 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh" task: - name: '[depends, sanitizers: thread (TSan), no gui] [hirsute]' + name: '[TSan, depends, no gui] [jammy]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:hirsute + image: ubuntu:jammy cpu: 6 # Increase CPU and Memory to avoid timeout memory: 24G env: @@ -220,7 +216,7 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" task: - name: '[depends, sanitizers: memory (MSan)] [focal]' + name: '[MSan, depends] [focal]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal @@ -229,16 +225,16 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" task: - name: '[no depends, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer] [hirsute]' + name: '[ASan + LSan + UBSan + integer, no depends] [jammy]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:hirsute + image: ubuntu:jammy env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" task: - name: '[no depends, sanitizers: fuzzer,address,undefined,integer] [focal]' + name: '[fuzzer,address,undefined,integer, no depends] [focal]' only_if: $CIRRUS_BRANCH == $CIRRUS_DEFAULT_BRANCH || $CIRRUS_BASE_BRANCH == $CIRRUS_DEFAULT_BRANCH << : *GLOBAL_TASK_TEMPLATE container: @@ -273,18 +269,22 @@ task: task: name: 'macOS 10.15 [gui, no tests] [focal]' - << : *DEPENDS_SDK_CACHE_TEMPLATE - << : *GLOBAL_TASK_TEMPLATE + << : *BASE_TEMPLATE + macos_sdk_cache: + folder: "depends/SDKs/$MACOS_SDK" + fingerprint_key: "$MACOS_SDK" + << : *MAIN_TEMPLATE container: image: ubuntu:focal env: + MACOS_SDK: "Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers" << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_mac.sh" task: name: 'macOS 11 native [gui] [no depends]' brew_install_script: - - brew install boost libevent berkeley-db4 qt@5 miniupnpc libnatpmp ccache zeromq qrencode sqlite libtool automake pkg-config gnu-getopt + - brew install boost libevent berkeley-db@4 qt@5 miniupnpc libnatpmp ccache zeromq qrencode sqlite libtool automake pkg-config gnu-getopt << : *GLOBAL_TASK_TEMPLATE osx_instance: # Use latest image, but hardcode version to avoid silent upgrades (and breaks) @@ -297,8 +297,10 @@ task: task: name: 'ARM64 Android APK [focal]' - << : *DEPENDS_SDK_CACHE_TEMPLATE << : *BASE_TEMPLATE + android_sdk_cache: + folder: "depends/SDKs/android" + fingerprint_key: "ANDROID_API_LEVEL=28 ANDROID_BUILD_TOOLS_VERSION=28.0.3 ANDROID_NDK_VERSION=23.1.7779620" depends_sources_cache: folder: "depends/sources" fingerprint_script: git rev-list -1 HEAD ./depends diff --git a/Makefile.am b/Makefile.am index af63cf0cbb..9f50e51ce0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -206,6 +206,7 @@ LCOV_FILTER_PATTERN = \ -p "src/bench/" \ -p "src/univalue" \ -p "src/crypto/ctaes" \ + -p "src/minisketch" \ -p "src/secp256k1" \ -p "depends" diff --git a/SECURITY.md b/SECURITY.md index 7ed96c7cea..25b6175c95 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -14,7 +14,7 @@ The following keys may be used to communicate sensitive information to developer | Name | Fingerprint | |------|-------------| | Wladimir van der Laan | 71A3 B167 3540 5025 D447 E8F2 7481 0B01 2346 C9A6 | -| Jonas Schnelli | 32EE 5C4C 3FA1 5CCA DB46 ABE5 29D4 BCB6 416F 53EC | | Pieter Wuille | 133E AC17 9436 F14A 5CF1 B794 860F EB80 4E66 9320 | +| Michael Ford | E777 299F C265 DD04 7930 70EB 944D 35F9 AC3D B76A | -You can import a key by running the following command with that individual’s fingerprint: `gpg --recv-keys "<fingerprint>"` Ensure that you put quotes around fingerprints containing spaces. +You can import a key by running the following command with that individual’s fingerprint: `gpg --keyserver hkps://keys.openpgp.org --recv-keys "<fingerprint>"` Ensure that you put quotes around fingerprints containing spaces. diff --git a/build-aux/m4/ax_check_compile_flag.m4 b/build-aux/m4/ax_check_compile_flag.m4 index ca3639715e..bd753b34d7 100644 --- a/build-aux/m4/ax_check_compile_flag.m4 +++ b/build-aux/m4/ax_check_compile_flag.m4 @@ -1,5 +1,5 @@ # =========================================================================== -# http://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html +# https://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html # =========================================================================== # # SYNOPSIS @@ -29,33 +29,12 @@ # Copyright (c) 2008 Guido U. Draheim <guidod@gmx.de> # Copyright (c) 2011 Maarten Bosmans <mkbosmans@gmail.com> # -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General -# Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. -# -# As a special exception, the respective Autoconf Macro's copyright owner -# gives unlimited permission to copy, distribute and modify the configure -# scripts that are the output of Autoconf when processing the Macro. You -# need not follow the terms of the GNU General Public License when using -# or distributing such scripts, even though portions of the text of the -# Macro appear in them. The GNU General Public License (GPL) does govern -# all other use of the material that constitutes the Autoconf Macro. -# -# This special exception to the GPL applies to versions of the Autoconf -# Macro released by the Autoconf Archive. When you make and distribute a -# modified version of the Autoconf Macro, you may extend this special -# exception to the GPL to apply to your modified version as well. +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. -#serial 4 +#serial 6 AC_DEFUN([AX_CHECK_COMPILE_FLAG], [AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF diff --git a/build-aux/m4/ax_check_link_flag.m4 b/build-aux/m4/ax_check_link_flag.m4 index eb01a6ce13..03a30ce4c7 100644 --- a/build-aux/m4/ax_check_link_flag.m4 +++ b/build-aux/m4/ax_check_link_flag.m4 @@ -1,5 +1,5 @@ # =========================================================================== -# http://www.gnu.org/software/autoconf-archive/ax_check_link_flag.html +# https://www.gnu.org/software/autoconf-archive/ax_check_link_flag.html # =========================================================================== # # SYNOPSIS @@ -29,33 +29,12 @@ # Copyright (c) 2008 Guido U. Draheim <guidod@gmx.de> # Copyright (c) 2011 Maarten Bosmans <mkbosmans@gmail.com> # -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General -# Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. -# -# As a special exception, the respective Autoconf Macro's copyright owner -# gives unlimited permission to copy, distribute and modify the configure -# scripts that are the output of Autoconf when processing the Macro. You -# need not follow the terms of the GNU General Public License when using -# or distributing such scripts, even though portions of the text of the -# Macro appear in them. The GNU General Public License (GPL) does govern -# all other use of the material that constitutes the Autoconf Macro. -# -# This special exception to the GPL applies to versions of the Autoconf -# Macro released by the Autoconf Archive. When you make and distribute a -# modified version of the Autoconf Macro, you may extend this special -# exception to the GPL to apply to your modified version as well. +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. -#serial 4 +#serial 6 AC_DEFUN([AX_CHECK_LINK_FLAG], [AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF diff --git a/build-aux/m4/ax_check_preproc_flag.m4 b/build-aux/m4/ax_check_preproc_flag.m4 index ca1d5ee2b6..e43560fbd3 100644 --- a/build-aux/m4/ax_check_preproc_flag.m4 +++ b/build-aux/m4/ax_check_preproc_flag.m4 @@ -1,5 +1,5 @@ # =========================================================================== -# http://www.gnu.org/software/autoconf-archive/ax_check_preproc_flag.html +# https://www.gnu.org/software/autoconf-archive/ax_check_preproc_flag.html # =========================================================================== # # SYNOPSIS @@ -29,33 +29,12 @@ # Copyright (c) 2008 Guido U. Draheim <guidod@gmx.de> # Copyright (c) 2011 Maarten Bosmans <mkbosmans@gmail.com> # -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General -# Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. -# -# As a special exception, the respective Autoconf Macro's copyright owner -# gives unlimited permission to copy, distribute and modify the configure -# scripts that are the output of Autoconf when processing the Macro. You -# need not follow the terms of the GNU General Public License when using -# or distributing such scripts, even though portions of the text of the -# Macro appear in them. The GNU General Public License (GPL) does govern -# all other use of the material that constitutes the Autoconf Macro. -# -# This special exception to the GPL applies to versions of the Autoconf -# Macro released by the Autoconf Archive. When you make and distribute a -# modified version of the Autoconf Macro, you may extend this special -# exception to the GPL to apply to your modified version as well. +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. -#serial 4 +#serial 6 AC_DEFUN([AX_CHECK_PREPROC_FLAG], [AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF diff --git a/build-aux/m4/bitcoin_find_bdb48.m4 b/build-aux/m4/bitcoin_find_bdb48.m4 index 3d6c8210ed..d2f01907c3 100644 --- a/build-aux/m4/bitcoin_find_bdb48.m4 +++ b/build-aux/m4/bitcoin_find_bdb48.m4 @@ -3,12 +3,12 @@ dnl Distributed under the MIT software license, see the accompanying dnl file COPYING or http://www.opensource.org/licenses/mit-license.php. AC_DEFUN([BITCOIN_FIND_BDB48],[ - AC_ARG_VAR(BDB_CFLAGS, [C compiler flags for BerkeleyDB, bypasses autodetection]) - AC_ARG_VAR(BDB_LIBS, [Linker flags for BerkeleyDB, bypasses autodetection]) + AC_ARG_VAR([BDB_CFLAGS], [C compiler flags for BerkeleyDB, bypasses autodetection]) + AC_ARG_VAR([BDB_LIBS], [Linker flags for BerkeleyDB, bypasses autodetection]) - if test "x$use_bdb" = "xno"; then + if test "$use_bdb" = "no"; then use_bdb=no - elif test "x$BDB_CFLAGS" = "x"; then + elif test "$BDB_CFLAGS" = ""; then AC_MSG_CHECKING([for Berkeley DB C++ headers]) BDB_CPPFLAGS= bdbpath=X @@ -28,7 +28,7 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ #error "failed to find bdb 4.8+" #endif ]])],[ - if test "x$bdbpath" = "xX"; then + if test "$bdbpath" = "X"; then bdbpath="${searchpath}" fi ],[ @@ -45,13 +45,13 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ break ],[]) done - if test "x$bdbpath" = "xX"; then + if test "$bdbpath" = "X"; then use_bdb=no AC_MSG_RESULT([no]) AC_MSG_WARN([libdb_cxx headers missing]) AC_MSG_WARN(AC_PACKAGE_NAME[ requires this library for BDB (legacy) wallet support]) AC_MSG_WARN([Passing --without-bdb will suppress this warning]) - elif test "x$bdb48path" = "xX"; then + elif test "$bdb48path" = "X"; then BITCOIN_SUBDIR_TO_INCLUDE(BDB_CPPFLAGS,[${bdbpath}],db_cxx) AC_ARG_WITH([incompatible-bdb],[AS_HELP_STRING([--with-incompatible-bdb], [allow using a bdb version other than 4.8])],[ AC_MSG_WARN([Found Berkeley DB other than 4.8]) @@ -74,9 +74,9 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ fi AC_SUBST(BDB_CPPFLAGS) - if test "x$use_bdb" = "xno"; then + if test "$use_bdb" = "no"; then use_bdb=no - elif test "x$BDB_LIBS" = "x"; then + elif test "$BDB_LIBS" = ""; then # TODO: Ideally this could find the library version and make sure it matches the headers being used for searchlib in db_cxx-4.8 db_cxx db4_cxx; do AC_CHECK_LIB([$searchlib],[main],[ @@ -84,13 +84,13 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ break ]) done - if test "x$BDB_LIBS" = "x"; then + if test "$BDB_LIBS" = ""; then AC_MSG_WARN([libdb_cxx headers missing]) AC_MSG_WARN(AC_PACKAGE_NAME[ requires this library for BDB (legacy) wallet support]) AC_MSG_WARN([Passing --without-bdb will suppress this warning]) fi fi - if test "x$use_bdb" != "xno"; then + if test "$use_bdb" != "no"; then AC_SUBST(BDB_LIBS) AC_DEFINE([USE_BDB], [1], [Define if BDB support should be compiled in]) use_bdb=yes diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index 1e979edf0f..fb14ff40ff 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -5,8 +5,8 @@ dnl file COPYING or http://www.opensource.org/licenses/mit-license.php. dnl Helper for cases where a qt dependency is not met. dnl Output: If qt version is auto, set bitcoin_enable_qt to false. Else, exit. AC_DEFUN([BITCOIN_QT_FAIL],[ - if test "x$bitcoin_qt_want_version" = xauto && test "x$bitcoin_qt_force" != xyes; then - if test "x$bitcoin_enable_qt" != xno; then + if test "$bitcoin_qt_want_version" = "auto" && test "$bitcoin_qt_force" != "yes"; then + if test "$bitcoin_enable_qt" != "no"; then AC_MSG_WARN([$1; bitcoin-qt frontend will not be built]) fi bitcoin_enable_qt=no @@ -17,7 +17,7 @@ AC_DEFUN([BITCOIN_QT_FAIL],[ ]) AC_DEFUN([BITCOIN_QT_CHECK],[ - if test "x$bitcoin_enable_qt" != xno && test "x$bitcoin_qt_want_version" != xno; then + if test "$bitcoin_enable_qt" != "no" && test "$bitcoin_qt_want_version" != "no"; then true $1 else @@ -35,12 +35,12 @@ dnl Inputs: $4: If "yes", don't fail if $2 is not found. dnl Output: $1 is set to the path of $2 if found. $2 are searched in order. AC_DEFUN([BITCOIN_QT_PATH_PROGS],[ BITCOIN_QT_CHECK([ - if test "x$3" != x; then - AC_PATH_PROGS($1,$2,,$3) + if test "$3" != ""; then + AC_PATH_PROGS([$1], [$2], [], [$3]) else - AC_PATH_PROGS($1,$2) + AC_PATH_PROGS([$1], [$2]) fi - if test "x$$1" = x && test "x$4" != xyes; then + if test "$$1" = "" && test "$4" != "yes"; then BITCOIN_QT_FAIL([$1 not found]) fi ]) @@ -57,20 +57,21 @@ AC_DEFUN([BITCOIN_QT_INIT],[ [build bitcoin-qt GUI (default=auto)])], [ bitcoin_qt_want_version=$withval - if test "x$bitcoin_qt_want_version" = xyes; then + if test "$bitcoin_qt_want_version" = "yes"; then bitcoin_qt_force=yes bitcoin_qt_want_version=auto fi ], [bitcoin_qt_want_version=auto]) - AS_IF([test "x$with_gui" = xqt5_debug], + AS_IF([test "$with_gui" = "qt5_debug"], [AS_CASE([$host], [*darwin*], [qt_lib_suffix=_debug], - [*mingw*], [qt_lib_suffix=d], [qt_lib_suffix= ]); bitcoin_qt_want_version=qt5], [qt_lib_suffix= ]) + AS_CASE([$host], [*android*], [qt_lib_suffix=_$ANDROID_ARCH]) + AC_ARG_WITH([qt-incdir],[AS_HELP_STRING([--with-qt-incdir=INC_DIR],[specify qt include path (overridden by pkgconfig)])], [qt_include_path=$withval], []) AC_ARG_WITH([qt-libdir],[AS_HELP_STRING([--with-qt-libdir=LIB_DIR],[specify qt lib path (overridden by pkgconfig)])], [qt_lib_path=$withval], []) AC_ARG_WITH([qt-plugindir],[AS_HELP_STRING([--with-qt-plugindir=PLUGIN_DIR],[specify qt plugin path (overridden by pkgconfig)])], [qt_plugin_path=$withval], []) @@ -86,7 +87,7 @@ AC_DEFUN([BITCOIN_QT_INIT],[ dnl Android doesn't support D-Bus and certainly doesn't use it for notifications case $host in *android*) - if test "x$use_dbus" != xyes; then + if test "$use_dbus" != "yes"; then use_dbus=no fi ;; @@ -118,10 +119,10 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ CPPFLAGS="$QT_INCLUDES $CPPFLAGS" CXXFLAGS="$PIC_FLAGS $CXXFLAGS" _BITCOIN_QT_IS_STATIC - if test "x$bitcoin_cv_static_qt" = xyes; then + if test "$bitcoin_cv_static_qt" = "yes"; then _BITCOIN_QT_CHECK_STATIC_LIBS - if test "x$qt_plugin_path" != x; then + if test "$qt_plugin_path" != ""; then if test -d "$qt_plugin_path/platforms"; then QT_LIBS="$QT_LIBS -L$qt_plugin_path/platforms" fi @@ -136,47 +137,45 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ fi fi - AC_DEFINE(QT_STATICPLUGIN, 1, [Define this symbol if qt plugins are static]) - if test "x$TARGET_OS" != xandroid; then + AC_DEFINE([QT_STATICPLUGIN], [1], [Define this symbol if qt plugins are static]) + if test "$TARGET_OS" != "android"; then _BITCOIN_QT_CHECK_STATIC_PLUGIN([QMinimalIntegrationPlugin], [-lqminimal]) - AC_DEFINE(QT_QPA_PLATFORM_MINIMAL, 1, [Define this symbol if the minimal qt platform exists]) + AC_DEFINE([QT_QPA_PLATFORM_MINIMAL], [1], [Define this symbol if the minimal qt platform exists]) fi - if test "x$TARGET_OS" = xwindows; then + if test "$TARGET_OS" = "windows"; then dnl Linking against wtsapi32 is required. See #17749 and dnl https://bugreports.qt.io/browse/QTBUG-27097. AX_CHECK_LINK_FLAG([-lwtsapi32], [QT_LIBS="$QT_LIBS -lwtsapi32"], [AC_MSG_ERROR([could not link against -lwtsapi32])]) _BITCOIN_QT_CHECK_STATIC_PLUGIN([QWindowsIntegrationPlugin], [-lqwindows]) _BITCOIN_QT_CHECK_STATIC_PLUGIN([QWindowsVistaStylePlugin], [-lqwindowsvistastyle]) - AC_DEFINE(QT_QPA_PLATFORM_WINDOWS, 1, [Define this symbol if the qt platform is windows]) - elif test "x$TARGET_OS" = xlinux; then - dnl workaround for https://bugreports.qt.io/browse/QTBUG-74874 - AX_CHECK_LINK_FLAG([-lxcb-shm], [QT_LIBS="$QT_LIBS -lxcb-shm"], [AC_MSG_ERROR([could not link against -lxcb-shm])]) + AC_DEFINE([QT_QPA_PLATFORM_WINDOWS], [1], [Define this symbol if the qt platform is windows]) + elif test "$TARGET_OS" = "linux"; then _BITCOIN_QT_CHECK_STATIC_PLUGIN([QXcbIntegrationPlugin], [-lqxcb]) - AC_DEFINE(QT_QPA_PLATFORM_XCB, 1, [Define this symbol if the qt platform is xcb]) - elif test "x$TARGET_OS" = xdarwin; then - AX_CHECK_LINK_FLAG([[-framework Carbon]],[QT_LIBS="$QT_LIBS -framework Carbon"],[AC_MSG_ERROR(could not link against Carbon framework)]) - AX_CHECK_LINK_FLAG([[-framework IOSurface]],[QT_LIBS="$QT_LIBS -framework IOSurface"],[AC_MSG_ERROR(could not link against IOSurface framework)]) - AX_CHECK_LINK_FLAG([[-framework Metal]],[QT_LIBS="$QT_LIBS -framework Metal"],[AC_MSG_ERROR(could not link against Metal framework)]) - AX_CHECK_LINK_FLAG([[-framework QuartzCore]],[QT_LIBS="$QT_LIBS -framework QuartzCore"],[AC_MSG_ERROR(could not link against QuartzCore framework)]) + AC_DEFINE([QT_QPA_PLATFORM_XCB], [1], [Define this symbol if the qt platform is xcb]) + elif test "$TARGET_OS" = "darwin"; then + AX_CHECK_LINK_FLAG([-framework Carbon], [QT_LIBS="$QT_LIBS -framework Carbon"], [AC_MSG_ERROR(could not link against Carbon framework)]) + AX_CHECK_LINK_FLAG([-framework IOSurface], [QT_LIBS="$QT_LIBS -framework IOSurface"], [AC_MSG_ERROR(could not link against IOSurface framework)]) + AX_CHECK_LINK_FLAG([-framework Metal], [QT_LIBS="$QT_LIBS -framework Metal"], [AC_MSG_ERROR(could not link against Metal framework)]) + AX_CHECK_LINK_FLAG([-framework QuartzCore], [QT_LIBS="$QT_LIBS -framework QuartzCore"], [AC_MSG_ERROR(could not link against QuartzCore framework)]) _BITCOIN_QT_CHECK_STATIC_PLUGIN([QCocoaIntegrationPlugin], [-lqcocoa]) _BITCOIN_QT_CHECK_STATIC_PLUGIN([QMacStylePlugin], [-lqmacstyle]) - AC_DEFINE(QT_QPA_PLATFORM_COCOA, 1, [Define this symbol if the qt platform is cocoa]) - elif test "x$TARGET_OS" = xandroid; then - QT_LIBS="-Wl,--export-dynamic,--undefined=JNI_OnLoad -lqtforandroid -ljnigraphics -landroid -lqtfreetype $QT_LIBS" - AC_DEFINE(QT_QPA_PLATFORM_ANDROID, 1, [Define this symbol if the qt platform is android]) + AC_DEFINE([QT_QPA_PLATFORM_COCOA], [1], [Define this symbol if the qt platform is cocoa]) + elif test "$TARGET_OS" = "android"; then + QT_LIBS="-Wl,--export-dynamic,--undefined=JNI_OnLoad -lplugins_platforms_qtforandroid_$ANDROID_ARCH -ljnigraphics -landroid -lqtfreetype_$ANDROID_ARCH $QT_LIBS" + AC_DEFINE([QT_QPA_PLATFORM_ANDROID], [1], [Define this symbol if the qt platform is android]) fi fi CPPFLAGS=$TEMP_CPPFLAGS CXXFLAGS=$TEMP_CXXFLAGS ]) - if test "x$qt_bin_path" = x; then + if test "$qt_bin_path" = ""; then qt_bin_path="`$PKG_CONFIG --variable=host_bins ${qt_lib_prefix}Core 2>/dev/null`" fi - if test "x$use_hardening" != xno; then + if test "$use_hardening" != "no"; then BITCOIN_QT_CHECK([ - AC_MSG_CHECKING(whether -fPIE can be used with this Qt config) + AC_MSG_CHECKING([whether -fPIE can be used with this Qt config]) TEMP_CPPFLAGS=$CPPFLAGS TEMP_CXXFLAGS=$CXXFLAGS CPPFLAGS="$QT_INCLUDES $CPPFLAGS" @@ -192,15 +191,15 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ choke #endif ]])], - [ AC_MSG_RESULT(yes); QT_PIE_FLAGS=$PIE_FLAGS ], - [ AC_MSG_RESULT(no); QT_PIE_FLAGS=$PIC_FLAGS] + [ AC_MSG_RESULT([yes]); QT_PIE_FLAGS=$PIE_FLAGS ], + [ AC_MSG_RESULT([no]); QT_PIE_FLAGS=$PIC_FLAGS] ) CPPFLAGS=$TEMP_CPPFLAGS CXXFLAGS=$TEMP_CXXFLAGS ]) else BITCOIN_QT_CHECK([ - AC_MSG_CHECKING(whether -fPIC is needed with this Qt config) + AC_MSG_CHECKING([whether -fPIC is needed with this Qt config]) TEMP_CPPFLAGS=$CPPFLAGS CPPFLAGS="$QT_INCLUDES $CPPFLAGS" AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ @@ -214,8 +213,8 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ choke #endif ]])], - [ AC_MSG_RESULT(no)], - [ AC_MSG_RESULT(yes); QT_PIE_FLAGS=$PIC_FLAGS] + [ AC_MSG_RESULT([no])], + [ AC_MSG_RESULT([yes]); QT_PIE_FLAGS=$PIC_FLAGS] ) CPPFLAGS=$TEMP_CPPFLAGS ]) @@ -234,12 +233,12 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ BITCOIN_QT_CHECK([ MOC_DEFS="${MOC_DEFS} -DQ_OS_MAC" base_frameworks="-framework Foundation -framework AppKit" - AX_CHECK_LINK_FLAG([[$base_frameworks]],[QT_LIBS="$QT_LIBS $base_frameworks"],[AC_MSG_ERROR(could not find base frameworks)]) + AX_CHECK_LINK_FLAG([$base_frameworks], [QT_LIBS="$QT_LIBS $base_frameworks"], [AC_MSG_ERROR(could not find base frameworks)]) ]) ;; *mingw*) BITCOIN_QT_CHECK([ - AX_CHECK_LINK_FLAG([[-mwindows]],[QT_LDFLAGS="$QT_LDFLAGS -mwindows"],[AC_MSG_WARN(-mwindows linker support not detected)]) + AX_CHECK_LINK_FLAG([-mwindows], [QT_LDFLAGS="$QT_LDFLAGS -mwindows"], [AC_MSG_WARN([-mwindows linker support not detected])]) ]) esac @@ -249,26 +248,26 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ BITCOIN_QT_CHECK([ bitcoin_enable_qt=yes bitcoin_enable_qt_test=yes - if test "x$have_qt_test" = xno; then + if test "$have_qt_test" = "no"; then bitcoin_enable_qt_test=no fi bitcoin_enable_qt_dbus=no - if test "x$use_dbus" != xno && test "x$have_qt_dbus" = xyes; then + if test "$use_dbus" != "no" && test "$have_qt_dbus" = "yes"; then bitcoin_enable_qt_dbus=yes fi - if test "x$use_dbus" = xyes && test "x$have_qt_dbus" = xno; then + if test "$use_dbus" = "yes" && test "$have_qt_dbus" = "no"; then AC_MSG_ERROR([libQtDBus not found. Install libQtDBus or remove --with-qtdbus.]) fi - if test "x$LUPDATE" = x; then + if test "$LUPDATE" = ""; then AC_MSG_WARN([lupdate tool is required to update Qt translations.]) fi - if test "x$LCONVERT" = x; then + if test "$LCONVERT" = ""; then AC_MSG_WARN([lconvert tool is required to update Qt translations.]) fi ],[ bitcoin_enable_qt=no ]) - if test x$bitcoin_enable_qt = xyes; then + if test $bitcoin_enable_qt = "yes"; then AC_MSG_RESULT([$bitcoin_enable_qt ($qt_lib_prefix)]) else AC_MSG_RESULT([$bitcoin_enable_qt]) @@ -349,18 +348,20 @@ AC_DEFUN([_BITCOIN_QT_CHECK_STATIC_LIBS], [ PKG_CHECK_MODULES([QT_FB], [${qt_lib_prefix}FbSupport${qt_lib_suffix}], [QT_LIBS="$QT_FB_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_FONTDATABASE], [${qt_lib_prefix}FontDatabaseSupport${qt_lib_suffix}], [QT_LIBS="$QT_FONTDATABASE_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_THEME], [${qt_lib_prefix}ThemeSupport${qt_lib_suffix}], [QT_LIBS="$QT_THEME_LIBS $QT_LIBS"]) - if test "x$TARGET_OS" = xlinux; then + if test "$TARGET_OS" = "linux"; then PKG_CHECK_MODULES([QT_INPUT], [${qt_lib_prefix}InputSupport], [QT_LIBS="$QT_INPUT_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_SERVICE], [${qt_lib_prefix}ServiceSupport], [QT_LIBS="$QT_SERVICE_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_XCBQPA], [${qt_lib_prefix}XcbQpa], [QT_LIBS="$QT_XCBQPA_LIBS $QT_LIBS"]) - elif test "x$TARGET_OS" = xdarwin; then + PKG_CHECK_MODULES([QT_XKBCOMMON], [${qt_lib_prefix}XkbCommonSupport], [QT_LIBS="$QT_XKBCOMMON_LIBS $QT_LIBS"]) + elif test "$TARGET_OS" = "darwin"; then PKG_CHECK_MODULES([QT_CLIPBOARD], [${qt_lib_prefix}ClipboardSupport${qt_lib_suffix}], [QT_LIBS="$QT_CLIPBOARD_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_GRAPHICS], [${qt_lib_prefix}GraphicsSupport${qt_lib_suffix}], [QT_LIBS="$QT_GRAPHICS_LIBS $QT_LIBS"]) PKG_CHECK_MODULES([QT_SERVICE], [${qt_lib_prefix}ServiceSupport${qt_lib_suffix}], [QT_LIBS="$QT_SERVICE_LIBS $QT_LIBS"]) - elif test "x$TARGET_OS" = xwindows; then + elif test "$TARGET_OS" = "windows"; then PKG_CHECK_MODULES([QT_WINDOWSUIAUTOMATION], [${qt_lib_prefix}WindowsUIAutomationSupport${qt_lib_suffix}], [QT_LIBS="$QT_WINDOWSUIAUTOMATION_LIBS $QT_LIBS"]) - elif test "x$TARGET_OS" = xandroid; then - PKG_CHECK_MODULES([QT_EGL], [${qt_lib_prefix}EglSupport], [QT_LIBS="$QT_EGL_LIBS $QT_LIBS"]) + elif test "$TARGET_OS" = "android"; then + PKG_CHECK_MODULES([QT_EGL], [${qt_lib_prefix}EglSupport${qt_lib_suffix}], [QT_LIBS="$QT_EGL_LIBS $QT_LIBS"]) + PKG_CHECK_MODULES([QT_SERVICE], [${qt_lib_prefix}ServiceSupport${qt_lib_suffix}], [QT_LIBS="$QT_SERVICE_LIBS $QT_LIBS"]) fi ]) @@ -391,7 +392,7 @@ AC_DEFUN([_BITCOIN_QT_FIND_LIBS],[ BITCOIN_QT_CHECK([ PKG_CHECK_MODULES([QT_TEST], [${qt_lib_prefix}Test${qt_lib_suffix} $qt_version], [QT_TEST_INCLUDES="$QT_TEST_CFLAGS"; have_qt_test=yes], [have_qt_test=no]) - if test "x$use_dbus" != xno; then + if test "$use_dbus" != "no"; then PKG_CHECK_MODULES([QT_DBUS], [${qt_lib_prefix}DBus $qt_version], [QT_DBUS_INCLUDES="$QT_DBUS_CFLAGS"; have_qt_dbus=yes], [have_qt_dbus=no]) fi ]) diff --git a/build-aux/m4/bitcoin_subdir_to_include.m4 b/build-aux/m4/bitcoin_subdir_to_include.m4 index 7841042ac8..736270afea 100644 --- a/build-aux/m4/bitcoin_subdir_to_include.m4 +++ b/build-aux/m4/bitcoin_subdir_to_include.m4 @@ -5,13 +5,13 @@ dnl file COPYING or http://www.opensource.org/licenses/mit-license.php. dnl BITCOIN_SUBDIR_TO_INCLUDE([CPPFLAGS-VARIABLE-NAME],[SUBDIRECTORY-NAME],[HEADER-FILE]) dnl SUBDIRECTORY-NAME must end with a path separator AC_DEFUN([BITCOIN_SUBDIR_TO_INCLUDE],[ - if test "x$2" = "x"; then + if test "$2" = ""; then AC_MSG_RESULT([default]) else echo "#include <$2$3.h>" >conftest.cpp newinclpath=`${CXXCPP} ${CPPFLAGS} -M conftest.cpp 2>/dev/null | [ tr -d '\\n\\r\\\\' | sed -e 's/^.*[[:space:]:]\(\/[^[:space:]]*\)]$3[\.h[[:space:]].*$/\1/' -e t -e d`] AC_MSG_RESULT([${newinclpath}]) - if test "x${newinclpath}" != "x"; then + if test "${newinclpath}" != ""; then eval "$1=\"\$$1\"' -I${newinclpath}'" fi fi diff --git a/build_msvc/README.md b/build_msvc/README.md index 36fb942c8e..cabe4d55ec 100644 --- a/build_msvc/README.md +++ b/build_msvc/README.md @@ -28,14 +28,14 @@ Qt --------------------- To build Bitcoin Core with the GUI, a static build of Qt is required. -1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-src-5.12.11.zip`](https://download.qt.io/official_releases/qt/5.12/5.12.11/single/qt-everywhere-src-5.12.11.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`. +1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-src-5.15.2.zip`](https://download.qt.io/official_releases/qt/5.15/5.15.2/single/qt-everywhere-src-5.15.2.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`. 2. Open "x64 Native Tools Command Prompt for VS 2019", and input the following commands: ```cmd cd C:\dev\qt-source mkdir build cd build -..\configure -release -silent -opensource -confirm-license -opengl desktop -no-shared -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -no-libjpeg -nomake examples -nomake tests -nomake tools -no-dbus -no-libudev -no-icu -no-gtk -no-opengles3 -no-angle -no-sql-sqlite -no-sql-odbc -no-sqlite -no-libudev -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcanvas3d -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquickcontrols -skip qtquickcontrols2 -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-sql -no-feature-sqlmodel -prefix C:\Qt_static +..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml -prefix C:\Qt_static nmake nmake install ``` diff --git a/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj b/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj index affb60425b..2ac0be9814 100644 --- a/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj +++ b/build_msvc/bitcoin-wallet/bitcoin-wallet.vcxproj @@ -18,27 +18,18 @@ <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj"> <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project> </ProjectReference> - <ProjectReference Include="..\libbitcoin_cli\libbitcoin_cli.vcxproj"> - <Project>{0667528c-d734-4009-adf9-c0d6c4a5a5a6}</Project> - </ProjectReference> <ProjectReference Include="..\libbitcoin_common\libbitcoin_common.vcxproj"> <Project>{7c87e378-df58-482e-aa2f-1bc129bc19ce}</Project> </ProjectReference> <ProjectReference Include="..\libbitcoin_crypto\libbitcoin_crypto.vcxproj"> <Project>{6190199c-6cf4-4dad-bfbd-93fa72a760c1}</Project> </ProjectReference> - <ProjectReference Include="..\libbitcoin_server\libbitcoin_server.vcxproj"> - <Project>{460fee33-1fe1-483f-b3bf-931ff8e969a5}</Project> - </ProjectReference> <ProjectReference Include="..\libbitcoin_util\libbitcoin_util.vcxproj"> <Project>{b53a5535-ee9d-4c6f-9a26-f79ee3bc3754}</Project> </ProjectReference> <ProjectReference Include="..\libbitcoin_wallet\libbitcoin_wallet.vcxproj"> <Project>{93b86837-b543-48a5-a89b-7c87abb77df2}</Project> </ProjectReference> - <ProjectReference Include="..\libbitcoin_zmq\libbitcoin_zmq.vcxproj"> - <Project>{792d487f-f14c-49fc-a9de-3fc150f31c3f}</Project> - </ProjectReference> <ProjectReference Include="..\libunivalue\libunivalue.vcxproj"> <Project>{5724ba7d-a09a-4ba8-800b-c4c1561b3d69}</Project> </ProjectReference> @@ -48,9 +39,6 @@ <ProjectReference Include="..\libsecp256k1\libsecp256k1.vcxproj"> <Project>{bb493552-3b8c-4a8c-bf69-a6e7a51d2ea6}</Project> </ProjectReference> - <ProjectReference Include="..\libleveldb\libleveldb.vcxproj"> - <Project>{18430fef-6b61-4c53-b396-718e02850f1b}</Project> - </ProjectReference> </ItemGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> diff --git a/build_msvc/bitcoin.sln b/build_msvc/bitcoin.sln index 1b1f27a8a9..b2ab64a34b 100644 --- a/build_msvc/bitcoin.sln +++ b/build_msvc/bitcoin.sln @@ -4,8 +4,6 @@ VisualStudioVersion = 16.0.28803.452 MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoinconsensus", "libbitcoinconsensus\libbitcoinconsensus.vcxproj", "{2B384FA8-9EE1-4544-93CB-0D733C25E8CE}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "testconsensus", "testconsensus\testconsensus.vcxproj", "{E78473E9-B850-456C-9120-276301E04C06}" -EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bitcoind", "bitcoind\bitcoind.vcxproj", "{D4513DDF-6013-44DC-ADCC-12EAF6D1F038}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libbitcoin_util", "libbitcoin_util\libbitcoin_util.vcxproj", "{B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}" @@ -50,6 +48,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libtest_util", "libtest_uti EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test_bitcoin-qt", "test_bitcoin-qt\test_bitcoin-qt.vcxproj", "{51201D5E-D939-4854-AE9D-008F03FF518E}" EndProject +Project("{542007E3-BE0D-4B0D-A6B0-AA8813E2558D}") = "libminisketch", "libminisketch\libminisketch.vcxproj", "{542007E3-BE0D-4B0D-A6B0-AA8813E2558D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -60,10 +60,6 @@ Global {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Debug|x64.Build.0 = Debug|x64 {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Release|x64.ActiveCfg = Release|x64 {2B384FA8-9EE1-4544-93CB-0D733C25E8CE}.Release|x64.Build.0 = Release|x64 - {E78473E9-B850-456C-9120-276301E04C06}.Debug|x64.ActiveCfg = Debug|x64 - {E78473E9-B850-456C-9120-276301E04C06}.Debug|x64.Build.0 = Debug|x64 - {E78473E9-B850-456C-9120-276301E04C06}.Release|x64.ActiveCfg = Release|x64 - {E78473E9-B850-456C-9120-276301E04C06}.Release|x64.Build.0 = Release|x64 {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Debug|x64.ActiveCfg = Debug|x64 {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Debug|x64.Build.0 = Debug|x64 {D4513DDF-6013-44DC-ADCC-12EAF6D1F038}.Release|x64.ActiveCfg = Release|x64 @@ -152,13 +148,15 @@ Global {51201D5E-D939-4854-AE9D-008F03FF518E}.Debug|x64.Build.0 = Debug|x64 {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x64.ActiveCfg = Release|x64 {51201D5E-D939-4854-AE9D-008F03FF518E}.Release|x64.Build.0 = Release|x64 + {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Debug|x64.ActiveCfg = Debug|x64 + {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Debug|x64.Build.0 = Debug|x64 + {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Release|x64.ActiveCfg = Release|x64 + {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {8AA72EDA-2CD4-4564-B1E4-688B760EEEE9} - SolutionGuid = {8607C0F4-F33D-41B8-8D51-18E366A0F8DF} SolutionGuid = {58AAB032-7274-49BD-845E-5EF4DBB69B70} EndGlobalSection EndGlobal diff --git a/build_msvc/bitcoin_config.h b/build_msvc/bitcoin_config.h.in index e2930f3ea9..83e53c8d56 100644 --- a/build_msvc/bitcoin_config.h +++ b/build_msvc/bitcoin_config.h.in @@ -6,16 +6,16 @@ #define BITCOIN_BITCOIN_CONFIG_H /* Version Build */ -#define CLIENT_VERSION_BUILD 0 +#define CLIENT_VERSION_BUILD $ /* Version is release */ -#define CLIENT_VERSION_IS_RELEASE false +#define CLIENT_VERSION_IS_RELEASE $ /* Major version */ -#define CLIENT_VERSION_MAJOR 22 +#define CLIENT_VERSION_MAJOR $ /* Minor version */ -#define CLIENT_VERSION_MINOR 99 +#define CLIENT_VERSION_MINOR $ /* Copyright holder(s) before %s replacement */ #define COPYRIGHT_HOLDERS "The %s developers" @@ -27,7 +27,7 @@ #define COPYRIGHT_HOLDERS_SUBSTITUTION "Bitcoin Core" /* Copyright year */ -#define COPYRIGHT_YEAR 2021 +#define COPYRIGHT_YEAR $ /* Define to 1 to enable wallet functions */ #define ENABLE_WALLET 1 @@ -184,13 +184,13 @@ #define PACKAGE_NAME "Bitcoin Core" /* Define to the full name and version of this package. */ -#define PACKAGE_STRING "Bitcoin Core 22.99.0" +#define PACKAGE_STRING $ /* Define to the home page for this package. */ #define PACKAGE_URL "https://bitcoincore.org/" /* Define to the version of this package. */ -#define PACKAGE_VERSION "22.99.0" +#define PACKAGE_VERSION $ /* Define this symbol if the minimal qt platform exists */ #define QT_QPA_PLATFORM_MINIMAL 1 diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj index fb4d70cc78..0cbe2effd5 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj @@ -91,7 +91,7 @@ <DisableSpecificWarnings>4018;4244;4267;4334;4715;4805;4834</DisableSpecificWarnings> <TreatWarningAsError>true</TreatWarningAsError> <PreprocessorDefinitions>_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions)</PreprocessorDefinitions> - <AdditionalIncludeDirectories>..\..\src;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories>..\..\src;..\..\src\minisketch\include;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> <Link> <SubSystem>Console</SubSystem> diff --git a/build_msvc/libminisketch/libminisketch.vcxproj b/build_msvc/libminisketch/libminisketch.vcxproj new file mode 100644 index 0000000000..b34593fe5c --- /dev/null +++ b/build_msvc/libminisketch/libminisketch.vcxproj @@ -0,0 +1,38 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <Import Project="..\common.init.vcxproj" /> + <PropertyGroup Label="Globals"> + <ProjectGuid>{542007E3-BE0D-4B0D-A6B0-AA8813E2558D}</ProjectGuid> + </PropertyGroup> + <PropertyGroup Label="Configuration"> + <ConfigurationType>StaticLibrary</ConfigurationType> + </PropertyGroup> + <ItemGroup> + <ClCompile Include="..\..\src\minisketch\src\minisketch.cpp" /> + <ClCompile Include="..\..\src\minisketch\src\fields\clmul_1byte.cpp" /> + <ClCompile Include="..\..\src\minisketch\src\fields\clmul_2bytes.cpp" /> + <ClCompile Include="..\..\src\minisketch\src\fields\clmul_3bytes.cpp" /> + <ClCompile Include="..\..\src\minisketch\src\fields\clmul_4bytes.cpp" /> + <ClCompile Include="..\..\src\minisketch\src\fields\clmul_5bytes.cpp" /> + <ClCompile Include="..\..\src\minisketch\src\fields\clmul_6bytes.cpp" /> + <ClCompile Include="..\..\src\minisketch\src\fields\clmul_7bytes.cpp" /> + <ClCompile Include="..\..\src\minisketch\src\fields\clmul_8bytes.cpp" /> + <ClCompile Include="..\..\src\minisketch\src\fields\generic_1byte.cpp" /> + <ClCompile Include="..\..\src\minisketch\src\fields\generic_2bytes.cpp" /> + <ClCompile Include="..\..\src\minisketch\src\fields\generic_3bytes.cpp" /> + <ClCompile Include="..\..\src\minisketch\src\fields\generic_4bytes.cpp" /> + <ClCompile Include="..\..\src\minisketch\src\fields\generic_5bytes.cpp" /> + <ClCompile Include="..\..\src\minisketch\src\fields\generic_6bytes.cpp" /> + <ClCompile Include="..\..\src\minisketch\src\fields\generic_7bytes.cpp" /> + <ClCompile Include="..\..\src\minisketch\src\fields\generic_8bytes.cpp" /> + </ItemGroup> + <ItemDefinitionGroup> + <ClCompile> + <DisableSpecificWarnings>4060;4065;4146;4244;4267;4554</DisableSpecificWarnings> + <PreprocessorDefinitions>HAVE_CLMUL;DISABLE_DEFAULT_FIELDS;ENABLE_FIELD_32;%(PreprocessorDefinitions)</PreprocessorDefinitions> + </ClCompile> + </ItemDefinitionGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> + <Import Project="..\common.vcxproj" /> +</Project> diff --git a/build_msvc/libsecp256k1_config.h b/build_msvc/libsecp256k1_config.h index 5978b9a0d9..57f2f144ff 100644 --- a/build_msvc/libsecp256k1_config.h +++ b/build_msvc/libsecp256k1_config.h @@ -29,4 +29,4 @@ #define ECMULT_GEN_PREC_BITS 4 #define ECMULT_WINDOW_SIZE 15 -#endif /* BITCOIN_LIBSECP256K1_CONFIG_H */ +#endif // BITCOIN_LIBSECP256K1_CONFIG_H diff --git a/build_msvc/msvc-autogen.py b/build_msvc/msvc-autogen.py index a1ed935996..6ce65f3fa3 100755 --- a/build_msvc/msvc-autogen.py +++ b/build_msvc/msvc-autogen.py @@ -57,6 +57,41 @@ def set_common_properties(toolset): with open(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), 'w', encoding='utf-8',newline='\n') as wfile: wfile.write(s) +def parse_config_into_btc_config(): + def find_between( s, first, last ): + try: + start = s.index( first ) + len( first ) + end = s.index( last, start ) + return s[start:end] + except ValueError: + return "" + + config_info = [] + with open(os.path.join(SOURCE_DIR,'../configure.ac'), encoding="utf8") as f: + for line in f: + if line.startswith("define"): + config_info.append(find_between(line, "(_", ")")) + + config_info = [c for c in config_info if not c.startswith("COPYRIGHT_HOLDERS")] + + config_dict = dict(item.split(", ") for item in config_info) + config_dict["PACKAGE_VERSION"] = f"\"{config_dict['CLIENT_VERSION_MAJOR']}.{config_dict['CLIENT_VERSION_MINOR']}.{config_dict['CLIENT_VERSION_BUILD']}\"" + version = config_dict["PACKAGE_VERSION"].strip('"') + config_dict["PACKAGE_STRING"] = f"\"Bitcoin Core {version}\"" + + with open(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h.in'), "r", encoding="utf8") as template_file: + template = template_file.readlines() + + for index, line in enumerate(template): + header = "" + if line.startswith("#define"): + header = line.split(" ")[1] + if header in config_dict: + template[index] = line.replace("$", f"{config_dict[header]}") + + with open(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), "w", encoding="utf8") as btc_config: + btc_config.writelines(template) + def main(): parser = argparse.ArgumentParser(description='Bitcoin-core msbuild configuration initialiser.') parser.add_argument('-toolset', nargs='?',help='Optionally sets the msbuild platform toolset, e.g. v142 for Visual Studio 2019.' @@ -79,6 +114,7 @@ def main(): with open(vcxproj_filename, 'w', encoding='utf-8') as vcxproj_file: vcxproj_file.write(vcxproj_in_file.read().replace( '@SOURCE_FILES@\n', content)) + parse_config_into_btc_config() copyfile(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), os.path.join(SOURCE_DIR, 'config/bitcoin-config.h')) copyfile(os.path.join(SOURCE_DIR,'../build_msvc/libsecp256k1_config.h'), os.path.join(SOURCE_DIR, 'secp256k1/src/libsecp256k1-config.h')) diff --git a/build_msvc/test_bitcoin/test_bitcoin.vcxproj b/build_msvc/test_bitcoin/test_bitcoin.vcxproj index bb1a780bfa..2fc0078b8d 100644 --- a/build_msvc/test_bitcoin/test_bitcoin.vcxproj +++ b/build_msvc/test_bitcoin/test_bitcoin.vcxproj @@ -19,6 +19,9 @@ <ClCompile Include="..\..\src\wallet\test\util.cpp" /> </ItemGroup> <ItemGroup> + <ProjectReference Include="..\libminisketch\libminisketch.vcxproj"> + <Project>{542007e3-be0d-4b0d-a6b0-aa8813e2558d}</Project> + </ProjectReference> <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj"> <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project> </ProjectReference> diff --git a/build_msvc/testconsensus/testconsensus.cpp b/build_msvc/testconsensus/testconsensus.cpp deleted file mode 100644 index f3c8517130..0000000000 --- a/build_msvc/testconsensus/testconsensus.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) 2018-2020 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <iostream> - -// bitcoin includes. -#include <..\src\script\bitcoinconsensus.h> -#include <..\src\primitives\transaction.h> -#include <..\src\script\script.h> -#include <..\src\streams.h> -#include <..\src\version.h> - -CMutableTransaction BuildSpendingTransaction(const CScript& scriptSig, const CScriptWitness& scriptWitness, int nValue = 0) -{ - CMutableTransaction txSpend; - txSpend.nVersion = 1; - txSpend.nLockTime = 0; - txSpend.vin.resize(1); - txSpend.vout.resize(1); - txSpend.vin[0].scriptWitness = scriptWitness; - txSpend.vin[0].prevout.hash = uint256(); - txSpend.vin[0].prevout.n = 0; - txSpend.vin[0].scriptSig = scriptSig; - txSpend.vin[0].nSequence = CTxIn::SEQUENCE_FINAL; - txSpend.vout[0].scriptPubKey = CScript(); - txSpend.vout[0].nValue = nValue; - - return txSpend; -} - -int main() -{ - std::cout << "bitcoinconsensus version: " << bitcoinconsensus_version() << std::endl; - - CScript pubKeyScript; - pubKeyScript << OP_1 << OP_0 << OP_1; - - int amount = 0; // 600000000; - - CScript scriptSig; - CScriptWitness scriptWitness; - CTransaction vanillaSpendTx = BuildSpendingTransaction(scriptSig, scriptWitness, amount); - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << vanillaSpendTx; - - bitcoinconsensus_error err; - auto op0Result = bitcoinconsensus_verify_script_with_amount(pubKeyScript.data(), pubKeyScript.size(), amount, stream.data(), stream.size(), 0, bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL, &err); - std::cout << "Op0 result: " << op0Result << ", error code " << err << std::endl; - - getchar(); - - return 0; -} diff --git a/build_msvc/testconsensus/testconsensus.vcxproj b/build_msvc/testconsensus/testconsensus.vcxproj deleted file mode 100644 index 776c40920a..0000000000 --- a/build_msvc/testconsensus/testconsensus.vcxproj +++ /dev/null @@ -1,28 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> - <Import Project="..\common.init.vcxproj" /> - <PropertyGroup Label="Globals"> - <ProjectGuid>{E78473E9-B850-456C-9120-276301E04C06}</ProjectGuid> - </PropertyGroup> - <PropertyGroup Label="Configuration"> - <ConfigurationType>Application</ConfigurationType> - <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir> - </PropertyGroup> - <ItemGroup> - <ClCompile Include="testconsensus.cpp" /> - </ItemGroup> - <ItemGroup> - <ProjectReference Include="..\libbitcoinconsensus\libbitcoinconsensus.vcxproj"> - <Project>{2B384FA8-9EE1-4544-93CB-0D733C25E8CE}</Project> - </ProjectReference> - <ProjectReference Include="..\libbitcoin_util\libbitcoin_util.vcxproj"> - <Project>{B53A5535-EE9D-4C6F-9A26-F79EE3BC3754}</Project> - </ProjectReference> - <ProjectReference Include="..\libsecp256k1\libsecp256k1.vcxproj"> - <Project>{BB493552-3B8C-4A8C-BF69-A6E7A51D2EA6}</Project> - </ProjectReference> - </ItemGroup> - <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> - <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> - <Import Project="..\common.vcxproj" /> -</Project> diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 991234a436..cf203b4940 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -8,8 +8,8 @@ export LC_ALL=C ${CI_RETRY_EXE} apt-get update ${CI_RETRY_EXE} apt-get install -y clang-format-9 python3-pip curl git gawk jq -update-alternatives --install /usr/bin/clang-format clang-format $(which clang-format-9 ) 100 -update-alternatives --install /usr/bin/clang-format-diff clang-format-diff $(which clang-format-diff-9) 100 +update-alternatives --install /usr/bin/clang-format clang-format "$(which clang-format-9 )" 100 +update-alternatives --install /usr/bin/clang-format-diff clang-format-diff "$(which clang-format-diff-9)" 100 ${CI_RETRY_EXE} pip3 install codespell==2.0.0 ${CI_RETRY_EXE} pip3 install flake8==3.8.3 @@ -17,6 +17,6 @@ ${CI_RETRY_EXE} pip3 install mypy==0.910 ${CI_RETRY_EXE} pip3 install pyzmq==22.3.0 ${CI_RETRY_EXE} pip3 install vulture==2.3 -SHELLCHECK_VERSION=v0.7.2 +SHELLCHECK_VERSION=v0.8.0 curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ export PATH="/tmp/shellcheck-${SHELLCHECK_VERSION}:${PATH}" diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index f7dacd8512..16bc00285f 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -8,8 +8,8 @@ export LC_ALL=C GIT_HEAD=$(git rev-parse HEAD) if [ -n "$CIRRUS_PR" ]; then - COMMIT_RANGE="$CIRRUS_BASE_SHA..$GIT_HEAD" - test/lint/commit-script-check.sh $COMMIT_RANGE + COMMIT_RANGE="${CIRRUS_BASE_SHA}..$GIT_HEAD" + test/lint/commit-script-check.sh "$COMMIT_RANGE" fi export COMMIT_RANGE @@ -17,6 +17,7 @@ export COMMIT_RANGE # check with -r to not have to fetch all the remotes. test/lint/git-subtree-check.sh src/crypto/ctaes test/lint/git-subtree-check.sh src/secp256k1 +test/lint/git-subtree-check.sh src/minisketch test/lint/git-subtree-check.sh src/univalue test/lint/git-subtree-check.sh src/leveldb test/lint/git-subtree-check.sh src/crc32c @@ -30,9 +31,12 @@ if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ "$CIRRUS_PR" = "" ] ; t # sanity checking only a few (10) commits seems sufficient and cheap. git log HEAD~10 -1 --format='%H' > ./contrib/verify-commits/trusted-sha512-root-commit git log HEAD~10 -1 --format='%H' > ./contrib/verify-commits/trusted-git-root - ${CI_RETRY_EXE} gpg --keyserver hkps://keys.openpgp.org --recv-keys $(<contrib/verify-commits/trusted-keys) && + mapfile -t KEYS < contrib/verify-commits/trusted-keys + ${CI_RETRY_EXE} gpg --keyserver hkps://keys.openpgp.org --recv-keys "${KEYS[@]}" && ./contrib/verify-commits/verify-commits.py; fi -echo -git log --no-merges --oneline $COMMIT_RANGE +if [ -n "$COMMIT_RANGE" ]; then + echo + git log --no-merges --oneline "$COMMIT_RANGE" +fi diff --git a/ci/test/00_setup_env_android.sh b/ci/test/00_setup_env_android.sh index 2f9d1f2a9f..e0b574aa20 100755 --- a/ci/test/00_setup_env_android.sh +++ b/ci/test/00_setup_env_android.sh @@ -16,10 +16,10 @@ export RUN_FUNCTIONAL_TESTS=false export ANDROID_API_LEVEL=28 export ANDROID_BUILD_TOOLS_VERSION=28.0.3 -export ANDROID_NDK_VERSION=22.1.7171670 +export ANDROID_NDK_VERSION=23.1.7779620 export ANDROID_TOOLS_URL=https://dl.google.com/android/repository/commandlinetools-linux-6609375_latest.zip export ANDROID_HOME="${DEPENDS_DIR}/SDKs/android" export ANDROID_NDK_HOME="${ANDROID_HOME}/ndk/${ANDROID_NDK_VERSION}" export DEP_OPTS="ANDROID_SDK=${ANDROID_HOME} ANDROID_NDK=${ANDROID_NDK_HOME} ANDROID_API_LEVEL=${ANDROID_API_LEVEL} ANDROID_TOOLCHAIN_BIN=${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/linux-x86_64/bin/" -export BITCOIN_CONFIG="--disable-ccache --disable-tests --enable-gui-tests --disable-bench --disable-fuzz-binary --without-utils --without-libs --without-daemon" +export BITCOIN_CONFIG="--disable-tests --enable-gui-tests --disable-bench --disable-fuzz-binary --without-utils --without-libs --without-daemon" diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index ab185b6e71..947c4b2891 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_asan export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev" -export DOCKER_NAME_TAG=ubuntu:hirsute +export DOCKER_NAME_TAG=ubuntu:22.04 export NO_DEPENDS=1 export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index b8ac691346..9c34cfdf50 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export DOCKER_NAME_TAG="ubuntu:20.04" export CONTAINER_NAME=ci_native_fuzz -export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev" +export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libsqlite3-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index 2cf672b91e..673326ded7 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export DOCKER_NAME_TAG="ubuntu:20.04" export CONTAINER_NAME=ci_native_fuzz_valgrind -export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev valgrind" +export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libsqlite3-dev valgrind" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 8176179f0b..14b7b98782 100755 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -15,5 +15,5 @@ export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1 v0.20.1" -export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-reduce-exports +export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-reduce-exports \ --enable-debug --disable-fuzz-binary CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" CC=gcc-8 CXX=g++-8" diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index a5082bdaab..5d0880ff4a 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan -export DOCKER_NAME_TAG=ubuntu:hirsute +export DOCKER_NAME_TAG=ubuntu:22.04 export PACKAGES="clang llvm libc++abi-dev libc++-dev python3-zmq" export DEP_OPTS="CC=clang CXX='clang++ -stdlib=libc++'" export GOAL="install" diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 2079d2ed2b..8a416513f9 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -12,6 +12,7 @@ fi if [ "$CI_OS_NAME" == "macos" ]; then sudo -H pip3 install --upgrade pip + # shellcheck disable=SC2086 IN_GETOPT_BIN="/usr/local/opt/gnu-getopt/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES fi @@ -39,6 +40,7 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then systemctl restart docker fi + # shellcheck disable=SC2086 DOCKER_ID=$(docker run $DOCKER_ADMIN --rm --interactive --detach --tty \ --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \ --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \ @@ -54,7 +56,7 @@ else fi DOCKER_EXEC () { - $DOCKER_CI_CMD_PREFIX bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*" + $DOCKER_CI_CMD_PREFIX bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd \"$P_CI_DIR\" && $*" } export -f DOCKER_EXEC @@ -64,11 +66,12 @@ fi if [[ $DOCKER_NAME_TAG == centos* ]]; then ${CI_RETRY_EXE} DOCKER_EXEC dnf -y install epel-release - ${CI_RETRY_EXE} DOCKER_EXEC dnf -y --allowerasing install $DOCKER_PACKAGES $PACKAGES + ${CI_RETRY_EXE} DOCKER_EXEC dnf -y --allowerasing install "$DOCKER_PACKAGES" "$PACKAGES" elif [ "$CI_USE_APT_INSTALL" != "no" ]; then ${CI_RETRY_EXE} DOCKER_EXEC apt-get update - ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES + ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y "$PACKAGES" "$DOCKER_PACKAGES" if [ -n "$PIP_PACKAGES" ]; then + # shellcheck disable=SC2086 ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES fi fi @@ -79,14 +82,14 @@ if [ "$CI_OS_NAME" == "macos" ]; then else DOCKER_EXEC free -m -h DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) - DOCKER_EXEC echo $(lscpu | grep Endian) + DOCKER_EXEC echo "$(lscpu | grep Endian)" fi DOCKER_EXEC echo "Free disk space:" DOCKER_EXEC df -h if [ "$RUN_FUZZ_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then - if [ ! -d ${DIR_QA_ASSETS} ]; then - DOCKER_EXEC git clone --depth=1 https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} + if [ ! -d "${DIR_QA_ASSETS}" ]; then + DOCKER_EXEC git clone --depth=1 https://github.com/bitcoin-core/qa-assets "${DIR_QA_ASSETS}" fi export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ @@ -106,17 +109,17 @@ fi if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then echo "Create $BASE_ROOT_DIR" - DOCKER_EXEC rsync -a /ro_base/ $BASE_ROOT_DIR + DOCKER_EXEC rsync -a /ro_base/ "$BASE_ROOT_DIR" fi if [ "$USE_BUSY_BOX" = "true" ]; then echo "Setup to use BusyBox utils" - DOCKER_EXEC mkdir -p $BASE_SCRATCH_DIR/bins/ + DOCKER_EXEC mkdir -p "${BASE_SCRATCH_DIR}/bins/" # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version) # find excluded for now because it does not recognize the -delete option in ./depends (fixed in later BusyBox version) # ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed) # shellcheck disable=SC1010 - DOCKER_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \| grep -v "^find$"\)\; do ln -s \$\(command -v busybox\) $BASE_SCRATCH_DIR/bins/\$util\; done + DOCKER_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \| grep -v "^find$"\)\; do ln -s \$\(command -v busybox\) "${BASE_SCRATCH_DIR}/bins/\$util"\; done # Print BusyBox version DOCKER_EXEC patch --help fi diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index 8dd489d7f8..d8c23bd26b 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -8,24 +8,29 @@ export LC_ALL=C.UTF-8 # Make sure default datadir does not exist and is never read by creating a dummy file if [ "$CI_OS_NAME" == "macos" ]; then - echo > $HOME/Library/Application\ Support/Bitcoin + echo > "${HOME}/Library/Application Support/Bitcoin" else DOCKER_EXEC echo \> \$HOME/.bitcoin fi -DOCKER_EXEC mkdir -p ${DEPENDS_DIR}/SDKs ${DEPENDS_DIR}/sdk-sources +DOCKER_EXEC mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources" -OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz" -OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_BASENAME}" +OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers" -if [ -n "$XCODE_VERSION" ] && [ ! -f "$OSX_SDK_PATH" ]; then - DOCKER_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH" +if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then + OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz" + OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}" + if [ ! -f "$OSX_SDK_PATH" ]; then + DOCKER_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH" + fi + DOCKER_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" fi -if [ -n "$ANDROID_TOOLS_URL" ]; then - ANDROID_TOOLS_PATH=$DEPENDS_DIR/sdk-sources/android-tools.zip - - DOCKER_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH" +if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then + ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip + if [ ! -f "$ANDROID_TOOLS_PATH" ]; then + DOCKER_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH" + fi DOCKER_EXEC mkdir -p "${ANDROID_HOME}/cmdline-tools" DOCKER_EXEC unzip -o "$ANDROID_TOOLS_PATH" -d "${ANDROID_HOME}/cmdline-tools" DOCKER_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/tools/bin/sdkmanager --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\"" @@ -38,11 +43,8 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then DOCKER_EXEC "contrib/install_db4.sh \$(pwd) --enable-umrw CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" fi -if [ -n "$XCODE_VERSION" ] && [ -f "$OSX_SDK_PATH" ]; then - DOCKER_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" -fi if [[ $HOST = *-mingw32 ]]; then - DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\) + DOCKER_EXEC update-alternatives --set "${HOST}-g++" \$\(which "${HOST}-g++-posix"\) fi if [ -z "$NO_DEPENDS" ]; then if [[ $DOCKER_NAME_TAG == centos* ]]; then @@ -53,7 +55,7 @@ if [ -z "$NO_DEPENDS" ]; then else SHELL_OPTS="CONFIG_SHELL=" fi - DOCKER_EXEC $SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS + DOCKER_EXEC "$SHELL_OPTS" make "$MAKEJOBS" -C depends HOST="$HOST" "$DEP_OPTS" fi if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then DOCKER_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}" diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index b1d83883d1..c63724d585 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8 if [ -n "$ANDROID_TOOLS_URL" ]; then DOCKER_EXEC make distclean || true DOCKER_EXEC ./autogen.sh - DOCKER_EXEC ./configure $BITCOIN_CONFIG --prefix=$DEPENDS_DIR/aarch64-linux-android || ( (DOCKER_EXEC cat config.log) && false) + DOCKER_EXEC ./configure "$BITCOIN_CONFIG" --prefix="${DEPENDS_DIR}/aarch64-linux-android" || ( (DOCKER_EXEC cat config.log) && false) DOCKER_EXEC "make $MAKEJOBS && cd src/qt && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk" exit 0 fi @@ -29,13 +29,13 @@ fi DOCKER_EXEC mkdir -p "${BASE_BUILD_DIR}" export P_CI_DIR="${BASE_BUILD_DIR}" -DOCKER_EXEC "${BASE_ROOT_DIR}/configure" --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) +DOCKER_EXEC "${BASE_ROOT_DIR}/configure" --cache-file=config.cache "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (DOCKER_EXEC cat config.log) && false) -DOCKER_EXEC make distdir VERSION=$HOST +DOCKER_EXEC make distdir VERSION="$HOST" export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST" -DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) +DOCKER_EXEC ./configure --cache-file=../config.cache "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (DOCKER_EXEC cat config.log) && false) set -o errtrace trap 'DOCKER_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR @@ -48,7 +48,7 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then DOCKER_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h' fi -DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false ) +DOCKER_EXEC make "$MAKEJOBS" "$GOAL" || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make "$GOAL" V=1 ; false ) DOCKER_EXEC "ccache --version | head -n 1 && ccache --show-stats" DOCKER_EXEC du -sh "${DEPENDS_DIR}"/*/ diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 311a43755a..ce9252026a 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -8,13 +8,13 @@ export LC_ALL=C.UTF-8 if [[ $HOST = *-mingw32 ]]; then # Generate all binaries, so that they can be wrapped - DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1 + DOCKER_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" fi if [ -n "$QEMU_USER_CMD" ]; then # Generate all binaries, so that they can be wrapped - DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1 + DOCKER_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" fi @@ -23,15 +23,15 @@ if [ -n "$USE_VALGRIND" ]; then fi if [ "$RUN_UNIT_TESTS" = "true" ]; then - DOCKER_EXEC ${TEST_RUNNER_ENV} DIR_UNIT_TEST_DATA=${DIR_UNIT_TEST_DATA} LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib make $MAKEJOBS check VERBOSE=1 + DOCKER_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" make "$MAKEJOBS" check VERBOSE=1 fi if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then - DOCKER_EXEC ${TEST_RUNNER_ENV} DIR_UNIT_TEST_DATA=${DIR_UNIT_TEST_DATA} LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib "${BASE_BUILD_DIR}/bitcoin-*/src/test/test_bitcoin*" --catch_system_errors=no -l test_suite + DOCKER_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${BASE_BUILD_DIR}/bitcoin-*/src/test/test_bitcoin*" --catch_system_errors=no -l test_suite fi if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then - DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ${TEST_RUNNER_ENV} test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA} --quiet --failfast + DOCKER_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${TEST_RUNNER_ENV}" test/functional/test_runner.py --ci "$MAKEJOBS" --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" "${TEST_RUNNER_EXTRA}" --quiet --failfast fi if [ "$RUN_SECURITY_TESTS" = "true" ]; then @@ -39,5 +39,5 @@ if [ "$RUN_SECURITY_TESTS" = "true" ]; then fi if [ "$RUN_FUZZ_TESTS" = "true" ]; then - DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} $MAKEJOBS -l DEBUG ${DIR_FUZZ_IN} + DOCKER_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" test/fuzz/test_runner.py "${FUZZ_TESTS_CONFIG}" "$MAKEJOBS" -l DEBUG "${DIR_FUZZ_IN}" fi diff --git a/ci/test/wrap-qemu.sh b/ci/test/wrap-qemu.sh index 2cd7c8cec2..320fca78ed 100755 --- a/ci/test/wrap-qemu.sh +++ b/ci/test/wrap-qemu.sh @@ -6,9 +6,9 @@ export LC_ALL=C.UTF-8 -for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}; do +for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{no_nul,test_json,unitester,object}}; do # shellcheck disable=SC2044 - for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name $(basename $b_name)); do + for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do echo "Wrap $b ..." mv "$b" "${b}_orig" echo '#!/usr/bin/env bash' > "$b" diff --git a/ci/test/wrap-valgrind.sh b/ci/test/wrap-valgrind.sh index 6b3e6eb7e7..38cc5e0ca7 100755 --- a/ci/test/wrap-valgrind.sh +++ b/ci/test/wrap-valgrind.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 for b_name in "${BASE_OUTDIR}/bin"/*; do # shellcheck disable=SC2044 - for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name $(basename $b_name)); do + for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do echo "Wrap $b ..." mv "$b" "${b}_orig" echo '#!/usr/bin/env bash' > "$b" diff --git a/ci/test/wrap-wine.sh b/ci/test/wrap-wine.sh index 82964897e1..2550abdb40 100755 --- a/ci/test/wrap-wine.sh +++ b/ci/test/wrap-wine.sh @@ -6,9 +6,9 @@ export LC_ALL=C.UTF-8 -for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}.exe; do +for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{no_nul,test_json,unitester,object}}.exe; do # shellcheck disable=SC2044 - for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename $b_name)"); do + for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do if (file "$b" | grep "Windows"); then echo "Wrap $b ..." mv "$b" "${b}_orig" diff --git a/configure.ac b/configure.ac index dc3b07e4bb..45def0908b 100644 --- a/configure.ac +++ b/configure.ac @@ -15,7 +15,7 @@ AC_CONFIG_MACRO_DIR([build-aux/m4]) m4_ifndef([PKG_PROG_PKG_CONFIG], [AC_MSG_ERROR([PKG_PROG_PKG_CONFIG macro not found. Please install pkg-config and re-run autogen.sh])]) PKG_PROG_PKG_CONFIG -if test "x$PKG_CONFIG" = x; then +if test "$PKG_CONFIG" = ""; then AC_MSG_ERROR([pkg-config not found]) fi @@ -30,8 +30,8 @@ BITCOIN_MP_NODE_NAME=bitcoin-node BITCOIN_MP_GUI_NAME=bitcoin-gui dnl Unless the user specified ARFLAGS, force it to be cr -AC_ARG_VAR(ARFLAGS, [Flags for the archiver, defaults to <cr> if not set]) -if test "x${ARFLAGS+set}" != "xset"; then +AC_ARG_VAR([ARFLAGS], [Flags for the archiver, defaults to <cr> if not set]) +if test "${ARFLAGS+set}" != "set"; then ARFLAGS="cr" fi @@ -55,7 +55,7 @@ dnl make the compilation flags quiet unless V=1 is used AM_SILENT_RULES([yes]) dnl Compiler checks (here before libtool). -if test "x${CXXFLAGS+set}" = "xset"; then +if test "${CXXFLAGS+set}" = "set"; then CXXFLAGS_overridden=yes else CXXFLAGS_overridden=no @@ -86,7 +86,7 @@ CHECK_ATOMIC dnl Unless the user specified OBJCXX, force it to be the same as CXX. This ensures dnl that we get the same -std flags for both. m4_ifdef([AC_PROG_OBJCXX],[ -if test "x${OBJCXX+set}" = "x"; then +if test "${OBJCXX+set}" = ""; then OBJCXX="${CXX}" fi AC_PROG_OBJCXX @@ -100,24 +100,24 @@ dnl Libtool init checks. LT_INIT([pic-only]) dnl Check/return PATH for base programs. -AC_PATH_TOOL(AR, ar) -AC_PATH_TOOL(RANLIB, ranlib) -AC_PATH_TOOL(STRIP, strip) -AC_PATH_TOOL(GCOV, gcov) -AC_PATH_TOOL(LLVM_COV, llvm-cov) -AC_PATH_PROG(LCOV, lcov) +AC_PATH_TOOL([AR], [ar]) +AC_PATH_TOOL([RANLIB], [ranlib]) +AC_PATH_TOOL([STRIP], [strip]) +AC_PATH_TOOL([GCOV], [gcov]) +AC_PATH_TOOL([LLVM_COV], [llvm-cov]) +AC_PATH_PROG([LCOV], [lcov]) dnl Python 3.6 is specified in .python-version and should be used if available, see doc/dependencies.md AC_PATH_PROGS([PYTHON], [python3.6 python3.7 python3.8 python3.9 python3.10 python3.11 python3 python]) -AC_PATH_PROG(GENHTML, genhtml) +AC_PATH_PROG([GENHTML], [genhtml]) AC_PATH_PROG([GIT], [git]) -AC_PATH_PROG(CCACHE,ccache) -AC_PATH_PROG(XGETTEXT,xgettext) -AC_PATH_PROG(HEXDUMP,hexdump) -AC_PATH_TOOL(OBJCOPY, objcopy) -AC_PATH_PROG(DOXYGEN, doxygen) +AC_PATH_PROG([CCACHE], [ccache]) +AC_PATH_PROG([XGETTEXT], [xgettext]) +AC_PATH_PROG([HEXDUMP], [hexdump]) +AC_PATH_TOOL([OBJCOPY], [objcopy]) +AC_PATH_PROG([DOXYGEN], [doxygen]) AM_CONDITIONAL([HAVE_DOXYGEN], [test -n "$DOXYGEN"]) -AC_ARG_VAR(PYTHONPATH, Augments the default search path for python module files) +AC_ARG_VAR([PYTHONPATH], [Augments the default search path for python module files]) AC_ARG_ENABLE([wallet], [AS_HELP_STRING([--disable-wallet], @@ -257,8 +257,8 @@ AC_ARG_ENABLE([asm], [use_asm=$enableval], [use_asm=yes]) -if test "x$use_asm" = xyes; then - AC_DEFINE(USE_ASM, 1, [Define this symbol to build in assembly routines]) +if test "$use_asm" = "yes"; then + AC_DEFINE([USE_ASM], [1], [Define this symbol to build in assembly routines]) fi AC_ARG_ENABLE([zmq], @@ -289,7 +289,7 @@ AC_ARG_ENABLE(man, [AS_HELP_STRING([--disable-man], [do not install man pages (default is to install)])],, enable_man=yes) -AM_CONDITIONAL(ENABLE_MAN, test "$enable_man" != no) +AM_CONDITIONAL([ENABLE_MAN], [test "$enable_man" != "no"]) dnl Enable debug AC_ARG_ENABLE([debug], @@ -323,6 +323,11 @@ AC_ARG_ENABLE([external-signer], [use_external_signer=$enableval], [use_external_signer=yes]) +AC_ARG_ENABLE([lto], + [AS_HELP_STRING([--enable-lto],[build using LTO (default is no)])], + [enable_lto=$enableval], + [enable_lto=no]) + AC_LANG_PUSH([C++]) dnl Check for a flag to turn compiler warnings into errors. This is helpful for checks which may @@ -332,7 +337,7 @@ dnl Note that this is not necessarily a check to see if -Werror is supported, bu dnl a compile with -Werror can succeed. This is important because the compiler may already be dnl warning about something unrelated, for example about some path issue. If that is the case, dnl -Werror cannot be used because all of those warnings would be turned into errors. -AX_CHECK_COMPILE_FLAG([-Werror],[CXXFLAG_WERROR="-Werror"],[CXXFLAG_WERROR=""]) +AX_CHECK_COMPILE_FLAG([-Werror], [CXXFLAG_WERROR="-Werror"], [CXXFLAG_WERROR=""]) dnl Check for a flag to turn linker warnings into errors. When flags are passed to linkers via the dnl compiler driver using a -Wl,-foo flag, linker warnings may be swallowed rather than bubbling up. @@ -341,42 +346,47 @@ dnl dnl LDFLAG_WERROR Should only be used when testing -Wl,* case $host in *darwin*) - AX_CHECK_LINK_FLAG([-Wl,-fatal_warnings],[LDFLAG_WERROR="-Wl,-fatal_warnings"],[LDFLAG_WERROR=""]) + AX_CHECK_LINK_FLAG([-Wl,-fatal_warnings], [LDFLAG_WERROR="-Wl,-fatal_warnings"], [LDFLAG_WERROR=""]) ;; *) - AX_CHECK_LINK_FLAG([-Wl,--fatal-warnings],[LDFLAG_WERROR="-Wl,--fatal-warnings"],[LDFLAG_WERROR=""]) + AX_CHECK_LINK_FLAG([-Wl,--fatal-warnings], [LDFLAG_WERROR="-Wl,--fatal-warnings"], [LDFLAG_WERROR=""]) ;; esac -if test "x$enable_debug" = xyes; then +if test "$enable_debug" = "yes"; then dnl Clear default -g -O2 flags - if test "x$CXXFLAGS_overridden" = xno; then + if test "$CXXFLAGS_overridden" = "no"; then CXXFLAGS="" fi dnl Disable all optimizations - AX_CHECK_COMPILE_FLAG([-O0], [[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -O0"]],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-O0], [DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -O0"], [], [$CXXFLAG_WERROR]) dnl Prefer -g3, fall back to -g if that is unavailable. AX_CHECK_COMPILE_FLAG( [-g3], - [[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -g3"]], - [AX_CHECK_COMPILE_FLAG([-g],[[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -g"]],,[[$CXXFLAG_WERROR]])], - [[$CXXFLAG_WERROR]]) - - AX_CHECK_PREPROC_FLAG([-DDEBUG],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG"]],,[[$CXXFLAG_WERROR]]) - AX_CHECK_PREPROC_FLAG([-DDEBUG_LOCKORDER],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG_LOCKORDER"]],,[[$CXXFLAG_WERROR]]) - AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"]],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-ftrapv],[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -ftrapv"],,[[$CXXFLAG_WERROR]]) + [DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -g3"], + [AX_CHECK_COMPILE_FLAG([-g], [DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -g"], [], [$CXXFLAG_WERROR])], + [$CXXFLAG_WERROR]) + + AX_CHECK_PREPROC_FLAG([-DDEBUG], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG"], [], [$CXXFLAG_WERROR]) + AX_CHECK_PREPROC_FLAG([-DDEBUG_LOCKORDER], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG_LOCKORDER"], [], [$CXXFLAG_WERROR]) + AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-ftrapv], [DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -ftrapv"], [], [$CXXFLAG_WERROR]) +fi + +if test "$enable_lto" = "yes"; then + AX_CHECK_COMPILE_FLAG([-flto], [LTO_CXXFLAGS="$LTO_CXXFLAGS -flto"], [AC_MSG_ERROR([compile failed with -flto])], [$CXXFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-flto], [LTO_LDFLAGS="$LTO_LDFLAGS -flto"], [AC_MSG_ERROR([link failed with -flto])], [$CXXFLAG_WERROR]) fi -if test x$use_sanitizers != x; then +if test "$use_sanitizers" != ""; then dnl First check if the compiler accepts flags. If an incompatible pair like dnl -fsanitize=address,thread is used here, this check will fail. This will also dnl fail if a bad argument is passed, e.g. -fsanitize=undfeined AX_CHECK_COMPILE_FLAG( - [[-fsanitize=$use_sanitizers]], - [[SANITIZER_CXXFLAGS=-fsanitize=$use_sanitizers]], + [-fsanitize=$use_sanitizers], + [SANITIZER_CXXFLAGS="-fsanitize=$use_sanitizers"], [AC_MSG_ERROR([compiler did not accept requested flags])]) dnl Some compilers (e.g. GCC) require additional libraries like libasan, @@ -385,8 +395,8 @@ if test x$use_sanitizers != x; then dnl the sanitize flags are supported by the compiler but the actual sanitizer dnl libs are missing. AX_CHECK_LINK_FLAG( - [[-fsanitize=$use_sanitizers]], - [[SANITIZER_LDFLAGS=-fsanitize=$use_sanitizers]], + [-fsanitize=$use_sanitizers], + [SANITIZER_LDFLAGS="-fsanitize=$use_sanitizers"], [AC_MSG_ERROR([linker did not accept requested flags, you are missing required libraries])], [], [AC_LANG_PROGRAM([[ @@ -398,9 +408,9 @@ if test x$use_sanitizers != x; then fi ERROR_CXXFLAGS= -if test "x$enable_werror" = "xyes"; then - if test "x$CXXFLAG_WERROR" = "x"; then - AC_MSG_ERROR("enable-werror set but -Werror is not usable") +if test "$enable_werror" = "yes"; then + if test "$CXXFLAG_WERROR" = ""; then + AC_MSG_ERROR([enable-werror set but -Werror is not usable]) fi ERROR_CXXFLAGS=$CXXFLAG_WERROR @@ -411,68 +421,86 @@ if test "x$enable_werror" = "xyes"; then int f(){ assert(false); }]])]) fi -if test "x$CXXFLAGS_overridden" = "xno"; then - AX_CHECK_COMPILE_FLAG([-Wall],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wextra],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wextra"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wgnu],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wgnu"],,[[$CXXFLAG_WERROR]]) +if test "$CXXFLAGS_overridden" = "no"; then + AX_CHECK_COMPILE_FLAG([-Wall], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wextra], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wextra"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wgnu], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wgnu"], [], [$CXXFLAG_WERROR]) dnl some compilers will ignore -Wformat-security without -Wformat, so just combine the two here. - AX_CHECK_COMPILE_FLAG([-Wformat -Wformat-security],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat -Wformat-security"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wvla],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wvla"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wshadow-field],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wshadow-field"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wthread-safety],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wloop-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wloop-analysis"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wredundant-decls],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wunused-member-function],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-member-function"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wdate-time],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdate-time"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wconditional-uninitialized],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wconditional-uninitialized"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wduplicated-branches],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-branches"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wduplicated-cond],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-cond"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wlogical-op],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wlogical-op"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Woverloaded-virtual],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Woverloaded-virtual"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wformat -Wformat-security], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat -Wformat-security"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wvla], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wvla"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wshadow-field], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wshadow-field"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wthread-safety], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wloop-analysis], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wredundant-decls], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wunused-member-function], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-member-function"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wdate-time], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdate-time"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wconditional-uninitialized], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wconditional-uninitialized"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wduplicated-branches], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-branches"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wduplicated-cond], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-cond"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wlogical-op], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wlogical-op"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Woverloaded-virtual], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Woverloaded-virtual"], [], [$CXXFLAG_WERROR]) dnl -Wsuggest-override is broken with GCC before 9.2 dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78010 - AX_CHECK_COMPILE_FLAG([-Wsuggest-override],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsuggest-override"],,[[$CXXFLAG_WERROR]], + AX_CHECK_COMPILE_FLAG([-Wsuggest-override], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsuggest-override"], [], [$CXXFLAG_WERROR], [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) - AX_CHECK_COMPILE_FLAG([-Wunreachable-code-loop-increment],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunreachable-code-loop-increment"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wunreachable-code-loop-increment], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunreachable-code-loop-increment"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wimplicit-fallthrough"], [], [$CXXFLAG_WERROR]) - if test x$suppress_external_warnings != xno ; then - AX_CHECK_COMPILE_FLAG([-Wdocumentation],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdocumentation"],,[[$CXXFLAG_WERROR]]) + if test "$suppress_external_warnings" != "no" ; then + AX_CHECK_COMPILE_FLAG([-Wdocumentation], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdocumentation"], [], [$CXXFLAG_WERROR]) fi dnl Some compilers (gcc) ignore unknown -Wno-* options, but warn about all dnl unknown options if any other warning is produced. Test the -Wfoo case, and dnl set the -Wno-foo case if it works. - AX_CHECK_COMPILE_FLAG([-Wunused-parameter],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-parameter"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wself-assign],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-self-assign"],,[[$CXXFLAG_WERROR]]) - if test x$suppress_external_warnings != xyes ; then - AX_CHECK_COMPILE_FLAG([-Wdeprecated-copy],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-deprecated-copy"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wunused-parameter], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-parameter"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wself-assign], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-self-assign"], [], [$CXXFLAG_WERROR]) + if test "$suppress_external_warnings" != "yes" ; then + AX_CHECK_COMPILE_FLAG([-Wdeprecated-copy], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-deprecated-copy"], [], [$CXXFLAG_WERROR]) fi fi dnl Don't allow extended (non-ASCII) symbols in identifiers. This is easier for code review. -AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers],[[CXXFLAGS="$CXXFLAGS -fno-extended-identifiers"]],,[[$CXXFLAG_WERROR]]) +AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers], [CXXFLAGS="$CXXFLAGS -fno-extended-identifiers"], [], [$CXXFLAG_WERROR]) enable_sse42=no enable_sse41=no enable_avx2=no enable_shani=no -if test "x$use_asm" = "xyes"; then +if test "$use_asm" = "yes"; then dnl Check for optional instruction set support. Enabling these does _not_ imply that all code will dnl be compiled with them, rather that specific objects/libs may use them after checking for runtime dnl compatibility. dnl x86 -AX_CHECK_COMPILE_FLAG([-msse4.2],[[SSE42_CXXFLAGS="-msse4.2"]],,[[$CXXFLAG_WERROR]]) -AX_CHECK_COMPILE_FLAG([-msse4.1],[[SSE41_CXXFLAGS="-msse4.1"]],,[[$CXXFLAG_WERROR]]) -AX_CHECK_COMPILE_FLAG([-mavx -mavx2],[[AVX2_CXXFLAGS="-mavx -mavx2"]],,[[$CXXFLAG_WERROR]]) -AX_CHECK_COMPILE_FLAG([-msse4 -msha],[[SHANI_CXXFLAGS="-msse4 -msha"]],,[[$CXXFLAG_WERROR]]) +AX_CHECK_COMPILE_FLAG([-msse4.2], [SSE42_CXXFLAGS="-msse4.2"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-msse4.1], [SSE41_CXXFLAGS="-msse4.1"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-mavx -mavx2], [AVX2_CXXFLAGS="-mavx -mavx2"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-msse4 -msha], [SHANI_CXXFLAGS="-msse4 -msha"], [], [$CXXFLAG_WERROR]) + +enable_clmul= +AX_CHECK_COMPILE_FLAG([-mpclmul], [enable_clmul=yes], [], [$CXXFLAG_WERROR], [AC_LANG_PROGRAM([ + #include <stdint.h> + #include <x86intrin.h> +], [ + __m128i a = _mm_cvtsi64_si128((uint64_t)7); + __m128i b = _mm_clmulepi64_si128(a, a, 37); + __m128i c = _mm_srli_epi64(b, 41); + __m128i d = _mm_xor_si128(b, c); + uint64_t e = _mm_cvtsi128_si64(d); + return e == 0; +])]) + +if test "$enable_clmul" = "yes"; then + CLMUL_CXXFLAGS="-mpclmul" + AC_DEFINE([HAVE_CLMUL], [1], [Define this symbol if clmul instructions can be used]) +fi TEMP_CXXFLAGS="$CXXFLAGS" CXXFLAGS="$CXXFLAGS $SSE42_CXXFLAGS" -AC_MSG_CHECKING(for SSE4.2 intrinsics) +AC_MSG_CHECKING([for SSE4.2 intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include <stdint.h> #if defined(_MSC_VER) @@ -487,14 +515,14 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ l = _mm_crc32_u64(l, 0); return l; ]])], - [ AC_MSG_RESULT(yes); enable_sse42=yes], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); enable_sse42=yes], + [ AC_MSG_RESULT([no])] ) CXXFLAGS="$TEMP_CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS" CXXFLAGS="$CXXFLAGS $SSE41_CXXFLAGS" -AC_MSG_CHECKING(for SSE4.1 intrinsics) +AC_MSG_CHECKING([for SSE4.1 intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include <stdint.h> #include <immintrin.h> @@ -502,14 +530,14 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ __m128i l = _mm_set1_epi32(0); return _mm_extract_epi32(l, 3); ]])], - [ AC_MSG_RESULT(yes); enable_sse41=yes; AC_DEFINE(ENABLE_SSE41, 1, [Define this symbol to build code that uses SSE4.1 intrinsics]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); enable_sse41=yes; AC_DEFINE([ENABLE_SSE41], [1], [Define this symbol to build code that uses SSE4.1 intrinsics]) ], + [ AC_MSG_RESULT([no])] ) CXXFLAGS="$TEMP_CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS" CXXFLAGS="$CXXFLAGS $AVX2_CXXFLAGS" -AC_MSG_CHECKING(for AVX2 intrinsics) +AC_MSG_CHECKING([for AVX2 intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include <stdint.h> #include <immintrin.h> @@ -517,14 +545,14 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ __m256i l = _mm256_set1_epi32(0); return _mm256_extract_epi32(l, 7); ]])], - [ AC_MSG_RESULT(yes); enable_avx2=yes; AC_DEFINE(ENABLE_AVX2, 1, [Define this symbol to build code that uses AVX2 intrinsics]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); enable_avx2=yes; AC_DEFINE([ENABLE_AVX2], [1], [Define this symbol to build code that uses AVX2 intrinsics]) ], + [ AC_MSG_RESULT([no])] ) CXXFLAGS="$TEMP_CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS" CXXFLAGS="$CXXFLAGS $SHANI_CXXFLAGS" -AC_MSG_CHECKING(for SHA-NI intrinsics) +AC_MSG_CHECKING([for SHA-NI intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include <stdint.h> #include <immintrin.h> @@ -534,17 +562,17 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ __m128i k = _mm_set1_epi32(2); return _mm_extract_epi32(_mm_sha256rnds2_epu32(i, i, k), 0); ]])], - [ AC_MSG_RESULT(yes); enable_shani=yes; AC_DEFINE(ENABLE_SHANI, 1, [Define this symbol to build code that uses SHA-NI intrinsics]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); enable_shani=yes; AC_DEFINE([ENABLE_SHANI], [1], [Define this symbol to build code that uses SHA-NI intrinsics]) ], + [ AC_MSG_RESULT([no])] ) CXXFLAGS="$TEMP_CXXFLAGS" # ARM -AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto],[[ARM_CRC_CXXFLAGS="-march=armv8-a+crc+crypto"]],,[[$CXXFLAG_WERROR]]) +AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto], [ARM_CRC_CXXFLAGS="-march=armv8-a+crc+crypto"], [], [$CXXFLAG_WERROR]) TEMP_CXXFLAGS="$CXXFLAGS" CXXFLAGS="$CXXFLAGS $ARM_CRC_CXXFLAGS" -AC_MSG_CHECKING(for AArch64 CRC32 intrinsics) +AC_MSG_CHECKING([for AArch64 CRC32 intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include <arm_acle.h> #include <arm_neon.h> @@ -556,14 +584,14 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #error "crc32c library does not support hardware acceleration on 32-bit ARM" #endif ]])], - [ AC_MSG_RESULT(yes); enable_arm_crc=yes; ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); enable_arm_crc=yes; ], + [ AC_MSG_RESULT([no])] ) CXXFLAGS="$TEMP_CXXFLAGS" fi -CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO -D__STDC_FORMAT_MACROS" +CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO" AC_ARG_WITH([utils], [AS_HELP_STRING([--with-utils], @@ -610,33 +638,33 @@ AC_ARG_WITH([daemon], case $host in *mingw*) TARGET_OS=windows - AC_CHECK_LIB([kernel32], [GetModuleFileNameA],, AC_MSG_ERROR(libkernel32 missing)) - AC_CHECK_LIB([user32], [main],, AC_MSG_ERROR(libuser32 missing)) - AC_CHECK_LIB([gdi32], [main],, AC_MSG_ERROR(libgdi32 missing)) - AC_CHECK_LIB([comdlg32], [main],, AC_MSG_ERROR(libcomdlg32 missing)) - AC_CHECK_LIB([winmm], [main],, AC_MSG_ERROR(libwinmm missing)) - AC_CHECK_LIB([shell32], [SHGetSpecialFolderPathW],, AC_MSG_ERROR(libshell32 missing)) - AC_CHECK_LIB([comctl32], [main],, AC_MSG_ERROR(libcomctl32 missing)) - AC_CHECK_LIB([ole32], [CoCreateInstance],, AC_MSG_ERROR(libole32 missing)) - AC_CHECK_LIB([oleaut32], [main],, AC_MSG_ERROR(liboleaut32 missing)) - AC_CHECK_LIB([uuid], [main],, AC_MSG_ERROR(libuuid missing)) - AC_CHECK_LIB([advapi32], [CryptAcquireContextW],, AC_MSG_ERROR(libadvapi32 missing)) - AC_CHECK_LIB([ws2_32], [WSAStartup],, AC_MSG_ERROR(libws2_32 missing)) - AC_CHECK_LIB([shlwapi], [PathRemoveFileSpecW],, AC_MSG_ERROR(libshlwapi missing)) - AC_CHECK_LIB([iphlpapi], [GetAdaptersAddresses],, AC_MSG_ERROR(libiphlpapi missing)) + AC_CHECK_LIB([kernel32], [GetModuleFileNameA], [], [AC_MSG_ERROR([libkernel32 missing])]) + AC_CHECK_LIB([user32], [main], [], [AC_MSG_ERROR([libuser32 missing])]) + AC_CHECK_LIB([gdi32], [main], [], [AC_MSG_ERROR([libgdi32 missing])]) + AC_CHECK_LIB([comdlg32], [main], [], [AC_MSG_ERROR([libcomdlg32 missing])]) + AC_CHECK_LIB([winmm], [main], [], [AC_MSG_ERROR([libwinmm missing])]) + AC_CHECK_LIB([shell32], [SHGetSpecialFolderPathW], [], [AC_MSG_ERROR([libshell32 missing])]) + AC_CHECK_LIB([comctl32], [main], [], [AC_MSG_ERROR([libcomctl32 missing])]) + AC_CHECK_LIB([ole32], [CoCreateInstance], [], [AC_MSG_ERROR([libole32 missing])]) + AC_CHECK_LIB([oleaut32], [main], [], [AC_MSG_ERROR([liboleaut32 missing])]) + AC_CHECK_LIB([uuid], [main], [], [AC_MSG_ERROR([libuuid missing])]) + AC_CHECK_LIB([advapi32], [CryptAcquireContextW], [], [AC_MSG_ERROR([libadvapi32 missing])]) + AC_CHECK_LIB([ws2_32], [WSAStartup], [], [AC_MSG_ERROR([libws2_32 missing])]) + AC_CHECK_LIB([shlwapi], [PathRemoveFileSpecW], [], [AC_MSG_ERROR([libshlwapi missing])]) + AC_CHECK_LIB([iphlpapi], [GetAdaptersAddresses], [], [AC_MSG_ERROR([libiphlpapi missing])]) dnl -static is interpreted by libtool, where it has a different meaning. dnl In libtool-speak, it's -all-static. - AX_CHECK_LINK_FLAG([[-static]],[LIBTOOL_APP_LDFLAGS="$LIBTOOL_APP_LDFLAGS -all-static"]) + AX_CHECK_LINK_FLAG([-static], [LIBTOOL_APP_LDFLAGS="$LIBTOOL_APP_LDFLAGS -all-static"]) - AC_PATH_PROG([MAKENSIS], [makensis], none) - if test x$MAKENSIS = xnone; then - AC_MSG_WARN("makensis not found. Cannot create installer.") + AC_PATH_PROG([MAKENSIS], [makensis], [none]) + if test "$MAKENSIS" = "none"; then + AC_MSG_WARN([makensis not found. Cannot create installer.]) fi - AC_PATH_TOOL(WINDRES, windres, none) - if test x$WINDRES = xnone; then - AC_MSG_ERROR("windres not found") + AC_PATH_TOOL([WINDRES], [windres], [none]) + if test "$WINDRES" = "none"; then + AC_MSG_ERROR([windres not found]) fi CPPFLAGS="$CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -D_WIN32_WINNT=0x0601 -D_WIN32_IE=0x0501 -DWIN32_LEAN_AND_MEAN" @@ -650,34 +678,34 @@ case $host in postdeps_CXX= dnl We require Windows 7 (NT 6.1) or later - AX_CHECK_LINK_FLAG([[-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1]],[LDFLAGS="$LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"],,[[$LDFLAG_WERROR]]) + AX_CHECK_LINK_FLAG([-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1], [LDFLAGS="$LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"], [], [$LDFLAG_WERROR]) ;; *darwin*) TARGET_OS=darwin - if test x$cross_compiling != xyes; then + if test $cross_compiling != "yes"; then BUILD_OS=darwin - AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg],rsvg-convert) - AC_CHECK_PROG([BREW],brew, brew) - if test x$BREW = xbrew; then + AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg], [rsvg-convert]) + AC_CHECK_PROG([BREW], [brew], [brew]) + if test "$BREW" = "brew"; then dnl These Homebrew packages may be keg-only, meaning that they won't be found dnl in expected paths because they may conflict with system files. Ask dnl Homebrew where each one is located, then adjust paths accordingly. dnl It's safe to add these paths even if the functionality is disabled by dnl the user (--without-wallet or --without-gui for example). - if test "x$use_bdb" != xno && $BREW list --versions berkeley-db4 >/dev/null && test "x$BDB_CFLAGS" = "x" && test "x$BDB_LIBS" = "x"; then - bdb_prefix=$($BREW --prefix berkeley-db4 2>/dev/null) + if test "$use_bdb" != "no" && $BREW list --versions berkeley-db@4 >/dev/null && test "$BDB_CFLAGS" = "" && test "$BDB_LIBS" = ""; then + bdb_prefix=$($BREW --prefix berkeley-db@4 2>/dev/null) dnl This must precede the call to BITCOIN_FIND_BDB48 below. BDB_CFLAGS="-I$bdb_prefix/include" BDB_LIBS="-L$bdb_prefix/lib -ldb_cxx-4.8" fi - if test "x$use_sqlite" != xno && $BREW list --versions sqlite3 >/dev/null; then + if test "$use_sqlite" != "no" && $BREW list --versions sqlite3 >/dev/null; then export PKG_CONFIG_PATH="$($BREW --prefix sqlite3 2>/dev/null)/lib/pkgconfig:$PKG_CONFIG_PATH" fi - if $BREW list --versions qt5 >/dev/null; then - export PKG_CONFIG_PATH="$($BREW --prefix qt5 2>/dev/null)/lib/pkgconfig:$PKG_CONFIG_PATH" + if $BREW list --versions qt@5 >/dev/null; then + export PKG_CONFIG_PATH="$($BREW --prefix qt@5 2>/dev/null)/lib/pkgconfig:$PKG_CONFIG_PATH" fi case $host in @@ -686,18 +714,18 @@ case $host in dnl Therefore, as we do not use pkg-config to detect miniupnpc and libnatpmp dnl packages, we should set the CPPFLAGS and LDFLAGS variables for them dnl explicitly. - if test "x$use_upnp" != xno && $BREW list --versions miniupnpc >/dev/null; then + if test "$use_upnp" != "no" && $BREW list --versions miniupnpc >/dev/null; then miniupnpc_prefix=$($BREW --prefix miniupnpc 2>/dev/null) - if test "x$suppress_external_warnings" != xno; then + if test "$suppress_external_warnings" != "no"; then CPPFLAGS="$CPPFLAGS -isystem $miniupnpc_prefix/include" else CPPFLAGS="$CPPFLAGS -I$miniupnpc_prefix/include" fi LDFLAGS="$LDFLAGS -L$miniupnpc_prefix/lib" fi - if test "x$use_natpmp" != xno && $BREW list --versions libnatpmp >/dev/null; then + if test "$use_natpmp" != "no" && $BREW list --versions libnatpmp >/dev/null; then libnatpmp_prefix=$($BREW --prefix libnatpmp 2>/dev/null) - if test "x$suppress_external_warnings" != xno; then + if test "$suppress_external_warnings" != "no"; then CPPFLAGS="$CPPFLAGS -isystem $libnatpmp_prefix/include" else CPPFLAGS="$CPPFLAGS -I$libnatpmp_prefix/include" @@ -713,14 +741,14 @@ case $host in BUILD_OS=darwin ;; *) - AC_PATH_TOOL([DSYMUTIL], [dsymutil], dsymutil) - AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], install_name_tool) - AC_PATH_TOOL([OTOOL], [otool], otool) - AC_PATH_PROGS([XORRISOFS], [xorrisofs], xorrisofs) - AC_PATH_PROGS([DMG], [dmg], dmg) - AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg],rsvg-convert) - AC_PATH_PROGS([IMAGEMAGICK_CONVERT], [convert],convert) - AC_PATH_PROGS([TIFFCP], [tiffcp],tiffcp) + AC_PATH_TOOL([DSYMUTIL], [dsymutil], [dsymutil]) + AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], [install_name_tool]) + AC_PATH_TOOL([OTOOL], [otool], [otool]) + AC_PATH_PROGS([XORRISOFS], [xorrisofs], [xorrisofs]) + AC_PATH_PROGS([DMG], [dmg], [dmg]) + AC_PATH_PROGS([RSVG_CONVERT], [rsvg-convert rsvg], [rsvg-convert]) + AC_PATH_PROGS([IMAGEMAGICK_CONVERT], [convert], [convert]) + AC_PATH_PROGS([TIFFCP], [tiffcp], [tiffcp]) dnl libtool will try to strip the static lib, which is a problem for dnl cross-builds because strip attempts to call a hard-coded ld, @@ -731,7 +759,7 @@ case $host in esac fi - AX_CHECK_LINK_FLAG([[-Wl,-headerpad_max_install_names]], [LDFLAGS="$LDFLAGS -Wl,-headerpad_max_install_names"],, [[$LDFLAG_WERROR]]) + AX_CHECK_LINK_FLAG([-Wl,-headerpad_max_install_names], [LDFLAGS="$LDFLAGS -Wl,-headerpad_max_install_names"], [], [$LDFLAG_WERROR]) CPPFLAGS="$CPPFLAGS -DMAC_OSX -DOBJC_OLD_DISPATCH_PROTOTYPES=0" OBJCXXFLAGS="$CXXFLAGS" ;; @@ -751,7 +779,7 @@ case $host in *i686*) ANDROID_ARCH=i686 ;; - *) AC_MSG_ERROR("Could not determine Android arch") ;; + *) AC_MSG_ERROR([Could not determine Android arch]) ;; esac ;; *linux*) @@ -759,19 +787,19 @@ case $host in ;; esac -if test x$use_extended_functional_tests != xno; then +if test "$use_extended_functional_tests" != "no"; then AC_SUBST(EXTENDED_FUNCTIONAL_TESTS, --extended) fi -if test x$use_lcov = xyes; then - if test x$LCOV = x; then - AC_MSG_ERROR("lcov testing requested but lcov not found") +if test "$use_lcov" = "yes"; then + if test "$LCOV" = ""; then + AC_MSG_ERROR([lcov testing requested but lcov not found]) fi - if test x$PYTHON = x; then - AC_MSG_ERROR("lcov testing requested but python not found") + if test "$PYTHON" = ""; then + AC_MSG_ERROR([lcov testing requested but python not found]) fi - if test x$GENHTML = x; then - AC_MSG_ERROR("lcov testing requested but genhtml not found") + if test "$GENHTML" = ""; then + AC_MSG_ERROR([lcov testing requested but genhtml not found]) fi AC_MSG_CHECKING([whether compiler is Clang]) @@ -783,13 +811,13 @@ if test x$use_lcov = xyes; then #endif ]])],[ AC_MSG_RESULT([yes]) - if test x$LLVM_COV = x; then + if test "$LLVM_COV" = ""; then AC_MSG_ERROR([lcov testing requested but llvm-cov not found]) fi COV_TOOL="$LLVM_COV gcov" ],[ AC_MSG_RESULT([no]) - if test x$GCOV = x; then + if test "$GCOV" = "x"; then AC_MSG_ERROR([lcov testing requested but gcov not found]) fi COV_TOOL="$GCOV" @@ -798,14 +826,14 @@ if test x$use_lcov = xyes; then AC_SUBST(COV_TOOL_WRAPPER, "cov_tool_wrapper.sh") LCOV="$LCOV --gcov-tool $(pwd)/$COV_TOOL_WRAPPER" - AX_CHECK_LINK_FLAG([[--coverage]], [LDFLAGS="$LDFLAGS --coverage"], - [AC_MSG_ERROR("lcov testing requested but --coverage linker flag does not work")]) + AX_CHECK_LINK_FLAG([--coverage], [LDFLAGS="$LDFLAGS --coverage"], + [AC_MSG_ERROR([lcov testing requested but --coverage linker flag does not work])]) AX_CHECK_COMPILE_FLAG([--coverage],[CXXFLAGS="$CXXFLAGS --coverage"], - [AC_MSG_ERROR("lcov testing requested but --coverage flag does not work")]) + [AC_MSG_ERROR([lcov testing requested but --coverage flag does not work])]) CXXFLAGS="$CXXFLAGS -Og" fi -if test x$use_lcov_branch != xno; then +if test "$use_lcov_branch" != "no"; then AC_SUBST(LCOV_OPTS, "$LCOV_OPTS --rc lcov_branch_coverage=1") fi @@ -825,55 +853,55 @@ AC_SYS_LARGEFILE dnl detect POSIX or GNU variant of strerror_r AC_FUNC_STRERROR_R -if test x$ac_cv_sys_file_offset_bits != x && - test x$ac_cv_sys_file_offset_bits != xno && - test x$ac_cv_sys_file_offset_bits != xunknown; then +if test "$ac_cv_sys_file_offset_bits" != "" && + test "$ac_cv_sys_file_offset_bits" != "no" && + test "$ac_cv_sys_file_offset_bits" != "unknown"; then CPPFLAGS="$CPPFLAGS -D_FILE_OFFSET_BITS=$ac_cv_sys_file_offset_bits" fi -if test x$ac_cv_sys_large_files != x && - test x$ac_cv_sys_large_files != xno && - test x$ac_cv_sys_large_files != xunknown; then +if test "$ac_cv_sys_large_files" != "" && + test "$ac_cv_sys_large_files" != "no" && + test "$ac_cv_sys_large_files" != "unknown"; then CPPFLAGS="$CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files" fi AC_SEARCH_LIBS([clock_gettime],[rt]) -if test "x$enable_gprof" = xyes; then +if test "$enable_gprof" = "yes"; then dnl -pg is incompatible with -pie. Since hardening and profiling together doesn't make sense, dnl we simply make them mutually exclusive here. Additionally, hardened toolchains may force dnl -pie by default, in which case it needs to be turned off with -no-pie. - if test x$use_hardening = xyes; then - AC_MSG_ERROR(gprof profiling is not compatible with hardening. Reconfigure with --disable-hardening or --disable-gprof) + if test "$use_hardening" = "yes"; then + AC_MSG_ERROR([gprof profiling is not compatible with hardening. Reconfigure with --disable-hardening or --disable-gprof]) fi use_hardening=no AX_CHECK_COMPILE_FLAG([-pg],[GPROF_CXXFLAGS="-pg"], - [AC_MSG_ERROR(gprof profiling requested but not available)], [[$CXXFLAG_WERROR]]) + [AC_MSG_ERROR([gprof profiling requested but not available])], [$CXXFLAG_WERROR]) - AX_CHECK_LINK_FLAG([[-no-pie]], [GPROF_LDFLAGS="-no-pie"]) - AX_CHECK_LINK_FLAG([[-pg]],[GPROF_LDFLAGS="$GPROF_LDFLAGS -pg"], - [AC_MSG_ERROR(gprof profiling requested but not available)], [[$GPROF_LDFLAGS]]) + AX_CHECK_LINK_FLAG([-no-pie], [GPROF_LDFLAGS="-no-pie"]) + AX_CHECK_LINK_FLAG([-pg], [GPROF_LDFLAGS="$GPROF_LDFLAGS -pg"], + [AC_MSG_ERROR([gprof profiling requested but not available])], [$GPROF_LDFLAGS]) fi -if test x$TARGET_OS != xwindows; then +if test "$TARGET_OS" != "windows"; then dnl All windows code is PIC, forcing it on just adds useless compile warnings - AX_CHECK_COMPILE_FLAG([-fPIC],[PIC_FLAGS="-fPIC"]) + AX_CHECK_COMPILE_FLAG([-fPIC], [PIC_FLAGS="-fPIC"]) fi dnl All versions of gcc that we commonly use for building are subject to bug dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90348. To work around that, set dnl -fstack-reuse=none for all gcc builds. (Only gcc understands this flag) -AX_CHECK_COMPILE_FLAG([-fstack-reuse=none],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-reuse=none"]) -if test x$use_hardening != xno; then +AX_CHECK_COMPILE_FLAG([-fstack-reuse=none], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-reuse=none"]) +if test "$use_hardening" != "no"; then use_hardening=yes - AX_CHECK_COMPILE_FLAG([-Wstack-protector],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -Wstack-protector"]) - AX_CHECK_COMPILE_FLAG([-fstack-protector-all],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-protector-all"]) + AX_CHECK_COMPILE_FLAG([-Wstack-protector], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -Wstack-protector"]) + AX_CHECK_COMPILE_FLAG([-fstack-protector-all], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-protector-all"]) dnl -fcf-protection used with Clang 7 causes ld to emit warnings: dnl ld: error: ... <corrupt x86 feature size: 0x8> dnl Use CHECK_LINK_FLAG & --fatal-warnings to ensure we won't use the flag in this case. - AX_CHECK_LINK_FLAG([-fcf-protection=full],[HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fcf-protection=full"],, [[$LDFLAG_WERROR]]) + AX_CHECK_LINK_FLAG([-fcf-protection=full], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fcf-protection=full"], [], [$LDFLAG_WERROR]) case $host in *mingw*) @@ -889,7 +917,7 @@ if test x$use_hardening != xno; then dnl When enable_debug is yes, all optimizations are disabled. dnl However, FORTIFY_SOURCE requires that there is some level of optimization, otherwise it does nothing and just creates a compiler warning. dnl Since FORTIFY_SOURCE is a no-op without optimizations, do not enable it when enable_debug is yes. - if test x$enable_debug != xyes; then + if test "$enable_debug" != "yes"; then AX_CHECK_PREPROC_FLAG([-D_FORTIFY_SOURCE=2],[ AX_CHECK_PREPROC_FLAG([-U_FORTIFY_SOURCE],[ HARDENED_CPPFLAGS="$HARDENED_CPPFLAGS -U_FORTIFY_SOURCE" @@ -898,18 +926,18 @@ if test x$use_hardening != xno; then ]) fi - AX_CHECK_LINK_FLAG([[-Wl,--enable-reloc-section]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--enable-reloc-section"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,--dynamicbase]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--dynamicbase"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,--nxcompat]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--nxcompat"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,--high-entropy-va]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--high-entropy-va"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,-z,relro]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,relro"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,-z,now]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,now"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,-z,separate-code]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,separate-code"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-fPIE -pie]], [PIE_FLAGS="-fPIE"; HARDENED_LDFLAGS="$HARDENED_LDFLAGS -pie"],, [[$CXXFLAG_WERROR]]) + AX_CHECK_LINK_FLAG([-Wl,--enable-reloc-section], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--enable-reloc-section"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,--dynamicbase], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--dynamicbase"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,--nxcompat], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--nxcompat"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,--high-entropy-va], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,--high-entropy-va"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-z,relro], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,relro"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-z,now], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,now"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-z,separate-code], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-z,separate-code"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-fPIE -pie], [PIE_FLAGS="-fPIE"; HARDENED_LDFLAGS="$HARDENED_LDFLAGS -pie"], [], [$CXXFLAG_WERROR]) case $host in *mingw*) - AC_CHECK_LIB([ssp], [main],, AC_MSG_ERROR(libssp missing)) + AC_CHECK_LIB([ssp], [main], [], [AC_MSG_ERROR([libssp missing])]) ;; esac fi @@ -917,10 +945,10 @@ fi dnl These flags are specific to ld64, and may cause issues with other linkers. dnl For example: GNU ld will interpret -dead_strip as -de and then try and use dnl "ad_strip" as the symbol for the entry point. -if test x$TARGET_OS = xdarwin; then - AX_CHECK_LINK_FLAG([[-Wl,-dead_strip]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,-dead_strip_dylibs]], [LDFLAGS="$LDFLAGS -Wl,-dead_strip_dylibs"],, [[$LDFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,-bind_at_load]], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-bind_at_load"],, [[$LDFLAG_WERROR]]) +if test "$TARGET_OS" = "darwin"; then + AX_CHECK_LINK_FLAG([-Wl,-dead_strip], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-dead_strip_dylibs], [LDFLAGS="$LDFLAGS -Wl,-dead_strip_dylibs"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-bind_at_load], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-bind_at_load"], [], [$LDFLAG_WERROR]) fi AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) @@ -949,40 +977,40 @@ AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64],,, #include <byteswap.h> #endif]) -AC_MSG_CHECKING(for __builtin_clzl) +AC_MSG_CHECKING([for __builtin_clzl]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ (void) __builtin_clzl(0); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZL, 1, [Define this symbol if you have __builtin_clzl])], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); have_clzl=yes; AC_DEFINE([HAVE_BUILTIN_CLZL], [1], [Define this symbol if you have __builtin_clzl])], + [ AC_MSG_RESULT([no]); have_clzl=no;] ) -AC_MSG_CHECKING(for __builtin_clzll) +AC_MSG_CHECKING([for __builtin_clzll]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ (void) __builtin_clzll(0); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZLL, 1, [Define this symbol if you have __builtin_clzll])], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); have_clzll=yes; AC_DEFINE([HAVE_BUILTIN_CLZLL], [1], [Define this symbol if you have __builtin_clzll])], + [ AC_MSG_RESULT([no]); have_clzll=no;] ) dnl Check for malloc_info (for memory statistics information in getmemoryinfo) -AC_MSG_CHECKING(for getmemoryinfo) +AC_MSG_CHECKING([for getmemoryinfo]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <malloc.h>]], [[ int f = malloc_info(0, NULL); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_MALLOC_INFO, 1,[Define this symbol if you have malloc_info]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_MALLOC_INFO], [1], [Define this symbol if you have malloc_info]) ], + [ AC_MSG_RESULT([no])] ) dnl Check for mallopt(M_ARENA_MAX) (to set glibc arenas) -AC_MSG_CHECKING(for mallopt M_ARENA_MAX) +AC_MSG_CHECKING([for mallopt M_ARENA_MAX]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <malloc.h>]], [[ mallopt(M_ARENA_MAX, 1); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_MALLOPT_ARENA_MAX, 1,[Define this symbol if you have mallopt with M_ARENA_MAX]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_MALLOPT_ARENA_MAX], [1], [Define this symbol if you have mallopt with M_ARENA_MAX]) ], + [ AC_MSG_RESULT([no])] ) dnl Check for posix_fallocate -AC_MSG_CHECKING(for posix_fallocate) +AC_MSG_CHECKING([for posix_fallocate]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ // same as in src/util/system.cpp #ifdef __linux__ @@ -993,8 +1021,8 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #endif // __linux__ #include <fcntl.h>]], [[ int f = posix_fallocate(0, 0, 0); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_POSIX_FALLOCATE, 1,[Define this symbol if you have posix_fallocate]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_POSIX_FALLOCATE], [1], [Define this symbol if you have posix_fallocate]) ], + [ AC_MSG_RESULT([no])] ) AC_MSG_CHECKING([for default visibility attribute]) @@ -1003,12 +1031,12 @@ AC_COMPILE_IFELSE([AC_LANG_SOURCE([ int main(){} ])], [ - AC_DEFINE(HAVE_DEFAULT_VISIBILITY_ATTRIBUTE,1,[Define if the visibility attribute is supported.]) - AC_MSG_RESULT(yes) + AC_DEFINE([HAVE_DEFAULT_VISIBILITY_ATTRIBUTE], [1], [Define if the visibility attribute is supported.]) + AC_MSG_RESULT([yes]) ], [ - AC_MSG_RESULT(no) - if test x$use_reduce_exports = xyes; then + AC_MSG_RESULT([no]) + if test "$use_reduce_exports" = "yes"; then AC_MSG_ERROR([Cannot find a working visibility attribute. Use --disable-reduce-exports.]) fi ] @@ -1020,16 +1048,13 @@ AC_COMPILE_IFELSE([AC_LANG_SOURCE([ int main(){} ])], [ - AC_DEFINE(HAVE_DLLEXPORT_ATTRIBUTE,1,[Define if the dllexport attribute is supported.]) - AC_MSG_RESULT(yes) + AC_DEFINE([HAVE_DLLEXPORT_ATTRIBUTE], [1], [Define if the dllexport attribute is supported.]) + AC_MSG_RESULT([yes]) ], - [AC_MSG_RESULT(no)] + [AC_MSG_RESULT([no])] ) -dnl thread_local is currently disabled when building with glibc back compat. -dnl Our minimum supported glibc is 2.17, however support for thread_local -dnl did not arrive in glibc until 2.18. -if test "x$use_thread_local" = xyes || test "x$use_thread_local" = xauto; then +if test "$use_thread_local" = "yes" || test "$use_thread_local" = "auto"; then TEMP_LDFLAGS="$LDFLAGS" LDFLAGS="$TEMP_LDFLAGS $PTHREAD_CFLAGS" AC_MSG_CHECKING([for thread_local support]) @@ -1048,21 +1073,21 @@ if test "x$use_thread_local" = xyes || test "x$use_thread_local" = xauto; then dnl mingw32's implementation of thread_local has also been shown to behave dnl erroneously under concurrent usage; see: dnl https://gist.github.com/jamesob/fe9a872051a88b2025b1aa37bfa98605 - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) ;; *freebsd*) dnl FreeBSD's implementation of thread_local is also buggy (per dnl https://groups.google.com/d/msg/bsdmailinglist/22ncTZAbDp4/Dii_pII5AwAJ) - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) ;; *) - AC_DEFINE(HAVE_THREAD_LOCAL,1,[Define if thread_local is supported.]) - AC_MSG_RESULT(yes) + AC_DEFINE([HAVE_THREAD_LOCAL], [1], [Define if thread_local is supported.]) + AC_MSG_RESULT([yes]) ;; esac ], [ - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) ] ) LDFLAGS="$TEMP_LDFLAGS" @@ -1070,57 +1095,57 @@ fi dnl check for gmtime_r(), fallback to gmtime_s() if that is unavailable dnl fail if neither are available. -AC_MSG_CHECKING(for gmtime_r) +AC_MSG_CHECKING([for gmtime_r]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <ctime>]], [[ gmtime_r((const time_t *) nullptr, (struct tm *) nullptr); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_GMTIME_R, 1, [Define this symbol if gmtime_r is available]) ], - [ AC_MSG_RESULT(no); - AC_MSG_CHECKING(for gmtime_s); + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_GMTIME_R], [1], [Define this symbol if gmtime_r is available]) ], + [ AC_MSG_RESULT([no]); + AC_MSG_CHECKING([for gmtime_s]); AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <ctime>]], [[ gmtime_s((struct tm *) nullptr, (const time_t *) nullptr); ]])], - [ AC_MSG_RESULT(yes)], - [ AC_MSG_RESULT(no); AC_MSG_ERROR(Both gmtime_r and gmtime_s are unavailable) ] + [ AC_MSG_RESULT([yes])], + [ AC_MSG_RESULT([no]); AC_MSG_ERROR([Both gmtime_r and gmtime_s are unavailable]) ] ) ] ) dnl Check for different ways of gathering OS randomness -AC_MSG_CHECKING(for Linux getrandom syscall) +AC_MSG_CHECKING([for Linux getrandom syscall]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h> #include <sys/syscall.h> #include <linux/random.h>]], [[ syscall(SYS_getrandom, nullptr, 32, 0); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYS_GETRANDOM, 1,[Define this symbol if the Linux getrandom system call is available]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_SYS_GETRANDOM], [1], [Define this symbol if the Linux getrandom system call is available]) ], + [ AC_MSG_RESULT([no])] ) -AC_MSG_CHECKING(for getentropy) +AC_MSG_CHECKING([for getentropy]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h>]], [[ getentropy(nullptr, 32) ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_GETENTROPY, 1,[Define this symbol if the BSD getentropy system call is available]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_GETENTROPY], [1], [Define this symbol if the BSD getentropy system call is available]) ], + [ AC_MSG_RESULT([no])] ) -AC_MSG_CHECKING(for getentropy via random.h) +AC_MSG_CHECKING([for getentropy via random.h]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h> #include <sys/random.h>]], [[ getentropy(nullptr, 32) ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_GETENTROPY_RAND, 1,[Define this symbol if the BSD getentropy system call is available with sys/random.h]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_GETENTROPY_RAND], [1], [Define this symbol if the BSD getentropy system call is available with sys/random.h]) ], + [ AC_MSG_RESULT([no])] ) -AC_MSG_CHECKING(for sysctl) +AC_MSG_CHECKING([for sysctl]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h> #include <sys/sysctl.h>]], [[ #ifdef __linux__ #error "Don't use sysctl on Linux, it's deprecated even when it works" #endif sysctl(nullptr, 2, nullptr, nullptr, nullptr, 0); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYSCTL, 1,[Define this symbol if the BSD sysctl() is available]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_SYSCTL], [1], [Define this symbol if the BSD sysctl() is available]) ], + [ AC_MSG_RESULT([no])] ) -AC_MSG_CHECKING(for sysctl KERN_ARND) +AC_MSG_CHECKING([for sysctl KERN_ARND]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h> #include <sys/sysctl.h>]], [[ #ifdef __linux__ @@ -1128,70 +1153,70 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h> #endif static int name[2] = {CTL_KERN, KERN_ARND}; sysctl(name, 2, nullptr, nullptr, nullptr, 0); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_SYSCTL_ARND, 1,[Define this symbol if the BSD sysctl(KERN_ARND) is available]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([HAVE_SYSCTL_ARND], [1], [Define this symbol if the BSD sysctl(KERN_ARND) is available]) ], + [ AC_MSG_RESULT([no])] ) -AC_MSG_CHECKING(for if type char equals int8_t) +AC_MSG_CHECKING([for if type char equals int8_t]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <stdint.h> #include <type_traits>]], [[ static_assert(std::is_same<int8_t, char>::value, ""); ]])], - [ AC_MSG_RESULT(yes); AC_DEFINE(CHAR_EQUALS_INT8, 1,[Define this symbol if type char equals int8_t]) ], - [ AC_MSG_RESULT(no)] + [ AC_MSG_RESULT([yes]); AC_DEFINE([CHAR_EQUALS_INT8], [1], [Define this symbol if type char equals int8_t]) ], + [ AC_MSG_RESULT([no])] ) -AC_MSG_CHECKING(for fdatasync) +AC_MSG_CHECKING([for fdatasync]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <unistd.h>]], [[ fdatasync(0); ]])], - [ AC_MSG_RESULT(yes); HAVE_FDATASYNC=1 ], - [ AC_MSG_RESULT(no); HAVE_FDATASYNC=0 ] + [ AC_MSG_RESULT([yes]); HAVE_FDATASYNC=1 ], + [ AC_MSG_RESULT([no]); HAVE_FDATASYNC=0 ] ) AC_DEFINE_UNQUOTED([HAVE_FDATASYNC], [$HAVE_FDATASYNC], [Define to 1 if fdatasync is available.]) -AC_MSG_CHECKING(for F_FULLFSYNC) +AC_MSG_CHECKING([for F_FULLFSYNC]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <fcntl.h>]], [[ fcntl(0, F_FULLFSYNC, 0); ]])], - [ AC_MSG_RESULT(yes); HAVE_FULLFSYNC=1 ], - [ AC_MSG_RESULT(no); HAVE_FULLFSYNC=0 ] + [ AC_MSG_RESULT([yes]); HAVE_FULLFSYNC=1 ], + [ AC_MSG_RESULT([no]); HAVE_FULLFSYNC=0 ] ) -AC_MSG_CHECKING(for O_CLOEXEC) +AC_MSG_CHECKING([for O_CLOEXEC]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <fcntl.h>]], [[ open("", O_CLOEXEC); ]])], - [ AC_MSG_RESULT(yes); HAVE_O_CLOEXEC=1 ], - [ AC_MSG_RESULT(no); HAVE_O_CLOEXEC=0 ] + [ AC_MSG_RESULT([yes]); HAVE_O_CLOEXEC=1 ], + [ AC_MSG_RESULT([no]); HAVE_O_CLOEXEC=0 ] ) AC_DEFINE_UNQUOTED([HAVE_O_CLOEXEC], [$HAVE_O_CLOEXEC], [Define to 1 if O_CLOEXEC flag is available.]) dnl crc32c platform checks -AC_MSG_CHECKING(for __builtin_prefetch) +AC_MSG_CHECKING([for __builtin_prefetch]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[ char data = 0; const char* address = &data; __builtin_prefetch(address, 0, 0); ]])], - [ AC_MSG_RESULT(yes); HAVE_BUILTIN_PREFETCH=1 ], - [ AC_MSG_RESULT(no); HAVE_BUILTIN_PREFETCH=0 ] + [ AC_MSG_RESULT([yes]); HAVE_BUILTIN_PREFETCH=1 ], + [ AC_MSG_RESULT([no]); HAVE_BUILTIN_PREFETCH=0 ] ) -AC_MSG_CHECKING(for _mm_prefetch) +AC_MSG_CHECKING([for _mm_prefetch]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <xmmintrin.h>]], [[ char data = 0; const char* address = &data; _mm_prefetch(address, _MM_HINT_NTA); ]])], - [ AC_MSG_RESULT(yes); HAVE_MM_PREFETCH=1 ], - [ AC_MSG_RESULT(no); HAVE_MM_PREFETCH=0 ] + [ AC_MSG_RESULT([yes]); HAVE_MM_PREFETCH=1 ], + [ AC_MSG_RESULT([no]); HAVE_MM_PREFETCH=0 ] ) -AC_MSG_CHECKING(for strong getauxval support in the system headers) +AC_MSG_CHECKING([for strong getauxval support in the system headers]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include <sys/auxv.h> ]], [[ getauxval(AT_HWCAP); ]])], - [ AC_MSG_RESULT(yes); HAVE_STRONG_GETAUXVAL=1; AC_DEFINE(HAVE_STRONG_GETAUXVAL, 1, [Define this symbol to build code that uses getauxval)]) ], - [ AC_MSG_RESULT(no); HAVE_STRONG_GETAUXVAL=0 ] + [ AC_MSG_RESULT([yes]); HAVE_STRONG_GETAUXVAL=1; AC_DEFINE([HAVE_STRONG_GETAUXVAL], [1], [Define this symbol to build code that uses getauxval)]) ], + [ AC_MSG_RESULT([no]); HAVE_STRONG_GETAUXVAL=0 ] ) have_any_system=no @@ -1201,8 +1226,8 @@ AC_LINK_IFELSE( [[ #include <cstdlib> ]], [[ int nErr = std::system(""); ]] )], - [ AC_MSG_RESULT(yes); have_any_system=yes], - [ AC_MSG_RESULT(no) ] + [ AC_MSG_RESULT([yes]); have_any_system=yes], + [ AC_MSG_RESULT([no]) ] ) AC_MSG_CHECKING([for ::_wsystem]) @@ -1211,12 +1236,12 @@ AC_LINK_IFELSE( [[ ]], [[ int nErr = ::_wsystem(""); ]] )], - [ AC_MSG_RESULT(yes); have_any_system=yes], - [ AC_MSG_RESULT(no) ] + [ AC_MSG_RESULT([yes]); have_any_system=yes], + [ AC_MSG_RESULT([no]) ] ) -if test "x$have_any_system" != "xno"; then - AC_DEFINE(HAVE_SYSTEM, 1, Define to 1 if std::system or ::wsystem is available.) +if test "$have_any_system" != "no"; then + AC_DEFINE([HAVE_SYSTEM], [1], [Define to 1 if std::system or ::wsystem is available.]) fi dnl SUPPRESSED_CPPFLAGS=SUPPRESS_WARNINGS([$SOME_CPPFLAGS]) @@ -1232,8 +1257,8 @@ AC_DEFUN([SUPPRESS_WARNINGS], [$(echo $1 |${SED} -E -e 's/(^| )-I/\1-isystem /g' -e 's;-isystem /usr/include([/ ]|$);-I/usr/include\1;g')]) dnl enable-fuzz should disable all other targets -if test "x$enable_fuzz" = "xyes"; then - AC_MSG_WARN(enable-fuzz will disable all other targets and force --enable-fuzz-binary=yes) +if test "$enable_fuzz" = "yes"; then + AC_MSG_WARN([enable-fuzz will disable all other targets and force --enable-fuzz-binary=yes]) build_bitcoin_utils=no build_bitcoin_cli=no build_bitcoin_tx=no @@ -1244,7 +1269,6 @@ if test "x$enable_fuzz" = "xyes"; then bitcoin_enable_qt=no bitcoin_enable_qt_test=no bitcoin_enable_qt_dbus=no - enable_wallet=no use_bench=no use_external_signer=no use_upnp=no @@ -1252,14 +1276,13 @@ if test "x$enable_fuzz" = "xyes"; then use_zmq=no enable_fuzz_binary=yes - AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME],[[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"]],,[[$CXXFLAG_WERROR]]) + AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"], [], [$CXXFLAG_WERROR]) AC_MSG_CHECKING([whether main function is needed for fuzz binary]) AX_CHECK_LINK_FLAG( - [[-fsanitize=$use_sanitizers]], + [-fsanitize=$use_sanitizers], [AC_MSG_RESULT([no])], - [AC_MSG_RESULT([yes]) - CPPFLAGS="$CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION"], + [AC_MSG_RESULT([yes]); CPPFLAGS="$CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION"], [], [AC_LANG_PROGRAM([[ #include <cstdint> @@ -1277,7 +1300,7 @@ else dnl Keep a copy of the original $QT_INCLUDES and use it when invoking qt's moc QT_INCLUDES_UNSUPPRESSED=$QT_INCLUDES - if test x$suppress_external_warnings != xno ; then + if test "$suppress_external_warnings" != "no" ; then QT_INCLUDES=SUPPRESS_WARNINGS($QT_INCLUDES) QT_DBUS_INCLUDES=SUPPRESS_WARNINGS($QT_DBUS_INCLUDES) QT_TEST_INCLUDES=SUPPRESS_WARNINGS($QT_TEST_INCLUDES) @@ -1286,29 +1309,29 @@ else CPPFLAGS="$CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION" fi -if test x$enable_wallet != xno; then +if test "$enable_wallet" != "no"; then dnl Check for libdb_cxx only if wallet enabled - if test "x$use_bdb" != "xno"; then + if test "$use_bdb" != "no"; then BITCOIN_FIND_BDB48 - if test x$suppress_external_warnings != xno ; then + if test "$suppress_external_warnings" != "no" ; then BDB_CPPFLAGS=SUPPRESS_WARNINGS($BDB_CPPFLAGS) fi fi dnl Check for sqlite3 - if test "x$use_sqlite" != "xno"; then + if test "$use_sqlite" != "no"; then PKG_CHECK_MODULES([SQLITE], [sqlite3 >= 3.7.17], [have_sqlite=yes], [have_sqlite=no]) fi AC_MSG_CHECKING([whether to build wallet with support for sqlite]) - if test "x$use_sqlite" = "xno"; then + if test "$use_sqlite" = "no"; then use_sqlite=no - elif test "x$have_sqlite" = "xno"; then - if test "x$use_sqlite" = "xyes"; then + elif test "$have_sqlite" = "no"; then + if test "$use_sqlite" = "yes"; then AC_MSG_ERROR([sqlite support requested but cannot be built. Use --without-sqlite]) fi use_sqlite=no else - if test x$use_sqlite != xno; then + if test "$use_sqlite" != "no"; then AC_DEFINE([USE_SQLITE],[1],[Define if sqlite support should be compiled in]) use_sqlite=yes fi @@ -1316,28 +1339,34 @@ if test x$enable_wallet != xno; then AC_MSG_RESULT([$use_sqlite]) dnl Disable wallet if both --without-bdb and --without-sqlite - if test "x$use_bdb$use_sqlite" = "xnono"; then - if test "x$enable_wallet" = "xyes"; then + if test "$use_bdb$use_sqlite" = "nono"; then + if test "$enable_wallet" = "yes"; then AC_MSG_ERROR([wallet functionality requested but no BDB or SQLite support available.]) fi enable_wallet=no fi fi -if test x$use_ebpf != xno; then +if test "$use_ebpf" != "no"; then AC_MSG_CHECKING([whether eBPF tracepoints are supported]) AC_COMPILE_IFELSE([ AC_LANG_PROGRAM( [#include <sys/sdt.h>], [DTRACE_PROBE("context", "event");] )], - [AC_MSG_RESULT(yes); have_sdt=yes; AC_DEFINE([ENABLE_TRACING], [1], [Define to 1 to enable eBPF user static defined tracepoints])], - [AC_MSG_RESULT(no); have_sdt=no;] + [AC_MSG_RESULT([yes]); have_sdt=yes; AC_DEFINE([ENABLE_TRACING], [1], [Define to 1 to enable eBPF user static defined tracepoints])], + [AC_MSG_RESULT([no]); have_sdt=no;] ) fi +if test "$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests" = "nonononononono"; then + use_upnp=no + use_natpmp=no + use_zmq=no +fi + dnl Check for libminiupnpc (optional) -if test x$use_upnp != xno; then +if test "$use_upnp" != "no"; then AC_CHECK_HEADERS( [miniupnpc/miniupnpc.h miniupnpc/upnpcommands.h miniupnpc/upnperrors.h], [AC_CHECK_LIB([miniupnpc], [upnpDiscover], [MINIUPNPC_LIBS=-lminiupnpc], [have_miniupnpc=no])], @@ -1345,7 +1374,7 @@ if test x$use_upnp != xno; then ) dnl The minimum supported miniUPnPc API version is set to 10. This keeps compatibility dnl with Ubuntu 16.04 LTS and Debian 8 libminiupnpc-dev packages. -if test x$have_miniupnpc != xno; then +if test "$have_miniupnpc" != "no"; then AC_MSG_CHECKING([whether miniUPnPc API version is supported]) AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ @%:@include <miniupnpc/miniupnpc.h> @@ -1356,9 +1385,9 @@ if test x$have_miniupnpc != xno; then # error miniUPnPc API version is too old #endif ]])],[ - AC_MSG_RESULT(yes) + AC_MSG_RESULT([yes]) ],[ - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) AC_MSG_WARN([miniUPnPc API version < 10 is unsupported, disabling UPnP support.]) have_miniupnpc=no ]) @@ -1366,48 +1395,48 @@ fi fi dnl Check for libnatpmp (optional). -if test "x$use_natpmp" != xno; then +if test "$use_natpmp" != "no"; then AC_CHECK_HEADERS([natpmp.h], [AC_CHECK_LIB([natpmp], [initnatpmp], [NATPMP_LIBS=-lnatpmp], [have_natpmp=no])], [have_natpmp=no]) fi -if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench = xnonononononono; then +if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench" = "nonononononono"; then use_boost=no else use_boost=yes fi -if test x$use_boost = xyes; then +if test "$use_boost" = "yes"; then dnl Check for Boost headers AX_BOOST_BASE([1.64.0],[],[AC_MSG_ERROR([Boost is not available!])]) - if test x$want_boost = xno; then - AC_MSG_ERROR([[only libbitcoinconsensus can be built without boost]]) + if test "$want_boost" = "no"; then + AC_MSG_ERROR([only libbitcoinconsensus can be built without Boost]) fi AX_BOOST_SYSTEM AX_BOOST_FILESYSTEM - if test x$suppress_external_warnings != xno; then + if test "$suppress_external_warnings" != "no"; then BOOST_CPPFLAGS=SUPPRESS_WARNINGS($BOOST_CPPFLAGS) fi BOOST_LIBS="$BOOST_LDFLAGS $BOOST_SYSTEM_LIB $BOOST_FILESYSTEM_LIB" fi -if test "x$use_external_signer" != xno; then - AC_DEFINE([ENABLE_EXTERNAL_SIGNER],,[define if external signer support is enabled]) +if test "$use_external_signer" != "no"; then + AC_DEFINE([ENABLE_EXTERNAL_SIGNER], [], [Define if external signer support is enabled]) fi -AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER], [test "x$use_external_signer" = "xyes"]) +AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER], [test "$use_external_signer" = "yes"]) dnl Do not compile with syscall sandbox support when compiling under the sanitizers. dnl The sanitizers introduce use of syscalls that are not typically used in bitcoind dnl (such as execve when the sanitizers execute llvm-symbolizer). -if test x$use_sanitizers != x; then - AC_MSG_WARN(Specifying --with-sanitizers forces --without-seccomp since the sanitizers introduce use of syscalls not allowed by the bitcoind syscall sandbox (-sandbox=<mode>).) +if test "$use_sanitizers" != ""; then + AC_MSG_WARN([Specifying --with-sanitizers forces --without-seccomp since the sanitizers introduce use of syscalls not allowed by the bitcoind syscall sandbox (-sandbox=<mode>).]) seccomp_found=no fi -if test "x$seccomp_found" != "xno"; then +if test "$seccomp_found" != "no"; then AC_MSG_CHECKING([for seccomp-bpf (Linux x86-64)]) AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ @%:@include <linux/seccomp.h> @@ -1416,11 +1445,11 @@ if test "x$seccomp_found" != "xno"; then # error Syscall sandbox is an experimental feature currently available only under Linux x86-64. #endif ]])],[ - AC_MSG_RESULT(yes) + AC_MSG_RESULT([yes]) seccomp_found="yes" - AC_DEFINE(USE_SYSCALL_SANDBOX, 1, [Define this symbol to build with syscall sandbox support.]) + AC_DEFINE([USE_SYSCALL_SANDBOX], [1], [Define this symbol to build with syscall sandbox support.]) ],[ - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) seccomp_found="no" ]) fi @@ -1428,22 +1457,22 @@ dnl Currently only enable -sandbox=<mode> feature if seccomp is found. dnl In the future, sandboxing could be also be supported with other dnl sandboxing mechanisms besides seccomp. use_syscall_sandbox=$seccomp_found -AM_CONDITIONAL([ENABLE_SYSCALL_SANDBOX], [test "x$use_syscall_sandbox" != "xno"]) +AM_CONDITIONAL([ENABLE_SYSCALL_SANDBOX], [test "$use_syscall_sandbox" != "no"]) dnl Check for reduced exports -if test x$use_reduce_exports = xyes; then - AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[CXXFLAGS="$CXXFLAGS -fvisibility=hidden"], - [AC_MSG_ERROR([Cannot set hidden symbol visibility. Use --disable-reduce-exports.])],[[$CXXFLAG_WERROR]]) - AX_CHECK_LINK_FLAG([[-Wl,--exclude-libs,ALL]],[RELDFLAGS="-Wl,--exclude-libs,ALL"],,[[$LDFLAG_WERROR]]) +if test "$use_reduce_exports" = "yes"; then + AX_CHECK_COMPILE_FLAG([-fvisibility=hidden], [CXXFLAGS="$CXXFLAGS -fvisibility=hidden"], + [AC_MSG_ERROR([Cannot set hidden symbol visibility. Use --disable-reduce-exports.])], [$CXXFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,--exclude-libs,ALL], [RELDFLAGS="-Wl,--exclude-libs,ALL"], [], [$LDFLAG_WERROR]) fi -if test x$use_tests = xyes; then +if test "$use_tests" = "yes"; then - if test x$HEXDUMP = x; then - AC_MSG_ERROR(hexdump is required for tests) + if test "$HEXDUMP" = ""; then + AC_MSG_ERROR([hexdump is required for tests]) fi - if test x$use_boost = xyes; then + if test "$use_boost" = "yes"; then AX_BOOST_UNIT_TEST_FRAMEWORK @@ -1459,9 +1488,9 @@ if test x$use_tests = xyes; then #include <boost/test/unit_test.hpp> ])], - [AC_MSG_RESULT(yes)] + [AC_MSG_RESULT([yes])] [TESTDEFS="$TESTDEFS -DBOOST_TEST_DYN_LINK"], - [AC_MSG_RESULT(no)]) + [AC_MSG_RESULT([no])]) LIBS="$TEMP_LIBS" CPPFLAGS="$TEMP_CPPFLAGS" @@ -1470,26 +1499,26 @@ fi dnl libevent check -if test x$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench != xnonononono; then +if test "$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench" != "nonononono"; then PKG_CHECK_MODULES([EVENT], [libevent >= 2.0.21], [use_libevent=yes], [AC_MSG_ERROR([libevent version 2.0.21 or greater not found.])]) - if test x$TARGET_OS != xwindows; then + if test "$TARGET_OS" != "windows"; then PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.0.21],, [AC_MSG_ERROR([libevent_pthreads version 2.0.21 or greater not found.])]) fi - if test x$suppress_external_warnings != xno; then + if test "$suppress_external_warnings" != "no"; then EVENT_CFLAGS=SUPPRESS_WARNINGS($EVENT_CFLAGS) fi fi dnl QR Code encoding library check -if test "x$use_qr" != xno; then +if test "$use_qr" != "no"; then BITCOIN_QT_CHECK([PKG_CHECK_MODULES([QR], [libqrencode], [have_qrencode=yes], [have_qrencode=no])]) fi dnl ZMQ check -if test "x$use_zmq" = xyes; then +if test "$use_zmq" = "yes"; then PKG_CHECK_MODULES([ZMQ], [libzmq >= 4], AC_DEFINE([ENABLE_ZMQ], [1], [Define to 1 to enable ZMQ functions]), [AC_DEFINE([ENABLE_ZMQ], [0], [Define to 1 to enable ZMQ functions]) @@ -1499,7 +1528,7 @@ else AC_DEFINE_UNQUOTED([ENABLE_ZMQ], [0], [Define to 1 to enable ZMQ functions]) fi -if test "x$use_zmq" = xyes; then +if test "$use_zmq" = "yes"; then dnl Assume libzmq was built for static linking case $host in *mingw*) @@ -1511,12 +1540,12 @@ fi dnl libmultiprocess library check libmultiprocess_found=no -if test "x$with_libmultiprocess" = xyes || test "x$with_libmultiprocess" = xauto; then +if test "$with_libmultiprocess" = "yes" || test "$with_libmultiprocess" = "auto"; then m4_ifdef([PKG_CHECK_MODULES], [PKG_CHECK_MODULES([LIBMULTIPROCESS], [libmultiprocess], [ libmultiprocess_found=yes; libmultiprocess_prefix=`$PKG_CONFIG --variable=prefix libmultiprocess`; ], [true])]) -elif test "x$with_libmultiprocess" != xno; then +elif test "$with_libmultiprocess" != "no"; then AC_MSG_ERROR([--with-libmultiprocess=$with_libmultiprocess value is not yes, auto, or no]) fi AC_SUBST(LIBMULTIPROCESS_CFLAGS) @@ -1524,66 +1553,66 @@ AC_SUBST(LIBMULTIPROCESS_LIBS) dnl Enable multiprocess check -if test "x$enable_multiprocess" = xyes; then - if test "x$libmultiprocess_found" != xyes; then +if test "$enable_multiprocess" = "yes"; then + if test "$libmultiprocess_found" != "yes"; then AC_MSG_ERROR([--enable-multiprocess=yes option specified but libmultiprocess library was not found. May need to install libmultiprocess library, or specify install path with PKG_CONFIG_PATH environment variable. Running 'pkg-config --debug libmultiprocess' may be helpful for debugging.]) fi build_multiprocess=yes -elif test "x$enable_multiprocess" = xauto; then +elif test "$enable_multiprocess" = "auto"; then build_multiprocess=$libmultiprocess_found else build_multiprocess=no fi -AM_CONDITIONAL([BUILD_MULTIPROCESS],[test "x$build_multiprocess" = xyes]) -AM_CONDITIONAL([BUILD_BITCOIN_NODE], [test "x$build_multiprocess" = xyes]) -AM_CONDITIONAL([BUILD_BITCOIN_GUI], [test "x$build_multiprocess" = xyes]) +AM_CONDITIONAL([BUILD_MULTIPROCESS], [test "$build_multiprocess" = "yes"]) +AM_CONDITIONAL([BUILD_BITCOIN_NODE], [test "$build_multiprocess" = "yes"]) +AM_CONDITIONAL([BUILD_BITCOIN_GUI], [test "$build_multiprocess" = "yes"]) dnl codegen tools check -if test x$build_multiprocess != xno; then - if test "x$with_mpgen" = xyes || test "x$with_mpgen" = xauto; then +if test "$build_multiprocess" != "no"; then + if test "$with_mpgen" = "yes" || test "$with_mpgen" = "auto"; then MPGEN_PREFIX="$libmultiprocess_prefix" - elif test "x$with_mpgen" != xno; then + elif test "$with_mpgen" != "no"; then MPGEN_PREFIX="$with_mpgen"; fi AC_SUBST(MPGEN_PREFIX) fi AC_MSG_CHECKING([whether to build bitcoind]) -AM_CONDITIONAL([BUILD_BITCOIND], [test x$build_bitcoind = xyes]) +AM_CONDITIONAL([BUILD_BITCOIND], [test $build_bitcoind = "yes"]) AC_MSG_RESULT($build_bitcoind) AC_MSG_CHECKING([whether to build bitcoin-cli]) -AM_CONDITIONAL([BUILD_BITCOIN_CLI], [test x$build_bitcoin_cli = xyes]) +AM_CONDITIONAL([BUILD_BITCOIN_CLI], [test $build_bitcoin_cli = "yes"]) AC_MSG_RESULT($build_bitcoin_cli) AC_MSG_CHECKING([whether to build bitcoin-tx]) -AM_CONDITIONAL([BUILD_BITCOIN_TX], [test x$build_bitcoin_tx = xyes]) +AM_CONDITIONAL([BUILD_BITCOIN_TX], [test $build_bitcoin_tx = "yes"]) AC_MSG_RESULT($build_bitcoin_tx) AC_MSG_CHECKING([whether to build bitcoin-wallet]) -AM_CONDITIONAL([BUILD_BITCOIN_WALLET], [test x$build_bitcoin_wallet = xyes]) +AM_CONDITIONAL([BUILD_BITCOIN_WALLET], [test $build_bitcoin_wallet = "yes"]) AC_MSG_RESULT($build_bitcoin_wallet) AC_MSG_CHECKING([whether to build bitcoin-util]) -AM_CONDITIONAL([BUILD_BITCOIN_UTIL], [test x$build_bitcoin_util = xyes]) +AM_CONDITIONAL([BUILD_BITCOIN_UTIL], [test $build_bitcoin_util = "yes"]) AC_MSG_RESULT($build_bitcoin_util) AC_MSG_CHECKING([whether to build libraries]) -AM_CONDITIONAL([BUILD_BITCOIN_LIBS], [test x$build_bitcoin_libs = xyes]) -if test x$build_bitcoin_libs = xyes; then - AC_DEFINE(HAVE_CONSENSUS_LIB, 1, [Define this symbol if the consensus lib has been built]) +AM_CONDITIONAL([BUILD_BITCOIN_LIBS], [test $build_bitcoin_libs = "yes"]) +if test "$build_bitcoin_libs" = "yes"; then + AC_DEFINE([HAVE_CONSENSUS_LIB], [1], [Define this symbol if the consensus lib has been built]) AC_CONFIG_FILES([libbitcoinconsensus.pc:libbitcoinconsensus.pc.in]) fi AC_MSG_RESULT($build_bitcoin_libs) AC_LANG_POP -if test "x$use_ccache" != "xno"; then - AC_MSG_CHECKING(if ccache should be used) - if test x$CCACHE = x; then - if test "x$use_ccache" = "xyes"; then +if test "$use_ccache" != "no"; then + AC_MSG_CHECKING([if ccache should be used]) + if test "$CCACHE" = ""; then + if test "$use_ccache" = "yes"; then AC_MSG_ERROR([ccache not found.]); else use_ccache=no @@ -1594,72 +1623,72 @@ if test "x$use_ccache" != "xno"; then CXX="$ac_cv_path_CCACHE $CXX" fi AC_MSG_RESULT($use_ccache) - if test "x$use_ccache" = "xyes"; then - AX_CHECK_COMPILE_FLAG([-fdebug-prefix-map=A=B],[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -fdebug-prefix-map=\$(abs_top_srcdir)=."],,[[$CXXFLAG_WERROR]]) - AX_CHECK_PREPROC_FLAG([-fmacro-prefix-map=A=B],[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -fmacro-prefix-map=\$(abs_top_srcdir)=."],,[[$CXXFLAG_WERROR]]) + if test "$use_ccache" = "yes"; then + AX_CHECK_COMPILE_FLAG([-fdebug-prefix-map=A=B], [DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -fdebug-prefix-map=\$(abs_top_srcdir)=."], [], [$CXXFLAG_WERROR]) + AX_CHECK_PREPROC_FLAG([-fmacro-prefix-map=A=B], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -fmacro-prefix-map=\$(abs_top_srcdir)=."], [], [$CXXFLAG_WERROR]) fi fi dnl enable wallet AC_MSG_CHECKING([if wallet should be enabled]) -if test x$enable_wallet != xno; then - AC_MSG_RESULT(yes) +if test "$enable_wallet" != "no"; then + AC_MSG_RESULT([yes]) AC_DEFINE_UNQUOTED([ENABLE_WALLET],[1],[Define to 1 to enable wallet functions]) enable_wallet=yes else - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) fi dnl enable upnp support AC_MSG_CHECKING([whether to build with support for UPnP]) -if test x$have_miniupnpc = xno; then - if test x$use_upnp = xyes; then - AC_MSG_ERROR("UPnP requested but cannot be built. Use --without-miniupnpc.") +if test "$have_miniupnpc" = "no"; then + if test "$use_upnp" = "yes"; then + AC_MSG_ERROR([UPnP requested but cannot be built. Use --without-miniupnpc]) fi - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) use_upnp=no else - if test x$use_upnp != xno; then - AC_MSG_RESULT(yes) + if test "$use_upnp" != "no"; then + AC_MSG_RESULT([yes]) AC_MSG_CHECKING([whether to build with UPnP enabled by default]) use_upnp=yes upnp_setting=0 - if test x$use_upnp_default != xno; then + if test "$use_upnp_default" != "no"; then use_upnp_default=yes upnp_setting=1 fi - AC_MSG_RESULT($use_upnp_default) + AC_MSG_RESULT([$use_upnp_default]) AC_DEFINE_UNQUOTED([USE_UPNP],[$upnp_setting],[UPnP support not compiled if undefined, otherwise value (0 or 1) determines default state]) - if test x$TARGET_OS = xwindows; then + if test "$TARGET_OS" = "windows"; then MINIUPNPC_CPPFLAGS="-DSTATICLIB -DMINIUPNP_STATICLIB" fi else - AC_MSG_RESULT(no) + AC_MSG_RESULT([no]) fi fi dnl Enable NAT-PMP support. AC_MSG_CHECKING([whether to build with support for NAT-PMP]) -if test "x$have_natpmp" = xno; then - if test "x$use_natpmp" = xyes; then +if test "$have_natpmp" = "no"; then + if test "$use_natpmp" = "yes"; then AC_MSG_ERROR([NAT-PMP requested but cannot be built. Use --without-natpmp]) fi AC_MSG_RESULT([no]) use_natpmp=no else - if test "x$use_natpmp" != xno; then + if test "$use_natpmp" != "no"; then AC_MSG_RESULT([yes]) AC_MSG_CHECKING([whether to build with NAT-PMP enabled by default]) use_natpmp=yes natpmp_setting=0 - if test "x$use_natpmp_default" != xno; then + if test "$use_natpmp_default" != "no"; then use_natpmp_default=yes natpmp_setting=1 fi AC_MSG_RESULT($use_natpmp_default) AC_DEFINE_UNQUOTED([USE_NATPMP], [$natpmp_setting], [NAT-PMP support not compiled if undefined, otherwise value (0 or 1) determines default state]) - if test x$TARGET_OS = xwindows; then + if test "$TARGET_OS" = "windows"; then NATPMP_CPPFLAGS="-DSTATICLIB -DNATPMP_STATICLIB" fi else @@ -1669,35 +1698,35 @@ fi dnl these are only used when qt is enabled BUILD_TEST_QT="" -if test x$bitcoin_enable_qt != xno; then +if test "$bitcoin_enable_qt" != "no"; then dnl enable dbus support AC_MSG_CHECKING([whether to build GUI with support for D-Bus]) - if test x$bitcoin_enable_qt_dbus != xno; then - AC_DEFINE([USE_DBUS],[1],[Define if dbus support should be compiled in]) + if test "$bitcoin_enable_qt_dbus" != "no"; then + AC_DEFINE([USE_DBUS], [1], [Define if dbus support should be compiled in]) fi - AC_MSG_RESULT($bitcoin_enable_qt_dbus) + AC_MSG_RESULT([$bitcoin_enable_qt_dbus]) dnl enable qr support AC_MSG_CHECKING([whether to build GUI with support for QR codes]) - if test x$have_qrencode = xno; then - if test x$use_qr = xyes; then + if test "$have_qrencode" = "no"; then + if test "$use_qr" = "yes"; then AC_MSG_ERROR([QR support requested but cannot be built. Use --without-qrencode]) fi use_qr=no else - if test x$use_qr != xno; then - AC_DEFINE([USE_QRCODE],[1],[Define if QR support should be compiled in]) + if test "$use_qr" != "no"; then + AC_DEFINE([USE_QRCODE], [1], [Define if QR support should be compiled in]) use_qr=yes fi fi AC_MSG_RESULT([$use_qr]) - if test x$XGETTEXT = x; then - AC_MSG_WARN("xgettext is required to update qt translations") + if test "$XGETTEXT" = ""; then + AC_MSG_WARN([xgettext is required to update qt translations]) fi AC_MSG_CHECKING([whether to build test_bitcoin-qt]) - if test x$use_gui_tests$bitcoin_enable_qt_test = xyesyes; then + if test "$use_gui_tests$bitcoin_enable_qt_test" = "yesyes"; then AC_MSG_RESULT([yes]) BUILD_TEST_QT="yes" else @@ -1705,11 +1734,11 @@ if test x$bitcoin_enable_qt != xno; then fi fi -AM_CONDITIONAL([ENABLE_ZMQ], [test "x$use_zmq" = "xyes"]) +AM_CONDITIONAL([ENABLE_ZMQ], [test "$use_zmq" = "yes"]) AC_MSG_CHECKING([whether to build test_bitcoin]) -if test x$use_tests = xyes; then - if test "x$enable_fuzz" = "xyes"; then +if test "$use_tests" = "yes"; then + if test "$enable_fuzz" = "yes"; then AC_MSG_RESULT([no, because fuzzing is enabled]) else AC_MSG_RESULT([yes]) @@ -1721,57 +1750,61 @@ else fi AC_MSG_CHECKING([whether to reduce exports]) -if test x$use_reduce_exports = xyes; then +if test "$use_reduce_exports" = "yes"; then AC_MSG_RESULT([yes]) else AC_MSG_RESULT([no]) fi -if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_libs$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests = xnononononononono; then +if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_libs$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests" = "nononononononono"; then AC_MSG_ERROR([No targets! Please specify at least one of: --with-utils --with-libs --with-daemon --with-gui --enable-bench or --enable-tests]) fi -if test x$enable_fuzz_binary = xyes; then +if test "$enable_fuzz_binary" = "yes"; then CHECK_RUNTIME_LIB fi -AM_CONDITIONAL([TARGET_DARWIN], [test x$TARGET_OS = xdarwin]) -AM_CONDITIONAL([BUILD_DARWIN], [test x$BUILD_OS = xdarwin]) -AM_CONDITIONAL([TARGET_LINUX], [test x$TARGET_OS = xlinux]) -AM_CONDITIONAL([TARGET_WINDOWS], [test x$TARGET_OS = xwindows]) -AM_CONDITIONAL([ENABLE_WALLET],[test x$enable_wallet = xyes]) -AM_CONDITIONAL([USE_SQLITE], [test "x$use_sqlite" = "xyes"]) -AM_CONDITIONAL([USE_BDB], [test "x$use_bdb" = "xyes"]) -AM_CONDITIONAL([ENABLE_TRACING],[test x$have_sdt = xyes]) -AM_CONDITIONAL([ENABLE_TESTS],[test x$BUILD_TEST = xyes]) -AM_CONDITIONAL([ENABLE_FUZZ],[test x$enable_fuzz = xyes]) -AM_CONDITIONAL([ENABLE_FUZZ_BINARY],[test x$enable_fuzz_binary = xyes]) -AM_CONDITIONAL([ENABLE_QT],[test x$bitcoin_enable_qt = xyes]) -AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$BUILD_TEST_QT = xyes]) -AM_CONDITIONAL([ENABLE_BENCH],[test x$use_bench = xyes]) -AM_CONDITIONAL([USE_QRCODE], [test x$use_qr = xyes]) -AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes]) -AM_CONDITIONAL([USE_LIBEVENT],[test x$use_libevent = xyes]) -AM_CONDITIONAL([HARDEN],[test x$use_hardening = xyes]) -AM_CONDITIONAL([ENABLE_SSE42],[test x$enable_sse42 = xyes]) -AM_CONDITIONAL([ENABLE_SSE41],[test x$enable_sse41 = xyes]) -AM_CONDITIONAL([ENABLE_AVX2],[test x$enable_avx2 = xyes]) -AM_CONDITIONAL([ENABLE_SHANI],[test x$enable_shani = xyes]) -AM_CONDITIONAL([ENABLE_ARM_CRC],[test x$enable_arm_crc = xyes]) -AM_CONDITIONAL([USE_ASM],[test x$use_asm = xyes]) -AM_CONDITIONAL([WORDS_BIGENDIAN],[test x$ac_cv_c_bigendian = xyes]) -AM_CONDITIONAL([USE_NATPMP],[test x$use_natpmp = xyes]) -AM_CONDITIONAL([USE_UPNP],[test x$use_upnp = xyes]) - -AC_DEFINE(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR, [Major version]) -AC_DEFINE(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR, [Minor version]) -AC_DEFINE(CLIENT_VERSION_BUILD, _CLIENT_VERSION_BUILD, [Version Build]) -AC_DEFINE(CLIENT_VERSION_IS_RELEASE, _CLIENT_VERSION_IS_RELEASE, [Version is release]) -AC_DEFINE(COPYRIGHT_YEAR, _COPYRIGHT_YEAR, [Copyright year]) -AC_DEFINE(COPYRIGHT_HOLDERS, "_COPYRIGHT_HOLDERS", [Copyright holder(s) before %s replacement]) -AC_DEFINE(COPYRIGHT_HOLDERS_SUBSTITUTION, "_COPYRIGHT_HOLDERS_SUBSTITUTION", [Replacement for %s in copyright holders string]) +AM_CONDITIONAL([TARGET_DARWIN], [test "$TARGET_OS" = "darwin"]) +AM_CONDITIONAL([BUILD_DARWIN], [test "$BUILD_OS" = "darwin"]) +AM_CONDITIONAL([TARGET_LINUX], [test "$TARGET_OS" = "linux"]) +AM_CONDITIONAL([TARGET_WINDOWS], [test "$TARGET_OS" = "windows"]) +AM_CONDITIONAL([ENABLE_WALLET], [test "$enable_wallet" = "yes"]) +AM_CONDITIONAL([USE_SQLITE], [test "$use_sqlite" = "yes"]) +AM_CONDITIONAL([USE_BDB], [test "$use_bdb" = "yes"]) +AM_CONDITIONAL([ENABLE_TRACING], [test "$have_sdt" = "yes"]) +AM_CONDITIONAL([ENABLE_TESTS], [test "$BUILD_TEST" = "yes"]) +AM_CONDITIONAL([ENABLE_FUZZ], [test "$enable_fuzz" = "yes"]) +AM_CONDITIONAL([ENABLE_FUZZ_BINARY], [test "$enable_fuzz_binary" = "yes"]) +AM_CONDITIONAL([ENABLE_QT], [test "$bitcoin_enable_qt" = "yes"]) +AM_CONDITIONAL([ENABLE_QT_TESTS], [test "$BUILD_TEST_QT" = "yes"]) +AM_CONDITIONAL([ENABLE_BENCH], [test "$use_bench" = "yes"]) +AM_CONDITIONAL([USE_QRCODE], [test "$use_qr" = "yes"]) +AM_CONDITIONAL([USE_LCOV], [test "$use_lcov" = "yes"]) +AM_CONDITIONAL([USE_LIBEVENT], [test "$use_libevent" = "yes"]) +AM_CONDITIONAL([HARDEN], [test "$use_hardening" = "yes"]) +AM_CONDITIONAL([ENABLE_SSE42], [test "$enable_sse42" = "yes"]) +AM_CONDITIONAL([ENABLE_SSE41], [test "$enable_sse41" = "yes"]) +AM_CONDITIONAL([ENABLE_AVX2], [test "$enable_avx2" = "yes"]) +AM_CONDITIONAL([ENABLE_SHANI], [test "$enable_shani" = "yes"]) +AM_CONDITIONAL([ENABLE_ARM_CRC], [test "$enable_arm_crc" = "yes"]) +AM_CONDITIONAL([USE_ASM], [test "$use_asm" = "yes"]) +AM_CONDITIONAL([WORDS_BIGENDIAN], [test "$ac_cv_c_bigendian" = "yes"]) +AM_CONDITIONAL([USE_NATPMP], [test "$use_natpmp" = "yes"]) +AM_CONDITIONAL([USE_UPNP], [test "$use_upnp" = "yes"]) + +dnl for minisketch +AM_CONDITIONAL([ENABLE_CLMUL], [test "$enable_clmul" = "yes"]) +AM_CONDITIONAL([HAVE_CLZ], [test "$have_clzl$have_clzll" = "yesyes"]) + +AC_DEFINE([CLIENT_VERSION_MAJOR], [_CLIENT_VERSION_MAJOR], [Major version]) +AC_DEFINE([CLIENT_VERSION_MINOR], [_CLIENT_VERSION_MINOR], [Minor version]) +AC_DEFINE([CLIENT_VERSION_BUILD], [_CLIENT_VERSION_BUILD], [Version Build]) +AC_DEFINE([CLIENT_VERSION_IS_RELEASE], [_CLIENT_VERSION_IS_RELEASE], [Version is release]) +AC_DEFINE([COPYRIGHT_YEAR], [_COPYRIGHT_YEAR], [Copyright year]) +AC_DEFINE([COPYRIGHT_HOLDERS], ["_COPYRIGHT_HOLDERS"], [Copyright holder(s) before %s replacement]) +AC_DEFINE([COPYRIGHT_HOLDERS_SUBSTITUTION], ["_COPYRIGHT_HOLDERS_SUBSTITUTION"], [Replacement for %s in copyright holders string]) define(_COPYRIGHT_HOLDERS_FINAL, [patsubst(_COPYRIGHT_HOLDERS, [%s], [_COPYRIGHT_HOLDERS_SUBSTITUTION])]) -AC_DEFINE(COPYRIGHT_HOLDERS_FINAL, "_COPYRIGHT_HOLDERS_FINAL", [Copyright holder(s)]) +AC_DEFINE([COPYRIGHT_HOLDERS_FINAL], ["_COPYRIGHT_HOLDERS_FINAL"], [Copyright holder(s)]) AC_SUBST(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR) AC_SUBST(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR) AC_SUBST(CLIENT_VERSION_BUILD, _CLIENT_VERSION_BUILD) @@ -1800,12 +1833,15 @@ AC_SUBST(GPROF_LDFLAGS) AC_SUBST(HARDENED_CXXFLAGS) AC_SUBST(HARDENED_CPPFLAGS) AC_SUBST(HARDENED_LDFLAGS) +AC_SUBST(LTO_CXXFLAGS) +AC_SUBST(LTO_LDFLAGS) AC_SUBST(PIC_FLAGS) AC_SUBST(PIE_FLAGS) AC_SUBST(SANITIZER_CXXFLAGS) AC_SUBST(SANITIZER_LDFLAGS) AC_SUBST(SSE42_CXXFLAGS) AC_SUBST(SSE41_CXXFLAGS) +AC_SUBST(CLMUL_CXXFLAGS) AC_SUBST(AVX2_CXXFLAGS) AC_SUBST(SHANI_CXXFLAGS) AC_SUBST(ARM_CRC_CXXFLAGS) @@ -1890,16 +1926,16 @@ echo " multiprocess = $build_multiprocess" echo " with experimental syscall sandbox support = $use_syscall_sandbox" echo " with libs = $build_bitcoin_libs" echo " with wallet = $enable_wallet" -if test "x$enable_wallet" != "xno"; then +if test "$enable_wallet" != "no"; then echo " with sqlite = $use_sqlite" echo " with bdb = $use_bdb" fi echo " with gui / qt = $bitcoin_enable_qt" -if test x$bitcoin_enable_qt != xno; then +if test $bitcoin_enable_qt != "no"; then echo " with qr = $use_qr" fi echo " with zmq = $use_zmq" -if test x$enable_fuzz == xno; then +if test $enable_fuzz = "no"; then echo " with test = $use_tests" else echo " with test = not building test_bitcoin because fuzzing is enabled" @@ -1914,6 +1950,7 @@ echo " sanitizers = $use_sanitizers" echo " debug enabled = $enable_debug" echo " gprof enabled = $enable_gprof" echo " werror = $enable_werror" +echo " LTO = $enable_lto" echo echo " target os = $TARGET_OS" echo " build os = $build_os" @@ -1922,7 +1959,7 @@ echo " CC = $CC" echo " CFLAGS = $PTHREAD_CFLAGS $CFLAGS" echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CPPFLAGS" echo " CXX = $CXX" -echo " CXXFLAGS = $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS" -echo " LDFLAGS = $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS" +echo " CXXFLAGS = $LTO_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS" +echo " LDFLAGS = $LTO_LDFLAGS $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS" echo " ARFLAGS = $ARFLAGS" echo diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index 9a555c70bb..d6914bf655 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -33,6 +33,7 @@ EXCLUDE_DIRS = [ # git subtrees "src/crypto/ctaes/", "src/leveldb/", + "src/minisketch", "src/secp256k1/", "src/univalue/", "src/crc32c/", diff --git a/contrib/devtools/gen-manpages.sh b/contrib/devtools/gen-manpages.sh index b7bf76ce77..753a5a2494 100755 --- a/contrib/devtools/gen-manpages.sh +++ b/contrib/devtools/gen-manpages.sh @@ -17,7 +17,7 @@ WALLET_TOOL=${WALLET_TOOL:-$BINDIR/bitcoin-wallet} BITCOINUTIL=${BITCOINQT:-$BINDIR/bitcoin-util} BITCOINQT=${BITCOINQT:-$BINDIR/qt/bitcoin-qt} -[ ! -x $BITCOIND ] && echo "$BITCOIND not found or not executable." && exit 1 +[ ! -x "$BITCOIND" ] && echo "$BITCOIND not found or not executable." && exit 1 # Don't allow man pages to be generated for binaries built from a dirty tree DIRTY="" @@ -30,7 +30,7 @@ done if [ -n "$DIRTY" ] then echo -e "WARNING: the following binaries were built from a dirty tree:\n" - echo -e $DIRTY + echo -e "$DIRTY" echo "man pages generated from dirty binaries should NOT be committed." echo "To properly generate man pages, please commit your changes to the above binaries, rebuild them, then run this script again." fi @@ -46,8 +46,8 @@ $BITCOIND --version | sed -n '1!p' >> footer.h2m for cmd in $BITCOIND $BITCOINCLI $BITCOINTX $WALLET_TOOL $BITCOINUTIL $BITCOINQT; do cmdname="${cmd##*/}" - help2man -N --version-string=${BTCVER[0]} --include=footer.h2m -o ${MANDIR}/${cmdname}.1 ${cmd} - sed -i "s/\\\-${BTCVER[1]}//g" ${MANDIR}/${cmdname}.1 + help2man -N --version-string="${BTCVER[0]}" --include=footer.h2m -o "${MANDIR}/${cmdname}.1" "${cmd}" + sed -i "s/\\\-${BTCVER[1]}//g" "${MANDIR}/${cmdname}.1" done rm -f footer.h2m diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index ef421aebb1..677557b8fa 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -121,6 +121,21 @@ def check_PE_RELOC_SECTION(binary) -> bool: '''Check for a reloc section. This is required for functional ASLR.''' return binary.has_relocations +def check_PE_control_flow(binary) -> bool: + ''' + Check for control flow instrumentation + ''' + main = binary.get_symbol('main').value + + section_addr = binary.section_from_rva(main).virtual_address + virtual_address = binary.optional_header.imagebase + section_addr + main + + content = binary.get_content_from_virtual_address(virtual_address, 4, lief.Binary.VA_TYPES.VA) + + if content == [243, 15, 30, 250]: # endbr64 + return True + return False + def check_MACHO_NOUNDEFS(binary) -> bool: ''' Check for no undefined references. @@ -177,7 +192,8 @@ CHECKS = { ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), ('NX', check_NX), - ('RELOC_SECTION', check_PE_RELOC_SECTION) + ('RELOC_SECTION', check_PE_RELOC_SECTION), + ('CONTROL_FLOW', check_PE_control_flow), ], 'MACHO': [ ('PIE', check_PIE), diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 136a9b70c1..ec3b7691e9 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -19,35 +19,31 @@ import lief #type:ignore # https://github.com/lief-project/LIEF/pull/562 LIEF_ELF_ARCH_RISCV = lief.ELF.ARCH(243) -# Debian 8 (Jessie) EOL: 2020. https://wiki.debian.org/DebianReleases#Production_Releases +# Debian 9 (Stretch) EOL: 2022. https://wiki.debian.org/DebianReleases#Production_Releases # -# - g++ version 4.9.2 (https://packages.debian.org/search?suite=jessie&arch=any&searchon=names&keywords=g%2B%2B) -# - libc version 2.19 (https://packages.debian.org/search?suite=jessie&arch=any&searchon=names&keywords=libc6) +# - g++ version 6.3.0 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=g%2B%2B) +# - libc version 2.24 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=libc6) # -# Ubuntu 16.04 (Xenial) EOL: 2024. https://wiki.ubuntu.com/Releases +# Ubuntu 16.04 (Xenial) EOL: 2026. https://wiki.ubuntu.com/Releases # -# - g++ version 5.3.1 (https://packages.ubuntu.com/search?keywords=g%2B%2B&searchon=names&suite=xenial§ion=all) -# - libc version 2.23.0 (https://packages.ubuntu.com/search?keywords=libc6&searchon=names&suite=xenial§ion=all) +# - g++ version 5.3.1 +# - libc version 2.23 # -# CentOS 7 EOL: 2024. https://wiki.centos.org/FAQ/General +# CentOS Stream 8 EOL: 2024. https://wiki.centos.org/About/Product # -# - g++ version 4.8.5 (http://mirror.centos.org/centos/7/os/x86_64/Packages/) -# - libc version 2.17 (http://mirror.centos.org/centos/7/os/x86_64/Packages/) -# -# Taking the minimum of these as our target. -# -# According to GNU ABI document (https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html) this corresponds to: -# GCC 4.8.5: GCC_4.8.0 -# (glibc) GLIBC_2_17 +# - g++ version 8.5.0 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) +# - libc version 2.28 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) # +# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info. + MAX_VERSIONS = { 'GCC': (4,8,0), 'GLIBC': { - lief.ELF.ARCH.i386: (2,17), - lief.ELF.ARCH.x86_64: (2,17), - lief.ELF.ARCH.ARM: (2,17), - lief.ELF.ARCH.AARCH64:(2,17), - lief.ELF.ARCH.PPC64: (2,17), + lief.ELF.ARCH.i386: (2,18), + lief.ELF.ARCH.x86_64: (2,18), + lief.ELF.ARCH.ARM: (2,18), + lief.ELF.ARCH.AARCH64:(2,18), + lief.ELF.ARCH.PPC64: (2,18), LIEF_ELF_ARCH_RISCV: (2,27), }, 'LIBATOMIC': (1,0), @@ -109,7 +105,19 @@ ELF_ALLOWED_LIBRARIES = { 'libxkbcommon-x11.so.0', # keyboard keymapping 'libfontconfig.so.1', # font support 'libfreetype.so.6', # font parsing -'libdl.so.2' # programming interface to dynamic linker +'libdl.so.2', # programming interface to dynamic linker +'libxcb-icccm.so.4', +'libxcb-image.so.0', +'libxcb-shm.so.0', +'libxcb-keysyms.so.1', +'libxcb-randr.so.0', +'libxcb-render-util.so.0', +'libxcb-render.so.0', +'libxcb-shape.so.0', +'libxcb-sync.so.1', +'libxcb-xfixes.so.0', +'libxcb-xinerama.so.0', +'libxcb-xkb.so.1', } MACHO_ALLOWED_LIBRARIES = { @@ -120,6 +128,7 @@ MACHO_ALLOWED_LIBRARIES = { 'AppKit', # user interface 'ApplicationServices', # common application tasks. 'Carbon', # deprecated c back-compat API +'ColorSync', 'CoreFoundation', # low level func, data types 'CoreGraphics', # 2D rendering 'CoreServices', # operating system services diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 0af7cdf5e6..01df863ac0 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -70,16 +70,18 @@ class TestSecurityChecks(unittest.TestCase): write_testcode(source) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION')) + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION CONTROL_FLOW')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION')) + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION CONTROL_FLOW')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA')) + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA')) # -pie -fPIE does nothing unless --dynamicbase is also supplied + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) # -pie -fPIE does nothing unless --dynamicbase is also supplied self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']), - (1, executable+': failed HIGH_ENTROPY_VA')) + (1, executable+': failed HIGH_ENTROPY_VA CONTROL_FLOW')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']), + (1, executable+': failed CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE', '-fcf-protection=full']), (0, '')) clean_files(source, executable) diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py index 5246375fe3..d699e85026 100755 --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -44,7 +44,7 @@ class TestSymbolChecks(unittest.TestCase): self.skipTest("test not available for RISC-V") # nextup was introduced in GLIBC 2.24, so is newer than our supported - # glibc (2.17), and available in our release build environment (2.24). + # glibc (2.18), and available in our release build environment (2.24). with open(source, 'w', encoding="utf8") as f: f.write(''' #define _GNU_SOURCE diff --git a/contrib/guix/guix-clean b/contrib/guix/guix-clean index 9fa17191e8..9af0a793cf 100755 --- a/contrib/guix/guix-clean +++ b/contrib/guix/guix-clean @@ -34,7 +34,7 @@ check_tools cat mkdir make git guix # under_dir() { local path_residue - path_residue="${2##${1}}" + path_residue="${2##"${1}"}" if [ -z "$path_residue" ] || [ "$path_residue" = "$2" ]; then return 1 else diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index e009f97c60..596d0ca1fb 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -238,9 +238,6 @@ mkdir -p "$OUTDIR" # CONFIGFLAGS CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disable-fuzz-binary" -case "$HOST" in - *linux*) CONFIGFLAGS+=" --disable-threadlocal" ;; -esac # CFLAGS HOST_CFLAGS="-O2 -g" @@ -263,7 +260,7 @@ case "$HOST" in *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; esac -# Using --no-tls-get-addr-optimize retains compatibility with glibc 2.17, by +# Using --no-tls-get-addr-optimize retains compatibility with glibc 2.18, by # avoiding a PowerPC64 optimisation available in glibc 2.22 and later. # https://sourceware.org/binutils/docs-2.35/ld/PowerPC64-ELF64.html case "$HOST" in diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index 40ae4b5208..1f287e9bbb 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -2,10 +2,10 @@ export LC_ALL=C set -e -o pipefail -# shellcheck source=../../shell/realpath.bash +# shellcheck source=contrib/shell/realpath.bash source contrib/shell/realpath.bash -# shellcheck source=../../shell/git-utils.bash +# shellcheck source=contrib/shell/git-utils.bash source contrib/shell/git-utils.bash ################ diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh index dd4d862dee..81c88a2ae7 100755 --- a/contrib/install_db4.sh +++ b/contrib/install_db4.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -20,7 +20,7 @@ expand_path() { cd "${1}" && pwd -P } -BDB_PREFIX="$(expand_path ${1})/db4"; shift; +BDB_PREFIX="$(expand_path "${1}")/db4"; shift; BDB_VERSION='db-4.8.30.NC' BDB_HASH='12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef' BDB_URL="https://download.oracle.com/berkeley-db/${BDB_VERSION}.tar.gz" @@ -62,6 +62,12 @@ http_get() { sha256_check "${3}" "${2}" } +# Ensure the commands we use exist on the system +if ! check_exists patch; then + echo "Command-line tool 'patch' not found. Install patch and try again." + exit 1 +fi + mkdir -p "${BDB_PREFIX}" http_get "${BDB_URL}" "${BDB_VERSION}.tar.gz" "${BDB_HASH}" tar -xzvf ${BDB_VERSION}.tar.gz -C "$BDB_PREFIX" diff --git a/contrib/macdeploy/detached-sig-apply.sh b/contrib/macdeploy/detached-sig-apply.sh index d481413cc3..813e99f9da 100755 --- a/contrib/macdeploy/detached-sig-apply.sh +++ b/contrib/macdeploy/detached-sig-apply.sh @@ -22,6 +22,6 @@ if [ -z "$SIGNATURE" ]; then exit 1 fi -${SIGNAPPLE} apply ${UNSIGNED} ${SIGNATURE} +${SIGNAPPLE} apply "${UNSIGNED}" "${SIGNATURE}" mv ${ROOTDIR} ${OUTDIR} echo "Signed: ${OUTDIR}" diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk index 457d8f5e64..ebef1d2db0 100755 --- a/contrib/macdeploy/gen-sdk +++ b/contrib/macdeploy/gen-sdk @@ -81,7 +81,7 @@ def run(): print("Creating output .tar.gz file...") with out_sdktgz_path.open("wb") as fp: - with gzip.GzipFile(fileobj=fp, compresslevel=9, mtime=0) as gzf: + with gzip.GzipFile(fileobj=fp, mode='wb', compresslevel=9, mtime=0) as gzf: with tarfile.open(mode="w", fileobj=gzf) as tarfp: print("Adding MacOSX SDK {} files...".format(sdk_version)) tarfp_add_with_base_change(tarfp, sdk_dir, out_name) diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py index dbecba7d1d..44345e3987 100755 --- a/contrib/seeds/generate-seeds.py +++ b/contrib/seeds/generate-seeds.py @@ -61,7 +61,7 @@ def name_to_bip155(addr): raise ValueError(f'Invalid I2P {vchAddr}') elif '.' in addr: # IPv4 return (BIP155Network.IPV4, bytes((int(x) for x in addr.split('.')))) - elif ':' in addr: # IPv6 + elif ':' in addr: # IPv6 or CJDNS sub = [[], []] # prefix, suffix x = 0 addr = addr.split(':') @@ -77,7 +77,14 @@ def name_to_bip155(addr): sub[x].append(val & 0xff) nullbytes = 16 - len(sub[0]) - len(sub[1]) assert((x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0)) - return (BIP155Network.IPV6, bytes(sub[0] + ([0] * nullbytes) + sub[1])) + addr_bytes = bytes(sub[0] + ([0] * nullbytes) + sub[1]) + if addr_bytes[0] == 0xfc: + # Assume that seeds with fc00::/8 addresses belong to CJDNS, + # not to the publicly unroutable "Unique Local Unicast" network, see + # RFC4193: https://datatracker.ietf.org/doc/html/rfc4193#section-8 + return (BIP155Network.CJDNS, addr_bytes) + else: + return (BIP155Network.IPV6, addr_bytes) else: raise ValueError('Could not parse address %s' % addr) diff --git a/contrib/tracing/README.md b/contrib/tracing/README.md index 1f93474fa0..b71ce2f34b 100644 --- a/contrib/tracing/README.md +++ b/contrib/tracing/README.md @@ -234,3 +234,62 @@ Histogram of block connection times in milliseconds (ms). [16, 32) 9 | | [32, 64) 4 | | ``` + +### log_utxocache_flush.py + +A BCC Python script to log the cache and index flushes. Based on the +`utxocache:flush` tracepoint. + +```bash +$ python3 contrib/tracing/log_utxocache_flush.py ./src/bitcoind +``` + +``` +Logging utxocache flushes. Ctrl-C to end... +Duration (µs) Mode Coins Count Memory Usage Prune Full Flush +0 PERIODIC 28484 3929.87 kB False False +1 PERIODIC 28485 3930.00 kB False False +0 PERIODIC 28489 3930.51 kB False False +1 PERIODIC 28490 3930.64 kB False False +0 PERIODIC 28491 3930.77 kB False False +0 PERIODIC 28491 3930.77 kB False False +0 PERIODIC 28496 3931.41 kB False False +1 PERIODIC 28496 3931.41 kB False False +0 PERIODIC 28497 3931.54 kB False False +1 PERIODIC 28497 3931.54 kB False False +1 PERIODIC 28499 3931.79 kB False False +. +. +. +53788 ALWAYS 30076 4136.27 kB False False +7463 ALWAYS 0 245.84 kB False False +``` + +### log_utxos.bt + +A `bpftrace` script to log information about the coins that are added, spent, or +uncached from the UTXO set. Based on the `utxocache:add`, `utxocache:spend` and +`utxocache:uncache` tracepoints. + +```bash +$ bpftrace contrib/tracing/log_utxos.bt +``` + +It should produce an output similar to the following. + +```bash +Attaching 4 probes... +OP Outpoint Value Height Coinbase +Added 6ba9ad857e1ef2eb2a2c94f06813c414c7ab273e3d6bd7ad64e000315a887e7c:1 10000 2094512 No +Spent fa7dc4db56637a151f6649d8f26732956d1c5424c82aae400a83d02b2cc2c87b:0 182264897 2094512 No +Added eeb2f099b1af6a2a12e6ddd2eeb16fc5968582241d7f08ba202d28b60ac264c7:0 10000 2094512 No +Added eeb2f099b1af6a2a12e6ddd2eeb16fc5968582241d7f08ba202d28b60ac264c7:1 182254756 2094512 No +Added a0c7f4ec9cccef2d89672a624a4e6c8237a17572efdd4679eea9e9ee70d2db04:0 10072679 2094513 Yes +Spent 25e0df5cc1aeb1b78e6056bf403e5e8b7e41f138060ca0a50a50134df0549a5e:2 540 2094508 No +Spent 42f383c04e09c26a2378272ec33aa0c1bf4883ca5ab739e8b7e06be5a5787d61:1 3848399 2007724 No +Added f85e3b4b89270863a389395cc9a4123e417ab19384cef96533c6649abd6b0561:0 3788399 2094513 No +Added f85e3b4b89270863a389395cc9a4123e417ab19384cef96533c6649abd6b0561:2 540 2094513 No +Spent a05880b8c77971ed0b9f73062c7c4cdb0ff3856ab14cbf8bc481ed571cd34b83:1 5591281046 2094511 No +Added eb689865f7d957938978d6207918748f74e6aa074f47874724327089445b0960:0 5589696005 2094513 No +Added eb689865f7d957938978d6207918748f74e6aa074f47874724327089445b0960:1 1565556 2094513 No +``` diff --git a/contrib/tracing/log_utxocache_flush.py b/contrib/tracing/log_utxocache_flush.py new file mode 100755 index 0000000000..df27dc193a --- /dev/null +++ b/contrib/tracing/log_utxocache_flush.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 + +import sys +import ctypes +from bcc import BPF, USDT + +"""Example logging Bitcoin Core utxo set cache flushes utilizing + the utxocache:flush tracepoint.""" + +# USAGE: ./contrib/tracing/log_utxocache_flush.py path/to/bitcoind + +# BCC: The C program to be compiled to an eBPF program (by BCC) and loaded into +# a sandboxed Linux kernel VM. +program = """ +# include <uapi/linux/ptrace.h> +struct data_t +{ + u64 duration; + u32 mode; + u64 coins_count; + u64 coins_mem_usage; + bool is_flush_prune; + bool is_full_flush; +}; + +// BPF perf buffer to push the data to user space. +BPF_PERF_OUTPUT(flush); + +int trace_flush(struct pt_regs *ctx) { + struct data_t data = {}; + bpf_usdt_readarg(1, ctx, &data.duration); + bpf_usdt_readarg(2, ctx, &data.mode); + bpf_usdt_readarg(3, ctx, &data.coins_count); + bpf_usdt_readarg(4, ctx, &data.coins_mem_usage); + bpf_usdt_readarg(5, ctx, &data.is_flush_prune); + bpf_usdt_readarg(5, ctx, &data.is_full_flush); + flush.perf_submit(ctx, &data, sizeof(data)); + return 0; +} +""" + +FLUSH_MODES = [ + 'NONE', + 'IF_NEEDED', + 'PERIODIC', + 'ALWAYS' +] + + +class Data(ctypes.Structure): + # define output data structure corresponding to struct data_t + _fields_ = [ + ("duration", ctypes.c_uint64), + ("mode", ctypes.c_uint32), + ("coins_count", ctypes.c_uint64), + ("coins_mem_usage", ctypes.c_uint64), + ("is_flush_prune", ctypes.c_bool), + ("is_full_flush", ctypes.c_bool) + ] + + +def print_event(event): + print("%-15d %-10s %-15d %-15s %-8s %-8s" % ( + event.duration, + FLUSH_MODES[event.mode], + event.coins_count, + "%.2f kB" % (event.coins_mem_usage/1000), + event.is_flush_prune, + event.is_full_flush + )) + + +def main(bitcoind_path): + bitcoind_with_usdts = USDT(path=str(bitcoind_path)) + + # attaching the trace functions defined in the BPF program + # to the tracepoints + bitcoind_with_usdts.enable_probe( + probe="flush", fn_name="trace_flush") + b = BPF(text=program, usdt_contexts=[bitcoind_with_usdts]) + + def handle_flush(_, data, size): + """ Coins Flush handler. + Called each time coin caches and indexes are flushed.""" + event = ctypes.cast(data, ctypes.POINTER(Data)).contents + print_event(event) + + b["flush"].open_perf_buffer(handle_flush) + print("Logging utxocache flushes. Ctrl-C to end...") + print("%-15s %-10s %-15s %-15s %-8s %-8s" % ("Duration (µs)", "Mode", + "Coins Count", "Memory Usage", + "Prune", "Full Flush")) + + while True: + try: + b.perf_buffer_poll() + except KeyboardInterrupt: + exit(0) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("USAGE: ", sys.argv[0], "path/to/bitcoind") + exit(1) + + path = sys.argv[1] + main(path) diff --git a/contrib/tracing/log_utxos.bt b/contrib/tracing/log_utxos.bt new file mode 100755 index 0000000000..0d47f3d62b --- /dev/null +++ b/contrib/tracing/log_utxos.bt @@ -0,0 +1,86 @@ +#!/usr/bin/env bpftrace + +/* + + USAGE: + + bpftrace contrib/tracing/log_utxos.bt + + This script requires a 'bitcoind' binary compiled with eBPF support and the + 'utxochache' tracepoints. By default, it's assumed that 'bitcoind' is + located in './src/bitcoind'. This can be modified in the script below. + + NOTE: requires bpftrace v0.12.0 or above. +*/ + +BEGIN +{ + printf("%-7s %-71s %16s %7s %8s\n", + "OP", "Outpoint", "Value", "Height", "Coinbase"); +} + +/* + Attaches to the 'utxocache:add' tracepoint and prints additions to the UTXO set cache. +*/ +usdt:./src/bitcoind:utxocache:add +{ + $txid = arg0; + $index = (uint32)arg1; + $height = (uint32)arg2; + $value = (int64)arg3; + $isCoinbase = arg4; + + printf("Added "); + $p = $txid + 31; + unroll(32) { + $b = *(uint8*)$p; + printf("%02x", $b); + $p-=1; + } + + printf(":%-6d %16ld %7d %s\n", $index, $value, $height, ($isCoinbase ? "Yes" : "No" )); +} + +/* + Attaches to the 'utxocache:spent' tracepoint and prints spents from the UTXO set cache. +*/ +usdt:./src/bitcoind:utxocache:spent +{ + $txid = arg0; + $index = (uint32)arg1; + $height = (uint32)arg2; + $value = (int64)arg3; + $isCoinbase = arg4; + + printf("Spent "); + $p = $txid + 31; + unroll(32) { + $b = *(uint8*)$p; + printf("%02x", $b); + $p-=1; + } + + printf(":%-6d %16ld %7d %s\n", $index, $value, $height, ($isCoinbase ? "Yes" : "No" )); +} + +/* + Attaches to the 'utxocache:uncache' tracepoint and uncache UTXOs from the UTXO set cache. +*/ +usdt:./src/bitcoind:utxocache:uncache +{ + $txid = arg0; + $index = (uint32)arg1; + $height = (uint32)arg2; + $value = (int64)arg3; + $isCoinbase = arg4; + + printf("Uncache "); + $p = $txid + 31; + unroll(32) { + $b = *(uint8*)$p; + printf("%02x", $b); + $p-=1; + } + + printf(":%-6d %16ld %7d %s\n", $index, $value, $height, ($isCoinbase ? "Yes" : "No" )); +} diff --git a/contrib/verify-commits/allow-revsig-commits b/contrib/verify-commits/allow-revsig-commits index 3abf82e529..0bb299b8fa 100644 --- a/contrib/verify-commits/allow-revsig-commits +++ b/contrib/verify-commits/allow-revsig-commits @@ -502,3 +502,144 @@ e2bf830bb6c1bfa038c943dd6f5d92a406bd723f 0ad104190465d8d65c2344bbe10dcf3df025d86c 5c7df7022bcd360e6af00b9458b1a3fd54e1cc9a 59ad56851a342d2c62f6b38bf15002b23ab439e1 +d8cd7b137fb075616f31d2b43b85fa2e27ea7477 +655937ebcbf681ededf86b1f0f60aac45c73393d +abdfd2d0e3ebec7dbead89317ee9192189a35809 +e439aeb30c0439001a781c5979aec41e1fc2aa50 +b9b26d9c3615d15669ae0a049c1dede39a9e59a9 +fdf146f3293c487afdc4d6d9f6b64099aa8bd28a +16e3b175781caacee403a2dc40cd6c70448e12ef +b30c62d4b954df05bf404cfbeb5b728282201496 +b3f377daaa86cd7755a552fa3adfeb195835f58e +0a8f519a0626d7cd385114ce610d23215c051b3f +544f3234384b2f6c290e987ad18576e1b50d7db9 +91482e5bf22d283d32c9f83c8057f10971848107 +e754c6e33194e9ed69ba5350c5139b0423b645fe +dc1e54206d76e5fa378d28a18ae1fb2bcf714485 +b2863c0685a5c12f829095cbabaf26ccc49e46ec +b14db5abab405a708f0166293f1ea12222a6bf03 +8010ded6da56842c09b14665343cd189d7e08401 +d387507aeca652a5569825af65243536f2ce26ea +27bf14f6f3e0fb1f348f13c1b54fc6b67b3bef6e +f8d470e24606297dab95e30b1d39ff664fbda31d +b25a4c2284babdf1e8cf0ec3b1402200dd25f33f +1329ef1f00e4fad83937ddd8721d0292ccfe7808 +9a1ad2c5cbdfa3114d05df57103c34f72e087f26 +1e90862f5d0b5f7dcc18fb018b2bbaa323dcca1d +ad552a54c56a420be84b47154882c3e4c76f9bdd +90b1c7e5c50551f39d4983008d1d5ab3b085803e +d6b2235ca45e072961e25a35e6a159e97c9e556b +2643fa50869f22672cbc72ac497d9c30234075b8 +01f909828d126d5443bc28758f51781eccdf5848 +f54f3738c8ce839c413d7b6b719be2ff341536ca +418ae49ee1eac2c9d6cd4ba83c036a41f1afe922 +5a666428b0f11d62af2002bd54a45ff2f79f30cd +a07e8caa5d5000286604458e6887f57fec7fdcbb +8b262eb2d80bfa27ae8501078ce47bc1407e9c55 +5df84de583c900e00fef63bedaef32786f205a33 +4ba6da55743a55189164e29e45ac9e73a074d808 +88430cbab4dca36b6a867364cab319cde6a9ebca +e0f7515f5500968c86e5a9f4912d83d4abc5b2b9 +9b8b1079ddab64ac955766536c38d23dc57bc499 +af20f9b1d485582b8c8aa8294bac4f2c540246d2 +7be9a9a570c1140048f8781ced1111e1d930e517 +2bac3e484114c30548e286972525dd799dbd0a5b +df529dcc65e8037c5a3a3ad74545be294a770f07 +6acd8700bc0ee1d10207a362c1e07372ba274041 +ffc6e48b2983189dc0ce7de0a038e5329bc07b1b +252ae7111cbff09a4cbc5caee9e02b6ed3580476 +b3ecb7bab6074377d87c700bf0c5d351e5d3174f +d9fdac130a5ed1d96fcac6bb87c10bec9d596b17 +a07e8caa5d5000286604458e6887f57fec7fdcbb +8b262eb2d80bfa27ae8501078ce47bc1407e9c55 +5df84de583c900e00fef63bedaef32786f205a33 +4ba6da55743a55189164e29e45ac9e73a074d808 +5bea05bc1d17aa43cbdf3a3413241f8132790d93 +c17f11f7b43ad3bd9e242c67db1f3679558a0581 +5ea932a51083837cdd27715e10a3a0d5d553af24 +033c78671b91b12d589ebff6c5ede8d94d7500f8 +ef8a634358848847e006c43ce621bc17a612fd1f +ba216b5fa63e7e6cae847d1e3621f5c54840f898 +26fee4f6bd9aec62c6caa60683ad66574cf16aa6 +6ab0e4cf49549640b903bf5fce0e6035b8116397 +326a5652e0d25fdb60c337ef4f1c98a63e0748f0 +424be03305143cbe5da5d5adb54d73d3dc3747b6 +38c201f47c0bc388a05cdb35d6137150fa90193e +12ed800ab870e0fc527a84d6e4584b10c8d239f5 +aeed345c9bade5d52a3fbf0a943203f6c82e6344 +c6223b3daab0328ca742b1cc3c15e89e698630bb +877678710800a4d78afc12519424f232f1a583d3 +6c4fecfaf7beefad0d1c3f8520bf50bb515a0716 +98212745c8acb5cc4e688bbb3979bfd46b25f98a +b9bceaf1c081a84d9fcc680372614e797b168a9e +1afc22a7667a7a5c66b4b5d7f50832356dd5ec12 +3255d6347b1f9eccbec3d6d93d4a424087a3b35b +ec20f01ba0945a3113797ac98a6b3500e24603d4 +75b5643c47c3b382ed97a9f5e2bdc883a0f98709 +fee0d803fb55c8d85b5cd1ff69d799c5ad522e18 +565494619d809655fa94d274bb2202d25553e485 +ad6fce67b9bb6eee864c8431ad3291aebaa2e5d2 +99c7db8731cc77f143b52f544b3fdd93033ed20d +b4d03be3cac04da8b5d5fa17e29c5220b75d970b +ef37f2033c4ae104585cd980141262f95d33166e +5cfdda2503c995cdd563b1a2a29162ac298d173d +c5904e871479514b2e2e18b4fdbbe468c4e5ec8e +10b22e3141a603ec891d2cfc7100c29c7409aabe +afd2fca911c4a5e3a4d1f0993a226d40f250aff4 +505955052e60e0681865f3064e005ca0d3aa90bf +8fdd23a224ba236874ef662c4ca311b002dbcab3 +1c011ff430106b5f727f2eaa0f7f4883cd2122a3 +ec8a50b8d786a8cd1192e692ab19b46979add582 +f90603ac6d24f5263649675d51233f1fce8b2ecd +b7d6623c76e1468f2a93db5a3120580e2784d74a +66270a416edb1610f276124483feceef9cba93ff +e4fcbf797ed3b472d352ac3794ec82f581209c50 +479afa0f8486146a35f1fb96be1826061ecbcf23 +2a09a3891fde052a585dc019eea9fba26d42445d +90a002ea647dcea57a2ed4294eab77897168ba1d +30c21306c17165c3925fea4ac9d1a4763c6d2a99 +b3eb0d6485510f2bdf36d256ab60ce29b8213744 +efbcf2b1d5ff4ee7132eae9c9e203d2b875c545b +b33ca14f594e2cf2a16ef27778169deb7cc9f4dc +d636f3943d39ec893dab2d2546f77f3f2607769d +cafe24f039e117d53288387c2720f44f27deecd0 +de8db47b7ff351f3287c5efb85102ba8836058d6 +d76e84a21416ef77e78138e326d4d249454e79dc +7a74f88a26cf251ba36b26f604f1ac9940fd9c92 +1ad3d4e1261f4a444d982a1470c257c78233bda3 +8d9f45ea6a5e4220e44d34139438eea75a07530b +c98ebf1bfb29a8203b5090412afeb333384213cd +f18bb49547095020a30e81b648075bc7e707515c +76f268b9bd1b69eb7784c5324abbb67f3e395b97 +e801084decf4542d57cf5ddb95820643766a172a +be3e042c20e2f3449b7b55d1cab0a80b0c6f00af +400fdd08cc95f1e85afafd07ddd9c0bed11483ea +098b01dc58ff555c473ae58c92c34b03a77eda5f +7cc2c670e3d7cf26454ac8547a94ec2c8ca90b34 +1088b02f0ccd7358d2b7076bb9e122d59d502d02 +f94b7d5bfa911ea7125920589723ee63a3eec9f0 +b4b057a3e0712dd16b50cbcfe7d613e4413ffa1c +b40ceed98a112f4f0d07351ce07270d9ff2bf796 +4cb8757aae1ae31e5519d81e854f44ed062d9836 +f2f7e97e8cc24cf7a2b7954cb74ecfd0f91a95ad +ae786098bc58b1ca92f596a698b23aeade9cd2cd +c33652576ce21694b33a94832378f737dd6959fb +e317c0d19201ff75fa7afedf93a9d1cd2c560af2 +bee35299716cc72cb7d5bd4daa9fddca05c58378 +318ea50a1c2f612e750a93e36620dd0c4531e9cf +b6ee855b411ee9bc39f935d0da3298a773a2ed37 +daf3e7def7b9e5db7a32f5a20b5c4e09e3f0dd18 +bc64b5aa0fc543fe8fd3dbaec275f89df44dc409 +3f57c55dba6ef1fda2bdf6fd9abd8ca7eb6828e4 +431a548faaf51c7a5fc89b6e479187a1c0e29805 +e4bbd3d230f22401ba0a0a72c8ed41ee1bd098a0 +c45da32047cac54afc99cf9b8a539389c577ded1 +ab1f1d32469180b3d011e9625d67c86a22b55903 +a550f6e415fd8aec8c45d4704712a408c37ecd18 +c73af5416b66f09cec0eb106f5a10f9bb6ef9cb1 +a077a90da88f12d9f10c8b85840bdb847a98b0a0 +c5e9e428a9198c8c4076f239b5eaa8dc95e7985b +b7365f0545b1a6862e3277b2b2139ee0d5aee1cf +4bd0e9b90a39c5c6a016b83882ae44cb4d28f1f8 +7438ceac716fdfe6621728c05e718eaa89dd89aa +4e3efd47e0d50c6cd1dc81ccc9669a5b2658f495 diff --git a/contrib/verify-commits/pre-push-hook.sh b/contrib/verify-commits/pre-push-hook.sh index 78873dc0c3..59d1813762 100755 --- a/contrib/verify-commits/pre-push-hook.sh +++ b/contrib/verify-commits/pre-push-hook.sh @@ -9,11 +9,11 @@ if ! [[ "$2" =~ ^(git@)?(www.)?github.com(:|/)bitcoin/bitcoin(.git)?$ ]]; then fi while read LINE; do - set -- A $LINE + set -- A "$LINE" if [ "$4" != "refs/heads/master" ]; then continue fi - if ! ./contrib/verify-commits/verify-commits.py $3 > /dev/null 2>&1; then + if ! ./contrib/verify-commits/verify-commits.py "$3" > /dev/null 2>&1; then echo "ERROR: A commit is not signed, can't push" ./contrib/verify-commits/verify-commits.py exit 1 diff --git a/contrib/windeploy/detached-sig-create.sh b/contrib/windeploy/detached-sig-create.sh index 29802e622e..fb5cb4a4d9 100755 --- a/contrib/windeploy/detached-sig-create.sh +++ b/contrib/windeploy/detached-sig-create.sh @@ -23,6 +23,7 @@ TIMESERVER=http://timestamp.comodoca.com CERTFILE="win-codesign.cert" mkdir -p "${OUTSUBDIR}" +# shellcheck disable=SC2046 basename -a $(ls -1 "${SRCDIR}"/*-unsigned.exe) | while read UNSIGNED; do echo Signing "${UNSIGNED}" "${OSSLSIGNCODE}" sign -certs "${CERTFILE}" -t "${TIMESERVER}" -h sha256 -in "${SRCDIR}/${UNSIGNED}" -out "${WORKDIR}/${UNSIGNED}" "$@" diff --git a/depends/Makefile b/depends/Makefile index a3b9cd2099..3dc99cc121 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -115,7 +115,7 @@ include builders/default.mk include packages/packages.mk # Previously, we directly invoked the well-known programs using $(shell ...) -# to contruct build_id_string. However, that was problematic because: +# to construct build_id_string. However, that was problematic because: # # 1. When invoking a shell, GNU Make special-cases exit code 127 (command not # found) by not capturing the output but instead passing it through. This is diff --git a/depends/README.md b/depends/README.md index 15c82cddf2..5c00807473 100644 --- a/depends/README.md +++ b/depends/README.md @@ -41,7 +41,7 @@ Common `host-platform-triplet`s for cross compilation are: - `i686-linux-android` for Android x86 32 bit - `x86_64-linux-android` for Android x86 64 bit -The paths are automatically configured and no other options are needed unless targeting [Android](#Android). +The paths are automatically configured and no other options are needed unless targeting [Android](../doc/build-android.md). ### Install the required dependencies: Ubuntu & Debian @@ -87,14 +87,6 @@ For linux S390X cross compilation: sudo apt-get install g++-s390x-linux-gnu binutils-s390x-linux-gnu -### Install the required dependencies: M1-based macOS - -To be able to build the `qt` package, ensure that Rosetta 2 is installed: - -``` -softwareupdate --install-rosetta -``` - ### Dependency Options The following can be set when running make: `make FOO=bar` @@ -133,18 +125,6 @@ options will be passed to bitcoin's configure. In this case, `--disable-wallet`. download-linux: run 'make download-linux' to fetch all sources needed for linux builds -### Android - -Before proceeding with an Android build one needs to get the [Android SDK](https://developer.android.com/studio) and use the "SDK Manager" tool to download the NDK and one or more "Platform packages" (these are Android versions and have a corresponding API level). -In order to build `ANDROID_API_LEVEL` (API level corresponding to the Android version targeted, e.g. Android 9.0 Pie is 28 and its "Platform package" needs to be available) and `ANDROID_TOOLCHAIN_BIN` (path to toolchain binaries depending on the platform the build is being performed on) need to be set. - -API levels from 24 to 29 have been tested to work. - -If the build includes Qt, environment variables `ANDROID_SDK` and `ANDROID_NDK` need to be set as well but can otherwise be omitted. -This is an example command for a default build with no disabled dependencies: - - ANDROID_SDK=/home/user/Android/Sdk ANDROID_NDK=/home/user/Android/Sdk/ndk-bundle make HOST=aarch64-linux-android ANDROID_API_LEVEL=28 ANDROID_TOOLCHAIN_BIN=/home/user/Android/Sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/bin - ### Other documentation - [description.md](description.md): General description of the depends system diff --git a/depends/config.site.in b/depends/config.site.in index 5cf107f19b..ed1a1f8d3d 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -62,7 +62,7 @@ if test -z "$with_gui" && test -n "@no_qt@"; then with_gui=no fi -if test -n "@debug@" && test -z "@no_qt@" && test "x$with_gui" != xno; then +if test -n "@debug@" && test -z "@no_qt@" && test "$with_gui" != "no"; then with_gui=qt5_debug fi @@ -72,7 +72,6 @@ fi if test "@host_os@" = darwin; then BREW=no - PORT=no fi PATH="${depends_prefix}/native/bin:${PATH}" diff --git a/depends/hosts/android.mk b/depends/hosts/android.mk index eabd84bbbe..fcc1c4f5c3 100644 --- a/depends/hosts/android.mk +++ b/depends/hosts/android.mk @@ -1,12 +1,11 @@ ifeq ($(HOST),armv7a-linux-android) -android_AR=$(ANDROID_TOOLCHAIN_BIN)/arm-linux-androideabi-ar android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)eabi$(ANDROID_API_LEVEL)-clang++ android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)eabi$(ANDROID_API_LEVEL)-clang -android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/arm-linux-androideabi-ranlib else -android_AR=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ar android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang++ android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang -android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ranlib endif +android_AR=$(ANDROID_TOOLCHAIN_BIN)/llvm-ar +android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/llvm-ranlib + android_cmake_system=Android diff --git a/depends/packages.md b/depends/packages.md index 7ed20ea129..4158b46d28 100644 --- a/depends/packages.md +++ b/depends/packages.md @@ -178,8 +178,8 @@ not sufficient to just say `libprimary`. For us, it's much easier to just link a static `libsecondary` into a shared `libprimary`. Especially because in our case, we are linking against a dummy `libprimary` anyway that we'll throw away. We don't care if the end-user has a -static or dynamic `libseconday`, that's not our concern. With a static -`libseconday`, when we need to link `libprimary` into our executable, there's no +static or dynamic `libsecondary`, that's not our concern. With a static +`libsecondary`, when we need to link `libprimary` into our executable, there's no dependency chain to worry about as `libprimary` has all the symbols. ## Build targets: diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index 21df50b040..5fe2b2bbb8 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -23,10 +23,11 @@ else $(package)_toolset_$(host_os)=gcc endif $(package)_config_libraries=filesystem,system,test -$(package)_cxxflags+=-std=c++17 -fvisibility=hidden +$(package)_cxxflags+=-std=c++17 $(package)_cxxflags_linux=-fPIC $(package)_cxxflags_android=-fPIC $(package)_cxxflags_x86_64_darwin=-fcf-protection=full +$(package)_cxxflags_mingw32=-fcf-protection=full endef define $(package)_preprocess_cmds @@ -42,5 +43,5 @@ define $(package)_build_cmds endef define $(package)_stage_cmds - b2 -d0 -j4 --prefix=$($(package)_staging_prefix_dir) $($(package)_config_opts) toolset=$($(package)_toolset_$(host_os)) install + b2 -d0 -j4 --prefix=$($(package)_staging_prefix_dir) $($(package)_config_opts) toolset=$($(package)_toolset_$(host_os)) --no-cmake-config install endef diff --git a/depends/packages/expat.mk b/depends/packages/expat.mk index 41c1114be0..c2089d1ccb 100644 --- a/depends/packages/expat.mk +++ b/depends/packages/expat.mk @@ -1,12 +1,13 @@ package=expat -$(package)_version=2.2.7 -$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_2_2_7/ -$(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=cbc9102f4a31a8dafd42d642e9a3aa31e79a0aedaa1f6efd2795ebc83174ec18 +$(package)_version=2.4.1 +$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_2_4_1/ +$(package)_file_name=$(package)-$($(package)_version).tar.xz +$(package)_sha256_hash=cf032d0dba9b928636548e32b327a2d66b1aab63c4f4a13dd132c2d1d2f2fb6a define $(package)_set_vars $(package)_config_opts=--disable-shared --without-docbook --without-tests --without-examples $(package)_config_opts += --disable-dependency-tracking --enable-option-checking + $(package)_config_opts += --without-xmlwf $(package)_config_opts_linux=--with-pic endef @@ -23,5 +24,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm -rf share lib/*.la + rm -rf share lib/cmake lib/*.la endef diff --git a/depends/packages/freetype.mk b/depends/packages/freetype.mk index aebc8a5f3b..6f5dbe0f01 100644 --- a/depends/packages/freetype.mk +++ b/depends/packages/freetype.mk @@ -1,12 +1,12 @@ package=freetype -$(package)_version=2.7.1 +$(package)_version=2.11.0 $(package)_download_path=https://download.savannah.gnu.org/releases/$(package) -$(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=3a3bb2c4e15ffb433f2032f50a5b5a92558206822e22bfe8cbe339af4aa82f88 +$(package)_file_name=$(package)-$($(package)_version).tar.xz +$(package)_sha256_hash=8bee39bd3968c4804b70614a0a3ad597299ad0e824bc8aad5ce8aaf48067bde7 define $(package)_set_vars $(package)_config_opts=--without-zlib --without-png --without-harfbuzz --without-bzip2 --disable-static - $(package)_config_opts += --enable-option-checking + $(package)_config_opts += --enable-option-checking --without-brotli $(package)_config_opts_linux=--with-pic endef diff --git a/depends/packages/libXau.mk b/depends/packages/libXau.mk index 24e0e9d325..b7e032c0b2 100644 --- a/depends/packages/libXau.mk +++ b/depends/packages/libXau.mk @@ -1,8 +1,8 @@ package=libXau -$(package)_version=1.0.8 +$(package)_version=1.0.9 $(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=fdd477320aeb5cdd67272838722d6b7d544887dfe7de46e1e7cc0c27c2bea4f2 +$(package)_sha256_hash=ccf8cbf0dbf676faa2ea0a6d64bcc3b6746064722b606c8c52917ed00dcb73ec $(package)_dependencies=xproto # When updating this package, check the default value of diff --git a/depends/packages/libxcb.mk b/depends/packages/libxcb.mk index 710f43959c..99a7ee2fbd 100644 --- a/depends/packages/libxcb.mk +++ b/depends/packages/libxcb.mk @@ -1,25 +1,21 @@ package=libxcb -$(package)_version=1.10 +$(package)_version=1.14 $(package)_download_path=https://xcb.freedesktop.org/dist -$(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=98d9ab05b636dd088603b64229dd1ab2d2cc02ab807892e107d674f9c3f2d5b5 +$(package)_file_name=$(package)-$($(package)_version).tar.xz +$(package)_sha256_hash=a55ed6db98d43469801262d81dc2572ed124edc3db31059d4e9916eb9f844c34 $(package)_dependencies=xcb_proto libXau define $(package)_set_vars -$(package)_config_opts=--disable-static --disable-build-docs --without-doxygen --without-launchd +$(package)_config_opts=--disable-static --disable-devel-docs --without-doxygen --without-launchd $(package)_config_opts += --disable-dependency-tracking --enable-option-checking -# Because we pass -qt-xcb to Qt, it will compile in a set of xcb helper libraries and extensions, -# so we skip building all of the extensions here. -# More info is available from: https://doc.qt.io/qt-5.9/linux-requirements.html +# Disable uneeded extensions. +# More info is available from: https://doc.qt.io/qt-5.15/linux-requirements.html $(package)_config_opts += --disable-composite --disable-damage --disable-dpms $(package)_config_opts += --disable-dri2 --disable-dri3 --disable-glx -$(package)_config_opts += --disable-present --disable-randr --disable-record -$(package)_config_opts += --disable-render --disable-resource --disable-screensaver -$(package)_config_opts += --disable-shape --disable-sync -$(package)_config_opts += --disable-xevie --disable-xfixes --disable-xfree86-dri -$(package)_config_opts += --disable-xinerama --disable-xinput -$(package)_config_opts += --disable-xprint --disable-selinux --disable-xtest -$(package)_config_opts += --disable-xv --disable-xvmc +$(package)_config_opts += --disable-present --disable-record --disable-resource +$(package)_config_opts += --disable-screensaver --disable-xevie --disable-xfree86-dri +$(package)_config_opts += --disable-xinput --disable-xprint --disable-selinux +$(package)_config_opts += --disable-xtest --disable-xv --disable-xvmc endef define $(package)_preprocess_cmds diff --git a/depends/packages/libxcb_util.mk b/depends/packages/libxcb_util.mk new file mode 100644 index 0000000000..2b65b58597 --- /dev/null +++ b/depends/packages/libxcb_util.mk @@ -0,0 +1,32 @@ +package=libxcb_util +$(package)_version=0.4.0 +$(package)_download_path=https://xcb.freedesktop.org/dist +$(package)_file_name=xcb-util-$($(package)_version).tar.bz2 +$(package)_sha256_hash=46e49469cb3b594af1d33176cd7565def2be3fa8be4371d62271fabb5eae50e9 +$(package)_dependencies=libxcb + +define $(package)_set_vars +$(package)_config_opts = --disable-shared --disable-devel-docs --without-doxygen +$(package)_config_opts += --disable-dependency-tracking --enable-option-checking +$(package)_cflags += -fPIC +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf share/man share/doc lib/*.la +endef diff --git a/depends/packages/libxcb_util_image.mk b/depends/packages/libxcb_util_image.mk new file mode 100644 index 0000000000..d12d67e8e8 --- /dev/null +++ b/depends/packages/libxcb_util_image.mk @@ -0,0 +1,31 @@ +package=libxcb_util_image +$(package)_version=0.4.0 +$(package)_download_path=https://xcb.freedesktop.org/dist +$(package)_file_name=xcb-util-image-$($(package)_version).tar.bz2 +$(package)_sha256_hash=2db96a37d78831d643538dd1b595d7d712e04bdccf8896a5e18ce0f398ea2ffc +$(package)_dependencies=libxcb libxcb_util + +define $(package)_set_vars +$(package)_config_opts=--disable-static --disable-devel-docs --without-doxygen +$(package)_config_opts+= --disable-dependency-tracking --enable-option-checking +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf share/man share/doc lib/*.la +endef diff --git a/depends/packages/libxcb_util_keysyms.mk b/depends/packages/libxcb_util_keysyms.mk new file mode 100644 index 0000000000..d4f72dedbe --- /dev/null +++ b/depends/packages/libxcb_util_keysyms.mk @@ -0,0 +1,31 @@ +package=libxcb_util_keysyms +$(package)_version=0.4.0 +$(package)_download_path=https://xcb.freedesktop.org/dist +$(package)_file_name=xcb-util-keysyms-$($(package)_version).tar.bz2 +$(package)_sha256_hash=0ef8490ff1dede52b7de533158547f8b454b241aa3e4dcca369507f66f216dd9 +$(package)_dependencies=libxcb xproto + +define $(package)_set_vars +$(package)_config_opts=--disable-static --disable-devel-docs --without-doxygen +$(package)_config_opts += --disable-dependency-tracking --enable-option-checking +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf share/man share/doc lib/*.la +endef diff --git a/depends/packages/libxcb_util_render.mk b/depends/packages/libxcb_util_render.mk new file mode 100644 index 0000000000..28f1fb073c --- /dev/null +++ b/depends/packages/libxcb_util_render.mk @@ -0,0 +1,31 @@ +package=libxcb_util_render +$(package)_version=0.3.9 +$(package)_download_path=https://xcb.freedesktop.org/dist +$(package)_file_name=xcb-util-renderutil-$($(package)_version).tar.bz2 +$(package)_sha256_hash=c6e97e48fb1286d6394dddb1c1732f00227c70bd1bedb7d1acabefdd340bea5b +$(package)_dependencies=libxcb + +define $(package)_set_vars +$(package)_config_opts=--disable-static --disable-devel-docs --without-doxygen +$(package)_config_opts += --disable-dependency-tracking --enable-option-checking +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf share/man share/doc lib/*.la +endef diff --git a/depends/packages/libxcb_util_wm.mk b/depends/packages/libxcb_util_wm.mk new file mode 100644 index 0000000000..3b905ba4ec --- /dev/null +++ b/depends/packages/libxcb_util_wm.mk @@ -0,0 +1,31 @@ +package=libxcb_util_wm +$(package)_version=0.4.1 +$(package)_download_path=https://xcb.freedesktop.org/dist +$(package)_file_name=xcb-util-wm-$($(package)_version).tar.bz2 +$(package)_sha256_hash=28bf8179640eaa89276d2b0f1ce4285103d136be6c98262b6151aaee1d3c2a3f +$(package)_dependencies=libxcb + +define $(package)_set_vars +$(package)_config_opts=--disable-static --disable-devel-docs --without-doxygen +$(package)_config_opts += --disable-dependency-tracking --enable-option-checking +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub . +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds + rm -rf share/man share/doc lib/*.la +endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 9094e327ef..426fa2e748 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -2,7 +2,7 @@ packages:=boost libevent qrencode_packages = qrencode -qt_linux_packages:=qt expat libxcb xcb_proto libXau xproto freetype fontconfig libxkbcommon +qt_linux_packages:=qt expat libxcb xcb_proto libXau xproto freetype fontconfig libxkbcommon libxcb_util libxcb_util_render libxcb_util_keysyms libxcb_util_image libxcb_util_wm qt_android_packages=qt qt_darwin_packages=qt diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 12e0494ad4..83dcb12001 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -1,24 +1,24 @@ -PACKAGE=qt -$(package)_version=5.12.11 -$(package)_download_path=https://download.qt.io/official_releases/qt/5.12/$($(package)_version)/submodules +package=qt +$(package)_version=5.15.2 +$(package)_download_path=https://download.qt.io/official_releases/qt/5.15/$($(package)_version)/submodules $(package)_suffix=everywhere-src-$($(package)_version).tar.xz $(package)_file_name=qtbase-$($(package)_suffix) -$(package)_sha256_hash=1c1b4e33137ca77881074c140d54c3c9747e845a31338cfe8680f171f0bc3a39 -$(package)_linux_dependencies=freetype fontconfig libxcb libxkbcommon +$(package)_sha256_hash=909fad2591ee367993a75d7e2ea50ad4db332f05e1c38dd7a5a274e156a4e0f8 +$(package)_linux_dependencies=freetype fontconfig libxcb libxkbcommon libxcb_util libxcb_util_render libxcb_util_keysyms libxcb_util_image libxcb_util_wm $(package)_qt_libs=corelib network widgets gui plugins testlib $(package)_linguist_tools = lrelease lupdate lconvert $(package)_patches = qt.pro qttools_src.pro $(package)_patches += fix_qt_pkgconfig.patch mac-qmake.conf fix_no_printer.patch no-xlib.patch -$(package)_patches += support_new_android_ndks.patch fix_android_jni_static.patch dont_hardcode_pwd.patch -$(package)_patches+= no_sdk_version_check.patch -$(package)_patches+= fix_lib_paths.patch fix_android_pch.patch -$(package)_patches+= qtbase-moc-ignore-gcc-macro.patch fix_limits_header.patch +$(package)_patches += dont_hardcode_x86_64.patch fix_montery_include.patch +$(package)_patches += fix_android_jni_static.patch dont_hardcode_pwd.patch +$(package)_patches += qtbase-moc-ignore-gcc-macro.patch fix_limits_header.patch +$(package)_patches += fix_bigsur_style.patch use_android_ndk23.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) -$(package)_qttranslations_sha256_hash=577b0668a777eb2b451c61e8d026d79285371597ce9df06b6dee6c814164b7c3 +$(package)_qttranslations_sha256_hash=d5788e86257b21d5323f1efd94376a213e091d1e5e03b45a95dd052b5f570db8 $(package)_qttools_file_name=qttools-$($(package)_suffix) -$(package)_qttools_sha256_hash=98b2aaca230458f65996f3534fd471d2ffd038dd58ac997c0589c06dc2385b4f +$(package)_qttools_sha256_hash=c189d0ce1ff7c739db9a3ace52ac3e24cb8fd6dbf234e49f075249b38f43c1cc $(package)_extra_sources = $($(package)_qttranslations_file_name) $(package)_extra_sources += $($(package)_qttools_file_name) @@ -29,14 +29,14 @@ $(package)_config_opts_release += -silent $(package)_config_opts_debug = -debug $(package)_config_opts_debug += -optimized-tools $(package)_config_opts += -bindir $(build_prefix)/bin -$(package)_config_opts += -c++std c++1z +$(package)_config_opts += -c++std c++17 $(package)_config_opts += -confirm-license $(package)_config_opts += -hostprefix $(build_prefix) $(package)_config_opts += -no-compile-examples $(package)_config_opts += -no-cups $(package)_config_opts += -no-egl $(package)_config_opts += -no-eglfs -$(package)_config_opts += -no-freetype +$(package)_config_opts += -no-evdev $(package)_config_opts += -no-gif $(package)_config_opts += -no-glib $(package)_config_opts += -no-icu @@ -51,6 +51,7 @@ $(package)_config_opts += -no-mtdev $(package)_config_opts += -no-openssl $(package)_config_opts += -no-openvg $(package)_config_opts += -no-reduce-relocations +$(package)_config_opts += -no-schannel $(package)_config_opts += -no-sctp $(package)_config_opts += -no-securetransport $(package)_config_opts += -no-sql-db2 @@ -64,6 +65,7 @@ $(package)_config_opts += -no-sql-sqlite $(package)_config_opts += -no-sql-sqlite2 $(package)_config_opts += -no-system-proxies $(package)_config_opts += -no-use-gold-linker +$(package)_config_opts += -no-zstd $(package)_config_opts += -nomake examples $(package)_config_opts += -nomake tests $(package)_config_opts += -nomake tools @@ -101,6 +103,7 @@ $(package)_config_opts += -no-feature-sqlmodel $(package)_config_opts += -no-feature-statemachine $(package)_config_opts += -no-feature-syntaxhighlighter $(package)_config_opts += -no-feature-textbrowser +$(package)_config_opts += -no-feature-textmarkdownwriter $(package)_config_opts += -no-feature-textodfwriter $(package)_config_opts += -no-feature-topleveldomain $(package)_config_opts += -no-feature-udpsocket @@ -116,6 +119,7 @@ $(package)_config_opts_darwin = -no-dbus $(package)_config_opts_darwin += -no-opengl $(package)_config_opts_darwin += -pch $(package)_config_opts_darwin += -no-feature-corewlan +$(package)_config_opts_darwin += -no-freetype $(package)_config_opts_darwin += QMAKE_MACOSX_DEPLOYMENT_TARGET=$(OSX_MIN_VERSION) ifneq ($(build_os),darwin) @@ -127,10 +131,12 @@ $(package)_config_opts_darwin += -device-option MAC_TARGET=$(host) $(package)_config_opts_darwin += -device-option XCODE_VERSION=$(XCODE_VERSION) endif -# for macOS on Apple Silicon (ARM) see https://bugreports.qt.io/browse/QTBUG-85279 +ifneq ($(build_arch),$(host_arch)) $(package)_config_opts_aarch64_darwin += -device-option QMAKE_APPLE_DEVICE_ARCHS=arm64 +$(package)_config_opts_x86_64_darwin += -device-option QMAKE_APPLE_DEVICE_ARCHS=x86_64 +endif -$(package)_config_opts_linux = -qt-xcb +$(package)_config_opts_linux = -xcb $(package)_config_opts_linux += -no-xcb-xlib $(package)_config_opts_linux += -no-feature-xlib $(package)_config_opts_linux += -system-freetype @@ -149,6 +155,7 @@ $(package)_config_opts_s390x_linux = -platform linux-g++ -xplatform bitcoin-linu $(package)_config_opts_mingw32 = -no-opengl $(package)_config_opts_mingw32 += -no-dbus +$(package)_config_opts_mingw32 += -no-freetype $(package)_config_opts_mingw32 += -xplatform win32-g++ $(package)_config_opts_mingw32 += "QMAKE_CFLAGS = '$($(package)_cflags) $($(package)_cppflags)'" $(package)_config_opts_mingw32 += "QMAKE_CXXFLAGS = '$($(package)_cflags) $($(package)_cppflags)'" @@ -163,7 +170,6 @@ $(package)_config_opts_android += -android-ndk-platform android-$(ANDROID_API_LE $(package)_config_opts_android += -device-option CROSS_COMPILE="$(host)-" $(package)_config_opts_android += -egl $(package)_config_opts_android += -qpa xcb -$(package)_config_opts_android += -no-eglfs $(package)_config_opts_android += -no-dbus $(package)_config_opts_android += -opengl es2 $(package)_config_opts_android += -qt-freetype @@ -224,14 +230,14 @@ define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/dont_hardcode_pwd.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch && \ - patch -p1 -i $($(package)_patch_dir)/support_new_android_ndks.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_android_jni_static.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_android_pch.patch && \ patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \ - patch -p1 -i $($(package)_patch_dir)/no_sdk_version_check.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_lib_paths.patch && \ + patch -p1 -i $($(package)_patch_dir)/dont_hardcode_x86_64.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_limits_header.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_montery_include.patch && \ + patch -p1 -i $($(package)_patch_dir)/fix_bigsur_style.patch && \ + patch -p1 -i $($(package)_patch_dir)/use_android_ndk23.patch && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ cp -f qtbase/mkspecs/macx-clang/qplatformdefs.h qtbase/mkspecs/macx-clang-linux/ &&\ cp -f $($(package)_patch_dir)/mac-qmake.conf qtbase/mkspecs/macx-clang-linux/qmake.conf && \ @@ -248,6 +254,7 @@ endef define $(package)_config_cmds export PKG_CONFIG_SYSROOT_DIR=/ && \ export PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig && \ + export QT_MAC_SDK_NO_VERSION_CHECK=1 && \ cd qtbase && \ ./configure -top-level $($(package)_config_opts) endef diff --git a/depends/packages/xcb_proto.mk b/depends/packages/xcb_proto.mk index 01203a0718..9be822506d 100644 --- a/depends/packages/xcb_proto.mk +++ b/depends/packages/xcb_proto.mk @@ -1,8 +1,8 @@ package=xcb_proto -$(package)_version=1.10 -$(package)_download_path=https://xcb.freedesktop.org/dist -$(package)_file_name=xcb-proto-$($(package)_version).tar.bz2 -$(package)_sha256_hash=7ef40ddd855b750bc597d2a435da21e55e502a0fefa85b274f2c922800baaf05 +$(package)_version=1.14.1 +$(package)_download_path=https://xorg.freedesktop.org/archive/individual/proto +$(package)_file_name=xcb-proto-$($(package)_version).tar.xz +$(package)_sha256_hash=f04add9a972ac334ea11d9d7eb4fc7f8883835da3e4859c9afa971efdf57fcc3 define $(package)_config_cmds $($(package)_autoconf) @@ -17,6 +17,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - find -name "*.pyc" -delete && \ - find -name "*.pyo" -delete + rm -rf lib/python*/site-packages/xcbgen/__pycache__ endef diff --git a/depends/packages/xproto.mk b/depends/packages/xproto.mk index 6bd867d02b..7a43c52faf 100644 --- a/depends/packages/xproto.mk +++ b/depends/packages/xproto.mk @@ -1,8 +1,8 @@ package=xproto -$(package)_version=7.0.26 +$(package)_version=7.0.31 $(package)_download_path=https://xorg.freedesktop.org/releases/individual/proto $(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=636162c1759805a5a0114a369dffdeccb8af8c859ef6e1445f26a4e6e046514f +$(package)_sha256_hash=c6f9747da0bd3a95f86b17fb8dd5e717c8f3ab7f0ece3ba1b247899ec1ef7747 define $(package)_set_vars $(package)_config_opts=--without-fop --without-xmlto --without-xsltproc --disable-specs diff --git a/depends/patches/qt/dont_hardcode_x86_64.patch b/depends/patches/qt/dont_hardcode_x86_64.patch new file mode 100644 index 0000000000..5c1e030fa4 --- /dev/null +++ b/depends/patches/qt/dont_hardcode_x86_64.patch @@ -0,0 +1,119 @@ +macOS: Don't hard-code x86_64 as the architecture when using qmake + +Upstream commit: + - Qt 6.1: 9082cc8e8d5a6441dabe5e7a95bc0cd9085b95fe + +For other Qt branches see +https://codereview.qt-project.org/q/I70db7e4c27f0d3da5d0af33cb491d72c312d3fa8 + + +--- old/qtbase/configure.json ++++ new/qtbase/configure.json +@@ -244,11 +244,18 @@ + + "testTypeDependencies": { + "linkerSupportsFlag": [ "use_bfd_linker", "use_gold_linker", "use_lld_linker" ], +- "verifySpec": [ "shared", "use_bfd_linker", "use_gold_linker", "use_lld_linker", "compiler-flags", "qmakeargs", "commit" ], ++ "verifySpec": [ ++ "shared", ++ "use_bfd_linker", "use_gold_linker", "use_lld_linker", ++ "compiler-flags", "qmakeargs", ++ "simulator_and_device", ++ "thread", ++ "commit" ], + "compile": [ "verifyspec" ], + "detectPkgConfig": [ "cross_compile", "machineTuple" ], + "library": [ "pkg-config", "compiler-flags" ], +- "getPkgConfigVariable": [ "pkg-config" ] ++ "getPkgConfigVariable": [ "pkg-config" ], ++ "architecture" : [ "verifyspec" ] + }, + + "testTypeAliases": { +@@ -762,7 +769,7 @@ + }, + "architecture": { + "label": "Architecture", +- "output": [ "architecture" ] ++ "output": [ "architecture", "commitConfig" ] + }, + "pkg-config": { + "label": "Using pkg-config", +diff --git a/configure.pri b/configure.pri +index 49755f7abfd..8be9b10d7d4 100644 +--- old/qtbase/configure.pri ++++ new/qtbase/configure.pri +@@ -662,6 +662,13 @@ defineTest(qtConfOutput_commitOptions) { + write_file($$QT_BUILD_TREE/mkspecs/qdevice.pri, $${currentConfig}.output.devicePro)|error() + } + ++# Output is written after configuring each Qt module, ++# but some tests within a module might depend on the ++# configuration output of previous tests. ++defineTest(qtConfOutput_commitConfig) { ++ qtConfProcessOutput() ++} ++ + # type (empty or 'host'), option name, default value + defineTest(processQtPath) { + out_var = config.rel_input.$${2} +diff --git a/mkspecs/common/macx.conf b/mkspecs/common/macx.conf +index d16b77acb8e..4ba0a8eaa36 100644 +--- old/qtbase/mkspecs/common/macx.conf ++++ new/qtbase/mkspecs/common/macx.conf +@@ -6,7 +6,6 @@ QMAKE_PLATFORM += macos osx macx + QMAKE_MAC_SDK = macosx + + QMAKE_MACOSX_DEPLOYMENT_TARGET = 10.13 +-QMAKE_APPLE_DEVICE_ARCHS = x86_64 + + # Should be 10.15, but as long as the CI builds with + # older SDKs we have to keep this. +diff --git a/mkspecs/features/mac/default_post.prf b/mkspecs/features/mac/default_post.prf +index 92a9112bca6..d888731ec8d 100644 +--- old/qtbase/mkspecs/features/mac/default_post.prf ++++ new/qtbase/mkspecs/features/mac/default_post.prf +@@ -90,6 +90,11 @@ app_extension_api_only { + QMAKE_LFLAGS += $$QMAKE_CFLAGS_APPLICATION_EXTENSION + } + ++# Non-universal builds do not set QMAKE_APPLE_DEVICE_ARCHS, ++# so we pick it up from what the arch test resolved instead. ++isEmpty(QMAKE_APPLE_DEVICE_ARCHS): \ ++ QMAKE_APPLE_DEVICE_ARCHS = $$QT_ARCH ++ + macx-xcode { + qmake_pkginfo_typeinfo.name = QMAKE_PKGINFO_TYPEINFO + !isEmpty(QMAKE_PKGINFO_TYPEINFO): \ +@@ -145,9 +150,6 @@ macx-xcode { + simulator: VALID_SIMULATOR_ARCHS = $$QMAKE_APPLE_SIMULATOR_ARCHS + VALID_ARCHS = $$VALID_DEVICE_ARCHS $$VALID_SIMULATOR_ARCHS + +- isEmpty(VALID_ARCHS): \ +- error("QMAKE_APPLE_DEVICE_ARCHS or QMAKE_APPLE_SIMULATOR_ARCHS must contain at least one architecture") +- + single_arch: VALID_ARCHS = $$first(VALID_ARCHS) + + ACTIVE_ARCHS = $(filter $(EXPORT_VALID_ARCHS), $(ARCHS)) +diff --git a/mkspecs/features/toolchain.prf b/mkspecs/features/toolchain.prf +index efbe7c1e55b..8add6dc8043 100644 +--- old/qtbase/mkspecs/features/toolchain.prf ++++ new/qtbase/mkspecs/features/toolchain.prf +@@ -182,9 +182,14 @@ isEmpty($${target_prefix}.INCDIRS) { + # UIKit simulator platforms will see the device SDK's sysroot in + # QMAKE_DEFAULT_*DIRS, because they're handled in a single build pass. + darwin { +- # Clang doesn't pick up the architecture from the sysroot, and will +- # default to the host architecture, so we need to manually set it. +- cxx_flags += -arch $$QMAKE_APPLE_DEVICE_ARCHS ++ uikit { ++ # Clang doesn't automatically pick up the architecture, just because ++ # we're passing the iOS sysroot below, and we will end up building the ++ # test for the host architecture, resulting in linker errors when ++ # linking against the iOS libraries. We work around this by passing ++ # the architecture explicitly. ++ cxx_flags += -arch $$first(QMAKE_APPLE_DEVICE_ARCHS) ++ } + + uikit:macx-xcode: \ + cxx_flags += -isysroot $$sdk_path_device.value diff --git a/depends/patches/qt/fix_android_jni_static.patch b/depends/patches/qt/fix_android_jni_static.patch index a186aeb8f6..bb64661761 100644 --- a/depends/patches/qt/fix_android_jni_static.patch +++ b/depends/patches/qt/fix_android_jni_static.patch @@ -1,6 +1,6 @@ --- old/qtbase/src/plugins/platforms/android/androidjnimain.cpp +++ new/qtbase/src/plugins/platforms/android/androidjnimain.cpp -@@ -898,6 +898,14 @@ +@@ -914,6 +914,14 @@ Q_DECL_EXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void */*reserved*/) __android_log_print(ANDROID_LOG_FATAL, "Qt", "registerNatives failed"); return -1; } diff --git a/depends/patches/qt/fix_android_pch.patch b/depends/patches/qt/fix_android_pch.patch deleted file mode 100644 index 195e1c5e59..0000000000 --- a/depends/patches/qt/fix_android_pch.patch +++ /dev/null @@ -1,10 +0,0 @@ ---- old/qtbase/mkspecs/common/android-base-head.conf -+++ new/qtbase/mkspecs/common/android-base-head.conf -@@ -72,6 +72,6 @@ CROSS_COMPILE = $$NDK_TOOLCHAIN_PATH/bin/$$NDK_TOOLS_PREFIX- - QMAKE_PCH_OUTPUT_EXT = .gch - - QMAKE_CFLAGS_PRECOMPILE = -x c-header -c ${QMAKE_PCH_INPUT} -o ${QMAKE_PCH_OUTPUT} --QMAKE_CFLAGS_USE_PRECOMPILE = -include ${QMAKE_PCH_OUTPUT_BASE} -+QMAKE_CFLAGS_USE_PRECOMPILE = -include-pch ${QMAKE_PCH_OUTPUT} - QMAKE_CXXFLAGS_PRECOMPILE = -x c++-header -c ${QMAKE_PCH_INPUT} -o ${QMAKE_PCH_OUTPUT} - QMAKE_CXXFLAGS_USE_PRECOMPILE = $$QMAKE_CFLAGS_USE_PRECOMPILE diff --git a/depends/patches/qt/fix_bigsur_style.patch b/depends/patches/qt/fix_bigsur_style.patch new file mode 100644 index 0000000000..0f6c848bb0 --- /dev/null +++ b/depends/patches/qt/fix_bigsur_style.patch @@ -0,0 +1,90 @@ +Fix rendering in macOS BigSur + +See: https://bugreports.qt.io/browse/QTBUG-86513. + +Upstream commits (combined in this patch): + - Qt 6.0: 40fb97e97f550b8afd13ecc3a038d9d0c2d82bbb + - Qt 6.0: 3857f104cac127f62e64e55a20613f0ac2e6b843 + - Qt 6.1: abee4cdd5925a8513f51784754fca8fa35031732 + +--- old/qtbase/src/plugins/styles/mac/qmacstyle_mac.mm ++++ new/qtbase/src/plugins/styles/mac/qmacstyle_mac.mm +@@ -3870,6 +3870,7 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter + const auto cs = d->effectiveAquaSizeConstrain(opt, w); + // Extra hacks to get the proper pressed appreance when not selected or selected and inactive + const bool needsInactiveHack = (!isActive && isSelected); ++ const bool isBigSurOrAbove = QOperatingSystemVersion::current() >= QOperatingSystemVersion::MacOSBigSur; + const auto ct = !needsInactiveHack && (isSelected || tp == QStyleOptionTab::OnlyOneTab) ? + QMacStylePrivate::Button_PushButton : + QMacStylePrivate::Button_PopupButton; +@@ -3878,6 +3879,12 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter + auto *pb = static_cast<NSButton *>(d->cocoaControl(cw)); + + auto vOffset = isPopupButton ? 1 : 2; ++ if (isBigSurOrAbove) { ++ // Make it 1, otherwise, offset is very visible compared ++ // to selected tab (which is not a popup button). ++ vOffset = 1; ++ } ++ + if (tabDirection == QMacStylePrivate::East) + vOffset -= 1; + const auto outerAdjust = isPopupButton ? 1 : 4; +@@ -3894,9 +3901,22 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter + frameRect = frameRect.adjusted(-innerAdjust, 0, outerAdjust, 0); + else + frameRect = frameRect.adjusted(-outerAdjust, 0, innerAdjust, 0); ++ ++ if (isSelected && isBigSurOrAbove) { ++ // 1 pixed of 'roundness' is still visible on the right ++ // (the left is OK, it's rounded). ++ frameRect = frameRect.adjusted(0, 0, 1, 0); ++ } ++ + break; + case QStyleOptionTab::Middle: + frameRect = frameRect.adjusted(-innerAdjust, 0, innerAdjust, 0); ++ ++ if (isSelected && isBigSurOrAbove) { ++ // 1 pixel of 'roundness' is still visible on both ++ // sides - left and right. ++ frameRect = frameRect.adjusted(-1, 0, 1, 0); ++ } + break; + case QStyleOptionTab::End: + // Pressed state hack: tweak adjustments in preparation for flip below +@@ -3904,6 +3924,11 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter + frameRect = frameRect.adjusted(-innerAdjust, 0, outerAdjust, 0); + else + frameRect = frameRect.adjusted(-outerAdjust, 0, innerAdjust, 0); ++ ++ if (isSelected && isBigSurOrAbove) { ++ // 1 pixel of 'roundness' is still visible on the left. ++ frameRect = frameRect.adjusted(-1, 0, 0, 0); ++ } + break; + case QStyleOptionTab::OnlyOneTab: + frameRect = frameRect.adjusted(-outerAdjust, 0, outerAdjust, 0); +@@ -3951,7 +3976,10 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter + NSPopUpArrowPosition oldPosition = NSPopUpArrowAtCenter; + NSPopUpButtonCell *pbCell = nil; + auto rAdjusted = r; +- if (isPopupButton && tp == QStyleOptionTab::OnlyOneTab) { ++ if (isPopupButton && (tp == QStyleOptionTab::OnlyOneTab || isBigSurOrAbove)) { ++ // Note: starting from macOS BigSur NSPopupButton has this ++ // arrow 'button' in a different place and it became ++ // quite visible 'in between' inactive tabs. + pbCell = static_cast<NSPopUpButtonCell *>(pb.cell); + oldPosition = pbCell.arrowPosition; + pbCell.arrowPosition = NSPopUpNoArrow; +@@ -3959,6 +3987,10 @@ void QMacStyle::drawControl(ControlElement ce, const QStyleOption *opt, QPainter + // NSPopUpButton in this state is smaller. + rAdjusted.origin.x -= 3; + rAdjusted.size.width += 6; ++ if (isBigSurOrAbove) { ++ if (tp == QStyleOptionTab::End) ++ rAdjusted.origin.x -= 2; ++ } + } + } + diff --git a/depends/patches/qt/fix_lib_paths.patch b/depends/patches/qt/fix_lib_paths.patch deleted file mode 100644 index d1a15373f4..0000000000 --- a/depends/patches/qt/fix_lib_paths.patch +++ /dev/null @@ -1,193 +0,0 @@ ---- old/qtbase/mkspecs/common/mac.conf -+++ new/qtbase/mkspecs/common/mac.conf -@@ -14,7 +14,6 @@ - - QMAKE_RESOURCE = /Developer/Tools/Rez - QMAKE_EXTENSION_SHLIB = dylib --QMAKE_EXTENSIONS_AUX_SHLIB = tbd - QMAKE_LIBDIR = - - # sdk.prf will prefix the proper SDK sysroot - ---- old/qtbase/mkspecs/features/qmake_use.prf -+++ new/qtbase/mkspecs/features/qmake_use.prf -@@ -22,6 +22,8 @@ - !defined(QMAKE_LIBS_$$nu, var): \ - error("Library '$$lower($$replace(nu, _, -))' is not defined.") - -+ QMAKE_LIBDIR += $$eval(QMAKE_LIBDIR_$$nu) -+ - debug: \ - LIBS$${suffix} += $$eval(QMAKE_LIBS_$${nu}_DEBUG) $$eval(QMAKE_LIBS_$$nu) - else: \ - ---- old/qtbase/mkspecs/features/qt_configure.prf -+++ new/qtbase/mkspecs/features/qt_configure.prf -@@ -526,98 +526,23 @@ - return($$sysrootified) - } - --# libs-var, libs, in-paths, out-paths-var -+# libs-var, libs, in-paths - defineTest(qtConfResolveLibs) { -- ret = true -- paths = $$3 -- out = -- copy = false -- for (l, 2) { -- $$copy { -- copy = false -- out += $$l -- } else: equals(l, "-s") { -- # em++ flag to link libraries from emscripten-ports; passed on literally. -- copy = true -- out += $$l -- } else: contains(l, "^-L.*") { -- lp = $$replace(l, "^-L", ) -- gcc: lp = $$qtGccSysrootifiedPath($$lp) -- !exists($$lp/.) { -- qtLog("Library path $$val_escape(lp) is invalid.") -- ret = false -- } else { -- paths += $$lp -- } -- } else: contains(l, "^-l.*") { -- lib = $$replace(l, "^-l", ) -- lcan = -- integrity:contains(lib, "^.*\\.a") { -- # INTEGRITY compiler searches for exact filename -- # if -l argument has .a suffix -- lcan += $${lib} -- } else: contains(lib, "^:.*") { -- # Use exact filename when -l:filename syntax is used. -- lib ~= s/^:// -- lcan += $${lib} -- } else: unix { -- # Under UNIX, we look for actual shared libraries, in addition -- # to static ones. -- shexts = $$QMAKE_EXTENSION_SHLIB $$QMAKE_EXTENSIONS_AUX_SHLIB -- for (ext, shexts) { -- lcan += $${QMAKE_PREFIX_SHLIB}$${lib}.$${ext} -- } -- lcan += \ -- $${QMAKE_PREFIX_STATICLIB}$${lib}.$${QMAKE_EXTENSION_STATICLIB} -- } else { -- # Under Windows, we look only for static libraries, as even for DLLs -- # one actually links against a static import library. -- mingw { -- lcan += \ -- # MinGW supports UNIX-style library naming in addition to -- # the MSVC style. -- lib$${lib}.dll.a lib$${lib}.a \ -- # Fun fact: prefix-less libraries are also supported. -- $${lib}.dll.a $${lib}.a -- } -- lcan += $${lib}.lib -- } -- l = $$qtConfFindInPathList($$lcan, $$paths $$EXTRA_LIBDIR $$QMAKE_DEFAULT_LIBDIRS) -- isEmpty(l) { -- qtLog("None of [$$val_escape(lcan)] found in [$$val_escape(paths)] and global paths.") -- ret = false -- } else { -- out += $$l -- } -- } else { -- out += $$l -- } -- } -- $$1 = $$out -+ for (path, 3): \ -+ pre_lflags += -L$$path -+ $$1 = $$pre_lflags $$2 - export($$1) -- !isEmpty(4) { -- $$4 = $$paths -- export($$4) -- } -- return($$ret) --} -- --# source-var --defineTest(qtConfResolveAllLibs) { -- ret = true -- !qtConfResolveLibs($${1}.libs, $$eval($${1}.libs), , $${1}.libdirs): \ -- ret = false -- for (b, $${1}.builds._KEYS_): \ -- !qtConfResolveLibs($${1}.builds.$${b}, $$eval($${1}.builds.$${b}), $$eval($${1}.libdirs), ): \ -- ret = false -- return($$ret) -+ return(true) - } - - # libs-var, in-paths, libs - defineTest(qtConfResolvePathLibs) { - ret = true -- gcc: 2 = $$qtGccSysrootifiedPaths($$2) -- for (libdir, 2) { -+ gcc: \ -+ local_paths = $$qtGccSysrootifiedPaths($$2) -+ else: \ -+ local_paths = $$2 -+ for (libdir, local_paths) { - !exists($$libdir/.) { - qtLog("Library path $$val_escape(libdir) is invalid.") - ret = false -@@ -667,8 +592,11 @@ - # includes-var, in-paths, test-object-var - defineTest(qtConfResolvePathIncs) { - ret = true -- gcc: 2 = $$qtGccSysrootifiedPaths($$2) -- for (incdir, 2) { -+ gcc: \ -+ local_paths = $$qtGccSysrootifiedPaths($$2) -+ else: \ -+ local_paths = $$2 -+ for (incdir, local_paths) { - !exists($$incdir/.) { - qtLog("Include path $$val_escape(incdir) is invalid.") - ret = false -@@ -727,6 +655,7 @@ - vars += $$eval(config.commandline.rev_assignments.$${iv}) - defined(config.input.$${iv}, var) { - eval($${1}.builds.$${b} = $$eval(config.input.$${iv})) -+ export($${1}.builds.$${b}) - $${1}.builds._KEYS_ *= $${b} - any = true - } else { -@@ -741,11 +670,14 @@ - export($${1}.builds._KEYS_) - # we also reset the generic libs, to avoid surprises. - $${1}.libs = -+ export($${1}.libs) - } - - # direct libs. overwrites inline libs. -- defined(config.input.$${input}.libs, var): \ -+ defined(config.input.$${input}.libs, var) { - eval($${1}.libs = $$eval(config.input.$${input}.libs)) -+ export($${1}.libs) -+ } - - includes = $$eval(config.input.$${input}.incdir) - -@@ -754,6 +686,7 @@ - !isEmpty(prefix) { - includes += $$prefix/include - $${1}.libs = -L$$prefix/lib $$eval($${1}.libs) -+ export($${1}.libs) - } - - libdir = $$eval(config.input.$${input}.libdir) -@@ -762,11 +695,9 @@ - for (ld, libdir): \ - libs += -L$$ld - $${1}.libs = $$libs $$eval($${1}.libs) -+ export($${1}.libs) - } - -- !qtConfResolveAllLibs($$1): \ -- return(false) -- - !qtConfResolvePathIncs($${1}.includedir, $$includes, $$2): \ - return(false) - diff --git a/depends/patches/qt/fix_limits_header.patch b/depends/patches/qt/fix_limits_header.patch index e4313770e5..cb5a8cd1b5 100644 --- a/depends/patches/qt/fix_limits_header.patch +++ b/depends/patches/qt/fix_limits_header.patch @@ -19,17 +19,16 @@ Upstream commits: #include <stdlib.h> #include <string.h> ---- old/qtbase/src/corelib/tools/qbytearraymatcher.h -+++ new/qtbase/src/corelib/tools/qbytearraymatcher.h -@@ -42,6 +42,8 @@ - - #include <QtCore/qbytearray.h> +--- old/qtbase/src/corelib/global/qfloat16.h ++++ new/qtbase/src/corelib/global/qfloat16.h +@@ -43,6 +43,7 @@ + #include <QtCore/qglobal.h> + #include <QtCore/qmetatype.h> +#include <limits> -+ - QT_BEGIN_NAMESPACE - + #include <string.h> + #if defined(QT_COMPILER_SUPPORTS_F16C) && defined(__AVX2__) && !defined(__F16C__) --- old/qtbase/src/tools/moc/generator.cpp +++ new/qtbase/src/tools/moc/generator.cpp @@ -42,3 +41,24 @@ Upstream commits: #include <math.h> #include <stdio.h> +--- old/qtbase/src/corelib/text/qbytearraymatcher.h ++++ new/qtbase/src/corelib/text/qbytearraymatcher.h +@@ -42,6 +42,8 @@ + + #include <QtCore/qbytearray.h> + ++#include <limits> ++ + QT_BEGIN_NAMESPACE + + +--- old/qtbase/src/corelib/tools/qoffsetstringarray_p.h ++++ new/qtbase/src/corelib/tools/qoffsetstringarray_p.h +@@ -55,6 +55,7 @@ + + #include <tuple> + #include <array> ++#include <limits> + + QT_BEGIN_NAMESPACE + diff --git a/depends/patches/qt/fix_montery_include.patch b/depends/patches/qt/fix_montery_include.patch new file mode 100644 index 0000000000..38b700addf --- /dev/null +++ b/depends/patches/qt/fix_montery_include.patch @@ -0,0 +1,21 @@ +From dece6f5840463ae2ddf927d65eb1b3680e34a547 +From: Øystein Heskestad <oystein.heskestad@qt.io> +Date: Wed, 27 Oct 2021 13:07:46 +0200 +Subject: [PATCH] Add missing macOS header file that was indirectly included before + +See: https://bugreports.qt.io/browse/QTBUG-97855 + +Upstream Commits: + - Qt 6.2: c884bf138a21dd7320e35cef34d24e22e74d7ce0 + +diff --git a/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h b/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h +index e070ba97..07c75b04 100644 +--- a/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h ++++ b/qtbase/src/plugins/platforms/cocoa/qiosurfacegraphicsbuffer.h +@@ -40,6 +40,7 @@ + #ifndef QIOSURFACEGRAPHICSBUFFER_H + #define QIOSURFACEGRAPHICSBUFFER_H + ++#include <CoreGraphics/CGColorSpace.h> + #include <qpa/qplatformgraphicsbuffer.h> + #include <private/qcore_mac_p.h> diff --git a/depends/patches/qt/fix_qt_pkgconfig.patch b/depends/patches/qt/fix_qt_pkgconfig.patch index a5de2b4b9e..73f4d89f73 100644 --- a/depends/patches/qt/fix_qt_pkgconfig.patch +++ b/depends/patches/qt/fix_qt_pkgconfig.patch @@ -4,20 +4,8 @@ load(qt_targets) # this builds on top of qt_common --!internal_module:if(unix|mingw) { +-!internal_module:if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) { +if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) { CONFIG += create_pc QMAKE_PKGCONFIG_DESTDIR = pkgconfig host_build: \ -@@ -284,9 +284,9 @@ load(qt_targets) - QMAKE_PKGCONFIG_CFLAGS = -D$$MODULE_DEFINE -I${includedir}/$$MODULE_INCNAME - } - QMAKE_PKGCONFIG_NAME = $$replace(TARGET, ^Qt, "Qt$$QT_MAJOR_VERSION ") -- QMAKE_PKGCONFIG_FILE = $$replace(TARGET, ^Qt, Qt$$QT_MAJOR_VERSION) -+ QMAKE_PKGCONFIG_FILE = $$replace(TARGET, ^Qt, Qt$$QT_MAJOR_VERSION)$$qtPlatformTargetSuffix() - for(i, MODULE_DEPENDS): \ -- QMAKE_PKGCONFIG_REQUIRES += $$replace(QT.$${i}.name, ^Qt, Qt$$section(QT.$${i}.VERSION, ., 0, 0)) -+ QMAKE_PKGCONFIG_REQUIRES += $$replace(QT.$${i}.name, ^Qt, Qt$$section(QT.$${i}.VERSION, ., 0, 0))$$qtPlatformTargetSuffix() - isEmpty(QMAKE_PKGCONFIG_DESCRIPTION): \ - QMAKE_PKGCONFIG_DESCRIPTION = $$replace(TARGET, ^Qt, "Qt ") module - !isEmpty(lib_replace0.match) { diff --git a/depends/patches/qt/mac-qmake.conf b/depends/patches/qt/mac-qmake.conf index 190ab7a160..e4bfaa1463 100644 --- a/depends/patches/qt/mac-qmake.conf +++ b/depends/patches/qt/mac-qmake.conf @@ -13,7 +13,6 @@ QMAKE_MAC_SDK.macosx.Path = $${MAC_SDK_PATH} QMAKE_MAC_SDK.macosx.platform_name = macosx QMAKE_MAC_SDK.macosx.SDKVersion = $${MAC_SDK_VERSION} QMAKE_MAC_SDK.macosx.PlatformPath = /phony -QMAKE_APPLE_DEVICE_ARCHS=x86_64 !host_build: QMAKE_CFLAGS += -target $${MAC_TARGET} !host_build: QMAKE_OBJECTIVE_CFLAGS += $$QMAKE_CFLAGS !host_build: QMAKE_CXXFLAGS += $$QMAKE_CFLAGS diff --git a/depends/patches/qt/no-xlib.patch b/depends/patches/qt/no-xlib.patch index f4a6f09ee4..d6846aaca2 100644 --- a/depends/patches/qt/no-xlib.patch +++ b/depends/patches/qt/no-xlib.patch @@ -22,7 +22,7 @@ index 7c62c2e2b3..c05c6c0a07 100644 #include <xcb/xfixes.h> #include <xcb/xcb_image.h> -@@ -391,6 +393,7 @@ void QXcbCursor::changeCursor(QCursor *cursor, QWindow *window) +@@ -391,6 +391,7 @@ void QXcbCursor::changeCursor(QCursor *cursor, QWindow *window) xcb_flush(xcb_connection()); } @@ -30,7 +30,7 @@ index 7c62c2e2b3..c05c6c0a07 100644 static int cursorIdForShape(int cshape) { int cursorId = 0; -@@ -444,6 +447,7 @@ static int cursorIdForShape(int cshape) +@@ -444,6 +445,7 @@ static int cursorIdForShape(int cshape) } return cursorId; } @@ -38,7 +38,7 @@ index 7c62c2e2b3..c05c6c0a07 100644 xcb_cursor_t QXcbCursor::createNonStandardCursor(int cshape) { -@@ -556,7 +560,9 @@ static xcb_cursor_t loadCursor(void *dpy, int cshape) +@@ -556,7 +558,9 @@ static xcb_cursor_t loadCursor(void *dpy, int cshape) xcb_cursor_t QXcbCursor::createFontCursor(int cshape) { xcb_connection_t *conn = xcb_connection(); @@ -48,16 +48,15 @@ index 7c62c2e2b3..c05c6c0a07 100644 xcb_cursor_t cursor = XCB_NONE; // Try Xcursor first -@@ -585,7 +591,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) - +@@ -586,6 +590,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) // Non-standard X11 cursors are created from bitmaps cursor = createNonStandardCursor(cshape); -- + +#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library) // Create a glpyh cursor if everything else failed if (!cursor && cursorId) { cursor = xcb_generate_id(conn); -@@ -593,6 +599,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) +@@ -593,6 +598,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape) cursorId, cursorId + 1, 0xFFFF, 0xFFFF, 0xFFFF, 0, 0, 0); } diff --git a/depends/patches/qt/no_sdk_version_check.patch b/depends/patches/qt/no_sdk_version_check.patch deleted file mode 100644 index b16635b572..0000000000 --- a/depends/patches/qt/no_sdk_version_check.patch +++ /dev/null @@ -1,20 +0,0 @@ -commit f5eb142cd04be2bc4ca610ed3b5b7e8ce3520ee3 -Author: fanquake <fanquake@gmail.com> -Date: Tue Jan 5 16:08:49 2021 +0800 - - Don't invoke macOS SDK version checking - - This tries to use xcrun which is not available when cross-compiling. - -diff --git a/qtbase/mkspecs/features/mac/default_post.prf b/qtbase/mkspecs/features/mac/default_post.prf -index 92a9112bca6..447e186eb26 100644 ---- a/qtbase/mkspecs/features/mac/default_post.prf -+++ b/qtbase/mkspecs/features/mac/default_post.prf -@@ -8,7 +8,6 @@ contains(TEMPLATE, .*app) { - !macx-xcode:if(isEmpty(BUILDS)|build_pass) { - # Detect changes to the platform SDK - QMAKE_EXTRA_VARIABLES += QMAKE_MAC_SDK QMAKE_MAC_SDK_VERSION QMAKE_XCODE_DEVELOPER_PATH -- QMAKE_EXTRA_INCLUDES += $$shell_quote($$PWD/sdk.mk) - } - - # Detect incompatible SDK versions diff --git a/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch b/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch index 0358bea6e9..f0c14a9400 100644 --- a/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch +++ b/depends/patches/qt/qtbase-moc-ignore-gcc-macro.patch @@ -7,7 +7,7 @@ Upstream report: https://bugreports.qt.io/browse/QTBUG-83160 diff --git a/qtbase/src/tools/moc/main.cpp b/qtbase/src/tools/moc/main.cpp --- a/qtbase/src/tools/moc/main.cpp +++ b/qtbase/src/tools/moc/main.cpp -@@ -188,6 +188,7 @@ int runMoc(int argc, char **argv) +@@ -238,6 +238,7 @@ int runMoc(int argc, char **argv) dummyVariadicFunctionMacro.arguments += Symbol(0, PP_IDENTIFIER, "__VA_ARGS__"); pp.macros["__attribute__"] = dummyVariadicFunctionMacro; pp.macros["__declspec"] = dummyVariadicFunctionMacro; diff --git a/depends/patches/qt/support_new_android_ndks.patch b/depends/patches/qt/support_new_android_ndks.patch deleted file mode 100644 index 85c8ae2132..0000000000 --- a/depends/patches/qt/support_new_android_ndks.patch +++ /dev/null @@ -1,122 +0,0 @@ -Follow Google's BuildSystemMaintainers doc to support future NDK releases. - -Upstream commit: - - Qt 5.14: 9b14950ff600a4ce5a8698b67ab38907c50417f1 - ---- old/qtbase/mkspecs/android-clang/qmake.conf -+++ new/qtbase/mkspecs/android-clang/qmake.conf -@@ -14,43 +14,29 @@ - QMAKE_CC = $$NDK_LLVM_PATH/bin/clang - QMAKE_CXX = $$NDK_LLVM_PATH/bin/clang++ - -+# Follow https://android.googlesource.com/platform/ndk/+/ndk-release-r20/docs/BuildSystemMaintainers.md -+ - equals(ANDROID_TARGET_ARCH, armeabi-v7a): \ -- QMAKE_CFLAGS += -target armv7-none-linux-androideabi --else: equals(ANDROID_TARGET_ARCH, armeabi): \ -- QMAKE_CFLAGS += -target armv5te-none-linux-androideabi -+ QMAKE_CFLAGS = -target armv7a-linux-androideabi$$replace(ANDROID_PLATFORM, "android-", "") - else: equals(ANDROID_TARGET_ARCH, arm64-v8a): \ -- QMAKE_CFLAGS += -target aarch64-none-linux-android -+ QMAKE_CFLAGS = -target aarch64-linux-android$$replace(ANDROID_PLATFORM, "android-", "") - else: equals(ANDROID_TARGET_ARCH, x86): \ -- QMAKE_CFLAGS += -target i686-none-linux-android -mstackrealign -+ QMAKE_CFLAGS = -target i686-linux-android$$replace(ANDROID_PLATFORM, "android-", "") -mstackrealign - else: equals(ANDROID_TARGET_ARCH, x86_64): \ -- QMAKE_CFLAGS += -target x86_64-none-linux-android --else: equals(ANDROID_TARGET_ARCH, mips): \ -- QMAKE_CFLAGS += -target mipsel-none-linux-android --else: equals(ANDROID_TARGET_ARCH, mips64): \ -- QMAKE_CFLAGS += -target mips64el-none-linux-android -- --QMAKE_CFLAGS += -gcc-toolchain $$NDK_TOOLCHAIN_PATH -fno-limit-debug-info -- --QMAKE_LINK = $$QMAKE_CXX $$QMAKE_CFLAGS -Wl,--exclude-libs,libgcc.a -Wl,--exclude-libs,libatomic.a -nostdlib++ --equals(ANDROID_TARGET_ARCH, armeabi-v7a): QMAKE_LINK += -Wl,--exclude-libs,libunwind.a -- --QMAKE_CFLAGS += -DANDROID_HAS_WSTRING --sysroot=$$NDK_ROOT/sysroot \ -- -isystem $$NDK_ROOT/sysroot/usr/include/$$NDK_TOOLS_PREFIX \ -- -isystem $$NDK_ROOT/sources/cxx-stl/llvm-libc++/include \ -- -isystem $$NDK_ROOT/sources/android/support/include \ -- -isystem $$NDK_ROOT/sources/cxx-stl/llvm-libc++abi/include -+ QMAKE_CFLAGS = -target x86_64-linux-android$$replace(ANDROID_PLATFORM, "android-", "") - --ANDROID_SOURCES_CXX_STL_LIBDIR = $$NDK_ROOT/sources/cxx-stl/llvm-libc++/libs/$$ANDROID_TARGET_ARCH -+QMAKE_CFLAGS += -fno-limit-debug-info - --ANDROID_STDCPP_PATH = $$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++_shared.so -+QMAKE_LINK = $$QMAKE_CXX $$QMAKE_CFLAGS - --ANDROID_USE_LLVM = true -+ANDROID_STDCPP_PATH = $$NDK_LLVM_PATH/sysroot/usr/lib/$$NDK_TOOLS_PREFIX/libc++_shared.so - --exists($$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++.so): \ -- ANDROID_CXX_STL_LIBS = -lc++ --else: \ -- ANDROID_CXX_STL_LIBS = $$ANDROID_SOURCES_CXX_STL_LIBDIR/libc++.so.$$replace(ANDROID_PLATFORM, "android-", "") -+ANDROID_USE_LLVM = true - --QMAKE_CFLAGS_OPTIMIZE_SIZE = -Oz -+QMAKE_CFLAGS_OPTIMIZE_SIZE = -Oz -+QMAKE_LIBDIR_POST = -+QMAKE_LFLAGS = -+QMAKE_LIBS_PRIVATE = -+ANDROID_CXX_STL_LIBS = - - include(../common/android-base-tail.conf) - ---- old/qtbase/mkspecs/common/android-base-head.conf -+++ new/qtbase/mkspecs/common/android-base-head.conf -@@ -64,7 +58,6 @@ - } - - CONFIG += $$ANDROID_PLATFORM --QMAKE_CFLAGS = -D__ANDROID_API__=$$replace(ANDROID_PLATFORM, "android-", "") - - ANDROID_PLATFORM_ROOT_PATH = $$NDK_ROOT/platforms/$$ANDROID_PLATFORM/arch-$$ANDROID_ARCHITECTURE/ - ---- old/qtbase/mkspecs/common/android-base-tail.conf -+++ new/qtbase/mkspecs/common/android-base-tail.conf -@@ -6,22 +6,17 @@ - QMAKE_CFLAGS += -fstack-protector-strong -DANDROID - - equals(ANDROID_TARGET_ARCH, armeabi-v7a): \ -- QMAKE_CFLAGS += -march=armv7-a -mfloat-abi=softfp -mfpu=vfp -fno-builtin-memmove -+ QMAKE_CFLAGS += -march=armv7-a -mfloat-abi=softfp -mfpu=vfp - else: equals(ANDROID_TARGET_ARCH, armeabi): \ -- QMAKE_CFLAGS += -march=armv5te -mtune=xscale -msoft-float -fno-builtin-memmove --# -fno-builtin-memmove is used to workaround https://code.google.com/p/android/issues/detail?id=81692 -+ QMAKE_CFLAGS += -march=armv5te -mtune=xscale -msoft-float - - QMAKE_CFLAGS_WARN_ON = -Wall -W - QMAKE_CFLAGS_WARN_OFF = - equals(ANDROID_TARGET_ARCH, armeabi-v7a) | equals(ANDROID_TARGET_ARCH, armeabi) { - CONFIG += optimize_size - QMAKE_CFLAGS_DEBUG = -g -marm -O0 -- equals(ANDROID_TARGET_ARCH, armeabi):if(equals(NDK_TOOLCHAIN_VERSION, 4.8)|equals(NDK_TOOLCHAIN_VERSION, 4.9)) { -- DEFINES += QT_OS_ANDROID_GCC_48_WORKAROUND -- } else { -- QMAKE_CFLAGS_RELEASE += -mthumb -- QMAKE_CFLAGS_RELEASE_WITH_DEBUGINFO += -mthumb -- } -+ QMAKE_CFLAGS_RELEASE += -mthumb -+ QMAKE_CFLAGS_RELEASE_WITH_DEBUGINFO += -mthumb - } - - QMAKE_CFLAGS_SHLIB = -fPIC -@@ -61,15 +56,12 @@ - QMAKE_RANLIB = $${CROSS_COMPILE}ranlib - - QMAKE_INCDIR_POST = --QMAKE_LIBDIR_POST = $$ANDROID_SOURCES_CXX_STL_LIBDIR - QMAKE_INCDIR_X11 = - QMAKE_LIBDIR_X11 = - QMAKE_INCDIR_OPENGL = - QMAKE_LIBDIR_OPENGL = - - QMAKE_LINK_SHLIB = $$QMAKE_LINK --QMAKE_LFLAGS = --sysroot=$$ANDROID_PLATFORM_ROOT_PATH --equals(ANDROID_TARGET_ARCH, x86_64) QMAKE_LFLAGS += -L$$ANDROID_PLATFORM_ROOT_PATH/usr/lib64 - QMAKE_LFLAGS_APP = -Wl,--no-undefined -Wl,-z,noexecstack -shared - QMAKE_LFLAGS_SHLIB = -Wl,--no-undefined -Wl,-z,noexecstack -shared - QMAKE_LFLAGS_PLUGIN = $$QMAKE_LFLAGS_SHLIB diff --git a/depends/patches/qt/use_android_ndk23.patch b/depends/patches/qt/use_android_ndk23.patch new file mode 100644 index 0000000000..85b8690218 --- /dev/null +++ b/depends/patches/qt/use_android_ndk23.patch @@ -0,0 +1,13 @@ +Use Android NDK r23 LTS + +--- old/qtbase/mkspecs/features/android/default_pre.prf ++++ new/qtbase/mkspecs/features/android/default_pre.prf +@@ -73,7 +73,7 @@ else: equals(QT_ARCH, x86_64): CROSS_COMPILE = $$NDK_LLVM_PATH/bin/x86_64-linux- + else: equals(QT_ARCH, arm64-v8a): CROSS_COMPILE = $$NDK_LLVM_PATH/bin/aarch64-linux-android- + else: CROSS_COMPILE = $$NDK_LLVM_PATH/bin/arm-linux-androideabi- + +-QMAKE_RANLIB = $${CROSS_COMPILE}ranlib ++QMAKE_RANLIB = $$NDK_LLVM_PATH/bin/llvm-ranlib + QMAKE_LINK_SHLIB = $$QMAKE_LINK + QMAKE_LFLAGS = + diff --git a/doc/README.md b/doc/README.md index aabfe220bc..4845f00ade 100644 --- a/doc/README.md +++ b/doc/README.md @@ -71,6 +71,7 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th ### Miscellaneous - [Assets Attribution](assets-attribution.md) +- [Assumeutxo design](assumeutxo.md) - [bitcoin.conf Configuration File](bitcoin-conf.md) - [Files](files.md) - [Fuzz-testing](fuzzing.md) diff --git a/doc/assumeutxo.md b/doc/assumeutxo.md new file mode 100644 index 0000000000..2726cf779b --- /dev/null +++ b/doc/assumeutxo.md @@ -0,0 +1,138 @@ +# assumeutxo + +Assumeutxo is a feature that allows fast bootstrapping of a validating bitcoind +instance with a very similar security model to assumevalid. + +The RPC commands `dumptxoutset` and `loadtxoutset` are used to respectively generate +and load UTXO snapshots. The utility script `./contrib/devtools/utxo_snapshot.sh` may +be of use. + +## General background + +- [assumeutxo proposal](https://github.com/jamesob/assumeutxo-docs/tree/2019-04-proposal/proposal) +- [Github issue](https://github.com/bitcoin/bitcoin/issues/15605) +- [draft PR](https://github.com/bitcoin/bitcoin/pull/15606) + +## Design notes + +- A new block index `nStatus` flag is introduced, `BLOCK_ASSUMED_VALID`, to mark block + index entries that are required to be assumed-valid by a chainstate created + from a UTXO snapshot. This flag is mostly used as a way to modify certain + CheckBlockIndex() logic to account for index entries that are pending validation by a + chainstate running asynchronously in the background. We also use this flag to control + which index entries are added to setBlockIndexCandidates during LoadBlockIndex(). + +- Indexing implementations via BaseIndex can no longer assume that indexation happens + sequentially, since background validation chainstates can submit BlockConnected + events out of order with the active chain. + +- The concept of UTXO snapshots is treated as an implementation detail that lives + behind the ChainstateManager interface. The external presentation of the changes + required to facilitate the use of UTXO snapshots is the understanding that there are + now certain regions of the chain that can be temporarily assumed to be valid (using + the nStatus flag mentioned above). In certain cases, e.g. wallet rescanning, this is + very similar to dealing with a pruned chain. + + Logic outside ChainstateManager should try not to know about snapshots, instead + preferring to work in terms of more general states like assumed-valid. + + +## Chainstate phases + +Chainstate within the system goes through a number of phases when UTXO snapshots are +used, as managed by `ChainstateManager`. At various points there can be multiple +`CChainState` objects in existence to facilitate both maintaining the network tip and +performing historical validation of the assumed-valid chain. + +It is worth noting that though there are multiple separate chainstates, those +chainstates share use of a common block index (i.e. they hold the same `BlockManager` +reference). + +The subheadings below outline the phases and the corresponding changes to chainstate +data. + +### "Normal" operation via initial block download + +`ChainstateManager` manages a single CChainState object, for which +`m_snapshot_blockhash` is null. This chainstate is (maybe obviously) +considered active. This is the "traditional" mode of operation for bitcoind. + +| | | +| ---------- | ----------- | +| number of chainstates | 1 | +| active chainstate | ibd | + +### User loads a UTXO snapshot via `loadtxoutset` RPC + +`ChainstateManager` initializes a new chainstate (see `ActivateSnapshot()`) to load the +snapshot contents into. During snapshot load and validation (see +`PopulateAndValidateSnapshot()`), the new chainstate is not considered active and the +original chainstate remains in use as active. + +| | | +| ---------- | ----------- | +| number of chainstates | 2 | +| active chainstate | ibd | + +Once the snapshot chainstate is loaded and validated, it is promoted to active +chainstate and a sync to tip begins. A new chainstate directory is created in the +datadir for the snapshot chainstate called +`chainstate_[SHA256 blockhash of snapshot base block]`. + +| | | +| ---------- | ----------- | +| number of chainstates | 2 | +| active chainstate | snapshot | + +The snapshot begins to sync to tip from its base block, technically in parallel with +the original chainstate, but it is given priority during block download and is +allocated most of the cache (see `MaybeRebalanceCaches()` and usages) as our chief +consideration is getting to network tip. + +**Failure consideration:** if shutdown happens at any point during this phase, both +chainstates will be detected during the next init and the process will resume. + +### Snapshot chainstate hits network tip + +Once the snapshot chainstate leaves IBD, caches are rebalanced +(via `MaybeRebalanceCaches()` in `ActivateBestChain()`) and more cache is given +to the background chainstate, which is responsible for doing full validation of the +assumed-valid parts of the chain. + +**Note:** at this point, ValidationInterface callbacks will be coming in from both +chainstates. Considerations here must be made for indexing, which may no longer be happening +sequentially. + +### Background chainstate hits snapshot base block + +Once the tip of the background chainstate hits the base block of the snapshot +chainstate, we stop use of the background chainstate by setting `m_stop_use` (not yet +committed - see #15606), in `CompleteSnapshotValidation()`, which is checked in +`ActivateBestChain()`). We hash the background chainstate's UTXO set contents and +ensure it matches the compiled value in `CMainParams::m_assumeutxo_data`. + +The background chainstate data lingers on disk until shutdown, when in +`ChainstateManager::Reset()`, the background chainstate is cleaned up with +`ValidatedSnapshotShutdownCleanup()`, which renames the `chainstate_[hash]` datadir as +`chainstate`. + +| | | +| ---------- | ----------- | +| number of chainstates | 2 (ibd has `m_stop_use=true`) | +| active chainstate | snapshot | + +**Failure consideration:** if bitcoind unexpectedly halts after `m_stop_use` is set on +the background chainstate but before `CompleteSnapshotValidation()` can finish, the +need to complete snapshot validation will be detected on subsequent init by +`ChainstateManager::CheckForUncleanShutdown()`. + +### Bitcoind restarts sometime after snapshot validation has completed + +When bitcoind initializes again, what began as the snapshot chainstate is now +indistinguishable from a chainstate that has been built from the traditional IBD +process, and will be initialized as such. + +| | | +| ---------- | ----------- | +| number of chainstates | 1 | +| active chainstate | ibd | diff --git a/doc/bips.md b/doc/bips.md index 45954bcfd8..b5fa9315d3 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -57,3 +57,11 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v22.0**): with mainnet activation as of **v0.21.1** ([PR 21377](https://github.com/bitcoin/bitcoin/pull/21377), [PR 21686](https://github.com/bitcoin/bitcoin/pull/21686)). * [`BIP 350`](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki): Addresses for native v1+ segregated Witness outputs use Bech32m instead of Bech32 as of **v22.0** ([PR 20861](https://github.com/bitcoin/bitcoin/pull/20861)). +* [`BIP 380`](https://github.com/bitcoin/bips/blob/master/bip-0380.mediawiki) + [`381`](https://github.com/bitcoin/bips/blob/master/bip-0381.mediawiki) + [`382`](https://github.com/bitcoin/bips/blob/master/bip-0382.mediawiki) + [`383`](https://github.com/bitcoin/bips/blob/master/bip-0383.mediawiki) + [`384`](https://github.com/bitcoin/bips/blob/master/bip-0384.mediawiki) + [`385`](https://github.com/bitcoin/bips/blob/master/bip-0385.mediawiki): + Output Script Descriptors, and most of Script Expressions are implemented as of **v0.17.0** ([PR 13697](https://github.com/bitcoin/bitcoin/pull/13697)). +* [`BIP 386`](https://github.com/bitcoin/bips/blob/master/bip-0386.mediawiki): tr() Output Script Descriptors are implemented as of **v22.0** ([PR 22051](https://github.com/bitcoin/bitcoin/pull/22051)). diff --git a/doc/build-android.md b/doc/build-android.md index 7a8a9e6a65..2f2e01c441 100644 --- a/doc/build-android.md +++ b/doc/build-android.md @@ -3,9 +3,22 @@ ANDROID BUILD NOTES This guide describes how to build and package the `bitcoin-qt` GUI for Android on Linux and macOS. -## Preparation -You will need to get the Android NDK and build dependencies for Android as described in [depends/README.md](../depends/README.md). +## Dependencies + +Before proceeding with an Android build one needs to get the [Android SDK](https://developer.android.com/studio) and use the "SDK Manager" tool to download the NDK and one or more "Platform packages" (these are Android versions and have a corresponding API level). + +The minimum supported Android NDK version is [r23](https://github.com/android/ndk/wiki/Changelog-r23). + +In order to build `ANDROID_API_LEVEL` (API level corresponding to the Android version targeted, e.g. Android 9.0 Pie is 28 and its "Platform package" needs to be available) and `ANDROID_TOOLCHAIN_BIN` (path to toolchain binaries depending on the platform the build is being performed on) need to be set. + +API levels from 24 to 29 have been tested to work. + +If the build includes Qt, environment variables `ANDROID_SDK` and `ANDROID_NDK` need to be set as well but can otherwise be omitted. +This is an example command for a default build with no disabled dependencies: + + ANDROID_SDK=/home/user/Android/Sdk ANDROID_NDK=/home/user/Android/Sdk/ndk-bundle make HOST=aarch64-linux-android ANDROID_API_LEVEL=28 ANDROID_TOOLCHAIN_BIN=/home/user/Android/Sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/bin + ## Building and packaging diff --git a/doc/build-unix.md b/doc/build-unix.md index 02c36eea7c..f50a9b23c0 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -42,13 +42,12 @@ Optional dependencies: ------------|------------------|---------------------- miniupnpc | UPnP Support | Firewall-jumping support libnatpmp | NAT-PMP Support | Firewall-jumping support - libdb4.8 | Berkeley DB | Optional, wallet storage (only needed when wallet enabled) + libdb4.8 | Berkeley DB | Wallet storage (only needed when legacy wallet enabled) qt | GUI | GUI toolkit (only needed when GUI enabled) - libqrencode | QR codes in GUI | Optional for generating QR codes (only needed when GUI enabled) - univalue | Utility | JSON parsing and encoding (bundled version will be used unless --with-system-univalue passed to configure) - libzmq3 | ZMQ notification | Optional, allows generating ZMQ notifications (requires ZMQ version >= 4.0.0) - sqlite3 | SQLite DB | Optional, wallet storage (only needed when wallet enabled) - systemtap | Tracing (USDT) | Optional, statically defined tracepoints + libqrencode | QR codes in GUI | QR code generation (only needed when GUI enabled) + libzmq3 | ZMQ notification | ZMQ notifications (requires ZMQ version >= 4.0.0) + sqlite3 | SQLite DB | Wallet storage (only needed when descriptor wallet enabled) + systemtap | Tracing (USDT) | Statically defined tracepoints For the versions used, see [dependencies.md](dependencies.md) @@ -85,19 +84,15 @@ Now, you can either build from self-compiled [depends](/depends/README.md) or in sudo apt-get install libevent-dev libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev -Berkeley DB is required for the wallet. - -Ubuntu and Debian have their own `libdb-dev` and `libdb++-dev` packages, but these will install -Berkeley DB 5.1 or later. This will break binary wallet compatibility with the distributed executables, which -are based on BerkeleyDB 4.8. If you do not care about wallet compatibility, -pass `--with-incompatible-bdb` to configure. - -Otherwise, you can build Berkeley DB [yourself](#berkeley-db). - SQLite is required for the descriptor wallet: sudo apt install libsqlite3-dev +Berkeley DB is required for the legacy wallet. Ubuntu and Debian have their own `libdb-dev` and `libdb++-dev` packages, +but these will install Berkeley DB 5.1 or later. This will break binary wallet compatibility with the distributed +executables, which are based on BerkeleyDB 4.8. If you do not care about wallet compatibility, pass +`--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley DB [yourself](#berkeley-db). + To build Bitcoin Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode) Optional port mapping libraries (see: `--with-miniupnpc`, `--enable-upnp-default`, and `--with-natpmp`, `--enable-natpmp-default`): @@ -146,20 +141,18 @@ Now, you can either build from self-compiled [depends](/depends/README.md) or in sudo dnf install libevent-devel boost-devel -Berkeley DB is required for the wallet: +SQLite is required for the descriptor wallet: + + sudo dnf install sqlite-devel + +Berkeley DB is required for the legacy wallet: sudo dnf install libdb4-devel libdb4-cxx-devel Newer Fedora releases, since Fedora 33, have only `libdb-devel` and `libdb-cxx-devel` packages, but these will install Berkeley DB 5.3 or later. This will break binary wallet compatibility with the distributed executables, which are based on Berkeley DB 4.8. If you do not care about wallet compatibility, -pass `--with-incompatible-bdb` to configure. - -Otherwise, you can build Berkeley DB [yourself](#berkeley-db). - -SQLite is required for the descriptor wallet: - - sudo dnf install sqlite-devel +pass `--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley DB [yourself](#berkeley-db). To build Bitcoin Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode) @@ -225,8 +218,10 @@ turned off by default. See the configure options for NAT-PMP behavior desired: Berkeley DB ----------- -It is recommended to use Berkeley DB 4.8. If you have to build it yourself, -you can use [the installation script included in contrib/](/contrib/install_db4.sh) + +The legacy wallet uses Berkeley DB. To ensure backwards compatibility it is +recommended to use Berkeley DB 4.8. If you have to build it yourself, you can +use [the installation script included in contrib/](/contrib/install_db4.sh) like so: ```shell @@ -239,15 +234,6 @@ Otherwise, you can build Bitcoin Core from self-compiled [depends](/depends/READ **Note**: You only need Berkeley DB if the wallet is enabled (see [*Disable-wallet mode*](#disable-wallet-mode)). -Boost ------ -If you need to build Boost yourself: - - sudo su - ./bootstrap.sh - ./bjam install - - Security -------- To help make your Bitcoin Core installation more secure by making certain attacks impossible to @@ -326,7 +312,7 @@ This example lists the steps necessary to setup and build a command line only, n Note: Enabling wallet support requires either compiling against a Berkeley DB newer than 4.8 (package `db`) using `--with-incompatible-bdb`, or building and depending on a local version of Berkeley DB 4.8. The readily available Arch Linux packages are currently built using -`--with-incompatible-bdb` according to the [PKGBUILD](https://projects.archlinux.org/svntogit/community.git/tree/bitcoin/trunk/PKGBUILD). +`--with-incompatible-bdb` according to the [PKGBUILD](https://github.com/archlinux/svntogit-community/blob/packages/bitcoin/trunk/PKGBUILD). As mentioned above, when maintaining portability of the wallet between the standard Bitcoin Core distributions and independently built node software is desired, Berkeley DB 4.8 must be used. diff --git a/doc/dependencies.md b/doc/dependencies.md index 0c1fd6ba98..f2b6760ac2 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -8,11 +8,10 @@ These are the dependencies currently used by Bitcoin Core. You can find instruct | Berkeley DB | [4.8.30](https://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html) | 4.8.x | No | | | | Boost | [1.71.0](https://www.boost.org/users/download/) | [1.64.0](https://github.com/bitcoin/bitcoin/pull/22320) | No | | | | Clang<sup>[ \* ](#note1)</sup> | | [7.0](https://releases.llvm.org/download.html) (C++17 & std::filesystem support) | | | | -| Expat | [2.2.7](https://libexpat.github.io/) | | No | Yes | | | fontconfig | [2.12.1](https://www.freedesktop.org/software/fontconfig/release/) | | No | Yes | | -| FreeType | [2.7.1](https://download.savannah.gnu.org/releases/freetype) | | No | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Android only) | +| FreeType | [2.11.0](https://download.savannah.gnu.org/releases/freetype) | | No | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Android only) | | GCC | | [8.1](https://gcc.gnu.org/) (C++17 & std::filesystem support) | | | | -| glibc | | [2.17](https://www.gnu.org/software/libc/) | | | | | +| glibc | | [2.18](https://www.gnu.org/software/libc/) | | | | | | HarfBuzz-NG | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | | libevent | [2.1.12-stable](https://github.com/libevent/libevent/releases) | [2.0.21](https://github.com/bitcoin/bitcoin/pull/18676) | No | | | | libnatpmp | git commit [4536032...](https://github.com/miniupnp/libnatpmp/tree/4536032ae32268a45c073a4d5e91bbab4534773a) | | No | | | diff --git a/doc/developer-notes.md b/doc/developer-notes.md index a05ea93a46..1888897856 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -1029,6 +1029,9 @@ Current subtrees include: - Subtree at https://github.com/bitcoin-core/univalue-subtree ; maintained by Core contributors. - Deviates from upstream https://github.com/jgarzik/univalue. +- src/minisketch + - Upstream at https://github.com/sipa/minisketch ; maintained by Core contributors. + Upgrading LevelDB --------------------- @@ -1251,6 +1254,12 @@ A few guidelines for introducing and reviewing new RPC interfaces: - *Rationale*: User-facing consistency. +- Use `fs::path::u8string()` and `fs::u8path()` functions when converting path + to JSON strings, not `fs::PathToString` and `fs::PathFromString` + + - *Rationale*: JSON strings are Unicode strings, not byte strings, and + RFC8259 requires JSON to be encoded as UTF-8. + Internal interface guidelines ----------------------------- diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 0880f9f581..73d04837f1 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -19,6 +19,10 @@ $ FUZZ=process_message src/test/fuzz/fuzz There is also a runner script to execute all fuzz targets. Refer to `./test/fuzz/test_runner.py --help` for more details. +## Overview of Bitcoin Core fuzzing + +[Google](https://github.com/google/fuzzing/) has a good overview of fuzzing in general, with contributions from key architects of some of the most-used fuzzers. [This paper](https://agroce.github.io/bitcoin_report.pdf) includes an external overview of the status of Bitcoin Core fuzzing, as of summer 2021. [John Regehr](https://blog.regehr.org/archives/1687) provides good advice on writing code that assists fuzzers in finding bugs, which is useful for developers to keep in mind. + ## Fuzzing harnesses and output [`process_message`](https://github.com/bitcoin/bitcoin/blob/master/src/test/fuzz/process_message.cpp) is a fuzzing harness for the [`ProcessMessage(...)` function (`net_processing`)](https://github.com/bitcoin/bitcoin/blob/master/src/net_processing.cpp). The available fuzzing harnesses are found in [`src/test/fuzz/`](https://github.com/bitcoin/bitcoin/tree/master/src/test/fuzz). diff --git a/doc/release-notes-23093.md b/doc/release-notes-23093.md deleted file mode 100644 index 68fbaec53c..0000000000 --- a/doc/release-notes-23093.md +++ /dev/null @@ -1,11 +0,0 @@ -Notable changes -=============== - -Updated RPCs ------------- - -- `upgradewallet` will now automatically flush the keypool if upgrading -from a non-HD wallet to an HD wallet, to immediately start using the -newly-generated HD keys. -- a new RPC `newkeypool` has been added, which will flush (entirely -clear and refill) the keypool. diff --git a/doc/release-notes.md b/doc/release-notes.md index 231195108b..230a56d2cd 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -77,6 +77,21 @@ Otherwise, please use the `rescanblockchain` RPC to trigger a rescan. (#23123) Updated RPCs ------------ +- `upgradewallet` will now automatically flush the keypool if upgrading + from a non-HD wallet to an HD wallet, to immediately start using the + newly-generated HD keys. (#23093) + +- a new RPC `newkeypool` has been added, which will flush (entirely + clear and refill) the keypool. (#23093) + +- The `validateaddress` RPC now returns an `error_locations` array for invalid + addresses, with the indices of invalid character locations in the address (if + known). For example, this will attempt to locate up to two Bech32 errors, and + return their locations if successful. Success and correctness are only guaranteed + if fewer than two substitution errors have been made. + The error message returned in the `error` field now also returns more specific + errors when decoding fails. (#16807) + - The `-deprecatedrpc=addresses` configuration option has been removed. RPCs `gettxout`, `getrawtransaction`, `decoderawtransaction`, `decodescript`, `gettransaction verbose=true` and REST endpoints `/rest/tx`, `/rest/getutxos`, @@ -136,6 +151,10 @@ Updated settings mean `-persistmempool=1`. Passing `-persistmempool=0`, `-persistmempool=1` and `-nopersistmempool` is unaffected. (#23061) +- `-maxuploadtarget` now allows human readable byte units [k|K|m|M|g|G|t|T]. + E.g. `-maxuploadtarget=500g`. No whitespace, +- or fractions allowed. + Default is `M` if no suffix provided. (#23249) + Tools and Utilities ------------------- diff --git a/doc/release-process.md b/doc/release-process.md index 14567d4f15..f786b345b1 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -19,8 +19,7 @@ Release Process * On both the master branch and the new release branch: - update `CLIENT_VERSION_MAJOR` in [`configure.ac`](../configure.ac) - - update `CLIENT_VERSION_MAJOR`, `PACKAGE_VERSION`, and `PACKAGE_STRING` in [`build_msvc/bitcoin_config.h`](/build_msvc/bitcoin_config.h) -* On the new release branch in [`configure.ac`](../configure.ac) and [`build_msvc/bitcoin_config.h`](/build_msvc/bitcoin_config.h) (see [this commit](https://github.com/bitcoin/bitcoin/commit/742f7dd)): +* On the new release branch in [`configure.ac`](../configure.ac)(see [this commit](https://github.com/bitcoin/bitcoin/commit/742f7dd)): - set `CLIENT_VERSION_MINOR` to `0` - set `CLIENT_VERSION_BUILD` to `0` - set `CLIENT_VERSION_IS_RELEASE` to `true` diff --git a/doc/tracing.md b/doc/tracing.md index 57104c43a0..5b9ba09c2f 100644 --- a/doc/tracing.md +++ b/doc/tracing.md @@ -108,6 +108,55 @@ Arguments passed: 5. SigOps in the Block (excluding coinbase SigOps) `uint64` 6. Time it took to connect the Block in microseconds (µs) as `uint64` +### Context `utxocache` + +#### Tracepoint `utxocache:flush` + +Is called *after* the caches and indexes are flushed depending on the mode +`CChainState::FlushStateToDisk` is called with. + +Arguments passed: +1. Duration in microseconds as `int64` +2. Flush state mode as `uint32`. It's an enumerator class with values `0` + (`NONE`), `1` (`IF_NEEDED`), `2` (`PERIODIC`), `3` (`ALWAYS`) +3. Number of coins flushed as `uint64` +4. Memory usage in bytes as `uint64` +5. If the flush was pruned as `bool` +6. If it was full flush as `bool` + +#### Tracepoint `utxocache:add` + +It is called when a new coin is added to the UTXO cache. + +Arguments passed: +1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Output index as `uint32` +3. Block height the coin was added to the UTXO-set as `uint32` +4. Value of the coin as `int64` +5. If the coin is a coinbase as `bool` + +#### Tracepoint `utxocache:spent` + +It is called when a coin is spent from the UTXO cache. + +Arguments passed: +1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Output index as `uint32` +3. Block height the coin was spent, as `uint32` +4. Value of the coin as `int64` +5. If the coin is a coinbase as `bool` + +#### Tracepoint `utxocache:uncache` + +It is called when the UTXO with the given outpoint is removed from the cache. + +Arguments passed: +1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Output index as `uint32` +3. Block height the coin was uncached, as `uint32` +4. Value of the coin as `int64` +. If the coin is a coinbase as `bool` + ## Adding tracepoints to Bitcoin Core To add a new tracepoint, `#include <util/trace.h>` in the compilation unit where diff --git a/share/genbuild.sh b/share/genbuild.sh index fe89ae1fde..2ced24cd7b 100755 --- a/share/genbuild.sh +++ b/share/genbuild.sh @@ -26,7 +26,7 @@ if [ "${BITCOIN_GENBUILD_NO_GIT}" != "1" ] && [ -e "$(command -v git)" ] && [ "$ # if latest commit is tagged and not dirty, then override using the tag name RAWDESC=$(git describe --abbrev=0 2>/dev/null) - if [ "$(git rev-parse HEAD)" = "$(git rev-list -1 $RAWDESC 2>/dev/null)" ]; then + if [ "$(git rev-parse HEAD)" = "$(git rev-list -1 "$RAWDESC" 2>/dev/null)" ]; then git diff-index --quiet HEAD -- && GIT_TAG=$RAWDESC fi diff --git a/src/.clang-format b/src/.clang-format index a69c57f3e0..791b3b8f9f 100644 --- a/src/.clang-format +++ b/src/.clang-format @@ -34,14 +34,6 @@ IndentWidth: 4 KeepEmptyLinesAtTheStartOfBlocks: false MaxEmptyLinesToKeep: 2 NamespaceIndentation: None -ObjCSpaceAfterProperty: false -ObjCSpaceBeforeProtocolList: false -PenaltyBreakBeforeFirstCallParameter: 1 -PenaltyBreakComment: 300 -PenaltyBreakFirstLessLess: 120 -PenaltyBreakString: 1000 -PenaltyExcessCharacter: 1000000 -PenaltyReturnTypeOnItsOwnLine: 200 PointerAlignment: Left SpaceBeforeAssignmentOperators: true SpaceBeforeParens: ControlStatements @@ -51,6 +43,5 @@ SpacesInAngles: false SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false SpacesInParentheses: false -Standard: Cpp11 -TabWidth: 8 +Standard: c++17 UseTab: Never diff --git a/src/Makefile.am b/src/Makefile.am index 3f5c3a2f4a..72f548c192 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -8,14 +8,14 @@ print-%: FORCE DIST_SUBDIRS = secp256k1 -AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) -AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) +AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) $(LTO_LDFLAGS) +AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) $(LTO_CXXFLAGS) AM_CPPFLAGS = $(DEBUG_CPPFLAGS) $(HARDENED_CPPFLAGS) AM_LIBTOOLFLAGS = --preserve-dup-deps PTHREAD_FLAGS = $(PTHREAD_CFLAGS) $(PTHREAD_LIBS) EXTRA_LIBRARIES = -BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) +BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) LIBBITCOIN_SERVER=libbitcoin_server.a LIBBITCOIN_COMMON=libbitcoin_common.a @@ -166,7 +166,6 @@ BITCOIN_CORE_H = \ mapport.h \ memusage.h \ merkleblock.h \ - miner.h \ net.h \ net_permissions.h \ net_processing.h \ @@ -178,6 +177,8 @@ BITCOIN_CORE_H = \ node/coin.h \ node/coinstats.h \ node/context.h \ + node/miner.h \ + node/minisketchwrapper.h \ node/psbt.h \ node/transaction.h \ node/ui_interface.h \ @@ -245,6 +246,7 @@ BITCOIN_CORE_H = \ util/macros.h \ util/message.h \ util/moneystr.h \ + util/overloaded.h \ util/rbf.h \ util/readwritefile.h \ util/serfloat.h \ @@ -281,6 +283,7 @@ BITCOIN_CORE_H = \ wallet/load.h \ wallet/receive.h \ wallet/rpcwallet.h \ + wallet/rpc/util.h \ wallet/salvage.h \ wallet/scriptpubkeyman.h \ wallet/spend.h \ @@ -333,7 +336,6 @@ libbitcoin_server_a_SOURCES = \ index/txindex.cpp \ init.cpp \ mapport.cpp \ - miner.cpp \ net.cpp \ net_processing.cpp \ node/blockstorage.cpp \ @@ -341,6 +343,8 @@ libbitcoin_server_a_SOURCES = \ node/coinstats.cpp \ node/context.cpp \ node/interfaces.cpp \ + node/miner.cpp \ + node/minisketchwrapper.cpp \ node/psbt.cpp \ node/transaction.cpp \ node/ui_interface.cpp \ @@ -406,7 +410,10 @@ libbitcoin_wallet_a_SOURCES = \ wallet/interfaces.cpp \ wallet/load.cpp \ wallet/receive.cpp \ - wallet/rpcdump.cpp \ + wallet/rpc/backup.cpp \ + wallet/rpc/encrypt.cpp \ + wallet/rpc/signmessage.cpp \ + wallet/rpc/util.cpp \ wallet/rpcwallet.cpp \ wallet/scriptpubkeyman.cpp \ wallet/spend.cpp \ @@ -707,7 +714,18 @@ bitcoin_wallet_SOURCES += init/bitcoin-wallet.cpp bitcoin_wallet_CPPFLAGS = $(bitcoin_bin_cppflags) bitcoin_wallet_CXXFLAGS = $(bitcoin_bin_cxxflags) bitcoin_wallet_LDFLAGS = $(bitcoin_bin_ldflags) -bitcoin_wallet_LDADD = $(LIBBITCOIN_WALLET_TOOL) $(bitcoin_bin_ldadd) +bitcoin_wallet_LDADD = \ + $(LIBBITCOIN_WALLET_TOOL) \ + $(LIBBITCOIN_WALLET) \ + $(LIBBITCOIN_COMMON) \ + $(LIBBITCOIN_UTIL) \ + $(LIBUNIVALUE) \ + $(LIBBITCOIN_CONSENSUS) \ + $(LIBBITCOIN_CRYPTO) \ + $(LIBSECP256K1) \ + $(BOOST_LIBS) \ + $(BDB_LIBS) \ + $(SQLITE_LIBS) if TARGET_WINDOWS bitcoin_wallet_SOURCES += bitcoin-wallet-res.rc @@ -842,8 +860,11 @@ nodist_libbitcoin_ipc_a_SOURCES = $(libbitcoin_ipc_mpgen_output) CLEANFILES += $(libbitcoin_ipc_mpgen_output) endif +include Makefile.minisketch.include + include Makefile.crc32c.include include Makefile.leveldb.include + include Makefile.test_util.include include Makefile.test_fuzz.include diff --git a/src/Makefile.minisketch.include b/src/Makefile.minisketch.include new file mode 100644 index 0000000000..b337f48349 --- /dev/null +++ b/src/Makefile.minisketch.include @@ -0,0 +1,43 @@ +include minisketch/sources.mk + +LIBMINISKETCH_CPPFLAGS= +LIBMINISKETCH_CPPFLAGS += -DDISABLE_DEFAULT_FIELDS -DENABLE_FIELD_32 + +LIBMINISKETCH = minisketch/libminisketch.a +MINISKETCH_LIBS = $(LIBMINISKETCH) + +if ENABLE_CLMUL +LIBMINISKETCH_CLMUL = minisketch/libminisketch_clmul.a +LIBMINISKETCH_CPPFLAGS += -DHAVE_CLMUL +MINISKETCH_LIBS += $(LIBMINISKETCH_CLMUL) +endif + +if HAVE_CLZ +LIBMINISKETCH_CPPFLAGS += -DHAVE_CLZ +endif + +EXTRA_LIBRARIES += $(MINISKETCH_LIBS) + +minisketch_libminisketch_clmul_a_SOURCES = $(MINISKETCH_FIELD_CLMUL_SOURCES_INT) $(MINISKETCH_FIELD_CLMUL_HEADERS_INT) +minisketch_libminisketch_clmul_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) $(CLMUL_CXXFLAGS) +minisketch_libminisketch_clmul_a_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS) + +minisketch_libminisketch_a_SOURCES = $(MINISKETCH_FIELD_GENERIC_SOURCES_INT) $(MINISKETCH_LIB_SOURCES_INT) +minisketch_libminisketch_a_SOURCES += $(MINISKETCH_FIELD_GENERIC_HEADERS_INT) $(MINISKETCH_LIB_HEADERS_INT) $(MINISKETCH_DIST_HEADERS_INT) +minisketch_libminisketch_a_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS) +minisketch_libminisketch_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) + +if ENABLE_TESTS +if !ENABLE_FUZZ +MINISKETCH_TEST = minisketch/test +TESTS += $(MINISKETCH_TEST) +noinst_PROGRAMS += $(MINISKETCH_TEST) + +minisketch_test_SOURCES = $(MINISKETCH_TEST_SOURCES_INT) +minisketch_test_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS) +minisketch_test_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +minisketch_test_LDADD = $(MINISKETCH_LIBS) +minisketch_test_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) + +endif +endif diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index e6ea8bdf61..35d5b0004a 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -391,7 +391,7 @@ QT_BASE_TLD = $(shell tar tf $(QT_BASE_PATH) --exclude='*/*') bitcoin_qt_apk: FORCE mkdir -p $(APK_LIB_DIR) - cp $(dir $(CC))../sysroot/usr/lib/$(host_alias)/libc++_shared.so $(APK_LIB_DIR) + cp $(dir $(lastword $(CC)))../sysroot/usr/lib/$(host_alias)/libc++_shared.so $(APK_LIB_DIR) tar xf $(QT_BASE_PATH) -C qt/android/src/ $(QT_BASE_TLD)src/android/jar/src --strip-components=5 tar xf $(QT_BASE_PATH) -C qt/android/src/ $(QT_BASE_TLD)src/android/java/src --strip-components=5 tar xf $(QT_BASE_PATH) -C qt/android/res/ $(QT_BASE_TLD)src/android/java/res --strip-components=5 diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 27f9382631..06d195aaaf 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -16,6 +16,7 @@ FUZZ_BINARY=test/fuzz/fuzz$(EXEEXT) JSON_TEST_FILES = \ test/data/script_tests.json \ + test/data/bip341_wallet_vectors.json \ test/data/base58_encode_decode.json \ test/data/blockfilters.json \ test/data/key_io_valid.json \ @@ -50,6 +51,7 @@ FUZZ_SUITE_LD_COMMON = \ $(BOOST_LIBS) \ $(LIBMEMENV) \ $(LIBSECP256K1) \ + $(MINISKETCH_LIBS) \ $(EVENT_LIBS) \ $(EVENT_PTHREADS_LIBS) @@ -63,11 +65,10 @@ endif # test_bitcoin binary # BITCOIN_TESTS =\ - test/arith_uint256_tests.cpp \ - test/scriptnum10.h \ test/addrman_tests.cpp \ - test/amount_tests.cpp \ test/allocator_tests.cpp \ + test/amount_tests.cpp \ + test/arith_uint256_tests.cpp \ test/base32_tests.cpp \ test/base58_tests.cpp \ test/base64_tests.cpp \ @@ -75,8 +76,8 @@ BITCOIN_TESTS =\ test/bip32_tests.cpp \ test/blockchain_tests.cpp \ test/blockencodings_tests.cpp \ - test/blockfilter_tests.cpp \ test/blockfilter_index_tests.cpp \ + test/blockfilter_tests.cpp \ test/bloom_tests.cpp \ test/bswap_tests.cpp \ test/checkqueue_tests.cpp \ @@ -86,6 +87,7 @@ BITCOIN_TESTS =\ test/compress_tests.cpp \ test/crypto_tests.cpp \ test/cuckoocache_tests.cpp \ + test/dbwrapper_tests.cpp \ test/denialofservice_tests.cpp \ test/descriptor_tests.cpp \ test/flatfile_tests.cpp \ @@ -97,12 +99,11 @@ BITCOIN_TESTS =\ test/key_io_tests.cpp \ test/key_tests.cpp \ test/logging_tests.cpp \ - test/dbwrapper_tests.cpp \ - test/validation_tests.cpp \ test/mempool_tests.cpp \ test/merkle_tests.cpp \ test/merkleblock_tests.cpp \ test/miner_tests.cpp \ + test/minisketch_tests.cpp \ test/multisig_tests.cpp \ test/net_peer_eviction_tests.cpp \ test/net_tests.cpp \ @@ -122,6 +123,7 @@ BITCOIN_TESTS =\ test/script_parse_tests.cpp \ test/script_standard_tests.cpp \ test/script_tests.cpp \ + test/scriptnum10.h \ test/scriptnum_tests.cpp \ test/serfloat_tests.cpp \ test/serialize_tests.cpp \ @@ -133,20 +135,22 @@ BITCOIN_TESTS =\ test/streams_tests.cpp \ test/sync_tests.cpp \ test/system_tests.cpp \ - test/util_threadnames_tests.cpp \ test/timedata_tests.cpp \ test/torcontrol_tests.cpp \ test/transaction_tests.cpp \ test/txindex_tests.cpp \ + test/txpackage_tests.cpp \ test/txrequest_tests.cpp \ test/txvalidation_tests.cpp \ test/txvalidationcache_tests.cpp \ test/uint256_tests.cpp \ test/util_tests.cpp \ + test/util_threadnames_tests.cpp \ test/validation_block_tests.cpp \ test/validation_chainstate_tests.cpp \ test/validation_chainstatemanager_tests.cpp \ test/validation_flush_tests.cpp \ + test/validation_tests.cpp \ test/validationinterface_tests.cpp \ test/versionbits_tests.cpp @@ -157,6 +161,7 @@ BITCOIN_TESTS += \ wallet/test/wallet_tests.cpp \ wallet/test/walletdb_tests.cpp \ wallet/test/wallet_crypto_tests.cpp \ + wallet/test/wallet_transaction_tests.cpp \ wallet/test/coinselector_tests.cpp \ wallet/test/init_tests.cpp \ wallet/test/ismine_tests.cpp \ @@ -170,6 +175,10 @@ if USE_BDB BITCOIN_TESTS += wallet/test/db_tests.cpp endif +if USE_SQLITE +FUZZ_WALLET_SRC = \ + wallet/test/fuzz/notifications.cpp +endif # USE_SQLITE BITCOIN_TEST_SUITE += \ wallet/test/util.cpp \ @@ -178,7 +187,7 @@ BITCOIN_TEST_SUITE += \ wallet/test/wallet_test_fixture.h \ wallet/test/init_test_fixture.cpp \ wallet/test/init_test_fixture.h -endif +endif # ENABLE_WALLET test_test_bitcoin_SOURCES = $(BITCOIN_TEST_SUITE) $(BITCOIN_TESTS) $(JSON_TEST_FILES) $(RAW_TEST_FILES) test_test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(TESTDEFS) $(EVENT_CFLAGS) @@ -188,7 +197,7 @@ test_test_bitcoin_LDADD += $(LIBBITCOIN_WALLET) endif test_test_bitcoin_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \ - $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) + $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) $(MINISKETCH_LIBS) test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_test_bitcoin_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) @@ -199,14 +208,13 @@ test_test_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) FUZZ_SUITE_LD_COMMON += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif -FUZZ_SUITE_LDFLAGS_COMMON = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) - if ENABLE_FUZZ_BINARY test_fuzz_fuzz_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_fuzz_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_fuzz_LDADD = $(FUZZ_SUITE_LD_COMMON) -test_fuzz_fuzz_LDFLAGS = $(FUZZ_SUITE_LDFLAGS_COMMON) $(RUNTIME_LDFLAGS) +test_fuzz_fuzz_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) $(RUNTIME_LDFLAGS) test_fuzz_fuzz_SOURCES = \ + $(FUZZ_WALLET_SRC) \ test/fuzz/addition_overflow.cpp \ test/fuzz/addrman.cpp \ test/fuzz/asmap.cpp \ @@ -253,6 +261,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/locale.cpp \ test/fuzz/merkleblock.cpp \ test/fuzz/message.cpp \ + test/fuzz/minisketch.cpp \ test/fuzz/muhash.cpp \ test/fuzz/multiplication_overflow.cpp \ test/fuzz/net.cpp \ diff --git a/src/addrman.cpp b/src/addrman.cpp index 825b5239c8..8a0433c40d 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -7,6 +7,8 @@ #include <addrman_impl.h> #include <hash.h> +#include <logging.h> +#include <logging/timer.h> #include <netaddress.h> #include <protocol.h> #include <random.h> @@ -387,7 +389,7 @@ void AddrManImpl::Unserialize(Stream& s_) LogPrint(BCLog::ADDRMAN, "addrman lost %i new and %i tried addresses due to collisions or invalid addresses\n", nLostUnk, nLost); } - const int check_code{ForceCheckAddrman()}; + const int check_code{CheckAddrman()}; if (check_code != 0) { throw std::ios_base::failure(strprintf( "Corrupt data. Consistency check failed with code %s", @@ -937,18 +939,19 @@ void AddrManImpl::Check() const if (m_consistency_check_ratio == 0) return; if (insecure_rand.randrange(m_consistency_check_ratio) >= 1) return; - const int err{ForceCheckAddrman()}; + const int err{CheckAddrman()}; if (err) { LogPrintf("ADDRMAN CONSISTENCY CHECK FAILED!!! err=%i\n", err); assert(false); } } -int AddrManImpl::ForceCheckAddrman() const +int AddrManImpl::CheckAddrman() const { AssertLockHeld(cs); - LogPrint(BCLog::ADDRMAN, "Addrman checks started: new %i, tried %i, total %u\n", nNew, nTried, vRandom.size()); + LOG_TIME_MILLIS_WITH_CATEGORY_MSG_ONCE( + strprintf("new %i, tried %i, total %u", nNew, nTried, vRandom.size()), BCLog::ADDRMAN); std::unordered_set<int> setTried; std::unordered_map<int, int> mapNew; @@ -1028,7 +1031,6 @@ int AddrManImpl::ForceCheckAddrman() const if (nKey.IsNull()) return -16; - LogPrint(BCLog::ADDRMAN, "Addrman checks completed successfully\n"); return 0; } diff --git a/src/addrman.h b/src/addrman.h index a9f697f66f..455d84ca71 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -53,6 +53,7 @@ static constexpr int32_t DEFAULT_ADDRMAN_CONSISTENCY_CHECKS{0}; */ class AddrMan { +protected: const std::unique_ptr<AddrManImpl> m_impl; public: @@ -135,9 +136,6 @@ public: void SetServices(const CService& addr, ServiceFlags nServices); const std::vector<bool>& GetAsmap() const; - - friend class AddrManTest; - friend class AddrManDeterministic; }; #endif // BITCOIN_ADDRMAN_H diff --git a/src/addrman_impl.h b/src/addrman_impl.h index e097932ee8..10a65871c1 100644 --- a/src/addrman_impl.h +++ b/src/addrman_impl.h @@ -6,6 +6,7 @@ #define BITCOIN_ADDRMAN_IMPL_H #include <logging.h> +#include <logging/timer.h> #include <netaddress.h> #include <protocol.h> #include <serialize.h> @@ -265,12 +266,13 @@ private: std::pair<CAddress, int64_t> SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs); - //! Consistency check, taking into account m_consistency_check_ratio. Will std::abort if an inconsistency is detected. + //! Consistency check, taking into account m_consistency_check_ratio. + //! Will std::abort if an inconsistency is detected. void Check() const EXCLUSIVE_LOCKS_REQUIRED(cs); //! Perform consistency check, regardless of m_consistency_check_ratio. //! @returns an error code or zero. - int ForceCheckAddrman() const EXCLUSIVE_LOCKS_REQUIRED(cs); + int CheckAddrman() const EXCLUSIVE_LOCKS_REQUIRED(cs); }; #endif // BITCOIN_ADDRMAN_IMPL_H diff --git a/src/banman.h b/src/banman.h index f495dab49d..6cb6304744 100644 --- a/src/banman.h +++ b/src/banman.h @@ -95,4 +95,4 @@ private: CRollingBloomFilter m_discouraged GUARDED_BY(m_cs_banned) {50000, 0.000001}; }; -#endif +#endif // BITCOIN_BANMAN_H diff --git a/src/base58.cpp b/src/base58.cpp index fb04673c5c..b122e0055e 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -149,7 +149,7 @@ std::string EncodeBase58Check(Span<const unsigned char> input) return false; } // re-calculate the checksum, ensure it matches the included 4-byte checksum - uint256 hash = Hash(MakeSpan(vchRet).first(vchRet.size() - 4)); + uint256 hash = Hash(Span{vchRet}.first(vchRet.size() - 4)); if (memcmp(&hash, &vchRet[vchRet.size() - 4], 4) != 0) { vchRet.clear(); return false; diff --git a/src/bech32.cpp b/src/bech32.cpp index 9da2488ef2..3cda1dfff5 100644 --- a/src/bech32.cpp +++ b/src/bech32.cpp @@ -1,11 +1,15 @@ // Copyright (c) 2017, 2021 Pieter Wuille +// Copyright (c) 2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bech32.h> #include <util/vector.h> +#include <array> #include <assert.h> +#include <numeric> +#include <optional> namespace bech32 { @@ -30,6 +34,90 @@ const int8_t CHARSET_REV[128] = { 1, 0, 3, 16, 11, 28, 12, 14, 6, 4, 2, -1, -1, -1, -1, -1 }; +/** We work with the finite field GF(1024) defined as a degree 2 extension of the base field GF(32) + * The defining polynomial of the extension is x^2 + 9x + 23. + * Let (e) be a root of this defining polynomial. Then (e) is a primitive element of GF(1024), + * that is, a generator of the field. Every non-zero element of the field can then be represented + * as (e)^k for some power k. + * The array GF1024_EXP contains all these powers of (e) - GF1024_EXP[k] = (e)^k in GF(1024). + * Conversely, GF1024_LOG contains the discrete logarithms of these powers, so + * GF1024_LOG[GF1024_EXP[k]] == k. + * The following function generates the two tables GF1024_EXP and GF1024_LOG as constexprs. */ +constexpr std::pair<std::array<int16_t, 1023>, std::array<int16_t, 1024>> GenerateGFTables() +{ + // Build table for GF(32). + // We use these tables to perform arithmetic in GF(32) below, when constructing the + // tables for GF(1024). + std::array<int8_t, 31> GF32_EXP{}; + std::array<int8_t, 32> GF32_LOG{}; + + // fmod encodes the defining polynomial of GF(32) over GF(2), x^5 + x^3 + 1. + // Because coefficients in GF(2) are binary digits, the coefficients are packed as 101001. + const int fmod = 41; + + // Elements of GF(32) are encoded as vectors of length 5 over GF(2), that is, + // 5 binary digits. Each element (b_4, b_3, b_2, b_1, b_0) encodes a polynomial + // b_4*x^4 + b_3*x^3 + b_2*x^2 + b_1*x^1 + b_0 (modulo fmod). + // For example, 00001 = 1 is the multiplicative identity. + GF32_EXP[0] = 1; + GF32_LOG[0] = -1; + GF32_LOG[1] = 0; + int v = 1; + for (int i = 1; i < 31; ++i) { + // Multiplication by x is the same as shifting left by 1, as + // every coefficient of the polynomial is moved up one place. + v = v << 1; + // If the polynomial now has an x^5 term, we subtract fmod from it + // to remain working modulo fmod. Subtraction is the same as XOR in characteristic + // 2 fields. + if (v & 32) v ^= fmod; + GF32_EXP[i] = v; + GF32_LOG[v] = i; + } + + // Build table for GF(1024) + std::array<int16_t, 1023> GF1024_EXP{}; + std::array<int16_t, 1024> GF1024_LOG{}; + + GF1024_EXP[0] = 1; + GF1024_LOG[0] = -1; + GF1024_LOG[1] = 0; + + // Each element v of GF(1024) is encoded as a 10 bit integer in the following way: + // v = v1 || v0 where v0, v1 are 5-bit integers (elements of GF(32)). + // The element (e) is encoded as 1 || 0, to represent 1*(e) + 0. Every other element + // a*(e) + b is represented as a || b (a and b are both GF(32) elements). Given (v), + // we compute (e)*(v) by multiplying in the following way: + // + // v0' = 23*v1 + // v1' = 9*v1 + v0 + // e*v = v1' || v0' + // + // Where 23, 9 are GF(32) elements encoded as described above. Multiplication in GF(32) + // is done using the log/exp tables: + // e^x * e^y = e^(x + y) so a * b = EXP[ LOG[a] + LOG [b] ] + // for non-zero a and b. + + v = 1; + for (int i = 1; i < 1023; ++i) { + int v0 = v & 31; + int v1 = v >> 5; + + int v0n = v1 ? GF32_EXP.at((GF32_LOG.at(v1) + GF32_LOG.at(23)) % 31) : 0; + int v1n = (v1 ? GF32_EXP.at((GF32_LOG.at(v1) + GF32_LOG.at(9)) % 31) : 0) ^ v0; + + v = v1n << 5 | v0n; + GF1024_EXP[i] = v; + GF1024_LOG[v] = i; + } + + return std::make_pair(GF1024_EXP, GF1024_LOG); +} + +constexpr auto tables = GenerateGFTables(); +constexpr const std::array<int16_t, 1023>& GF1024_EXP = tables.first; +constexpr const std::array<int16_t, 1024>& GF1024_LOG = tables.second; + /* Determine the final constant to use for the specified encoding. */ uint32_t EncodingConstant(Encoding encoding) { assert(encoding == Encoding::BECH32 || encoding == Encoding::BECH32M); @@ -127,12 +215,98 @@ uint32_t PolyMod(const data& v) return c; } +/** Syndrome computes the values s_j = R(e^j) for j in [997, 998, 999]. As described above, the + * generator polynomial G is the LCM of the minimal polynomials of (e)^997, (e)^998, and (e)^999. + * + * Consider a codeword with errors, of the form R(x) = C(x) + E(x). The residue is the bit-packed + * result of computing R(x) mod G(X), where G is the generator of the code. Because C(x) is a valid + * codeword, it is a multiple of G(X), so the residue is in fact just E(x) mod G(x). Note that all + * of the (e)^j are roots of G(x) by definition, so R((e)^j) = E((e)^j). + * + * Let R(x) = r1*x^5 + r2*x^4 + r3*x^3 + r4*x^2 + r5*x + r6 + * + * To compute R((e)^j), we are really computing: + * r1*(e)^(j*5) + r2*(e)^(j*4) + r3*(e)^(j*3) + r4*(e)^(j*2) + r5*(e)^j + r6 + * + * Now note that all of the (e)^(j*i) for i in [5..0] are constants and can be precomputed. + * But even more than that, we can consider each coefficient as a bit-string. + * For example, take r5 = (b_5, b_4, b_3, b_2, b_1) written out as 5 bits. Then: + * r5*(e)^j = b_1*(e)^j + b_2*(2*(e)^j) + b_3*(4*(e)^j) + b_4*(8*(e)^j) + b_5*(16*(e)^j) + * where all the (2^i*(e)^j) are constants and can be precomputed. + * + * Then we just add each of these corresponding constants to our final value based on the + * bit values b_i. This is exactly what is done in the Syndrome function below. + */ +constexpr std::array<uint32_t, 25> GenerateSyndromeConstants() { + std::array<uint32_t, 25> SYNDROME_CONSTS{}; + for (int k = 1; k < 6; ++k) { + for (int shift = 0; shift < 5; ++shift) { + int16_t b = GF1024_LOG.at(1 << shift); + int16_t c0 = GF1024_EXP.at((997*k + b) % 1023); + int16_t c1 = GF1024_EXP.at((998*k + b) % 1023); + int16_t c2 = GF1024_EXP.at((999*k + b) % 1023); + uint32_t c = c2 << 20 | c1 << 10 | c0; + int ind = 5*(k-1) + shift; + SYNDROME_CONSTS[ind] = c; + } + } + return SYNDROME_CONSTS; +} +constexpr std::array<uint32_t, 25> SYNDROME_CONSTS = GenerateSyndromeConstants(); + +/** + * Syndrome returns the three values s_997, s_998, and s_999 described above, + * packed into a 30-bit integer, where each group of 10 bits encodes one value. + */ +uint32_t Syndrome(const uint32_t residue) { + // low is the first 5 bits, corresponding to the r6 in the residue + // (the constant term of the polynomial). + uint32_t low = residue & 0x1f; + + // We begin by setting s_j = low = r6 for all three values of j, because these are unconditional. + uint32_t result = low ^ (low << 10) ^ (low << 20); + + // Then for each following bit, we add the corresponding precomputed constant if the bit is 1. + // For example, 0x31edd3c4 is 1100011110 1101110100 1111000100 when unpacked in groups of 10 + // bits, corresponding exactly to a^999 || a^998 || a^997 (matching the corresponding values in + // GF1024_EXP above). In this way, we compute all three values of s_j for j in (997, 998, 999) + // simultaneously. Recall that XOR corresponds to addition in a characteristic 2 field. + for (int i = 0; i < 25; ++i) { + result ^= ((residue >> (5+i)) & 1 ? SYNDROME_CONSTS.at(i) : 0); + } + return result; +} + /** Convert to lower case. */ inline unsigned char LowerCase(unsigned char c) { return (c >= 'A' && c <= 'Z') ? (c - 'A') + 'a' : c; } +/** Return indices of invalid characters in a Bech32 string. */ +bool CheckCharacters(const std::string& str, std::vector<int>& errors) { + bool lower = false, upper = false; + for (size_t i = 0; i < str.size(); ++i) { + unsigned char c = str[i]; + if (c >= 'a' && c <= 'z') { + if (upper) { + errors.push_back(i); + } else { + lower = true; + } + } else if (c >= 'A' && c <= 'Z') { + if (lower) { + errors.push_back(i); + } else { + upper = true; + } + } else if (c < 33 || c > 126) { + errors.push_back(i); + } + } + return errors.empty(); +} + /** Expand a HRP for use in checksum computation. */ data ExpandHRP(const std::string& hrp) { @@ -196,14 +370,8 @@ std::string Encode(Encoding encoding, const std::string& hrp, const data& values /** Decode a Bech32 or Bech32m string. */ DecodeResult Decode(const std::string& str) { - bool lower = false, upper = false; - for (size_t i = 0; i < str.size(); ++i) { - unsigned char c = str[i]; - if (c >= 'a' && c <= 'z') lower = true; - else if (c >= 'A' && c <= 'Z') upper = true; - else if (c < 33 || c > 126) return {}; - } - if (lower && upper) return {}; + std::vector<int> errors; + if (!CheckCharacters(str, errors)) return {}; size_t pos = str.rfind('1'); if (str.size() > 90 || pos == str.npos || pos == 0 || pos + 7 > str.size()) { return {}; @@ -227,4 +395,174 @@ DecodeResult Decode(const std::string& str) { return {result, std::move(hrp), data(values.begin(), values.end() - 6)}; } +/** Find index of an incorrect character in a Bech32 string. */ +std::pair<std::string, std::vector<int>> LocateErrors(const std::string& str) { + std::vector<int> error_locations{}; + + if (str.size() > 90) { + error_locations.resize(str.size() - 90); + std::iota(error_locations.begin(), error_locations.end(), 90); + return std::make_pair("Bech32 string too long", std::move(error_locations)); + } + + if (!CheckCharacters(str, error_locations)){ + return std::make_pair("Invalid character or mixed case", std::move(error_locations)); + } + + size_t pos = str.rfind('1'); + if (pos == str.npos) { + return std::make_pair("Missing separator", std::vector<int>{}); + } + if (pos == 0 || pos + 7 > str.size()) { + error_locations.push_back(pos); + return std::make_pair("Invalid separator position", std::move(error_locations)); + } + + std::string hrp; + for (size_t i = 0; i < pos; ++i) { + hrp += LowerCase(str[i]); + } + + size_t length = str.size() - 1 - pos; // length of data part + data values(length); + for (size_t i = pos + 1; i < str.size(); ++i) { + unsigned char c = str[i]; + int8_t rev = CHARSET_REV[c]; + if (rev == -1) { + error_locations.push_back(i); + return std::make_pair("Invalid Base 32 character", std::move(error_locations)); + } + values[i - pos - 1] = rev; + } + + // We attempt error detection with both bech32 and bech32m, and choose the one with the fewest errors + // We can't simply use the segwit version, because that may be one of the errors + std::optional<Encoding> error_encoding; + for (Encoding encoding : {Encoding::BECH32, Encoding::BECH32M}) { + std::vector<int> possible_errors; + // Recall that (ExpandHRP(hrp) ++ values) is interpreted as a list of coefficients of a polynomial + // over GF(32). PolyMod computes the "remainder" of this polynomial modulo the generator G(x). + uint32_t residue = PolyMod(Cat(ExpandHRP(hrp), values)) ^ EncodingConstant(encoding); + + // All valid codewords should be multiples of G(x), so this remainder (after XORing with the encoding + // constant) should be 0 - hence 0 indicates there are no errors present. + if (residue != 0) { + // If errors are present, our polynomial must be of the form C(x) + E(x) where C is the valid + // codeword (a multiple of G(x)), and E encodes the errors. + uint32_t syn = Syndrome(residue); + + // Unpack the three 10-bit syndrome values + int s0 = syn & 0x3FF; + int s1 = (syn >> 10) & 0x3FF; + int s2 = syn >> 20; + + // Get the discrete logs of these values in GF1024 for more efficient computation + int l_s0 = GF1024_LOG.at(s0); + int l_s1 = GF1024_LOG.at(s1); + int l_s2 = GF1024_LOG.at(s2); + + // First, suppose there is only a single error. Then E(x) = e1*x^p1 for some position p1 + // Then s0 = E((e)^997) = e1*(e)^(997*p1) and s1 = E((e)^998) = e1*(e)^(998*p1) + // Therefore s1/s0 = (e)^p1, and by the same logic, s2/s1 = (e)^p1 too. + // Hence, s1^2 == s0*s2, which is exactly the condition we check first: + if (l_s0 != -1 && l_s1 != -1 && l_s2 != -1 && (2 * l_s1 - l_s2 - l_s0 + 2046) % 1023 == 0) { + // Compute the error position p1 as l_s1 - l_s0 = p1 (mod 1023) + size_t p1 = (l_s1 - l_s0 + 1023) % 1023; // the +1023 ensures it is positive + // Now because s0 = e1*(e)^(997*p1), we get e1 = s0/((e)^(997*p1)). Remember that (e)^1023 = 1, + // so 1/((e)^997) = (e)^(1023-997). + int l_e1 = l_s0 + (1023 - 997) * p1; + // Finally, some sanity checks on the result: + // - The error position should be within the length of the data + // - e1 should be in GF(32), which implies that e1 = (e)^(33k) for some k (the 31 non-zero elements + // of GF(32) form an index 33 subgroup of the 1023 non-zero elements of GF(1024)). + if (p1 < length && !(l_e1 % 33)) { + // Polynomials run from highest power to lowest, so the index p1 is from the right. + // We don't return e1 because it is dangerous to suggest corrections to the user, + // the user should check the address themselves. + possible_errors.push_back(str.size() - p1 - 1); + } + // Otherwise, suppose there are two errors. Then E(x) = e1*x^p1 + e2*x^p2. + } else { + // For all possible first error positions p1 + for (size_t p1 = 0; p1 < length; ++p1) { + // We have guessed p1, and want to solve for p2. Recall that E(x) = e1*x^p1 + e2*x^p2, so + // s0 = E((e)^997) = e1*(e)^(997^p1) + e2*(e)^(997*p2), and similar for s1 and s2. + // + // Consider s2 + s1*(e)^p1 + // = 2e1*(e)^(999^p1) + e2*(e)^(999*p2) + e2*(e)^(998*p2)*(e)^p1 + // = e2*(e)^(999*p2) + e2*(e)^(998*p2)*(e)^p1 + // (Because we are working in characteristic 2.) + // = e2*(e)^(998*p2) ((e)^p2 + (e)^p1) + // + int s2_s1p1 = s2 ^ (s1 == 0 ? 0 : GF1024_EXP.at((l_s1 + p1) % 1023)); + if (s2_s1p1 == 0) continue; + int l_s2_s1p1 = GF1024_LOG.at(s2_s1p1); + + // Similarly, s1 + s0*(e)^p1 + // = e2*(e)^(997*p2) ((e)^p2 + (e)^p1) + int s1_s0p1 = s1 ^ (s0 == 0 ? 0 : GF1024_EXP.at((l_s0 + p1) % 1023)); + if (s1_s0p1 == 0) continue; + int l_s1_s0p1 = GF1024_LOG.at(s1_s0p1); + + // So, putting these together, we can compute the second error position as + // (e)^p2 = (s2 + s1^p1)/(s1 + s0^p1) + // p2 = log((e)^p2) + size_t p2 = (l_s2_s1p1 - l_s1_s0p1 + 1023) % 1023; + + // Sanity checks that p2 is a valid position and not the same as p1 + if (p2 >= length || p1 == p2) continue; + + // Now we want to compute the error values e1 and e2. + // Similar to above, we compute s1 + s0*(e)^p2 + // = e1*(e)^(997*p1) ((e)^p1 + (e)^p2) + int s1_s0p2 = s1 ^ (s0 == 0 ? 0 : GF1024_EXP.at((l_s0 + p2) % 1023)); + if (s1_s0p2 == 0) continue; + int l_s1_s0p2 = GF1024_LOG.at(s1_s0p2); + + // And compute (the log of) 1/((e)^p1 + (e)^p2)) + int inv_p1_p2 = 1023 - GF1024_LOG.at(GF1024_EXP.at(p1) ^ GF1024_EXP.at(p2)); + + // Then (s1 + s0*(e)^p1) * (1/((e)^p1 + (e)^p2))) + // = e2*(e)^(997*p2) + // Then recover e2 by dividing by (e)^(997*p2) + int l_e2 = l_s1_s0p1 + inv_p1_p2 + (1023 - 997) * p2; + // Check that e2 is in GF(32) + if (l_e2 % 33) continue; + + // In the same way, (s1 + s0*(e)^p2) * (1/((e)^p1 + (e)^p2))) + // = e1*(e)^(997*p1) + // So recover e1 by dividing by (e)^(997*p1) + int l_e1 = l_s1_s0p2 + inv_p1_p2 + (1023 - 997) * p1; + // Check that e1 is in GF(32) + if (l_e1 % 33) continue; + + // Again, we do not return e1 or e2 for safety. + // Order the error positions from the left of the string and return them + if (p1 > p2) { + possible_errors.push_back(str.size() - p1 - 1); + possible_errors.push_back(str.size() - p2 - 1); + } else { + possible_errors.push_back(str.size() - p2 - 1); + possible_errors.push_back(str.size() - p1 - 1); + } + break; + } + } + } else { + // No errors + return std::make_pair("", std::vector<int>{}); + } + + if (error_locations.empty() || (!possible_errors.empty() && possible_errors.size() < error_locations.size())) { + error_locations = std::move(possible_errors); + if (!error_locations.empty()) error_encoding = encoding; + } + } + std::string error_message = error_encoding == Encoding::BECH32M ? "Invalid Bech32m checksum" + : error_encoding == Encoding::BECH32 ? "Invalid Bech32 checksum" + : "Invalid checksum"; + + return std::make_pair(error_message, std::move(error_locations)); +} + } // namespace bech32 diff --git a/src/bech32.h b/src/bech32.h index e9450ccc2b..5e89e6efda 100644 --- a/src/bech32.h +++ b/src/bech32.h @@ -1,4 +1,5 @@ // Copyright (c) 2017, 2021 Pieter Wuille +// Copyright (c) 2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -44,6 +45,9 @@ struct DecodeResult /** Decode a Bech32 or Bech32m string. */ DecodeResult Decode(const std::string& str); +/** Return the positions of errors in a Bech32 string. */ +std::pair<std::string, std::vector<int>> LocateErrors(const std::string& str); + } // namespace bech32 #endif // BITCOIN_BECH32_H diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index b4b33d115f..0577ab80e3 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -34,10 +34,10 @@ static void AssembleBlock(benchmark::Bench& bench) txs.at(b) = MakeTransactionRef(tx); } { - LOCK(::cs_main); // Required for ::AcceptToMemoryPool. + LOCK(::cs_main); for (const auto& txr : txs) { - const MempoolAcceptResult res = ::AcceptToMemoryPool(test_setup->m_node.chainman->ActiveChainstate(), *test_setup->m_node.mempool, txr, false /* bypass_limits */); + const MempoolAcceptResult res = test_setup->m_node.chainman->ProcessTransaction(txr); assert(res.m_result_type == MempoolAcceptResult::ResultType::VALID); } } diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index d5275b0b76..dae3a47cd7 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -45,7 +45,7 @@ static void CCoinsCaching(benchmark::Bench& bench) // Benchmark. const CTransaction tx_1(t1); bench.run([&] { - bool success = AreInputsStandard(tx_1, coins, false); + bool success{AreInputsStandard(tx_1, coins)}; assert(success); }); ECC_Stop(); diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index fd5145950b..3c24fee60f 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -18,7 +18,7 @@ static void addCoin(const CAmount& nValue, const CWallet& wallet, std::vector<st tx.nLockTime = nextLockTime++; // so all transactions get different hashes tx.vout.resize(1); tx.vout[0].nValue = nValue; - wtxs.push_back(std::make_unique<CWalletTx>(MakeTransactionRef(std::move(tx)))); + wtxs.push_back(std::make_unique<CWalletTx>(MakeTransactionRef(std::move(tx)), TxStateInactive{})); } // Simple benchmark for wallet coin selection. Note that it maybe be necessary @@ -32,7 +32,7 @@ static void CoinSelection(benchmark::Bench& bench) { NodeContext node; auto chain = interfaces::MakeChain(node); - CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(chain.get(), "", gArgs, CreateDummyWalletDatabase()); std::vector<std::unique_ptr<CWalletTx>> wtxs; LOCK(wallet.cs_wallet); diff --git a/src/bench/rpc_mempool.cpp b/src/bench/rpc_mempool.cpp index f1eeef8885..67c827d0d3 100644 --- a/src/bench/rpc_mempool.cpp +++ b/src/bench/rpc_mempool.cpp @@ -12,7 +12,7 @@ static void AddTx(const CTransactionRef& tx, const CAmount& fee, CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs) { LockPoints lp; - pool.addUnchecked(CTxMemPoolEntry(tx, fee, /* time */ 0, /* height */ 1, /* spendsCoinbase */ false, /* sigOpCost */ 4, lp)); + pool.addUnchecked(CTxMemPoolEntry(tx, fee, /*time=*/0, /*entry_height=*/1, /*spends_coinbase=*/false, /*sigops_cost=*/4, lp)); } static void RpcMempool(benchmark::Bench& bench) diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index 166ed16042..8e3ca59496 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -20,7 +20,7 @@ static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const b const auto& ADDRESS_WATCHONLY = ADDRESS_BCRT1_UNSPENDABLE; - CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockWalletDatabase()}; + CWallet wallet{test_setup->m_node.chain.get(), "", gArgs, CreateMockWalletDatabase()}; { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 338d32ef6b..279521a761 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -49,6 +49,7 @@ static constexpr int DEFAULT_WAIT_CLIENT_TIMEOUT = 0; static const bool DEFAULT_NAMED=false; static const int CONTINUE_EXECUTION=-1; static constexpr int8_t UNKNOWN_NETWORK{-1}; +static constexpr std::array NETWORKS{"ipv4", "ipv6", "onion", "i2p", "cjdns"}; /** Default number of blocks to generate for RPC generatetoaddress. */ static const std::string DEFAULT_NBLOCKS = "1"; @@ -242,11 +243,10 @@ public: class AddrinfoRequestHandler : public BaseRequestHandler { private: - static constexpr std::array m_networks{"ipv4", "ipv6", "onion", "i2p"}; int8_t NetworkStringToId(const std::string& str) const { - for (size_t i = 0; i < m_networks.size(); ++i) { - if (str == m_networks.at(i)) return i; + for (size_t i = 0; i < NETWORKS.size(); ++i) { + if (str == NETWORKS[i]) return i; } return UNKNOWN_NETWORK; } @@ -269,7 +269,7 @@ public: throw std::runtime_error("-addrinfo requires bitcoind server to be running v22.0 and up"); } // Count the number of peers known to our node, by network. - std::array<uint64_t, m_networks.size()> counts{{}}; + std::array<uint64_t, NETWORKS.size()> counts{{}}; for (const UniValue& node : nodes) { std::string network_name{node["network"].get_str()}; const int8_t network_id{NetworkStringToId(network_name)}; @@ -279,8 +279,8 @@ public: // Prepare result to return to user. UniValue result{UniValue::VOBJ}, addresses{UniValue::VOBJ}; uint64_t total{0}; // Total address count - for (size_t i = 0; i < m_networks.size(); ++i) { - addresses.pushKV(m_networks.at(i), counts.at(i)); + for (size_t i = 0; i < NETWORKS.size(); ++i) { + addresses.pushKV(NETWORKS[i], counts.at(i)); total += counts.at(i); } addresses.pushKV("total", total); @@ -363,14 +363,13 @@ class NetinfoRequestHandler : public BaseRequestHandler { private: static constexpr uint8_t MAX_DETAIL_LEVEL{4}; - static constexpr std::array m_networks{"ipv4", "ipv6", "onion", "i2p"}; - std::array<std::array<uint16_t, m_networks.size() + 1>, 3> m_counts{{{}}}; //!< Peer counts by (in/out/total, networks/total) + std::array<std::array<uint16_t, NETWORKS.size() + 1>, 3> m_counts{{{}}}; //!< Peer counts by (in/out/total, networks/total) uint8_t m_block_relay_peers_count{0}; uint8_t m_manual_peers_count{0}; int8_t NetworkStringToId(const std::string& str) const { - for (size_t i = 0; i < m_networks.size(); ++i) { - if (str == m_networks.at(i)) return i; + for (size_t i = 0; i < NETWORKS.size(); ++i) { + if (str == NETWORKS[i]) return i; } return UNKNOWN_NETWORK; } @@ -471,10 +470,10 @@ public: const bool is_outbound{!peer["inbound"].get_bool()}; const bool is_block_relay{!peer["relaytxes"].get_bool()}; const std::string conn_type{peer["connection_type"].get_str()}; - ++m_counts.at(is_outbound).at(network_id); // in/out by network - ++m_counts.at(is_outbound).at(m_networks.size()); // in/out overall - ++m_counts.at(2).at(network_id); // total by network - ++m_counts.at(2).at(m_networks.size()); // total overall + ++m_counts.at(is_outbound).at(network_id); // in/out by network + ++m_counts.at(is_outbound).at(NETWORKS.size()); // in/out overall + ++m_counts.at(2).at(network_id); // total by network + ++m_counts.at(2).at(NETWORKS.size()); // total overall if (conn_type == "block-relay-only") ++m_block_relay_peers_count; if (conn_type == "manual") ++m_manual_peers_count; if (DetailsRequested()) { @@ -571,7 +570,7 @@ public: for (int8_t n : reachable_networks) { result += strprintf("%8i", m_counts.at(i).at(n)); // network peers count } - result += strprintf(" %5i", m_counts.at(i).at(m_networks.size())); // total peers count + result += strprintf(" %5i", m_counts.at(i).at(NETWORKS.size())); // total peers count if (i == 1) { // the outbound row has two extra columns for block relay and manual peer counts result += strprintf(" %5i", m_block_relay_peers_count); if (m_manual_peers_count) result += strprintf(" %5i", m_manual_peers_count); diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index 41fa0b6fa0..69d4eacd87 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -81,7 +81,7 @@ GCSFilter::GCSFilter(const Params& params) GCSFilter::GCSFilter(const Params& params, std::vector<unsigned char> encoded_filter) : m_params(params), m_encoded(std::move(encoded_filter)) { - VectorReader stream(GCS_SER_TYPE, GCS_SER_VERSION, m_encoded, 0); + SpanReader stream{GCS_SER_TYPE, GCS_SER_VERSION, m_encoded}; uint64_t N = ReadCompactSize(stream); m_N = static_cast<uint32_t>(N); @@ -92,7 +92,7 @@ GCSFilter::GCSFilter(const Params& params, std::vector<unsigned char> encoded_fi // Verify that the encoded filter contains exactly N elements. If it has too much or too little // data, a std::ios_base::failure exception will be raised. - BitStreamReader<VectorReader> bitreader(stream); + BitStreamReader<SpanReader> bitreader{stream}; for (uint64_t i = 0; i < m_N; ++i) { GolombRiceDecode(bitreader, m_params.m_P); } @@ -133,13 +133,13 @@ GCSFilter::GCSFilter(const Params& params, const ElementSet& elements) bool GCSFilter::MatchInternal(const uint64_t* element_hashes, size_t size) const { - VectorReader stream(GCS_SER_TYPE, GCS_SER_VERSION, m_encoded, 0); + SpanReader stream{GCS_SER_TYPE, GCS_SER_VERSION, m_encoded}; // Seek forward by size of N uint64_t N = ReadCompactSize(stream); assert(N == m_N); - BitStreamReader<VectorReader> bitreader(stream); + BitStreamReader<SpanReader> bitreader{stream}; uint64_t value = 0; size_t hashes_index = 0; diff --git a/src/chain.cpp b/src/chain.cpp index c09113a866..5d182e1af8 100644 --- a/src/chain.cpp +++ b/src/chain.cpp @@ -5,9 +5,6 @@ #include <chain.h> -/** - * CChain implementation - */ void CChain::SetTip(CBlockIndex *pindex) { if (pindex == nullptr) { vChain.clear(); diff --git a/src/checkqueue.h b/src/checkqueue.h index 7c20e2013c..760dfbddc1 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -167,16 +167,24 @@ public: //! Add a batch of checks to the queue void Add(std::vector<T>& vChecks) { - LOCK(m_mutex); - for (T& check : vChecks) { - queue.push_back(T()); - check.swap(queue.back()); + if (vChecks.empty()) { + return; } - nTodo += vChecks.size(); - if (vChecks.size() == 1) + + { + LOCK(m_mutex); + for (T& check : vChecks) { + queue.emplace_back(); + check.swap(queue.back()); + } + nTodo += vChecks.size(); + } + + if (vChecks.size() == 1) { m_worker_cv.notify_one(); - else if (vChecks.size() > 1) + } else { m_worker_cv.notify_all(); + } } //! Stop all of the worker threads. diff --git a/src/coins.cpp b/src/coins.cpp index ce0b131de6..daead6055e 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -7,6 +7,7 @@ #include <consensus/consensus.h> #include <logging.h> #include <random.h> +#include <util/trace.h> #include <version.h> bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; } @@ -95,6 +96,12 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi it->second.coin = std::move(coin); it->second.flags |= CCoinsCacheEntry::DIRTY | (fresh ? CCoinsCacheEntry::FRESH : 0); cachedCoinsUsage += it->second.coin.DynamicMemoryUsage(); + TRACE5(utxocache, add, + outpoint.hash.data(), + (uint32_t)outpoint.n, + (uint32_t)coin.nHeight, + (int64_t)coin.out.nValue, + (bool)coin.IsCoinBase()); } void CCoinsViewCache::EmplaceCoinInternalDANGER(COutPoint&& outpoint, Coin&& coin) { @@ -120,6 +127,12 @@ bool CCoinsViewCache::SpendCoin(const COutPoint &outpoint, Coin* moveout) { CCoinsMap::iterator it = FetchCoin(outpoint); if (it == cacheCoins.end()) return false; cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); + TRACE5(utxocache, spent, + outpoint.hash.data(), + (uint32_t)outpoint.n, + (uint32_t)it->second.coin.nHeight, + (int64_t)it->second.coin.out.nValue, + (bool)it->second.coin.IsCoinBase()); if (moveout) { *moveout = std::move(it->second.coin); } @@ -231,6 +244,12 @@ void CCoinsViewCache::Uncache(const COutPoint& hash) CCoinsMap::iterator it = cacheCoins.find(hash); if (it != cacheCoins.end() && it->second.flags == 0) { cachedCoinsUsage -= it->second.coin.DynamicMemoryUsage(); + TRACE5(utxocache, uncache, + hash.hash.data(), + (uint32_t)hash.n, + (uint32_t)it->second.coin.nHeight, + (int64_t)it->second.coin.out.nValue, + (bool)it->second.coin.IsCoinBase()); cacheCoins.erase(it); } } diff --git a/src/compressor.h b/src/compressor.h index 40b2496f06..00b777a111 100644 --- a/src/compressor.h +++ b/src/compressor.h @@ -65,12 +65,12 @@ struct ScriptCompression void Ser(Stream &s, const CScript& script) { CompressedScript compr; if (CompressScript(script, compr)) { - s << MakeSpan(compr); + s << Span{compr}; return; } unsigned int nSize = script.size() + nSpecialScripts; s << VARINT(nSize); - s << MakeSpan(script); + s << Span{script}; } template<typename Stream> @@ -79,7 +79,7 @@ struct ScriptCompression s >> VARINT(nSize); if (nSize < nSpecialScripts) { CompressedScript vch(GetSpecialScriptSize(nSize), 0x00); - s >> MakeSpan(vch); + s >> Span{vch}; DecompressScript(script, nSize, vch); return; } @@ -90,7 +90,7 @@ struct ScriptCompression s.ignore(nSize); } else { script.resize(nSize); - s >> MakeSpan(script); + s >> Span{script}; } } }; diff --git a/src/consensus/amount.h b/src/consensus/amount.h index 96566ea13f..59b8e3417a 100644 --- a/src/consensus/amount.h +++ b/src/consensus/amount.h @@ -26,4 +26,4 @@ static constexpr CAmount COIN = 100000000; static constexpr CAmount MAX_MONEY = 21000000 * COIN; inline bool MoneyRange(const CAmount& nValue) { return (nValue >= 0 && nValue <= MAX_MONEY); } -#endif // BITCOIN_CONSENSUS_AMOUNT_H +#endif // BITCOIN_CONSENSUS_AMOUNT_H diff --git a/src/consensus/validation.h b/src/consensus/validation.h index c4d305434a..05416d0aca 100644 --- a/src/consensus/validation.h +++ b/src/consensus/validation.h @@ -53,6 +53,7 @@ enum class TxValidationResult { */ TX_CONFLICT, TX_MEMPOOL_POLICY, //!< violated mempool's fee/size/descendant/RBF/etc limits + TX_NO_MEMPOOL, //!< this node does not have a mempool so can't validate the transaction }; /** A "reason" why a block was invalid, suitable for determining whether the diff --git a/src/cuckoocache.h b/src/cuckoocache.h index 1166466771..15cb55c3ce 100644 --- a/src/cuckoocache.h +++ b/src/cuckoocache.h @@ -89,7 +89,7 @@ public: */ inline void bit_set(uint32_t s) { - mem[s >> 3].fetch_or(1 << (s & 7), std::memory_order_relaxed); + mem[s >> 3].fetch_or(uint8_t(1 << (s & 7)), std::memory_order_relaxed); } /** bit_unset marks an entry as something that should not be overwritten. @@ -100,7 +100,7 @@ public: */ inline void bit_unset(uint32_t s) { - mem[s >> 3].fetch_and(~(1 << (s & 7)), std::memory_order_relaxed); + mem[s >> 3].fetch_and(uint8_t(~(1 << (s & 7))), std::memory_order_relaxed); } /** bit_is_set queries the table for discardability at `s`. diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 2fdc54464a..dbae2c45f2 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -136,6 +136,10 @@ CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bo TryCreateDirectories(path); LogPrintf("Opening LevelDB in %s\n", fs::PathToString(path)); } + // PathToString() return value is safe to pass to leveldb open function, + // because on POSIX leveldb passes the byte string directly to ::open(), and + // on Windows it converts from UTF-8 to UTF-16 before calling ::CreateFileW + // (see env_posix.cc and env_windows.cc). leveldb::Status status = leveldb::DB::Open(options, fs::PathToString(path), &pdb); dbwrapper_private::HandleError(status); LogPrintf("Opened LevelDB successfully\n"); @@ -94,31 +94,34 @@ static inline path operator+(path p1, path p2) /** * Convert path object to byte string. On POSIX, paths natively are byte - * strings so this is trivial. On Windows, paths natively are Unicode, so an - * encoding step is necessary. + * strings, so this is trivial. On Windows, paths natively are Unicode, so an + * encoding step is necessary. The inverse of \ref PathToString is \ref + * PathFromString. The strings returned and parsed by these functions can be + * used to call POSIX APIs, and for roundtrip conversion, logging, and + * debugging. * - * The inverse of \ref PathToString is \ref PathFromString. The strings - * returned and parsed by these functions can be used to call POSIX APIs, and - * for roundtrip conversion, logging, and debugging. But they are not - * guaranteed to be valid UTF-8, and are generally meant to be used internally, - * not externally. When communicating with external programs and libraries that - * require UTF-8, fs::path::u8string() and fs::u8path() methods can be used. - * For other applications, if support for non UTF-8 paths is required, or if - * higher-level JSON or XML or URI or C-style escapes are preferred, it may be - * also be appropriate to use different path encoding functions. - * - * Implementation note: On Windows, the std::filesystem::path(string) - * constructor and std::filesystem::path::string() method are not safe to use - * here, because these methods encode the path using C++'s narrow multibyte - * encoding, which on Windows corresponds to the current "code page", which is - * unpredictable and typically not able to represent all valid paths. So - * std::filesystem::path::u8string() and std::filesystem::u8path() functions - * are used instead on Windows. On POSIX, u8string/u8path functions are not - * safe to use because paths are not always valid UTF-8, so plain string - * methods which do not transform the path there are used. + * Because \ref PathToString and \ref PathFromString functions don't specify an + * encoding, they are meant to be used internally, not externally. They are not + * appropriate to use in applications requiring UTF-8, where + * fs::path::u8string() and fs::u8path() methods should be used instead. Other + * applications could require still different encodings. For example, JSON, XML, + * or URI applications might prefer to use higher level escapes (\uXXXX or + * &XXXX; or %XX) instead of multibyte encoding. Rust, Python, Java applications + * may require encoding paths with their respective UTF-8 derivatives WTF-8, + * PEP-383, and CESU-8 (see https://en.wikipedia.org/wiki/UTF-8#Derivatives). */ static inline std::string PathToString(const path& path) { + // Implementation note: On Windows, the std::filesystem::path(string) + // constructor and std::filesystem::path::string() method are not safe to + // use here, because these methods encode the path using C++'s narrow + // multibyte encoding, which on Windows corresponds to the current "code + // page", which is unpredictable and typically not able to represent all + // valid paths. So std::filesystem::path::u8string() and + // std::filesystem::u8path() functions are used instead on Windows. On + // POSIX, u8string/u8path functions are not safe to use because paths are + // not always valid UTF-8, so plain string methods which do not transform + // the path there are used. #ifdef WIN32 return path.u8string(); #else diff --git a/src/httprpc.h b/src/httprpc.h index 5a3b990646..6daf7d28f5 100644 --- a/src/httprpc.h +++ b/src/httprpc.h @@ -31,4 +31,4 @@ void InterruptREST(); */ void StopREST(); -#endif +#endif // BITCOIN_HTTPRPC_H @@ -267,4 +267,4 @@ private: } // namespace sam } // namespace i2p -#endif /* BITCOIN_I2P_H */ +#endif // BITCOIN_I2P_H diff --git a/src/init.cpp b/src/init.cpp index e554a18fbf..22c0d928b5 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -29,13 +29,13 @@ #include <interfaces/init.h> #include <interfaces/node.h> #include <mapport.h> -#include <miner.h> #include <net.h> #include <net_permissions.h> #include <net_processing.h> #include <netbase.h> #include <node/blockstorage.h> #include <node/context.h> +#include <node/miner.h> #include <node/ui_interface.h> #include <policy/feerate.h> #include <policy/fees.h> @@ -59,6 +59,7 @@ #include <util/asmap.h> #include <util/check.h> #include <util/moneystr.h> +#include <util/strencodings.h> #include <util/string.h> #include <util/syscall_sandbox.h> #include <util/system.h> @@ -422,6 +423,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-asmap=<file>", strprintf("Specify asn mapping used for bucketing of the peers (default: %s). Relative paths will be prefixed by the net-specific datadir location.", DEFAULT_ASMAP_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-bantime=<n>", strprintf("Default duration (in seconds) of manually configured bans (default: %u)", DEFAULT_MISBEHAVING_BANTIME), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-bind=<addr>[:<port>][=onion]", strprintf("Bind to given address and always listen on it (default: 0.0.0.0). Use [host]:port notation for IPv6. Append =onion to tag any incoming connections to that address and port as incoming Tor connections (default: 127.0.0.1:%u=onion, testnet: 127.0.0.1:%u=onion, signet: 127.0.0.1:%u=onion, regtest: 127.0.0.1:%u=onion)", defaultBaseParams->OnionServiceTargetPort(), testnetBaseParams->OnionServiceTargetPort(), signetBaseParams->OnionServiceTargetPort(), regtestBaseParams->OnionServiceTargetPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); + argsman.AddArg("-cjdnsreachable", "If set then this host is configured for CJDNS (connecting to fc00::/8 addresses would lead us to the CJDNS network) (default: 0)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-connect=<ip>", "Connect only to the specified node; -noconnect disables automatic connections (the rules for this peer are the same as for -addnode). This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); argsman.AddArg("-discover", "Discover own IP addresses (default: 1 when listening and no -externalip or -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-dns", strprintf("Allow DNS lookups for -addnode, -seednode and -connect (default: %u)", DEFAULT_NAME_LOOKUP), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -435,7 +437,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-maxreceivebuffer=<n>", strprintf("Maximum per-connection receive buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXRECEIVEBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxsendbuffer=<n>", strprintf("Maximum per-connection send buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXSENDBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by peers forward or backward by this amount. (default: %u seconds)", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-maxuploadtarget=<n>", strprintf("Tries to keep outbound traffic under the given target (in MiB per 24h). Limit does not apply to peers with 'download' permission. 0 = no limit (default: %d)", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-maxuploadtarget=<n>", strprintf("Tries to keep outbound traffic under the given target per 24h. Limit does not apply to peers with 'download' permission or blocks created within past week. 0 = no limit (default: %s). Optional suffix units [k|K|m|M|g|G|t|T] (default: M). Lowercase is 1000 base while uppercase is 1024 base", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-onion=<ip:port>", "Use separate SOCKS5 proxy to reach peers via Tor onion services, set -noonion to disable (default: -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-i2psam=<ip:port>", "I2P SAM proxy to reach I2P peers and accept I2P connections (default: none)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-i2pacceptincoming", "If set and -i2psam is also set then incoming I2P connections are accepted via the SAM proxy. If this is not set but -i2psam is set then only outgoing connections will be made to the I2P network. Ignored if -i2psam is not set. Listening for incoming I2P connections is done through the SAM proxy, not by binding to a local address and port (default: 1)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -1101,11 +1103,6 @@ bool AppInitLockDataDirectory() bool AppInitInterfaces(NodeContext& node) { node.chain = node.init->makeChain(); - // Create client interfaces for wallets that are supposed to be loaded - // according to -wallet and -disablewallet options. This only constructs - // the interfaces, it doesn't load wallet data. Wallets actually get loaded - // when load() and start() interface methods are called below. - g_wallet_init_interface.Construct(node); return true; } @@ -1113,6 +1110,12 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) { const ArgsManager& args = *Assert(node.args); const CChainParams& chainparams = Params(); + + auto opt_max_upload = ParseByteUnits(args.GetArg("-maxuploadtarget", DEFAULT_MAX_UPLOAD_TARGET), ByteUnit::M); + if (!opt_max_upload) { + return InitError(strprintf(_("Unable to parse -maxuploadtarget: '%s' (possible integer overflow?)"), args.GetArg("-maxuploadtarget", ""))); + } + // ********************************************************* Step 4a: application initialization if (!CreatePidFile(args)) { // Detailed error printed inside CreatePidFile(). @@ -1169,6 +1172,13 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) GetMainSignals().RegisterBackgroundSignalScheduler(*node.scheduler); + // Create client interfaces for wallets that are supposed to be loaded + // according to -wallet and -disablewallet options. This only constructs + // the interfaces, it doesn't load wallet data. Wallets actually get loaded + // when load() and start() interface methods are called below. + g_wallet_init_interface.Construct(node); + uiInterface.InitWallet(); + /* Register RPC commands regardless of -server setting so they will be * available in the GUI RPC console even if external calls are disabled. */ @@ -1294,6 +1304,14 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } } + if (!args.IsArgSet("-cjdnsreachable")) { + SetReachable(NET_CJDNS, false); + } + // Now IsReachable(NET_CJDNS) is true if: + // 1. -cjdnsreachable is given and + // 2.1. -onlynet is not given or + // 2.2. -onlynet=cjdns is given + // Check for host lookup allowed before parsing any network related parameters fNameLookup = args.GetBoolArg("-dns", DEFAULT_NAME_LOOKUP); @@ -1315,6 +1333,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) SetProxy(NET_IPV4, addrProxy); SetProxy(NET_IPV6, addrProxy); SetProxy(NET_ONION, addrProxy); + SetProxy(NET_CJDNS, addrProxy); SetNameProxy(addrProxy); SetReachable(NET_ONION, true); // by default, -proxy sets onion as reachable, unless -noonion later } @@ -1542,7 +1561,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) const CBlockIndex* tip = chainstate->m_chain.Tip(); RPCNotifyBlockChange(tip); - if (tip && tip->nTime > GetAdjustedTime() + 2 * 60 * 60) { + if (tip && tip->nTime > GetTime() + MAX_FUTURE_BLOCK_TIME) { strLoadError = _("The block database contains a block which appears to be from the future. " "This may be due to your computer's date and time being set incorrectly. " "Only rebuild the block database if you are sure that your computer's date and time are correct"); @@ -1748,8 +1767,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) connOptions.nSendBufferMaxSize = 1000 * args.GetIntArg("-maxsendbuffer", DEFAULT_MAXSENDBUFFER); connOptions.nReceiveFloodSize = 1000 * args.GetIntArg("-maxreceivebuffer", DEFAULT_MAXRECEIVEBUFFER); connOptions.m_added_nodes = args.GetArgs("-addnode"); - - connOptions.nMaxOutboundLimit = 1024 * 1024 * args.GetIntArg("-maxuploadtarget", DEFAULT_MAX_UPLOAD_TARGET); + connOptions.nMaxOutboundLimit = *opt_max_upload; connOptions.m_peer_connect_timeout = peer_connect_timeout; for (const std::string& bind_arg : args.GetArgs("-bind")) { diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index d4ceb517dd..601c677a20 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -217,9 +217,6 @@ public: //! Check if shutdown requested. virtual bool shutdownRequested() = 0; - //! Get adjusted time. - virtual int64_t getAdjustedTime() = 0; - //! Send init message. virtual void initMessage(const std::string& message) = 0; @@ -287,9 +284,6 @@ public: //! to be prepared to handle this by ignoring notifications about unknown //! removed transactions and already added new transactions. virtual void requestMempoolTransactions(Notifications& notifications) = 0; - - //! Check if Taproot has activated - virtual bool isTaprootActive() = 0; }; //! Interface to let node manage chain clients (wallets, or maybe tools for diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 34fdde3774..974156e6e1 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -6,7 +6,6 @@ #define BITCOIN_INTERFACES_NODE_H #include <consensus/amount.h> -#include <external_signer.h> #include <net.h> // For NodeId #include <net_types.h> // For banmap_t #include <netaddress.h> // For Network @@ -31,6 +30,7 @@ class RPCTimerInterface; class UniValue; class proxyType; enum class SynchronizationState; +enum class TransactionError; struct CNodeStateStats; struct NodeContext; struct bilingual_str; @@ -50,6 +50,16 @@ struct BlockAndHeaderTipInfo double verification_progress; }; +//! External signer interface used by the GUI. +class ExternalSigner +{ +public: + virtual ~ExternalSigner() {}; + + //! Get signer display name + virtual std::string getName() = 0; +}; + //! Top-level interface for a bitcoin node (bitcoind process). class Node { @@ -111,8 +121,8 @@ public: //! Disconnect node by id. virtual bool disconnectById(NodeId id) = 0; - //! List external signers - virtual std::vector<ExternalSigner> externalSigners() = 0; + //! Return list of external signers (attached devices which can sign transactions). + virtual std::vector<std::unique_ptr<ExternalSigner>> listExternalSigners() = 0; //! Get total bytes recv. virtual int64_t getTotalBytesRecv() = 0; @@ -174,6 +184,9 @@ public: //! Get unspent outputs associated with a transaction. virtual bool getUnspentOutput(const COutPoint& output, Coin& coin) = 0; + //! Broadcast transaction. + virtual TransactionError broadcastTransaction(CTransactionRef tx, CAmount max_tx_fee, std::string& err_string) = 0; + //! Get wallet client. virtual WalletClient& walletClient() = 0; @@ -197,6 +210,10 @@ public: using ShowProgressFn = std::function<void(const std::string& title, int progress, bool resume_possible)>; virtual std::unique_ptr<Handler> handleShowProgress(ShowProgressFn fn) = 0; + //! Register handler for wallet client constructed messages. + using InitWalletFn = std::function<void()>; + virtual std::unique_ptr<Handler> handleInitWallet(InitWalletFn fn) = 0; + //! Register handler for number of connections changed messages. using NotifyNumConnectionsChangedFn = std::function<void(int new_num_connections)>; virtual std::unique_ptr<Handler> handleNotifyNumConnectionsChanged(NotifyNumConnectionsChangedFn fn) = 0; diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index 490563426c..a56ed8802d 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -13,12 +13,13 @@ #include <util/message.h> #include <util/ui_change_type.h> +#include <cstdint> #include <functional> #include <map> #include <memory> -#include <stdint.h> #include <string> #include <tuple> +#include <type_traits> #include <utility> #include <vector> @@ -34,7 +35,7 @@ struct CRecipient; struct PartiallySignedTransaction; struct WalletContext; struct bilingual_str; -typedef uint8_t isminefilter; +using isminefilter = std::underlying_type<isminetype>::type; namespace interfaces { diff --git a/src/key.cpp b/src/key.cpp index 39155e4311..86081b3464 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -229,6 +229,12 @@ bool CKey::Sign(const uint256 &hash, std::vector<unsigned char>& vchSig, bool gr assert(ret); secp256k1_ecdsa_signature_serialize_der(secp256k1_context_sign, vchSig.data(), &nSigLen, &sig); vchSig.resize(nSigLen); + // Additional verification step to prevent using a potentially corrupted signature + secp256k1_pubkey pk; + ret = secp256k1_ec_pubkey_create(secp256k1_context_sign, &pk, begin()); + assert(ret); + ret = secp256k1_ecdsa_verify(GetVerifyContext(), &sig, hash.begin(), &pk); + assert(ret); return true; } @@ -251,17 +257,25 @@ bool CKey::SignCompact(const uint256 &hash, std::vector<unsigned char>& vchSig) return false; vchSig.resize(CPubKey::COMPACT_SIGNATURE_SIZE); int rec = -1; - secp256k1_ecdsa_recoverable_signature sig; - int ret = secp256k1_ecdsa_sign_recoverable(secp256k1_context_sign, &sig, hash.begin(), begin(), secp256k1_nonce_function_rfc6979, nullptr); + secp256k1_ecdsa_recoverable_signature rsig; + int ret = secp256k1_ecdsa_sign_recoverable(secp256k1_context_sign, &rsig, hash.begin(), begin(), secp256k1_nonce_function_rfc6979, nullptr); assert(ret); - ret = secp256k1_ecdsa_recoverable_signature_serialize_compact(secp256k1_context_sign, &vchSig[1], &rec, &sig); + ret = secp256k1_ecdsa_recoverable_signature_serialize_compact(secp256k1_context_sign, &vchSig[1], &rec, &rsig); assert(ret); assert(rec != -1); vchSig[0] = 27 + rec + (fCompressed ? 4 : 0); + // Additional verification step to prevent using a potentially corrupted signature + secp256k1_pubkey epk, rpk; + ret = secp256k1_ec_pubkey_create(secp256k1_context_sign, &epk, begin()); + assert(ret); + ret = secp256k1_ecdsa_recover(GetVerifyContext(), &rpk, &rsig, hash.begin()); + assert(ret); + ret = secp256k1_ec_pubkey_cmp(GetVerifyContext(), &epk, &rpk); + assert(ret == 0); return true; } -bool CKey::SignSchnorr(const uint256& hash, Span<unsigned char> sig, const uint256* merkle_root, const uint256* aux) const +bool CKey::SignSchnorr(const uint256& hash, Span<unsigned char> sig, const uint256* merkle_root, const uint256& aux) const { assert(sig.size() == 64); secp256k1_keypair keypair; @@ -274,7 +288,14 @@ bool CKey::SignSchnorr(const uint256& hash, Span<unsigned char> sig, const uint2 uint256 tweak = XOnlyPubKey(pubkey_bytes).ComputeTapTweakHash(merkle_root->IsNull() ? nullptr : merkle_root); if (!secp256k1_keypair_xonly_tweak_add(GetVerifyContext(), &keypair, tweak.data())) return false; } - bool ret = secp256k1_schnorrsig_sign(secp256k1_context_sign, sig.data(), hash.data(), &keypair, aux ? (unsigned char*)aux->data() : nullptr); + bool ret = secp256k1_schnorrsig_sign(secp256k1_context_sign, sig.data(), hash.data(), &keypair, (unsigned char*)aux.data()); + if (ret) { + // Additional verification step to prevent using a potentially corrupted signature + secp256k1_xonly_pubkey pubkey_verify; + ret = secp256k1_keypair_xonly_pub(GetVerifyContext(), &pubkey_verify, nullptr, &keypair); + ret &= secp256k1_schnorrsig_verify(GetVerifyContext(), sig.data(), hash.begin(), 32, &pubkey_verify); + } + if (!ret) memory_cleanse(sig.data(), sig.size()); memory_cleanse(&keypair, sizeof(keypair)); return ret; } @@ -319,10 +340,11 @@ bool CExtKey::Derive(CExtKey &out, unsigned int _nChild) const { return key.Derive(out.key, out.chaincode, _nChild, chaincode); } -void CExtKey::SetSeed(const unsigned char *seed, unsigned int nSeedLen) { +void CExtKey::SetSeed(Span<const uint8_t> seed) +{ static const unsigned char hashkey[] = {'B','i','t','c','o','i','n',' ','s','e','e','d'}; std::vector<unsigned char, secure_allocator<unsigned char>> vout(64); - CHMAC_SHA512(hashkey, sizeof(hashkey)).Write(seed, nSeedLen).Finalize(vout.data()); + CHMAC_SHA512{hashkey, sizeof(hashkey)}.Write(seed.data(), seed.size()).Finalize(vout.data()); key.Set(vout.data(), vout.data() + 32, true); memcpy(chaincode.begin(), vout.data() + 32, 32); nDepth = 0; @@ -85,6 +85,7 @@ public: //! Simple read-only vector-like interface. unsigned int size() const { return (fValid ? keydata.size() : 0); } + const unsigned char* data() const { return keydata.data(); } const unsigned char* begin() const { return keydata.data(); } const unsigned char* end() const { return keydata.data() + size(); } @@ -129,7 +130,7 @@ public: /** * Create a BIP-340 Schnorr signature, for the xonly-pubkey corresponding to *this, - * optionally tweaked by *merkle_root. Additional nonce entropy can be provided through + * optionally tweaked by *merkle_root. Additional nonce entropy is provided through * aux. * * merkle_root is used to optionally perform tweaking of the private key, as specified @@ -142,7 +143,7 @@ public: * (this is used for key path spending, with specific * Merkle root of the script tree). */ - bool SignSchnorr(const uint256& hash, Span<unsigned char> sig, const uint256* merkle_root = nullptr, const uint256* aux = nullptr) const; + bool SignSchnorr(const uint256& hash, Span<unsigned char> sig, const uint256* merkle_root, const uint256& aux) const; //! Derive BIP32 child key. bool Derive(CKey& keyChild, ChainCode &ccChild, unsigned int nChild, const ChainCode& cc) const; @@ -177,7 +178,7 @@ struct CExtKey { void Decode(const unsigned char code[BIP32_EXTKEY_SIZE]); bool Derive(CExtKey& out, unsigned int nChild) const; CExtPubKey Neuter() const; - void SetSeed(const unsigned char* seed, unsigned int nSeedLen); + void SetSeed(Span<const uint8_t> seed); }; /** Initialize the elliptic curve support. May not be called twice without calling ECC_Stop first. */ diff --git a/src/key_io.cpp b/src/key_io.cpp index 615f4c9312..3559f3a03d 100644 --- a/src/key_io.cpp +++ b/src/key_io.cpp @@ -76,12 +76,16 @@ public: std::string operator()(const CNoDestination& no) const { return {}; } }; -CTxDestination DecodeDestination(const std::string& str, const CChainParams& params, std::string& error_str) +CTxDestination DecodeDestination(const std::string& str, const CChainParams& params, std::string& error_str, std::vector<int>* error_locations) { std::vector<unsigned char> data; uint160 hash; error_str = ""; - if (DecodeBase58Check(str, data, 21)) { + + // Note this will be false if it is a valid Bech32 address for a different network + bool is_bech32 = (ToLower(str.substr(0, params.Bech32HRP().size())) == params.Bech32HRP()); + + if (!is_bech32 && DecodeBase58Check(str, data, 21)) { // base58-encoded Bitcoin addresses. // Public-key-hash-addresses have version 0 (or 111 testnet). // The data vector contains RIPEMD160(SHA256(pubkey)), where pubkey is the serialized public key. @@ -98,15 +102,30 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par return ScriptHash(hash); } - // Set potential error message. - // This message may be changed if the address can also be interpreted as a Bech32 address. - error_str = "Invalid prefix for Base58-encoded address"; + // If the prefix of data matches either the script or pubkey prefix, the length must have been wrong + if ((data.size() >= script_prefix.size() && + std::equal(script_prefix.begin(), script_prefix.end(), data.begin())) || + (data.size() >= pubkey_prefix.size() && + std::equal(pubkey_prefix.begin(), pubkey_prefix.end(), data.begin()))) { + error_str = "Invalid length for Base58 address"; + } else { + error_str = "Invalid prefix for Base58-encoded address"; + } + return CNoDestination(); + } else if (!is_bech32) { + // Try Base58 decoding without the checksum, using a much larger max length + if (!DecodeBase58(str, data, 100)) { + error_str = "Not a valid Bech32 or Base58 encoding"; + } else { + error_str = "Invalid checksum or length of Base58 address"; + } + return CNoDestination(); } + data.clear(); const auto dec = bech32::Decode(str); if ((dec.encoding == bech32::Encoding::BECH32 || dec.encoding == bech32::Encoding::BECH32M) && dec.data.size() > 0) { // Bech32 decoding - error_str = ""; if (dec.hrp != params.Bech32HRP()) { error_str = "Invalid prefix for Bech32 address"; return CNoDestination(); @@ -168,9 +187,10 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par } } - // Set error message if address can't be interpreted as Base58 or Bech32. - if (error_str.empty()) error_str = "Invalid address format"; - + // Perform Bech32 error location + auto res = bech32::LocateErrors(str); + error_str = res.first; + if (error_locations) *error_locations = std::move(res.second); return CNoDestination(); } } // namespace @@ -258,9 +278,9 @@ std::string EncodeDestination(const CTxDestination& dest) return std::visit(DestinationEncoder(Params()), dest); } -CTxDestination DecodeDestination(const std::string& str, std::string& error_msg) +CTxDestination DecodeDestination(const std::string& str, std::string& error_msg, std::vector<int>* error_locations) { - return DecodeDestination(str, Params(), error_msg); + return DecodeDestination(str, Params(), error_msg, error_locations); } CTxDestination DecodeDestination(const std::string& str) @@ -272,7 +292,7 @@ CTxDestination DecodeDestination(const std::string& str) bool IsValidDestinationString(const std::string& str, const CChainParams& params) { std::string error_msg; - return IsValidDestination(DecodeDestination(str, params, error_msg)); + return IsValidDestination(DecodeDestination(str, params, error_msg, nullptr)); } bool IsValidDestinationString(const std::string& str) diff --git a/src/key_io.h b/src/key_io.h index bd81f7847e..2062bb4c44 100644 --- a/src/key_io.h +++ b/src/key_io.h @@ -23,7 +23,7 @@ std::string EncodeExtPubKey(const CExtPubKey& extpubkey); std::string EncodeDestination(const CTxDestination& dest); CTxDestination DecodeDestination(const std::string& str); -CTxDestination DecodeDestination(const std::string& str, std::string& error_msg); +CTxDestination DecodeDestination(const std::string& str, std::string& error_msg, std::vector<int>* error_locations = nullptr); bool IsValidDestinationString(const std::string& str); bool IsValidDestinationString(const std::string& str, const CChainParams& params); diff --git a/src/logging/timer.h b/src/logging/timer.h index 79627b1fe3..b77a0e17c7 100644 --- a/src/logging/timer.h +++ b/src/logging/timer.h @@ -27,10 +27,12 @@ public: Timer( std::string prefix, std::string end_msg, - BCLog::LogFlags log_category = BCLog::LogFlags::ALL) : + BCLog::LogFlags log_category = BCLog::LogFlags::ALL, + bool msg_on_completion = true) : m_prefix(std::move(prefix)), m_title(std::move(end_msg)), - m_log_category(log_category) + m_log_category(log_category), + m_message_on_completion(msg_on_completion) { this->Log(strprintf("%s started", m_title)); m_start_t = GetTime<std::chrono::microseconds>(); @@ -38,7 +40,11 @@ public: ~Timer() { - this->Log(strprintf("%s completed", m_title)); + if (m_message_on_completion) { + this->Log(strprintf("%s completed", m_title)); + } else { + this->Log("completed"); + } } void Log(const std::string& msg) @@ -74,14 +80,17 @@ private: std::chrono::microseconds m_start_t{}; //! Log prefix; usually the name of the function this was created in. - const std::string m_prefix{}; + const std::string m_prefix; //! A descriptive message of what is being timed. - const std::string m_title{}; + const std::string m_title; //! Forwarded on to LogPrint if specified - has the effect of only //! outputting the timing log when a particular debug= category is specified. - const BCLog::LogFlags m_log_category{}; + const BCLog::LogFlags m_log_category; + + //! Whether to output the message again on completion. + const bool m_message_on_completion; }; } // namespace BCLog @@ -91,6 +100,8 @@ private: BCLog::Timer<std::chrono::microseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, log_category) #define LOG_TIME_MILLIS_WITH_CATEGORY(end_msg, log_category) \ BCLog::Timer<std::chrono::milliseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, log_category) +#define LOG_TIME_MILLIS_WITH_CATEGORY_MSG_ONCE(end_msg, log_category) \ + BCLog::Timer<std::chrono::milliseconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg, log_category, /* msg_on_completion=*/false) #define LOG_TIME_SECONDS(end_msg) \ BCLog::Timer<std::chrono::seconds> PASTE2(logging_timer, __COUNTER__)(__func__, end_msg) diff --git a/src/minisketch/.cirrus.yml b/src/minisketch/.cirrus.yml new file mode 100644 index 0000000000..4a5353f137 --- /dev/null +++ b/src/minisketch/.cirrus.yml @@ -0,0 +1,154 @@ +env: + BUILD: check + HOST: + MAKEFLAGS: -j4 + BENCH: yes + TESTRUNS: + EXEC_CMD: + ENABLE_FIELDS: + +cat_logs_snippet: &CAT_LOGS + on_failure: + cat_test_log_script: + - cat test-suite.log || true + cat_config_log_script: + - cat config.log || true + cat_test_env_script: + - cat test_env.log || true + cat_ci_env_script: + - env + +merge_base_script_snippet: &MERGE_BASE + merge_base_script: + - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi + - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH + - git config --global user.email "ci@ci.ci" + - git config --global user.name "ci" + - git merge FETCH_HEAD # Merge base to detect silent merge conflicts + +env_matrix_snippet: &ENV_MATRIX_VALGRIND + - env: + ENABLE_FIELDS: "7,32,58" + - env: + BUILD: distcheck + - env: + EXEC_CMD: valgrind --error-exitcode=42 + TESTRUNS: 1 + BUILD: + +env_matrix_snippet: &ENV_MATRIX_SAN + - env: + ENABLE_FIELDS: 28 + - env: + BUILD: distcheck + - env: + CXXFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer" + LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer" + UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1" + BENCH: no + +env_matrix_snippet: &ENV_MATRIX_SAN_VALGRIND + - env: + ENABLE_FIELDS: "11,64,37" + - env: + BUILD: distcheck + - env: + EXEC_CMD: valgrind --error-exitcode=42 + TESTRUNS: 1 + BUILD: + - env: + CXXFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer" + LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer" + UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1" + BENCH: no + +task: + name: "x86_64: Linux (Debian stable)" + container: + dockerfile: ci/linux-debian.Dockerfile + memory: 2G + cpu: 4 + matrix: + << : *ENV_MATRIX_SAN_VALGRIND + matrix: + - env: + CC: gcc + - env: + CC: clang + << : *MERGE_BASE + test_script: + - ./ci/cirrus.sh + << : *CAT_LOGS + +task: + name: "i686: Linux (Debian stable)" + container: + dockerfile: ci/linux-debian.Dockerfile + memory: 2G + cpu: 4 + env: + HOST: i686-linux-gnu + matrix: + << : *ENV_MATRIX_VALGRIND + matrix: + - env: + CC: i686-linux-gnu-gcc + - env: + CC: clang --target=i686-pc-linux-gnu -isystem /usr/i686-linux-gnu/include + test_script: + - ./ci/cirrus.sh + << : *CAT_LOGS + +task: + name: "x86_64: macOS Catalina" + macos_instance: + image: catalina-base + env: + # Cirrus gives us a fixed number of 12 virtual CPUs. + MAKEFLAGS: -j13 + matrix: + << : *ENV_MATRIX_SAN + matrix: + - env: + CC: gcc-9 + - env: + CC: clang + brew_script: + - brew update + - brew install automake libtool gcc@9 + << : *MERGE_BASE + test_script: + - ./ci/cirrus.sh + << : *CAT_LOGS + +task: + name: "s390x (big-endian): Linux (Debian stable, QEMU)" + container: + dockerfile: ci/linux-debian.Dockerfile + cpu: 4 + memory: 2G + env: + EXEC_CMD: qemu-s390x -L /usr/s390x-linux-gnu + HOST: s390x-linux-gnu + BUILD: + << : *MERGE_BASE + test_script: + # https://sourceware.org/bugzilla/show_bug.cgi?id=27008 + - rm /etc/ld.so.cache + - ./ci/cirrus.sh + << : *CAT_LOGS + +task: + name: "x86_64-w64-mingw32: Linux (Debian stable, Wine)" + container: + dockerfile: ci/linux-debian.Dockerfile + cpu: 4 + memory: 2G + env: + EXEC_CMD: wine + HOST: x86_64-w64-mingw32 + BUILD: + << : *MERGE_BASE + test_script: + - ./ci/cirrus.sh + << : *CAT_LOGS diff --git a/src/minisketch/.gitignore b/src/minisketch/.gitignore new file mode 100644 index 0000000000..f5be6fab88 --- /dev/null +++ b/src/minisketch/.gitignore @@ -0,0 +1,35 @@ +*.o +*.lo +*.la +*.dll +*.dylib +*.so.* +.* +*.a +*~ + +Makefile +Makefile.in +aclocal.m4 +autom4te.cache/ +build-aux/config.guess +build-aux/config.sub +build-aux/depcomp +build-aux/install-sh +build-aux/ltmain.sh +build-aux/m4/libtool.m4 +build-aux/m4/lt~obsolete.m4 +build-aux/m4/ltoptions.m4 +build-aux/m4/ltsugar.m4 +build-aux/m4/ltversion.m4 +build-aux/missing +build-aux/compile +build-aux/test-driver +config.log +config.status +configure +libtool +stamp-h1 + +test* +bench diff --git a/src/minisketch/LICENSE b/src/minisketch/LICENSE new file mode 100644 index 0000000000..b25e3caee3 --- /dev/null +++ b/src/minisketch/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/minisketch/Makefile.am b/src/minisketch/Makefile.am new file mode 100644 index 0000000000..86a5c9dc90 --- /dev/null +++ b/src/minisketch/Makefile.am @@ -0,0 +1,92 @@ +ACLOCAL_AMFLAGS = -I build-aux/m4 +AM_CXXFLAGS = $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) + +include sources.mk + +include_HEADERS = $(MINISKETCH_DIST_HEADERS_INT) +noinst_HEADERS = $(MINISKETCH_LIB_HEADERS_INT) $(MINISKETCH_FIELD_GENERIC_HEADERS_INT) $(MINISKETCH_FIELD_CLMUL_HEADERS_INT) + +LIBMINISKETCH = libminisketch.la +LIBMINISKETCH_FIELD_GENERIC = libminisketch_field_generic.la +if ENABLE_CLMUL +LIBMINISKETCH_FIELD_CLMUL = libminisketch_field_clmul.la +endif +if USE_TESTS +LIBMINISKETCH_VERIFY=libminisketch_verify.la +LIBMINISKETCH_FIELD_GENERIC_VERIFY=libminisketch_field_generic_verify.la +if ENABLE_CLMUL +LIBMINISKETCH_FIELD_CLMUL_VERIFY=libminisketch_field_clmul_verify.la +endif +endif + +lib_LTLIBRARIES = +lib_LTLIBRARIES += $(LIBMINISKETCH) + +noinst_LTLIBRARIES = +noinst_LTLIBRARIES += $(LIBMINISKETCH_FIELD_GENERIC) +noinst_LTLIBRARIES += $(LIBMINISKETCH_FIELD_GENERIC_VERIFY) +noinst_LTLIBRARIES += $(LIBMINISKETCH_FIELD_CLMUL) +noinst_LTLIBRARIES += $(LIBMINISKETCH_FIELD_CLMUL_VERIFY) +noinst_LTLIBRARIES += $(LIBMINISKETCH_VERIFY) + +# Release libs +libminisketch_field_generic_la_SOURCES = $(MINISKETCH_FIELD_GENERIC_SOURCES_INT) +libminisketch_field_generic_la_CPPFLAGS = $(AM_CPPFLAGS) $(RELEASE_DEFINES) + +libminisketch_field_clmul_la_SOURCES = $(MINISKETCH_FIELD_CLMUL_SOURCES_INT) +libminisketch_field_clmul_la_CPPFLAGS = $(AM_CPPFLAGS) $(RELEASE_DEFINES) +libminisketch_field_clmul_la_CXXFLAGS = $(AM_CXXFLAGS) $(CLMUL_CXXFLAGS) + +libminisketch_la_SOURCES = $(MINISKETCH_LIB_SOURCES_INT) +libminisketch_la_CPPFLAGS = $(AM_CPPFLAGS) $(RELEASE_DEFINES) +libminisketch_la_LIBADD = $(LIBMINISKETCH_FIELD_CLMUL) $(LIBMINISKETCH_FIELD_GENERIC) + +# Libs with extra verification checks +libminisketch_field_generic_verify_la_SOURCES = $(MINISKETCH_FIELD_GENERIC_SOURCES_INT) +libminisketch_field_generic_verify_la_CPPFLAGS = $(AM_CPPFLAGS) $(VERIFY_DEFINES) + +libminisketch_field_clmul_verify_la_SOURCES = $(MINISKETCH_FIELD_CLMUL_SOURCES_INT) +libminisketch_field_clmul_verify_la_CPPFLAGS = $(AM_CPPFLAGS) $(VERIFY_DEFINES) +libminisketch_field_clmul_verify_la_CXXFLAGS = $(AM_CXXFLAGS) $(CLMUL_CXXFLAGS) + +libminisketch_verify_la_SOURCES = $(MINISKETCH_LIB_SOURCES_INT) +libminisketch_verify_la_CPPFLAGS = $(AM_CPPFLAGS) $(VERIFY_DEFINES) +libminisketch_verify_la_LIBADD = $(LIBMINISKETCH_FIELD_CLMUL_VERIFY) $(LIBMINISKETCH_FIELD_GENERIC_VERIFY) + +noinst_PROGRAMS = +if USE_BENCHMARK +noinst_PROGRAMS += bench +endif +if USE_TESTS +noinst_PROGRAMS += test test-verify +TESTS = test test-verify +endif + +bench_SOURCES = $(MINISKETCH_BENCH_SOURCES_INT) +bench_CPPFLAGS = $(AM_CPPFLAGS) $(RELEASE_DEFINES) +bench_LDADD = $(LIBMINISKETCH) +bench_LDFLAGS = $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) + +test_SOURCES = $(MINISKETCH_TEST_SOURCES_INT) +test_CPPFLAGS = $(AM_CPPFLAGS) $(RELEASE_DEFINES) +test_LDADD = $(LIBMINISKETCH) +test_LDFLAGS = $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) + +test_verify_SOURCES = $(MINISKETCH_TEST_SOURCES_INT) +test_verify_CPPFLAGS = $(AM_CPPFLAGS) $(VERIFY_DEFINES) +test_verify_LDADD = $(LIBMINISKETCH_VERIFY) +test_verify_LDFLAGS = $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) + +EXTRA_DIST= +EXTRA_DIST += LICENSE +EXTRA_DIST += README.md +EXTRA_DIST += doc/example.c +EXTRA_DIST += doc/gen_params.sage +EXTRA_DIST += doc/math.md +EXTRA_DIST += doc/moduli.md +EXTRA_DIST += doc/plot_bits.png +EXTRA_DIST += doc/plot_capacity.png +EXTRA_DIST += doc/plot_diff.png +EXTRA_DIST += doc/plot_size.png +EXTRA_DIST += doc/protocoltips.md +EXTRA_DIST += tests/pyminisketch.py diff --git a/src/minisketch/README.md b/src/minisketch/README.md new file mode 100644 index 0000000000..c0cfdc1623 --- /dev/null +++ b/src/minisketch/README.md @@ -0,0 +1,210 @@ +# Minisketch: a library for [BCH](https://en.wikipedia.org/wiki/BCH_code)-based set reconciliation +<img align="right" src="doc/minisketch-vs.png" /> + +`libminisketch` is an optimized standalone MIT-licensed library with C API for constructing and decoding *set sketches*, which can be used for compact set reconciliation and other applications. +It is an implementation of the PinSketch<sup>[[1]](#myfootnote1)</sup> algorithm. An explanation of the algorithm can be found [here](doc/math.md). + +## Sketches for set reconciliation + +Sketches, as produced by this library, can be seen as "set checksums" with two peculiar properties: +* Sketches have a predetermined capacity, and when the number of elements in the set is not higher than the capacity, `libminisketch` will always recover the entire set from the sketch. A sketch of *b*-bit elements with capacity *c* can be stored in *bc* bits. +* The sketches of two sets can be combined by adding them (XOR) to obtain a sketch of the [symmetric difference](https://en.wikipedia.org/wiki/Symmetric_difference) between the two sets (*i.e.*, all elements that occur in one but not both input sets). + +This makes them appropriate for a very bandwidth-efficient set reconciliation protocol. If Alice and Bob each have a set of elements, and they suspect that the sets largely but not entirely overlap, +they can use the following protocol to let both parties learn all the elements: +* Alice and Bob both compute a sketch of their set elements. +* Alice sends her sketch to Bob. +* Bob combines the two sketches, and obtains a sketch of the symmetric difference. +* Bob tries to recover the elements from the difference sketch. +* Bob sends every element in the difference that he has to Alice. + +This will always succeed when the size of the difference (elements that Alice has but Bob doesn't plus elements that Bob has but Alice doesn't) does not exceed the +capacity of the sketch that Alice sent. The interesting part is that this works regardless of the actual set sizes—only the difference matters. + +If the elements are large, it may be preferable to compute the sketches over *hashes* of the set elements. In that case an additional step is added to the protocol, where Bob also sends the hash +of every element he does not have to Alice, who responds with the requested elements. + +The doc/ directory has additional [tips for designing reconciliation protocols using libminisketch](doc/protocoltips.md). + +## Evaluation + +<img src="doc/plot_capacity.png" width="432" height="324" /> <img src="doc/plot_diff.png" width="432" height="324" /> + +<img src="doc/plot_size.png" width="432" height="324" /> <img src="doc/plot_bits.png" width="432" height="324" /> + +**The first graph** above shows a benchmark of `libminisketch` against three other set reconciliation algorithms/implementations. The benchmarks were performed using a single core on a system with an Intel Core i7-7820HQ CPU with clock speed locked at 2.4 GHz. The diagram shows the time needed for merging of two sketches and decoding the result. The creation of a sketch on the same machine takes around 5 ns per capacity and per set element. The other implementations are: +* [`pinsketch`](https://www.cs.bu.edu/~reyzin/code/fuzzy.html), the original PinSketch implementation. +* [`cpisync`](https://github.com/trachten/cpisync), a software project which implements a number of set reconciliation algorithms and protocols. The included benchmark analyzes the non-probabilistic version of the original CPISync algorithm<sup>[[5]](#myfootnote5)</sup> only. +* A high-performance custom IBLT implementation using 4 hash functions and 32-bit checksums. + +For the largest sizes currently of interest to the authors, such as a set of capacity 4096 with 1024 differences, `libminisketch` is forty-nine times faster than `pinsketch` and over eight thousand times faster than `cpisync`. `libminisketch` is fast enough on realistic set sizes for use on high-traffic network servers where computational resources are limited. + +Even where performance is latency-limited, small minisketches can be fast enough to improve performance. On the above i7-7820HQ, a set of 2500 30-bit entries with a difference of 20 elements can be communicated in less time with a minisketch than sending the raw set so long as the communications bandwidth is 1 gigabit per second or less; an eight-element difference can be communicated in better than one-fifth the time on a gigabit link. + +**The second graph** above shows the performance of the same algorithms on the same system, but this time keeping the capacity constant at 128, while varying the number of differences to reconcile between 1 and 128. It shows how `cpisync`'s reconciliation speed is mostly dependent on capacity, while `pinsketch`/`libminisketch` are more dependent on number of differences. + +**The third graph** above shows the size overhead of a typical IBLT scheme over the other algorithms (which are near-optimal bandwidth), for various levels of failure probability. IBLT takes tens of times the bandwidth of `libminisketch` sketches when the set difference size is small and the required failure rate is low. + +**The fourth graph** above shows the effect of the field size on speed in `libminisketch`. The three lines correspond to: +* CLMUL 64-bit: Intel Core i7-7820HQ system at 2.4 GHz +* Generic 64-bit: POWER9 CP9M06 system at 2.8 GHz (Talos II) +* Generic 32-bit: Cortex-A53 at 1.2 GHz (Raspberry Pi 3B) + +It shows how CLMUL implementations are faster for certain fields (specifically, field sizes for which an irreducible polynomial of the form *x<sup>b</sup> + x + 1* over *GF(2)* exists, and to a lesser extent, fields which are a multiple of 8 bits). It also shows how (for now) a significant performance drop exists for fields larger than 32 bits on 32-bit platforms. Note that the three lines are not at the same scale (the Raspberry Pi 3B is around 10x slower for 32-bit fields than the Core i7; the POWER9 is around 1.3x slower). + +Below we compare the PinSketch algorithm (which `libminisketch` is an implementation of) with other set reconciliation algorithms: + +| Algorithm | Sketch size | Decode success | Decoding complexity | Difference type | Secure sketch | +| ----------------------------------------------------- | ------------------------- | ---------------| ------------------- | --------------- | ------------- | +| CPISync<sup>[[2]](#myfootnote2)</sup> | *(b+1)c* | Always | *O(n<sup>3</sup>)* | Both | Yes | +| PinSketch<sup>[[1]](#myfootnote1)</sup> | *bc* | Always | *O(n<sup>2</sup>)* | Symmetric only | Yes | +| IBLT<sup>[[6]](#myfootnote1)[[7]](#myfootnote1)</sup> | *αbc* (see graph 3) | Probabilistic | *O(n)* | Depends | No | + +* **Sketch size:** This column shows the size in bits of a sketch designed for reconciling *c* different *b*-bit elements. PinSketch and CPISync have a near-optimal<sup>[[11]](#myfootnote11)</sup> communication overhead, which in practice means the sketch size is very close (or equal to) *bc* bits. That is the same size as would be needed to transfer the elements of the difference naively (which is remarkable, as the difference isn't even known by the sender). For IBLT there is an overhead factor *α*, which depends on various design parameters, but is often between *2* and *10*. +* **Decode success:** Whenever a sketch is designed with a capacity not lower than the actual difference size, CPISync and PinSketch guarantee that decoding of the difference will always succeed. IBLT always has a chance of failure, though that chance can be made arbitrarily small by increasing the communication overhead. +* **Decoding complexity:** The space savings achieved by near-optimal algorithms come at a cost in performance, as their asymptotic decode complexity is quadratic or cubic, while IBLT is linear. This means that using near-optimal algorithms can be too expensive for applications where the difference is sufficiently large. +* **Difference type:** PinSketch can only compute the symmetric difference from a merged sketch, while CPISync and IBLT can distinguish which side certain elements were missing on. When the decoder has access to one of the sets, this generally doesn't matter, as he can look up each of the elements in the symmetric difference with one of the sets. +* **Secure sketch:** Whether the sketch satisfies the definition of a secure sketch<sup>[[1]](#myfootnote1)</sup>, which implies a minimal amount about a set can be extracted from a sketch by anyone who does not know most of the elements already. This makes the algorithm appropriate for applications like fingerprint authentication. + +## Building + +The build system is very rudimentary for now, and [improvements](https://github.com/sipa/minisketch/pulls) are welcome. + +The following may work and produce a `libminisketch.a` file you can link against: + +```bash +git clone https://github.com/sipa/minisketch +cd minisketch +./autogen.sh && ./configure && make +``` + +## Usage + +In this section Alice and Bob are trying to find the difference between their sets. +Alice has the set *[3000 ... 3009]*, while Bob has *[3002 ... 3011]*. + +First, Alice creates a sketch: + +```c +#include <stdio.h> +#include <assert.h> +#include "../include/minisketch.h" +int main(void) { + + minisketch *sketch_a = minisketch_create(12, 0, 4); +``` + +The arguments are: +* The field size *b*, which specifies the size of the elements being reconciled. With a field size *b*, the supported range of set elements is the integers from *1* to *2<sup>b</sub>* *- 1*, inclusive. Note that elements cannot be zero. +* The implementation number. Implementation *0* is always supported, but more efficient algorithms may be available on some hardware. The serialized form of a sketch is independent of the implementation, so different implementations can interoperate. +* The capacity *c*, which specifies how many differences the resulting sketch can reconcile. + +Then Alice adds her elements to her sketch. Note that adding the same element a second time removes it again, as sketches have set semantics, not multiset semantics. + +```c + for (int i = 3000; i < 3010; ++i) { + minisketch_add_uint64(sketch_a, i); + } +``` + +The next step is serializing the sketch into a byte array: + +```c + size_t sersize = minisketch_serialized_size(sketch_a); + assert(sersize == 12 * 4 / 8); // 4 12-bit values is 6 bytes. + unsigned char *buffer_a = malloc(sersize); + minisketch_serialize(sketch_a, buffer_a); + minisketch_destroy(sketch_a); +``` + +The contents of the buffer can then be submitted to Bob, who can create his own sketch: + +```c + minisketch *sketch_b = minisketch_create(12, 0, 4); // Bob's own sketch + for (int i = 3002; i < 3012; ++i) { + minisketch_add_uint64(sketch_b, i); + } +``` + +After Bob receives Alice's serialized sketch, he can reconcile: + +```c + sketch_a = minisketch_create(12, 0, 4); // Alice's sketch + minisketch_deserialize(sketch_a, buffer_a); // Load Alice's sketch + free(buffer_a); + + // Merge the elements from sketch_a into sketch_b. The result is a sketch_b + // which contains all elements that occurred in Alice's or Bob's sets, but not + // in both. + minisketch_merge(sketch_b, sketch_a); + + uint64_t differences[4]; + ssize_t num_differences = minisketch_decode(sketch_b, 4, differences); + minisketch_destroy(sketch_a); + minisketch_destroy(sketch_b); + if (num_differences < 0) { + printf("More than 4 differences!\n"); + } else { + ssize_t i; + for (i = 0; i < num_differences; ++i) { + printf("%u is in only one of the two sets\n", (unsigned)differences[i]); + } + } +} +``` + +In this example Bob would see output such as: + +``` +$ gcc -std=c99 -Wall -Wextra -o example ./doc/example.c -Lsrc/ -lminisketch -lstdc++ && ./example +3000 is in only one of the two sets +3011 is in only one of the two sets +3001 is in only one of the two sets +3010 is in only one of the two sets +``` + +The order of the output is arbitrary and will differ on different runs of minisketch_decode(). + +## Applications + +Communications efficient set reconciliation has been proposed to optimize Bitcoin transaction distribution<sup>[[8]](#myfootnote8)</sup>, which would allow Bitcoin nodes to have many more peers while reducing bandwidth usage. It could also be used for Bitcoin block distribution<sup>[[9]](#myfootnote9)</sup>, particularly for very low bandwidth links such as satellite. A similar approach (CPISync) is used by PGP SKS keyservers to synchronize their databases efficiently. Secure sketches can also be used as helper data to reliably extract a consistent cryptographic key from fuzzy biometric data while leaking minimal information<sup>[[1]](#myfootnote1)</sup>. They can be combined with [dcnets](https://en.wikipedia.org/wiki/Dining_cryptographers_problem) to create cryptographic multiparty anonymous communication<sup>[[10]](#myfootnote10)</sup>. + +## Implementation notes + +`libminisketch` is written in C++11, but has a [C API](include/minisketch.h) for compatibility reasons. + +Specific algorithms and optimizations used: +* Finite field implementations: + * A generic implementation using C unsigned integer bit operations, and one using the [CLMUL instruction](https://en.wikipedia.org/wiki/CLMUL_instruction_set) where available. The latter has specializations for different classes of fields that permit optimizations (those with trinomial irreducible polynomials, and those whose size is a multiple of 8 bits). + * Precomputed tables for (repeated) squaring, and for solving equations of the form *x<sup>2</sup> + x = a*<sup>[[2]](#myfootnote2)</sup>. + * Inverses are computed using an [exponentiation ladder](https://en.wikipedia.org/w/index.php?title=Exponentiation_by_squaring&oldid=868883860)<sup>[[12]](#myfootnote12)</sup> on systems where multiplications are relatively fast, and using an [extended GCD algorithm](https://en.wikipedia.org/w/index.php?title=Extended_Euclidean_algorithm&oldid=865802511#Computing_multiplicative_inverses_in_modular_structures) otherwise. + * Repeated multiplications are accelerated using runtime precomputations on systems where multiplications are relatively slow. + * The serialization of field elements always represents them as bits that are coefficients of the lowest-weight (using lexicographic order as tie breaker) irreducible polynomials over *GF(2)* (see [this list](doc/moduli.md)), but for some implementations they are converted to a different representation internally. +* The sketch algorithms are specialized for each separate field implementation, permitting inlining and specific optimizations while avoiding dynamic allocations and branching costs. +* Decoding of sketches uses the [Berlekamp-Massey algorithm](https://en.wikipedia.org/w/index.php?title=Berlekamp%E2%80%93Massey_algorithm&oldid=870768940)<sup>[[3]](#myfootnote3)</sup> to compute the characteristic polynomial. +* Finding the roots of polynomials is done using the Berlekamp trace algorithm with explicit formula for quadratic polynomials<sup>[[4]](#myfootnote4)</sup>. The root finding is randomized to prevent adversarial inputs that intentionally trigger worst-case decode time. +* A (possibly) novel optimization combines a test for unique roots with the Berlekamp trace algorithm. + +Some improvements that are still TODO: +* Explicit formulas for the roots of polynomials of higher degree than 2 +* Subquadratic multiplication and modulus algorithms +* The [Half-GCD algorithm](http://mathworld.wolfram.com/Half-GCD.html) for faster GCDs +* An interface for incremental decoding: most of the computation in most failed decodes can be reused when attempting to decode a longer sketch of the same set +* Platform specific optimizations for platforms other than x86 +* Avoid using slow uint64_t for calculations on 32-bit hosts +* Optional IBLT / Hybrid and set entropy coder under the same interface + +## References + +* <a name="myfootnote1">[1]</a> Dodis, Ostrovsky, Reyzin and Smith. *Fuzzy Extractors: How to Generate Strong Keys from Biometrics and Other Noisy Data.* SIAM Journal on Computing, volume 38, number 1, pages 97-139, 2008). [[URL]](http://eprint.iacr.org/2003/235) [[PDF]](https://eprint.iacr.org/2003/235.pdf) +* <a name="myfootnote5">[5]</a> A. Trachtenberg, D. Starobinski and S. Agarwal. *Fast PDA synchronization using characteristic polynomial interpolation.* Proceedings, Twenty-First Annual Joint Conference of the IEEE Computer and Communications Societies, New York, NY, USA, 2002, pp. 1510-1519 vol.3. [[PDF]](https://pdfs.semanticscholar.org/43da/2070b6b7b2320a1fed2fd5e70e87332c9c5e.pdf) +* <a name="myfootnote2">[2]</a> Cherly, Jørgen, Luis Gallardo, Leonid Vaserstein, and Ethel Wheland. *Solving quadratic equations over polynomial rings of characteristic two.* Publicacions Matemà tiques (1998): 131-142. [[PDF]](https://www.raco.cat/index.php/PublicacionsMatematiques/article/viewFile/37927/40412) +* <a name="myfootnote3">[3]</a> J. Massey. *Shift-register synthesis and BCH decoding.* IEEE Transactions on Information Theory, vol. 15, no. 1, pp. 122-127, January 1969. [[PDF]](http://crypto.stanford.edu/~mironov/cs359/massey.pdf) +* <a name="myfootnote4">[4]</a> Bhaskar Biswas, Vincent Herbert. *Efficient Root Finding of Polynomials over Fields of Characteristic 2.* 2009. hal-00626997. [[URL]](https://hal.archives-ouvertes.fr/hal-00626997) [[PDF]](https://hal.archives-ouvertes.fr/hal-00626997/document) +* <a name="myfootnote6">[6]</a> Eppstein, David, Michael T. Goodrich, Frank Uyeda, and George Varghese. *What's the difference?: efficient set reconciliation without prior context.* ACM SIGCOMM Computer Communication Review, vol. 41, no. 4, pp. 218-229. ACM, 2011. [[PDF]](https://www.ics.uci.edu/~eppstein/pubs/EppGooUye-SIGCOMM-11.pdf) +* <a name="myfootnote7">[7]</a> Goodrich, Michael T. and Michael Mitzenmacher. *Invertible bloom lookup tables.* 2011 49th Annual Allerton Conference on Communication, Control, and Computing (Allerton) (2011): 792-799. [[PDF]](https://arxiv.org/pdf/1101.2245.pdf) +* <a name="myfootnote8">[8]</a> Maxwell, Gregory F. *[Blocksonly mode BW savings, the limits of efficient block xfer, and better relay](https://bitcointalk.org/index.php?topic=1377345.0)* Bitcointalk 2016, *[Technical notes on mempool synchronizing relay](https://people.xiph.org/~greg/mempool_sync_relay.txt)* #bitcoin-wizards 2016. +* <a name="myfootnote9">[9]</a> Maxwell, Gregory F. *[Block network coding](https://en.bitcoin.it/wiki/User:Gmaxwell/block_network_coding)* Bitcoin Wiki 2014, *[Technical notes on efficient block xfer](https://people.xiph.org/~greg/efficient.block.xfer.txt)* #bitcoin-wizards 2015. +* <a name="myfootnote10">[10]</a> Ruffing, Tim, Moreno-Sanchez, Pedro, Aniket, Kate, *P2P Mixing and Unlinkable Bitcoin Transactions* NDSS Symposium 2017 [[URL]](https://eprint.iacr.org/2016/824) [[PDF]](https://eprint.iacr.org/2016/824.pdf) +* <a name="myfootnote11">[11]</a> Y. Misky, A. Trachtenberg, R. Zippel. *Set Reconciliation with Nearly Optimal Communication Complexity.* Cornell University, 2000. [[URL]](https://ecommons.cornell.edu/handle/1813/5803) [[PDF]](https://ecommons.cornell.edu/bitstream/handle/1813/5803/2000-1813.pdf) +* <a name="myfootnote12">[12]</a> Itoh, Toshiya, and Shigeo Tsujii. "A fast algorithm for computing multiplicative inverses in GF (2m) using normal bases." Information and computation 78, no. 3 (1988): 171-177. [[URL]](https://www.sciencedirect.com/science/article/pii/0890540188900247) diff --git a/src/minisketch/autogen.sh b/src/minisketch/autogen.sh new file mode 100755 index 0000000000..27417daf76 --- /dev/null +++ b/src/minisketch/autogen.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# Copyright (c) 2013-2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +set -e +srcdir="$(dirname $0)" +cd "$srcdir" +if [ -z ${LIBTOOLIZE} ] && GLIBTOOLIZE="`which glibtoolize 2>/dev/null`"; then + LIBTOOLIZE="${GLIBTOOLIZE}" + export LIBTOOLIZE +fi +which autoreconf >/dev/null || \ + (echo "configuration failed, please install autoconf first" && exit 1) +autoreconf --install --force --warnings=all diff --git a/src/minisketch/build-aux/m4/ax_check_compile_flag.m4 b/src/minisketch/build-aux/m4/ax_check_compile_flag.m4 new file mode 100644 index 0000000000..bd753b34d7 --- /dev/null +++ b/src/minisketch/build-aux/m4/ax_check_compile_flag.m4 @@ -0,0 +1,53 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_COMPILE_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS], [INPUT]) +# +# DESCRIPTION +# +# Check whether the given FLAG works with the current language's compiler +# or gives an error. (Warnings, however, are ignored) +# +# ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on +# success/failure. +# +# If EXTRA-FLAGS is defined, it is added to the current language's default +# flags (e.g. CFLAGS) when the check is done. The check is thus made with +# the flags: "CFLAGS EXTRA-FLAGS FLAG". This can for example be used to +# force the compiler to issue an error when a bad flag is given. +# +# INPUT gives an alternative input source to AC_COMPILE_IFELSE. +# +# NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this +# macro in sync with AX_CHECK_{PREPROC,LINK}_FLAG. +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim <guidod@gmx.de> +# Copyright (c) 2011 Maarten Bosmans <mkbosmans@gmail.com> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 6 + +AC_DEFUN([AX_CHECK_COMPILE_FLAG], +[AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF +AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]flags_$4_$1])dnl +AC_CACHE_CHECK([whether _AC_LANG compiler accepts $1], CACHEVAR, [ + ax_check_save_flags=$[]_AC_LANG_PREFIX[]FLAGS + _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $4 $1" + AC_COMPILE_IFELSE([m4_default([$5],[AC_LANG_PROGRAM()])], + [AS_VAR_SET(CACHEVAR,[yes])], + [AS_VAR_SET(CACHEVAR,[no])]) + _AC_LANG_PREFIX[]FLAGS=$ax_check_save_flags]) +AS_VAR_IF(CACHEVAR,yes, + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_COMPILE_FLAGS diff --git a/src/minisketch/build-aux/m4/ax_check_link_flag.m4 b/src/minisketch/build-aux/m4/ax_check_link_flag.m4 new file mode 100644 index 0000000000..03a30ce4c7 --- /dev/null +++ b/src/minisketch/build-aux/m4/ax_check_link_flag.m4 @@ -0,0 +1,53 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_check_link_flag.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_LINK_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS], [INPUT]) +# +# DESCRIPTION +# +# Check whether the given FLAG works with the linker or gives an error. +# (Warnings, however, are ignored) +# +# ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on +# success/failure. +# +# If EXTRA-FLAGS is defined, it is added to the linker's default flags +# when the check is done. The check is thus made with the flags: "LDFLAGS +# EXTRA-FLAGS FLAG". This can for example be used to force the linker to +# issue an error when a bad flag is given. +# +# INPUT gives an alternative input source to AC_LINK_IFELSE. +# +# NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this +# macro in sync with AX_CHECK_{PREPROC,COMPILE}_FLAG. +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim <guidod@gmx.de> +# Copyright (c) 2011 Maarten Bosmans <mkbosmans@gmail.com> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 6 + +AC_DEFUN([AX_CHECK_LINK_FLAG], +[AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF +AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_ldflags_$4_$1])dnl +AC_CACHE_CHECK([whether the linker accepts $1], CACHEVAR, [ + ax_check_save_flags=$LDFLAGS + LDFLAGS="$LDFLAGS $4 $1" + AC_LINK_IFELSE([m4_default([$5],[AC_LANG_PROGRAM()])], + [AS_VAR_SET(CACHEVAR,[yes])], + [AS_VAR_SET(CACHEVAR,[no])]) + LDFLAGS=$ax_check_save_flags]) +AS_VAR_IF(CACHEVAR,yes, + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_LINK_FLAGS diff --git a/src/minisketch/build-aux/m4/ax_check_preproc_flag.m4 b/src/minisketch/build-aux/m4/ax_check_preproc_flag.m4 new file mode 100644 index 0000000000..e43560fbd3 --- /dev/null +++ b/src/minisketch/build-aux/m4/ax_check_preproc_flag.m4 @@ -0,0 +1,53 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_check_preproc_flag.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_PREPROC_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS], [INPUT]) +# +# DESCRIPTION +# +# Check whether the given FLAG works with the current language's +# preprocessor or gives an error. (Warnings, however, are ignored) +# +# ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on +# success/failure. +# +# If EXTRA-FLAGS is defined, it is added to the preprocessor's default +# flags when the check is done. The check is thus made with the flags: +# "CPPFLAGS EXTRA-FLAGS FLAG". This can for example be used to force the +# preprocessor to issue an error when a bad flag is given. +# +# INPUT gives an alternative input source to AC_PREPROC_IFELSE. +# +# NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this +# macro in sync with AX_CHECK_{COMPILE,LINK}_FLAG. +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim <guidod@gmx.de> +# Copyright (c) 2011 Maarten Bosmans <mkbosmans@gmail.com> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 6 + +AC_DEFUN([AX_CHECK_PREPROC_FLAG], +[AC_PREREQ(2.64)dnl for _AC_LANG_PREFIX and AS_VAR_IF +AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]cppflags_$4_$1])dnl +AC_CACHE_CHECK([whether _AC_LANG preprocessor accepts $1], CACHEVAR, [ + ax_check_save_flags=$CPPFLAGS + CPPFLAGS="$CPPFLAGS $4 $1" + AC_PREPROC_IFELSE([m4_default([$5],[AC_LANG_PROGRAM()])], + [AS_VAR_SET(CACHEVAR,[yes])], + [AS_VAR_SET(CACHEVAR,[no])]) + CPPFLAGS=$ax_check_save_flags]) +AS_VAR_IF(CACHEVAR,yes, + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_PREPROC_FLAGS diff --git a/src/minisketch/build-aux/m4/ax_cxx_compile_stdcxx.m4 b/src/minisketch/build-aux/m4/ax_cxx_compile_stdcxx.m4 new file mode 100644 index 0000000000..f7e5137003 --- /dev/null +++ b/src/minisketch/build-aux/m4/ax_cxx_compile_stdcxx.m4 @@ -0,0 +1,962 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX(VERSION, [ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the specified +# version of the C++ standard. If necessary, add switches to CXX and +# CXXCPP to enable support. VERSION may be '11' (for the C++11 standard) +# or '14' (for the C++14 standard). +# +# The second argument, if specified, indicates whether you insist on an +# extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g. +# -std=c++11). If neither is specified, you get whatever works, with +# preference for no added switch, and then for an extended mode. +# +# The third argument, if specified 'mandatory' or if left unspecified, +# indicates that baseline support for the specified C++ standard is +# required and that the macro should error out if no mode with that +# support is found. If specified 'optional', then configuration proceeds +# regardless, after defining HAVE_CXX${VERSION} if and only if a +# supporting mode is found. +# +# LICENSE +# +# Copyright (c) 2008 Benjamin Kosnik <bkoz@redhat.com> +# Copyright (c) 2012 Zack Weinberg <zackw@panix.com> +# Copyright (c) 2013 Roy Stogner <roystgnr@ices.utexas.edu> +# Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov <sokolov@google.com> +# Copyright (c) 2015 Paul Norman <penorman@mac.com> +# Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu> +# Copyright (c) 2016, 2018 Krzesimir Nowak <qdlacz@gmail.com> +# Copyright (c) 2019 Enji Cooper <yaneurabeya@gmail.com> +# Copyright (c) 2020 Jason Merrill <jason@redhat.com> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 12 + +dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro +dnl (serial version number 13). + +AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl + m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"], + [$1], [14], [ax_cxx_compile_alternatives="14 1y"], + [$1], [17], [ax_cxx_compile_alternatives="17 1z"], + [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$2], [], [], + [$2], [ext], [], + [$2], [noext], [], + [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$3], [], [ax_cxx_compile_cxx$1_required=true], + [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true], + [$3], [optional], [ax_cxx_compile_cxx$1_required=false], + [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])]) + AC_LANG_PUSH([C++])dnl + ac_success=no + + m4_if([$2], [], [dnl + AC_CACHE_CHECK(whether $CXX supports C++$1 features by default, + ax_cv_cxx_compile_cxx$1, + [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [ax_cv_cxx_compile_cxx$1=yes], + [ax_cv_cxx_compile_cxx$1=no])]) + if test x$ax_cv_cxx_compile_cxx$1 = xyes; then + ac_success=yes + fi]) + + m4_if([$2], [noext], [], [dnl + if test x$ac_success = xno; then + for alternative in ${ax_cxx_compile_alternatives}; do + switch="-std=gnu++${alternative}" + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + fi]) + + m4_if([$2], [ext], [], [dnl + if test x$ac_success = xno; then + dnl HP's aCC needs +std=c++11 according to: + dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf + dnl Cray's crayCC needs "-h std=c++11" + for alternative in ${ax_cxx_compile_alternatives}; do + for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}"; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + if test x$ac_success = xyes; then + break + fi + done + fi]) + AC_LANG_POP([C++]) + if test x$ax_cxx_compile_cxx$1_required = xtrue; then + if test x$ac_success = xno; then + AC_MSG_ERROR([*** A compiler with support for C++$1 language features is required.]) + fi + fi + if test x$ac_success = xno; then + HAVE_CXX$1=0 + AC_MSG_NOTICE([No compiler with C++$1 support was found]) + else + HAVE_CXX$1=1 + AC_DEFINE(HAVE_CXX$1,1, + [define if the compiler supports basic C++$1 syntax]) + fi + AC_SUBST(HAVE_CXX$1) +]) + + +dnl Test body for checking C++11 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 +) + + +dnl Test body for checking C++14 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 +) + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_17 +) + +dnl Tests for new features in C++11 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[ + +// If the compiler admits that it is not ready for C++11, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201103L + +#error "This is not a C++11 compiler" + +#else + +namespace cxx11 +{ + + namespace test_static_assert + { + + template <typename T> + struct check + { + static_assert(sizeof(int) <= sizeof(T), "not big enough"); + }; + + } + + namespace test_final_override + { + + struct Base + { + virtual ~Base() {} + virtual void f() {} + }; + + struct Derived : public Base + { + virtual ~Derived() override {} + virtual void f() override {} + }; + + } + + namespace test_double_right_angle_brackets + { + + template < typename T > + struct check {}; + + typedef check<void> single_type; + typedef check<check<void>> double_type; + typedef check<check<check<void>>> triple_type; + typedef check<check<check<check<void>>>> quadruple_type; + + } + + namespace test_decltype + { + + int + f() + { + int a = 1; + decltype(a) b = 2; + return a + b; + } + + } + + namespace test_type_deduction + { + + template < typename T1, typename T2 > + struct is_same + { + static const bool value = false; + }; + + template < typename T > + struct is_same<T, T> + { + static const bool value = true; + }; + + template < typename T1, typename T2 > + auto + add(T1 a1, T2 a2) -> decltype(a1 + a2) + { + return a1 + a2; + } + + int + test(const int c, volatile int v) + { + static_assert(is_same<int, decltype(0)>::value == true, ""); + static_assert(is_same<int, decltype(c)>::value == false, ""); + static_assert(is_same<int, decltype(v)>::value == false, ""); + auto ac = c; + auto av = v; + auto sumi = ac + av + 'x'; + auto sumf = ac + av + 1.0; + static_assert(is_same<int, decltype(ac)>::value == true, ""); + static_assert(is_same<int, decltype(av)>::value == true, ""); + static_assert(is_same<int, decltype(sumi)>::value == true, ""); + static_assert(is_same<int, decltype(sumf)>::value == false, ""); + static_assert(is_same<int, decltype(add(c, v))>::value == true, ""); + return (sumf > 0.0) ? sumi : add(c, v); + } + + } + + namespace test_noexcept + { + + int f() { return 0; } + int g() noexcept { return 0; } + + static_assert(noexcept(f()) == false, ""); + static_assert(noexcept(g()) == true, ""); + + } + + namespace test_constexpr + { + + template < typename CharT > + unsigned long constexpr + strlen_c_r(const CharT *const s, const unsigned long acc) noexcept + { + return *s ? strlen_c_r(s + 1, acc + 1) : acc; + } + + template < typename CharT > + unsigned long constexpr + strlen_c(const CharT *const s) noexcept + { + return strlen_c_r(s, 0UL); + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("1") == 1UL, ""); + static_assert(strlen_c("example") == 7UL, ""); + static_assert(strlen_c("another\0example") == 7UL, ""); + + } + + namespace test_rvalue_references + { + + template < int N > + struct answer + { + static constexpr int value = N; + }; + + answer<1> f(int&) { return answer<1>(); } + answer<2> f(const int&) { return answer<2>(); } + answer<3> f(int&&) { return answer<3>(); } + + void + test() + { + int i = 0; + const int c = 0; + static_assert(decltype(f(i))::value == 1, ""); + static_assert(decltype(f(c))::value == 2, ""); + static_assert(decltype(f(0))::value == 3, ""); + } + + } + + namespace test_uniform_initialization + { + + struct test + { + static const int zero {}; + static const int one {1}; + }; + + static_assert(test::zero == 0, ""); + static_assert(test::one == 1, ""); + + } + + namespace test_lambdas + { + + void + test1() + { + auto lambda1 = [](){}; + auto lambda2 = lambda1; + lambda1(); + lambda2(); + } + + int + test2() + { + auto a = [](int i, int j){ return i + j; }(1, 2); + auto b = []() -> int { return '0'; }(); + auto c = [=](){ return a + b; }(); + auto d = [&](){ return c; }(); + auto e = [a, &b](int x) mutable { + const auto identity = [](int y){ return y; }; + for (auto i = 0; i < a; ++i) + a += b--; + return x + identity(a + b); + }(0); + return a + b + c + d + e; + } + + int + test3() + { + const auto nullary = [](){ return 0; }; + const auto unary = [](int x){ return x; }; + using nullary_t = decltype(nullary); + using unary_t = decltype(unary); + const auto higher1st = [](nullary_t f){ return f(); }; + const auto higher2nd = [unary](nullary_t f1){ + return [unary, f1](unary_t f2){ return f2(unary(f1())); }; + }; + return higher1st(nullary) + higher2nd(nullary)(unary); + } + + } + + namespace test_variadic_templates + { + + template <int...> + struct sum; + + template <int N0, int... N1toN> + struct sum<N0, N1toN...> + { + static constexpr auto value = N0 + sum<N1toN...>::value; + }; + + template <> + struct sum<> + { + static constexpr auto value = 0; + }; + + static_assert(sum<>::value == 0, ""); + static_assert(sum<1>::value == 1, ""); + static_assert(sum<23>::value == 23, ""); + static_assert(sum<1, 2>::value == 3, ""); + static_assert(sum<5, 5, 11>::value == 21, ""); + static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, ""); + + } + + // http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae + // Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function + // because of this. + namespace test_template_alias_sfinae + { + + struct foo {}; + + template<typename T> + using member = typename T::member_type; + + template<typename T> + void func(...) {} + + template<typename T> + void func(member<T>*) {} + + void test(); + + void test() { func<foo>(0); } + + } + +} // namespace cxx11 + +#endif // __cplusplus >= 201103L + +]]) + + +dnl Tests for new features in C++14 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_14], [[ + +// If the compiler admits that it is not ready for C++14, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201402L + +#error "This is not a C++14 compiler" + +#else + +namespace cxx14 +{ + + namespace test_polymorphic_lambdas + { + + int + test() + { + const auto lambda = [](auto&&... args){ + const auto istiny = [](auto x){ + return (sizeof(x) == 1UL) ? 1 : 0; + }; + const int aretiny[] = { istiny(args)... }; + return aretiny[0]; + }; + return lambda(1, 1L, 1.0f, '1'); + } + + } + + namespace test_binary_literals + { + + constexpr auto ivii = 0b0000000000101010; + static_assert(ivii == 42, "wrong value"); + + } + + namespace test_generalized_constexpr + { + + template < typename CharT > + constexpr unsigned long + strlen_c(const CharT *const s) noexcept + { + auto length = 0UL; + for (auto p = s; *p; ++p) + ++length; + return length; + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("x") == 1UL, ""); + static_assert(strlen_c("test") == 4UL, ""); + static_assert(strlen_c("another\0test") == 7UL, ""); + + } + + namespace test_lambda_init_capture + { + + int + test() + { + auto x = 0; + const auto lambda1 = [a = x](int b){ return a + b; }; + const auto lambda2 = [a = lambda1(x)](){ return a; }; + return lambda2(); + } + + } + + namespace test_digit_separators + { + + constexpr auto ten_million = 100'000'000; + static_assert(ten_million == 100000000, ""); + + } + + namespace test_return_type_deduction + { + + auto f(int& x) { return x; } + decltype(auto) g(int& x) { return x; } + + template < typename T1, typename T2 > + struct is_same + { + static constexpr auto value = false; + }; + + template < typename T > + struct is_same<T, T> + { + static constexpr auto value = true; + }; + + int + test() + { + auto x = 0; + static_assert(is_same<int, decltype(f(x))>::value, ""); + static_assert(is_same<int&, decltype(g(x))>::value, ""); + return x; + } + + } + +} // namespace cxx14 + +#endif // __cplusplus >= 201402L + +]]) + + +dnl Tests for new features in C++17 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_17], [[ + +// If the compiler admits that it is not ready for C++17, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201703L + +#error "This is not a C++17 compiler" + +#else + +#include <initializer_list> +#include <utility> +#include <type_traits> + +namespace cxx17 +{ + + namespace test_constexpr_lambdas + { + + constexpr int foo = [](){return 42;}(); + + } + + namespace test::nested_namespace::definitions + { + + } + + namespace test_fold_expression + { + + template<typename... Args> + int multiply(Args... args) + { + return (args * ... * 1); + } + + template<typename... Args> + bool all(Args... args) + { + return (args && ...); + } + + } + + namespace test_extended_static_assert + { + + static_assert (true); + + } + + namespace test_auto_brace_init_list + { + + auto foo = {5}; + auto bar {5}; + + static_assert(std::is_same<std::initializer_list<int>, decltype(foo)>::value); + static_assert(std::is_same<int, decltype(bar)>::value); + } + + namespace test_typename_in_template_template_parameter + { + + template<template<typename> typename X> struct D; + + } + + namespace test_fallthrough_nodiscard_maybe_unused_attributes + { + + int f1() + { + return 42; + } + + [[nodiscard]] int f2() + { + [[maybe_unused]] auto unused = f1(); + + switch (f1()) + { + case 17: + f1(); + [[fallthrough]]; + case 42: + f1(); + } + return f1(); + } + + } + + namespace test_extended_aggregate_initialization + { + + struct base1 + { + int b1, b2 = 42; + }; + + struct base2 + { + base2() { + b3 = 42; + } + int b3; + }; + + struct derived : base1, base2 + { + int d; + }; + + derived d1 {{1, 2}, {}, 4}; // full initialization + derived d2 {{}, {}, 4}; // value-initialized bases + + } + + namespace test_general_range_based_for_loop + { + + struct iter + { + int i; + + int& operator* () + { + return i; + } + + const int& operator* () const + { + return i; + } + + iter& operator++() + { + ++i; + return *this; + } + }; + + struct sentinel + { + int i; + }; + + bool operator== (const iter& i, const sentinel& s) + { + return i.i == s.i; + } + + bool operator!= (const iter& i, const sentinel& s) + { + return !(i == s); + } + + struct range + { + iter begin() const + { + return {0}; + } + + sentinel end() const + { + return {5}; + } + }; + + void f() + { + range r {}; + + for (auto i : r) + { + [[maybe_unused]] auto v = i; + } + } + + } + + namespace test_lambda_capture_asterisk_this_by_value + { + + struct t + { + int i; + int foo() + { + return [*this]() + { + return i; + }(); + } + }; + + } + + namespace test_enum_class_construction + { + + enum class byte : unsigned char + {}; + + byte foo {42}; + + } + + namespace test_constexpr_if + { + + template <bool cond> + int f () + { + if constexpr(cond) + { + return 13; + } + else + { + return 42; + } + } + + } + + namespace test_selection_statement_with_initializer + { + + int f() + { + return 13; + } + + int f2() + { + if (auto i = f(); i > 0) + { + return 3; + } + + switch (auto i = f(); i + 4) + { + case 17: + return 2; + + default: + return 1; + } + } + + } + + namespace test_template_argument_deduction_for_class_templates + { + + template <typename T1, typename T2> + struct pair + { + pair (T1 p1, T2 p2) + : m1 {p1}, + m2 {p2} + {} + + T1 m1; + T2 m2; + }; + + void f() + { + [[maybe_unused]] auto p = pair{13, 42u}; + } + + } + + namespace test_non_type_auto_template_parameters + { + + template <auto n> + struct B + {}; + + B<5> b1; + B<'a'> b2; + + } + + namespace test_structured_bindings + { + + int arr[2] = { 1, 2 }; + std::pair<int, int> pr = { 1, 2 }; + + auto f1() -> int(&)[2] + { + return arr; + } + + auto f2() -> std::pair<int, int>& + { + return pr; + } + + struct S + { + int x1 : 2; + volatile double y1; + }; + + S f3() + { + return {}; + } + + auto [ x1, y1 ] = f1(); + auto& [ xr1, yr1 ] = f1(); + auto [ x2, y2 ] = f2(); + auto& [ xr2, yr2 ] = f2(); + const auto [ x3, y3 ] = f3(); + + } + + namespace test_exception_spec_type_system + { + + struct Good {}; + struct Bad {}; + + void g1() noexcept; + void g2(); + + template<typename T> + Bad + f(T*, T*); + + template<typename T1, typename T2> + Good + f(T1*, T2*); + + static_assert (std::is_same_v<Good, decltype(f(g1, g2))>); + + } + + namespace test_inline_variables + { + + template<class T> void f(T) + {} + + template<class T> inline T g(T) + { + return T{}; + } + + template<> inline void f<>(int) + {} + + template<> int g<>(int) + { + return 5; + } + + } + +} // namespace cxx17 + +#endif // __cplusplus < 201703L + +]]) diff --git a/src/minisketch/ci/cirrus.sh b/src/minisketch/ci/cirrus.sh new file mode 100755 index 0000000000..02f737ca7f --- /dev/null +++ b/src/minisketch/ci/cirrus.sh @@ -0,0 +1,41 @@ +#!/bin/sh + +set -e +set -x + +export LC_ALL=C + +env >> test_env.log + +$CC -v || true +valgrind --version || true + +./autogen.sh + +FIELDS= +if [ -n "$ENABLE_FIELDS" ]; then + FIELDS="--enable-fields=$ENABLE_FIELDS" +fi +./configure --host="$HOST" --enable-benchmark="$BENCH" $FIELDS + +# We have set "-j<n>" in MAKEFLAGS. +make + +# Print information about binaries so that we can see that the architecture is correct +file test* || true +file bench* || true +file .libs/* || true + +if [ -n "$BUILD" ] +then + make "$BUILD" +fi + +if [ -n "$EXEC_CMD" ]; then + $EXEC_CMD ./test $TESTRUNS + $EXEC_CMD ./test-verify $TESTRUNS +fi + +if [ "$BENCH" = "yes" ]; then + $EXEC_CMD ./bench +fi diff --git a/src/minisketch/ci/linux-debian.Dockerfile b/src/minisketch/ci/linux-debian.Dockerfile new file mode 100644 index 0000000000..63e5412ee7 --- /dev/null +++ b/src/minisketch/ci/linux-debian.Dockerfile @@ -0,0 +1,17 @@ +FROM debian:stable + +RUN dpkg --add-architecture i386 +RUN dpkg --add-architecture s390x +RUN apt-get update + +# dkpg-dev: to make pkg-config work in cross-builds +RUN apt-get install --no-install-recommends --no-upgrade -y \ + git ca-certificates \ + make automake libtool pkg-config dpkg-dev valgrind qemu-user \ + gcc g++ clang libc6-dbg \ + gcc-i686-linux-gnu g++-i686-linux-gnu libc6-dev-i386-cross libc6-dbg:i386 \ + g++-s390x-linux-gnu gcc-s390x-linux-gnu libc6-dev-s390x-cross libc6-dbg:s390x \ + wine g++-mingw-w64-x86-64 + +# Run a dummy command in wine to make it set up configuration +RUN wine true || true diff --git a/src/minisketch/configure.ac b/src/minisketch/configure.ac new file mode 100644 index 0000000000..9dc66e7fd2 --- /dev/null +++ b/src/minisketch/configure.ac @@ -0,0 +1,162 @@ +AC_INIT([minisketch], [0.0.1], [http://github.com/sipa/minisketch/]) + +m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) + +AC_PREREQ(2.60) +AC_CONFIG_SRCDIR([src/minisketch.cpp]) +AC_CONFIG_AUX_DIR([build-aux]) +AC_CONFIG_MACRO_DIR([build-aux/m4]) +AM_INIT_AUTOMAKE([subdir-objects foreign]) + +LT_INIT +LT_LANG([C++]) +AC_LANG([C++]) + +AC_PATH_PROG(CCACHE,ccache) + +AC_ARG_ENABLE([ccache], + [AS_HELP_STRING([--disable-ccache], + [do not use ccache for building (default is to use if found)])], + [use_ccache=$enableval], + [use_ccache=auto]) + +AC_ARG_ENABLE(tests, + AS_HELP_STRING([--enable-tests],[compile tests (default is yes)]), + [use_tests=$enableval], + [use_tests=yes]) + +AC_ARG_ENABLE(benchmark, + AS_HELP_STRING([--enable-benchmark],[compile benchmark (default is no)]), + [use_benchmark=$enableval], + [use_benchmark=no]) + +m4_define([SUPPORTED_FIELDS], [2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64]) + +AC_MSG_CHECKING([which field sizes to build]) +AC_ARG_ENABLE([fields], AS_HELP_STRING([--enable-fields=LIST], [Comma-separated list of field sizes to build. Default=all. Available sizes:] m4_translit(m4_defn([SUPPORTED_FIELDS]), [,], [ ])), [], [enable_fields=SUPPORTED_FIELDS]) +have_disabled_fields=no +have_enabled_fields=no +m4_foreach([FIELD], [SUPPORTED_FIELDS], [ + case ",$enable_fields," in + *,FIELD,*) + have_enabled_fields=yes + ;; + *) + AC_DEFINE(DISABLE_FIELD_[]FIELD, [1], + [Define to 1 to remove support for field size] FIELD [.]) + have_disabled_fields=yes + ;; + esac +]) +AC_MSG_RESULT([$enable_fields]) +if test "x$have_enabled_fields" = xno; then + AC_MSG_ERROR([No field sizes are enabled.]) +fi + +AX_CHECK_COMPILE_FLAG([-Werror],[CXXFLAG_WERROR="-Werror"],[CXXFLAG_WERROR=""]) + +AX_CXX_COMPILE_STDCXX([11], [noext], [mandatory]) +enable_clmul= +AX_CHECK_COMPILE_FLAG([-mpclmul],[[enable_clmul=yes]],,[[$CXXFLAG_WERROR]],[AC_LANG_PROGRAM([ + #include <stdint.h> + #include <x86intrin.h> +], [ + __m128i a = _mm_cvtsi64_si128((uint64_t)7); + __m128i b = _mm_clmulepi64_si128(a, a, 37); + __m128i c = _mm_srli_epi64(b, 41); + __m128i d = _mm_xor_si128(b, c); + uint64_t e = _mm_cvtsi128_si64(d); + return e == 0; +])]) +if test x$enable_clmul = xyes; then + CLMUL_CXXFLAGS="-mpclmul" + AC_DEFINE(HAVE_CLMUL, 1, [Define this symbol if clmul instructions can be used]) +fi + + +AC_MSG_CHECKING(for working clz builtins) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [ + unsigned a = __builtin_clz(1); + unsigned long b = __builtin_clzl(1); + unsigned long long c = __builtin_clzll(1); + ])], + [ + AC_DEFINE(HAVE_CLZ, 1, [Define this symbol if clz builtins are present and working]) + AC_MSG_RESULT(yes) + ],[ + AC_MSG_RESULT(no) + ] +) + +AX_CHECK_LINK_FLAG([[-Wl,--exclude-libs,ALL]],[LDFLAGS="-Wl,--exclude-libs,ALL $LDFLAGS"]) + +case $host in + *mingw*) + dnl -static is interpreted by libtool, where it has a different meaning. + dnl In libtool-speak, it's -all-static. + AX_CHECK_LINK_FLAG([[-static]],[LIBTOOL_APP_LDFLAGS="$LIBTOOL_APP_LDFLAGS -all-static"]) + ;; + *) + AX_CHECK_LINK_FLAG([[-static]],[LIBTOOL_APP_LDFLAGS="-static"]) + ;; +esac + +AX_CHECK_COMPILE_FLAG([-Wall],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"],,[[$CXXFLAG_WERROR]]) +AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[CXXFLAGS="$CXXFLAGS -fvisibility=hidden"],[],[$CXXFLAG_WERROR]) + +## Some compilers (gcc) ignore unknown -Wno-* options, but warn about all +## unknown options if any other warning is produced. Test the -Wfoo case, and +## set the -Wno-foo case if it works. +AX_CHECK_COMPILE_FLAG([-Wshift-count-overflow],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-shift-count-overflow"],,[[$CXXFLAG_WERROR]]) + +if test "x$use_ccache" != "xno"; then + AC_MSG_CHECKING(if ccache should be used) + if test x$CCACHE = x; then + if test "x$use_ccache" = "xyes"; then + AC_MSG_ERROR([ccache not found.]); + else + use_ccache=no + fi + else + use_ccache=yes + CC="$ac_cv_path_CCACHE $CC" + CXX="$ac_cv_path_CCACHE $CXX" + fi + AC_MSG_RESULT($use_ccache) +fi +if test "x$use_ccache" = "xyes"; then + AX_CHECK_COMPILE_FLAG([-Qunused-arguments],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Qunused-arguments"],,[[$CXXFLAG_WERROR]]) +fi + +VERIFY_DEFINES=-DMINISKETCH_VERIFY +RELEASE_DEFINES= + +AC_CONFIG_FILES([ + Makefile +]) + +AC_SUBST(CLMUL_CXXFLAGS) +AC_SUBST(WARN_CXXFLAGS) +AC_SUBST(NOWARN_CXXFLAGS) +AC_SUBST(VERIFY_DEFINES) +AC_SUBST(RELEASE_DEFINES) +AC_SUBST(LIBTOOL_APP_LDFLAGS) +AM_CONDITIONAL([ENABLE_CLMUL],[test x$enable_clmul = xyes]) +AM_CONDITIONAL([USE_BENCHMARK], [test x"$use_benchmark" = x"yes"]) +AM_CONDITIONAL([USE_TESTS], [test x"$use_tests" != x"no"]) +AC_OUTPUT + +echo +echo "Build Options:" +echo " with benchmarks = $use_benchmark" +echo " with tests = $use_tests" +echo " enable clmul fields = $enable_clmul" +echo " CXX = $CXX" +echo " CXXFLAGS = $CXXFLAGS" +echo " CPPFLAGS = $CPPFLAGS" +echo " LDFLAGS = $LDFLAGS" +if test "$have_disabled_fields" = "yes"; then +echo +echo "Only compiling in support for field sizes: $enable_fields" +echo "WARNING: this means the library will lack support for other field sizes entirely" +fi diff --git a/src/minisketch/doc/example.c b/src/minisketch/doc/example.c new file mode 100644 index 0000000000..7279165845 --- /dev/null +++ b/src/minisketch/doc/example.c @@ -0,0 +1,51 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#include <stdio.h> +#include <assert.h> +#include "../include/minisketch.h" + +int main(void) { + + minisketch *sketch_a = minisketch_create(12, 0, 4); + + for (int i = 3000; i < 3010; ++i) { + minisketch_add_uint64(sketch_a, i); + } + + size_t sersize = minisketch_serialized_size(sketch_a); + assert(sersize == 12 * 4 / 8); // 4 12-bit values is 6 bytes. + unsigned char *buffer_a = malloc(sersize); + minisketch_serialize(sketch_a, buffer_a); + minisketch_destroy(sketch_a); + + minisketch *sketch_b = minisketch_create(12, 0, 4); // Bob's own sketch + for (int i = 3002; i < 3012; ++i) { + minisketch_add_uint64(sketch_b, i); + } + + sketch_a = minisketch_create(12, 0, 4); // Alice's sketch + minisketch_deserialize(sketch_a, buffer_a); // Load Alice's sketch + free(buffer_a); + + // Merge the elements from sketch_a into sketch_b. The result is a sketch_b + // which contains all elements that occurred in Alice's or Bob's sets, but not + // in both. + minisketch_merge(sketch_b, sketch_a); + + uint64_t differences[4]; + ssize_t num_differences = minisketch_decode(sketch_b, 4, differences); + minisketch_destroy(sketch_a); + minisketch_destroy(sketch_b); + if (num_differences < 0) { + printf("More than 4 differences!\n"); + } else { + ssize_t i; + for (i = 0; i < num_differences; ++i) { + printf("%u is in only one of the two sets\n", (unsigned)differences[i]); + } + } +} diff --git a/src/minisketch/doc/gen_basefpbits.sage b/src/minisketch/doc/gen_basefpbits.sage new file mode 100644 index 0000000000..d1e75a6e29 --- /dev/null +++ b/src/minisketch/doc/gen_basefpbits.sage @@ -0,0 +1,78 @@ +# Require exact values up to +FPBITS = 256 + +# Overkill accuracy +F = RealField(400) + +def BaseFPBits(bits, capacity): + return bits * capacity - int(ceil(F(log(sum(binomial(2**bits - 1, i) for i in range(capacity+1)), 2)))) + +def Log2Factorial(capacity): + return int(floor(log(factorial(capacity), 2))) + +print("uint64_t BaseFPBits(uint32_t bits, uint32_t capacity) {") +print(" // Correction table for low bits/capacities") +TBLS={} +FARS={} +SKIPS={} +for bits in range(1, 32): + TBL = [] + for capacity in range(1, min(2**bits, FPBITS)): + exact = BaseFPBits(bits, capacity) + approx = Log2Factorial(capacity) + TBL.append((exact, approx)) + MIN = 10000000000 + while len(TBL) and ((TBL[-1][0] == TBL[-1][1]) or (TBL[-1][0] >= FPBITS and TBL[-1][1] >= FPBITS)): + MIN = min(MIN, TBL[-1][0] - TBL[-1][1]) + TBL.pop() + while len(TBL) and (TBL[-1][0] - TBL[-1][1] == MIN): + TBL.pop() + SKIP = 0 + while SKIP < len(TBL) and TBL[SKIP][0] == TBL[SKIP][1]: + SKIP += 1 + DIFFS = [TBL[i][0] - TBL[i][1] for i in range(SKIP, len(TBL))] + if len(DIFFS) > 0 and len(DIFFS) * Integer(max(DIFFS)).nbits() > 64: + print(" static constexpr uint8_t ADD%i[] = {%s};" % (bits, ", ".join(("%i" % (TBL[i][0] - TBL[i][1])) for i in range(SKIP, len(TBL))))) + TBLS[bits] = DIFFS + FARS[bits] = MIN + SKIPS[bits] = SKIP +print("") +print(" if (capacity == 0) return 0;") +print(" uint64_t ret = 0;") +print(" if (bits < 32 && capacity >= (1U << bits)) {") +print(" ret = uint64_t{bits} * (capacity - (1U << bits) + 1);") +print(" capacity = (1U << bits) - 1;") +print(" }") +print(" ret += Log2Factorial(capacity);") +print(" switch (bits) {") +for bits in sorted(TBLS.keys()): + if len(TBLS[bits]) == 0: + continue + width = Integer(max(TBLS[bits])).nbits() + if len(TBLS[bits]) == 1: + add = "%i" % TBLS[bits][0] + elif len(TBLS[bits]) * width <= 64: + code = sum((2**(width*i) * TBLS[bits][i]) for i in range(len(TBLS[bits]))) + if width == 1: + add = "(0x%x >> (capacity - %i)) & 1" % (code, 1 + SKIPS[bits]) + else: + add = "(0x%x >> %i * (capacity - %i)) & %i" % (code, width, 1 + SKIPS[bits], 2**width - 1) + else: + add = "ADD%i[capacity - %i]" % (bits, 1 + SKIPS[bits]) + if len(TBLS[bits]) + SKIPS[bits] == 2**bits - 1: + print(" case %i: return ret + (capacity <= %i ? 0 : %s);" % (bits, SKIPS[bits], add)) + else: + print(" case %i: return ret + (capacity <= %i ? 0 : capacity > %i ? %i : %s);" % (bits, SKIPS[bits], len(TBLS[bits]) + SKIPS[bits], FARS[bits], add)) +print(" default: return ret;") +print(" }") +print("}") + +print("void TestBaseFPBits() {") +print(" static constexpr uint16_t TBL[20][100] = {%s};" % (", ".join("{" + ", ".join(("%i" % BaseFPBits(bits, capacity)) for capacity in range(0, 100)) + "}" for bits in range(1, 21)))) +print(" for (int bits = 1; bits <= 20; ++bits) {") +print(" for (int capacity = 0; capacity < 100; ++capacity) {") +print(" uint64_t computed = BaseFPBits(bits, capacity), exact = TBL[bits - 1][capacity];") +print(" CHECK(exact == computed || (exact >= 256 && computed >= 256));") +print(" }") +print(" }") +print("}") diff --git a/src/minisketch/doc/gen_params.sage b/src/minisketch/doc/gen_params.sage new file mode 100755 index 0000000000..1cf036adb4 --- /dev/null +++ b/src/minisketch/doc/gen_params.sage @@ -0,0 +1,333 @@ +#!/usr/bin/env sage +r""" +Generate finite field parameters for minisketch. + +This script selects the finite fields used by minisketch + for various sizes and generates the required tables for + the implementation. + +The output (after formatting) can be found in src/fields/*.cpp. + +""" +B.<b> = GF(2) +P.<p> = B[] + +def apply_map(m, v): + r = 0 + i = 0 + while v != 0: + if (v & 1): + r ^^= m[i] + i += 1 + v >>= 1 + return r + +def recurse_moduli(acc, maxweight, maxdegree): + for pos in range(maxweight, maxdegree + 1, 1): + poly = acc + p^pos + if maxweight == 1: + if poly.is_irreducible(): + return (pos, poly) + else: + (deg, ret) = recurse_moduli(poly, maxweight - 1, pos - 1) + if ret is not None: + return (pos, ret) + return (None, None) + +def compute_moduli(bits): + # Return all optimal irreducible polynomials for GF(2^bits) + # The result is a list of tuples (weight, degree of second-highest nonzero coefficient, polynomial) + maxdegree = bits - 1 + result = [] + for weight in range(1, bits, 2): + deg, res = None, None + while True: + ret = recurse_moduli(p^bits + 1, weight, maxdegree) + if ret[0] is not None: + (deg, res) = ret + maxdegree = deg - 1 + else: + break + if res is not None: + result.append((weight + 2, deg, res)) + return result + +def bits_to_int(vals): + ret = 0 + base = 1 + for val in vals: + ret += Integer(val) * base + base *= 2 + return ret + +def sqr_table(f, bits, n=1): + ret = [] + for i in range(bits): + ret.append((f^(2^n*i)).integer_representation()) + return ret + +# Compute x**(2**n) +def pow2(x, n): + for i in range(n): + x = x**2 + return x + +def qrt_table(F, f, bits): + # Table for solving x2 + x = a + # This implements the technique from https://www.raco.cat/index.php/PublicacionsMatematiques/article/viewFile/37927/40412, Lemma 1 + for i in range(bits): + if (f**i).trace() != 0: + u = f**i + ret = [] + for i in range(0, bits): + d = f^i + y = sum(pow2(d, j) * sum(pow2(u, k) for k in range(j)) for j in range(1, bits)) + ret.append(y.integer_representation() ^^ (y.integer_representation() & 1)) + return ret + +def conv_tables(F, NF, bits): + # Generate a F(2) linear projection that maps elements from one field + # to an isomorphic field with a different modulus. + f = F.gen() + fp = f.minimal_polynomial() + assert(fp == F.modulus()) + nfp = fp.change_ring(NF) + nf = sorted(nfp.roots(multiplicities=False))[0] + ret = [] + matrepr = [[B(0) for x in range(bits)] for y in range(bits)] + for i in range(bits): + val = (nf**i).integer_representation() + ret.append(val) + for j in range(bits): + matrepr[j][i] = B((val >> j) & 1) + mat = Matrix(matrepr).inverse().transpose() + ret2 = [] + for i in range(bits): + ret2.append(bits_to_int(mat[i])) + + for t in range(100): + f1a = F.random_element() + f1b = F.random_element() + f1r = f1a * f1b + f2a = NF.fetch_int(apply_map(ret, f1a.integer_representation())) + f2b = NF.fetch_int(apply_map(ret, f1b.integer_representation())) + f2r = NF.fetch_int(apply_map(ret, f1r.integer_representation())) + f2s = f2a * f2b + assert(f2r == f2s) + + for t in range(100): + f2a = NF.random_element() + f2b = NF.random_element() + f2r = f2a * f2b + f1a = F.fetch_int(apply_map(ret2, f2a.integer_representation())) + f1b = F.fetch_int(apply_map(ret2, f2b.integer_representation())) + f1r = F.fetch_int(apply_map(ret2, f2r.integer_representation())) + f1s = f1a * f1b + assert(f1r == f1s) + + return (ret, ret2) + +def fmt(i,typ): + if i == 0: + return "0" + else: + return "0x%x" % i + +def lintranstype(typ, bits, maxtbl): + gsize = min(maxtbl, bits) + array_size = (bits + gsize - 1) // gsize + bits_list = [] + total = 0 + for i in range(array_size): + rsize = (bits - total + array_size - i - 1) // (array_size - i) + total += rsize + bits_list.append(rsize) + return "RecLinTrans<%s, %s>" % (typ, ", ".join("%i" % x for x in bits_list)) + +INT=0 +CLMUL=1 +CLMUL_TRI=2 +MD=3 + +def print_modulus_md(mod): + ret = "" + pos = mod.degree() + for c in reversed(list(mod)): + if c: + if ret: + ret += " + " + if pos == 0: + ret += "1" + elif pos == 1: + ret += "x" + else: + ret += "x<sup>%i</sup>" % pos + pos -= 1 + return ret + +def pick_modulus(bits, style): + # Choose the lexicographicly-first lowest-weight modulus + # optionally subject to implementation specific constraints. + moduli = compute_moduli(bits) + if style == INT or style == MD: + multi_sqr = False + need_trans = False + elif style == CLMUL: + # Fast CLMUL reduction requires that bits + the highest + # set bit are less than 66. + moduli = list(filter((lambda x: bits+x[1] <= 66), moduli)) + moduli + multi_sqr = True + need_trans = True + if not moduli or moduli[0][2].change_ring(ZZ)(2) == 3 + 2**bits: + # For modulus 3, CLMUL_TRI is obviously better. + return None + elif style == CLMUL_TRI: + moduli = list(filter(lambda x: bits+x[1] <= 66, moduli)) + moduli + moduli = list(filter(lambda x: x[0] == 3, moduli)) + multi_sqr = True + need_trans = True + else: + assert(False) + if not moduli: + return None + return moduli[0][2] + +def print_result(bits, style): + if style == INT: + multi_sqr = False + need_trans = False + table_id = "%i" % bits + elif style == MD: + pass + elif style == CLMUL: + multi_sqr = True + need_trans = True + table_id = "%i" % bits + elif style == CLMUL_TRI: + multi_sqr = True + need_trans = True + table_id = "TRI%i" % bits + else: + assert(False) + + nmodulus = pick_modulus(bits, INT) + modulus = pick_modulus(bits, style) + if modulus is None: + return + + if style == MD: + print("* *%s*" % print_modulus_md(modulus)) + return + + if bits > 32: + typ = "uint64_t" + elif bits > 16: + typ = "uint32_t" + elif bits > 8: + typ = "uint16_t" + else: + typ = "uint8_t" + + ttyp = lintranstype(typ, bits, 4) + rtyp = lintranstype(typ, bits, 6) + + F.<f> = GF(2**bits, modulus=modulus) + + include_table = True + if style != INT and style != CLMUL: + cmodulus = pick_modulus(bits, CLMUL) + if cmodulus == modulus: + include_table = False + table_id = "%i" % bits + + if include_table: + print("typedef %s StatTable%s;" % (rtyp, table_id)) + rtyp = "StatTable%s" % table_id + if (style == INT): + print("typedef %s DynTable%s;" % (ttyp, table_id)) + ttyp = "DynTable%s" % table_id + + if need_trans: + if modulus != nmodulus: + # If the bitstream modulus is not the best modulus for + # this implementation a conversion table will be needed. + ctyp = rtyp + NF.<nf> = GF(2**bits, modulus=nmodulus) + ctables = conv_tables(NF, F, bits) + loadtbl = "&LOAD_TABLE_%s" % table_id + savetbl = "&SAVE_TABLE_%s" % table_id + if include_table: + print("constexpr %s LOAD_TABLE_%s({%s});" % (ctyp, table_id, ", ".join([fmt(x,typ) for x in ctables[0]]))) + print("constexpr %s SAVE_TABLE_%s({%s});" % (ctyp, table_id, ", ".join([fmt(x,typ) for x in ctables[1]]))) + else: + ctyp = "IdTrans" + loadtbl = "&ID_TRANS" + savetbl = "&ID_TRANS" + else: + assert(modulus == nmodulus) + + if include_table: + print("constexpr %s SQR_TABLE_%s({%s});" % (rtyp, table_id, ", ".join([fmt(x,typ) for x in sqr_table(f, bits, 1)]))) + if multi_sqr: + # Repeated squaring is a linearised polynomial so in F(2^n) it is + # F(2) linear and can be computed by a simple bit-matrix. + # Repeated squaring is especially useful in powering ladders such as + # for inversion. + # When certain repeated squaring tables are not in use, use the QRT + # table instead to make the C++ compiler happy (it always has the + # same type). + sqr2 = "&QRT_TABLE_%s" % table_id + sqr4 = "&QRT_TABLE_%s" % table_id + sqr8 = "&QRT_TABLE_%s" % table_id + sqr16 = "&QRT_TABLE_%s" % table_id + if ((bits - 1) >= 4): + if include_table: + print("constexpr %s SQR2_TABLE_%s({%s});" % (rtyp, table_id, ", ".join([fmt(x,typ) for x in sqr_table(f, bits, 2)]))) + sqr2 = "&SQR2_TABLE_%s" % table_id + if ((bits - 1) >= 8): + if include_table: + print("constexpr %s SQR4_TABLE_%s({%s});" % (rtyp, table_id, ", ".join([fmt(x,typ) for x in sqr_table(f, bits, 4)]))) + sqr4 = "&SQR4_TABLE_%s" % table_id + if ((bits - 1) >= 16): + if include_table: + print("constexpr %s SQR8_TABLE_%s({%s});" % (rtyp, table_id, ", ".join([fmt(x,typ) for x in sqr_table(f, bits, 8)]))) + sqr8 = "&SQR8_TABLE_%s" % table_id + if ((bits - 1) >= 32): + if include_table: + print("constexpr %s SQR16_TABLE_%s({%s});" % (rtyp, table_id, ", ".join([fmt(x,typ) for x in sqr_table(f, bits, 16)]))) + sqr16 = "&SQR16_TABLE_%s" % table_id + if include_table: + print("constexpr %s QRT_TABLE_%s({%s});" % (rtyp, table_id, ", ".join([fmt(x,typ) for x in qrt_table(F, f, bits)]))) + + modulus_weight = modulus.hamming_weight() + modulus_degree = (modulus - p**bits).degree() + modulus_int = (modulus - p**bits).change_ring(ZZ)(2) + + lfsr = "" + + if style == INT: + print("typedef Field<%s, %i, %i, %s, %s, &SQR_TABLE_%s, &QRT_TABLE_%s%s> Field%i;" % (typ, bits, modulus_int, rtyp, ttyp, table_id, table_id, lfsr, bits)) + elif style == CLMUL: + print("typedef Field<%s, %i, %i, %s, &SQR_TABLE_%s, %s, %s, %s, %s, &QRT_TABLE_%s, %s, %s, %s%s> Field%i;" % (typ, bits, modulus_int, rtyp, table_id, sqr2, sqr4, sqr8, sqr16, table_id, ctyp, loadtbl, savetbl, lfsr, bits)) + elif style == CLMUL_TRI: + print("typedef FieldTri<%s, %i, %i, %s, &SQR_TABLE_%s, %s, %s, %s, %s, &QRT_TABLE_%s, %s, %s, %s> FieldTri%i;" % (typ, bits, modulus_degree, rtyp, table_id, sqr2, sqr4, sqr8, sqr16, table_id, ctyp, loadtbl, savetbl, bits)) + else: + assert(False) + +for bits in range(2, 65): + print("#ifdef ENABLE_FIELD_INT_%i" % bits) + print("// %i bit field" % bits) + print_result(bits, INT) + print("#endif") + print("") + +for bits in range(2, 65): + print("#ifdef ENABLE_FIELD_INT_%i" % bits) + print("// %i bit field" % bits) + print_result(bits, CLMUL) + print_result(bits, CLMUL_TRI) + print("#endif") + print("") + +for bits in range(2, 65): + print_result(bits, MD) diff --git a/src/minisketch/doc/log2_factorial.sage b/src/minisketch/doc/log2_factorial.sage new file mode 100644 index 0000000000..afc6d66c57 --- /dev/null +++ b/src/minisketch/doc/log2_factorial.sage @@ -0,0 +1,85 @@ +import bisect + +INPUT_BITS = 32 +TABLE_BITS = 5 +INT_BITS = 64 +EXACT_FPBITS = 256 + +F = RealField(100) # overkill + +def BestOverApproxInvLog2(mulof, maxd): + """ + Compute denominator of an approximation of 1/log(2). + + Specifically, find the value of d (<= maxd, and a multiple of mulof) + such that ceil(d/log(2))/d is the best approximation of 1/log(2). + """ + dist=1 + best=0 + # Precomputed denominators that lead to good approximations of 1/log(2) + for d in [1, 2, 9, 70, 131, 192, 445, 1588, 4319, 11369, 18419, 25469, 287209, 836158, 3057423, 8336111, 21950910, 35565709, 49180508, 161156323, 273132138, 385107953, 882191721]: + kd = lcm(mulof, d) + if kd <= maxd: + n = ceil(kd / log(2)) + dis = F((n / kd) - 1 / log(2)) + if dis < dist: + dist = dis + best = kd + return best + + +LOG2_TABLE = [] +A = 0 +B = 0 +C = 0 +D = 0 +K = 0 + +def Setup(k): + global LOG2_TABLE, A, B, C, D, K + K = k + LOG2_TABLE = [] + for i in range(2 ** TABLE_BITS): + LOG2_TABLE.append(int(floor(F(K * log(1 + i / 2**TABLE_BITS, 2))))) + + # Maximum for (2*x+1)*LogK2(x) + max_T = (2^(INPUT_BITS + 1) - 1) * (INPUT_BITS*K - 1) + # Maximum for A + max_A = (2^INT_BITS - 1) // max_T + D = BestOverApproxInvLog2(2 * K, max_A * 2 * K) + A = D // (2 * K) + B = int(ceil(F(D/log(2)))) + C = int(floor(F(D*log(2*pi,2)/2))) + +def LogK2(n): + assert(n >= 1 and n < (1 << INPUT_BITS)) + bits = Integer(n).nbits() + return K * (bits - 1) + LOG2_TABLE[((n << (INPUT_BITS - bits)) >> (INPUT_BITS - TABLE_BITS - 1)) - 2**TABLE_BITS] + +def Log2Fact(n): + # Use formula (A*(2*x+1)*LogK2(x) - B*x + C) / D + return (A*(2*n+1)*LogK2(n) - B*n + C) // D + (n < 3) + +RES = [int(F(log(factorial(i),2))) for i in range(EXACT_FPBITS * 10)] + +best_worst_ratio = 0 + +for K in range(1, 10000): + Setup(K) + assert(LogK2(1) == 0) + assert(LogK2(2) == K) + assert(LogK2(4) == 2 * K) + good = True + worst_ratio = 1 + for i in range(1, EXACT_FPBITS * 10): + exact = RES[i] + approx = Log2Fact(i) + if not (approx <= exact and ((approx == exact) or (approx >= EXACT_FPBITS and exact >= EXACT_FPBITS))): + good = False + break + if worst_ratio * exact > approx: + worst_ratio = approx / exact + if good and worst_ratio > best_worst_ratio: + best_worst_ratio = worst_ratio + print("Formula: (%i*(2*x+1)*floor(%i*log2(x)) - %i*x + %i) / %i; log(max_ratio)=%f" % (A, K, B, C, D, RR(-log(worst_ratio)))) + print("LOG2K_TABLE: %r" % LOG2_TABLE) diff --git a/src/minisketch/doc/math.md b/src/minisketch/doc/math.md new file mode 100644 index 0000000000..cf46f193ab --- /dev/null +++ b/src/minisketch/doc/math.md @@ -0,0 +1,117 @@ +# The mathematics of Minisketch sketches + +This is an unconventional mathematical overview of the PinSketch algorithm without references to coding theory<sup>[[1]](#myfootnote1)</sup>. + +## Set sketches + +A sketch, for the purpose of this description, can be seen as a "set checksum" with two peculiar properties: + +* Sketches have a predetermined capacity, and when the number of elements in the set is not higher than the capacity, minisketch will always recover the entire set from the sketch. A sketch of *b*-bit elements with capacity *c* can be stored in *bc* bits. +* The sketches of two sets can be combined by adding them (XOR) to obtain a sketch of the [symmetric difference](https://en.wikipedia.org/wiki/Symmetric_difference) between the two sets (*i.e.*, all elements that occur in one but not both input sets). + +This overview explains how sets can be converted into a sketch and how a set can be recovered from a sketch. + +## From field elements to sketches + +**Data entries as field elements** + +Every integer in the range *[1...2<sup>b</sup>-1]* (the acceptable data elements for a Minisketch sketch with field size *b*) can be mapped to a nonzero field element of *GF(2<sup>b</sup>)*. In this [finite field](https://en.wikipedia.org/wiki/Finite_field), we can add and multiply elements together, with many of the expected properties for those operations. Addition (and subtraction!) of field elements corresponds to bitwise XOR of the integers they correspond to, though multiplication is more involved. + +**Sets as power series** + +We define a function *S* which maps field elements *m* to the following [formal power series](https://en.wikipedia.org/wiki/Formal_power_series) (similar to a polynomial, except there can be an infinite number of terms, and we don't care about concepts like convergence as we're never going to actually evaluate it for a specific value of *x*): + +* *S(m) = 1 + mx + m<sup>2</sup>x<sup>2</sup> + m<sup>3</sup>x<sup>3</sup> + ...*. + +We then extend this function to operate on sets of field elements, by adding together the images of every set element. If *M = {m<sub>1</sub>, m<sub>2</sub>, ... }*: + +* *S(M) = S({m<sub>1</sub>,m<sub>2</sub>,...}) = S(m<sub>1</sub>) + S(m<sub>2</sub>) + ... = (1 + 1 + ...) + (m<sub>1</sub> + m<sub>2</sub> + ...)x + (m<sub>1</sub><sup>2</sup> + m<sub>2</sub><sup>2</sup> + ...)x<sup>2</sup> + (m<sub>1</sub><sup>3</sup> + ...* + +Because in our field addition corresponds to XOR of integers, it holds for every *a* that *a + a = 0*. This carries over to the *S* function, meaning that *S(a) + S(a) = 0* for every *a*. This means that the coefficients of these power series have the second of the properties we +desire from a sketch, namely that an efficient operation exists to +combine two sketches such that the result is a sketch of the symmetric +difference of the sets. It holds that +*S({m<sub>1</sub>,m<sub>2</sub>}) + S({m<sub>2</sub>,m<sub>3</sub>}) = S(m<sub>1</sub>) + (S(m<sub>2</sub>) + S(m<sub>2</sub>)) + S(m<sub>3</sub>) = S(m<sub>1</sub>) + S(m<sub>3</sub>) = S({m<sub>1</sub>,m<sub>3</sub>})*. The question is whether we can also efficiently recover the elements from their power series' coefficients. + +**An infinity of coefficients is hard** + +To make reasoning about these power series easier, notice that the series for a single element is in fact a [geometric series](https://en.wikipedia.org/wiki/Geometric_series). If we were working over real numbers rather than a finite field and *|mx| < 1*, it would converge to *(1 - mx)<sup>-1</sup>*. Convergence has no meaning in formal power series, however it is still the case that: + +* *(1 - mx) S(m) = 1* + +You can verify this by seeing that every coefficient except the constant one gets cancelled out by the multiplication. This can be generalized to the series for multiple set elements. For two elements we have: + +* *(1 - m<sub>1</sub>x) (1 - m<sub>2</sub>x) S({m<sub>1</sub>,m<sub>2</sub>}) = (1 - m<sub>1</sub>x) (1 - m<sub>2</sub>x) (S(m<sub>1</sub>) + S(m<sub>2</sub>)) = (1 - m<sub>2</sub>x) + (1 - m<sub>1</sub>x)* + +And for three: + +* *(1 - m<sub>1</sub>x) (1 - m<sub>2</sub>x) (1 - m<sub>3</sub>x) S({m<sub>1</sub>,m<sub>2</sub>,m<sub>3</sub>}) = (1 - m<sub>1</sub>x) (1 - m<sub>2</sub>x) (1 - m<sub>3</sub>x) (S(m<sub>1</sub>) + S(m<sub>2</sub>) + S(m<sub>3</sub>)) = (1 - m<sub>2</sub>x)(1 - m<sub>3</sub>x) + (1 - m<sub>1</sub>x)(1 - m<sub>3</sub>x) + (1 - m<sub>1</sub>x)(1 - m<sub>2</sub>x)* + +In each case, we notice that multiplying *S(M)* with *(1 - m<sub>i</sub>x)* for each element *m<sub>i</sub> ∈ M* results in a polynomial of degree *n-1*. + +**Solving for the set elements** + +The above insight lets us build a solver that extracts the set elements from the coefficients of a power series. If we can find a polynomial *L* that is the product of *n* different *(1 - m<sub>i</sub>x)* factors for various values of *m<sub>i</sub>*, such that *P = S(M)L* is an *n-1* degree polynomial, then those values *m<sub>i</sub>* are the elements of *M*. + +The coefficients of *P* are nontrivial expressions of the set elements themselves. However, we can just focus on the coefficients of degree *n* and higher in *P*, as those are all 0. Let *s<sub>i</sub>* be the coefficients of *S(M)*, and *l<sub>i</sub>* the coefficients of L. In other words, *S(M) = s<sub>0</sub> + s<sub>1</sub>x + s<sub>2</sub>x<sup>2</sup> + s<sub>3</sub>x<sup>3</sup> + ...* and *L = l<sub>0</sub> + l<sub>1</sub>x + l<sub>2</sub>x<sup>2</sup> + l<sub>3</sub>x<sup>3</sup> + ... + l<sub>n</sub>x<sup>n</sup>*. Note that *l<sub>0</sub> = 1*, as it is the product of all the *1* terms in the *(1 - m<sub>i</sub>x)* factors. + +Here are the equations for the coefficients of *S(M)L* of degree *n+1* through *2n*: +* *s<sub>n+1</sub> + s<sub>n+0</sub>l<sub>1</sub> + s<sub>n-1</sub>l<sub>2</sub> + s<sub>n-2</sub>l<sub>3</sub> + ... + s<sub>1</sub>l<sub>n</sub> = 0* +* *s<sub>n+2</sub> + s<sub>n+1</sub>l<sub>1</sub> + s<sub>n+0</sub>l<sub>2</sub> + s<sub>n-1</sub>l<sub>3</sub> + ... + s<sub>2</sub>l<sub>n</sub> = 0* +* *s<sub>n+3</sub> + s<sub>n+2</sub>l<sub>1</sub> + s<sub>n+1</sub>l<sub>2</sub> + s<sub>n+0</sub>l<sub>3</sub> + ... + s<sub>3</sub>l<sub>n</sub> = 0* +* ... +* *s<sub>2n</sub> + s<sub>2n-1</sub>l<sub>1</sub> + s<sub>2n-2</sub>l<sub>2</sub> + s<sub>2n-3</sub>l<sub>3</sub> + ... + s<sub>n</sub>l<sub>n</sub> = 0* + +These are *n* linear equations with *n* unknowns (the *l<sub>i<sub>* +values, for *i=1..n*), which can be solved using [Gaussian elimination](https://en.wikipedia.org/wiki/Gaussian_elimination). After doing so, +we have the coefficients of *L*, which can then be [factored](https://en.wikipedia.org/wiki/Factorization_of_polynomials_over_finite_fields) +into first degree factors of the form *(1 - m<sub>i</sub>x)*. The resulting *m* values are our set elements. + +**Putting it all together** + +Interestingly, only *2n* coefficients of *S(M)* were needed for solving +the set of equations above. This means we have our answer: the +coefficients *1* through *2n* of *S(M)*, or the list +*[m<sub>1</sub> + m<sub>2</sub> + ..., m<sub>1</sub><sup>2</sup> + m<sub>2</sub><sup>2</sup> + ..., ..., m<sub>1</sub><sup>2n</sup> + m<sub>2</sub><sup>2n</sup> + ...]* +functions as a sketch, satisfying the two properties we want: + +* Sketches can be combined to form the sketch of their symmetric difference, by simply pairwise adding the list elements together. +* With *2n* list elements we can efficiently recover *n* elements from a sketch. + +**Capacity and difference** + +The approach above only works when the number of elements *n* in the sketch is known. Of course we want to support cases where only an upper bound on the number of elements in the sketch is known, the capacity *c*. Given that we can reconstruct a set of size *c* from a sketch with *2c* terms, we should be able to reconstruct a set of size *n* too as long as *n ≤ c*. This is simply a matter of trying to solve the above set of equations assuming values of *n* that count down from *c* until a solution is found for one. This is known as the [Peterson-Gorenstein-Zierler algorithm](https://en.wikipedia.org/wiki/BCH_code#Peterson%E2%80%93Gorenstein%E2%80%93Zierler_algorithm). + +## Optimizations + +**Halving the sketch size** + +We can in fact only include the odd terms in the sketch, and reconstruct the even ones before solving the equation to find *L*. This means the size of a sketch becomes just *c* field elements, the same size as would be needed to send its contents naively. + +To see how this is possible, we need the [Frobenius endomorphism](https://en.wikipedia.org/wiki/Frobenius_endomorphism), which in short states that in fields where *x + x = 0* it holds that *(x + y)<sup>2</sup> = x<sup>2</sup> + y<sup>2</sup>* for every *x* and *y* (the dream of every high school math student!). This means that: + +* *s<sub>2</sub> = m<sub>1</sub><sup>2</sup> + m<sub>2</sub><sup>2</sup> + ... = (m<sub>1</sub> + m<sub>2</sub> + ...)<sup>2</sup> = s<sub>1</sub><sup>2</sup>*. +* *s<sub>4</sub> = m<sub>1</sub><sup>4</sup> + m<sub>2</sub><sup>4</sup> + ... = (m<sub>1</sub><sup>2</sup> + m<sub>2</sub><sup>2</sup> + ...)<sup>2</sup> = s<sub>2</sub><sup>2</sup>*. +* *s<sub>6</sub> = m<sub>1</sub><sup>6</sup> + m<sub>2</sub><sup>6</sup> + ... = (m<sub>1</sub><sup>3</sup> + m<sub>2</sub><sup>3</sup> + ...)<sup>2</sup> = s<sub>3</sub><sup>2</sup>*. +* ... + +In other words, we only need to send *s<sub>1</sub>, s<sub>3</sub>, s<sub>5</sub>, ..., s<sub>2n-1</sub>* to recover all *2n* *s<sub>i</sub>* values, and proceed with reconstruction. + +**Quadratic performance rather than cubic** + +Using Gaussian elimination to solve the set of equations above for the *l<sub>i</sub>* values requires *O(n<sup>3</sup>)* field operations. However, due to the special structure in the equations (look at the repeated *s<sub>i</sub>* values), it can be solved in *O(n<sup>2</sup>)* time using a number of techniques, including the [Berlekamp-Massey algorithm](https://en.wikipedia.org/wiki/Berlekamp%E2%80%93Massey_algorithm) (BM). + +**Roots instead of factorization** + +As explained above, the polynomial *L* can be factored into *(1 - m<sub>i</sub>x)* factors, where the values *m<sub>i</sub>* are the set elements. However, since we know that a decodable sketch must result in a polynomial that is fully factorizable into degree-*1* factors, we can instead use a more efficient root-finding algorithm rather than a factorization algorithm. As the root of each *(1 - m<sub>i</sub>x)* factor is *m<sub>i</sub><sup>-1</sup>*, we conclude that the set elements are in fact the inverses of the roots of *L*. + +**Avoiding inversions** + +As inversions are a relatively expensive operation, it would be useful to avoid them. + +Say that we're trying to find the inverses of the roots of *L = 1 + l<sub>1</sub>x + l<sub>2</sub>x<sup>2</sup> + ... + l<sub>n</sub>x<sup>n</sup>*, then we're really interested in the solutions *y* for *1 + l<sub>1</sub>y<sup>-1</sup> + l<sub>2</sub>y<sup>-2</sup> + ... + l<sub>n</sub>y<sup>-n</sup> = 0*. By multiplying both sides in the equations with *y<sup>n</sup>*, we find *l<sub>n</sub> + l<sub>n-1</sub>y + l<sub>n-2</sub>y<sup>2</sup> + ... + y<sup>n</sup> = 0*. + +In other words, we can find the inverses of the roots of *L* by instead factoring the polynomial with the coefficients of *L* in reverse order. + +* <a name="myfootnote1">[1]</a> For those familiar with coding theory: PinSketch communicates a set difference by encoding the set members as errors in a binary [BCH](https://en.wikipedia.org/wiki/BCH_code) codeword 2<sup>bits</sup> in size and sends the syndromes. + The linearity of the syndromes provides all the properties needed for a sketch. Sketch decoding is simply finding the error locations. Decode is much faster than an ordinary BCH decoder for such a large codeword because the need to take a discrete log is avoided by storing the set in the roots directly instead of in an exponent (logically permuting the bits of the codeword). diff --git a/src/minisketch/doc/minisketch-vs.png b/src/minisketch/doc/minisketch-vs.png Binary files differnew file mode 100644 index 0000000000..aed810de8a --- /dev/null +++ b/src/minisketch/doc/minisketch-vs.png diff --git a/src/minisketch/doc/moduli.md b/src/minisketch/doc/moduli.md new file mode 100644 index 0000000000..379ac481b3 --- /dev/null +++ b/src/minisketch/doc/moduli.md @@ -0,0 +1,65 @@ +These are the irreducible polynomials over *GF(2)* used to represent field elements: + +* *x<sup>2</sup> + x + 1* +* *x<sup>3</sup> + x + 1* +* *x<sup>4</sup> + x + 1* +* *x<sup>5</sup> + x<sup>2</sup> + 1* +* *x<sup>6</sup> + x + 1* +* *x<sup>7</sup> + x + 1* +* *x<sup>8</sup> + x<sup>4</sup> + x<sup>3</sup> + x + 1* +* *x<sup>9</sup> + x + 1* +* *x<sup>10</sup> + x<sup>3</sup> + 1* +* *x<sup>11</sup> + x<sup>2</sup> + 1* +* *x<sup>12</sup> + x<sup>3</sup> + 1* +* *x<sup>13</sup> + x<sup>4</sup> + x<sup>3</sup> + x + 1* +* *x<sup>14</sup> + x<sup>5</sup> + 1* +* *x<sup>15</sup> + x + 1* +* *x<sup>16</sup> + x<sup>5</sup> + x<sup>3</sup> + x + 1* +* *x<sup>17</sup> + x<sup>3</sup> + 1* +* *x<sup>18</sup> + x<sup>3</sup> + 1* +* *x<sup>19</sup> + x<sup>5</sup> + x<sup>2</sup> + x + 1* +* *x<sup>20</sup> + x<sup>3</sup> + 1* +* *x<sup>21</sup> + x<sup>2</sup> + 1* +* *x<sup>22</sup> + x + 1* +* *x<sup>23</sup> + x<sup>5</sup> + 1* +* *x<sup>24</sup> + x<sup>4</sup> + x<sup>3</sup> + x + 1* +* *x<sup>25</sup> + x<sup>3</sup> + 1* +* *x<sup>26</sup> + x<sup>4</sup> + x<sup>3</sup> + x + 1* +* *x<sup>27</sup> + x<sup>5</sup> + x<sup>2</sup> + x + 1* +* *x<sup>28</sup> + x + 1* +* *x<sup>29</sup> + x<sup>2</sup> + 1* +* *x<sup>30</sup> + x + 1* +* *x<sup>31</sup> + x<sup>3</sup> + 1* +* *x<sup>32</sup> + x<sup>7</sup> + x<sup>3</sup> + x<sup>2</sup> + 1* +* *x<sup>33</sup> + x<sup>10</sup> + 1* +* *x<sup>34</sup> + x<sup>7</sup> + 1* +* *x<sup>35</sup> + x<sup>2</sup> + 1* +* *x<sup>36</sup> + x<sup>9</sup> + 1* +* *x<sup>37</sup> + x<sup>6</sup> + x<sup>4</sup> + x + 1* +* *x<sup>38</sup> + x<sup>6</sup> + x<sup>5</sup> + x + 1* +* *x<sup>39</sup> + x<sup>4</sup> + 1* +* *x<sup>40</sup> + x<sup>5</sup> + x<sup>4</sup> + x<sup>3</sup> + 1* +* *x<sup>41</sup> + x<sup>3</sup> + 1* +* *x<sup>42</sup> + x<sup>7</sup> + 1* +* *x<sup>43</sup> + x<sup>6</sup> + x<sup>4</sup> + x<sup>3</sup> + 1* +* *x<sup>44</sup> + x<sup>5</sup> + 1* +* *x<sup>45</sup> + x<sup>4</sup> + x<sup>3</sup> + x + 1* +* *x<sup>46</sup> + x + 1* +* *x<sup>47</sup> + x<sup>5</sup> + 1* +* *x<sup>48</sup> + x<sup>5</sup> + x<sup>3</sup> + x<sup>2</sup> + 1* +* *x<sup>49</sup> + x<sup>9</sup> + 1* +* *x<sup>50</sup> + x<sup>4</sup> + x<sup>3</sup> + x<sup>2</sup> + 1* +* *x<sup>51</sup> + x<sup>6</sup> + x<sup>3</sup> + x + 1* +* *x<sup>52</sup> + x<sup>3</sup> + 1* +* *x<sup>53</sup> + x<sup>6</sup> + x<sup>2</sup> + x + 1* +* *x<sup>54</sup> + x<sup>9</sup> + 1* +* *x<sup>55</sup> + x<sup>7</sup> + 1* +* *x<sup>56</sup> + x<sup>7</sup> + x<sup>4</sup> + x<sup>2</sup> + 1* +* *x<sup>57</sup> + x<sup>4</sup> + 1* +* *x<sup>58</sup> + x<sup>19</sup> + 1* +* *x<sup>59</sup> + x<sup>7</sup> + x<sup>4</sup> + x<sup>2</sup> + 1* +* *x<sup>60</sup> + x + 1* +* *x<sup>61</sup> + x<sup>5</sup> + x<sup>2</sup> + x + 1* +* *x<sup>62</sup> + x<sup>29</sup> + 1* +* *x<sup>63</sup> + x + 1* +* *x<sup>64</sup> + x<sup>4</sup> + x<sup>3</sup> + x + 1* diff --git a/src/minisketch/doc/plot_bits.png b/src/minisketch/doc/plot_bits.png Binary files differnew file mode 100644 index 0000000000..6e907d6b20 --- /dev/null +++ b/src/minisketch/doc/plot_bits.png diff --git a/src/minisketch/doc/plot_capacity.png b/src/minisketch/doc/plot_capacity.png Binary files differnew file mode 100644 index 0000000000..b4f760da36 --- /dev/null +++ b/src/minisketch/doc/plot_capacity.png diff --git a/src/minisketch/doc/plot_diff.png b/src/minisketch/doc/plot_diff.png Binary files differnew file mode 100644 index 0000000000..08ab6a86b9 --- /dev/null +++ b/src/minisketch/doc/plot_diff.png diff --git a/src/minisketch/doc/plot_size.png b/src/minisketch/doc/plot_size.png Binary files differnew file mode 100644 index 0000000000..b21921776a --- /dev/null +++ b/src/minisketch/doc/plot_size.png diff --git a/src/minisketch/doc/protocoltips.md b/src/minisketch/doc/protocoltips.md new file mode 100644 index 0000000000..610407ebc2 --- /dev/null +++ b/src/minisketch/doc/protocoltips.md @@ -0,0 +1,30 @@ +# Tips for designing protocols using `libminisketch` + +Sending a sketch is less efficient than just sending your whole set with efficient entropy coding if the number of differences is larger than *log<sub>2</sub>( 2<sup>b</sup> choose set_size ) / b*. + +In most applications your set can be hashed to entries just large enough to make the probability of collision negligible. This can be a considerable speedup and bandwidth savings. Short hashes (<128 bits) should be salted with an unpredictable value to prevent malicious inputs from intentionally causing collisions. Salting also allows an entry missed due to a collision to be reconciled on a later run with a different salt. Pre-hashing may not be possible in some applications, such as where there is only one-way communication, where the confidentiality of entry origin matters, or where security depends on the total absence of collisions. + +Some element sizes are faster to decode than others; see the benchmarks in the readme. + +Almost all the computational burden of reconciliation is in minisketch_decode(). Denial-of-service attacks can be mitigated by arranging protocol flow so that a party requests a sketch and decodes it rather than a construction where the participants will decode unsolicited sketches. Decode times can be constrained by limiting sketch capacity or via the max_count argument to minisketch_decode(). + +In most cases you don't actually know the size of the set difference in advance, but often you know a lower bound on it (the difference in set sizes). + +* There are difference size estimation techniques such as min-wise hashing<sup>[[1]](#myfootnote1)</sup> or random projections<sup>[[2]](#myfootnote2)</sup>, but complex estimators can end up using more bandwidth than they save. + +* It may be useful to always overestimate the sketch size needed to amortize communications overheads (*e.g.* packet headers, round trip delays). + +* If the total data sent would end up leaving you better off having just sent the whole set, per above, then you can send the set in response to a failure but leave out as many elements as the size of the previously sent sketch. The receiver can decode the partial set and use the data they already have to complete it, reducing bandwidth waste. + +* Additional elements can be sent for a sketch as few as one at a time with little decode cost until enough data is received to decode. This is most easily implemented by always computing the largest sketch size and sending it incrementally as needed. + +* Because sketches are linear you can adaptively subdivide to decode an overfull set. The sender uses a hash function to select approximately half their set members and sends a sketch of those members. The receiver can do the same and combine the result with the initially sent sketch to get two sketches with roughly half the number of members and attempt to decode them. Repeat recursively on failure. This adaptive subdivision procedure makes decode time essentially linear at the cost of communications inefficiency. Minisketches can also be used as the cells in an IBLT for similar reasons. + +Less efficient reconciliation techniques like IBLT or adaptive subdivision, or overheads like complex estimators effectively lower the threshold where sending the whole set efficiently would use less bandwidth. + +When the number of differences is more than 2<sup>b/2-1</sup> an alternative sketch encoding is possible that is somewhat smaller, but requires a table of size 2<sup>b</sup>; contact the authors if you have an application where that might be useful. + +## References + +* <a name="myfootnote1">[1]</a> Broder, A. *On the Resemblance and Containment of Documents* Proceedings of the Compression and Complexity of Sequences 1997 [[PDF]](https://www.cs.princeton.edu/courses/archive/spring13/cos598C/broder97resemblance.pdf) +* <a name="myfootnote2">[2]</a> Feigenbaum, Joan and Kannan, Sampath and Strauss, Martin J. and Viswanathan, Mahesh. *An Approximate L1-Difference Algorithm for Massive Data Streams* SIAM J. Comput. 2003 [[PDF]](http://www.cs.yale.edu/homes/jf/FKSV1.pdf) diff --git a/src/minisketch/include/minisketch.h b/src/minisketch/include/minisketch.h new file mode 100644 index 0000000000..0b5d8372e8 --- /dev/null +++ b/src/minisketch/include/minisketch.h @@ -0,0 +1,367 @@ +#ifndef _MINISKETCH_H_ +#define _MINISKETCH_H_ 1 + +#include <stdint.h> +#include <stdlib.h> + +#ifdef _MSC_VER +# include <compat.h> +#else +# include <unistd.h> +#endif + +#ifndef MINISKETCH_API +# if defined(_WIN32) +# ifdef MINISKETCH_BUILD +# define MINISKETCH_API __declspec(dllexport) +# else +# define MINISKETCH_API +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(MINISKETCH_BUILD) +# define MINISKETCH_API __attribute__ ((visibility ("default"))) +# else +# define MINISKETCH_API +# endif +#endif + +#ifdef __cplusplus +# if __cplusplus >= 201103L +# include <memory> +# include <vector> +# include <cassert> +# if __cplusplus >= 201703L +# include <optional> +# endif // __cplusplus >= 201703L +# endif // __cplusplus >= 201103L +extern "C" { +#endif // __cplusplus + +/** Opaque type for decoded sketches. */ +typedef struct minisketch minisketch; + +/** Determine whether support for elements of `bits` bits was compiled in. */ +MINISKETCH_API int minisketch_bits_supported(uint32_t bits); + +/** Determine the maximum number of implementations available. + * + * Multiple implementations may be available for a given element size, with + * different performance characteristics on different hardware. + * + * Each implementation is identified by a number from 0 to the output of this + * function call, inclusive. Note that not every combination of implementation + * and element size may exist (see further). +*/ +MINISKETCH_API uint32_t minisketch_implementation_max(void); + +/** Determine if the a combination of bits and implementation number is available. + * + * Returns 1 if it is, 0 otherwise. + */ +MINISKETCH_API int minisketch_implementation_supported(uint32_t bits, uint32_t implementation); + +/** Construct a sketch for a given element size, implementation and capacity. + * + * If the combination of `bits` and `implementation` is unavailable, or when + * OOM occurs, NULL is returned. If minisketch_implementation_supported + * returns 1 for the specified bits and implementation, this will always succeed + * (except when allocation fails). + * + * If the result is not NULL, it must be destroyed using minisketch_destroy. + */ +MINISKETCH_API minisketch* minisketch_create(uint32_t bits, uint32_t implementation, size_t capacity); + +/** Get the element size of a sketch in bits. */ +MINISKETCH_API uint32_t minisketch_bits(const minisketch* sketch); + +/** Get the capacity of a sketch. */ +MINISKETCH_API size_t minisketch_capacity(const minisketch* sketch); + +/** Get the implementation of a sketch. */ +MINISKETCH_API uint32_t minisketch_implementation(const minisketch* sketch); + +/** Set the seed for randomizing algorithm choices to a fixed value. + * + * By default, sketches are initialized with a random seed. This is important + * to avoid scenarios where an attacker could force worst-case behavior. + * + * This function initializes the seed to a user-provided value (any 64-bit + * integer is acceptable, regardless of field size). + * + * When seed is -1, a fixed internal value with predictable behavior is + * used. It is only intended for testing. + */ +MINISKETCH_API void minisketch_set_seed(minisketch* sketch, uint64_t seed); + +/** Clone a sketch. + * + * The result must be destroyed using minisketch_destroy. + */ +MINISKETCH_API minisketch* minisketch_clone(const minisketch* sketch); + +/** Destroy a sketch. + * + * The pointer that was passed in may not be used anymore afterwards. + */ +MINISKETCH_API void minisketch_destroy(minisketch* sketch); + +/** Compute the size in bytes for serializing a given sketch. */ +MINISKETCH_API size_t minisketch_serialized_size(const minisketch* sketch); + +/** Serialize a sketch to bytes. */ +MINISKETCH_API void minisketch_serialize(const minisketch* sketch, unsigned char* output); + +/** Deserialize a sketch from bytes. */ +MINISKETCH_API void minisketch_deserialize(minisketch* sketch, const unsigned char* input); + +/** Add an element to a sketch. + * + * If the element to be added is too large for the sketch, the most significant + * bits of the element are dropped. More precisely, if the element size of + * `sketch` is b bits, then this function adds the unsigned integer represented + * by the b least significant bits of `element` to `sketch`. + * + * If the element to be added is 0 (after potentially dropping the most significant + * bits), then this function is a no-op. Sketches cannot contain an element with + * the value 0. + * + * Note that adding the same element a second time removes it again. + */ +MINISKETCH_API void minisketch_add_uint64(minisketch* sketch, uint64_t element); + +/** Merge the elements of another sketch into this sketch. + * + * After merging, `sketch` will contain every element that existed in one but not + * both of the input sketches. It can be seen as an exclusive or operation on + * the set elements. If the capacity of `other_sketch` is lower than `sketch`'s, + * merging reduces the capacity of `sketch` to that of `other_sketch`. + * + * This function returns the capacity of `sketch` after merging has been performed + * (where this capacity is at least 1), or 0 to indicate that merging has failed because + * the two input sketches differ in their element size or implementation. If 0 is + * returned, `sketch` (and its capacity) have not been modified. + * + * It is also possible to perform this operation directly on the serializations + * of two sketches with the same element size and capacity by performing a bitwise XOR + * of the serializations. + */ +MINISKETCH_API size_t minisketch_merge(minisketch* sketch, const minisketch* other_sketch); + +/** Decode a sketch. + * + * `output` is a pointer to an array of `max_element` uint64_t's, which will be + * filled with the elements in this sketch. + * + * The return value is the number of decoded elements, or -1 if decoding failed. + */ +MINISKETCH_API ssize_t minisketch_decode(const minisketch* sketch, size_t max_elements, uint64_t* output); + +/** Compute the capacity needed to achieve a certain rate of false positives. + * + * A sketch with capacity c and no more than c elements can always be decoded + * correctly. However, if it has more than c elements, or contains just random + * bytes, it is possible that it will still decode, but the result will be + * nonsense. This can be counteracted by increasing the capacity slightly. + * + * Given a field size bits, an intended number of elements that can be decoded + * max_elements, and a false positive probability of 1 in 2**fpbits, this + * function computes the necessary capacity. It is only guaranteed to be + * accurate up to fpbits=256. + */ +MINISKETCH_API size_t minisketch_compute_capacity(uint32_t bits, size_t max_elements, uint32_t fpbits); + +/** Compute what max_elements can be decoded for a certain rate of false positives. + * + * This is the inverse operation of minisketch_compute_capacity. It determines, + * given a field size bits, a capacity of a sketch, and an acceptable false + * positive probability of 1 in 2**fpbits, what the maximum allowed + * max_elements value is. If no value of max_elements would give the desired + * false positive probability, 0 is returned. + * + * Note that this is not an exact inverse of minisketch_compute_capacity. For + * example, with bits=32, fpbits=16, and max_elements=8, + * minisketch_compute_capacity will return 9, as capacity 8 would only have a + * false positive chance of 1 in 2^15.3. Increasing the capacity to 9 however + * decreases the fp chance to 1 in 2^47.3, enough for max_elements=9 (with fp + * chance of 1 in 2^18.5). Therefore, minisketch_compute_max_elements with + * capacity=9 will return 9. + */ +MINISKETCH_API size_t minisketch_compute_max_elements(uint32_t bits, size_t capacity, uint32_t fpbits); + +#ifdef __cplusplus +} + +#if __cplusplus >= 201103L +/** Simple RAII C++11 wrapper around the minisketch API. */ +class Minisketch +{ + struct Deleter + { + void operator()(minisketch* ptr) const + { + minisketch_destroy(ptr); + } + }; + + std::unique_ptr<minisketch, Deleter> m_minisketch; + +public: + /** Check whether the library supports fields of the given size. */ + static bool BitsSupported(uint32_t bits) noexcept { return minisketch_bits_supported(bits); } + + /** Get the highest supported implementation number. */ + static uint32_t MaxImplementation() noexcept { return minisketch_implementation_max(); } + + /** Check whether the library supports fields with a given size and implementation number. + * If a particular field size `bits` is supported, implementation 0 is always supported for it. + * Higher implementation numbers may or may not be available as well, up to MaxImplementation(). + */ + static bool ImplementationSupported(uint32_t bits, uint32_t implementation) noexcept { return minisketch_implementation_supported(bits, implementation); } + + /** Given field size and a maximum number of decodable elements n, compute what capacity c to + * use so that sketches with more elements than n have a chance no higher than 2^-fpbits of + * being decoded incorrectly (and will instead fail when decoding for up to n elements). + * + * See minisketch_compute_capacity for more details. */ + static size_t ComputeCapacity(uint32_t bits, size_t max_elements, uint32_t fpbits) noexcept { return minisketch_compute_capacity(bits, max_elements, fpbits); } + + /** Reverse operation of ComputeCapacity. See minisketch_compute_max_elements. */ + static size_t ComputeMaxElements(uint32_t bits, size_t capacity, uint32_t fpbits) noexcept { return minisketch_compute_max_elements(bits, capacity, fpbits); } + + /** Construct a clone of the specified sketch. */ + Minisketch(const Minisketch& sketch) noexcept + { + if (sketch.m_minisketch) { + m_minisketch = std::unique_ptr<minisketch, Deleter>(minisketch_clone(sketch.m_minisketch.get())); + } + } + + /** Make this Minisketch a clone of the specified one. */ + Minisketch& operator=(const Minisketch& sketch) noexcept + { + if (sketch.m_minisketch) { + m_minisketch = std::unique_ptr<minisketch, Deleter>(minisketch_clone(sketch.m_minisketch.get())); + } + return *this; + } + + /** Check whether this Minisketch object is valid. */ + explicit operator bool() const noexcept { return bool{m_minisketch}; } + + /** Construct an (invalid) Minisketch object. */ + Minisketch() noexcept = default; + + /** Move constructor. */ + Minisketch(Minisketch&&) noexcept = default; + + /** Move assignment. */ + Minisketch& operator=(Minisketch&&) noexcept = default; + + /** Construct a Minisketch object with the specified parameters. + * + * If bits is not BitsSupported(), or the combination of bits and capacity is not + * ImplementationSupported(), or OOM occurs internally, an invalid Minisketch + * object will be constructed. Use operator bool() to check that this isn't the + * case before performing any other operations. */ + Minisketch(uint32_t bits, uint32_t implementation, size_t capacity) noexcept + { + m_minisketch = std::unique_ptr<minisketch, Deleter>(minisketch_create(bits, implementation, capacity)); + } + + /** Create a Minisketch object sufficiently large for the specified number of elements at given fpbits. + * It may construct an invalid object, which you may need to check for. */ + static Minisketch CreateFP(uint32_t bits, uint32_t implementation, size_t max_elements, uint32_t fpbits) noexcept + { + return Minisketch(bits, implementation, ComputeCapacity(bits, max_elements, fpbits)); + } + + /** Return the field size for a (valid) Minisketch object. */ + uint32_t GetBits() const noexcept { return minisketch_bits(m_minisketch.get()); } + + /** Return the capacity for a (valid) Minisketch object. */ + size_t GetCapacity() const noexcept { return minisketch_capacity(m_minisketch.get()); } + + /** Return the implementation number for a (valid) Minisketch object. */ + uint32_t GetImplementation() const noexcept { return minisketch_implementation(m_minisketch.get()); } + + /** Set the seed for a (valid) Minisketch object. See minisketch_set_seed(). */ + Minisketch& SetSeed(uint64_t seed) noexcept + { + minisketch_set_seed(m_minisketch.get(), seed); + return *this; + } + + /** Add (or remove, if already present) an element to a (valid) Minisketch object. + * See minisketch_add_uint64(). */ + Minisketch& Add(uint64_t element) noexcept + { + minisketch_add_uint64(m_minisketch.get(), element); + return *this; + } + + /** Merge sketch into *this; both have to be valid Minisketch objects. + * See minisketch_merge for details. */ + Minisketch& Merge(const Minisketch& sketch) noexcept + { + minisketch_merge(m_minisketch.get(), sketch.m_minisketch.get()); + return *this; + } + + /** Decode this (valid) Minisketch object into the result vector, up to as many elements as the + * vector's size permits. */ + bool Decode(std::vector<uint64_t>& result) const + { + ssize_t ret = minisketch_decode(m_minisketch.get(), result.size(), result.data()); + if (ret == -1) return false; + result.resize(ret); + return true; + } + + /** Get the serialized size in bytes for this (valid) Minisketch object.. */ + size_t GetSerializedSize() const noexcept { return minisketch_serialized_size(m_minisketch.get()); } + + /** Serialize this (valid) Minisketch object as a byte vector. */ + std::vector<unsigned char> Serialize() const + { + std::vector<unsigned char> result(GetSerializedSize()); + minisketch_serialize(m_minisketch.get(), result.data()); + return result; + } + + /** Deserialize into this (valid) Minisketch from an object containing its bytes (which has data() + * and size() members). */ + template<typename T> + Minisketch& Deserialize( + const T& obj, + typename std::enable_if< + std::is_convertible<typename std::remove_pointer<decltype(obj.data())>::type (*)[], const unsigned char (*)[]>::value && + std::is_convertible<decltype(obj.size()), std::size_t>::value, + std::nullptr_t + >::type = nullptr) noexcept + { + assert(GetSerializedSize() == obj.size()); + minisketch_deserialize(m_minisketch.get(), obj.data()); + return *this; + } + +#if __cplusplus >= 201703L + /** C++17 only: like Decode(), but up to a specified number of elements into an optional vector. */ + std::optional<std::vector<uint64_t>> Decode(size_t max_elements) const + { + std::vector<uint64_t> result(max_elements); + ssize_t ret = minisketch_decode(m_minisketch.get(), max_elements, result.data()); + if (ret == -1) return {}; + result.resize(ret); + return result; + } + + /** C++17 only: similar to Decode(), but with specified false positive probability. */ + std::optional<std::vector<uint64_t>> DecodeFP(uint32_t fpbits) const + { + return Decode(ComputeMaxElements(GetBits(), GetCapacity(), fpbits)); + } +#endif // __cplusplus >= 201703L +}; +#endif // __cplusplus >= 201103L +#endif // __cplusplus + +#endif // _MINISKETCH_H_ diff --git a/src/minisketch/sources.mk b/src/minisketch/sources.mk new file mode 100644 index 0000000000..386a4fcc23 --- /dev/null +++ b/src/minisketch/sources.mk @@ -0,0 +1,58 @@ +# - All variables are namespaced with MINISKETCH_ to avoid colliding with +# downstream makefiles. +# - All Variables ending in _HEADERS or _SOURCES confuse automake, so the +# _INT postfix is applied. +# - Convenience variables, for example a MINISKETCH_FIELDS_DIR should not be used +# as they interfere with automatic dependency generation +# - The %reldir% is the relative path from the Makefile.am. This allows +# downstreams to use these variables without having to manually account for +# the path change. + +MINISKETCH_INCLUDE_DIR_INT = %reldir%/include + +MINISKETCH_DIST_HEADERS_INT = +MINISKETCH_DIST_HEADERS_INT += %reldir%/include/minisketch.h + +MINISKETCH_LIB_HEADERS_INT = +MINISKETCH_LIB_HEADERS_INT += %reldir%/src/false_positives.h +MINISKETCH_LIB_HEADERS_INT += %reldir%/src/fielddefines.h +MINISKETCH_LIB_HEADERS_INT += %reldir%/src/int_utils.h +MINISKETCH_LIB_HEADERS_INT += %reldir%/src/lintrans.h +MINISKETCH_LIB_HEADERS_INT += %reldir%/src/sketch.h +MINISKETCH_LIB_HEADERS_INT += %reldir%/src/sketch_impl.h +MINISKETCH_LIB_HEADERS_INT += %reldir%/src/util.h + +MINISKETCH_LIB_SOURCES_INT = +MINISKETCH_LIB_SOURCES_INT += %reldir%/src/minisketch.cpp + +MINISKETCH_FIELD_GENERIC_HEADERS_INT = +MINISKETCH_FIELD_GENERIC_HEADERS_INT += %reldir%/src/fields/generic_common_impl.h + +MINISKETCH_FIELD_GENERIC_SOURCES_INT = +MINISKETCH_FIELD_GENERIC_SOURCES_INT += %reldir%/src/fields/generic_1byte.cpp +MINISKETCH_FIELD_GENERIC_SOURCES_INT += %reldir%/src/fields/generic_2bytes.cpp +MINISKETCH_FIELD_GENERIC_SOURCES_INT += %reldir%/src/fields/generic_3bytes.cpp +MINISKETCH_FIELD_GENERIC_SOURCES_INT += %reldir%/src/fields/generic_4bytes.cpp +MINISKETCH_FIELD_GENERIC_SOURCES_INT += %reldir%/src/fields/generic_5bytes.cpp +MINISKETCH_FIELD_GENERIC_SOURCES_INT += %reldir%/src/fields/generic_6bytes.cpp +MINISKETCH_FIELD_GENERIC_SOURCES_INT += %reldir%/src/fields/generic_7bytes.cpp +MINISKETCH_FIELD_GENERIC_SOURCES_INT += %reldir%/src/fields/generic_8bytes.cpp + +MINISKETCH_FIELD_CLMUL_HEADERS_INT = +MINISKETCH_FIELD_CLMUL_HEADERS_INT += %reldir%/src/fields/clmul_common_impl.h + +MINISKETCH_FIELD_CLMUL_SOURCES_INT = +MINISKETCH_FIELD_CLMUL_SOURCES_INT += %reldir%/src/fields/clmul_1byte.cpp +MINISKETCH_FIELD_CLMUL_SOURCES_INT += %reldir%/src/fields/clmul_2bytes.cpp +MINISKETCH_FIELD_CLMUL_SOURCES_INT += %reldir%/src/fields/clmul_3bytes.cpp +MINISKETCH_FIELD_CLMUL_SOURCES_INT += %reldir%/src/fields/clmul_4bytes.cpp +MINISKETCH_FIELD_CLMUL_SOURCES_INT += %reldir%/src/fields/clmul_5bytes.cpp +MINISKETCH_FIELD_CLMUL_SOURCES_INT += %reldir%/src/fields/clmul_6bytes.cpp +MINISKETCH_FIELD_CLMUL_SOURCES_INT += %reldir%/src/fields/clmul_7bytes.cpp +MINISKETCH_FIELD_CLMUL_SOURCES_INT += %reldir%/src/fields/clmul_8bytes.cpp + +MINISKETCH_BENCH_SOURCES_INT = +MINISKETCH_BENCH_SOURCES_INT += %reldir%/src/bench.cpp + +MINISKETCH_TEST_SOURCES_INT = +MINISKETCH_TEST_SOURCES_INT += %reldir%/src/test.cpp diff --git a/src/minisketch/src/bench.cpp b/src/minisketch/src/bench.cpp new file mode 100644 index 0000000000..f55944a448 --- /dev/null +++ b/src/minisketch/src/bench.cpp @@ -0,0 +1,122 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#include "../include/minisketch.h" +#include <string.h> +#include <memory> +#include <vector> +#include <chrono> +#include <random> +#include <set> +#include <algorithm> + +int main(int argc, char** argv) { + if (argc < 1 || argc > 4) { + printf("Usage: %s [syndromes=150] [errors=syndromes] [iters=10]\n", argv[0]); + return 1; + } + int syndromes = argc > 1 ? strtoul(argv[1], NULL, 10) : 150; + int errors = argc > 2 ? strtoul(argv[2], NULL, 10) : syndromes; + int iters = argc > 3 ? strtoul(argv[3], NULL, 10) : 10; + if (syndromes < 0 || syndromes > 1000000) { + printf("Number of syndromes (%i) out of range 0..1000000\n", syndromes); + return 1; + } + if (errors < 0) { + printf("Number of errors (%i) is negative(%i)\n", errors, syndromes); + return 1; + } + if (iters < 0 || iters > 1000000000) { + printf("Number of iterations (%i) out of range 0..1000000000\n", iters); + return 1; + } + uint32_t max_impl = minisketch_implementation_max(); + for (int bits = 2; bits <= 64; ++bits) { + if (errors > pow(2.0, bits - 1)) continue; + if (!minisketch_bits_supported(bits)) continue; + printf("recover[ms]\t% 3i\t", bits); + for (uint32_t impl = 0; impl <= max_impl; ++impl) { + std::vector<minisketch*> states; + std::vector<uint64_t> roots(2 * syndromes); + std::random_device rng; + std::uniform_int_distribution<uint64_t> dist(1, (uint64_t(1) << bits) - 1); + states.resize(iters); + std::vector<double> benches; + benches.reserve(iters); + for (int i = 0; i < iters; ++i) { + states[i] = minisketch_create(bits, impl, syndromes); + if (!states[i]) break; + std::set<uint64_t> done; + for (int j = 0; j < errors; ++j) { + uint64_t r; + do { + r = dist(rng); + } while (done.count(r)); + done.insert(r); + minisketch_add_uint64(states[i], r); + } + } + if (!states[0]) { + printf(" -\t"); + } else { + double total = 0.0; + for (auto& state : states) { + auto start = std::chrono::steady_clock::now(); + minisketch_decode(state, 2 * syndromes, roots.data()); + auto stop = std::chrono::steady_clock::now(); + std::chrono::duration<double> dur(stop - start); + total += dur.count(); + benches.push_back(dur.count()); + } + std::sort(benches.begin(), benches.end()); + printf("% 10.5f\t", benches[0] * 1000.0); + } + for (auto& state : states) { + minisketch_destroy(state); + } + } + printf("\n"); + printf("create[ns]\t% 3i\t", bits); + for (uint32_t impl = 0; impl <= max_impl; ++impl) { + std::vector<minisketch*> states; + std::random_device rng; + std::uniform_int_distribution<uint64_t> dist; + std::vector<uint64_t> data; + data.resize(errors * 10); + states.resize(iters); + std::vector<double> benches; + benches.reserve(iters); + for (int i = 0; i < iters; ++i) { + states[i] = minisketch_create(bits, impl, syndromes); + } + for (size_t i = 0; i < data.size(); ++i) { + data[i] = dist(rng); + } + if (!states[0]) { + printf(" -\t"); + } else { + double total = 0.0; + for (auto& state : states) { + auto start = std::chrono::steady_clock::now(); + for (auto val : data) { + minisketch_add_uint64(state, val); + } + auto stop = std::chrono::steady_clock::now(); + std::chrono::duration<double> dur(stop - start); + total += dur.count(); + benches.push_back(dur.count()); + } + std::sort(benches.begin(), benches.end()); + printf("% 10.5f\t", benches[0] * 1000000000.0 / data.size() / syndromes); + } + for (auto& state : states) { + minisketch_destroy(state); + } + } + printf("\n"); + } + return 0; +} diff --git a/src/minisketch/src/false_positives.h b/src/minisketch/src/false_positives.h new file mode 100644 index 0000000000..44ebb3e94c --- /dev/null +++ b/src/minisketch/src/false_positives.h @@ -0,0 +1,110 @@ +/********************************************************************** + * Copyright (c) 2020 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _MINISKETCH_FALSE_POSITIVES_H_ +#define _MINISKETCH_FALSE_POSITIVES_H_ + +#include "util.h" + +#include "int_utils.h" + +#include <stdint.h> + +namespace { + +/** Compute floor(log2(x!)), exactly up to x=57; an underestimate up to x=2^32-1. */ +uint64_t Log2Factorial(uint32_t x) { + //! Values of floor(106*log2(1 + i/32)) for i=0..31 + static constexpr uint8_t T[32] = { + 0, 4, 9, 13, 18, 22, 26, 30, 34, 37, 41, 45, 48, 52, 55, 58, 62, 65, 68, + 71, 74, 77, 80, 82, 85, 88, 90, 93, 96, 98, 101, 103 + }; + int bits = CountBits(x, 32); + // Compute an (under)estimate of floor(106*log2(x)). + // This works by relying on floor(log2(x)) = countbits(x)-1, and adding + // precision using the top 6 bits of x (the highest one of which is always + // one). + unsigned l2_106 = 106 * (bits - 1) + T[((x << (32 - bits)) >> 26) & 31]; + // Based on Stirling approximation for log2(x!): + // log2(x!) = log(x!) / log(2) + // = ((x + 1/2) * log(x) - x + log(2*pi)/2 + ...) / log(2) + // = (x + 1/2) * log2(x) - x/log(2) + log2(2*pi)/2 + ... + // = 1/2*(2*x+1)*log2(x) - (1/log(2))*x + log2(2*pi)/2 + ... + // = 1/212*(2*x+1)*(106*log2(x)) + (-1/log(2))*x + log2(2*pi)/2 + ... + // where 418079/88632748 is exactly 1/212 + // -127870026/88632748 is slightly less than -1/log(2) + // 117504694/88632748 is less than log2(2*pi)/2 + // A correction term is only needed for x < 3. + // + // See doc/log2_factorial.sage for how these constants were obtained. + return (418079 * (2 * uint64_t{x} + 1) * l2_106 - 127870026 * uint64_t{x} + 117504694 + 88632748 * (x < 3)) / 88632748; +} + +/** Compute floor(log2(2^(bits * capacity) / sum((2^bits - 1) choose k, k=0..capacity))), for bits>1 + * + * See doc/gen_basefpbits.sage for how the tables were obtained. */ +uint64_t BaseFPBits(uint32_t bits, uint32_t capacity) { + // Correction table for low bits/capacities + static constexpr uint8_t ADD5[] = {1, 1, 1, 1, 2, 2, 2, 3, 4, 4, 5, 5, 6, 7, 8, 8, 9, 10, 10, 10, 11, 11, 11, 12, 12, 12, 12}; + static constexpr uint8_t ADD6[] = {1, 0, 0, 0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, 4, 4, 5, 6, 6, 6, 7, 8, 8, 10, 10, 11, 12, 12, 13, 14, 15, 15, 16, 17, 18, 18, 19, 20, 20, 21, 21, 22, 22, 23, 23, 23, 24, 24, 24, 24}; + static constexpr uint8_t ADD7[] = {1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 7, 7, 8, 7, 8, 9, 9, 9, 10, 11, 11, 12, 12, 13, 13, 15, 15, 15, 16, 17, 17, 18, 19, 20, 20}; + static constexpr uint8_t ADD8[] = {1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 3, 4, 4, 5, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 8, 9, 9}; + static constexpr uint8_t ADD9[] = {1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 2, 1, 1, 1, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 3, 2, 3, 3, 3, 3, 4, 3, 3, 4, 4, 4, 4}; + + if (capacity == 0) return 0; + uint64_t ret = 0; + if (bits < 32 && capacity >= (1U << bits)) { + ret = uint64_t{bits} * (capacity - (1U << bits) + 1); + capacity = (1U << bits) - 1; + } + ret += Log2Factorial(capacity); + switch (bits) { + case 2: return ret + (capacity <= 2 ? 0 : 1); + case 3: return ret + (capacity <= 2 ? 0 : (0x2a5 >> 2 * (capacity - 3)) & 3); + case 4: return ret + (capacity <= 3 ? 0 : (0xb6d91a449 >> 3 * (capacity - 4)) & 7); + case 5: return ret + (capacity <= 4 ? 0 : ADD5[capacity - 5]); + case 6: return ret + (capacity <= 4 ? 0 : capacity > 54 ? 25 : ADD6[capacity - 5]); + case 7: return ret + (capacity <= 4 ? 0 : capacity > 57 ? 21 : ADD7[capacity - 5]); + case 8: return ret + (capacity <= 9 ? 0 : capacity > 56 ? 10 : ADD8[capacity - 10]); + case 9: return ret + (capacity <= 11 ? 0 : capacity > 54 ? 5 : ADD9[capacity - 12]); + case 10: return ret + (capacity <= 21 ? 0 : capacity > 50 ? 2 : (0x1a6665545555041 >> 2 * (capacity - 22)) & 3); + case 11: return ret + (capacity <= 21 ? 0 : capacity > 45 ? 1 : (0x5b3dc1 >> (capacity - 22)) & 1); + case 12: return ret + (capacity <= 21 ? 0 : capacity > 57 ? 0 : (0xe65522041 >> (capacity - 22)) & 1); + case 13: return ret + (capacity <= 27 ? 0 : capacity > 55 ? 0 : (0x8904081 >> (capacity - 28)) & 1); + case 14: return ret + (capacity <= 47 ? 0 : capacity > 48 ? 0 : 1); + default: return ret; + } +} + +size_t ComputeCapacity(uint32_t bits, size_t max_elements, uint32_t fpbits) { + if (bits == 0) return 0; + uint64_t base_fpbits = BaseFPBits(bits, max_elements); + // The fpbits provided by the base max_elements==capacity case are sufficient. + if (base_fpbits >= fpbits) return max_elements; + // Otherwise, increment capacity by ceil(fpbits / bits) beyond that. + return max_elements + (fpbits - base_fpbits + bits - 1) / bits; +} + +size_t ComputeMaxElements(uint32_t bits, size_t capacity, uint32_t fpbits) { + if (bits == 0) return 0; + // Start with max_elements=capacity, and decrease max_elements until the corresponding capacity is capacity. + size_t max_elements = capacity; + while (true) { + size_t capacity_for_max_elements = ComputeCapacity(bits, max_elements, fpbits); + CHECK_SAFE(capacity_for_max_elements >= capacity); + if (capacity_for_max_elements <= capacity) return max_elements; + size_t adjust = capacity_for_max_elements - capacity; + // Decrementing max_elements by N will at most decrement the corresponding capacity by N. + // As the observed capacity is adjust too high, we can safely decrease max_elements by adjust. + // If that brings us into negative max_elements territory, no solution exists and we return 0. + if (max_elements < adjust) return 0; + max_elements -= adjust; + } +} + +} // namespace + +#endif diff --git a/src/minisketch/src/fielddefines.h b/src/minisketch/src/fielddefines.h new file mode 100644 index 0000000000..510cb81f42 --- /dev/null +++ b/src/minisketch/src/fielddefines.h @@ -0,0 +1,560 @@ +/********************************************************************** + * Copyright (c) 2021 Cory Fields * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _MINISKETCH_FIELDDEFINES_H_ +#define _MINISKETCH_FIELDDEFINES_H_ + +/* + +This file translates external defines ENABLE_FIELD_FOO, DISABLE_FIELD_FOO, and +DISABLE_DEFAULT_FIELDS to internal ones: ENABLE_FIELD_INT_FOO. Only the +resulting internal includes should be used. + +Default: All fields enabled +-DDISABLE_FIELD_3: All fields except 3 are enabled +-DENABLE_FIELD_3: All fields enabled +-DDISABLE_DEFAULT_FIELDS: Error, no fields enabled +-DDISABLE_DEFAULT_FIELDS -DENABLE_FIELD_3: Only field 3 enabled +-DDISABLE_DEFAULT_FIELDS -DENABLE_FIELD_2 -DENABLE_FIELD_3: Only fields 2 and 3 are enabled +-DDISABLE_DEFAULT_FIELDS -DENABLE_FIELD_2 -DENABLE_FIELD_3 -DDISABLE_FIELD_3: Only field 2 enabled + +*/ + +#ifdef DISABLE_DEFAULT_FIELDS +#if defined(ENABLE_FIELD_2) && !defined(DISABLE_FIELD_2) +#define ENABLE_FIELD_INT_2 +#endif +#if defined(ENABLE_FIELD_3) && !defined(DISABLE_FIELD_3) +#define ENABLE_FIELD_INT_3 +#endif +#if defined(ENABLE_FIELD_4) && !defined(DISABLE_FIELD_4) +#define ENABLE_FIELD_INT_4 +#endif +#if defined(ENABLE_FIELD_5) && !defined(DISABLE_FIELD_5) +#define ENABLE_FIELD_INT_5 +#endif +#if defined(ENABLE_FIELD_6) && !defined(DISABLE_FIELD_6) +#define ENABLE_FIELD_INT_6 +#endif +#if defined(ENABLE_FIELD_7) && !defined(DISABLE_FIELD_7) +#define ENABLE_FIELD_INT_7 +#endif +#if defined(ENABLE_FIELD_8) && !defined(DISABLE_FIELD_8) +#define ENABLE_FIELD_INT_8 +#endif +#if defined(ENABLE_FIELD_9) && !defined(DISABLE_FIELD_9) +#define ENABLE_FIELD_INT_9 +#endif +#if defined(ENABLE_FIELD_10) && !defined(DISABLE_FIELD_10) +#define ENABLE_FIELD_INT_10 +#endif +#if defined(ENABLE_FIELD_11) && !defined(DISABLE_FIELD_11) +#define ENABLE_FIELD_INT_11 +#endif +#if defined(ENABLE_FIELD_12) && !defined(DISABLE_FIELD_12) +#define ENABLE_FIELD_INT_12 +#endif +#if defined(ENABLE_FIELD_13) && !defined(DISABLE_FIELD_13) +#define ENABLE_FIELD_INT_13 +#endif +#if defined(ENABLE_FIELD_14) && !defined(DISABLE_FIELD_14) +#define ENABLE_FIELD_INT_14 +#endif +#if defined(ENABLE_FIELD_15) && !defined(DISABLE_FIELD_15) +#define ENABLE_FIELD_INT_15 +#endif +#if defined(ENABLE_FIELD_16) && !defined(DISABLE_FIELD_16) +#define ENABLE_FIELD_INT_16 +#endif +#if defined(ENABLE_FIELD_17) && !defined(DISABLE_FIELD_17) +#define ENABLE_FIELD_INT_17 +#endif +#if defined(ENABLE_FIELD_18) && !defined(DISABLE_FIELD_18) +#define ENABLE_FIELD_INT_18 +#endif +#if defined(ENABLE_FIELD_19) && !defined(DISABLE_FIELD_19) +#define ENABLE_FIELD_INT_19 +#endif +#if defined(ENABLE_FIELD_20) && !defined(DISABLE_FIELD_20) +#define ENABLE_FIELD_INT_20 +#endif +#if defined(ENABLE_FIELD_21) && !defined(DISABLE_FIELD_21) +#define ENABLE_FIELD_INT_21 +#endif +#if defined(ENABLE_FIELD_22) && !defined(DISABLE_FIELD_22) +#define ENABLE_FIELD_INT_22 +#endif +#if defined(ENABLE_FIELD_23) && !defined(DISABLE_FIELD_23) +#define ENABLE_FIELD_INT_23 +#endif +#if defined(ENABLE_FIELD_24) && !defined(DISABLE_FIELD_24) +#define ENABLE_FIELD_INT_24 +#endif +#if defined(ENABLE_FIELD_25) && !defined(DISABLE_FIELD_25) +#define ENABLE_FIELD_INT_25 +#endif +#if defined(ENABLE_FIELD_26) && !defined(DISABLE_FIELD_26) +#define ENABLE_FIELD_INT_26 +#endif +#if defined(ENABLE_FIELD_27) && !defined(DISABLE_FIELD_27) +#define ENABLE_FIELD_INT_27 +#endif +#if defined(ENABLE_FIELD_28) && !defined(DISABLE_FIELD_28) +#define ENABLE_FIELD_INT_28 +#endif +#if defined(ENABLE_FIELD_29) && !defined(DISABLE_FIELD_29) +#define ENABLE_FIELD_INT_29 +#endif +#if defined(ENABLE_FIELD_30) && !defined(DISABLE_FIELD_30) +#define ENABLE_FIELD_INT_30 +#endif +#if defined(ENABLE_FIELD_31) && !defined(DISABLE_FIELD_31) +#define ENABLE_FIELD_INT_31 +#endif +#if defined(ENABLE_FIELD_32) && !defined(DISABLE_FIELD_32) +#define ENABLE_FIELD_INT_32 +#endif +#if defined(ENABLE_FIELD_33) && !defined(DISABLE_FIELD_33) +#define ENABLE_FIELD_INT_33 +#endif +#if defined(ENABLE_FIELD_34) && !defined(DISABLE_FIELD_34) +#define ENABLE_FIELD_INT_34 +#endif +#if defined(ENABLE_FIELD_35) && !defined(DISABLE_FIELD_35) +#define ENABLE_FIELD_INT_35 +#endif +#if defined(ENABLE_FIELD_36) && !defined(DISABLE_FIELD_36) +#define ENABLE_FIELD_INT_36 +#endif +#if defined(ENABLE_FIELD_37) && !defined(DISABLE_FIELD_37) +#define ENABLE_FIELD_INT_37 +#endif +#if defined(ENABLE_FIELD_38) && !defined(DISABLE_FIELD_38) +#define ENABLE_FIELD_INT_38 +#endif +#if defined(ENABLE_FIELD_39) && !defined(DISABLE_FIELD_39) +#define ENABLE_FIELD_INT_39 +#endif +#if defined(ENABLE_FIELD_40) && !defined(DISABLE_FIELD_40) +#define ENABLE_FIELD_INT_40 +#endif +#if defined(ENABLE_FIELD_41) && !defined(DISABLE_FIELD_41) +#define ENABLE_FIELD_INT_41 +#endif +#if defined(ENABLE_FIELD_42) && !defined(DISABLE_FIELD_42) +#define ENABLE_FIELD_INT_42 +#endif +#if defined(ENABLE_FIELD_43) && !defined(DISABLE_FIELD_43) +#define ENABLE_FIELD_INT_43 +#endif +#if defined(ENABLE_FIELD_44) && !defined(DISABLE_FIELD_44) +#define ENABLE_FIELD_INT_44 +#endif +#if defined(ENABLE_FIELD_45) && !defined(DISABLE_FIELD_45) +#define ENABLE_FIELD_INT_45 +#endif +#if defined(ENABLE_FIELD_46) && !defined(DISABLE_FIELD_46) +#define ENABLE_FIELD_INT_46 +#endif +#if defined(ENABLE_FIELD_47) && !defined(DISABLE_FIELD_47) +#define ENABLE_FIELD_INT_47 +#endif +#if defined(ENABLE_FIELD_48) && !defined(DISABLE_FIELD_48) +#define ENABLE_FIELD_INT_48 +#endif +#if defined(ENABLE_FIELD_49) && !defined(DISABLE_FIELD_49) +#define ENABLE_FIELD_INT_49 +#endif +#if defined(ENABLE_FIELD_50) && !defined(DISABLE_FIELD_50) +#define ENABLE_FIELD_INT_50 +#endif +#if defined(ENABLE_FIELD_51) && !defined(DISABLE_FIELD_51) +#define ENABLE_FIELD_INT_51 +#endif +#if defined(ENABLE_FIELD_52) && !defined(DISABLE_FIELD_52) +#define ENABLE_FIELD_INT_52 +#endif +#if defined(ENABLE_FIELD_53) && !defined(DISABLE_FIELD_53) +#define ENABLE_FIELD_INT_53 +#endif +#if defined(ENABLE_FIELD_54) && !defined(DISABLE_FIELD_54) +#define ENABLE_FIELD_INT_54 +#endif +#if defined(ENABLE_FIELD_55) && !defined(DISABLE_FIELD_55) +#define ENABLE_FIELD_INT_55 +#endif +#if defined(ENABLE_FIELD_56) && !defined(DISABLE_FIELD_56) +#define ENABLE_FIELD_INT_56 +#endif +#if defined(ENABLE_FIELD_57) && !defined(DISABLE_FIELD_57) +#define ENABLE_FIELD_INT_57 +#endif +#if defined(ENABLE_FIELD_58) && !defined(DISABLE_FIELD_58) +#define ENABLE_FIELD_INT_58 +#endif +#if defined(ENABLE_FIELD_59) && !defined(DISABLE_FIELD_59) +#define ENABLE_FIELD_INT_59 +#endif +#if defined(ENABLE_FIELD_60) && !defined(DISABLE_FIELD_60) +#define ENABLE_FIELD_INT_60 +#endif +#if defined(ENABLE_FIELD_61) && !defined(DISABLE_FIELD_61) +#define ENABLE_FIELD_INT_61 +#endif +#if defined(ENABLE_FIELD_62) && !defined(DISABLE_FIELD_62) +#define ENABLE_FIELD_INT_62 +#endif +#if defined(ENABLE_FIELD_63) && !defined(DISABLE_FIELD_63) +#define ENABLE_FIELD_INT_63 +#endif +#if defined(ENABLE_FIELD_64) && !defined(DISABLE_FIELD_64) +#define ENABLE_FIELD_INT_64 +#endif +#else +#if !defined(DISABLE_FIELD_2) +#define ENABLE_FIELD_INT_2 +#endif +#if !defined(DISABLE_FIELD_3) +#define ENABLE_FIELD_INT_3 +#endif +#if !defined(DISABLE_FIELD_4) +#define ENABLE_FIELD_INT_4 +#endif +#if !defined(DISABLE_FIELD_5) +#define ENABLE_FIELD_INT_5 +#endif +#if !defined(DISABLE_FIELD_6) +#define ENABLE_FIELD_INT_6 +#endif +#if !defined(DISABLE_FIELD_7) +#define ENABLE_FIELD_INT_7 +#endif +#if !defined(DISABLE_FIELD_8) +#define ENABLE_FIELD_INT_8 +#endif +#if !defined(DISABLE_FIELD_9) +#define ENABLE_FIELD_INT_9 +#endif +#if !defined(DISABLE_FIELD_10) +#define ENABLE_FIELD_INT_10 +#endif +#if !defined(DISABLE_FIELD_11) +#define ENABLE_FIELD_INT_11 +#endif +#if !defined(DISABLE_FIELD_12) +#define ENABLE_FIELD_INT_12 +#endif +#if !defined(DISABLE_FIELD_13) +#define ENABLE_FIELD_INT_13 +#endif +#if !defined(DISABLE_FIELD_14) +#define ENABLE_FIELD_INT_14 +#endif +#if !defined(DISABLE_FIELD_15) +#define ENABLE_FIELD_INT_15 +#endif +#if !defined(DISABLE_FIELD_16) +#define ENABLE_FIELD_INT_16 +#endif +#if !defined(DISABLE_FIELD_17) +#define ENABLE_FIELD_INT_17 +#endif +#if !defined(DISABLE_FIELD_18) +#define ENABLE_FIELD_INT_18 +#endif +#if !defined(DISABLE_FIELD_19) +#define ENABLE_FIELD_INT_19 +#endif +#if !defined(DISABLE_FIELD_20) +#define ENABLE_FIELD_INT_20 +#endif +#if !defined(DISABLE_FIELD_21) +#define ENABLE_FIELD_INT_21 +#endif +#if !defined(DISABLE_FIELD_22) +#define ENABLE_FIELD_INT_22 +#endif +#if !defined(DISABLE_FIELD_23) +#define ENABLE_FIELD_INT_23 +#endif +#if !defined(DISABLE_FIELD_24) +#define ENABLE_FIELD_INT_24 +#endif +#if !defined(DISABLE_FIELD_25) +#define ENABLE_FIELD_INT_25 +#endif +#if !defined(DISABLE_FIELD_26) +#define ENABLE_FIELD_INT_26 +#endif +#if !defined(DISABLE_FIELD_27) +#define ENABLE_FIELD_INT_27 +#endif +#if !defined(DISABLE_FIELD_28) +#define ENABLE_FIELD_INT_28 +#endif +#if !defined(DISABLE_FIELD_29) +#define ENABLE_FIELD_INT_29 +#endif +#if !defined(DISABLE_FIELD_30) +#define ENABLE_FIELD_INT_30 +#endif +#if !defined(DISABLE_FIELD_31) +#define ENABLE_FIELD_INT_31 +#endif +#if !defined(DISABLE_FIELD_32) +#define ENABLE_FIELD_INT_32 +#endif +#if !defined(DISABLE_FIELD_33) +#define ENABLE_FIELD_INT_33 +#endif +#if !defined(DISABLE_FIELD_34) +#define ENABLE_FIELD_INT_34 +#endif +#if !defined(DISABLE_FIELD_35) +#define ENABLE_FIELD_INT_35 +#endif +#if !defined(DISABLE_FIELD_36) +#define ENABLE_FIELD_INT_36 +#endif +#if !defined(DISABLE_FIELD_37) +#define ENABLE_FIELD_INT_37 +#endif +#if !defined(DISABLE_FIELD_38) +#define ENABLE_FIELD_INT_38 +#endif +#if !defined(DISABLE_FIELD_39) +#define ENABLE_FIELD_INT_39 +#endif +#if !defined(DISABLE_FIELD_40) +#define ENABLE_FIELD_INT_40 +#endif +#if !defined(DISABLE_FIELD_41) +#define ENABLE_FIELD_INT_41 +#endif +#if !defined(DISABLE_FIELD_42) +#define ENABLE_FIELD_INT_42 +#endif +#if !defined(DISABLE_FIELD_43) +#define ENABLE_FIELD_INT_43 +#endif +#if !defined(DISABLE_FIELD_44) +#define ENABLE_FIELD_INT_44 +#endif +#if !defined(DISABLE_FIELD_45) +#define ENABLE_FIELD_INT_45 +#endif +#if !defined(DISABLE_FIELD_46) +#define ENABLE_FIELD_INT_46 +#endif +#if !defined(DISABLE_FIELD_47) +#define ENABLE_FIELD_INT_47 +#endif +#if !defined(DISABLE_FIELD_48) +#define ENABLE_FIELD_INT_48 +#endif +#if !defined(DISABLE_FIELD_49) +#define ENABLE_FIELD_INT_49 +#endif +#if !defined(DISABLE_FIELD_50) +#define ENABLE_FIELD_INT_50 +#endif +#if !defined(DISABLE_FIELD_51) +#define ENABLE_FIELD_INT_51 +#endif +#if !defined(DISABLE_FIELD_52) +#define ENABLE_FIELD_INT_52 +#endif +#if !defined(DISABLE_FIELD_53) +#define ENABLE_FIELD_INT_53 +#endif +#if !defined(DISABLE_FIELD_54) +#define ENABLE_FIELD_INT_54 +#endif +#if !defined(DISABLE_FIELD_55) +#define ENABLE_FIELD_INT_55 +#endif +#if !defined(DISABLE_FIELD_56) +#define ENABLE_FIELD_INT_56 +#endif +#if !defined(DISABLE_FIELD_57) +#define ENABLE_FIELD_INT_57 +#endif +#if !defined(DISABLE_FIELD_58) +#define ENABLE_FIELD_INT_58 +#endif +#if !defined(DISABLE_FIELD_59) +#define ENABLE_FIELD_INT_59 +#endif +#if !defined(DISABLE_FIELD_60) +#define ENABLE_FIELD_INT_60 +#endif +#if !defined(DISABLE_FIELD_61) +#define ENABLE_FIELD_INT_61 +#endif +#if !defined(DISABLE_FIELD_62) +#define ENABLE_FIELD_INT_62 +#endif +#if !defined(DISABLE_FIELD_63) +#define ENABLE_FIELD_INT_63 +#endif +#if !defined(DISABLE_FIELD_64) +#define ENABLE_FIELD_INT_64 +#endif +#endif + +#if !defined(ENABLE_FIELD_INT_2) && \ + !defined(ENABLE_FIELD_INT_3) && \ + !defined(ENABLE_FIELD_INT_4) && \ + !defined(ENABLE_FIELD_INT_5) && \ + !defined(ENABLE_FIELD_INT_6) && \ + !defined(ENABLE_FIELD_INT_7) && \ + !defined(ENABLE_FIELD_INT_8) && \ + !defined(ENABLE_FIELD_INT_9) && \ + !defined(ENABLE_FIELD_INT_10) && \ + !defined(ENABLE_FIELD_INT_11) && \ + !defined(ENABLE_FIELD_INT_12) && \ + !defined(ENABLE_FIELD_INT_13) && \ + !defined(ENABLE_FIELD_INT_14) && \ + !defined(ENABLE_FIELD_INT_15) && \ + !defined(ENABLE_FIELD_INT_16) && \ + !defined(ENABLE_FIELD_INT_17) && \ + !defined(ENABLE_FIELD_INT_18) && \ + !defined(ENABLE_FIELD_INT_19) && \ + !defined(ENABLE_FIELD_INT_20) && \ + !defined(ENABLE_FIELD_INT_21) && \ + !defined(ENABLE_FIELD_INT_22) && \ + !defined(ENABLE_FIELD_INT_23) && \ + !defined(ENABLE_FIELD_INT_24) && \ + !defined(ENABLE_FIELD_INT_25) && \ + !defined(ENABLE_FIELD_INT_26) && \ + !defined(ENABLE_FIELD_INT_27) && \ + !defined(ENABLE_FIELD_INT_28) && \ + !defined(ENABLE_FIELD_INT_29) && \ + !defined(ENABLE_FIELD_INT_30) && \ + !defined(ENABLE_FIELD_INT_31) && \ + !defined(ENABLE_FIELD_INT_32) && \ + !defined(ENABLE_FIELD_INT_33) && \ + !defined(ENABLE_FIELD_INT_34) && \ + !defined(ENABLE_FIELD_INT_35) && \ + !defined(ENABLE_FIELD_INT_36) && \ + !defined(ENABLE_FIELD_INT_37) && \ + !defined(ENABLE_FIELD_INT_38) && \ + !defined(ENABLE_FIELD_INT_39) && \ + !defined(ENABLE_FIELD_INT_40) && \ + !defined(ENABLE_FIELD_INT_41) && \ + !defined(ENABLE_FIELD_INT_42) && \ + !defined(ENABLE_FIELD_INT_43) && \ + !defined(ENABLE_FIELD_INT_44) && \ + !defined(ENABLE_FIELD_INT_45) && \ + !defined(ENABLE_FIELD_INT_46) && \ + !defined(ENABLE_FIELD_INT_47) && \ + !defined(ENABLE_FIELD_INT_48) && \ + !defined(ENABLE_FIELD_INT_49) && \ + !defined(ENABLE_FIELD_INT_50) && \ + !defined(ENABLE_FIELD_INT_51) && \ + !defined(ENABLE_FIELD_INT_52) && \ + !defined(ENABLE_FIELD_INT_53) && \ + !defined(ENABLE_FIELD_INT_54) && \ + !defined(ENABLE_FIELD_INT_55) && \ + !defined(ENABLE_FIELD_INT_56) && \ + !defined(ENABLE_FIELD_INT_57) && \ + !defined(ENABLE_FIELD_INT_58) && \ + !defined(ENABLE_FIELD_INT_59) && \ + !defined(ENABLE_FIELD_INT_60) && \ + !defined(ENABLE_FIELD_INT_61) && \ + !defined(ENABLE_FIELD_INT_62) && \ + !defined(ENABLE_FIELD_INT_63) && \ + !defined(ENABLE_FIELD_INT_64) +#error No fields enabled +#endif + +#if defined(ENABLE_FIELD_INT_2) || \ + defined(ENABLE_FIELD_INT_3) || \ + defined(ENABLE_FIELD_INT_4) || \ + defined(ENABLE_FIELD_INT_5) || \ + defined(ENABLE_FIELD_INT_6) || \ + defined(ENABLE_FIELD_INT_7) || \ + defined(ENABLE_FIELD_INT_8) +#define ENABLE_FIELD_BYTES_INT_1 +#endif + +#if defined(ENABLE_FIELD_INT_9) || \ + defined(ENABLE_FIELD_INT_10) || \ + defined(ENABLE_FIELD_INT_11) || \ + defined(ENABLE_FIELD_INT_12) || \ + defined(ENABLE_FIELD_INT_13) || \ + defined(ENABLE_FIELD_INT_14) || \ + defined(ENABLE_FIELD_INT_15) || \ + defined(ENABLE_FIELD_INT_16) +#define ENABLE_FIELD_BYTES_INT_2 +#endif + +#if defined(ENABLE_FIELD_INT_17) || \ + defined(ENABLE_FIELD_INT_18) || \ + defined(ENABLE_FIELD_INT_19) || \ + defined(ENABLE_FIELD_INT_20) || \ + defined(ENABLE_FIELD_INT_21) || \ + defined(ENABLE_FIELD_INT_22) || \ + defined(ENABLE_FIELD_INT_23) || \ + defined(ENABLE_FIELD_INT_24) +#define ENABLE_FIELD_BYTES_INT_3 +#endif + +#if defined(ENABLE_FIELD_INT_25) || \ + defined(ENABLE_FIELD_INT_26) || \ + defined(ENABLE_FIELD_INT_27) || \ + defined(ENABLE_FIELD_INT_28) || \ + defined(ENABLE_FIELD_INT_29) || \ + defined(ENABLE_FIELD_INT_30) || \ + defined(ENABLE_FIELD_INT_31) || \ + defined(ENABLE_FIELD_INT_32) +#define ENABLE_FIELD_BYTES_INT_4 +#endif + +#if defined(ENABLE_FIELD_INT_33) || \ + defined(ENABLE_FIELD_INT_34) || \ + defined(ENABLE_FIELD_INT_35) || \ + defined(ENABLE_FIELD_INT_36) || \ + defined(ENABLE_FIELD_INT_37) || \ + defined(ENABLE_FIELD_INT_38) || \ + defined(ENABLE_FIELD_INT_39) || \ + defined(ENABLE_FIELD_INT_40) +#define ENABLE_FIELD_BYTES_INT_5 +#endif + +#if defined(ENABLE_FIELD_INT_41) || \ + defined(ENABLE_FIELD_INT_42) || \ + defined(ENABLE_FIELD_INT_43) || \ + defined(ENABLE_FIELD_INT_44) || \ + defined(ENABLE_FIELD_INT_45) || \ + defined(ENABLE_FIELD_INT_46) || \ + defined(ENABLE_FIELD_INT_47) || \ + defined(ENABLE_FIELD_INT_48) +#define ENABLE_FIELD_BYTES_INT_6 +#endif + +#if defined(ENABLE_FIELD_INT_49) || \ + defined(ENABLE_FIELD_INT_50) || \ + defined(ENABLE_FIELD_INT_51) || \ + defined(ENABLE_FIELD_INT_52) || \ + defined(ENABLE_FIELD_INT_53) || \ + defined(ENABLE_FIELD_INT_54) || \ + defined(ENABLE_FIELD_INT_55) || \ + defined(ENABLE_FIELD_INT_56) +#define ENABLE_FIELD_BYTES_INT_7 +#endif + +#if defined(ENABLE_FIELD_INT_57) || \ + defined(ENABLE_FIELD_INT_58) || \ + defined(ENABLE_FIELD_INT_59) || \ + defined(ENABLE_FIELD_INT_60) || \ + defined(ENABLE_FIELD_INT_61) || \ + defined(ENABLE_FIELD_INT_62) || \ + defined(ENABLE_FIELD_INT_63) || \ + defined(ENABLE_FIELD_INT_64) +#define ENABLE_FIELD_BYTES_INT_8 +#endif +#endif // _MINISKETCH_FIELDDEFINES_H_ diff --git a/src/minisketch/src/fields/clmul_1byte.cpp b/src/minisketch/src/fields/clmul_1byte.cpp new file mode 100644 index 0000000000..8826af9605 --- /dev/null +++ b/src/minisketch/src/fields/clmul_1byte.cpp @@ -0,0 +1,119 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_1) + +#include "clmul_common_impl.h" + +#include "../int_utils.h" +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_2 +// 2 bit field +typedef RecLinTrans<uint8_t, 2> StatTableTRI2; +constexpr StatTableTRI2 SQR_TABLE_TRI2({0x1, 0x3}); +constexpr StatTableTRI2 QRT_TABLE_TRI2({0x2, 0}); +typedef FieldTri<uint8_t, 2, 1, StatTableTRI2, &SQR_TABLE_TRI2, &QRT_TABLE_TRI2, &QRT_TABLE_TRI2, &QRT_TABLE_TRI2, &QRT_TABLE_TRI2, &QRT_TABLE_TRI2, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri2; +#endif + +#ifdef ENABLE_FIELD_INT_3 +// 3 bit field +typedef RecLinTrans<uint8_t, 3> StatTableTRI3; +constexpr StatTableTRI3 SQR_TABLE_TRI3({0x1, 0x4, 0x6}); +constexpr StatTableTRI3 QRT_TABLE_TRI3({0, 0x4, 0x6}); +typedef FieldTri<uint8_t, 3, 1, StatTableTRI3, &SQR_TABLE_TRI3, &QRT_TABLE_TRI3, &QRT_TABLE_TRI3, &QRT_TABLE_TRI3, &QRT_TABLE_TRI3, &QRT_TABLE_TRI3, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri3; +#endif + +#ifdef ENABLE_FIELD_INT_4 +// 4 bit field +typedef RecLinTrans<uint8_t, 4> StatTableTRI4; +constexpr StatTableTRI4 SQR_TABLE_TRI4({0x1, 0x4, 0x3, 0xc}); +constexpr StatTableTRI4 QRT_TABLE_TRI4({0x6, 0xa, 0x8, 0}); +typedef FieldTri<uint8_t, 4, 1, StatTableTRI4, &SQR_TABLE_TRI4, &QRT_TABLE_TRI4, &QRT_TABLE_TRI4, &QRT_TABLE_TRI4, &QRT_TABLE_TRI4, &QRT_TABLE_TRI4, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri4; +#endif + +#ifdef ENABLE_FIELD_INT_5 +// 5 bit field +typedef RecLinTrans<uint8_t, 5> StatTable5; +constexpr StatTable5 SQR_TABLE_5({0x1, 0x4, 0x10, 0xa, 0xd}); +constexpr StatTable5 SQR2_TABLE_5({0x1, 0x10, 0xd, 0xe, 0x1b}); +constexpr StatTable5 QRT_TABLE_5({0x14, 0x8, 0xa, 0, 0xe}); +typedef Field<uint8_t, 5, 5, StatTable5, &SQR_TABLE_5, &SQR2_TABLE_5, &QRT_TABLE_5, &QRT_TABLE_5, &QRT_TABLE_5, &QRT_TABLE_5, IdTrans, &ID_TRANS, &ID_TRANS> Field5; +typedef FieldTri<uint8_t, 5, 2, RecLinTrans<uint8_t, 5>, &SQR_TABLE_5, &SQR2_TABLE_5, &QRT_TABLE_5, &QRT_TABLE_5, &QRT_TABLE_5, &QRT_TABLE_5, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri5; +#endif + +#ifdef ENABLE_FIELD_INT_6 +// 6 bit field +typedef RecLinTrans<uint8_t, 6> StatTableTRI6; +constexpr StatTableTRI6 SQR_TABLE_TRI6({0x1, 0x4, 0x10, 0x3, 0xc, 0x30}); +constexpr StatTableTRI6 SQR2_TABLE_TRI6({0x1, 0x10, 0xc, 0x5, 0x13, 0x3c}); +constexpr StatTableTRI6 QRT_TABLE_TRI6({0x3a, 0x26, 0x24, 0x14, 0x20, 0}); +typedef FieldTri<uint8_t, 6, 1, StatTableTRI6, &SQR_TABLE_TRI6, &SQR2_TABLE_TRI6, &QRT_TABLE_TRI6, &QRT_TABLE_TRI6, &QRT_TABLE_TRI6, &QRT_TABLE_TRI6, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri6; +#endif + +#ifdef ENABLE_FIELD_INT_7 +// 7 bit field +typedef RecLinTrans<uint8_t, 4, 3> StatTableTRI7; +constexpr StatTableTRI7 SQR_TABLE_TRI7({0x1, 0x4, 0x10, 0x40, 0x6, 0x18, 0x60}); +constexpr StatTableTRI7 SQR2_TABLE_TRI7({0x1, 0x10, 0x6, 0x60, 0x14, 0x46, 0x78}); +constexpr StatTableTRI7 QRT_TABLE_TRI7({0, 0x14, 0x16, 0x72, 0x12, 0x40, 0x7a}); +typedef FieldTri<uint8_t, 7, 1, StatTableTRI7, &SQR_TABLE_TRI7, &SQR2_TABLE_TRI7, &QRT_TABLE_TRI7, &QRT_TABLE_TRI7, &QRT_TABLE_TRI7, &QRT_TABLE_TRI7, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri7; +#endif + +#ifdef ENABLE_FIELD_INT_8 +// 8 bit field +typedef RecLinTrans<uint8_t, 4, 4> StatTable8; +constexpr StatTable8 SQR_TABLE_8({0x1, 0x4, 0x10, 0x40, 0x1b, 0x6c, 0xab, 0x9a}); +constexpr StatTable8 SQR2_TABLE_8({0x1, 0x10, 0x1b, 0xab, 0x5e, 0x97, 0xb3, 0xc5}); +constexpr StatTable8 QRT_TABLE_8({0xbc, 0x2a, 0x28, 0x86, 0x2c, 0xde, 0x8e, 0}); +typedef Field<uint8_t, 8, 27, StatTable8, &SQR_TABLE_8, &SQR2_TABLE_8, &QRT_TABLE_8, &QRT_TABLE_8, &QRT_TABLE_8, &QRT_TABLE_8, IdTrans, &ID_TRANS, &ID_TRANS> Field8; +#endif +} + +Sketch* ConstructClMul1Byte(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_5 + case 5: return new SketchImpl<Field5>(implementation, 5); +#endif +#ifdef ENABLE_FIELD_INT_8 + case 8: return new SketchImpl<Field8>(implementation, 8); +#endif + } + return nullptr; +} + +Sketch* ConstructClMulTri1Byte(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_2 + case 2: return new SketchImpl<FieldTri2>(implementation, 2); +#endif +#ifdef ENABLE_FIELD_INT_3 + case 3: return new SketchImpl<FieldTri3>(implementation, 3); +#endif +#ifdef ENABLE_FIELD_INT_4 + case 4: return new SketchImpl<FieldTri4>(implementation, 4); +#endif +#ifdef ENABLE_FIELD_INT_5 + case 5: return new SketchImpl<FieldTri5>(implementation, 5); +#endif +#ifdef ENABLE_FIELD_INT_6 + case 6: return new SketchImpl<FieldTri6>(implementation, 6); +#endif +#ifdef ENABLE_FIELD_INT_7 + case 7: return new SketchImpl<FieldTri7>(implementation, 7); +#endif + } + return nullptr; +} diff --git a/src/minisketch/src/fields/clmul_2bytes.cpp b/src/minisketch/src/fields/clmul_2bytes.cpp new file mode 100644 index 0000000000..43930254dd --- /dev/null +++ b/src/minisketch/src/fields/clmul_2bytes.cpp @@ -0,0 +1,154 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_2) + +#include "clmul_common_impl.h" + +#include "../int_utils.h" +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_9 +// 9 bit field +typedef RecLinTrans<uint16_t, 5, 4> StatTableTRI9; +constexpr StatTableTRI9 SQR_TABLE_TRI9({0x1, 0x4, 0x10, 0x40, 0x100, 0x6, 0x18, 0x60, 0x180}); +constexpr StatTableTRI9 SQR2_TABLE_TRI9({0x1, 0x10, 0x100, 0x18, 0x180, 0x14, 0x140, 0x1e, 0x1e0}); +constexpr StatTableTRI9 SQR4_TABLE_TRI9({0x1, 0x180, 0x1e0, 0x198, 0x1fe, 0x80, 0xa0, 0x88, 0xaa}); +constexpr StatTableTRI9 QRT_TABLE_TRI9({0, 0x4e, 0x4c, 0x1aa, 0x48, 0x22, 0x1a2, 0x100, 0x58}); +typedef FieldTri<uint16_t, 9, 1, StatTableTRI9, &SQR_TABLE_TRI9, &SQR2_TABLE_TRI9, &SQR4_TABLE_TRI9, &QRT_TABLE_TRI9, &QRT_TABLE_TRI9, &QRT_TABLE_TRI9, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri9; +#endif + +#ifdef ENABLE_FIELD_INT_10 +// 10 bit field +typedef RecLinTrans<uint16_t, 5, 5> StatTable10; +constexpr StatTable10 SQR_TABLE_10({0x1, 0x4, 0x10, 0x40, 0x100, 0x9, 0x24, 0x90, 0x240, 0x112}); +constexpr StatTable10 SQR2_TABLE_10({0x1, 0x10, 0x100, 0x24, 0x240, 0x41, 0x19, 0x190, 0x136, 0x344}); +constexpr StatTable10 SQR4_TABLE_10({0x1, 0x240, 0x136, 0x141, 0x35d, 0x18, 0x265, 0x2e6, 0x227, 0x36b}); +constexpr StatTable10 QRT_TABLE_10({0xec, 0x86, 0x84, 0x30e, 0x80, 0x3c2, 0x306, 0, 0x90, 0x296}); +typedef Field<uint16_t, 10, 9, StatTable10, &SQR_TABLE_10, &SQR2_TABLE_10, &SQR4_TABLE_10, &QRT_TABLE_10, &QRT_TABLE_10, &QRT_TABLE_10, IdTrans, &ID_TRANS, &ID_TRANS> Field10; +typedef FieldTri<uint16_t, 10, 3, RecLinTrans<uint16_t, 5, 5>, &SQR_TABLE_10, &SQR2_TABLE_10, &SQR4_TABLE_10, &QRT_TABLE_10, &QRT_TABLE_10, &QRT_TABLE_10, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri10; +#endif + +#ifdef ENABLE_FIELD_INT_11 +// 11 bit field +typedef RecLinTrans<uint16_t, 6, 5> StatTable11; +constexpr StatTable11 SQR_TABLE_11({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0xa, 0x28, 0xa0, 0x280, 0x205}); +constexpr StatTable11 SQR2_TABLE_11({0x1, 0x10, 0x100, 0xa, 0xa0, 0x205, 0x44, 0x440, 0x428, 0x2a8, 0x291}); +constexpr StatTable11 SQR4_TABLE_11({0x1, 0xa0, 0x428, 0x1a, 0x645, 0x3a9, 0x144, 0x2d5, 0x9e, 0x4e7, 0x649}); +constexpr StatTable11 QRT_TABLE_11({0x734, 0x48, 0x4a, 0x1de, 0x4e, 0x35e, 0x1d6, 0x200, 0x5e, 0, 0x37e}); +typedef Field<uint16_t, 11, 5, StatTable11, &SQR_TABLE_11, &SQR2_TABLE_11, &SQR4_TABLE_11, nullptr, nullptr, &QRT_TABLE_11, IdTrans, &ID_TRANS, &ID_TRANS> Field11; +typedef FieldTri<uint16_t, 11, 2, RecLinTrans<uint16_t, 6, 5>, &SQR_TABLE_11, &SQR2_TABLE_11, &SQR4_TABLE_11, &QRT_TABLE_11, &QRT_TABLE_11, &QRT_TABLE_11, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri11; +#endif + +#ifdef ENABLE_FIELD_INT_12 +// 12 bit field +typedef RecLinTrans<uint16_t, 6, 6> StatTable12; +constexpr StatTable12 SQR_TABLE_12({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x9, 0x24, 0x90, 0x240, 0x900, 0x412}); +constexpr StatTable12 SQR2_TABLE_12({0x1, 0x10, 0x100, 0x9, 0x90, 0x900, 0x41, 0x410, 0x124, 0x249, 0x482, 0x804}); +constexpr StatTable12 SQR4_TABLE_12({0x1, 0x90, 0x124, 0x8, 0x480, 0x920, 0x40, 0x412, 0x924, 0x200, 0x82, 0x904}); +constexpr StatTable12 QRT_TABLE_12({0x48, 0xc10, 0xc12, 0x208, 0xc16, 0xd82, 0x200, 0x110, 0xc06, 0, 0xda2, 0x5a4}); +typedef Field<uint16_t, 12, 9, StatTable12, &SQR_TABLE_12, &SQR2_TABLE_12, &SQR4_TABLE_12, &QRT_TABLE_12, &QRT_TABLE_12, &QRT_TABLE_12, IdTrans, &ID_TRANS, &ID_TRANS> Field12; +typedef FieldTri<uint16_t, 12, 3, RecLinTrans<uint16_t, 6, 6>, &SQR_TABLE_12, &SQR2_TABLE_12, &SQR4_TABLE_12, &QRT_TABLE_12, &QRT_TABLE_12, &QRT_TABLE_12, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri12; +#endif + +#ifdef ENABLE_FIELD_INT_13 +// 13 bit field +typedef RecLinTrans<uint16_t, 5, 4, 4> StatTable13; +constexpr StatTable13 SQR_TABLE_13({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x36, 0xd8, 0x360, 0xd80, 0x161b, 0x185a}); +constexpr StatTable13 SQR2_TABLE_13({0x1, 0x10, 0x100, 0x1000, 0xd8, 0xd80, 0x185a, 0x514, 0x1176, 0x17b8, 0x1b75, 0x17ff, 0x1f05}); +constexpr StatTable13 SQR4_TABLE_13({0x1, 0xd8, 0x1176, 0x1f05, 0xd96, 0x18e8, 0x68, 0xbdb, 0x1a61, 0x1af2, 0x1a37, 0x3b9, 0x1440}); +constexpr StatTable13 QRT_TABLE_13({0xcfc, 0x1500, 0x1502, 0x382, 0x1506, 0x149c, 0x38a, 0x118, 0x1516, 0, 0x14bc, 0x100e, 0x3ca}); +typedef Field<uint16_t, 13, 27, StatTable13, &SQR_TABLE_13, &SQR2_TABLE_13, &SQR4_TABLE_13, &QRT_TABLE_13, &QRT_TABLE_13, &QRT_TABLE_13, IdTrans, &ID_TRANS, &ID_TRANS> Field13; +#endif + +#ifdef ENABLE_FIELD_INT_14 +// 14 bit field +typedef RecLinTrans<uint16_t, 5, 5, 4> StatTable14; +constexpr StatTable14 SQR_TABLE_14({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x21, 0x84, 0x210, 0x840, 0x2100, 0x442, 0x1108}); +constexpr StatTable14 SQR2_TABLE_14({0x1, 0x10, 0x100, 0x1000, 0x84, 0x840, 0x442, 0x401, 0x31, 0x310, 0x3100, 0x118c, 0x1844, 0x486}); +constexpr StatTable14 SQR4_TABLE_14({0x1, 0x84, 0x31, 0x1844, 0x501, 0x15ce, 0x3552, 0x3101, 0x8c5, 0x3a5, 0x1cf3, 0xd74, 0xc8a, 0x3411}); +constexpr StatTable14 QRT_TABLE_14({0x13f2, 0x206, 0x204, 0x3e06, 0x200, 0x1266, 0x3e0e, 0x114, 0x210, 0, 0x1246, 0x2848, 0x3e4e, 0x2258}); +typedef Field<uint16_t, 14, 33, StatTable14, &SQR_TABLE_14, &SQR2_TABLE_14, &SQR4_TABLE_14, &QRT_TABLE_14, &QRT_TABLE_14, &QRT_TABLE_14, IdTrans, &ID_TRANS, &ID_TRANS> Field14; +typedef FieldTri<uint16_t, 14, 5, RecLinTrans<uint16_t, 5, 5, 4>, &SQR_TABLE_14, &SQR2_TABLE_14, &SQR4_TABLE_14, &QRT_TABLE_14, &QRT_TABLE_14, &QRT_TABLE_14, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri14; +#endif + +#ifdef ENABLE_FIELD_INT_15 +// 15 bit field +typedef RecLinTrans<uint16_t, 5, 5, 5> StatTableTRI15; +constexpr StatTableTRI15 SQR_TABLE_TRI15({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x6, 0x18, 0x60, 0x180, 0x600, 0x1800, 0x6000}); +constexpr StatTableTRI15 SQR2_TABLE_TRI15({0x1, 0x10, 0x100, 0x1000, 0x6, 0x60, 0x600, 0x6000, 0x14, 0x140, 0x1400, 0x4006, 0x78, 0x780, 0x7800}); +constexpr StatTableTRI15 SQR4_TABLE_TRI15({0x1, 0x6, 0x14, 0x78, 0x110, 0x660, 0x1540, 0x7f80, 0x106, 0x614, 0x1478, 0x7910, 0x1666, 0x7554, 0x3ffe}); +constexpr StatTableTRI15 QRT_TABLE_TRI15({0, 0x114, 0x116, 0x428, 0x112, 0x137a, 0x420, 0x6d62, 0x102, 0x73a, 0x135a, 0x6460, 0x460, 0x4000, 0x6de2}); +typedef FieldTri<uint16_t, 15, 1, StatTableTRI15, &SQR_TABLE_TRI15, &SQR2_TABLE_TRI15, &SQR4_TABLE_TRI15, &QRT_TABLE_TRI15, &QRT_TABLE_TRI15, &QRT_TABLE_TRI15, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri15; +#endif + +#ifdef ENABLE_FIELD_INT_16 +// 16 bit field +typedef RecLinTrans<uint16_t, 6, 5, 5> StatTable16; +constexpr StatTable16 SQR_TABLE_16({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x2b, 0xac, 0x2b0, 0xac0, 0x2b00, 0xac00, 0xb056, 0xc10e}); +constexpr StatTable16 SQR2_TABLE_16({0x1, 0x10, 0x100, 0x1000, 0x2b, 0x2b0, 0x2b00, 0xb056, 0x445, 0x4450, 0x45ac, 0x5a6c, 0xa647, 0x657e, 0x571a, 0x7127}); +constexpr StatTable16 SQR4_TABLE_16({0x1, 0x2b, 0x445, 0xa647, 0x12a1, 0xf69d, 0x7f07, 0x9825, 0x6fad, 0x399d, 0xb515, 0xd7d1, 0x3fb4, 0x4b06, 0xe4df, 0x93c7}); +constexpr StatTable16 QRT_TABLE_16({0x732, 0x72b8, 0x72ba, 0x7e96, 0x72be, 0x78b2, 0x7e9e, 0x8cba, 0x72ae, 0xfa24, 0x7892, 0x5892, 0x7ede, 0xbec6, 0x8c3a, 0}); +typedef Field<uint16_t, 16, 43, StatTable16, &SQR_TABLE_16, &SQR2_TABLE_16, &SQR4_TABLE_16, &QRT_TABLE_16, &QRT_TABLE_16, &QRT_TABLE_16, IdTrans, &ID_TRANS, &ID_TRANS> Field16; +#endif +} + +Sketch* ConstructClMul2Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_10 + case 10: return new SketchImpl<Field10>(implementation, 10); +#endif +#ifdef ENABLE_FIELD_INT_11 + case 11: return new SketchImpl<Field11>(implementation, 11); +#endif +#ifdef ENABLE_FIELD_INT_12 + case 12: return new SketchImpl<Field12>(implementation, 12); +#endif +#ifdef ENABLE_FIELD_INT_13 + case 13: return new SketchImpl<Field13>(implementation, 13); +#endif +#ifdef ENABLE_FIELD_INT_14 + case 14: return new SketchImpl<Field14>(implementation, 14); +#endif +#ifdef ENABLE_FIELD_INT_16 + case 16: return new SketchImpl<Field16>(implementation, 16); +#endif + } + return nullptr; +} + +Sketch* ConstructClMulTri2Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_9 + case 9: return new SketchImpl<FieldTri9>(implementation, 9); +#endif +#ifdef ENABLE_FIELD_INT_10 + case 10: return new SketchImpl<FieldTri10>(implementation, 10); +#endif +#ifdef ENABLE_FIELD_INT_11 + case 11: return new SketchImpl<FieldTri11>(implementation, 11); +#endif +#ifdef ENABLE_FIELD_INT_12 + case 12: return new SketchImpl<FieldTri12>(implementation, 12); +#endif +#ifdef ENABLE_FIELD_INT_14 + case 14: return new SketchImpl<FieldTri14>(implementation, 14); +#endif +#ifdef ENABLE_FIELD_INT_15 + case 15: return new SketchImpl<FieldTri15>(implementation, 15); +#endif + } + return nullptr; +} diff --git a/src/minisketch/src/fields/clmul_3bytes.cpp b/src/minisketch/src/fields/clmul_3bytes.cpp new file mode 100644 index 0000000000..b473f66ba2 --- /dev/null +++ b/src/minisketch/src/fields/clmul_3bytes.cpp @@ -0,0 +1,166 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_3) + +#include "clmul_common_impl.h" + +#include "../int_utils.h" +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_17 +// 17 bit field +typedef RecLinTrans<uint32_t, 6, 6, 5> StatTable17; +constexpr StatTable17 SQR_TABLE_17({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x12, 0x48, 0x120, 0x480, 0x1200, 0x4800, 0x12000, 0x8012}); +constexpr StatTable17 SQR2_TABLE_17({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x48, 0x480, 0x4800, 0x8012, 0x104, 0x1040, 0x10400, 0x4048, 0x492, 0x4920, 0x9212, 0x12104}); +constexpr StatTable17 SQR4_TABLE_17({0x1, 0x10000, 0x8012, 0x4048, 0x12104, 0x1480, 0x5840, 0x14d20, 0x19202, 0x8112, 0x44c8, 0x13144, 0x5da0, 0x15850, 0x1cd7a, 0x1d34e, 0x1a484}); +constexpr StatTable17 SQR8_TABLE_17({0x1, 0x1a484, 0x1f24a, 0x1d572, 0x1eec4, 0x15448, 0xf9de, 0x9af0, 0x1ab78, 0x6048, 0xdc9a, 0x1eb24, 0x2ef4, 0x7c5e, 0x170b2, 0x16c1a, 0xa660}); +constexpr StatTable17 QRT_TABLE_17({0, 0x4c3e, 0x4c3c, 0x1a248, 0x4c38, 0x428, 0x1a240, 0x1b608, 0x4c28, 0x206, 0x408, 0x4000, 0x1a200, 0x18006, 0x1b688, 0x14d2e, 0x4d28}); +typedef Field<uint32_t, 17, 9, StatTable17, &SQR_TABLE_17, &SQR2_TABLE_17, &SQR4_TABLE_17, &SQR8_TABLE_17, &QRT_TABLE_17, &QRT_TABLE_17, IdTrans, &ID_TRANS, &ID_TRANS> Field17; +typedef FieldTri<uint32_t, 17, 3, RecLinTrans<uint32_t, 6, 6, 5>, &SQR_TABLE_17, &SQR2_TABLE_17, &SQR4_TABLE_17, &SQR8_TABLE_17, &QRT_TABLE_17, &QRT_TABLE_17, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri17; +#endif + +#ifdef ENABLE_FIELD_INT_18 +// 18 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6> StatTable18; +constexpr StatTable18 SQR_TABLE_18({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x9, 0x24, 0x90, 0x240, 0x900, 0x2400, 0x9000, 0x24000, 0x10012}); +constexpr StatTable18 SQR2_TABLE_18({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x24, 0x240, 0x2400, 0x24000, 0x41, 0x410, 0x4100, 0x1009, 0x10090, 0x924, 0x9240, 0x12412, 0x24104}); +constexpr StatTable18 SQR4_TABLE_18({0x1, 0x10000, 0x24000, 0x1009, 0x12412, 0x124, 0x201, 0x10480, 0x24820, 0x241, 0x10410, 0x24924, 0x8, 0x12, 0x20024, 0x8048, 0x12082, 0x920}); +constexpr StatTable18 SQR8_TABLE_18({0x1, 0x12082, 0x20904, 0x1000, 0x92, 0x904, 0x240, 0x12012, 0x4104, 0x41, 0x10080, 0x4924, 0x1009, 0x2412, 0x24804, 0x9240, 0x12410, 0x20}); +constexpr StatTable18 QRT_TABLE_18({0x9208, 0x422, 0x420, 0x8048, 0x424, 0x68b0, 0x8040, 0x30086, 0x434, 0x1040, 0x6890, 0x30ca2, 0x8000, 0x32896, 0x30006, 0, 0x534, 0x20532}); +typedef Field<uint32_t, 18, 9, StatTable18, &SQR_TABLE_18, &SQR2_TABLE_18, &SQR4_TABLE_18, &SQR8_TABLE_18, &QRT_TABLE_18, &QRT_TABLE_18, IdTrans, &ID_TRANS, &ID_TRANS> Field18; +typedef FieldTri<uint32_t, 18, 3, RecLinTrans<uint32_t, 6, 6, 6>, &SQR_TABLE_18, &SQR2_TABLE_18, &SQR4_TABLE_18, &SQR8_TABLE_18, &QRT_TABLE_18, &QRT_TABLE_18, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri18; +#endif + +#ifdef ENABLE_FIELD_INT_19 +// 19 bit field +typedef RecLinTrans<uint32_t, 5, 5, 5, 4> StatTable19; +constexpr StatTable19 SQR_TABLE_19({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x4e, 0x138, 0x4e0, 0x1380, 0x4e00, 0x13800, 0x4e000, 0x3804e, 0x6011f}); +constexpr StatTable19 SQR2_TABLE_19({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x4e, 0x4e0, 0x4e00, 0x4e000, 0x6011f, 0x1054, 0x10540, 0x544e, 0x544e0, 0x44f76, 0x4f658, 0x7649f, 0x6481a, 0x48004}); +constexpr StatTable19 SQR4_TABLE_19({0x1, 0x10000, 0x4e000, 0x544e, 0x7649f, 0x15f0, 0x5afa, 0x35b7d, 0x17dca, 0x7390f, 0x151ae, 0x3902b, 0x41e9c, 0x7f117, 0x23ec7, 0x62c2f, 0x5e852, 0x69238, 0x775c}); +constexpr StatTable19 SQR8_TABLE_19({0x1, 0x5e852, 0x394a3, 0x29f41, 0x618e5, 0x4210, 0x7add9, 0x31105, 0x5d098, 0x7bb13, 0x44f00, 0x966, 0x11ae6, 0x70901, 0x664bf, 0x67449, 0x3d2bf, 0x4cbf9, 0x54e0c}); +constexpr StatTable19 QRT_TABLE_19({0x5d6b0, 0x2f476, 0x2f474, 0x1d6a2, 0x2f470, 0x42a, 0x1d6aa, 0x1060, 0x2f460, 0x19e92, 0x40a, 0x1da98, 0x1d6ea, 0x28c78, 0x10e0, 0xf56a, 0x2f560, 0, 0x19c92}); +typedef Field<uint32_t, 19, 39, StatTable19, &SQR_TABLE_19, &SQR2_TABLE_19, &SQR4_TABLE_19, &SQR8_TABLE_19, &QRT_TABLE_19, &QRT_TABLE_19, IdTrans, &ID_TRANS, &ID_TRANS> Field19; +#endif + +#ifdef ENABLE_FIELD_INT_20 +// 20 bit field +typedef RecLinTrans<uint32_t, 5, 5, 5, 5> StatTable20; +constexpr StatTable20 SQR_TABLE_20({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x9, 0x24, 0x90, 0x240, 0x900, 0x2400, 0x9000, 0x24000, 0x90000, 0x40012}); +constexpr StatTable20 SQR2_TABLE_20({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x9, 0x90, 0x900, 0x9000, 0x90000, 0x41, 0x410, 0x4100, 0x41000, 0x10024, 0x249, 0x2490, 0x24900, 0x49012, 0x90104}); +constexpr StatTable20 SQR4_TABLE_20({0x1, 0x10000, 0x9000, 0x4100, 0x2490, 0x1001, 0x10900, 0x9410, 0x4349, 0x92594, 0x91, 0x10041, 0x19024, 0x4d112, 0x2599, 0x91091, 0x51941, 0x3dd34, 0x5d34b, 0x9b494}); +constexpr StatTable20 SQR8_TABLE_20({0x1, 0x51941, 0x880b5, 0x66d0, 0x46103, 0x19025, 0x45a49, 0x8a4b4, 0x80b45, 0x81f9f, 0xb081, 0x41040, 0xd19f5, 0xc11be, 0x4634b, 0xd8d70, 0x11027, 0xf8651, 0x141fa, 0xdc63}); +constexpr StatTable20 QRT_TABLE_20({0xc5dea, 0xc0110, 0xc0112, 0xe11de, 0xc0116, 0x24814, 0xe11d6, 0x20080, 0xc0106, 0xfe872, 0x24834, 0xe4106, 0xe1196, 0x1d9a4, 0x20000, 0x31190, 0xc0006, 0, 0xfea72, 0x7ea74}); +typedef Field<uint32_t, 20, 9, StatTable20, &SQR_TABLE_20, &SQR2_TABLE_20, &SQR4_TABLE_20, &SQR8_TABLE_20, &QRT_TABLE_20, &QRT_TABLE_20, IdTrans, &ID_TRANS, &ID_TRANS> Field20; +typedef FieldTri<uint32_t, 20, 3, RecLinTrans<uint32_t, 5, 5, 5, 5>, &SQR_TABLE_20, &SQR2_TABLE_20, &SQR4_TABLE_20, &SQR8_TABLE_20, &QRT_TABLE_20, &QRT_TABLE_20, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri20; +#endif + +#ifdef ENABLE_FIELD_INT_21 +// 21 bit field +typedef RecLinTrans<uint32_t, 6, 5, 5, 5> StatTable21; +constexpr StatTable21 SQR_TABLE_21({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0xa, 0x28, 0xa0, 0x280, 0xa00, 0x2800, 0xa000, 0x28000, 0xa0000, 0x80005}); +constexpr StatTable21 SQR2_TABLE_21({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x28, 0x280, 0x2800, 0x28000, 0x80005, 0x44, 0x440, 0x4400, 0x44000, 0x4000a, 0xaa, 0xaa0, 0xaa00, 0xaa000, 0xa0011}); +constexpr StatTable21 SQR4_TABLE_21({0x1, 0x10000, 0x2800, 0x440, 0xaa, 0xa0011, 0x101000, 0x28280, 0x4444, 0x40aaa, 0xaa101, 0x128, 0x8002d, 0xc4005, 0x4ea00, 0xba10, 0x101290, 0x1282c4, 0x6c44e, 0xeeeaa, 0xbaaa1}); +constexpr StatTable21 SQR8_TABLE_21({0x1, 0x101290, 0xc412d, 0x1ab101, 0x986d1, 0x1c6cc5, 0x3aa8c, 0x14b0fe, 0x1e7301, 0xb491d, 0x10d23e, 0xa4015, 0x4c2fa, 0xce8e5, 0xadfd9, 0xf110, 0x5220c, 0xf225f, 0xb8bdb, 0x159467, 0xc0df9}); +constexpr StatTable21 QRT_TABLE_21({0x1bd5fc, 0xbc196, 0xbc194, 0x74b96, 0xbc190, 0x1048, 0x74b9e, 0x672c8, 0xbc180, 0x4080, 0x1068, 0xc8200, 0x74bde, 0x64280, 0x67248, 0xc4280, 0xbc080, 0x80000, 0x4280, 0, 0x1468}); +typedef Field<uint32_t, 21, 5, StatTable21, &SQR_TABLE_21, &SQR2_TABLE_21, &SQR4_TABLE_21, &SQR8_TABLE_21, &QRT_TABLE_21, &QRT_TABLE_21, IdTrans, &ID_TRANS, &ID_TRANS> Field21; +typedef FieldTri<uint32_t, 21, 2, RecLinTrans<uint32_t, 6, 5, 5, 5>, &SQR_TABLE_21, &SQR2_TABLE_21, &SQR4_TABLE_21, &SQR8_TABLE_21, &QRT_TABLE_21, &QRT_TABLE_21, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri21; +#endif + +#ifdef ENABLE_FIELD_INT_22 +// 22 bit field +typedef RecLinTrans<uint32_t, 6, 6, 5, 5> StatTableTRI22; +constexpr StatTableTRI22 SQR_TABLE_TRI22({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x3, 0xc, 0x30, 0xc0, 0x300, 0xc00, 0x3000, 0xc000, 0x30000, 0xc0000, 0x300000}); +constexpr StatTableTRI22 SQR2_TABLE_TRI22({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0xc, 0xc0, 0xc00, 0xc000, 0xc0000, 0x5, 0x50, 0x500, 0x5000, 0x50000, 0x100003, 0x3c, 0x3c0, 0x3c00, 0x3c000, 0x3c0000}); +constexpr StatTableTRI22 SQR4_TABLE_TRI22({0x1, 0x10000, 0xc00, 0x50, 0x100003, 0x3c000, 0x1100, 0xcc, 0xc0005, 0x55000, 0x3fc0, 0x101, 0x1000c, 0xc0c00, 0x5050, 0x1003c3, 0x3c011, 0x111100, 0xcccc, 0xc0555, 0x15503f, 0x3fffc0}); +constexpr StatTableTRI22 SQR8_TABLE_TRI22({0x1, 0x3c011, 0x3ec1, 0x101103, 0x14503e, 0x28282, 0xd0009, 0x1d9c, 0xcc598, 0x25c81, 0x47304, 0xc0004, 0x3cc41, 0xcf758, 0x11415f, 0x1d11f7, 0x128280, 0x1b9027, 0x1070ce, 0x10eb5e, 0x5c0ec, 0x2097e0}); +constexpr StatTableTRI22 QRT_TABLE_TRI22({0x210d16, 0x104a, 0x1048, 0x4088, 0x104c, 0x200420, 0x4080, 0x492dc, 0x105c, 0x1a67f0, 0x200400, 0x21155c, 0x40c0, 0x20346c, 0x4925c, 0x1af7ac, 0x115c, 0x2274ac, 0x1a65f0, 0x2a65f0, 0x200000, 0}); +typedef FieldTri<uint32_t, 22, 1, StatTableTRI22, &SQR_TABLE_TRI22, &SQR2_TABLE_TRI22, &SQR4_TABLE_TRI22, &SQR8_TABLE_TRI22, &QRT_TABLE_TRI22, &QRT_TABLE_TRI22, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri22; +#endif + +#ifdef ENABLE_FIELD_INT_23 +// 23 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6, 5> StatTable23; +constexpr StatTable23 SQR_TABLE_23({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x42, 0x108, 0x420, 0x1080, 0x4200, 0x10800, 0x42000, 0x108000, 0x420000, 0x80042, 0x200108}); +constexpr StatTable23 SQR2_TABLE_23({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x42, 0x420, 0x4200, 0x42000, 0x420000, 0x200108, 0x1004, 0x10040, 0x100400, 0x4042, 0x40420, 0x404200, 0x42108, 0x421080, 0x210908, 0x109004, 0x90002}); +constexpr StatTable23 SQR4_TABLE_23({0x1, 0x10000, 0x4200, 0x1004, 0x40420, 0x210908, 0x52, 0x520000, 0x142400, 0x52148, 0x494202, 0x10c204, 0x1104, 0x40462, 0x630908, 0x100452, 0x562108, 0x1d2402, 0x57348, 0x495626, 0x34c72c, 0x21584e, 0x4614b0}); +constexpr StatTable23 SQR8_TABLE_23({0x1, 0x562108, 0x662840, 0x5304, 0x6d3842, 0x738f46, 0x50472, 0x6ff79e, 0x7cf204, 0x436274, 0x3e4bde, 0x42a93e, 0x147704, 0x6c3810, 0x28bff4, 0x78815c, 0x7ab4b0, 0x62852a, 0x255b30, 0x5653d0, 0x1afd36, 0x5f118, 0x601dd4}); +constexpr StatTable23 QRT_TABLE_23({0, 0x1040, 0x1042, 0x43056, 0x1046, 0x121d76, 0x4305e, 0x40a0, 0x1056, 0x15176, 0x121d56, 0x7ee1f6, 0x4301e, 0x40000, 0x4020, 0x4f0be, 0x1156, 0x7cf0a0, 0x15376, 0x1ee9e8, 0x121956, 0x3ac9f6, 0x7ee9f6}); +typedef Field<uint32_t, 23, 33, StatTable23, &SQR_TABLE_23, &SQR2_TABLE_23, &SQR4_TABLE_23, &SQR8_TABLE_23, &QRT_TABLE_23, &QRT_TABLE_23, IdTrans, &ID_TRANS, &ID_TRANS> Field23; +typedef FieldTri<uint32_t, 23, 5, RecLinTrans<uint32_t, 6, 6, 6, 5>, &SQR_TABLE_23, &SQR2_TABLE_23, &SQR4_TABLE_23, &SQR8_TABLE_23, nullptr, &QRT_TABLE_23, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri23; +#endif + +#ifdef ENABLE_FIELD_INT_24 +// 24 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6, 6> StatTable24; +constexpr StatTable24 SQR_TABLE_24({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1b, 0x6c, 0x1b0, 0x6c0, 0x1b00, 0x6c00, 0x1b000, 0x6c000, 0x1b0000, 0x6c0000, 0xb0001b, 0xc0005a}); +constexpr StatTable24 SQR2_TABLE_24({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1b, 0x1b0, 0x1b00, 0x1b000, 0x1b0000, 0xb0001b, 0x145, 0x1450, 0x14500, 0x145000, 0x45001b, 0x5001dc, 0x1db7, 0x1db70, 0x1db700, 0xdb701b, 0xb7011f, 0x701105}); +constexpr StatTable24 SQR4_TABLE_24({0x1, 0x10000, 0x1b00, 0x145, 0x45001b, 0x1db700, 0x11011, 0x111ab0, 0xb1aa5e, 0x51450e, 0x96db7, 0xb7c60f, 0x1a1a, 0x1a015e, 0x5f5e1b, 0x1ceef2, 0xf30ca2, 0xabbdb4, 0xba1aff, 0xf0bf5e, 0x579fc9, 0xce3da9, 0xa2c07f, 0x71dd40}); +constexpr StatTable24 SQR8_TABLE_24({0x1, 0xf30ca2, 0x573345, 0xb0a14e, 0xafd77d, 0x1419b, 0xb616a2, 0xba7db, 0xbe1560, 0xe0d0a3, 0x15bf5, 0x1056dd, 0xa29845, 0xf83d32, 0x13e0e9, 0xe2d8d3, 0xa10841, 0x57ac5a, 0x1c432f, 0x57044e, 0x454fba, 0x2bb37c, 0xf50fa, 0x85d5b9}); +constexpr StatTable24 QRT_TABLE_24({0x104e, 0xaf42a8, 0xaf42aa, 0xb78186, 0xaf42ae, 0x4090, 0xb7818e, 0x4a37c, 0xaf42be, 0x3688c0, 0x40b0, 0x80080e, 0xb781ce, 0xaf2232, 0x4a3fc, 0x856a82, 0xaf43be, 0x29c970, 0x368ac0, 0x968ace, 0x44b0, 0x77d570, 0x80000e, 0}); +typedef Field<uint32_t, 24, 27, StatTable24, &SQR_TABLE_24, &SQR2_TABLE_24, &SQR4_TABLE_24, &SQR8_TABLE_24, &QRT_TABLE_24, &QRT_TABLE_24, IdTrans, &ID_TRANS, &ID_TRANS> Field24; +#endif +} + +Sketch* ConstructClMul3Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_17 + case 17: return new SketchImpl<Field17>(implementation, 17); +#endif +#ifdef ENABLE_FIELD_INT_18 + case 18: return new SketchImpl<Field18>(implementation, 18); +#endif +#ifdef ENABLE_FIELD_INT_19 + case 19: return new SketchImpl<Field19>(implementation, 19); +#endif +#ifdef ENABLE_FIELD_INT_20 + case 20: return new SketchImpl<Field20>(implementation, 20); +#endif +#ifdef ENABLE_FIELD_INT_21 + case 21: return new SketchImpl<Field21>(implementation, 21); +#endif +#ifdef ENABLE_FIELD_INT_23 + case 23: return new SketchImpl<Field23>(implementation, 23); +#endif +#ifdef ENABLE_FIELD_INT_24 + case 24: return new SketchImpl<Field24>(implementation, 24); +#endif + } + return nullptr; +} + +Sketch* ConstructClMulTri3Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_17 + case 17: return new SketchImpl<FieldTri17>(implementation, 17); +#endif +#ifdef ENABLE_FIELD_INT_18 + case 18: return new SketchImpl<FieldTri18>(implementation, 18); +#endif +#ifdef ENABLE_FIELD_INT_20 + case 20: return new SketchImpl<FieldTri20>(implementation, 20); +#endif +#ifdef ENABLE_FIELD_INT_21 + case 21: return new SketchImpl<FieldTri21>(implementation, 21); +#endif +#ifdef ENABLE_FIELD_INT_22 + case 22: return new SketchImpl<FieldTri22>(implementation, 22); +#endif +#ifdef ENABLE_FIELD_INT_23 + case 23: return new SketchImpl<FieldTri23>(implementation, 23); +#endif + } + return nullptr; +} diff --git a/src/minisketch/src/fields/clmul_4bytes.cpp b/src/minisketch/src/fields/clmul_4bytes.cpp new file mode 100644 index 0000000000..c65974394c --- /dev/null +++ b/src/minisketch/src/fields/clmul_4bytes.cpp @@ -0,0 +1,158 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_4) + +#include "clmul_common_impl.h" + +#include "../int_utils.h" +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_25 +// 25 bit field +typedef RecLinTrans<uint32_t, 5, 5, 5, 5, 5> StatTable25; +constexpr StatTable25 SQR_TABLE_25({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x12, 0x48, 0x120, 0x480, 0x1200, 0x4800, 0x12000, 0x48000, 0x120000, 0x480000, 0x1200000, 0x800012}); +constexpr StatTable25 SQR2_TABLE_25({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x48, 0x480, 0x4800, 0x48000, 0x480000, 0x800012, 0x104, 0x1040, 0x10400, 0x104000, 0x1040000, 0x400048, 0x492, 0x4920, 0x49200, 0x492000, 0x920012, 0x1200104}); +constexpr StatTable25 SQR4_TABLE_25({0x1, 0x10000, 0x480, 0x800012, 0x104000, 0x4920, 0x1200104, 0x1001000, 0x48048, 0x481040, 0x410448, 0x492492, 0x930002, 0x580, 0x1800012, 0x14c000, 0x5960, 0x160014c, 0x1493000, 0x58058, 0x5814c0, 0xc14c5a, 0x596596, 0x1974922, 0x1249684}); +constexpr StatTable25 SQR8_TABLE_25({0x1, 0x5960, 0x1411448, 0x1860922, 0x1d814d2, 0x1cdede8, 0x1e15e16, 0x1b79686, 0xfdf116, 0x1efe4c8, 0x1b839a8, 0x10ced66, 0xae05ce, 0x1459400, 0xa29fa6, 0x85e4d2, 0x7eecee, 0x183a96, 0x1eb2fa8, 0xede876, 0xf6e440, 0x1f7140a, 0xd07d7c, 0x10e4ea2, 0x1222a54}); +constexpr StatTable25 QRT_TABLE_25({0, 0x482110, 0x482112, 0x1b3c3e6, 0x482116, 0x4960ae, 0x1b3c3ee, 0x4088, 0x482106, 0x58a726, 0x49608e, 0x5ce52e, 0x1b3c3ae, 0x2006, 0x4008, 0x1c1a8, 0x482006, 0x1e96488, 0x58a526, 0x400000, 0x49648e, 0x1800006, 0x5ced2e, 0xb3d3a8, 0x1b3d3ae}); +typedef Field<uint32_t, 25, 9, StatTable25, &SQR_TABLE_25, &SQR2_TABLE_25, &SQR4_TABLE_25, &SQR8_TABLE_25, &QRT_TABLE_25, &QRT_TABLE_25, IdTrans, &ID_TRANS, &ID_TRANS> Field25; +typedef FieldTri<uint32_t, 25, 3, RecLinTrans<uint32_t, 5, 5, 5, 5, 5>, &SQR_TABLE_25, &SQR2_TABLE_25, &SQR4_TABLE_25, &SQR8_TABLE_25, &QRT_TABLE_25, &QRT_TABLE_25, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri25; +#endif + +#ifdef ENABLE_FIELD_INT_26 +// 26 bit field +typedef RecLinTrans<uint32_t, 6, 5, 5, 5, 5> StatTable26; +constexpr StatTable26 SQR_TABLE_26({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x1b, 0x6c, 0x1b0, 0x6c0, 0x1b00, 0x6c00, 0x1b000, 0x6c000, 0x1b0000, 0x6c0000, 0x1b00000, 0x2c0001b, 0x300005a}); +constexpr StatTable26 SQR2_TABLE_26({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x6c, 0x6c0, 0x6c00, 0x6c000, 0x6c0000, 0x2c0001b, 0x145, 0x1450, 0x14500, 0x145000, 0x1450000, 0x500077, 0x100076b, 0x76dc, 0x76dc0, 0x76dc00, 0x36dc01b, 0x2dc011f, 0x1c01105}); +constexpr StatTable26 SQR4_TABLE_26({0x1, 0x10000, 0x6c0, 0x2c0001b, 0x145000, 0x76dc, 0x2dc011f, 0x1101100, 0x106ac6c, 0x6ad515, 0x1145127, 0x121b6dc, 0x2da1d0f, 0x10007c1, 0x3c7c01b, 0x128290, 0x29062e0, 0x2ee8d68, 0x167abcd, 0x3cabbce, 0x3c7a862, 0x6b83ce, 0x3cf5620, 0x229b787, 0x38a6b0f, 0x3071ade}); +constexpr StatTable26 SQR8_TABLE_26({0x1, 0x29062e0, 0x2b2942d, 0x34ab63, 0x3bddebb, 0x7b1823, 0x58b9ae, 0x391720e, 0x1385e18, 0x3891746, 0x13069c5, 0x2dfd089, 0x12a35ff, 0x3e534f, 0x172c6a2, 0x55338f, 0x3887137, 0x3f45b03, 0x164a695, 0x2c7e7ef, 0x29c907d, 0x636c85, 0x3db4007, 0x97e7ff, 0x3cbfe55, 0x31c0d96}); +constexpr StatTable26 QRT_TABLE_26({0x217b530, 0x2ae82a8, 0x2ae82aa, 0x2001046, 0x2ae82ae, 0x2de032e, 0x200104e, 0x70c10c, 0x2ae82be, 0x20151f2, 0x2de030e, 0xbc1400, 0x200100e, 0x178570, 0x70c18c, 0x2ae4232, 0x2ae83be, 0x211d742, 0x20153f2, 0x21f54f2, 0x2de070e, 0x5e0700, 0xbc1c00, 0x3abb97e, 0x200000e, 0}); +typedef Field<uint32_t, 26, 27, StatTable26, &SQR_TABLE_26, &SQR2_TABLE_26, &SQR4_TABLE_26, &SQR8_TABLE_26, &QRT_TABLE_26, &QRT_TABLE_26, IdTrans, &ID_TRANS, &ID_TRANS> Field26; +#endif + +#ifdef ENABLE_FIELD_INT_27 +// 27 bit field +typedef RecLinTrans<uint32_t, 6, 6, 5, 5, 5> StatTable27; +constexpr StatTable27 SQR_TABLE_27({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x4e, 0x138, 0x4e0, 0x1380, 0x4e00, 0x13800, 0x4e000, 0x138000, 0x4e0000, 0x1380000, 0x4e00000, 0x380004e, 0x600011f}); +constexpr StatTable27 SQR2_TABLE_27({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x4e, 0x4e0, 0x4e00, 0x4e000, 0x4e0000, 0x4e00000, 0x600011f, 0x1054, 0x10540, 0x105400, 0x1054000, 0x54004e, 0x54004e0, 0x4004f76, 0x4f658, 0x4f6580, 0x4f65800, 0x765811f, 0x658101a, 0x5810004}); +constexpr StatTable27 SQR4_TABLE_27({0x1, 0x10000, 0x4e0, 0x4e00000, 0x105400, 0x4004f76, 0x765811f, 0x1001110, 0x114e04e, 0x4abe54, 0x6551445, 0x45e212e, 0x13ccbdc, 0x3d805ef, 0x5e10100, 0x114b0e0, 0xe4bf22, 0x721c505, 0x51b3ba8, 0x3bf04d5, 0x4dabba0, 0x3b0aa45, 0x24a80cb, 0xc3d4b0, 0x4b34626, 0x6372e18, 0x6028c1b}); +constexpr StatTable27 SQR8_TABLE_27({0x1, 0xe4bf22, 0x430cb3c, 0x73b7225, 0x6526539, 0x3c278e3, 0x4724a6e, 0x48b39b4, 0x1dbf7de, 0x106508, 0x3564785, 0x33ae33f, 0x61d6685, 0x6adaca3, 0x2786b6f, 0x4e76784, 0x869f42, 0x466b048, 0x415e00e, 0x46c3c9a, 0x73ffd91, 0x49002e0, 0x3734fed, 0x3c04a43, 0x191d3ee, 0xe828b9, 0xfab68c}); +constexpr StatTable27 QRT_TABLE_27({0x6bf0530, 0x2be4496, 0x2be4494, 0x2bf0522, 0x2be4490, 0x1896cca, 0x2bf052a, 0x408a, 0x2be4480, 0x368ae72, 0x1896cea, 0x18d2ee0, 0x2bf056a, 0x1c76d6a, 0x400a, 0x336e9f8, 0x2be4580, 0x36baf12, 0x368ac72, 0x430360, 0x18968ea, 0x34a6b80, 0x18d26e0, 0xbf1560, 0x2bf156a, 0, 0x1c74d6a}); +typedef Field<uint32_t, 27, 39, StatTable27, &SQR_TABLE_27, &SQR2_TABLE_27, &SQR4_TABLE_27, &SQR8_TABLE_27, &QRT_TABLE_27, &QRT_TABLE_27, IdTrans, &ID_TRANS, &ID_TRANS> Field27; +#endif + +#ifdef ENABLE_FIELD_INT_28 +// 28 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6, 5, 5> StatTableTRI28; +constexpr StatTableTRI28 SQR_TABLE_TRI28({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x3, 0xc, 0x30, 0xc0, 0x300, 0xc00, 0x3000, 0xc000, 0x30000, 0xc0000, 0x300000, 0xc00000, 0x3000000, 0xc000000}); +constexpr StatTableTRI28 SQR2_TABLE_TRI28({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x3, 0x30, 0x300, 0x3000, 0x30000, 0x300000, 0x3000000, 0x5, 0x50, 0x500, 0x5000, 0x50000, 0x500000, 0x5000000, 0xf, 0xf0, 0xf00, 0xf000, 0xf0000, 0xf00000, 0xf000000}); +constexpr StatTableTRI28 SQR4_TABLE_TRI28({0x1, 0x10000, 0x30, 0x300000, 0x500, 0x5000000, 0xf000, 0x11, 0x110000, 0x330, 0x3300000, 0x5500, 0x500000f, 0xff000, 0x101, 0x1010000, 0x3030, 0x300005, 0x50500, 0x50000f0, 0xf0f000, 0x1111, 0x1110003, 0x33330, 0x3300055, 0x555500, 0x5000fff, 0xffff000}); +constexpr StatTableTRI28 SQR8_TABLE_TRI28({0x1, 0x3030, 0x5000500, 0xf0e111, 0x3210000, 0x6300faa, 0x40ef10e, 0x501, 0xf0c030, 0x5110630, 0x395b444, 0x621010e, 0x6010f9b, 0x13bc4cb, 0x110001, 0x3303065, 0xff50f, 0xf0e120, 0x3243530, 0x330fabb, 0x5ec232c, 0x511050e, 0x3c1c064, 0x2ec60a, 0x3954175, 0x7c5c43d, 0x20acba, 0x943bc43}); +constexpr StatTableTRI28 QRT_TABLE_TRI28({0x121d57a, 0x40216, 0x40214, 0x8112578, 0x40210, 0x10110, 0x8112570, 0x12597ec, 0x40200, 0x6983e00, 0x10130, 0x972b99c, 0x8112530, 0x8002000, 0x125976c, 0x815a76c, 0x40300, 0x936b29c, 0x6983c00, 0x97bb8ac, 0x10530, 0x9103000, 0x972b19c, 0xf6384ac, 0x8113530, 0x4113530, 0x8000000, 0}); +typedef FieldTri<uint32_t, 28, 1, StatTableTRI28, &SQR_TABLE_TRI28, &SQR2_TABLE_TRI28, &SQR4_TABLE_TRI28, &SQR8_TABLE_TRI28, &QRT_TABLE_TRI28, &QRT_TABLE_TRI28, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri28; +#endif + +#ifdef ENABLE_FIELD_INT_29 +// 29 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6, 6, 5> StatTable29; +constexpr StatTable29 SQR_TABLE_29({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0xa, 0x28, 0xa0, 0x280, 0xa00, 0x2800, 0xa000, 0x28000, 0xa0000, 0x280000, 0xa00000, 0x2800000, 0xa000000, 0x8000005}); +constexpr StatTable29 SQR2_TABLE_29({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x28, 0x280, 0x2800, 0x28000, 0x280000, 0x2800000, 0x8000005, 0x44, 0x440, 0x4400, 0x44000, 0x440000, 0x4400000, 0x400000a, 0xaa, 0xaa0, 0xaa00, 0xaa000, 0xaa0000, 0xaa00000, 0xa000011}); +constexpr StatTable29 SQR4_TABLE_29({0x1, 0x10000, 0x28, 0x280000, 0x440, 0x4400000, 0xaa00, 0xa000011, 0x101000, 0x10000280, 0x2828000, 0x4444, 0x444000a, 0xaaaa0, 0xaa00101, 0x1000100, 0x1002800, 0x8002805, 0x8044005, 0x440aa, 0xaa00aa, 0xaa1010, 0x10101010, 0x10128280, 0x28282c4, 0x2c44444, 0x4444eaa, 0xeaaaaaa, 0xaaba001}); +constexpr StatTable29 SQR8_TABLE_29({0x1, 0x1002800, 0x4680000, 0xae50ba, 0x2822a00, 0x14545eba, 0x110aed64, 0xc6eeaaf, 0x4ee00a0, 0x10aba290, 0x1bd6efc1, 0x8222b29, 0x1c791ebf, 0x174e85da, 0x1cc66c7f, 0x29292c4, 0x2886c20, 0xea04467, 0xc0eeb87, 0xccd4115, 0x16d5fa2e, 0x1cf8fe75, 0xe45a4e1, 0x19018b3f, 0x1d64778, 0x2e0bdf8, 0xa1bd96b, 0xff5b70e, 0x14d89770}); +constexpr StatTable29 QRT_TABLE_29({0x1b8351dc, 0xb87135e, 0xb87135c, 0xda7b35e, 0xb871358, 0x621a116, 0xda7b356, 0x40200, 0xb871348, 0xc9e2620, 0x621a136, 0x478b16, 0xda7b316, 0x6762e20, 0x40280, 0x6202000, 0xb871248, 0x627a316, 0xc9e2420, 0xcd1ad36, 0x621a536, 0x760e20, 0x478316, 0xa760e20, 0xda7a316, 0x8000000, 0x6760e20, 0, 0x44280}); +typedef Field<uint32_t, 29, 5, StatTable29, &SQR_TABLE_29, &SQR2_TABLE_29, &SQR4_TABLE_29, &SQR8_TABLE_29, &QRT_TABLE_29, &QRT_TABLE_29, IdTrans, &ID_TRANS, &ID_TRANS> Field29; +typedef FieldTri<uint32_t, 29, 2, RecLinTrans<uint32_t, 6, 6, 6, 6, 5>, &SQR_TABLE_29, &SQR2_TABLE_29, &SQR4_TABLE_29, &SQR8_TABLE_29, &QRT_TABLE_29, &QRT_TABLE_29, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri29; +#endif + +#ifdef ENABLE_FIELD_INT_30 +// 30 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6, 6, 6> StatTableTRI30; +constexpr StatTableTRI30 SQR_TABLE_TRI30({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x3, 0xc, 0x30, 0xc0, 0x300, 0xc00, 0x3000, 0xc000, 0x30000, 0xc0000, 0x300000, 0xc00000, 0x3000000, 0xc000000, 0x30000000}); +constexpr StatTableTRI30 SQR2_TABLE_TRI30({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0xc, 0xc0, 0xc00, 0xc000, 0xc0000, 0xc00000, 0xc000000, 0x5, 0x50, 0x500, 0x5000, 0x50000, 0x500000, 0x5000000, 0x10000003, 0x3c, 0x3c0, 0x3c00, 0x3c000, 0x3c0000, 0x3c00000, 0x3c000000}); +constexpr StatTableTRI30 SQR4_TABLE_TRI30({0x1, 0x10000, 0xc, 0xc0000, 0x50, 0x500000, 0x3c0, 0x3c00000, 0x1100, 0x11000000, 0xcc00, 0xc000005, 0x55000, 0x1000003f, 0x3fc000, 0x101, 0x1010000, 0xc0c, 0xc0c0000, 0x5050, 0x10500003, 0x3c3c0, 0x3c00011, 0x111100, 0x110000cc, 0xcccc00, 0xc000555, 0x5555000, 0x10003fff, 0x3fffc000}); +constexpr StatTableTRI30 SQR8_TABLE_TRI30({0x1, 0x1010000, 0xc000c, 0xc0c5050, 0x390, 0x13900012, 0x12c012c0, 0x121ddddd, 0x54100, 0x1003f33, 0xc3f0d04, 0x9555558, 0xd379000, 0x105d3fa2, 0x1d615e9e, 0x1101, 0x100100cc, 0xc0ccc09, 0x5590505, 0x3a9390, 0x3913fec, 0x13fedfcd, 0x121ddd8c, 0x11544103, 0x2cc3cff, 0x3e24c45, 0x9558bc8, 0x3a7958b, 0x1e98b158, 0x29d629e9}); +constexpr StatTableTRI30 QRT_TABLE_TRI30({0x2159df4a, 0x109134a, 0x1091348, 0x10114, 0x109134c, 0x3a203420, 0x1011c, 0x20004080, 0x109135c, 0x2005439c, 0x3a203400, 0x100400, 0x1015c, 0x3eb21930, 0x20004000, 0x20504c00, 0x109125c, 0x3b2b276c, 0x2005419c, 0x210450c0, 0x3a203000, 0x3e93186c, 0x100c00, 0x3aa23530, 0x1115c, 0x6b3286c, 0x3eb23930, 0xeb23930, 0x20000000, 0}); +typedef FieldTri<uint32_t, 30, 1, StatTableTRI30, &SQR_TABLE_TRI30, &SQR2_TABLE_TRI30, &SQR4_TABLE_TRI30, &SQR8_TABLE_TRI30, &QRT_TABLE_TRI30, &QRT_TABLE_TRI30, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri30; +#endif + +#ifdef ENABLE_FIELD_INT_31 +// 31 bit field +typedef RecLinTrans<uint32_t, 6, 5, 5, 5, 5, 5> StatTable31; +constexpr StatTable31 SQR_TABLE_31({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x12, 0x48, 0x120, 0x480, 0x1200, 0x4800, 0x12000, 0x48000, 0x120000, 0x480000, 0x1200000, 0x4800000, 0x12000000, 0x48000000, 0x20000012}); +constexpr StatTable31 SQR2_TABLE_31({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x12, 0x120, 0x1200, 0x12000, 0x120000, 0x1200000, 0x12000000, 0x20000012, 0x104, 0x1040, 0x10400, 0x104000, 0x1040000, 0x10400000, 0x4000012, 0x40000120, 0x1248, 0x12480, 0x124800, 0x1248000, 0x12480000, 0x24800012, 0x48000104}); +constexpr StatTable31 SQR4_TABLE_31({0x1, 0x10000, 0x12, 0x120000, 0x104, 0x1040000, 0x1248, 0x12480000, 0x10010, 0x100012, 0x120120, 0x1200104, 0x1041040, 0x10401248, 0x12492480, 0x24810002, 0x112, 0x1120000, 0x1304, 0x13040000, 0x11648, 0x16480012, 0x134810, 0x48100116, 0x1121120, 0x11201304, 0x13053040, 0x3041165a, 0x16596492, 0x64934922, 0x49248016}); +constexpr StatTable31 SQR8_TABLE_31({0x1, 0x112, 0x10104, 0x1131648, 0x10002, 0x1120224, 0x106021a, 0x146e3f86, 0x16, 0x174c, 0x161658, 0x175b1130, 0x16002c, 0x174c2e98, 0x16742dfc, 0x3f877966, 0x114, 0x10768, 0x1151050, 0x66b75b2, 0x1140228, 0x76a0ec2, 0x127a33da, 0x79648102, 0x1738, 0x1665f0, 0x172f64e0, 0x73cc668c, 0x17382e70, 0x65dccaac, 0x4abf956e}); +constexpr StatTable31 QRT_TABLE_31({0, 0x10110, 0x10112, 0x15076e, 0x10116, 0x117130e, 0x150766, 0x4743fa0, 0x10106, 0x1121008, 0x117132e, 0x176b248e, 0x150726, 0x172a2c88, 0x4743f20, 0x7eb81e86, 0x10006, 0x20008, 0x1121208, 0x56b2c8e, 0x117172e, 0x133f1bae, 0x176b2c8e, 0x7f2a0c8e, 0x151726, 0x10000000, 0x172a0c88, 0x60000006, 0x4747f20, 0x3eb89e80, 0x7eb89e86}); +typedef Field<uint32_t, 31, 9, StatTable31, &SQR_TABLE_31, &SQR2_TABLE_31, &SQR4_TABLE_31, &SQR8_TABLE_31, &QRT_TABLE_31, &QRT_TABLE_31, IdTrans, &ID_TRANS, &ID_TRANS> Field31; +typedef FieldTri<uint32_t, 31, 3, RecLinTrans<uint32_t, 6, 5, 5, 5, 5, 5>, &SQR_TABLE_31, &SQR2_TABLE_31, &SQR4_TABLE_31, &SQR8_TABLE_31, &QRT_TABLE_31, &QRT_TABLE_31, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri31; +#endif + +#ifdef ENABLE_FIELD_INT_32 +// 32 bit field +typedef RecLinTrans<uint32_t, 6, 6, 5, 5, 5, 5> StatTable32; +constexpr StatTable32 SQR_TABLE_32({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x8d, 0x234, 0x8d0, 0x2340, 0x8d00, 0x23400, 0x8d000, 0x234000, 0x8d0000, 0x2340000, 0x8d00000, 0x23400000, 0x8d000000, 0x3400011a, 0xd0000468, 0x40001037}); +constexpr StatTable32 SQR2_TABLE_32({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x8d, 0x8d0, 0x8d00, 0x8d000, 0x8d0000, 0x8d00000, 0x8d000000, 0xd0000468, 0x4051, 0x40510, 0x405100, 0x4051000, 0x40510000, 0x5100234, 0x51002340, 0x100236b9, 0x236b1d, 0x236b1d0, 0x236b1d00, 0x36b1d11a, 0x6b1d1037, 0xb1d1005e, 0x1d10001f, 0xd100017d}); +constexpr StatTable32 SQR4_TABLE_32({0x1, 0x10000, 0x8d, 0x8d0000, 0x4051, 0x40510000, 0x236b1d, 0x6b1d1037, 0x10001101, 0x1109d000, 0xd00859e5, 0x59881468, 0x144737e8, 0x37e2c4e3, 0xc4f9a67a, 0xa61d8c55, 0x8c010001, 0x41dc8d, 0xdc8d23cd, 0x23a60c51, 0xc41630e, 0x63087fcd, 0x7ffe7368, 0x735580f6, 0x80cd8e29, 0x8e6fe311, 0xe350f32b, 0xf35edc90, 0xdced0bd6, 0xbbd3eb1, 0x3eb4a621, 0xa63f6bc4}); +constexpr StatTable32 SQR8_TABLE_32({0x1, 0x8c010001, 0x6b9010bb, 0x7faf6b, 0xc4da8d37, 0xc10ab646, 0x445f546c, 0xe389129e, 0xd8aa2d3e, 0x85249468, 0xd599253f, 0x458976f9, 0xc9c86411, 0xccc2f34b, 0xa79e37dc, 0x9068e3c4, 0x3a30447f, 0x674c3398, 0x94f38a7, 0x402d3532, 0x116fffc7, 0x1c6b5ba2, 0xcd6a32e4, 0x49067a77, 0xa7f6a61e, 0x3cc3746, 0xeebe962e, 0x599276e1, 0x7b5fa4d9, 0x2aa3ce1, 0x990f8767, 0x1c3b66cb}); +constexpr StatTable32 QRT_TABLE_32({0x54fd1264, 0xc26fcd64, 0xc26fcd66, 0x238a7462, 0xc26fcd62, 0x973bccaa, 0x238a746a, 0x77766712, 0xc26fcd72, 0xc1bdd556, 0x973bcc8a, 0x572a094c, 0x238a742a, 0xb693be84, 0x77766792, 0x9555c03e, 0xc26fcc72, 0x568419f8, 0xc1bdd756, 0x96c3d2ca, 0x973bc88a, 0x54861fdc, 0x572a014c, 0xb79badc4, 0x238a642a, 0xb9b99fe0, 0xb6939e84, 0xc519fa86, 0x77762792, 0, 0x9555403e, 0x377627ba}); +typedef Field<uint32_t, 32, 141, StatTable32, &SQR_TABLE_32, &SQR2_TABLE_32, &SQR4_TABLE_32, &SQR8_TABLE_32, &QRT_TABLE_32, &QRT_TABLE_32, IdTrans, &ID_TRANS, &ID_TRANS> Field32; +#endif +} + +Sketch* ConstructClMul4Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_25 + case 25: return new SketchImpl<Field25>(implementation, 25); +#endif +#ifdef ENABLE_FIELD_INT_26 + case 26: return new SketchImpl<Field26>(implementation, 26); +#endif +#ifdef ENABLE_FIELD_INT_27 + case 27: return new SketchImpl<Field27>(implementation, 27); +#endif +#ifdef ENABLE_FIELD_INT_29 + case 29: return new SketchImpl<Field29>(implementation, 29); +#endif +#ifdef ENABLE_FIELD_INT_31 + case 31: return new SketchImpl<Field31>(implementation, 31); +#endif +#ifdef ENABLE_FIELD_INT_32 + case 32: return new SketchImpl<Field32>(implementation, 32); +#endif + } + return nullptr; +} + +Sketch* ConstructClMulTri4Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_25 + case 25: return new SketchImpl<FieldTri25>(implementation, 25); +#endif +#ifdef ENABLE_FIELD_INT_28 + case 28: return new SketchImpl<FieldTri28>(implementation, 28); +#endif +#ifdef ENABLE_FIELD_INT_29 + case 29: return new SketchImpl<FieldTri29>(implementation, 29); +#endif +#ifdef ENABLE_FIELD_INT_30 + case 30: return new SketchImpl<FieldTri30>(implementation, 30); +#endif +#ifdef ENABLE_FIELD_INT_31 + case 31: return new SketchImpl<FieldTri31>(implementation, 31); +#endif + } + return nullptr; +} diff --git a/src/minisketch/src/fields/clmul_5bytes.cpp b/src/minisketch/src/fields/clmul_5bytes.cpp new file mode 100644 index 0000000000..29c3fb10e7 --- /dev/null +++ b/src/minisketch/src/fields/clmul_5bytes.cpp @@ -0,0 +1,174 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_5) + +#include "clmul_common_impl.h" + +#include "../int_utils.h" +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_33 +// 33 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 5, 5, 5> StatTable33; +constexpr StatTable33 SQR_TABLE_33({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x802, 0x2008, 0x8020, 0x20080, 0x80200, 0x200800, 0x802000, 0x2008000, 0x8020000, 0x20080000, 0x80200000, 0x800401, 0x2001004, 0x8004010, 0x20010040, 0x80040100}); +constexpr StatTable33 SQR2_TABLE_33({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x2008, 0x20080, 0x200800, 0x2008000, 0x20080000, 0x800401, 0x8004010, 0x80040100, 0x400004, 0x4000040, 0x40000400, 0x4802, 0x48020, 0x480200, 0x4802000, 0x48020000, 0x80200802, 0x2009024, 0x20090240, 0x902001, 0x9020010, 0x90200100, 0x102000004, 0x20002048}); +constexpr StatTable33 SQR4_TABLE_33({0x1, 0x10000, 0x100000000, 0x2008000, 0x80040100, 0x4802, 0x48020000, 0x902001, 0x20002048, 0x20081000, 0x10400004, 0x248820, 0x88204812, 0x49020410, 0x4822081, 0x20880641, 0x6000044, 0x480300, 0x3009024, 0x90220180, 0xa00c11, 0xc104050, 0x40482608, 0x2688b024, 0xb0690344, 0x102248834, 0x8a30c912, 0xc8062518, 0x24886803, 0x684a0244, 0x294a025, 0xa020294a, 0x280a1010}); +constexpr StatTable33 SQR8_TABLE_33({0x1, 0x6000044, 0x280a1010, 0x122ac8e75, 0x83209926, 0x4a7a8a1, 0xcada863d, 0x6f2ab824, 0x6b4a8654, 0x70484bd6, 0x164c04e0b, 0x2fbc1617, 0xe095e5a3, 0xeaf7847d, 0xe5625e26, 0xa6aaa3e5, 0xc0164126, 0xd06217c0, 0x1ae58d21, 0xa8600250, 0xbaf87951, 0x8e12c19a, 0xa9b413b9, 0xb75ef087, 0x17e9214d9, 0x85968f33, 0x1e299478f, 0x92bc9a0f, 0x1975d642, 0x11af0b3f1, 0x4e86ee77, 0xe75f4726, 0x38026cce}); +constexpr StatTable33 SQR16_TABLE_33({0x1, 0x185df5e91, 0x193fb40eb, 0xd464f9e4, 0x1ba2d73a6, 0x1d9288c5e, 0x5de03a49, 0x1869ea37b, 0x13faaf379, 0x195d1a8f5, 0x6afd5625, 0xf9d75bab, 0xaf44fe50, 0x101034b9e, 0xcc889caf, 0x5ec7455, 0x7d232a66, 0x17dcfe2c3, 0x1c66ff8d0, 0x17107e836, 0x1939cdead, 0x9852afa0, 0x1b946909a, 0x1846638c5, 0xdd5fa94c, 0x1cb2600fe, 0x19241c856, 0x15fe05ccd, 0xc9f9a425, 0x89e0f463, 0x37b01b39, 0xab0410e0, 0x1ace4ca03}); +constexpr StatTable33 QRT_TABLE_33({0xba504dd4, 0x1e2798ef2, 0x1e2798ef0, 0x6698a4ec, 0x1e2798ef4, 0x1c7f1bef0, 0x6698a4e4, 0x16da1b384, 0x1e2798ee4, 0x661ca6ec, 0x1c7f1bed0, 0x1483b87a6, 0x6698a4a4, 0x800000, 0x16da1b304, 0x1a185101c, 0x1e2798fe4, 0xaa400954, 0x661ca4ec, 0x667caeec, 0x1c7f1bad0, 0x400800, 0x1483b8fa6, 0, 0x6698b4a4, 0x1c61da4b8, 0x802000, 0x16e5dadec, 0x16da1f304, 0x62fc8eec, 0x1a185901c, 0x1661da5ec, 0x1e2788fe4}); +typedef Field<uint64_t, 33, 1025, StatTable33, &SQR_TABLE_33, &SQR2_TABLE_33, &SQR4_TABLE_33, &SQR8_TABLE_33, &SQR16_TABLE_33, &QRT_TABLE_33, IdTrans, &ID_TRANS, &ID_TRANS> Field33; +typedef FieldTri<uint64_t, 33, 10, RecLinTrans<uint64_t, 6, 6, 6, 5, 5, 5>, &SQR_TABLE_33, &SQR2_TABLE_33, &SQR4_TABLE_33, &SQR8_TABLE_33, &SQR16_TABLE_33, &QRT_TABLE_33, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri33; +#endif + +#ifdef ENABLE_FIELD_INT_34 +// 34 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5> StatTable34; +constexpr StatTable34 SQR_TABLE_34({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x81, 0x204, 0x810, 0x2040, 0x8100, 0x20400, 0x81000, 0x204000, 0x810000, 0x2040000, 0x8100000, 0x20400000, 0x81000000, 0x204000000, 0x10000102, 0x40000408, 0x100001020}); +constexpr StatTable34 SQR2_TABLE_34({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x204, 0x2040, 0x20400, 0x204000, 0x2040000, 0x20400000, 0x204000000, 0x40000408, 0x4001, 0x40010, 0x400100, 0x4001000, 0x40010000, 0x100081, 0x1000810, 0x10008100, 0x100081000, 0x810204, 0x8102040, 0x81020400, 0x10204102, 0x102041020, 0x20410004, 0x204100040, 0x41000008}); +constexpr StatTable34 SQR4_TABLE_34({0x1, 0x10000, 0x100000000, 0x204000, 0x40000408, 0x4001000, 0x10008100, 0x81020400, 0x204100040, 0x304, 0x3040000, 0x6041, 0x60410000, 0x1000c1010, 0x10304183, 0x4181020c, 0x102042060, 0x20400001, 0x50010, 0x100100081, 0xa14204, 0x142041428, 0x14001001, 0x10038500, 0x385020400, 0x204704140, 0x41000f1c, 0xf143040, 0x3041e145, 0x1e1430410, 0x3042c5050, 0x5030448b, 0x4481120c, 0x112048120}); +constexpr StatTable34 SQR8_TABLE_34({0x1, 0x102042060, 0x4481120c, 0x1523455ab, 0x307081050, 0x21410f1c, 0x275d0e309, 0x3f676408a, 0x143a54d38, 0x304100344, 0x181774550, 0x1003cd092, 0x3f36b6421, 0x164d51695, 0x3e7c7f2ab, 0x9309b234, 0x354f8d24c, 0x1f5431410, 0x142012478, 0xc5225409, 0x14033f3cf, 0x123bd530c, 0x1100ee58, 0x35490c368, 0x2e1f3dcba, 0x2018108d2, 0x3c61a735d, 0xbf8fa918, 0x282ab07ea, 0x19c32af, 0x175e54c02, 0x2e4dfe2bb, 0x3374ab928, 0x3124a055}); +constexpr StatTable34 SQR16_TABLE_34({0x1, 0x3448e6f02, 0x352590eb9, 0xb173da17, 0x264977d39, 0x172d45e48, 0x1e026e5d6, 0x357b54017, 0x2925d27a4, 0x1f6a32696, 0x2f49f220c, 0x3a7383d9e, 0x28111d79b, 0x5580fcf1, 0x276ede679, 0x175b379f8, 0x34d67b66, 0xc7019416, 0x3f3d9d59f, 0x2a7c2c032, 0x2b3482ba7, 0x177cd0128, 0x1d6f4bd2e, 0x31647a632, 0x41353027, 0x56292eea, 0x2733c0501, 0x6d7ed066, 0x2f3db9a75, 0x3225bc5cc, 0x3f22da089, 0xd0a7588e, 0xb60b22d1, 0xc2fddb7e}); +constexpr StatTable34 QRT_TABLE_34({0x2f973a1f6, 0x40202, 0x40200, 0x348102060, 0x40204, 0x8000420, 0x348102068, 0x1092195c8, 0x40214, 0x3f6881b6e, 0x8000400, 0x3f810383e, 0x348102028, 0x340002068, 0x109219548, 0x24015a774, 0x40314, 0x3f050343e, 0x3f688196e, 0x3f81c3a3a, 0x8000000, 0x24031a560, 0x3f810303e, 0xb08c1a12, 0x348103028, 0xb2881906, 0x340000068, 0, 0x10921d548, 0x2e131e576, 0x240152774, 0x18921d55e, 0x50314, 0x14015271c}); +typedef Field<uint64_t, 34, 129, StatTable34, &SQR_TABLE_34, &SQR2_TABLE_34, &SQR4_TABLE_34, &SQR8_TABLE_34, &SQR16_TABLE_34, &QRT_TABLE_34, IdTrans, &ID_TRANS, &ID_TRANS> Field34; +typedef FieldTri<uint64_t, 34, 7, RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5>, &SQR_TABLE_34, &SQR2_TABLE_34, &SQR4_TABLE_34, &SQR8_TABLE_34, &SQR16_TABLE_34, &QRT_TABLE_34, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri34; +#endif + +#ifdef ENABLE_FIELD_INT_35 +// 35 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5> StatTable35; +constexpr StatTable35 SQR_TABLE_35({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0xa, 0x28, 0xa0, 0x280, 0xa00, 0x2800, 0xa000, 0x28000, 0xa0000, 0x280000, 0xa00000, 0x2800000, 0xa000000, 0x28000000, 0xa0000000, 0x280000000, 0x200000005}); +constexpr StatTable35 SQR2_TABLE_35({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0xa, 0xa0, 0xa00, 0xa000, 0xa0000, 0xa00000, 0xa000000, 0xa0000000, 0x200000005, 0x44, 0x440, 0x4400, 0x44000, 0x440000, 0x4400000, 0x44000000, 0x440000000, 0x400000028, 0x2a8, 0x2a80, 0x2a800, 0x2a8000, 0x2a80000, 0x2a800000, 0x2a8000000, 0x280000011}); +constexpr StatTable35 SQR4_TABLE_35({0x1, 0x10000, 0x100000000, 0xa000, 0xa0000000, 0x4400, 0x44000000, 0x2a80, 0x2a800000, 0x1010, 0x10100000, 0xa0a, 0xa0a0000, 0x200000445, 0x4444000, 0x4400002a8, 0x2aaa800, 0x2a8000101, 0x1000100, 0x10000a0, 0xa000a0, 0xa00044, 0x440044, 0x400440028, 0x4002a8028, 0x2802a8011, 0x280101011, 0x1010100a, 0x100a0a0a, 0x20a0a0a05, 0x20a044445, 0x444444440, 0x44442aaa8, 0x2aaaaaaa8, 0x2aaa90001}); +constexpr StatTable35 SQR8_TABLE_35({0x1, 0x2aaa800, 0x44442aaa8, 0x6400006ed, 0x64e4e4e45, 0x14544000, 0x8a145454, 0x2000034df, 0x49a749a36, 0xaa0a0000, 0x10aa0aaa, 0x1ba1a, 0x393a91ba, 0x3febaaaa9, 0x285105155, 0xa0ad9ad4, 0x269ce8d3b, 0x4de74f4e6, 0x42aaa8028, 0x4002aeea8, 0x400e46eec, 0x544e4006c, 0x145440144, 0x2abede545, 0x44309e74c, 0xa74eeda4, 0x64444ee49, 0x1aa1aaaa, 0x2b90bb1b1, 0x393902109, 0x16bc47bb2, 0x271ad1511, 0x6c8f98767, 0x69d3aa74c, 0x27790dc3b}); +constexpr StatTable35 SQR16_TABLE_35({0x1, 0x4c80f98a4, 0x763684437, 0x5a1cc86a0, 0x38922db8, 0x71755e12d, 0x2ca94c627, 0x388a2bc7f, 0x406596de0, 0x1818c6958, 0x174a92efe, 0x1a80c764e, 0x2f23eacbf, 0xd611ea8, 0x64d783fd5, 0x4fdfe0798, 0x31459de8d, 0x62c889d99, 0x9c419962, 0x2d8d865b3, 0x1ac7e7ffc, 0x38a0c12f3, 0x9fbc1076, 0x6f76d3b89, 0x6e472c757, 0x5f240de42, 0x10176ecc0, 0x20c1cef8, 0x8f77f91c, 0x3f6e533b9, 0x62017c147, 0x5ce81e2fa, 0x371fe4ad9, 0x2552b5046, 0xc3f3696c}); +constexpr StatTable35 QRT_TABLE_35({0x5c2038114, 0x2bf547ee8, 0x2bf547eea, 0x2bf1074e8, 0x2bf547eee, 0x1883d0736, 0x2bf1074e0, 0x100420, 0x2bf547efe, 0x400800, 0x1883d0716, 0x5e90e4a0, 0x2bf1074a0, 0x4e70ac20, 0x1004a0, 0x2f060c880, 0x2bf547ffe, 0x37d55fffe, 0x400a00, 0x3372573de, 0x1883d0316, 0x700c20, 0x5e90eca0, 0x10604880, 0x2bf1064a0, 0x18f35377e, 0x4e708c20, 0x33f557ffe, 0x1044a0, 0x1bf557ffe, 0x2f0604880, 0x200000000, 0x2bf557ffe, 0, 0x37d57fffe}); +typedef Field<uint64_t, 35, 5, StatTable35, &SQR_TABLE_35, &SQR2_TABLE_35, &SQR4_TABLE_35, &SQR8_TABLE_35, &SQR16_TABLE_35, &QRT_TABLE_35, IdTrans, &ID_TRANS, &ID_TRANS> Field35; +typedef FieldTri<uint64_t, 35, 2, RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5>, &SQR_TABLE_35, &SQR2_TABLE_35, &SQR4_TABLE_35, &SQR8_TABLE_35, &SQR16_TABLE_35, &QRT_TABLE_35, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri35; +#endif + +#ifdef ENABLE_FIELD_INT_36 +// 36 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6> StatTable36; +constexpr StatTable36 SQR_TABLE_36({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x201, 0x804, 0x2010, 0x8040, 0x20100, 0x80400, 0x201000, 0x804000, 0x2010000, 0x8040000, 0x20100000, 0x80400000, 0x201000000, 0x804000000, 0x10000402, 0x40001008, 0x100004020, 0x400010080}); +constexpr StatTable36 SQR2_TABLE_36({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x201, 0x2010, 0x20100, 0x201000, 0x2010000, 0x20100000, 0x201000000, 0x10000402, 0x100004020, 0x40001, 0x400010, 0x4000100, 0x40001000, 0x400010000, 0x100804, 0x1008040, 0x10080400, 0x100804000, 0x8040201, 0x80402010, 0x804020100, 0x40200008, 0x402000080, 0x20000004, 0x200000040, 0x2, 0x20}); +constexpr StatTable36 SQR4_TABLE_36({0x1, 0x10000, 0x100000000, 0x201000, 0x10000402, 0x4000100, 0x1008040, 0x80402010, 0x20000004, 0x200, 0x2000000, 0x4020, 0x40200000, 0x80002, 0x800020000, 0x201008000, 0x80400010, 0x4, 0x40000, 0x400000000, 0x804000, 0x40001008, 0x10000400, 0x4020100, 0x201008040, 0x80000010, 0x800, 0x8000000, 0x10080, 0x100800000, 0x200008, 0x80402, 0x804020000, 0x201000040, 0x10, 0x100000}); +constexpr StatTable36 SQR8_TABLE_36({0x1, 0x80400010, 0x804020000, 0x201008, 0x2000080, 0x20000804, 0x1008000, 0x402, 0x800000, 0x200, 0x80000010, 0x804020100, 0x40201000, 0x400010000, 0x100004, 0x201000000, 0x80400, 0x100000000, 0x40000, 0x10, 0x804000100, 0x40201008, 0x2010080, 0x20000800, 0x200008040, 0x10080000, 0x4020, 0x8000000, 0x2000, 0x800000100, 0x40200008, 0x402010000, 0x100804, 0x1000040, 0x10000402, 0x804000}); +constexpr StatTable36 SQR16_TABLE_36({0x1, 0x402000000, 0x100800020, 0x201000, 0x10080402, 0x800000000, 0x1008040, 0x400000, 0x20000800, 0x200, 0x400010080, 0x100000020, 0x40200000, 0x10080002, 0x20100, 0x201008000, 0x80000000, 0x100804, 0x40000, 0x2000080, 0x20, 0x40001008, 0x10000002, 0x4020000, 0x201008040, 0x2010, 0x20100800, 0x8000000, 0x400010000, 0x4000, 0x200008, 0x2, 0x804000000, 0x201000040, 0x402000, 0x20100804}); +constexpr StatTable36 QRT_TABLE_36({0x40200, 0x8b0526186, 0x8b0526184, 0x240001000, 0x8b0526180, 0xcb6894d94, 0x240001008, 0xdb6880c22, 0x8b0526190, 0x8000200, 0xcb6894db4, 0x500424836, 0x240001048, 0x406cb2834, 0xdb6880ca2, 0x241200008, 0x8b0526090, 0xdb05021a6, 0x8000000, 0xdb01829b2, 0xcb68949b4, 0x1001000, 0x500424036, 0x106116406, 0x240000048, 0xcb29968a4, 0x406cb0834, 0, 0xdb6884ca2, 0x110010516, 0x241208008, 0x430434520, 0x8b0536090, 0x41208040, 0xdb05221a6, 0xb6884d14}); +typedef Field<uint64_t, 36, 513, StatTable36, &SQR_TABLE_36, &SQR2_TABLE_36, &SQR4_TABLE_36, &SQR8_TABLE_36, &SQR16_TABLE_36, &QRT_TABLE_36, IdTrans, &ID_TRANS, &ID_TRANS> Field36; +typedef FieldTri<uint64_t, 36, 9, RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6>, &SQR_TABLE_36, &SQR2_TABLE_36, &SQR4_TABLE_36, &SQR8_TABLE_36, &SQR16_TABLE_36, &QRT_TABLE_36, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri36; +#endif + +#ifdef ENABLE_FIELD_INT_37 +// 37 bit field +typedef RecLinTrans<uint64_t, 6, 6, 5, 5, 5, 5, 5> StatTable37; +constexpr StatTable37 SQR_TABLE_37({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0xa6, 0x298, 0xa60, 0x2980, 0xa600, 0x29800, 0xa6000, 0x298000, 0xa60000, 0x2980000, 0xa600000, 0x29800000, 0xa6000000, 0x298000000, 0xa60000000, 0x980000053, 0x60000011f, 0x180000047c}); +constexpr StatTable37 SQR2_TABLE_37({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x298, 0x2980, 0x29800, 0x298000, 0x2980000, 0x29800000, 0x298000000, 0x980000053, 0x180000047c, 0x4414, 0x44140, 0x441400, 0x4414000, 0x44140000, 0x441400000, 0x4140000a6, 0x140000ac6, 0x140000ac60, 0xac43e, 0xac43e0, 0xac43e00, 0xac43e000, 0xac43e0000, 0xc43e0011f, 0x43e00101a, 0x3e0010106, 0x1e00101033}); +constexpr StatTable37 SQR4_TABLE_37({0x1, 0x10000, 0x100000000, 0x29800, 0x298000000, 0x44140, 0x441400000, 0xac43e, 0xac43e0000, 0x1e00101033, 0x1010011000, 0x11029a980, 0x9a982b1d3, 0x2b1c45014, 0x4501005f2, 0x1005f8ef80, 0x18efa98941, 0x9897de117, 0x1de10002ad, 0x2990398, 0x190398047c, 0x180443dee4, 0x3ded94ac6, 0x194ac071fa, 0x71c56e1a, 0x56e1adff2, 0x1adffa1690, 0x1a16a9ab31, 0x9ab0957cf, 0x957d85468, 0x18547edba2, 0x1edb9fc515, 0x1fc526c1a4, 0x6c1956aab, 0x156aa5b9d4, 0x5b9f59def, 0x159de6d961}); +constexpr StatTable37 SQR8_TABLE_37({0x1, 0x18efa98941, 0x1fc526c1a4, 0x11352e16c4, 0xba7aa5340, 0x17346e075f, 0xe91c746aa, 0xe560ac1bd, 0xa4544c5d9, 0x11bd3c631f, 0xd70c4b63c, 0xfe77d107c, 0x10548e5288, 0x1183954fb3, 0x19b3aa4bb, 0x782a2943c, 0x1c19ba61de, 0x6ad01fe38, 0xa22701577, 0xb96546ca0, 0x1d7c6c8b9c, 0xffef807e2, 0x16fcc14dc2, 0x110cc4e83c, 0xc3a35629a, 0x1062330476, 0xb2e5d1de1, 0x1ca4e3d229, 0x67826b51b, 0xe7e4c36e7, 0x59f1ac963, 0x12777f22c6, 0x13963d623a, 0x9e305ac92, 0x219b91d13, 0x175bebeb0d, 0xc6b7b5572}); +constexpr StatTable37 SQR16_TABLE_37({0x1, 0xcb88f2f8b, 0x1a2a0be7af, 0xb93048ada, 0x113ed92190, 0xc95a18e2b, 0x1e1cd4a85b, 0x19584a1a66, 0x1b947c28c2, 0x1b52b48e27, 0xe64e7b169, 0x14a256d011, 0xda657196d, 0x1947c1dcb4, 0x18b2fa3851, 0xae3d4171a, 0x658f1f4b9, 0x91852c314, 0x69346cf8e, 0x8224bf36c, 0x1086c810ed, 0x10419bc782, 0x57d6a4e36, 0xfbb31a43e, 0x18b502de05, 0x786795174, 0x1de0f1b7f3, 0x1d456b87dc, 0x1aabb2f3bc, 0xc5b80ef0c, 0x1ce4fd7543, 0x7ca740ca1, 0x29eaec26a, 0x1eb0b42043, 0xca3b2b17, 0x3453101c1, 0x1714c59187}); +constexpr StatTable37 QRT_TABLE_37({0xa3c62e7ba, 0xdc7a0c16a, 0xdc7a0c168, 0x12f7484546, 0xdc7a0c16c, 0xa9803a20, 0x12f748454e, 0xda07064a4, 0xdc7a0c17c, 0x123908de8e, 0xa9803a00, 0x122a888a8e, 0x12f748450e, 0x6790add8, 0xda0706424, 0x12e0a0384c, 0xdc7a0c07c, 0xcb28a2c2, 0x123908dc8e, 0xd09f85e86, 0xa9803e00, 0x124d682b6e, 0x122a88828e, 0x1738711a, 0x12f748550e, 0x73035b8, 0x67908dd8, 0xa0702438, 0xda0702424, 0xe0a0b860, 0x12e0a0b84c, 0x1c7a1c060, 0xdc7a1c07c, 0, 0xcb2aa2c2, 0x100000002c, 0x12390cdc8e}); +typedef Field<uint64_t, 37, 83, StatTable37, &SQR_TABLE_37, &SQR2_TABLE_37, &SQR4_TABLE_37, &SQR8_TABLE_37, &SQR16_TABLE_37, &QRT_TABLE_37, IdTrans, &ID_TRANS, &ID_TRANS> Field37; +#endif + +#ifdef ENABLE_FIELD_INT_38 +// 38 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 5, 5, 5, 5> StatTable38; +constexpr StatTable38 SQR_TABLE_38({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x63, 0x18c, 0x630, 0x18c0, 0x6300, 0x18c00, 0x63000, 0x18c000, 0x630000, 0x18c0000, 0x6300000, 0x18c00000, 0x63000000, 0x18c000000, 0x630000000, 0x18c0000000, 0x2300000063, 0xc0000014a, 0x3000000528}); +constexpr StatTable38 SQR2_TABLE_38({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x18c, 0x18c0, 0x18c00, 0x18c000, 0x18c0000, 0x18c00000, 0x18c000000, 0x18c0000000, 0xc0000014a, 0x1405, 0x14050, 0x140500, 0x1405000, 0x14050000, 0x140500000, 0x1405000000, 0x500001ef, 0x500001ef0, 0x100001ef63, 0x1ef7bc, 0x1ef7bc0, 0x1ef7bc00, 0x1ef7bc000, 0x1ef7bc0000, 0x2f7bc00129, 0x37bc00112d, 0x3bc0011027, 0x3c00110022}); +constexpr StatTable38 SQR4_TABLE_38({0x1, 0x10000, 0x100000000, 0x18c00, 0x18c000000, 0x14050, 0x140500000, 0x100001ef63, 0x1ef7bc000, 0x3bc0011027, 0x110001100, 0x110194c0, 0x194c0194c, 0x194d5455, 0xd5455154f, 0x151544a193, 0x4a18c631f, 0xc6319c6ca, 0x19c6c00014, 0x18c8d, 0x18c8d0000, 0xd00014096, 0x1409ddc00, 0x1ddc01efc6, 0x1efd5ab90, 0x15ab9110e1, 0x1110fe85b2, 0x3e85ab5465, 0x2b5445c97a, 0x5c9450993, 0x50994148f, 0x141488b12a, 0x8b134ee36, 0x34ee3a8ecc, 0x3a8ee3edc8, 0x23edeef7ed, 0x2ef7de8bf9, 0x1e8bc14041}); +constexpr StatTable38 SQR8_TABLE_38({0x1, 0x4a18c631f, 0x8b134ee36, 0x10b5c9474c, 0x3330e98ecb, 0x939897650, 0xd74b026b9, 0x860251dd9, 0x3afbe829b4, 0x3ae6afc308, 0x239ecafe00, 0x2acbc94749, 0x3a5770e19e, 0x4052e180b, 0x321fa15712, 0x3a8a4869ef, 0x1948598082, 0x3b1bd98542, 0xc1deb9112, 0x1b5c9242e, 0x338ba58e8b, 0x8abe06d20, 0x145bb1d2a9, 0x1d6e10fbf0, 0x197d522629, 0x2ff1bbe50d, 0xcc1594a16, 0xc94db1b03, 0x3b20e51c56, 0x101d1e5d07, 0x19472478f7, 0x269635a968, 0x2fd4a35802, 0x1b63e116b6, 0x19fdf9d22a, 0x2ef0e4d419, 0x3e80f730f4, 0x29869b04b9}); +constexpr StatTable38 SQR16_TABLE_38({0x1, 0x3f5fe2afaa, 0x4216541b5, 0x33b362f56a, 0x9d630d7e1, 0x11127694c1, 0x3f8daab2d6, 0x153ca20edc, 0x22a747a3de, 0xc6ab16040, 0x19cc9a7e37, 0x449d96001, 0x45a7e7c46, 0x36d11561ce, 0x114b93f52a, 0x42a87f1b3, 0x23112a30bc, 0x400df9212, 0x3aca9544df, 0x140c4b0bcf, 0x2ae2efa6d3, 0x2f7051159c, 0x19cca2f62e, 0x102023d8c0, 0xccc793f0b, 0x2ff4789b55, 0x339e4cd9ba, 0x2b02ab5052, 0x8c1b5db82, 0x2e461e4e32, 0xd93541605, 0x1acf12087, 0x33b88dca2b, 0x1e91723c8b, 0xd81047b2b, 0x2e5e54b97c, 0x85bb507d8, 0x2145b1864b}); +constexpr StatTable38 QRT_TABLE_38({0x34b0ac6430, 0x2223262fa, 0x2223262f8, 0x35554405fe, 0x2223262fc, 0x355514098a, 0x35554405f6, 0x400840, 0x2223262ec, 0x1777726532, 0x35551409aa, 0x15c06fc0, 0x35554405b6, 0x1f5303fec, 0x4008c0, 0x236a21030, 0x2223263ec, 0x1a9008c00, 0x1777726732, 0x3692c60ab6, 0x3555140daa, 0x15556007ee, 0x15c067c0, 0x14a0b030f2, 0x35554415b6, 0x227c06d168, 0x1f5301fec, 0x16c3928fc2, 0x4048c0, 0x3a942c4c0, 0x236a29030, 0x1636a2902e, 0x2223363ec, 0x3a6e898276, 0x1a9028c00, 0x6de74eb2c, 0x1777766732, 0}); +typedef Field<uint64_t, 38, 99, StatTable38, &SQR_TABLE_38, &SQR2_TABLE_38, &SQR4_TABLE_38, &SQR8_TABLE_38, &SQR16_TABLE_38, &QRT_TABLE_38, IdTrans, &ID_TRANS, &ID_TRANS> Field38; +#endif + +#ifdef ENABLE_FIELD_INT_39 +// 39 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5, 5> StatTable39; +constexpr StatTable39 SQR_TABLE_39({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x22, 0x88, 0x220, 0x880, 0x2200, 0x8800, 0x22000, 0x88000, 0x220000, 0x880000, 0x2200000, 0x8800000, 0x22000000, 0x88000000, 0x220000000, 0x880000000, 0x2200000000, 0x800000011, 0x2000000044}); +constexpr StatTable39 SQR2_TABLE_39({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x22, 0x220, 0x2200, 0x22000, 0x220000, 0x2200000, 0x22000000, 0x220000000, 0x2200000000, 0x2000000044, 0x404, 0x4040, 0x40400, 0x404000, 0x4040000, 0x40400000, 0x404000000, 0x4040000000, 0x400000088, 0x4000000880, 0x8888, 0x88880, 0x888800, 0x8888000, 0x88880000, 0x888800000, 0x888000011, 0x880000101, 0x800001001}); +constexpr StatTable39 SQR4_TABLE_39({0x1, 0x10000, 0x100000000, 0x2200, 0x22000000, 0x404, 0x4040000, 0x400000088, 0x888800, 0x888000011, 0x100010, 0x1000100000, 0x1000022000, 0x220022000, 0x220004040, 0x40404040, 0x4040400880, 0x4008888880, 0x888888101, 0x881000001, 0x122, 0x1220000, 0x2200000022, 0x260400, 0x2604000000, 0x48c88, 0x48c880000, 0x800009889, 0x98881000, 0x810001221, 0x12201220, 0x2012200264, 0x2002604264, 0x6042604044, 0x604048c8c4, 0x48c8c8c880, 0x48c8898881, 0x988888881, 0x888802201}); +constexpr StatTable39 SQR8_TABLE_39({0x1, 0x4040400880, 0x2002604264, 0xaa8022011, 0x810049ea9, 0x100100010, 0xc04008101, 0x644048ea4c, 0x18c1764441, 0x60f8e8526c, 0x22000122, 0x48c88989a3, 0xae0032001, 0x2a7aeafae5, 0x6a76641225, 0x2036245242, 0x3e9ab0308b, 0x1c49f6fe41, 0x681b069e2d, 0x4edee8cae5, 0x898c04, 0x660daa8880, 0x69cae9ccc1, 0x4881320991, 0xd06280001, 0x1cc8c8e3d9, 0x445fc65628, 0x4c889a8a49, 0x300b8caeec, 0x50d842fc94, 0x1811acb89d, 0x9d22101c, 0x2025aa407e, 0x20370a744a, 0x3cf77cb80b, 0x54a13e66e7, 0x34c17e2e04, 0x5c19fe54c1, 0x6a72cc767d}); +constexpr StatTable39 SQR16_TABLE_39({0x1, 0x37214861ce, 0x689e897065, 0x5678d6ee60, 0x619da834c4, 0x28352752d3, 0x14fed69ec6, 0x5b3d4aa637, 0x682fb8da4d, 0x2ce48c5615, 0x1591ac539c, 0x72d4fbcd0, 0x346b547296, 0x1e7065d419, 0x4e6eb48571, 0x26615d4c2c, 0x60d1c6122e, 0x78d0e2a2eb, 0x52bb3e2980, 0x3c2592d0ab, 0x701ba76b58, 0x5fdf53b685, 0x57cfd2d120, 0x75559e4344, 0x3837a46907, 0x15f961a4ce, 0x397b9a03e9, 0x5a8dd4ab69, 0x3a6ab3356f, 0x215d39c25e, 0x5bbaf82443, 0x6759e3c88c, 0x3c0b862ca1, 0x37eec7e79e, 0x6ce865e38, 0x4a56a338c0, 0x5684636aee, 0x325a019126, 0x24f18a4ef6}); +constexpr StatTable39 QRT_TABLE_39({0x66b02a408c, 0x100420, 0x100422, 0x14206080, 0x100426, 0x5dccefab1c, 0x14206088, 0x9fc11e5b6, 0x100436, 0x5466bea62a, 0x5dccefab3c, 0x9aa110536, 0x142060c8, 0x54739ed6e2, 0x9fc11e536, 0xe7a82c080, 0x100536, 0x4002000, 0x5466bea42a, 0x6a4022000, 0x5dccefaf3c, 0x9e8118536, 0x9aa110d36, 0x5680e080, 0x142070c8, 0x7d293c5b6, 0x54739ef6e2, 0x8d680e080, 0x9fc11a536, 0x6d282c080, 0xe7a824080, 0x800000000, 0x110536, 0x2d680e080, 0x4022000, 0, 0x5466baa42a, 0x46b03a44aa, 0x6a40a2000}); +typedef Field<uint64_t, 39, 17, StatTable39, &SQR_TABLE_39, &SQR2_TABLE_39, &SQR4_TABLE_39, &SQR8_TABLE_39, &SQR16_TABLE_39, &QRT_TABLE_39, IdTrans, &ID_TRANS, &ID_TRANS> Field39; +typedef FieldTri<uint64_t, 39, 4, RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5, 5>, &SQR_TABLE_39, &SQR2_TABLE_39, &SQR4_TABLE_39, &SQR8_TABLE_39, &SQR16_TABLE_39, &QRT_TABLE_39, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri39; +#endif + +#ifdef ENABLE_FIELD_INT_40 +// 40 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5, 5> StatTable40; +constexpr StatTable40 SQR_TABLE_40({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x39, 0xe4, 0x390, 0xe40, 0x3900, 0xe400, 0x39000, 0xe4000, 0x390000, 0xe40000, 0x3900000, 0xe400000, 0x39000000, 0xe4000000, 0x390000000, 0xe40000000, 0x3900000000, 0xe400000000, 0x900000004b, 0x400000015e}); +constexpr StatTable40 SQR2_TABLE_40({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x39, 0x390, 0x3900, 0x39000, 0x390000, 0x3900000, 0x39000000, 0x390000000, 0x3900000000, 0x900000004b, 0x541, 0x5410, 0x54100, 0x541000, 0x5410000, 0x54100000, 0x541000000, 0x5410000000, 0x41000000dd, 0x1000000d34, 0xd379, 0xd3790, 0xd37900, 0xd379000, 0xd3790000, 0xd37900000, 0xd379000000, 0x3790000115, 0x790000111b, 0x900001111f}); +constexpr StatTable40 SQR4_TABLE_40({0x1, 0x10000, 0x100000000, 0x3900, 0x39000000, 0x541, 0x5410000, 0x41000000dd, 0xd37900, 0xd379000000, 0x111001, 0x1110010000, 0x10003aa90, 0x3aa903900, 0x903900511a, 0x51051541, 0x515410de9, 0x410de9de4d, 0xe9de437815, 0x437801010e, 0x101000038, 0x383939, 0x3839390000, 0x3900057d41, 0x57d444100, 0x444100d6e5, 0xd6ebaa79, 0xebaa7911c6, 0x7911d2791a, 0xd2791102a9, 0x1102b82901, 0xb82902a972, 0x2a96bfed1, 0x6bfed16851, 0xd16859f42e, 0x59f43f61a8, 0x3f61a43794, 0xa43791de59, 0x91de42401f, 0x424000390e}); +constexpr StatTable40 SQR8_TABLE_40({0x1, 0x515410de9, 0x2a96bfed1, 0x13ba41ea90, 0x45bffe2b75, 0x5836900, 0x3887d7e690, 0xd34b688712, 0xc7a3d51557, 0xd1151ada71, 0x51442a740, 0x41cc5cbdb6, 0xc61a5701e9, 0x8757946d91, 0xa99e8b9e65, 0x80a0aca777, 0xc242b5c0e9, 0x6826eccb25, 0xad687ebd2d, 0xad5c69d802, 0x7ed2f8390, 0x51fa78eedf, 0xc0718c96f6, 0xaf4672a8c2, 0xc67436f2fd, 0x56ddb12767, 0x535afb0326, 0xbce1edda33, 0xef36202f0f, 0x45d13015ec, 0x104ab11aef, 0xef96c86d49, 0xc1b790bfc9, 0x2fa610e77f, 0x2a10a27d6e, 0xca5bb10773, 0xfdaf2b4642, 0xb3b4b7e20d, 0xe8bbe4d22e, 0xf9986bd2df}); +constexpr StatTable40 SQR16_TABLE_40({0x1, 0xe88450a7de, 0x3a0a56c3e8, 0x1684757d36, 0xc7f40bf3e9, 0x38aa7009c0, 0x2b6f129659, 0xd1e0fc42e5, 0x96150bc554, 0x9774ef4cc1, 0xd34eebf74d, 0x2d183441ec, 0xeedf6d1c78, 0x3f93c5d217, 0xb924305809, 0xc383bb7c14, 0x3f242bb94e, 0x9313556f6b, 0x2f5e1ecc6b, 0x2e7f9df195, 0xac8b882870, 0xd14f457f55, 0xf9f936148d, 0x719190770, 0x6838b41a21, 0xb95ff30106, 0xc1527dd1c5, 0xe858b5f9b6, 0x9368a791c2, 0x7de23878af, 0x95c610d398, 0xed0edcb032, 0x9548a680b0, 0xc133469e7b, 0x68c96ccbb2, 0x7773231ebb, 0xbd5ef4207c, 0xdf8bd59374, 0xb862414268, 0xfa62b39e42}); +constexpr StatTable40 QRT_TABLE_40({0x624b3cecc, 0xbc5c3f4c6, 0xbc5c3f4c4, 0xde1603e2c, 0xbc5c3f4c0, 0xaabec06cea, 0xde1603e24, 0x6cd9f724c2, 0xbc5c3f4d0, 0xcde1743818, 0xaabec06cca, 0xa138c314ca, 0xde1603e64, 0xaafc00f01a, 0x6cd9f72442, 0xcdca11bb4, 0xbc5c3f5d0, 0xa00002001a, 0xcde1743a18, 0xdf1407b90, 0xaabec068ca, 0xc043b482c8, 0xa138c31cca, 0xcb86977e3c, 0xde1602e64, 0x604596a326, 0xaafc00d01a, 0xcc1c165d0, 0x6cd9f76442, 0x673c94da26, 0xcdca19bb4, 0x67c0940a26, 0xbc5c2f5d0, 0xa4dca19bae, 0xa00000001a, 0x1bc5c2f5d0, 0xcde1703a18, 0, 0xdf1487b90, 0x8df1487b8a}); +typedef Field<uint64_t, 40, 57, StatTable40, &SQR_TABLE_40, &SQR2_TABLE_40, &SQR4_TABLE_40, &SQR8_TABLE_40, &SQR16_TABLE_40, &QRT_TABLE_40, IdTrans, &ID_TRANS, &ID_TRANS> Field40; +#endif +} + +Sketch* ConstructClMul5Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_33 + case 33: return new SketchImpl<Field33>(implementation, 33); +#endif +#ifdef ENABLE_FIELD_INT_34 + case 34: return new SketchImpl<Field34>(implementation, 34); +#endif +#ifdef ENABLE_FIELD_INT_35 + case 35: return new SketchImpl<Field35>(implementation, 35); +#endif +#ifdef ENABLE_FIELD_INT_36 + case 36: return new SketchImpl<Field36>(implementation, 36); +#endif +#ifdef ENABLE_FIELD_INT_37 + case 37: return new SketchImpl<Field37>(implementation, 37); +#endif +#ifdef ENABLE_FIELD_INT_38 + case 38: return new SketchImpl<Field38>(implementation, 38); +#endif +#ifdef ENABLE_FIELD_INT_39 + case 39: return new SketchImpl<Field39>(implementation, 39); +#endif +#ifdef ENABLE_FIELD_INT_40 + case 40: return new SketchImpl<Field40>(implementation, 40); +#endif + } + return nullptr; +} + +Sketch* ConstructClMulTri5Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_33 + case 33: return new SketchImpl<FieldTri33>(implementation, 33); +#endif +#ifdef ENABLE_FIELD_INT_34 + case 34: return new SketchImpl<FieldTri34>(implementation, 34); +#endif +#ifdef ENABLE_FIELD_INT_35 + case 35: return new SketchImpl<FieldTri35>(implementation, 35); +#endif +#ifdef ENABLE_FIELD_INT_36 + case 36: return new SketchImpl<FieldTri36>(implementation, 36); +#endif +#ifdef ENABLE_FIELD_INT_39 + case 39: return new SketchImpl<FieldTri39>(implementation, 39); +#endif + } + return nullptr; +} diff --git a/src/minisketch/src/fields/clmul_6bytes.cpp b/src/minisketch/src/fields/clmul_6bytes.cpp new file mode 100644 index 0000000000..d0e712400a --- /dev/null +++ b/src/minisketch/src/fields/clmul_6bytes.cpp @@ -0,0 +1,170 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_6) + +#include "clmul_common_impl.h" + +#include "../int_utils.h" +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_41 +// 41 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5> StatTable41; +constexpr StatTable41 SQR_TABLE_41({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x12, 0x48, 0x120, 0x480, 0x1200, 0x4800, 0x12000, 0x48000, 0x120000, 0x480000, 0x1200000, 0x4800000, 0x12000000, 0x48000000, 0x120000000, 0x480000000, 0x1200000000, 0x4800000000, 0x12000000000, 0x8000000012}); +constexpr StatTable41 SQR2_TABLE_41({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x48, 0x480, 0x4800, 0x48000, 0x480000, 0x4800000, 0x48000000, 0x480000000, 0x4800000000, 0x8000000012, 0x104, 0x1040, 0x10400, 0x104000, 0x1040000, 0x10400000, 0x104000000, 0x1040000000, 0x10400000000, 0x4000000048, 0x492, 0x4920, 0x49200, 0x492000, 0x4920000, 0x49200000, 0x492000000, 0x4920000000, 0x9200000012, 0x12000000104}); +constexpr StatTable41 SQR4_TABLE_41({0x1, 0x10000, 0x100000000, 0x480, 0x4800000, 0x8000000012, 0x104000, 0x1040000000, 0x4920, 0x49200000, 0x12000000104, 0x1001000, 0x10010000000, 0x48048, 0x480480000, 0x4800001040, 0x10410400, 0x4104000048, 0x492492, 0x4924920000, 0x9200010002, 0x100000100, 0x1000480, 0x10004800000, 0x8000048012, 0x480104000, 0x1040001040, 0x10404920, 0x4049200048, 0x12000492104, 0x4921001000, 0x10010010010, 0x100148048, 0x1480480480, 0x4804805840, 0x8058410412, 0x410410414c, 0x10414d2492, 0x14d24924920, 0x9249259202, 0x12592000004}); +constexpr StatTable41 SQR8_TABLE_41({0x1, 0x10410400, 0x100148048, 0x13040000104, 0x11044801040, 0x924da49202, 0x9680490002, 0x82510514c, 0x8481485932, 0xc83144d832, 0x134d34d6db6, 0x18010048012, 0x165db20004c, 0x4800597052, 0x10131135cd0, 0xcd6cc30d32, 0x160586101cc, 0x15c64969da8, 0x179715681cc, 0x12f3c0ffc74, 0xc7dd3dd3ce, 0x10014c968, 0x1b040048116, 0x35d6801044, 0xda4deda6d0, 0x1de94c85852, 0x1083500114c, 0xc4c9685dfa, 0x18515c6d592, 0x17de69aed7e, 0x16c6c8a6c6c, 0x165cfe1044c, 0xdb004cf018, 0x7075031c98, 0x1d9a90b0d72, 0x1bb2485caee, 0x1cbe4dfd48a, 0x1f1540b7400, 0xc62bc7fd02, 0x147b5103f2e, 0x390ee8bcc6}); +constexpr StatTable41 SQR16_TABLE_41({0x1, 0x61deee38fe, 0xe00adae2e, 0x1ea53eaa95a, 0x503e540566, 0xabc8e7f89a, 0x1bf760d86ac, 0x94cce9c722, 0x15c8006ee5c, 0x7aba20c1da, 0x12662a9603e, 0x5fe76acec4, 0x1e6beca9e42, 0x1efc8f7a000, 0x165997c6d7e, 0xee947a07ee, 0xd9bd741142, 0xaa304566c0, 0x5fe336e356, 0x11f1021b80c, 0xd34e5a1674, 0x99ed56b9dc, 0x9afae0eca, 0x1a5830b390e, 0x1be1a63eb7e, 0x141e77e141c, 0xee3be92168, 0xa93823d65c, 0x18a59f4b19c, 0xce69942af6, 0x3f7b319c0e, 0xba83a4a7b4, 0x7da4b6fcde, 0x17f79268f10, 0x1222602d048, 0x1b4b2f326b8, 0x159abff0786, 0xb35534a7a2, 0x84bbc48050, 0x173d5cbf330, 0x2897dd6f58}); +constexpr StatTable41 QRT_TABLE_41({0, 0x1599a5e0b0, 0x1599a5e0b2, 0x105c119e0, 0x1599a5e0b6, 0x1a2030452a6, 0x105c119e8, 0x1a307c55b2e, 0x1599a5e0a6, 0x1ee3f47bc8e, 0x1a203045286, 0x400808, 0x105c119a8, 0x1a3038573a6, 0x1a307c55bae, 0x4d2882a520, 0x1599a5e1a6, 0x1ffbaa0b720, 0x1ee3f47be8e, 0x4d68c22528, 0x1a203045686, 0x200006, 0x400008, 0x1b79a21b200, 0x105c109a8, 0x1ef3886a526, 0x1a3038553a6, 0x1b692209200, 0x1a307c51bae, 0x5d99a4e1a6, 0x4d28822520, 0x185e109ae, 0x1599a4e1a6, 0x4e3f43be88, 0x1ffbaa2b720, 0x4000000000, 0x1ee3f43be8e, 0x18000000006, 0x4d68ca2528, 0xa203145680, 0x1a203145686}); +typedef Field<uint64_t, 41, 9, StatTable41, &SQR_TABLE_41, &SQR2_TABLE_41, &SQR4_TABLE_41, &SQR8_TABLE_41, &SQR16_TABLE_41, &QRT_TABLE_41, IdTrans, &ID_TRANS, &ID_TRANS> Field41; +typedef FieldTri<uint64_t, 41, 3, RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5>, &SQR_TABLE_41, &SQR2_TABLE_41, &SQR4_TABLE_41, &SQR8_TABLE_41, &SQR16_TABLE_41, &QRT_TABLE_41, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri41; +#endif + +#ifdef ENABLE_FIELD_INT_42 +// 42 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6> StatTable42; +constexpr StatTable42 SQR_TABLE_42({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x81, 0x204, 0x810, 0x2040, 0x8100, 0x20400, 0x81000, 0x204000, 0x810000, 0x2040000, 0x8100000, 0x20400000, 0x81000000, 0x204000000, 0x810000000, 0x2040000000, 0x8100000000, 0x20400000000, 0x1000000102, 0x4000000408, 0x10000001020}); +constexpr StatTable42 SQR2_TABLE_42({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x204, 0x2040, 0x20400, 0x204000, 0x2040000, 0x20400000, 0x204000000, 0x2040000000, 0x20400000000, 0x4000000408, 0x4001, 0x40010, 0x400100, 0x4001000, 0x40010000, 0x400100000, 0x4001000000, 0x10000081, 0x100000810, 0x1000008100, 0x10000081000, 0x810204, 0x8102040, 0x81020400, 0x810204000, 0x8102040000, 0x1020400102, 0x10204001020, 0x2040010004, 0x20400100040, 0x4001000008}); +constexpr StatTable42 SQR4_TABLE_42({0x1, 0x10000, 0x100000000, 0x2040, 0x20400000, 0x4000000408, 0x4001000, 0x10000081, 0x810204, 0x8102040000, 0x20400100040, 0x1000000100, 0x1020400, 0x10204000000, 0x200001, 0x2000010000, 0x100040800, 0x408002040, 0x20408002, 0x4080020408, 0x204000020, 0x204001, 0x2040010000, 0x100040010, 0x400102040, 0x1020408100, 0x4081020008, 0x10200000020, 0x80, 0x800000, 0x8000000000, 0x102000, 0x1020000000, 0x20008, 0x200080000, 0x800004080, 0x40810200, 0x8102000810, 0x20008000040, 0x8102, 0x81020000, 0x10200001020}); +constexpr StatTable42 SQR8_TABLE_42({0x1, 0x100040800, 0x1020000000, 0x10200001000, 0x2040810000, 0x408102040, 0x4080000408, 0x10000000, 0x8002040810, 0x20008000, 0x10200080000, 0x40000004, 0x20408000040, 0x4000020000, 0x204000, 0x8002040010, 0x408102, 0x200081020, 0x40810000, 0x2000, 0x81000408, 0x4001, 0x2040010, 0x1020008002, 0x10204081020, 0x800004, 0x20000000000, 0x4081000400, 0x10000081, 0x100040010, 0x8102, 0x10000000020, 0x40010200, 0x408000000, 0x4080000400, 0x810204000, 0x102040810, 0x1020000102, 0x4000000, 0x2000810204, 0x8002000, 0x4080020000}); +constexpr StatTable42 SQR16_TABLE_42({0x1, 0x40800204, 0x2000800, 0x20008002040, 0x408100, 0x20000, 0x10004081020, 0x10000081, 0x40010204, 0x8102000000, 0x400000000, 0x1020008102, 0x1020408, 0x204081020, 0x200001, 0x10200, 0x102000010, 0x20408100000, 0x1020408002, 0x4000020000, 0x204000000, 0x204001, 0x2000800004, 0x8100000010, 0x400102000, 0x8002, 0x4080020000, 0x10000001000, 0x80, 0x2040010200, 0x100040000, 0x400100040, 0x20408000, 0x1000000, 0x204080020, 0x800004080, 0x2000810200, 0x8100000810, 0x20000000000, 0x1000408002, 0x81020400, 0x10204081000}); +constexpr StatTable42 QRT_TABLE_42({0x810200080, 0x120810806, 0x120810804, 0x1068c1a1000, 0x120810800, 0x34005023008, 0x1068c1a1008, 0x800004080, 0x120810810, 0x162818a10, 0x34005023028, 0x42408a14, 0x1068c1a1048, 0x1001040, 0x800004000, 0xb120808906, 0x120810910, 0x34000020068, 0x162818810, 0x68c021400, 0x34005023428, 0x10004000, 0x42408214, 0x162418214, 0x1068c1a0048, 0xb002018116, 0x1003040, 0x10008180448, 0x800000000, 0x62c08b04, 0xb120800906, 0x2408d1a3060, 0x120800910, 0x34401003028, 0x34000000068, 0, 0x162858810, 0xa042058116, 0x68c0a1400, 0x8162858806, 0x34005123428, 0x3068c0a1468}); +typedef Field<uint64_t, 42, 129, StatTable42, &SQR_TABLE_42, &SQR2_TABLE_42, &SQR4_TABLE_42, &SQR8_TABLE_42, &SQR16_TABLE_42, &QRT_TABLE_42, IdTrans, &ID_TRANS, &ID_TRANS> Field42; +typedef FieldTri<uint64_t, 42, 7, RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6>, &SQR_TABLE_42, &SQR2_TABLE_42, &SQR4_TABLE_42, &SQR8_TABLE_42, &SQR16_TABLE_42, &QRT_TABLE_42, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri42; +#endif + +#ifdef ENABLE_FIELD_INT_43 +// 43 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 5, 5, 5, 5, 5> StatTable43; +constexpr StatTable43 SQR_TABLE_43({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0xb2, 0x2c8, 0xb20, 0x2c80, 0xb200, 0x2c800, 0xb2000, 0x2c8000, 0xb20000, 0x2c80000, 0xb200000, 0x2c800000, 0xb2000000, 0x2c8000000, 0xb20000000, 0x2c80000000, 0xb200000000, 0x2c800000000, 0x32000000059, 0x4800000013d, 0x20000000446}); +constexpr StatTable43 SQR2_TABLE_43({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0xb2, 0xb20, 0xb200, 0xb2000, 0xb20000, 0xb200000, 0xb2000000, 0xb20000000, 0xb200000000, 0x32000000059, 0x20000000446, 0x4504, 0x45040, 0x450400, 0x4504000, 0x45040000, 0x450400000, 0x4504000000, 0x45040000000, 0x504000002c8, 0x4000002efa, 0x4000002efa0, 0x2ef8c8, 0x2ef8c80, 0x2ef8c800, 0x2ef8c8000, 0x2ef8c80000, 0x2ef8c800000, 0x6f8c800013d, 0x78c80001025, 0xc800010117, 0x48000101129}); +constexpr StatTable43 SQR4_TABLE_43({0x1, 0x10000, 0x100000000, 0xb20, 0xb200000, 0x32000000059, 0x450400, 0x4504000000, 0x4000002efa0, 0x2ef8c8000, 0x78c80001025, 0x10110010, 0x11001000b2, 0x1000b2b920, 0xb2b920b200, 0x120b204545f, 0x20454554046, 0x45540506efa, 0x506ed4df68, 0x6d4df6a79f5, 0x76a79c80133, 0x1c801010007, 0x101000b2100, 0xb210b2b20, 0x10b2b204504, 0x320450f655d, 0x50f654106c8, 0x54106efcb4c, 0x6efcb696320, 0x369631c93e1, 0x31c93ff9d8c, 0x3ff9d91a2a2, 0x591a2b9839b, 0x2b983b98dd4, 0x3b98dc651b0, 0x5c651a971e9, 0x1a971c9c0ba, 0x1c9c0b5853e, 0xb585322d78, 0x5322d7c6430, 0x57c6416617d, 0x4166159c82c, 0x159c8000b6c}); +constexpr StatTable43 SQR8_TABLE_43({0x1, 0x20454554046, 0x591a2b9839b, 0x722ff9f6fe9, 0x6a269c1eb12, 0xa3ce9f234e, 0x4d9ba8aae0b, 0x392cf0cf99b, 0x465f8594525, 0x4f9c1fb1524, 0x3b1a1dd441c, 0x381edd42255, 0x37a599424b, 0x554caee8670, 0x5335bb91d81, 0x69288c8a1a3, 0x3df2b6e68e5, 0x75330d31d56, 0x51a604b090c, 0x32e0d5a7ca3, 0x41eb9d4896e, 0x633e2855c9f, 0x4780d70e32f, 0x73b0cd728c3, 0x16627402bad, 0x4418f2a818a, 0x5cdd06cf7e5, 0x2a8da97a3ae, 0x446864a8976, 0x5a7bcbd45ea, 0x4034a4b8b04, 0x6bdaac9c5fa, 0x18769ce3a67, 0x560278257c, 0x41c06d6b64c, 0x69f6f61bd4b, 0x16cc45f84fd, 0x53f6b42f0f0, 0x6cac3d234f3, 0x1f94e8f24d5, 0x319342c7148, 0x8685bca86d, 0x6b694a6ea66}); +constexpr StatTable43 SQR16_TABLE_43({0x1, 0x1ce77599049, 0x191715250a, 0xc1573d8dff, 0x118e73ab5e4, 0x4b6a83225fe, 0x72b4bc8e0f5, 0x4a4b2b6bb02, 0x66daf4741e9, 0x50baba19898, 0x5eb38771912, 0x6fb458aad3c, 0x5ce3b10bde9, 0x5575f3498f0, 0x5f075aa8a0a, 0x41d0aa8ee20, 0x609e3c78c28, 0xe2e45a8018, 0x523ac062837, 0x738388a569d, 0x6616ec46da9, 0x1a75cc16d96, 0x49b0b43bbc3, 0x400416b3c9a, 0x25813f41ffe, 0x309fdb9d0bc, 0x489f45b2cbf, 0xa141f4f88e, 0x739e0d11fb3, 0x44971f51cc0, 0x6490576e60e, 0x6c6674c5355, 0x6978126a4e1, 0x3d04eae5a5, 0x312eed633f2, 0x1de4b98d6b9, 0x118a106fb0a, 0x26dae025f4, 0x5c179312ebb, 0x75870ef1921, 0x60e9fed95c0, 0x209ab92427a, 0x1c5014a1937}); +constexpr StatTable43 QRT_TABLE_43({0x2bccc2d6f6c, 0x4bccc2d6f54, 0x4bccc2d6f56, 0x7cc7bc61df0, 0x4bccc2d6f52, 0x7d13b404b10, 0x7cc7bc61df8, 0x37456e9ac5a, 0x4bccc2d6f42, 0x4e042c6a6, 0x7d13b404b30, 0x4a56de9ef4c, 0x7cc7bc61db8, 0x14bc18d8e, 0x37456e9acda, 0x7c89f84fb1e, 0x4bccc2d6e42, 0x7ffae40d210, 0x4e042c4a6, 0x366f45dd06, 0x7d13b404f30, 0x496fcaf8cca, 0x4a56de9e74c, 0x370b62b6af4, 0x7cc7bc60db8, 0x1498185a8, 0x14bc1ad8e, 0x7e602c46a98, 0x37456e9ecda, 0x36ccc2c6e74, 0x7c89f847b1e, 0x7e27d06d516, 0x4bccc2c6e42, 0x7f93302c396, 0x7ffae42d210, 0x3dd3440706, 0x4e046c4a6, 0x78bbc09da36, 0x366f4ddd06, 0, 0x7d13b504f30, 0x8bbc09da00, 0x496fc8f8cca}); +typedef Field<uint64_t, 43, 89, StatTable43, &SQR_TABLE_43, &SQR2_TABLE_43, &SQR4_TABLE_43, &SQR8_TABLE_43, &SQR16_TABLE_43, &QRT_TABLE_43, IdTrans, &ID_TRANS, &ID_TRANS> Field43; +#endif + +#ifdef ENABLE_FIELD_INT_44 +// 44 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5, 5, 5> StatTable44; +constexpr StatTable44 SQR_TABLE_44({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x21, 0x84, 0x210, 0x840, 0x2100, 0x8400, 0x21000, 0x84000, 0x210000, 0x840000, 0x2100000, 0x8400000, 0x21000000, 0x84000000, 0x210000000, 0x840000000, 0x2100000000, 0x8400000000, 0x21000000000, 0x84000000000, 0x10000000042, 0x40000000108}); +constexpr StatTable44 SQR2_TABLE_44({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x21, 0x210, 0x2100, 0x21000, 0x210000, 0x2100000, 0x21000000, 0x210000000, 0x2100000000, 0x21000000000, 0x10000000042, 0x401, 0x4010, 0x40100, 0x401000, 0x4010000, 0x40100000, 0x401000000, 0x4010000000, 0x40100000000, 0x1000000084, 0x10000000840, 0x8421, 0x84210, 0x842100, 0x8421000, 0x84210000, 0x842100000, 0x8421000000, 0x84210000000, 0x42100000108, 0x21000001004, 0x10000010002}); +constexpr StatTable44 SQR4_TABLE_44({0x1, 0x10000, 0x100000000, 0x210, 0x2100000, 0x21000000000, 0x40100, 0x401000000, 0x10000000840, 0x8421000, 0x84210000000, 0x100001, 0x1000010000, 0x100002100, 0x21000210, 0x10002100042, 0x21000401000, 0x4010040100, 0x401008421, 0x10084210840, 0x42108421108, 0x84211000010, 0x10000000001, 0x31000, 0x310000000, 0x611, 0x6110000, 0x61100000000, 0xc4310, 0xc43100000, 0x31000001844, 0x18421100, 0x84211000021, 0x10000310001, 0x3100031000, 0x310006110, 0x61100611, 0x110061100c6, 0x61100c43100, 0xc4310c4310, 0x10c43118423, 0x31184210844, 0x42108421218, 0x84212100010}); +constexpr StatTable44 SQR8_TABLE_44({0x1, 0x21000401000, 0x84211000021, 0xa521000, 0x42108421719, 0x311e5310e55, 0x10401008c61, 0x3210031000, 0x10c43058522, 0x74110050101, 0xf5312e97201, 0x42108421109, 0x21061501611, 0x94311002961, 0xf59421000, 0x52d4b56923a, 0x201e5300e54, 0x71501c8fe71, 0xb6002131043, 0xb5e4c168432, 0xb320f5619ae, 0xf5000f97201, 0x10c43118422, 0x24010451100, 0x942113d4330, 0x8421a521001, 0x6310e130719, 0xf72fc731c6c, 0x4ae739631, 0x70008417e4d, 0x20ca6358b77, 0x64110094a51, 0xd4002e97200, 0xf7f5a13853a, 0xb12664417f6, 0x843322d6860, 0xf7c4100194d, 0x7382d17842b, 0xf67fd71a10c, 0x6efcca4b731, 0xf4e5901ea2d, 0xcb8278a46, 0xa32050401ee, 0xb7218bb6518}); +constexpr StatTable44 SQR16_TABLE_44({0x1, 0xc6cdb660138, 0x13de5a69a7b, 0x80bcafe7981, 0x60eb6f976d1, 0x677fbef6cce, 0x1549bb4cdec, 0x3b1ddf6859, 0xc01b8da28a6, 0xf3e11efbf8c, 0xd3e6faf8ee3, 0xa3dbc5712c8, 0x72361d7ca84, 0xe59e509337d, 0x15fca12a6f4, 0x33ce445498c, 0x44406de91fb, 0x9784b690571, 0xb0fb81753af, 0xb53a7c2c977, 0x34fbd3dba9b, 0xc758c22e647, 0xd5ff69aa469, 0x41e6d42b47d, 0xa4d1a3d02e7, 0x365db54ae9f, 0xd2293b8770b, 0xf1bf95c7746, 0x337fbe1d950, 0x726879e26a7, 0xa4be5ec2171, 0x7080da9df82, 0x7560017ce2, 0xd03997e34ae, 0x27ad4309a78, 0xb7b0ead892b, 0xf45bedb915d, 0xc4f0e25a52c, 0xe774a9d7fe8, 0xece6c1d7a26, 0xf20ea9ab655, 0x159bb624dc2, 0x12f2780b45f, 0x840cc52f19d}); +constexpr StatTable44 QRT_TABLE_44({0xf05334f4f6e, 0x4002016, 0x4002014, 0xf04350e6246, 0x4002010, 0x4935b379a26, 0xf04350e624e, 0xf84250c228e, 0x4002000, 0xf04300e521e, 0x4935b379a06, 0xb966838dd48, 0xf04350e620e, 0xf7b8b80feda, 0xf84250c220e, 0xf972e097d5e, 0x4002100, 0x8000020000, 0xf04300e501e, 0x430025000, 0x4935b379e06, 0xf976a09dc5e, 0xb966838d548, 0xf84218c029a, 0xf04350e720e, 0x4925f36bf06, 0xf7b8b80deda, 0xb047d3ee758, 0xf84250c620e, 0xf80350e720e, 0xf972e09fd5e, 0x8091825284, 0x4012100, 0x9015063210, 0x8000000000, 0xff31a028c5e, 0xf04300a501e, 0x44340b7100, 0x4300a5000, 0, 0x4935b279e06, 0xa976b2dce18, 0xf976a29dc5e, 0x8935b279e18}); +typedef Field<uint64_t, 44, 33, StatTable44, &SQR_TABLE_44, &SQR2_TABLE_44, &SQR4_TABLE_44, &SQR8_TABLE_44, &SQR16_TABLE_44, &QRT_TABLE_44, IdTrans, &ID_TRANS, &ID_TRANS> Field44; +typedef FieldTri<uint64_t, 44, 5, RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5, 5, 5>, &SQR_TABLE_44, &SQR2_TABLE_44, &SQR4_TABLE_44, &SQR8_TABLE_44, &SQR16_TABLE_44, &QRT_TABLE_44, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri44; +#endif + +#ifdef ENABLE_FIELD_INT_45 +// 45 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5, 5, 5> StatTable45; +constexpr StatTable45 SQR_TABLE_45({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x36, 0xd8, 0x360, 0xd80, 0x3600, 0xd800, 0x36000, 0xd8000, 0x360000, 0xd80000, 0x3600000, 0xd800000, 0x36000000, 0xd8000000, 0x360000000, 0xd80000000, 0x3600000000, 0xd800000000, 0x36000000000, 0xd8000000000, 0x16000000001b, 0x18000000005a}); +constexpr StatTable45 SQR2_TABLE_45({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0xd8, 0xd80, 0xd800, 0xd8000, 0xd80000, 0xd800000, 0xd8000000, 0xd80000000, 0xd800000000, 0xd8000000000, 0x18000000005a, 0x514, 0x5140, 0x51400, 0x514000, 0x5140000, 0x51400000, 0x514000000, 0x5140000000, 0x51400000000, 0x114000000036, 0x1400000003b8, 0x3b6e, 0x3b6e0, 0x3b6e00, 0x3b6e000, 0x3b6e0000, 0x3b6e00000, 0x3b6e000000, 0x3b6e0000000, 0x1b6e0000001b, 0x16e00000011f, 0xe0000001105}); +constexpr StatTable45 SQR4_TABLE_45({0x1, 0x10000, 0x100000000, 0xd8, 0xd80000, 0xd800000000, 0x5140, 0x51400000, 0x114000000036, 0x3b6e00, 0x3b6e000000, 0xe0000001105, 0x11011000, 0x110110000000, 0x1000000d58d8, 0xd58d58000, 0x18d58000054e, 0x5451454, 0x54514540000, 0x145400038db8, 0x38db6d8e0, 0xdb6d8e00104, 0x18e00101000a, 0x10100010100, 0x10100d8d8, 0x100d8d800d8, 0x18d800d8d85a, 0xd8d8511140, 0x18511140511a, 0x114051117b58, 0x11117b556e36, 0x1b556e3b5575, 0xe3b557f1015, 0x157f1011011e, 0x101101101c48, 0x1101c458d58, 0x1c458d58d580, 0xd58d58815d4, 0x158815d1451a, 0x15d1451452c0, 0x51452ce6f6e, 0x12ce6f6db6d6, 0xf6db6da6e3d, 0x16da6e39e00f, 0xe39e00000dd}); +constexpr StatTable45 SQR8_TABLE_45({0x1, 0x18d58000054e, 0xe3b557f1015, 0x51985198, 0xdb68cb55452, 0x1d0cc1d84f02, 0x140110c19ae, 0x11a16a14e7fe, 0x1e7ca7c62aa9, 0xe0eae26629b, 0x12182bee80ab, 0x1a38a68f28d3, 0x8581419c8c, 0x1d47f6f12ebc, 0x19fd34c3806e, 0x12ddba59f3cd, 0x10fa07f12a0e, 0x1d93eb544486, 0x1cf42cd119be, 0x1ff32d4b62c3, 0xf34ae031191, 0xada837715bf, 0xd368a753f92, 0x2ba87b17a03, 0x10374c3e4088, 0x1a6f539c11bd, 0x16548a5473c7, 0x1eb70011a8c9, 0x1ee5435ba1a3, 0x1173c0537680, 0xa1a3668dd6b, 0x119faad25e8, 0xd3909240e00, 0x1b560d018881, 0x127ecb9095ed, 0x306b507e701, 0x12b934c21ea3, 0x1a9d258c5b8b, 0x10452fbf0b1c, 0xae92fee120d, 0x183eb4b419fa, 0xc24d2391313, 0x4e6c4746f6, 0x2815fe7c395, 0xe4ab383747f}); +constexpr StatTable45 SQR16_TABLE_45({0x1, 0x14af92df932, 0x484e0190bdc, 0xda69889e16e, 0xcf70dfdb150, 0x18c6743571a8, 0x1b2c3ad7fa79, 0x5f0cbe204f6, 0xee973392a75, 0x3e86ef79673, 0xb2a9bef7181, 0x19b5347ff116, 0x1cae0ec79856, 0x69093f18f81, 0x1964382be09a, 0x92c894b073e, 0x1d99d2922eb2, 0x647905ad0eb, 0x1695971acdd3, 0x8f3292bc8c4, 0x1ee4057ad94, 0x17f02dc60e01, 0x1bb8e05ab4d5, 0x14de5d2a05d6, 0x13a019a02983, 0xcd7097c3616, 0x1bd798639b8f, 0x1cf0ca5ac7b2, 0xa93b983cf05, 0x159a955a2aa8, 0x69e5ba33397, 0x3a6b3392237, 0x26aeab71e13, 0x26fe04d38b9, 0x1fa9df0e8c45, 0x104e85c234b0, 0x1792853f8767, 0x81573b15f20, 0x127d6bfb06d3, 0x8110e6957e8, 0x11f59cbcc110, 0xad68264cad8, 0x61438575b35, 0x56e4446dc, 0x1cc9cb28b150}); +constexpr StatTable45 QRT_TABLE_45({0xede34e3e0fc, 0x1554148191aa, 0x1554148191a8, 0x1767be1dc4a6, 0x1554148191ac, 0x26bd4931492, 0x1767be1dc4ae, 0x233ab9c454a, 0x1554148191bc, 0x16939e8bb3dc, 0x26bd49314b2, 0x3c6ca8bac52, 0x1767be1dc4ee, 0x16caa5054c16, 0x233ab9c45ca, 0x14a1649628bc, 0x1554148190bc, 0x3c382881252, 0x16939e8bb1dc, 0x3c7ca0aa160, 0x26bd49310b2, 0x27f40158000, 0x3c6ca8ba452, 0x173fc092853c, 0x1767be1dd4ee, 0x16cbe284f25c, 0x16caa5056c16, 0x155559002f96, 0x233ab9c05ca, 0x26eb8908b32, 0x14a16496a8bc, 0x15440885333c, 0x1554148090bc, 0x17d60702e0, 0x3c3828a1252, 0x54548d10b2, 0x16939e8fb1dc, 0x3ac1e81b1d2, 0x3c7ca02a160, 0x166bd48310bc, 0x26bd48310b2, 0, 0x27f40358000, 0x10000000000e, 0x3c6cacba452}); +typedef Field<uint64_t, 45, 27, StatTable45, &SQR_TABLE_45, &SQR2_TABLE_45, &SQR4_TABLE_45, &SQR8_TABLE_45, &SQR16_TABLE_45, &QRT_TABLE_45, IdTrans, &ID_TRANS, &ID_TRANS> Field45; +#endif + +#ifdef ENABLE_FIELD_INT_46 +// 46 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5, 5> StatTableTRI46; +constexpr StatTableTRI46 SQR_TABLE_TRI46({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x3, 0xc, 0x30, 0xc0, 0x300, 0xc00, 0x3000, 0xc000, 0x30000, 0xc0000, 0x300000, 0xc00000, 0x3000000, 0xc000000, 0x30000000, 0xc0000000, 0x300000000, 0xc00000000, 0x3000000000, 0xc000000000, 0x30000000000, 0xc0000000000, 0x300000000000}); +constexpr StatTableTRI46 SQR2_TABLE_TRI46({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0xc, 0xc0, 0xc00, 0xc000, 0xc0000, 0xc00000, 0xc000000, 0xc0000000, 0xc00000000, 0xc000000000, 0xc0000000000, 0x5, 0x50, 0x500, 0x5000, 0x50000, 0x500000, 0x5000000, 0x50000000, 0x500000000, 0x5000000000, 0x50000000000, 0x100000000003, 0x3c, 0x3c0, 0x3c00, 0x3c000, 0x3c0000, 0x3c00000, 0x3c000000, 0x3c0000000, 0x3c00000000, 0x3c000000000, 0x3c0000000000}); +constexpr StatTableTRI46 SQR4_TABLE_TRI46({0x1, 0x10000, 0x100000000, 0xc, 0xc0000, 0xc00000000, 0x50, 0x500000, 0x5000000000, 0x3c0, 0x3c00000, 0x3c000000000, 0x1100, 0x11000000, 0x110000000000, 0xcc00, 0xcc000000, 0xc0000000005, 0x55000, 0x550000000, 0x10000000003f, 0x3fc000, 0x3fc0000000, 0x101, 0x1010000, 0x10100000000, 0xc0c, 0xc0c0000, 0xc0c00000000, 0x5050, 0x50500000, 0x105000000003, 0x3c3c0, 0x3c3c00000, 0x3c000000011, 0x111100, 0x1111000000, 0x1100000000cc, 0xcccc00, 0xcccc000000, 0xc0000000555, 0x5555000, 0x55550000000, 0x100000003fff, 0x3fffc000, 0x3fffc0000000}); +constexpr StatTableTRI46 SQR8_TABLE_TRI46({0x1, 0xcc000000, 0x3c3c0, 0x10000000c, 0x550055000, 0x3c000111111, 0xc0050, 0x103fc000003f, 0x1111cccc00, 0x5000500390, 0x13ec1010101, 0xcccc9999955, 0x5000001100, 0xc0c01010000, 0xc003fffc555, 0x12c003c0cc00, 0x10505c5c0c0f, 0x155450003ffe, 0x1100cc054100, 0xfcc5053c3d1, 0xd3ff2c00d, 0xd059c3a5c3a, 0x2828282d21e, 0x5000000001, 0xcd010000, 0xc000003c695, 0x3c103c0000c, 0x55c095c0c, 0x169550112eee, 0x1100000c1150, 0x1c339050003f, 0x112e320c01, 0x1d50cc50cf95, 0x116d5292c2c2, 0xcccc9959954, 0x1050cc00113f, 0xc1d1002c3c0, 0xc013fafc509, 0x12fa93c59d01, 0x135c9081d11e, 0x150453cc3fae, 0x13f0d044d33, 0x688119f0a84, 0x39d2d62d29d, 0x3751370167, 0x24e4e4e4b4b4}); +constexpr StatTableTRI46 SQR16_TABLE_TRI46({0x1, 0x1fe6e1ab503e, 0xbbae1f3f55b, 0x1d51cc530c59, 0x163a6a22e14a, 0x5847feb7f81, 0x1ec9cc5fd281, 0xf6cc7b80c70, 0x8f46b31e374, 0xc13bf2ed37d, 0x148a1595bffe, 0x581ad245849, 0x1ea6920b83c1, 0x9d9a8355c7d, 0x6bcf393d5ff, 0x1d4e245085c0, 0x602a8c5e62c, 0x1922dd69197f, 0x7945d3a2aad, 0xf82a823f768, 0xdd24665599b, 0x13b43f6a29d, 0x4df114f238d, 0x1ee783c75ec0, 0xfb670f65c31, 0xf855dc973d2, 0x61ede5f2651, 0x6c1a1266403, 0x1f66ed2a96a, 0xbbbdf683148, 0x1ecc83e160c0, 0x1a2778c4bc0c, 0x10e154273753, 0x1704f8873c23, 0x1b4d3172da99, 0x2b3be805044, 0x5bb08848b9d, 0x1967d2b99be5, 0x7fa55262740, 0xe761a27cc28, 0x17dedd7181b5, 0x155b0344714a, 0x15187b38816e, 0xc5a679b5300, 0x1096cbf94c5d, 0x3f6b3cc122da}); +constexpr StatTableTRI46 QRT_TABLE_TRI46({0x211c4fd486ba, 0x100104a, 0x1001048, 0x104d0492d4, 0x100104c, 0x20005040c820, 0x104d0492dc, 0x40008080, 0x100105c, 0x24835068ce00, 0x20005040c800, 0x200000400800, 0x104d04929c, 0x100904325c, 0x40008000, 0x25da9e77daf0, 0x100115c, 0x1184e1696f0, 0x24835068cc00, 0x24825169dd5c, 0x20005040cc00, 0x3ea3241c60c0, 0x200000400000, 0x211c4e5496f0, 0x104d04829c, 0x20005340d86c, 0x100904125c, 0x24835968de5c, 0x4000c000, 0x6400a0c0, 0x25da9e775af0, 0x118cf1687ac, 0x101115c, 0x1ea1745cacc0, 0x1184e1496f0, 0x20181e445af0, 0x2483506ccc00, 0x20240060c0, 0x24825161dd5c, 0x1e21755dbd9c, 0x20005050cc00, 0x26a3746cacc0, 0x3ea3243c60c0, 0xea3243c60c0, 0x200000000000, 0}); +typedef FieldTri<uint64_t, 46, 1, StatTableTRI46, &SQR_TABLE_TRI46, &SQR2_TABLE_TRI46, &SQR4_TABLE_TRI46, &SQR8_TABLE_TRI46, &SQR16_TABLE_TRI46, &QRT_TABLE_TRI46, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri46; +#endif + +#ifdef ENABLE_FIELD_INT_47 +// 47 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5> StatTable47; +constexpr StatTable47 SQR_TABLE_47({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x42, 0x108, 0x420, 0x1080, 0x4200, 0x10800, 0x42000, 0x108000, 0x420000, 0x1080000, 0x4200000, 0x10800000, 0x42000000, 0x108000000, 0x420000000, 0x1080000000, 0x4200000000, 0x10800000000, 0x42000000000, 0x108000000000, 0x420000000000, 0x80000000042, 0x200000000108}); +constexpr StatTable47 SQR2_TABLE_47({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x42, 0x420, 0x4200, 0x42000, 0x420000, 0x4200000, 0x42000000, 0x420000000, 0x4200000000, 0x42000000000, 0x420000000000, 0x200000000108, 0x1004, 0x10040, 0x100400, 0x1004000, 0x10040000, 0x100400000, 0x1004000000, 0x10040000000, 0x100400000000, 0x4000000042, 0x40000000420, 0x400000004200, 0x42108, 0x421080, 0x4210800, 0x42108000, 0x421080000, 0x4210800000, 0x42108000000, 0x421080000000, 0x210800000108, 0x108000001004, 0x80000010002}); +constexpr StatTable47 SQR4_TABLE_47({0x1, 0x10000, 0x100000000, 0x42, 0x420000, 0x4200000000, 0x1004, 0x10040000, 0x100400000000, 0x42108, 0x421080000, 0x210800000108, 0x1000010, 0x10000100000, 0x1000004200, 0x42000420, 0x420004200000, 0x42000100400, 0x1004010040, 0x40100400420, 0x4004210842, 0x42108421080, 0x84210810002, 0x108100000004, 0x142, 0x1420000, 0x14200000000, 0x5204, 0x52040000, 0x520400000000, 0x142508, 0x1425080000, 0x250800000528, 0x5210810, 0x52108100000, 0x81000014202, 0x142001420, 0x420014200042, 0x142000520400, 0x5204052040, 0x40520401424, 0x20401425094a, 0x142509425080, 0x9425085210a, 0x508521084204, 0x21084210804a, 0x421080420010}); +constexpr StatTable47 SQR8_TABLE_47({0x1, 0x420004200000, 0x250800000528, 0x11004, 0xc6210910402, 0x142005730c10, 0x101000010, 0x1056050040, 0x55a429184204, 0x111404110002, 0x532408500562, 0x3d196251d62e, 0x420142, 0x44524611c66, 0x1142047728, 0x46205e2508, 0x67339c2519da, 0x5661384b5880, 0x434346200424, 0x392938cb55aa, 0x724b0c31058c, 0x7f4f1cf703fc, 0x4fe303d32d1e, 0x75de250d676c, 0x100400011004, 0xc6210910540, 0x102525331834, 0x1101046318, 0x1057532548, 0x37f4bd7f4b5e, 0x115021380840, 0x52674a501142, 0x297a4a1b86ae, 0x666b0c2010ca, 0x776839031b88, 0x4b5316a05622, 0x254e215e2030, 0x6733ce2009de, 0xa8609d21e86, 0x567347420874, 0x6e0d31db55ba, 0x4357182485c4, 0x2afb35ef02ba, 0x5af227961c30, 0x64faad1b0116, 0x2d5d2527ef40, 0x6e27e3bc1978}); +constexpr StatTable47 SQR16_TABLE_47({0x1, 0x421ac25c8774, 0x30581389b510, 0x423b9c671db0, 0xa4537914208, 0x38f9be0dbf38, 0x351c5a8b92a8, 0xc38b9920da2, 0x508d34674f2a, 0x1f8c359a6b76, 0x5ac4bf86daaa, 0x51d6a6616df2, 0xe2717a0378a, 0x13353e783e6e, 0x55a55ac09ec6, 0x3f17cde43402, 0x760584b64b6c, 0x6acbecc99a02, 0x16be80e45b76, 0x2d5069e0005a, 0x3388f5759aa6, 0x2f98f891f4e2, 0x7657f368d924, 0x48f81e34f5b0, 0x51a9087f072e, 0x1de01ba9001c, 0x560b4b374bfc, 0x13f576988ff0, 0x3673cd322294, 0x595959f7c5fe, 0xbfa426eb4a4, 0x2b68fd7c02c2, 0x2a3c1437913a, 0x6e4b179fcf9e, 0x69ddf09bbdee, 0x7b91973d5e52, 0x1329cefd9514, 0x6a5f380b7ab0, 0x48e6620529c4, 0x60589a4b95b6, 0x5e4bd1d1aa34, 0x4b1ec7645cc2, 0x5cfb8785aec6, 0x34e47cf10c3a, 0x7b6c363eee10, 0x1dc52d768b32, 0x3585af9113a0}); +constexpr StatTable47 QRT_TABLE_47({0, 0x1001040, 0x1001042, 0x1047043076, 0x1001046, 0x112471c241e, 0x104704307e, 0x4304e052168, 0x1001056, 0x10004000, 0x112471c243e, 0x172a09c949d6, 0x104704303e, 0x4002020, 0x4304e0521e8, 0x5400e220, 0x1001156, 0x172b08c85080, 0x10004200, 0x41200b0800, 0x112471c203e, 0x172f0cca50a0, 0x172a09c941d6, 0x7eb88a11c1d6, 0x104704203e, 0x1044042020, 0x4000020, 0x42001011156, 0x4304e0561e8, 0x172a28c95880, 0x54006220, 0x112931cc21e, 0x1011156, 0x53670f283e, 0x172b08ca5080, 0x7a80c414a03e, 0x10044200, 0x40000000000, 0x4120030800, 0x1928318801e, 0x112470c203e, 0x799283188000, 0x172f0cea50a0, 0x1eb88a91c1c8, 0x172a098941d6, 0x3ea8cc95e1f6, 0x7eb88a91c1d6}); +typedef Field<uint64_t, 47, 33, StatTable47, &SQR_TABLE_47, &SQR2_TABLE_47, &SQR4_TABLE_47, &SQR8_TABLE_47, &SQR16_TABLE_47, &QRT_TABLE_47, IdTrans, &ID_TRANS, &ID_TRANS> Field47; +typedef FieldTri<uint64_t, 47, 5, RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5>, &SQR_TABLE_47, &SQR2_TABLE_47, &SQR4_TABLE_47, &SQR8_TABLE_47, &SQR16_TABLE_47, &QRT_TABLE_47, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri47; +#endif + +#ifdef ENABLE_FIELD_INT_48 +// 48 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6> StatTable48; +constexpr StatTable48 SQR_TABLE_48({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x2d, 0xb4, 0x2d0, 0xb40, 0x2d00, 0xb400, 0x2d000, 0xb4000, 0x2d0000, 0xb40000, 0x2d00000, 0xb400000, 0x2d000000, 0xb4000000, 0x2d0000000, 0xb40000000, 0x2d00000000, 0xb400000000, 0x2d000000000, 0xb4000000000, 0x2d0000000000, 0xb40000000000, 0xd0000000005a, 0x40000000011f}); +constexpr StatTable48 SQR2_TABLE_48({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x2d, 0x2d0, 0x2d00, 0x2d000, 0x2d0000, 0x2d00000, 0x2d000000, 0x2d0000000, 0x2d00000000, 0x2d000000000, 0x2d0000000000, 0xd0000000005a, 0x451, 0x4510, 0x45100, 0x451000, 0x4510000, 0x45100000, 0x451000000, 0x4510000000, 0x45100000000, 0x451000000000, 0x5100000000b4, 0x100000000bd9, 0xbdbd, 0xbdbd0, 0xbdbd00, 0xbdbd000, 0xbdbd0000, 0xbdbd00000, 0xbdbd000000, 0xbdbd0000000, 0xbdbd00000000, 0xdbd00000011f, 0xbd0000001001, 0xd0000001010f}); +constexpr StatTable48 SQR4_TABLE_48({0x1, 0x10000, 0x100000000, 0x2d, 0x2d0000, 0x2d00000000, 0x451, 0x4510000, 0x45100000000, 0xbdbd, 0xbdbd0000, 0xbdbd00000000, 0x101101, 0x1011010000, 0x1101000002d0, 0x2d2fd2d, 0x2d2fd2d0000, 0xfd2d0000454a, 0x45514551, 0x455145510000, 0x4551000bd0bd, 0xbd0b6d0bd, 0xd0b6d0bd011f, 0xd0bd0100011e, 0x10001010001, 0x10100012d00, 0x12d002d2d, 0x2d002d2d002d, 0x2d2d00295100, 0x2951045551, 0x5104555104e5, 0x555104ecbdb4, 0x4ecbdbd00bd, 0xbdbd00bdadbc, 0xbdadac1101, 0xadac11011001, 0x11011013c3fc, 0x1013c3fefd2d, 0xc3fefd2fd2a7, 0xfd2fd2baac36, 0xd2baac2d450b, 0xac2d45145ac2, 0x45145ad0f851, 0x5ad0f85adb64, 0xf85adb6cbd10, 0xdb6cbd0bd0a2, 0xbd0bd0bc003c, 0xd0bc002c001f}); +constexpr StatTable48 SQR8_TABLE_48({0x1, 0x2d2fd2d0000, 0x4ecbdbd00bd, 0x2c0000002c, 0x47882d9b95ec, 0xb9eec2eefc90, 0xbd900450, 0x9734009bfc8f, 0x93070a51e9b7, 0xb9d0aa02ec00, 0x8630787dd0ab, 0x6b3468ab98c9, 0x4100001bc1bd, 0x3ffeec1692fd, 0x2e0cce80414a, 0x11a77fc95626, 0x7c9856084ffe, 0x2f41c702f193, 0xd260e95bfeeb, 0x3b86220b54eb, 0x5735c802071a, 0x44626fc7ba84, 0x2b9cda923f5b, 0xc57d0a962e45, 0xd1685001450b, 0x6d78400145b6, 0x9114412978c1, 0x7d9ece1f5b0c, 0xfd2419988960, 0xd12ac3eaeaa5, 0x7d67e75f441d, 0xf86c2ba5457c, 0x40db7617aa6a, 0x80ee292186, 0xbd0f8525d34c, 0xa87ce27ca699, 0xacf3315d7a6d, 0x1a289bca977, 0x92975374e0f1, 0x3fcf113826ab, 0xff4d9be19a5e, 0x1412e5091900, 0x82c721f22d43, 0x1d773380ff32, 0xfed661cca7b1, 0x308072e06846, 0xd3eb44e91aa0, 0x819a669cbb14}); +constexpr StatTable48 SQR16_TABLE_48({0x1, 0x50c24311dfa9, 0xfc08c1e39482, 0xa9ff91b620a3, 0x54954a59d16c, 0xec45c0a9fb02, 0xba7004022837, 0xc1ea19828166, 0xee9a3efecffe, 0x57ed421a20bb, 0x69d387b19141, 0x9105d02d728f, 0xd2f24d4006da, 0x39195005f508, 0xd0206ff5333c, 0x8592e734a441, 0x787b36a1c435, 0x1151e6f03f85, 0xfe0429bf95ab, 0xab2b20b47651, 0x2a65fc935212, 0x7f73ae670e2e, 0x697c17f0fc4a, 0x55dc5681f013, 0xadbd09a289bc, 0x418414f64940, 0x927c737efd40, 0x38535d08fc98, 0xe811b107691c, 0x856c3bbf4cf6, 0x47e629ad3757, 0xf9c82b4b2c09, 0x64312c99e2f, 0xd4936c978dfd, 0x782ff8716675, 0xaf853e867dd7, 0x457143c1fa6d, 0x84c4dda48e91, 0xbac9aacda41e, 0x6a6e0ffb2dc1, 0xfef377f00194, 0x3a129790acc1, 0x541e49c6f92a, 0x73e821aca96d, 0x3a6f15c03f57, 0x1bf377c66f3b, 0xbff5e192fe3b, 0x346360ee74a}); +constexpr StatTable48 QRT_TABLE_48({0xc00442c284f0, 0xc16b7fda410a, 0xc16b7fda4108, 0xada3b5c79fbe, 0xc16b7fda410c, 0x16f3c18d5b0, 0xada3b5c79fb6, 0x7090a381f64, 0xc16b7fda411c, 0xcafc15d179f8, 0x16f3c18d590, 0x6630880e534e, 0xada3b5c79ff6, 0xa13dd1f49826, 0x7090a381fe4, 0xb87560f6a74, 0xc16b7fda401c, 0xaaaaffff0012, 0xcafc15d17bf8, 0xaafd15f07bf6, 0x16f3c18d190, 0x60000020000e, 0x6630880e5b4e, 0xcb977fcb401c, 0xada3b5c78ff6, 0x6663420cad0, 0xa13dd1f4b826, 0xc0045fc2f41c, 0x7090a385fe4, 0x6762e24b834, 0xb87560fea74, 0xc6351fed241c, 0xc16b7fdb401c, 0x60065622ea7a, 0xaaaafffd0012, 0xdf9562bea74, 0xcafc15d57bf8, 0x6657ea057bea, 0xaafd15f87bf6, 0xa79329ddaa66, 0x16f3c08d190, 0xa39229f0aa66, 0x60000000000e, 0x175fb4468ad0, 0x6630884e5b4e, 0, 0xcb977f4b401c, 0x2630884e5b40}); +typedef Field<uint64_t, 48, 45, StatTable48, &SQR_TABLE_48, &SQR2_TABLE_48, &SQR4_TABLE_48, &SQR8_TABLE_48, &SQR16_TABLE_48, &QRT_TABLE_48, IdTrans, &ID_TRANS, &ID_TRANS> Field48; +#endif +} + +Sketch* ConstructClMul6Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_41 + case 41: return new SketchImpl<Field41>(implementation, 41); +#endif +#ifdef ENABLE_FIELD_INT_42 + case 42: return new SketchImpl<Field42>(implementation, 42); +#endif +#ifdef ENABLE_FIELD_INT_43 + case 43: return new SketchImpl<Field43>(implementation, 43); +#endif +#ifdef ENABLE_FIELD_INT_44 + case 44: return new SketchImpl<Field44>(implementation, 44); +#endif +#ifdef ENABLE_FIELD_INT_45 + case 45: return new SketchImpl<Field45>(implementation, 45); +#endif +#ifdef ENABLE_FIELD_INT_47 + case 47: return new SketchImpl<Field47>(implementation, 47); +#endif +#ifdef ENABLE_FIELD_INT_48 + case 48: return new SketchImpl<Field48>(implementation, 48); +#endif + } + return nullptr; +} + +Sketch* ConstructClMulTri6Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_41 + case 41: return new SketchImpl<FieldTri41>(implementation, 41); +#endif +#ifdef ENABLE_FIELD_INT_42 + case 42: return new SketchImpl<FieldTri42>(implementation, 42); +#endif +#ifdef ENABLE_FIELD_INT_44 + case 44: return new SketchImpl<FieldTri44>(implementation, 44); +#endif +#ifdef ENABLE_FIELD_INT_46 + case 46: return new SketchImpl<FieldTri46>(implementation, 46); +#endif +#ifdef ENABLE_FIELD_INT_47 + case 47: return new SketchImpl<FieldTri47>(implementation, 47); +#endif + } + return nullptr; +} diff --git a/src/minisketch/src/fields/clmul_7bytes.cpp b/src/minisketch/src/fields/clmul_7bytes.cpp new file mode 100644 index 0000000000..2050dc32dd --- /dev/null +++ b/src/minisketch/src/fields/clmul_7bytes.cpp @@ -0,0 +1,170 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_7) + +#include "clmul_common_impl.h" + +#include "../int_utils.h" +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_49 +// 49 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5, 5, 5, 5> StatTable49; +constexpr StatTable49 SQR_TABLE_49({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x402, 0x1008, 0x4020, 0x10080, 0x40200, 0x100800, 0x402000, 0x1008000, 0x4020000, 0x10080000, 0x40200000, 0x100800000, 0x402000000, 0x1008000000, 0x4020000000, 0x10080000000, 0x40200000000, 0x100800000000, 0x402000000000, 0x1008000000000, 0x20000000402, 0x80000001008, 0x200000004020, 0x800000010080}); +constexpr StatTable49 SQR2_TABLE_49({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x1008, 0x10080, 0x100800, 0x1008000, 0x10080000, 0x100800000, 0x1008000000, 0x10080000000, 0x100800000000, 0x1008000000000, 0x80000001008, 0x800000010080, 0x100004, 0x1000040, 0x10000400, 0x100004000, 0x1000040000, 0x10000400000, 0x100004000000, 0x1000040000000, 0x400001008, 0x4000010080, 0x40000100800, 0x400001008000, 0x10080402, 0x100804020, 0x1008040200, 0x10080402000, 0x100804020000, 0x1008040200000, 0x80402001008, 0x804020010080, 0x40200100004, 0x402001000040, 0x20010000002, 0x200100000020}); +constexpr StatTable49 SQR4_TABLE_49({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x1008000, 0x10080000000, 0x800000010080, 0x100004000, 0x1000040000000, 0x400001008000, 0x10080402000, 0x804020010080, 0x200100000020, 0x1000000001000, 0x11008000, 0x110080000000, 0x800000110880, 0x1108004000, 0x1080040001008, 0x400011008400, 0x110084402000, 0x844020110880, 0x201108040220, 0x1080402000008, 0x20001008002, 0x10080000100, 0x800001010080, 0x10100004000, 0x1000040010080, 0x400101808000, 0x1018080402000, 0x8040210100c0, 0x210100400020, 0x1004000011080, 0x11180c020, 0x11180c0200000, 0xc020011108c0, 0x11108004010, 0x1080040111088, 0x401111808400, 0x1118084403008, 0x8440311908c0, 0x311908440220, 0x108440211008c, 0x2110184c022, 0x10184c0201108, 0xc020100904c2, 0x100904024010, 0x1040240000004}); +constexpr StatTable49 SQR8_TABLE_49({0x1, 0x800000110880, 0x210100400020, 0x1040240000004, 0x130081008002, 0x191c0a54130c8, 0x900804130890, 0x210111408020, 0x582c0402004, 0xd320910984c0, 0xb1d1ad4522e8, 0x1d80945829818, 0x1218540601128, 0x1240340000024, 0x11300c1018082, 0x193d1a4c5f0ea, 0x148a0522810, 0xe0201051c8e0, 0x2dc7c25120a8, 0x1d2205148a440, 0x33c0adc0e24a, 0x17850e987bab8, 0xa8a40071c9e0, 0x10d47d391c1a8, 0x9740b01888c2, 0x91c0e54130c8, 0x920805138892, 0x130819500b028, 0x8583c0516884, 0x1fa25934984e8, 0x1f5c2fcc5a6ec, 0x15a094483189a, 0x1014cc242300, 0xac020000582c, 0x3b05524100aa, 0x1520255145c6e, 0x4279b95aec40, 0x1f1a0951178e8, 0xbcc646004828, 0x3b055219aca8, 0x57d2fc40666e, 0xba50b987beba, 0x8aa44d913bea, 0x944717d1b9a0, 0x776562491022, 0x1701305105c4c, 0x19039cd5be842, 0x1b281d8e58082, 0x134dac80532a4}); +constexpr StatTable49 SQR16_TABLE_49({0x1, 0x790181b552e0, 0xeb19044e00a, 0xc6bf7911f7ae, 0x447f77c1a0c4, 0x19d2a0d21c480, 0x13d4e22aadedc, 0x18fa344c8f0a6, 0x1481c1bbfde92, 0x41547e22f6e0, 0xf5ad96335088, 0xd7e4db3adaa0, 0x197fc8d7b53d0, 0x37781564b82a, 0xa52ef2139cbc, 0x153c6a0949498, 0x18d7401fc152e, 0xc4b5d8597752, 0xd15cd891aa2, 0x217903427da8, 0x13ec9e269a0e0, 0xc01720774514, 0x389aeb1d788a, 0x64a914a860a4, 0xa09aebec6188, 0x15c3239e150c8, 0x38f8fe110ce, 0xc1ea415c5006, 0x3209972f2ff0, 0x41bfc6b2ad88, 0x1ccc2fd5f73c8, 0x7bed1f863c00, 0x1a46d9b9844f4, 0x12e3ca6573ff6, 0x290c26cca98c, 0x514cb03b3b2e, 0x11168909cbc2c, 0x8e6dc910afda, 0x11311def1c440, 0x3e42d62664d8, 0x1c2bb2d75fe80, 0x2db5d58b45ca, 0x3d14059fd338, 0x109e8f457ebf8, 0x43b071b62a64, 0x185242247c010, 0x5e0c7721c092, 0x1c94950e46b82, 0x1761170f76a40}); +constexpr StatTable49 QRT_TABLE_49({0, 0x10004196, 0x10004194, 0x5099461f080, 0x10004190, 0x40840600c20, 0x5099461f088, 0x58a56349cfde, 0x10004180, 0x48641a0c03fe, 0x40840600c00, 0x10084002848, 0x5099461f0c8, 0x4002048, 0x58a56349cf5e, 0x5088460a048, 0x10004080, 0x4c2852624dde, 0x48641a0c01fe, 0x14893129c280, 0x40840600800, 0x1eb23c323ace8, 0x10084002048, 0x48740a09417e, 0x5099461e0c8, 0x40852604d96, 0x4000048, 0x5cad2b29c37e, 0x58a563498f5e, 0x20000200, 0x50884602048, 0x10000000000, 0x10014080, 0x4c2a56624d96, 0x4c2852604dde, 0x1ee2347438ca0, 0x48641a0801fe, 0x480000000048, 0x14893121c280, 0x14091121c080, 0x40840700800, 0x1a5099561e17e, 0x1eb23c303ace8, 0x8740a894136, 0x10084402048, 0x18101c501ace8, 0x48740a89417e, 0x15dace6286f96, 0x5099561e0c8}); +typedef Field<uint64_t, 49, 513, StatTable49, &SQR_TABLE_49, &SQR2_TABLE_49, &SQR4_TABLE_49, &SQR8_TABLE_49, &SQR16_TABLE_49, &QRT_TABLE_49, IdTrans, &ID_TRANS, &ID_TRANS> Field49; +typedef FieldTri<uint64_t, 49, 9, RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5, 5, 5, 5>, &SQR_TABLE_49, &SQR2_TABLE_49, &SQR4_TABLE_49, &SQR8_TABLE_49, &SQR16_TABLE_49, &QRT_TABLE_49, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri49; +#endif + +#ifdef ENABLE_FIELD_INT_50 +// 50 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5, 5, 5, 5> StatTable50; +constexpr StatTable50 SQR_TABLE_50({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x1d, 0x74, 0x1d0, 0x740, 0x1d00, 0x7400, 0x1d000, 0x74000, 0x1d0000, 0x740000, 0x1d00000, 0x7400000, 0x1d000000, 0x74000000, 0x1d0000000, 0x740000000, 0x1d00000000, 0x7400000000, 0x1d000000000, 0x74000000000, 0x1d0000000000, 0x740000000000, 0x1d00000000000, 0x340000000001d, 0x1000000000053}); +constexpr StatTable50 SQR2_TABLE_50({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x74, 0x740, 0x7400, 0x74000, 0x740000, 0x7400000, 0x74000000, 0x740000000, 0x7400000000, 0x74000000000, 0x740000000000, 0x340000000001d, 0x151, 0x1510, 0x15100, 0x151000, 0x1510000, 0x15100000, 0x151000000, 0x1510000000, 0x15100000000, 0x151000000000, 0x1510000000000, 0x1100000000069, 0x10000000006e4, 0x6e34, 0x6e340, 0x6e3400, 0x6e34000, 0x6e340000, 0x6e3400000, 0x6e34000000, 0x6e340000000, 0x6e3400000000, 0x2e3400000001d, 0x234000000011f, 0x3400000001118}); +constexpr StatTable50 SQR4_TABLE_50({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x74000, 0x740000000, 0x340000000001d, 0x151000, 0x1510000000, 0x1100000000069, 0x6e3400, 0x6e34000000, 0x234000000011f, 0x1110100, 0x11101000000, 0x1010000000734, 0x7334740, 0x73347400000, 0x347400000145c, 0x14540510, 0x145405100000, 0x510000068b9, 0x68b91a34, 0x68b91a340000, 0x11a3400010106, 0x101010001, 0x1010100010000, 0x1000100074740, 0x1000747474000, 0x347474007401d, 0x340074015050d, 0x340150505101d, 0x1050510151069, 0x11015106e5a5d, 0x1106e5a5a3469, 0x25a5a346e351f, 0x2346e3510111e, 0x235101110001f, 0x111000110634, 0x1106347334, 0x1063473340074, 0x733400735301, 0x7353004541, 0x353004541014c, 0x454101446dc0, 0x101446dc1cb90, 0x6dc1cb97468d, 0x1cb97468c1a30, 0x3468c1a350009, 0x1a3500010007}); +constexpr StatTable50 SQR8_TABLE_50({0x1, 0x7334740, 0x1050510151069, 0x3468c1a350009, 0x341624173531c, 0x245791a347b50, 0x23179d1a40682, 0x1671402235203, 0x321023818751e, 0x143ca5b716dd5, 0x171633ad257de, 0x33860681a5d1d, 0x5e572f82a317, 0x10e7512224646, 0x32d6b56300005, 0x350ab39687414, 0x25c47550c1a8a, 0x23a2e2d91533f, 0x2211af19c2381, 0x352073a863a68, 0x37f43380f0ac4, 0x233516127052a, 0x25ad4785169cf, 0x237b6a609b0b6, 0x132fd372b5dac, 0x1f311727562e, 0x345bd7e275754, 0x352fe5b3d7708, 0x259a328ca3376, 0x25101aab53ece, 0x32701d9da5ace, 0x17809a9c86099, 0x72b4752a7323, 0x202d22dc33a7c, 0x5a8c0dbc19a2, 0x14a86b37416ad, 0x5c574289fe12, 0x3627f3bf0f37b, 0x27349052a4f83, 0x2436d71033de5, 0x22fab345e0bce, 0x27ea796d5a27a, 0x1e4f33562d17, 0x31a1f9c3f2154, 0x1638db7753f96, 0x2256163f33b5f, 0x11a6ecf28882e, 0x1bd4cf35f47cc, 0x25e19aeb21e64, 0x371612d0b4dcd}); +constexpr StatTable50 SQR16_TABLE_50({0x1, 0x14db3a1b1531f, 0x270a39b5e8c48, 0x26536a58442bd, 0x7f158d4b869e, 0x12663760f7d, 0x29634a2c8876, 0x15271f7ec5d31, 0x17fbb0726d0f0, 0x7f0f7bf826bb, 0x115135d3c7c4c, 0x348ffaaa125e5, 0x1887695a20d9, 0x25e41181c0de, 0x2670d7f17fb35, 0x356079737f513, 0x22bebda8a1574, 0x315f9649d2b50, 0x13abe45aa6ac8, 0x723d536b5242, 0x24263520a22a9, 0x15860c0156a69, 0x271d0bbeed892, 0x146920f281d19, 0x117d5d46e7991, 0x278d8273551fc, 0x15d73a9745614, 0x7e5e966bbfe0, 0x687b14e62abb, 0x178acea79fa5c, 0x3363c557e9662, 0x3153c79bf06ef, 0x15c8ff9daf7ce, 0x243b030f4617a, 0x20663fbd2383a, 0x25c5dbd448872, 0x21fc8dfbd2429, 0x229f9fb8f01b0, 0x17a180ae72359, 0x1c8e2f554ad9, 0x174596d1e774f, 0x3264c5da47f53, 0x333817d45b05c, 0x321907ec10dfd, 0x3a12b2018ada, 0x23ab0599cd08, 0x23028d60c00e5, 0x8ca05e2a1eab, 0x3537bf673a228, 0x32f8cf8611080}); +constexpr StatTable50 QRT_TABLE_50({0xfbdfa3ae9d4c, 0x38143245a4878, 0x38143245a487a, 0x38527487e7492, 0x38143245a487e, 0x3124c61f56d2a, 0x38527487e749a, 0xfa8c91b087c0, 0x38143245a486e, 0x3eca48c6196be, 0x3124c61f56d0a, 0x380000040080a, 0x38527487e74da, 0x976b2d8b39b4, 0xfa8c91b08740, 0xfa8cd5b02724, 0x38143245a496e, 0x316291dd013fe, 0x3eca48c6194be, 0x10344122064, 0x3124c61f5690a, 0x68c5f006ee40, 0x380000040000a, 0x852749fe64d0, 0x38527487e64da, 0x37ef8e9d0e9da, 0x976b2d8b19b4, 0x37fabd1cef34a, 0xfa8c91b0c740, 0x96282d9159b4, 0xfa8cd5b0a724, 0x464a8249dd0, 0x38143245b496e, 0x37eaa8ddc94be, 0x316291dd213fe, 0x392446035690a, 0x3eca48c6594be, 0x974b258b4964, 0x103441a2064, 0x385a7c87fb4da, 0x3124c61e5690a, 0xeb8ad5d9a724, 0x68c5f026ee40, 0x3724c61e5690a, 0x380000000000a, 0x3a8c5f026ee4a, 0x8527497e64d0, 0, 0x38527497e64da, 0x2fbdfa2ae8d0a}); +typedef Field<uint64_t, 50, 29, StatTable50, &SQR_TABLE_50, &SQR2_TABLE_50, &SQR4_TABLE_50, &SQR8_TABLE_50, &SQR16_TABLE_50, &QRT_TABLE_50, IdTrans, &ID_TRANS, &ID_TRANS> Field50; +#endif + +#ifdef ENABLE_FIELD_INT_51 +// 51 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5, 5, 5> StatTable51; +constexpr StatTable51 SQR_TABLE_51({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x96, 0x258, 0x960, 0x2580, 0x9600, 0x25800, 0x96000, 0x258000, 0x960000, 0x2580000, 0x9600000, 0x25800000, 0x96000000, 0x258000000, 0x960000000, 0x2580000000, 0x9600000000, 0x25800000000, 0x96000000000, 0x258000000000, 0x960000000000, 0x2580000000000, 0x160000000004b, 0x580000000012c, 0x6000000000426}); +constexpr StatTable51 SQR2_TABLE_51({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x96, 0x960, 0x9600, 0x96000, 0x960000, 0x9600000, 0x96000000, 0x960000000, 0x9600000000, 0x96000000000, 0x960000000000, 0x160000000004b, 0x6000000000426, 0x4114, 0x41140, 0x411400, 0x4114000, 0x41140000, 0x411400000, 0x4114000000, 0x41140000000, 0x411400000000, 0x4114000000000, 0x1140000000258, 0x1400000002516, 0x40000000251f6, 0x251d38, 0x251d380, 0x251d3800, 0x251d38000, 0x251d380000, 0x251d3800000, 0x251d38000000, 0x251d380000000, 0x51d380000012c, 0x1d3800000100e, 0x538000001003d, 0x380000010011e}); +constexpr StatTable51 SQR4_TABLE_51({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x96000, 0x960000000, 0x160000000004b, 0x411400, 0x4114000000, 0x1140000000258, 0x251d380, 0x251d3800000, 0x1d3800000100e, 0x10010110, 0x100101100000, 0x1011000009600, 0x960969f6, 0x960969f60000, 0x169f60004110b, 0x6000411015132, 0x4110151054000, 0x1510540251f60, 0x540251f6ba760, 0x51f6ba74eb92c, 0x3a74eb900001f, 0x6b90000010033, 0x100010860, 0x1000108600000, 0x1086000096000, 0x960092874, 0x160092874004b, 0x128740041144b, 0x40041144304e2, 0x1144304c78258, 0x304c78251d1d8, 0x78251d1c38368, 0x1d1c38352800e, 0x3835280011188, 0x280011197096e, 0x1119709787000, 0x709787009fb46, 0x7009fb7861c9, 0x1fb7861cae24b, 0x61cae2456109, 0x2e245610a7bc3, 0x5610a7bd61498, 0x27bd614b79d2b, 0x614b79d3a74de, 0x79d3a74e9f68a, 0x274e9f6b06011, 0x1f6b06000000f}); +constexpr StatTable51 SQR8_TABLE_51({0x1, 0x960969f6, 0x40041144304e2, 0x79d3a74e9f68a, 0x61005961939d4, 0x2e2108dfdafb5, 0xe7e61b897f73, 0x493a58b330d18, 0x7882105dc65ec, 0x5f00774200d11, 0x63ef4cd371ef3, 0x660b24b8d214b, 0x7ab791e669e3d, 0x10821820969f6, 0x1544b9d4c3f3e, 0x831185e3da14, 0x1eb0831983187, 0x1d8699ae87312, 0x586e000eb5f1a, 0x3ea794ef9c821, 0x2ab1c63209cc1, 0x7f434bcc29855, 0x673d370c40117, 0x6a668249ddd8b, 0x48be019e56bbe, 0x57d1a751be823, 0x5621931ca6d5f, 0x68c5a37844a68, 0xefa69123b6b1, 0x5804da97df62a, 0x30c29b82f3986, 0x5b808f6ddc779, 0x2c8b4e7596cbe, 0x2c5a432ec7a14, 0x7f178a4d63277, 0x77112a07b99b7, 0x56cf47ad50529, 0x73a2180190a41, 0x25cbc68f1f1a8, 0x1c27dc22e6950, 0x2fbf4aafee2ad, 0x554b728a595ca, 0x52726d34627e, 0x6dcc716c9e860, 0x36ade274d5eff, 0x1fa23a55b359a, 0x1bc6260896059, 0x53a74c5798bc1, 0x50e671fc54a4a, 0x251a72b3c4c3c, 0x6d9623f5d3a1e}); +constexpr StatTable51 SQR16_TABLE_51({0x1, 0x27b32044e9663, 0x528c08dd195bf, 0x5d461228d5764, 0x616db8f131bf6, 0x9d910988ca4, 0x1e7a7a29c55fd, 0x512a2e6297818, 0x688d44453ead0, 0x70e0b6e1b3be2, 0x4313e5612d70, 0x132241d43589d, 0x7ca688c29c89a, 0x1d6b8caeb8958, 0x36d06e8e76e3b, 0x18ebafc89388e, 0x1cb5f93b2c29c, 0x5137bd7b7b6ec, 0x6e3ae8731000b, 0x359203e5e12fe, 0x1822ded1f1e16, 0x3ee9c50cbcb89, 0x5cc0b4564ab4, 0x695b235bd9236, 0x283c619a1ecb, 0x6f37f1f6ef70d, 0x7f394b6fbdd53, 0x3f482b36793f, 0x4055274e56dfa, 0x1a85d9d434f33, 0x37aa8f3df2031, 0x5f4e77b2bb063, 0x6e9702d84f07b, 0x25f16f8ffd4c2, 0x22c591d8277cb, 0x59435d9bae242, 0x46eaf9f69ddd9, 0x3098c1e26bd6e, 0x6c6544847a1d, 0x254946c0c33ce, 0x23970a6118811, 0x67f6c55082b49, 0x6592c83ebde46, 0x716418f089ed8, 0x8cb8de463166, 0x37cb1794fac42, 0x94ac55c1ac68, 0x3ab0d33bb4fdf, 0x1669c2f7ae3c5, 0x4d4e4f61d1f04, 0x476980d17eef5}); +constexpr StatTable51 QRT_TABLE_51({0x778bf2703d152, 0x2aaaafbff2092, 0x2aaaafbff2090, 0x4d2119c7e7780, 0x2aaaafbff2094, 0x65de1df8ae194, 0x4d2119c7e7788, 0x67d63d7ba262c, 0x2aaaafbff2084, 0x28ff003f4167c, 0x65de1df8ae1b4, 0x658397fb1d034, 0x4d2119c7e77c8, 0x4d7c9284526ba, 0x67d63d7ba26ac, 0x6666333fc0cbe, 0x2aaaafbff2184, 0x295b807ab55ee, 0x28ff003f4147c, 0x2aaabfffe0016, 0x65de1df8ae5b4, 0x209210349d60, 0x658397fb1d834, 0x4d215dc7cf1c8, 0x4d2119c7e67c8, 0x662b2b3d7b4be, 0x4d7c9284506ba, 0x255af00b36e0, 0x67d63d7ba66ac, 0x65de1fb8ac1a6, 0x6666333fc8cbe, 0x662f3b3ded4be, 0x2aaaafbfe2184, 0x663a9dbc3a426, 0x295b807a955ee, 0x4cdc9ec128928, 0x28ff003f0147c, 0x28a0c93cd511c, 0x2aaabfff60016, 0x65d73cf8e78d4, 0x65de1df9ae5b4, 0x4d5eddc44f1c8, 0x209210149d60, 0x357fcc506c8a, 0x658397ff1d834, 0, 0x4d215dcfcf1c8, 0x63f536f5d4554, 0x4d2119d7e67c8, 0x4000000000022, 0x662b2b1d7b4be}); +typedef Field<uint64_t, 51, 75, StatTable51, &SQR_TABLE_51, &SQR2_TABLE_51, &SQR4_TABLE_51, &SQR8_TABLE_51, &SQR16_TABLE_51, &QRT_TABLE_51, IdTrans, &ID_TRANS, &ID_TRANS> Field51; +#endif + +#ifdef ENABLE_FIELD_INT_52 +// 52 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5, 5> StatTable52; +constexpr StatTable52 SQR_TABLE_52({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x9, 0x24, 0x90, 0x240, 0x900, 0x2400, 0x9000, 0x24000, 0x90000, 0x240000, 0x900000, 0x2400000, 0x9000000, 0x24000000, 0x90000000, 0x240000000, 0x900000000, 0x2400000000, 0x9000000000, 0x24000000000, 0x90000000000, 0x240000000000, 0x900000000000, 0x2400000000000, 0x9000000000000, 0x4000000000012}); +constexpr StatTable52 SQR2_TABLE_52({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x9, 0x90, 0x900, 0x9000, 0x90000, 0x900000, 0x9000000, 0x90000000, 0x900000000, 0x9000000000, 0x90000000000, 0x900000000000, 0x9000000000000, 0x41, 0x410, 0x4100, 0x41000, 0x410000, 0x4100000, 0x41000000, 0x410000000, 0x4100000000, 0x41000000000, 0x410000000000, 0x4100000000000, 0x1000000000024, 0x249, 0x2490, 0x24900, 0x249000, 0x2490000, 0x24900000, 0x249000000, 0x2490000000, 0x24900000000, 0x249000000000, 0x2490000000000, 0x4900000000012, 0x9000000000104}); +constexpr StatTable52 SQR4_TABLE_52({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x9000, 0x90000000, 0x900000000000, 0x4100, 0x41000000, 0x410000000000, 0x2490, 0x24900000, 0x249000000000, 0x1001, 0x10010000, 0x100100000000, 0x1000000000900, 0x9009000, 0x90090000000, 0x900000000410, 0x4104100, 0x41041000000, 0x410000000249, 0x2492490, 0x24924900000, 0x9249000000104, 0x1000001, 0x10000010000, 0x100000090, 0x1000000900000, 0x9000009000, 0x90000041, 0x900000410000, 0x4100004100, 0x1000041000024, 0x410000249000, 0x2490002490, 0x4900024900012, 0x249000100100, 0x1001001001, 0x10010010009, 0x100100090090, 0x1000900900900, 0x9009009009000, 0x90090041041, 0x900410410410, 0x4104104104100, 0x1041041024924, 0x410249249249, 0x2492492492490, 0x4924924910002, 0x9249100000004}); +constexpr StatTable52 SQR8_TABLE_52({0x1, 0x1000000000900, 0x900000410000, 0x410249249249, 0x349100000000, 0x1900000d1, 0x10d10d1065965, 0x51010000, 0x2d924909000, 0x4114114114109, 0xe591, 0x1000007c96591, 0x1903981d13981, 0x249000000001, 0x1000100000990, 0x990090451041, 0x41022cb49249, 0x37d824910000, 0x90191890190d8, 0x10d10d106edf5, 0x54114101, 0x102f4b400bd90, 0x5c04114114108, 0x8f5900000ebcc, 0x1e59107b5f401, 0x8f5ab89f5abcc, 0x24924900001, 0x1010010010909, 0x90000041d100, 0x41024f7df7d9, 0x34a591003491, 0x9001900000d0, 0x4c10d106522c, 0xb49051500100, 0x43dafdb40249, 0x428c0c5114109, 0x59001f590e576, 0x49f59f25facf6, 0x19039c4c17881, 0x26fdb4902491, 0x10110010998, 0x109009045cc51, 0x90022a8867d9, 0x9527ffcb5a6dc, 0x9bc01190190d9, 0x14c119006e608, 0x42fd9e0d55042, 0x143f711b5bfd9, 0x5fa58e1809008, 0x23d647ac81eb2, 0x57ac8f223a466, 0x865abc8b4abcc}); +constexpr StatTable52 SQR16_TABLE_52({0x1, 0x4f881c2d96599, 0xd7eb53011fc41, 0x81d7387961fef, 0xd9afe338982c3, 0x17590c140da98, 0x141a99a87a04e, 0x10036ba4083d9, 0x8f4f72ffb12c7, 0xc8b70df241e1b, 0x18bd9e5d46c40, 0x18331d76266bd, 0x4d915f264a4e0, 0x46aeffb8e4037, 0x4800042de37b5, 0xdb172953272e8, 0x17a9c2edf826a, 0x191cf7053e3f2, 0x82036da842cea, 0x5891da126c1e, 0x1e536e9e4af49, 0x451b5638f5449, 0x5a006c6c4f8c8, 0x5ac71a535fb44, 0xd39a4d489ebd0, 0x4704e31bc006d, 0xc4b327f6ffae1, 0x46980b709bd00, 0xd755405154c11, 0x5741be2d0b797, 0xcb3d48ed630cb, 0x98a66c9f4f599, 0x4caa324b91629, 0x816b5015eeeaf, 0xa595e92a8ed4, 0xc93c6d9f5a073, 0x4250068b39e2, 0x105add98997b5, 0x408b030c0bce0, 0xced5e4a4a2028, 0x1eb59d68e7f25, 0x189756a5b6db0, 0xc49c5a7c98b01, 0x18c9a496767cb, 0xde650554b3d49, 0x11077035fd81c, 0x8b37c5e95a659, 0x45b9226c2c25e, 0xdd2b5e20c7c8b, 0x6de972f0e7025, 0x84e80092f5681, 0x8dfcf97183cbc}); +constexpr StatTable52 QRT_TABLE_52({0xc108165459b0e, 0x10004086, 0x10004084, 0xc00000100104e, 0x10004080, 0x2041810a545b0, 0xc000001001046, 0x1181e055efc0, 0x10004090, 0x40810214390, 0x2041810a54590, 0xc000141019106, 0xc000001001006, 0x10816045ab40, 0x1181e055ef40, 0xc000111015196, 0x10004190, 0xe045c19af44a2, 0x40810214190, 0xe045809ad0532, 0x2041810a54190, 0xdb387a03fe646, 0xc000141019906, 0x2000000800000, 0xc000001000006, 0x2486548199c34, 0x108160458b40, 0x2041808a50534, 0x1181e055af40, 0xc0408312153d6, 0xc00011101d196, 0x21499f0e0eed0, 0x10014190, 0xe15dff9faabe2, 0xe045c19ad44a2, 0xdb787b01ea7d6, 0x40810254190, 0xe484409180532, 0xe045809a50532, 0xc14095164d896, 0x2041810b54190, 0x217dee8fb7a74, 0xdb387a01fe646, 0x441810b54190, 0xc000141419906, 0xc3386e15e7f46, 0x2000000000000, 0x1000141419900, 0xc000000000006, 0, 0x248654a199c34, 0xa48654a199c32}); +typedef Field<uint64_t, 52, 9, StatTable52, &SQR_TABLE_52, &SQR2_TABLE_52, &SQR4_TABLE_52, &SQR8_TABLE_52, &SQR16_TABLE_52, &QRT_TABLE_52, IdTrans, &ID_TRANS, &ID_TRANS> Field52; +typedef FieldTri<uint64_t, 52, 3, RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5, 5>, &SQR_TABLE_52, &SQR2_TABLE_52, &SQR4_TABLE_52, &SQR8_TABLE_52, &SQR16_TABLE_52, &QRT_TABLE_52, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri52; +#endif + +#ifdef ENABLE_FIELD_INT_53 +// 53 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 5> StatTable53; +constexpr StatTable53 SQR_TABLE_53({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x8e, 0x238, 0x8e0, 0x2380, 0x8e00, 0x23800, 0x8e000, 0x238000, 0x8e0000, 0x2380000, 0x8e00000, 0x23800000, 0x8e000000, 0x238000000, 0x8e0000000, 0x2380000000, 0x8e00000000, 0x23800000000, 0x8e000000000, 0x238000000000, 0x8e0000000000, 0x2380000000000, 0x8e00000000000, 0x3800000000047, 0xe00000000011c, 0x18000000000437}); +constexpr StatTable53 SQR2_TABLE_53({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x238, 0x2380, 0x23800, 0x238000, 0x2380000, 0x23800000, 0x238000000, 0x2380000000, 0x23800000000, 0x238000000000, 0x2380000000000, 0x3800000000047, 0x18000000000437, 0x4054, 0x40540, 0x405400, 0x4054000, 0x40540000, 0x405400000, 0x4054000000, 0x40540000000, 0x405400000000, 0x4054000000000, 0x54000000008e, 0x54000000008e0, 0x14000000008e8e, 0x8ea56, 0x8ea560, 0x8ea5600, 0x8ea56000, 0x8ea560000, 0x8ea5600000, 0x8ea56000000, 0x8ea560000000, 0x8ea5600000000, 0xea5600000011c, 0xa560000001015, 0x560000001000b, 0x1600000010003e}); +constexpr StatTable53 SQR4_TABLE_53({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x23800, 0x238000000, 0x2380000000000, 0x40540, 0x405400000, 0x4054000000000, 0x8ea56, 0x8ea560000, 0x8ea5600000000, 0x1600000010003e, 0x1000111000, 0x10001110000000, 0x11100000238000, 0x2380219b80, 0x380219b800047, 0x19b8000405447, 0x4054441114, 0x54441114008e, 0x41114008ea5ee, 0x14008ea5e6c1b8, 0xea5e6c193611c, 0x6c1936100000d, 0x13610000000124, 0x1010338, 0x10103380000, 0x1033800000238, 0x180000023a3e0f, 0x23a3e3d4000, 0x1a3e3d40000437, 0x1d400004014997, 0x40149af1600, 0x149af160008e0, 0xf160008e2a4a3, 0x8e2a4bc4710, 0x2a4bc47101015, 0x1c4710101022bb, 0x101010228120a8, 0x102281208ba380, 0x1208ba3a3c26c, 0xba3a3c26e7e1c, 0x3c26e7e0ad413, 0xe7e0ad414deb9, 0xad414dea4a7d0, 0x14dea4a7c40960, 0x4a7c4094acd8b, 0x4094acd82b43a, 0xacd82b432f376, 0x2b432f3623804, 0x12f36238010027}); +constexpr StatTable53 SQR8_TABLE_53({0x1, 0x11100000238000, 0x1a3e3d40000437, 0x4a7c4094acd8b, 0x1eea7d6a679bbe, 0x1c423906384897, 0x2168da5f2c08c, 0x1b8259ec8ea11, 0x19d1f388b2d3d6, 0x1959a5720001a7, 0x1a6c8fa147c79, 0x191868056d58df, 0x19b717beaf7eb0, 0x1d37e92df66e7f, 0x16ec165c8535b7, 0x7da5e73dba0b3, 0x14d2bece5702b1, 0xadfaa30a5cf7e, 0x101934bec0d066, 0xaae7f006690ce, 0x6bfdbd85eb297, 0x6b2cb00d8c2b5, 0x52ee73aae547e, 0x67461baa976c2, 0xf8a44f459c7d, 0x2579abd0b5fc6, 0x6e7e5e9b82057, 0x1ab0a0fcf2d91c, 0x385dc87020053, 0xfc75a891c9df0, 0xca67b851d0c1a, 0x4c8d3234fd4f7, 0xf3ea564798c7f, 0x16881f479c0d6b, 0x60b0e8e33fe90, 0x18259a2869066d, 0xc52b463fb1475, 0x8229075c3475d, 0x6725108ff0948, 0xd5edf67d5a509, 0xbf52bb2383664, 0xd5b84ac7ed2ab, 0xbb5901d009d56, 0xcb380bfcebc5, 0x5f411d4594745, 0x18bdcb9f9d25da, 0xf0d3abe76ec15, 0x8b1a6404ca3b5, 0x15b7c7c793b65f, 0x11ff16f08569ff, 0x19c1d4c5eb3442, 0x5deb92ff5fc40, 0xa8009f9410cbc}); +constexpr StatTable53 SQR16_TABLE_53({0x1, 0x5a65e677a526f, 0x142b8f50195f72, 0x12b0ca8e8b1225, 0x1b892547f268cc, 0x1239ca3a4824b6, 0x4249dac026ea8, 0x38080cba150e5, 0x903882481cefb, 0x1ad11e5cf99bf0, 0x14fa149116ab75, 0x6cbd888de21e5, 0x1388c718c37a69, 0x89d1eb38e9978, 0xf12019f00f91f, 0xb377986c7da1f, 0x1c780b06da5cb9, 0x1e10c7eee3249d, 0xe1afb7bd8111d, 0xc821f2a6fa090, 0x1a26caa65e1d59, 0x4280741c8cc4c, 0xb9c507337dad8, 0x65bffa0a097b6, 0x12068bb8ed4ac0, 0x6d751e7056355, 0xbccc3fbdcf084, 0x17ed82d58ea927, 0x125a30b543b4b8, 0xaf1ce3f5f84ce, 0x1082e42090b649, 0xf8d6a6212c41a, 0x1f89211d4982d, 0x1910bdfe092d07, 0x9363da9b9b9d3, 0x8a7196ef7b84e, 0x33fe46ddf1dc, 0x1f3f3291cf719d, 0x91a5da69f1035, 0x5a8dc6eb62cfb, 0xaf99fcc57728a, 0x15e73f1aa49f47, 0x2d82e50796b97, 0x1072fcbb074200, 0x15180f0fc7904, 0xa3a194b750f79, 0xb053c3eea9bb3, 0x1e58da5ae123de, 0x10b47afec00861, 0x17cd9ea910639d, 0x1ecf806dbf8c36, 0xf93d00fe6145b, 0x1247d788a3eda}); +constexpr StatTable53 QRT_TABLE_53({0xf940b90844076, 0x1f940b90844052, 0x1f940b90844050, 0x9d2a063b43e64, 0x1f940b90844054, 0x936f69323ec14, 0x9d2a063b43e6c, 0xe12270a88898, 0x1f940b90844044, 0x1f917f00bb5a3c, 0x936f69323ec34, 0x1f622df85b46ee, 0x9d2a063b43e2c, 0x9bc65ab040b66, 0xe12270a88818, 0x958330b931986, 0x1f940b90844144, 0x98e2a06e32e0, 0x1f917f00bb583c, 0x1f877970dc1024, 0x936f69323e834, 0x16cc3c9b1558c2, 0x1f622df85b4eee, 0x16de1c3351dae8, 0x9d2a063b42e2c, 0x1fecdc7855f8ee, 0x9bc65ab042b66, 0x933821b1cb6fe, 0xe12270a8c818, 0x1f675958641c0e, 0x958330b939986, 0x9d97e050e960, 0x1f940b90854144, 0x1f820fa0e38adc, 0x98e2a06c32e0, 0x1650f0e358a010, 0x1f917f00bf583c, 0x1643af4b037a3a, 0x1f877970d41024, 0x1ffe2c281d8c16, 0x936f69333e834, 0xf00d50ffccf8, 0x16cc3c9b3558c2, 0x16bc31cbca943a, 0x1f622df81b4eee, 0xa6cbd8007232, 0x16de1c33d1dae8, 0x15d2a062b42e10, 0x9d2a062b42e2c, 0x1aa77896586ca, 0x1fecdc7a55f8ee, 0, 0x9bc65af042b66}); +typedef Field<uint64_t, 53, 71, StatTable53, &SQR_TABLE_53, &SQR2_TABLE_53, &SQR4_TABLE_53, &SQR8_TABLE_53, &SQR16_TABLE_53, &QRT_TABLE_53, IdTrans, &ID_TRANS, &ID_TRANS> Field53; +#endif + +#ifdef ENABLE_FIELD_INT_54 +// 54 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 6> StatTable54; +constexpr StatTable54 SQR_TABLE_54({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x201, 0x804, 0x2010, 0x8040, 0x20100, 0x80400, 0x201000, 0x804000, 0x2010000, 0x8040000, 0x20100000, 0x80400000, 0x201000000, 0x804000000, 0x2010000000, 0x8040000000, 0x20100000000, 0x80400000000, 0x201000000000, 0x804000000000, 0x2010000000000, 0x8040000000000, 0x20100000000000, 0x400000000402, 0x1000000001008, 0x4000000004020, 0x10000000010080}); +constexpr StatTable54 SQR2_TABLE_54({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x804, 0x8040, 0x80400, 0x804000, 0x8040000, 0x80400000, 0x804000000, 0x8040000000, 0x80400000000, 0x804000000000, 0x8040000000000, 0x400000000402, 0x4000000004020, 0x40001, 0x400010, 0x4000100, 0x40001000, 0x400010000, 0x4000100000, 0x40001000000, 0x400010000000, 0x4000100000000, 0x1000000201, 0x10000002010, 0x100000020100, 0x1000000201000, 0x10000002010000, 0x20100804, 0x201008040, 0x2010080400, 0x20100804000, 0x201008040000, 0x2010080400000, 0x20100804000000, 0x1008040001008, 0x10080400010080, 0x804000100004, 0x8040001000040, 0x400010000002, 0x4000100000020}); +constexpr StatTable54 SQR4_TABLE_54({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x80400, 0x804000000, 0x8040000000000, 0x400010, 0x4000100000, 0x1000000201, 0x10000002010000, 0x20100804000, 0x1008040001008, 0x400010000002, 0x100000000100, 0x1008040, 0x10080400000, 0x804000000804, 0x8000001, 0x80000010000, 0x100004020, 0x1000040200000, 0x402000080400, 0x20000804020100, 0x8040200008000, 0x2000080400010, 0x804000000800, 0x8040001, 0x80400010000, 0x4000100004020, 0x1000040001000, 0x400010080400, 0x100804020100, 0x8040201008040, 0x2010080000010, 0x800000000004, 0x200, 0x2000000, 0x20000000000, 0x1008, 0x10080000, 0x100800000000, 0x8000000008040, 0x80002000, 0x800020000000, 0x200000040200, 0x402010080, 0x4020100800000, 0x1008000200008, 0x2000000002, 0x20000000020000, 0x201008000, 0x2010080000000, 0x800000100004}); +constexpr StatTable54 SQR8_TABLE_54({0x1, 0x10080400000, 0x100804020100, 0x1008000200008, 0x10080002000000, 0x4000000804, 0x8000201000040, 0x402000000000, 0x20100004020, 0x1000000000, 0x80000010, 0x20100800000100, 0x201008, 0x80002010080, 0x804020100000, 0x201000040, 0x2000080000, 0x4020100004000, 0x8040000, 0x10000402000, 0x20000, 0x1008000001008, 0x10080402010080, 0x4000000004, 0x40001000040, 0x10080402, 0x4020000004000, 0x40001, 0x2000080402010, 0x20000000000000, 0x1000000200000, 0x10000000000080, 0x4020100000, 0x40201008040, 0x402000080002, 0x4020000800000, 0x1000000201, 0x2000080400010, 0x100800000000, 0x8040001008, 0x400000000, 0x20000004, 0x8040200000040, 0x80402, 0x20000804020, 0x201008040000, 0x80400010, 0x800020000, 0x1008040001000, 0x2010000, 0x4000100800, 0x8000, 0x402000000402, 0x4020100804020}); +constexpr StatTable54 SQR16_TABLE_54({0x1, 0x80002010000, 0x4020000000000, 0x1000000201008, 0x402010000402, 0x20100804020000, 0x8000001000040, 0x2000000000000, 0x804020000800, 0x1000000201, 0x80002000080, 0x804020, 0x8, 0x400010080000, 0x20100000000000, 0x8000001008040, 0x2010080002010, 0x804020100804, 0x8000001, 0x10000000000000, 0x4020100004000, 0x8000001008, 0x400010000400, 0x4020100, 0x40, 0x2000080400000, 0x800000000804, 0x8040001, 0x10080400010080, 0x4020100804020, 0x40000008, 0x402, 0x20100800020000, 0x40000008040, 0x2000080002000, 0x20100800, 0x200, 0x10000402000000, 0x4000000004020, 0x40200008, 0x402000080002, 0x20100804020100, 0x200000040, 0x2010, 0x804000100804, 0x200000040200, 0x10000400010000, 0x100804000, 0x1000, 0x2010000402, 0x20000000020100, 0x201000040, 0x2010000400010, 0x804020100004}); +constexpr StatTable54 QRT_TABLE_54({0x201008000200, 0x26c10916494994, 0x26c10916494996, 0x40008008, 0x26c10916494992, 0x141a2434c12d12, 0x40008000, 0x36c00110594c22, 0x26c10916494982, 0x200000040200, 0x141a2434c12d32, 0x10010816104534, 0x40008040, 0x36da60b01308b2, 0x36c00110594ca2, 0x48200209000, 0x26c10916494882, 0x41b6da2d86106, 0x200000040000, 0x32db2c228965b0, 0x141a2434c12932, 0x9000000200048, 0x10010816104d34, 0x32db68b2832da4, 0x40009040, 0x40045928b4902, 0x36da60b01328b2, 0x1000040000, 0x36c00110590ca2, 0x101b69865a4120, 0x48200201000, 0x22da6434912884, 0x26c10916484882, 0x9000240208008, 0x41b6da2da6106, 0x22c14484c20180, 0x200000000000, 0x4016db29b6812, 0x32db2c228165b0, 0x9008200201048, 0x141a2434d12932, 0x32c36ca2c264b0, 0x9000000000048, 0x140a65b48a2c32, 0x10010816504d34, 0, 0x32db68b2032da4, 0x404490824814, 0x41009040, 0x14da60a4536126, 0x40045908b4902, 0x8000041009008, 0x36da60b41328b2, 0x6db68b2032c12}); +typedef Field<uint64_t, 54, 513, StatTable54, &SQR_TABLE_54, &SQR2_TABLE_54, &SQR4_TABLE_54, &SQR8_TABLE_54, &SQR16_TABLE_54, &QRT_TABLE_54, IdTrans, &ID_TRANS, &ID_TRANS> Field54; +typedef FieldTri<uint64_t, 54, 9, RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 6>, &SQR_TABLE_54, &SQR2_TABLE_54, &SQR4_TABLE_54, &SQR8_TABLE_54, &SQR16_TABLE_54, &QRT_TABLE_54, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri54; +#endif + +#ifdef ENABLE_FIELD_INT_55 +// 55 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5> StatTable55; +constexpr StatTable55 SQR_TABLE_55({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x102, 0x408, 0x1020, 0x4080, 0x10200, 0x40800, 0x102000, 0x408000, 0x1020000, 0x4080000, 0x10200000, 0x40800000, 0x102000000, 0x408000000, 0x1020000000, 0x4080000000, 0x10200000000, 0x40800000000, 0x102000000000, 0x408000000000, 0x1020000000000, 0x4080000000000, 0x10200000000000, 0x40800000000000, 0x2000000000102, 0x8000000000408, 0x20000000001020}); +constexpr StatTable55 SQR2_TABLE_55({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x102, 0x1020, 0x10200, 0x102000, 0x1020000, 0x10200000, 0x102000000, 0x1020000000, 0x10200000000, 0x102000000000, 0x1020000000000, 0x10200000000000, 0x2000000000102, 0x20000000001020, 0x10004, 0x100040, 0x1000400, 0x10004000, 0x100040000, 0x1000400000, 0x10004000000, 0x100040000000, 0x1000400000000, 0x10004000000000, 0x40000000102, 0x400000001020, 0x4000000010200, 0x40000000102000, 0x1020408, 0x10204080, 0x102040800, 0x1020408000, 0x10204080000, 0x102040800000, 0x1020408000000, 0x10204080000000, 0x2040800000102, 0x20408000001020, 0x4080000010004, 0x40800000100040, 0x8000001000008}); +constexpr StatTable55 SQR4_TABLE_55({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x10200, 0x102000000, 0x1020000000000, 0x10004, 0x100040000, 0x1000400000000, 0x4000000010200, 0x102040800, 0x1020408000000, 0x4080000010004, 0x100000010, 0x1000000100000, 0x1000010200, 0x10000102000000, 0x1020000102000, 0x1020010004, 0x10200100040000, 0x1000400100040, 0x4001000410200, 0x10004102040800, 0x41020408102000, 0x4081020418004, 0x10204180000010, 0x41800000000040, 0x10300, 0x103000000, 0x1030000000000, 0x10106, 0x101060000, 0x1010600000000, 0x6000000010302, 0x103040c00, 0x103040c000000, 0x40c0000010106, 0x101020418, 0x1010204180000, 0x2041800010302, 0x18000103000008, 0x1030000103000, 0x1030010106, 0x10300101060000, 0x1010600101060, 0x6001010610302, 0x10106103040c00, 0x6103040c103020, 0x40c103041c106, 0x103041c1020418, 0x41c10204081060, 0x2040810214282, 0x8102142800008, 0x21428000000020}); +constexpr StatTable55 SQR8_TABLE_55({0x1, 0x1000010200, 0x101060000, 0x6103040c103020, 0x1001400010200, 0x4081121478004, 0x7903050f103028, 0x1100010200, 0x1020101162000, 0x6703040c113322, 0x113055c1030618, 0x50a102353a804, 0x3e83050f11336a, 0x103040f102071e, 0x101070200, 0x7123050c143020, 0x3100c010200, 0x60c193166c286, 0x6d2b040f15302c, 0x103140f010200, 0x2070f112772e2, 0x7621050c153322, 0x143341cd420418, 0x70e193270ee9e, 0xbe9840f14334e, 0x143355fe53051e, 0x3050e103555fc, 0x5153e50f143c00, 0x1001500050200, 0x450a152957a004, 0x7b071d0f11332a, 0x1000040200, 0x5000040b060000, 0x64061a0c103020, 0x153c44f147c71e, 0x4008142b428a04, 0x2ca75c8f103078, 0x113341f117371e, 0x5402040b162000, 0x62064aac143336, 0x50c003f51ff06, 0x7cf1f3d7ef2e6, 0x6e2d180714332e, 0x103150f050100, 0x43350b1a3152e2, 0x74261e06143020, 0x113f54fd17c65e, 0x56301c3b66cd98, 0x10dff9c7953054, 0x153055fe47360e, 0x46340a1b2277fc, 0x4574bfb5753c14, 0x50f003345fc52, 0x41f653264c9962, 0x7612375be93322}); +constexpr StatTable55 SQR16_TABLE_55({0x1, 0x4ba7488f00015a, 0x30ce9d3a61c1e4, 0x4a2e76980aff84, 0x4e44f5b9d2f610, 0x7b479e4450115c, 0x248c18e86b39b2, 0x1ba74c8406015a, 0x35e93420af76aa, 0x7f282c7c68ad54, 0x7f8b356ad7bc5a, 0x527272878d3b24, 0x587495a40395a4, 0x43c4d0fd51f96c, 0x57ce893a71f0c6, 0x62c68a94803da, 0x1b32bc920e6546, 0x5073c39b469c78, 0x2fba08c009b110, 0x10bd0559ba45c, 0x3bbbd0ca4b3246, 0x243ad4b7c193b8, 0x335d7f186b5db2, 0x5590f3a0fd73f0, 0x72953f208233ba, 0x5210b9a31e6c62, 0x744bb124e351da, 0x4929f00a730244, 0x736ff5bdc1c63c, 0x4c1da1fb246e2e, 0x553c18b46d95cc, 0x268f5c8c143376, 0x438f5a59cbf094, 0x6a718b25fd3946, 0x67053b1bf54fe0, 0x441c5323cb0288, 0x5def8fcd41d5a8, 0x40446cdfcdb062, 0x1043009fb20072, 0xef08d6ed9e9c6, 0xbdf8adea645be, 0x76b092b499c072, 0xd754f98b724c2, 0x5a21d55c8f8752, 0x4f0f36a62eeb0c, 0x262f651fb93b18, 0x3336d340aa0aaa, 0x69375d0e9970fa, 0x2e0997225afe66, 0x6692008b83364e, 0x230856519bc3ae, 0x2c0a54962f8378, 0x2a6460de8a4266, 0x2f14d8fa237452, 0x25934cd7ae0030}); +constexpr StatTable55 QRT_TABLE_55({0, 0x121d57b6623fde, 0x121d57b6623fdc, 0x68908340d10e00, 0x121d57b6623fd8, 0x100300510e20, 0x68908340d10e08, 0x10004096, 0x121d57b6623fc8, 0x100010000, 0x100300510e00, 0x7ea8c890a088e8, 0x68908340d10e48, 0x68809540871648, 0x10004016, 0x68808000808068, 0x121d57b6623ec8, 0x68909240d41c48, 0x100010200, 0x6884c170ad0216, 0x100300510a00, 0x68848160a50200, 0x7ea8c890a080e8, 0x7eecbca04ab4b6, 0x68908340d11e48, 0x120c54b62234c8, 0x68809540873648, 0x69929240d61c48, 0x10000016, 0x68808060800000, 0x68808000800068, 0x80000080, 0x121d57b6633ec8, 0x7ea8cb90a18ae8, 0x68909240d61c48, 0x16284090200080, 0x100050200, 0x474302a345e, 0x6884c170a50216, 0x166cbca0cab4de, 0x100300410a00, 0x1000000000000, 0x68848160850200, 0x688cc1f0a50296, 0x7ea8c890e080e8, 0x7e8cc1f0a50280, 0x7eecbca0cab4b6, 0x68000000000068, 0x68908341d11e48, 0x7880954487365e, 0x120c54b42234c8, 0x9929248d61c20, 0x68809544873648, 0x41121208561c20, 0x69929248d61c48}); +typedef Field<uint64_t, 55, 129, StatTable55, &SQR_TABLE_55, &SQR2_TABLE_55, &SQR4_TABLE_55, &SQR8_TABLE_55, &SQR16_TABLE_55, &QRT_TABLE_55, IdTrans, &ID_TRANS, &ID_TRANS> Field55; +typedef FieldTri<uint64_t, 55, 7, RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5>, &SQR_TABLE_55, &SQR2_TABLE_55, &SQR4_TABLE_55, &SQR8_TABLE_55, &SQR16_TABLE_55, &QRT_TABLE_55, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri55; +#endif + +#ifdef ENABLE_FIELD_INT_56 +// 56 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5> StatTable56; +constexpr StatTable56 SQR_TABLE_56({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x95, 0x254, 0x950, 0x2540, 0x9500, 0x25400, 0x95000, 0x254000, 0x950000, 0x2540000, 0x9500000, 0x25400000, 0x95000000, 0x254000000, 0x950000000, 0x2540000000, 0x9500000000, 0x25400000000, 0x95000000000, 0x254000000000, 0x950000000000, 0x2540000000000, 0x9500000000000, 0x25400000000000, 0x95000000000000, 0x5400000000012a, 0x5000000000043d, 0x40000000001061}); +constexpr StatTable56 SQR2_TABLE_56({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x95, 0x950, 0x9500, 0x95000, 0x950000, 0x9500000, 0x95000000, 0x950000000, 0x9500000000, 0x95000000000, 0x950000000000, 0x9500000000000, 0x95000000000000, 0x5000000000043d, 0x4111, 0x41110, 0x411100, 0x4111000, 0x41110000, 0x411100000, 0x4111000000, 0x41110000000, 0x411100000000, 0x4111000000000, 0x41110000000000, 0x11100000000254, 0x110000000025d5, 0x10000000025dc5, 0x25dcc5, 0x25dcc50, 0x25dcc500, 0x25dcc5000, 0x25dcc50000, 0x25dcc500000, 0x25dcc5000000, 0x25dcc50000000, 0x25dcc500000000, 0x5dcc500000012a, 0xdcc50000001061, 0xcc500000010079, 0xc500000010016c, 0x5000000100103c}); +constexpr StatTable56 SQR4_TABLE_56({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x9500, 0x95000000, 0x950000000000, 0x4111, 0x41110000, 0x411100000000, 0x110000000025d5, 0x25dcc500, 0x25dcc5000000, 0xdcc50000001061, 0x10010101, 0x100101010000, 0x1010100000950, 0x1000009509595, 0x95095959500, 0x5095959500043d, 0x95950004115111, 0x41151505011, 0x11515050110254, 0x505011025de985, 0x11025de9a93c10, 0x5de9a93c19c42a, 0xa93c19c400005d, 0x19c4000001000c, 0x100010094, 0x1000100940000, 0x1009400009500, 0x94000095009500, 0x950095418400, 0x954184004111, 0x41840041114111, 0x411141349dd4, 0x1141349dd425d5, 0x349dd425dce0d5, 0xd425dce0cce1b9, 0xdce0cce1ddd461, 0xcce1ddd4011160, 0xddd401110941f5, 0x1110941959dc4, 0x941959dc49cc5, 0x959dc49cc118d5, 0xc49cc1189454b9, 0xc1189454d4d12c, 0x9454d4d14358f8, 0xd4d14358b9aa44, 0x4358b9aa20a205, 0xb9aa20a221d7b8, 0x20a221d7ed10a2, 0x21d7ed10b0f90a, 0xed10b0f918507b, 0xb0f91850000050, 0x1850000001000d}); +constexpr StatTable56 SQR8_TABLE_56({0x1, 0x1010100000950, 0x950095418400, 0xd4d14358b9aa44, 0x1135dd851025d5, 0x2c3e45b8a8a9d9, 0xcc39c4d816cc89, 0x51109400, 0x8496c8edb8f151, 0x1c2d7d88406199, 0x3856af0918b2ea, 0x2c26c02be43364, 0x7c13f0a9492898, 0x887abc757e3b3c, 0x10100411009c5, 0x850b98e029a995, 0x18309e7d346f24, 0x49e692600134d8, 0xf902789abce101, 0xed998d1d57187b, 0xa5488e663e846e, 0x84267921a952d0, 0x3d464f2d15176e, 0x801aac9d710b04, 0xfc00d6eb916343, 0x5c7fb78f391c1, 0x745ee236e80ea0, 0x81f8c65be65eac, 0x1940095415941, 0x1188025e2103d0, 0x49c166e0b13f34, 0xbd26558f28a2b3, 0xe147d131ae4b81, 0x25b501ad8ba86d, 0x75fb4e24c70a79, 0x88172901f1684d, 0xb090520bb570a2, 0x963c9b97aad59, 0x39b1e5f12c85a6, 0xd90de4c2bf3055, 0x9c921257e4a1b, 0x45f1f318fef834, 0x48161e1eb09635, 0x10685b397538ce, 0xbc8d4a0c6bc62a, 0xdce738247bfad9, 0x1115b3337d25a4, 0x195bf5a6f0999b, 0xb85101388b2f37, 0x8ee1b2833544cf, 0x49bc1efef7da90, 0x346e404662e355, 0x8c0dab6a1217d6, 0x3cb782ec54c968, 0x5efe07d4f59f4, 0x55f19c0f482900}); +constexpr StatTable56 SQR16_TABLE_56({0x1, 0x2563e0b70105c4, 0x48ce07ef5576bb, 0xb94064d844f117, 0x207d2f511ffe3c, 0xf8f6dd1e2a3e6b, 0xe4cc405e0c6cdb, 0xd053f9b827b2bf, 0x550ae8d22edcbf, 0x29f7570f88728b, 0xa06a9e2dfd84a6, 0x55567b9483b3ff, 0x197c6c0d004df6, 0xe106c03f218a16, 0xc50dd2aaf0a388, 0x39473f6702a06c, 0xc8c1736b312ded, 0x992dc692bb707d, 0x24bb9a8dcad06f, 0x9cc45f9e3c2075, 0x455e7271eb130b, 0x847157a5326f59, 0xdc8ccb4ab3f5bd, 0x9463c02c46910d, 0xe1debd0b794514, 0x4c5128db660cde, 0x11910a685416a3, 0x11dfa5b9552a3d, 0x5d902ced822708, 0x794ff735e94601, 0xf1dc5fd7efcf7e, 0x19fb7ff8d06993, 0x7069119ac28a09, 0x98ba5a77d83e7f, 0xf4923dbc1b24e5, 0x7c2dcc84668312, 0xc27e2f5f2243f, 0x78c6d8ebe4bede, 0xad39495debf1a5, 0xa1564b894b50f0, 0x5898ae4e965be9, 0x28aa991e046567, 0x585e95889bb734, 0xc59e73661cf916, 0xed70696926d95d, 0xcca6630954309a, 0x8c4b12ac111264, 0xe8413ad0493e05, 0x1acea73bc9166, 0x9a7f11cd38d12d, 0x390dd1972139ec, 0x9146bc1a4fbff0, 0xd5a1c594335b01, 0x2566272e74ef1a, 0xd4a8baf259e7d0, 0x71e7efd8f20703}); +constexpr StatTable56 QRT_TABLE_56({0x10004084, 0xd058f12fd5925e, 0xd058f12fd5925c, 0x41a60b5566d9f0, 0xd058f12fd59258, 0xbda60a142740ba, 0x41a60b5566d9f8, 0xd059f1afc5e688, 0xd058f12fd59248, 0xfc040841615a22, 0xbda60a1427409a, 0xbda60b5426c1ca, 0x41a60b5566d9b8, 0x1a60b4166b950, 0xd059f1afc5e608, 0xfc000041409822, 0xd058f12fd59348, 0xd1ee7a4ef4185c, 0xfc040841615822, 0x9049759b80b4a4, 0xbda60a1427449a, 0xd258e06f301e18, 0xbda60b5426c9ca, 0x6dfeeb3bf6d7d2, 0x41a60b5566c9b8, 0xbdef3ed4ae398a, 0x1a60b41669950, 0xd1ef3f8eeff04c, 0xd059f1afc5a608, 0xbda203340783de, 0xfc000041401822, 0x2dfefbaff2b27a, 0xd058f12fd49348, 0xfdb788a0706776, 0xd1ee7a4ef6185c, 0x2e5de0ae41337a, 0xfc040841655822, 0x41eb17d5ceecf8, 0x9049759b88b4a4, 0x40048874211afc, 0xbda60a1437449a, 0xd04a720f93400c, 0xd258e06f101e18, 0xbc559cf5ac7fce, 0xbda60b5466c9ca, 0x6dc9759b88b4d6, 0x6dfeeb3b76d7d2, 0x92feea7b275af0, 0x41a60b5466c9b8, 0, 0xbdef3ed6ae398a, 0x2811d5edd8ee2a, 0x1a60b45669950, 0xb1a60b5466c9ca, 0xd1ef3f86eff04c, 0xec493582c8f032}); +typedef Field<uint64_t, 56, 149, StatTable56, &SQR_TABLE_56, &SQR2_TABLE_56, &SQR4_TABLE_56, &SQR8_TABLE_56, &SQR16_TABLE_56, &QRT_TABLE_56, IdTrans, &ID_TRANS, &ID_TRANS> Field56; +#endif +} + +Sketch* ConstructClMul7Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_49 + case 49: return new SketchImpl<Field49>(implementation, 49); +#endif +#ifdef ENABLE_FIELD_INT_50 + case 50: return new SketchImpl<Field50>(implementation, 50); +#endif +#ifdef ENABLE_FIELD_INT_51 + case 51: return new SketchImpl<Field51>(implementation, 51); +#endif +#ifdef ENABLE_FIELD_INT_52 + case 52: return new SketchImpl<Field52>(implementation, 52); +#endif +#ifdef ENABLE_FIELD_INT_53 + case 53: return new SketchImpl<Field53>(implementation, 53); +#endif +#ifdef ENABLE_FIELD_INT_54 + case 54: return new SketchImpl<Field54>(implementation, 54); +#endif +#ifdef ENABLE_FIELD_INT_55 + case 55: return new SketchImpl<Field55>(implementation, 55); +#endif +#ifdef ENABLE_FIELD_INT_56 + case 56: return new SketchImpl<Field56>(implementation, 56); +#endif + } + return nullptr; +} + +Sketch* ConstructClMulTri7Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_49 + case 49: return new SketchImpl<FieldTri49>(implementation, 49); +#endif +#ifdef ENABLE_FIELD_INT_52 + case 52: return new SketchImpl<FieldTri52>(implementation, 52); +#endif +#ifdef ENABLE_FIELD_INT_54 + case 54: return new SketchImpl<FieldTri54>(implementation, 54); +#endif +#ifdef ENABLE_FIELD_INT_55 + case 55: return new SketchImpl<FieldTri55>(implementation, 55); +#endif + } + return nullptr; +} diff --git a/src/minisketch/src/fields/clmul_8bytes.cpp b/src/minisketch/src/fields/clmul_8bytes.cpp new file mode 100644 index 0000000000..8dc1089fee --- /dev/null +++ b/src/minisketch/src/fields/clmul_8bytes.cpp @@ -0,0 +1,175 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_8) + +#include "clmul_common_impl.h" + +#include "../int_utils.h" +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_57 +// 57 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5> StatTable57; +constexpr StatTable57 SQR_TABLE_57({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x22, 0x88, 0x220, 0x880, 0x2200, 0x8800, 0x22000, 0x88000, 0x220000, 0x880000, 0x2200000, 0x8800000, 0x22000000, 0x88000000, 0x220000000, 0x880000000, 0x2200000000, 0x8800000000, 0x22000000000, 0x88000000000, 0x220000000000, 0x880000000000, 0x2200000000000, 0x8800000000000, 0x22000000000000, 0x88000000000000, 0x20000000000011, 0x80000000000044}); +constexpr StatTable57 SQR2_TABLE_57({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x100000000000000, 0x88, 0x880, 0x8800, 0x88000, 0x880000, 0x8800000, 0x88000000, 0x880000000, 0x8800000000, 0x88000000000, 0x880000000000, 0x8800000000000, 0x88000000000000, 0x80000000000044, 0x404, 0x4040, 0x40400, 0x404000, 0x4040000, 0x40400000, 0x404000000, 0x4040000000, 0x40400000000, 0x404000000000, 0x4040000000000, 0x40400000000000, 0x4000000000022, 0x40000000000220, 0x2222, 0x22220, 0x222200, 0x2222000, 0x22220000, 0x222200000, 0x2222000000, 0x22220000000, 0x222200000000, 0x2222000000000, 0x22220000000000, 0x22200000000011, 0x22000000000101, 0x20000000001001}); +constexpr StatTable57 SQR4_TABLE_57({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x880, 0x8800000, 0x88000000000, 0x80000000000044, 0x404000, 0x4040000000, 0x40400000000000, 0x22220, 0x222200000, 0x2222000000000, 0x20000000001001, 0x10001000, 0x100010000000, 0x100000000088, 0x880088, 0x8800880000, 0x88008800000000, 0x88000000040400, 0x404040400, 0x4040404000000, 0x4040000002222, 0x22222222, 0x222222220000, 0x22222200000101, 0x22000001000001, 0x10000000100, 0x100000001000000, 0x10000088000, 0x100000880000000, 0x8800000088000, 0x880004040, 0x8800040400000, 0x404000004040, 0x40000040400220, 0x404002222000, 0x40022220000220, 0x22200002222011, 0x22220100010, 0x22201000100011, 0x10001000100010, 0x10001000108800, 0x10001088008800, 0x10880088008800, 0x880088008c04, 0x88008c040404, 0x8c0404040404, 0x4040404040426, 0x4040404262222, 0x4042622222222, 0x26222222222222, 0x22222222232201, 0x22222322000001, 0x23220000000001}); +constexpr StatTable57 SQR8_TABLE_57({0x1, 0x100010000000, 0x100000880000000, 0x88008c040404, 0x80000000022264, 0x26262604000101, 0x50023220100230, 0x222aa222000001, 0x20000404041401, 0x100404003222000, 0x32aa22aa23aa01, 0x2326048800088, 0x808100222ea722, 0x508e36ec548e34, 0x26222000022223, 0x22000223200001, 0x32001001108801, 0x91001000108844, 0x85048c04880044, 0x4c86ae64c80220, 0x6ea40546662003, 0x66662726ae22ab, 0x40622aa2aa40c8, 0x26063ea6364477, 0x2406744c950437, 0x8a33606aa727aa, 0xd09332ca7e2d9a, 0x7e2c14ce3e6c17, 0x22aa2626260405, 0x22200002626011, 0x27260000002223, 0x22208812aa3011, 0x2ba30040488001, 0x48c8c9caebe26, 0x28898040489001, 0x1048c008922aa26, 0x30c0c08a22b801, 0x12222666fa72601, 0x9afa60a8048eaa, 0x6aa1400afba131, 0xbc9c168c00800d, 0xc083aa60588bb0, 0xbeeefeae26e6e7, 0x8c44ee22226ae2, 0x8c44ee32205042, 0x2667443228d143, 0x36af14ba381d17, 0x72eb33981a3f35, 0x72c911aab20d9d, 0x72ca2b2aaacccd, 0x66463fae3f44ff, 0xac345eee3b4077, 0xe4977caefbe1fd, 0xf01b70a0dd0f9a, 0xf40f74bc580bbd, 0x17e2d1c6c5c0f35, 0x3624140232aa33}); +constexpr StatTable57 SQR16_TABLE_57({0x1, 0xaeeec814065447, 0x110889c99ba3004, 0x1c59ed582798e0e, 0x1c9c766272a2a74, 0x1422928a5250a00, 0x2c97eb48f402a1, 0x10c6d916b128dea, 0x3f2e6ca66ebf67, 0x93ac75fcd63ec8, 0x19263128e42246, 0x1fd1ca54b556091, 0x60ff38200c4e09, 0x1381808ede9982b, 0xc7a9ace2f9dc2a, 0x6c2ee414271c57, 0x3c16f4cffdbe17, 0xc627ec6fe179ee, 0x178f994adf6525b, 0x18be0b635ca1650, 0x4afcb2ae2e98b6, 0x6f81f53a7688dd, 0x45319b3e02c15c, 0x1044be090058910, 0xaa02d012fca063, 0x11fba4c5b80dbfa, 0xf9f44be142268b, 0x1e351a44eb480bf, 0xacf5c17bd0aedf, 0x6f2d74bab851eb, 0x1b8ac3589da9915, 0x1afeb885d3fdd67, 0x7d7d596dd60bbd, 0x1329567316f5723, 0xfdfe23b549fcef, 0xc985ed1e7a009e, 0x71f794bbac1b03, 0xc740582125f7d0, 0x1b3584e031e3b77, 0x29978a3c559ed3, 0xde04d46b4ae516, 0x2f6d6e1c749405, 0x1ec95b44a4251f3, 0xb95b0a5f451f2d, 0x1dc80aedaab9bf2, 0xd0354d3ff74808, 0x180889b484b0364, 0x196895708367d90, 0x104575064a09414, 0x19e88f14fc111ec, 0x1cf4088d3cffd88, 0x1e6c28b9a76c6d5, 0x81ba060c9e485e, 0x12b811107188d68, 0x5e6f10ca82cd88, 0x120882748af043d, 0x145fb82467c596e}); +constexpr StatTable57 QRT_TABLE_57({0xd0c3a82c902426, 0x232aa54103915e, 0x232aa54103915c, 0x1763e291e61699c, 0x232aa541039158, 0x1f424d678bb15e, 0x1763e291e616994, 0x26fd8122f10d36, 0x232aa541039148, 0x1e0a0206002000, 0x1f424d678bb17e, 0x5d72563f39d7e, 0x1763e291e6169d4, 0x1519beb9d597df4, 0x26fd8122f10db6, 0x150c3a87c90e4aa, 0x232aa541039048, 0x15514891f6179d4, 0x1e0a0206002200, 0x14ec9ba7a94c6aa, 0x1f424d678bb57e, 0x1e0f4286382420, 0x5d72563f3957e, 0x4000080000, 0x1763e291e6179d4, 0x1ac0e804882000, 0x1519beb9d595df4, 0x1f430d6793b57e, 0x26fd8122f14db6, 0x3c68e806882000, 0x150c3a87c9064aa, 0x1484fe18b915e, 0x232aa541029048, 0x14f91eb9b595df4, 0x15514891f6379d4, 0x48f6a82380420, 0x1e0a0206042200, 0x14b1beb99595df4, 0x14ec9ba7a9cc6aa, 0x4cf2a82b00420, 0x1f424d679bb57e, 0x26aa0002000000, 0x1e0f4286182420, 0x173f1039dd17df4, 0x5d72563b3957e, 0x4aa0002000000, 0x4000880000, 0x16d31eb9b595df4, 0x1763e291f6179d4, 0x20000000000000, 0x1ac0e806882000, 0x2caa0002000000, 0x1519beb99595df4, 0, 0x1f430d6f93b57e, 0x73e90d6d93b57e, 0x26fd8132f14db6}); +typedef Field<uint64_t, 57, 17, StatTable57, &SQR_TABLE_57, &SQR2_TABLE_57, &SQR4_TABLE_57, &SQR8_TABLE_57, &SQR16_TABLE_57, &QRT_TABLE_57, IdTrans, &ID_TRANS, &ID_TRANS> Field57; +typedef FieldTri<uint64_t, 57, 4, RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5>, &SQR_TABLE_57, &SQR2_TABLE_57, &SQR4_TABLE_57, &SQR8_TABLE_57, &SQR16_TABLE_57, &QRT_TABLE_57, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri57; +#endif + +#ifdef ENABLE_FIELD_INT_58 +// 58 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5> StatTable58; +constexpr StatTable58 LOAD_TABLE_58({0x1, 0x77dd188d5d600, 0x41a7213270def0, 0x10921661867b40a, 0x880b92a6f74da3, 0x143e72cde8f4484, 0x1863cb65c3eee0e, 0x2a1aa4a82154057, 0x22265135db9e135, 0x1446f023770d6d0, 0x183c4be7b4fae6, 0x1cc8a1187e99bd4, 0x3a1ea282e1e2ff1, 0x2700aae9dbcd275, 0x1571f84428a416e, 0xc8eb0b234b8a57, 0x23227afc0d9ba75, 0x1de9497779018c7, 0x5898e896d43329, 0x1501bd1b83bb55f, 0xbb56c28ce180b, 0x188e087d2dbf7e0, 0x36eee77125d8ec9, 0xde4235cbe95166, 0x1c71e4d57306163, 0x2a7e1b1ae5d87ee, 0x3a685560450c909, 0x1cd0545bc185c4b, 0x151779b11f09892, 0x2ab069803c4d787, 0x3bf279c0b825ad5, 0x15edc1ef3d2513c, 0x37bf223b4d0d045, 0x262a786b0324cd3, 0x27658294b696713, 0x33771167b0137e8, 0x86a73ef2dc3271, 0xc64453d2ff05, 0x14c55bc975ce8c, 0x3581164b1e4826e, 0x461dde5468bc26, 0x3f31528346e9451, 0x3f2669a5324a555, 0xd0b1c042854400, 0x32cc9899ea263e3, 0x2a423a8a96f4e95, 0x3e64acd727b470b, 0x2f1f1c1de9c997d, 0x268f2df0a8ab060, 0x14fd82231712442, 0x106e14e5e9e8f8c, 0x2686a15d911a24c, 0x182f831c5142b40, 0x36ca3e60fc7c678, 0x22ba581841d83ff, 0x11539696ce13d17, 0x558c3670aecb8e, 0x2b9b2d828d6d864}); +constexpr StatTable58 SAVE_TABLE_58({0x1, 0xde56167ca60c8e, 0x166391328282dfb, 0x2c6ed660535e701, 0xf83470e499e0e, 0x3686f752cff05fd, 0x1ada23d28022d0b, 0x2a2ac069b41ffd4, 0x2d40f316b40053e, 0x3fb69372877a1f3, 0x13def6e665e9b30, 0x23eb4222bc98b90, 0x2991c5ab618f62c, 0x1c4b63ee1e37a86, 0xdcb10179c77602, 0x708837c2f0ee59, 0x151fe1b533a6d99, 0x44613653cb9d83, 0x33dc64f2b5abae6, 0x27d704726f1f9ba, 0x2fdef2de96892ad, 0x3fd032a21834dbe, 0x1ce2548191e42ab, 0x431410a40ab44b, 0x206f1338c9a75e1, 0x130035675a32179, 0xdf781bb8adbd09, 0x1aaaf085ea624e0, 0x1df0605123c28e9, 0x28d3b3631320c9c, 0x81951a3af55e95, 0x388c776adc88ca1, 0x3ebed178f719885, 0x3c4546b19b0fe51, 0x129564a29700d09, 0x3f642277d82c520, 0x3a46d24ff0ac3fd, 0x1e75e367d627740, 0x33b01746a0f4aad, 0x2af930ca2fa61f, 0x3fcea0ca3af7aac, 0x230722de56e3f4a, 0x3541e58cc5afefd, 0x32cf711ae15ba7e, 0x11d3670d510fc6f, 0x6ddd78eec82112, 0x216210641885856, 0x87535f37c08809, 0x1fa464b5f82155b, 0xdbd43e91708494, 0x1500e23396dd2c4, 0x16cf4098632235f, 0x37e9117da8979ba, 0x6f8bfa04f66a7, 0x18dff008060e626, 0x196286fd9dbad1c, 0x35078156610f8ab, 0x7a669ff8398fea}); +constexpr StatTable58 SQR_TABLE_58({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x63, 0x18c, 0x630, 0x18c0, 0x6300, 0x18c00, 0x63000, 0x18c000, 0x630000, 0x18c0000, 0x6300000, 0x18c00000, 0x63000000, 0x18c000000, 0x630000000, 0x18c0000000, 0x6300000000, 0x18c00000000, 0x63000000000, 0x18c000000000, 0x630000000000, 0x18c0000000000, 0x6300000000000, 0x18c00000000000, 0x63000000000000, 0x18c000000000000, 0x230000000000063, 0xc000000000014a, 0x300000000000528}); +constexpr StatTable58 SQR2_TABLE_58({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x100000000000000, 0x18c, 0x18c0, 0x18c00, 0x18c000, 0x18c0000, 0x18c00000, 0x18c000000, 0x18c0000000, 0x18c00000000, 0x18c000000000, 0x18c0000000000, 0x18c00000000000, 0x18c000000000000, 0xc000000000014a, 0x1405, 0x14050, 0x140500, 0x1405000, 0x14050000, 0x140500000, 0x1405000000, 0x14050000000, 0x140500000000, 0x1405000000000, 0x14050000000000, 0x140500000000000, 0x50000000001ef, 0x50000000001ef0, 0x10000000001ef63, 0x1ef7bc, 0x1ef7bc0, 0x1ef7bc00, 0x1ef7bc000, 0x1ef7bc0000, 0x1ef7bc00000, 0x1ef7bc000000, 0x1ef7bc0000000, 0x1ef7bc00000000, 0x1ef7bc000000000, 0x2f7bc0000000129, 0x37bc0000000112d, 0x3bc000000011027, 0x3c0000000110022}); +constexpr StatTable58 SQR4_TABLE_58({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x18c0, 0x18c00000, 0x18c000000000, 0xc000000000014a, 0x1405000, 0x14050000000, 0x140500000000000, 0x1ef7bc, 0x1ef7bc0000, 0x1ef7bc00000000, 0x3bc000000011027, 0x110001100, 0x1100011000000, 0x11000000194c, 0x1000000194c018c, 0x194c0194c000, 0x14c0194c000014a, 0x194c00001545500, 0x15455154550, 0x154551545500000, 0x1154550001f18df, 0x150001f18c63193, 0x1f18c6318c7c00, 0xc6318c7c01010a, 0x18c7c0101000014, 0x1010000000101, 0x1000000010118c0, 0x10118d8c000, 0x10118d8c0000000, 0xd8c0000018d98a, 0x18d9811050, 0x18d98110500000, 0x18110500001411a, 0x500001410eb94c, 0x1410eb94bbc00, 0x10eb94bbc001ef0, 0x14bbc001ee85ab2, 0x1ee85aac1111, 0x2e85aac11110129, 0x2ac11110111097a, 0x111011109445c8c, 0x11109445c9554c0, 0x1445c9554d95406, 0x9554d954189419, 0xd954189414901f, 0x1894149014051f, 0x149014051e478f, 0x14051e478ee2ec, 0x11e478ee2edef63, 0x38ee2edef7aded3, 0x2edef7adef6bdc8, 0x37adef6bdf07c2d, 0x2f6bdf07c0018f9, 0x1f07c0018c018d1}); +constexpr StatTable58 SQR8_TABLE_58({0x1, 0x1100011000000, 0x10118d8c0000000, 0xd954189414901f, 0xc018def7a2f6f6, 0x1e19cc6d44444e, 0x2b2e8450d1ef706, 0x196294c624791e5, 0x2a9441aa2b74da8, 0x1c6810fa7a2fe66, 0x4e0f0eff6badbb, 0x26faf3a76e59127, 0x11aa58d6498919f, 0x3b3ce4e04f23b30, 0x2ed3f70684ae8d7, 0x8d64c737fc5014, 0x1516c589c4fd458, 0x5ba5cee14ea182, 0x368ad344d93d4ae, 0x15e2547ea25ba2a, 0xdecb4283969d9a, 0x2f2a95e5c791149, 0x3fc958586bc93d2, 0x3216bfeab663783, 0xced412d3f6e530, 0x85fb7bcb26d797, 0x19be97fdbcede01, 0x192a5409529ebf4, 0x98b4f8527795b1, 0x192e8188bbc9aac, 0x322f07e9abdf6d0, 0x2a4a5cd6239de91, 0x4c97dec82e63a, 0x37d6397e1d26aee, 0x1939dc6d77a98db, 0x2e23b8e5b0a982a, 0x2787751f5aa0dba, 0x3f81252033f3cc8, 0x1171b73d009f511, 0x8811f0328040bb, 0x3a659ae0b1d2417, 0xc8b454d91baa72, 0x197b01428520b86, 0x1872c8c17f7fe81, 0x143f7913f4c7f5, 0x3a71b7542e7ec68, 0x3e60d3d49155d34, 0x11d5f10402402c8, 0x2be8db11809a1df, 0x3667129f17b1d6a, 0x2749715db24cd0a, 0x185d6130cdfee96, 0x3abdc4d78640154, 0x39bd5ea2e22f89a, 0x3f9a113c1095209, 0xdb4c8bd4f72f4e, 0x32ae35f0ad0b4ee, 0x2f3770997f16cc}); +constexpr StatTable58 SQR16_TABLE_58({0x1, 0x2fcd2228a7c16ab, 0x846ef4a277243e, 0x1d2bf9061084cfb, 0x23598fddcd64f64, 0xcc36f3a7174e2a, 0x365a50c11b89583, 0x611bed1afae48, 0x22a03fae7957244, 0x45e6546308ff3e, 0x3aebd6f3893b9b4, 0x2bf4a9d5586f8db, 0x32fd7d2d5d6f867, 0x14819feeb813a6f, 0x100ab4d9ad808fc, 0x11c0fd674209c71, 0x3701211690581e7, 0x5c33087013a39a, 0x188935ebbc048bb, 0x10787f930a52538, 0xd49849206986b2, 0x17d1298ba5b565, 0x5d465e006f3142, 0x569a5ce90e9bff, 0x2b591716524b4cb, 0x32f7d39faa352cf, 0x10f701fea440dc0, 0x11c5f10a9d3c9d5, 0x18457154a0bf6ea, 0x15516f140726673, 0x1cf780781353aa4, 0x2a7d7e0e83c4bbc, 0x276c009e3198958, 0x220b8531adc2c11, 0x937d7effc370ab, 0x27632fc1b91dac1, 0x3b36628aa135d3f, 0x37230eddd77f21a, 0x1c1b5e0f410eca9, 0x3200c9c78a9127f, 0x3a55e6fb19e6dc4, 0x150cb064eb271f7, 0x5c74759db43ae1, 0x37046240fba02a9, 0x1937118eb920f04, 0x2795ad9a663a0c9, 0x1d4297ad3d62e8a, 0x3b927d82816e04d, 0x15b56f89c278c21, 0x2b8e4ef675619d6, 0x2e0823575b9bb28, 0xdeb4b405ed7e9c, 0x83d627c04e5155, 0x391134c52f7ae67, 0x9e2c9657999608, 0x3b1e574e9a4eb3a, 0x2b58dd062cd0021, 0x38d1fb86f1978ab}); +constexpr StatTable58 QRT_TABLE_58({0x21b9dfe73454bc2, 0x351ca3a13788360, 0x351ca3a13788362, 0x1ad5a042934094, 0x351ca3a13788366, 0x48f62c33f34cc, 0x1ad5a04293409c, 0x14b1f9a41eb8342, 0x351ca3a13788376, 0x3682437996f7786, 0x48f62c33f34ec, 0x21ad5a152920174, 0x1ad5a0429340dc, 0x3766ef998858a86, 0x14b1f9a41eb83c2, 0x151ca3a437843c2, 0x351ca3a13788276, 0x1e5ac7c1aff42c, 0x3682437996f7586, 0x3767ee558c7856a, 0x48f62c33f30ec, 0x39fb408a690330, 0x21ad5a152920974, 0x372f1d7dbf4255a, 0x1ad5a0429350dc, 0x39bb888af33330, 0x3766ef99885aa86, 0x27b58e0ba2df00, 0x14b1f9a41ebc3c2, 0x1540d06c191bcf2, 0x151ca3a4378c3c2, 0x39ee0d0a17f4c0, 0x351ca3a13798276, 0x2049f6c5379fdb4, 0x1e5ac7c1adf42c, 0x1ac5a182d64bf0, 0x3682437996b7586, 0x16cbe3c0a2c7c1e, 0x3767ee558cf856a, 0x372a1d35b20aa6a, 0x48f62c32f30ec, 0x26ab144a891cdc, 0x39fb408a490330, 0x205df3712ae76a8, 0x21ad5a152d20974, 0x34fb58f12e386b6, 0x372f1d7db74255a, 0x21b4a5f53871674, 0x1ad5a0439350dc, 0x1d602e40318fdc, 0x39bb8888f33330, 0x179bb8888f3332e, 0x3766ef99c85aa86, 0x2cec9eb2f5d0aa8, 0x27b58e03a2df00, 0x6caa1452491cdc, 0x14b1f9a51ebc3c2, 0}); +typedef Field<uint64_t, 58, 99, StatTable58, &SQR_TABLE_58, &SQR2_TABLE_58, &SQR4_TABLE_58, &SQR8_TABLE_58, &SQR16_TABLE_58, &QRT_TABLE_58, StatTable58, &LOAD_TABLE_58, &SAVE_TABLE_58> Field58; +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5> StatTableTRI58; +constexpr StatTableTRI58 SQR_TABLE_TRI58({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x80001, 0x200004, 0x800010, 0x2000040, 0x8000100, 0x20000400, 0x80001000, 0x200004000, 0x800010000, 0x2000040000, 0x8000100000, 0x20000400000, 0x80001000000, 0x200004000000, 0x800010000000, 0x2000040000000, 0x8000100000000, 0x20000400000000, 0x80001000000000, 0x200004000000000, 0x10000100002, 0x40000400008, 0x100001000020, 0x400004000080, 0x1000010000200, 0x4000040000800, 0x10000100002000, 0x40000400008000, 0x100001000020000}); +constexpr StatTableTRI58 SQR2_TABLE_TRI58({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x100000000000000, 0x200004, 0x2000040, 0x20000400, 0x200004000, 0x2000040000, 0x20000400000, 0x200004000000, 0x2000040000000, 0x20000400000000, 0x200004000000000, 0x40000400008, 0x400004000080, 0x4000040000800, 0x40000400008000, 0x4000000001, 0x40000000010, 0x400000000100, 0x4000000001000, 0x40000000010000, 0x180001, 0x1800010, 0x18000100, 0x180001000, 0x1800010000, 0x18000100000, 0x180001000000, 0x1800010000000, 0x18000100000000, 0x180001000000000, 0x10000300006, 0x100003000060, 0x1000030000600, 0x10000300006000, 0x100003000060000, 0x30000400004, 0x300004000040, 0x3000040000400, 0x30000400004000, 0x300004000040000, 0x4000020000c, 0x4000020000c0, 0x4000020000c00, 0x4000020000c000}); +constexpr StatTableTRI58 SQR4_TABLE_TRI58({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x2000040, 0x20000400000, 0x200004000000000, 0x40000400008000, 0x4000000001000, 0x18000100, 0x180001000000, 0x10000300006, 0x100003000060000, 0x30000400004000, 0x4000020000c00, 0x200004000100, 0x40001400008, 0x14000000001, 0x140000000010000, 0x380005000, 0x3800050000000, 0x5000070000e0, 0x70000400014, 0x3000040001c0001, 0x40001a0001c000, 0x1a000140001000, 0x1400024000680, 0x240004000050, 0x40000180009, 0x1800010001, 0x18000100010000, 0x1000130000600, 0x1300004000040, 0x4000220004c, 0x22000440001, 0x220004400010000, 0x44000540008800, 0x5400000001100, 0x1b800150, 0x1b8001500000, 0x380015000300006, 0x15000370006e000, 0x37000440005400, 0x440003a000dc0, 0x3a0005400110, 0x20005400160000e, 0x140016400068001, 0x164000400015000, 0x4000398005900, 0x3980051000100, 0x5100063000e6, 0x100063000460014, 0x2300044001c4001, 0x44001820018c00, 0x18200104001100, 0x1040021400608, 0x214004000041, 0x140040000010008}); +constexpr StatTableTRI58 SQR8_TABLE_TRI58({0x1, 0x40001400008, 0x1300004000040, 0x4000398005900, 0x21a004140001000, 0x15007370046e014, 0x14004001b810158, 0x18022100441001, 0x1005130063600e6, 0x4200004181059, 0x447a1a3f41ccd0, 0x383d151573100e6, 0x101600544019940, 0x270027c0059c000, 0x4818070100e1, 0x20545402560168e, 0x76071e40419414, 0x38001501bb00157, 0x18054101401149, 0x104343116260a0c, 0x17b9c06c9180809, 0x35b793b6107d791, 0x2e706624276b452, 0x3a543cf805118, 0x278004063619444, 0x22aae45555d4155, 0x105597ba5075e80, 0x364504676052ce, 0x35800790000401, 0x264044141418809, 0x3313274051a405d, 0x52c1b85195848, 0x57f03b8205e9e, 0x22c5044070044e0, 0x1547370047f115, 0x10402383a848d51, 0x16220024510a1, 0x4050579e30c9e7, 0x15e201b4605c018, 0x297e7fd6e672cc2, 0x286f01429f08ff7, 0x31c56646279854c, 0x36fd34ece6e98e6, 0x31e6939431f00b9, 0x311386d18673a0a, 0x2b6524f5cf195aa, 0x2dd63711ff50016, 0x1585649073391ae, 0x1004431143e1ab5, 0x13be61cf659d4d9, 0x98a87036371777, 0x66673706472d14, 0x273867fcbd99159, 0x27c4c58464098e9, 0x347304213c56db, 0x721f05c140cc15, 0x38144503ed007d9, 0x2e054541404549}); +constexpr StatTableTRI58 SQR16_TABLE_TRI58({0x1, 0x3f4d56f7779e1f0, 0xe27368ee2eeacd, 0x135c653e9699a2f, 0x6b0f78c5b96a46, 0x25fa3044c7e0248, 0x2a078335aa8c788, 0x2b2fb5e8ec09222, 0x214fe2bd0b14a22, 0x10b6f34977f0f41, 0x3dc4a1564361cee, 0xa2ae7c793a9fcf, 0x7fc45e1a362304, 0x3ec19729047ce58, 0x1ef9b26acd27396, 0x225a72a9b2db21a, 0xaaa90ccba715d8, 0x2da6362d54cd62, 0x37dae1e3484d433, 0x1ced37972ce3594, 0x164d907773ab8b9, 0xbeaf6f3fc883a1, 0x1d8ac7ee4682652, 0x102fa1481f0470a, 0x3e17062fd515fba, 0x21652276c35fe65, 0x57862a59d3fa78, 0x36b077a8057cde3, 0x287ce593d9cee2f, 0x290b965ae5d215a, 0x2cc2a18d887125c, 0xc46c603fd8423b, 0xdcd705a0e16776, 0x3307e00c6585a3f, 0x2d82d4b6c18532d, 0x28efe74f174d530, 0x2ddbc57b95adaac, 0x31d41679a107eb4, 0x1f24f6f872cb97f, 0x32718f9b0a03ff6, 0x1f283546f68ca0c, 0x158f309c150c885, 0x1ccaf78ea1873ea, 0x30e3b732bf1875f, 0xcce47efdb9ecb1, 0xcf3954987b5601, 0xebdc136185c456, 0x388046727963e11, 0x22e117909faee51, 0x3215b67613a2a60, 0x172480d3a2f11de, 0x382552280610b4d, 0x3c53c5d9c350cce, 0x6edc0d3330295e, 0x3452a6b8c868f37, 0x398cd7e93017ecc, 0x2e1ec37c30a741e, 0xb00d11006ffa14}); +constexpr StatTableTRI58 QRT_TABLE_TRI58({0x2450096792a5c5c, 0x610014271011c, 0x610014271011e, 0x1f0cb811314ea88, 0x610014271011a, 0x8000000420, 0x1f0cb811314ea80, 0x265407ad8a20bcc, 0x610014271010a, 0x3d18be98392ebd0, 0x8000000400, 0xc29b930e407056, 0x1f0cb811314eac0, 0x1fcef001154dee8, 0x265407ad8a20b4c, 0xc69b924c61f94a, 0x610014271000a, 0x211006895845190, 0x3d18be98392e9d0, 0x54007accac09cc, 0x8000000000, 0xc08b934e107854, 0xc29b930e407856, 0x275407adc220bcc, 0x1f0cb811314fac0, 0x1f6db815164ea8a, 0x1fcef001154fee8, 0x1b2db801945e396, 0x265407ad8a24b4c, 0x21100ec95865590, 0xc69b924c61794a, 0x273507b1e530ad6, 0x610014270000a, 0x1b4cb835b34e29c, 0x211006895865190, 0x3839bf20d47e016, 0x3d18be98396e9d0, 0x3858bd34f36e01c, 0x54007acca409cc, 0, 0x8000100000, 0xc29a130e507856, 0xc08b934e307854, 0x13253921d448296, 0xc29b930e007856, 0x13c60935f6486bc, 0x275407adca20bcc, 0x3571be8c5e6c9da, 0x1f0cb811214fac0, 0x410014261011c, 0x1f6db815364ea8a, 0x13a50921d1486b6, 0x1fcef001554fee8, 0x64001249245a5c, 0x1b2db801145e396, 0x8610014670200a, 0x265407ac8a24b4c, 0x1a5cbfbdeb0f30c}); +typedef FieldTri<uint64_t, 58, 19, StatTableTRI58, &SQR_TABLE_TRI58, &SQR2_TABLE_TRI58, &SQR4_TABLE_TRI58, &SQR8_TABLE_TRI58, &SQR16_TABLE_TRI58, &QRT_TABLE_TRI58, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri58; +#endif + +#ifdef ENABLE_FIELD_INT_59 +// 59 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5> StatTable59; +constexpr StatTable59 SQR_TABLE_59({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x12a, 0x4a8, 0x12a0, 0x4a80, 0x12a00, 0x4a800, 0x12a000, 0x4a8000, 0x12a0000, 0x4a80000, 0x12a00000, 0x4a800000, 0x12a000000, 0x4a8000000, 0x12a0000000, 0x4a80000000, 0x12a00000000, 0x4a800000000, 0x12a000000000, 0x4a8000000000, 0x12a0000000000, 0x4a80000000000, 0x12a00000000000, 0x4a800000000000, 0x12a000000000000, 0x4a8000000000000, 0x2a000000000012a, 0x28000000000043d, 0x200000000001061}); +constexpr StatTable59 SQR2_TABLE_59({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x100000000000000, 0x12a, 0x12a0, 0x12a00, 0x12a000, 0x12a0000, 0x12a00000, 0x12a000000, 0x12a0000000, 0x12a00000000, 0x12a000000000, 0x12a0000000000, 0x12a00000000000, 0x12a000000000000, 0x2a000000000012a, 0x200000000001061, 0x10444, 0x104440, 0x1044400, 0x10444000, 0x104440000, 0x1044400000, 0x10444000000, 0x104440000000, 0x1044400000000, 0x10444000000000, 0x104440000000000, 0x4440000000012a, 0x4440000000012a0, 0x440000000012ea8, 0x40000000012ee28, 0x12ee628, 0x12ee6280, 0x12ee62800, 0x12ee628000, 0x12ee6280000, 0x12ee62800000, 0x12ee628000000, 0x12ee6280000000, 0x12ee62800000000, 0x2ee62800000012a, 0x6e6280000001061, 0x662800000010079, 0x62800000010016c, 0x28000000100103c}); +constexpr StatTable59 SQR4_TABLE_59({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x12a0, 0x12a00000, 0x12a000000000, 0x2a000000000012a, 0x1044400, 0x10444000000, 0x104440000000000, 0x40000000012ee28, 0x12ee628000, 0x12ee6280000000, 0x662800000010079, 0x100101010, 0x1001010100000, 0x101010000012a0, 0x10000012a12b2a, 0x12a12b2b2a00, 0x2a12b2b2a00012a, 0x32b2a0001045461, 0x200010454541421, 0x104545414044000, 0x45414044012ef02, 0x4044012ef4d49aa, 0x12ef4d49e0ce28, 0x74d49e0ce290079, 0x1e0ce290000011d, 0x62900000100016d, 0x100010013a0, 0x100010013a00000, 0x10013a00012a000, 0x3a00012a012a12a, 0x12a012a116e400, 0x12a116e4010444, 0x116e40104450444, 0x40104450457ea6c, 0x4450457ea2692a0, 0x457ea2692ee7020, 0x22692ee706f7059, 0x2ee706f707e73ba, 0x6f707e73901116, 0x7e7390111013b6, 0x390111013b12b16, 0x11013b12b299b2a, 0x3b12b299b398b2a, 0x3299b398b17de61, 0x3398b17de543b4f, 0x317de543b7b1065, 0x6543b7b1053bb27, 0x37b1053bb4d0b6b, 0x53bb4d0b5b95ca, 0x34d0b5b95cfbf5b, 0x35b95cfbf6885b5, 0x5cfbf688587c89a, 0x7688587c8cf3adb, 0x587c8cf3aa00050, 0xcf3aa00001000d}); +constexpr StatTable59 SQR8_TABLE_59({0x1, 0x1001010100000, 0x10013a00012a000, 0x3398b17de543b4f, 0x2a00116b8c2812a, 0x7cbf06ffa4d5cd6, 0x1288f1cf576c2e0, 0x3047cfc394d3391, 0x322d00452b2c451, 0x226dcb1999949d1, 0x2e2e5ab30351bc0, 0x10b2afcfca2edc6, 0x7ff39b98a3372a8, 0x2d7b439441ae332, 0x5603b26a2dae616, 0x3a13899c470338a, 0x16e8a14f0113f3c, 0x754f4aa46d3bb2, 0x38aa45436b16334, 0x634468d6b3f47b5, 0x248ca58bd03241d, 0x255d1fbddf51ae7, 0x7f4b46a330ef6bc, 0x3b3159b37b1a654, 0x7bbff798b50cf3e, 0x568afef7a72512a, 0x701d7955e599ab3, 0x3e7aed5ec2e2c82, 0x5c4d118847ff477, 0x21264d599c12421, 0x4d287fc89bb5a71, 0x6d1f30202fff956, 0x6c54d2de7c68bf8, 0x350c930ed65aed3, 0x5630ddede4ba32c, 0x7c18282af602d36, 0x198a362bf3c8a07, 0x40dde880541e01c, 0x49c0e7e7438c0c7, 0x3ade2abe6845a50, 0x6ffad83e7ac09c4, 0x52185a0d23e667a, 0x2e8c821b63a858a, 0x770e59a57577b23, 0x2fe0ea55e7032a6, 0x23cf0c9a1565a09, 0x1c53d32d80a4427, 0x23164f78db9fa8b, 0x691c4ffab038e2a, 0x33fc91a8a831d85, 0x48039e34eec4e05, 0x2581dbb898c10b5, 0x374067097dfc9f9, 0x241611fdbd3f8e7, 0x1b9f2934941d831, 0x1940c046b9b4a62, 0x5333ac5e7a608f6, 0xe9fa1f11b06830, 0x3d3bbe0ab819c34}); +constexpr StatTable59 SQR16_TABLE_59({0x1, 0x857cdd2d43d447, 0xf829d2f68520b5, 0x19fe843a13f84fd, 0xcac85f3b30aa13, 0x5c7d9cd6997e169, 0x21e7ab9693a08f3, 0xe5cda6478df23f, 0xc3e206ed797b25, 0x755908ad7cca1c1, 0x16236a14b269480, 0x5fedfd73877a5e3, 0x6d66cb2c634cab2, 0x1b60fade310cb41, 0x5dcfd76c147e4ff, 0x2e686c220dcdc6a, 0x1d348a9dfc46113, 0x4e97ec4ce1b1081, 0x20ccc4ae0ada275, 0x5ec224932d09f73, 0x385cecd0572d2a0, 0x520f6a5162503d4, 0x3ff8003ba0976e, 0x5a314f7726ffcb7, 0x505c4f556b43e5a, 0x259ddd3f8c27783, 0x25441858e820409, 0x2714ab44ef6c58b, 0x53437cae5c3011c, 0x122c6454cb53ac0, 0x349b57934525af9, 0x394e01a9ab9a786, 0x665a91eb8e73f0d, 0x4c4e86cc5c98631, 0x7983a92ec037fe2, 0x67919ad3e0a3d69, 0x685c3d6c72af62e, 0x4eafca0e4b49fd7, 0x69534a8afbbeee, 0x720f8307d28c8cb, 0x49828239c03d1b7, 0x4c7e6edd9907a53, 0x1fe81ca4466f8fb, 0x19a865c194c7a23, 0x518bbfec9151454, 0x5b7bfbc756a7e4d, 0x146cc66da8b0754, 0x58e7cba08f0b29b, 0x1b578332a8f1985, 0x72d1c4f9eacac25, 0x6fc4f312025b99a, 0x199f6741974302b, 0x3edcb2e16193874, 0x38b45862414392c, 0x3a6669ab6604f52, 0x227da450a65496e, 0x4e85a5c57a7f719, 0x36b5dbf304b88be, 0x2ba8a1264ef68a0}); +constexpr StatTable59 QRT_TABLE_59({0x38d905ab028567a, 0x789fa6ed3b44d72, 0x789fa6ed3b44d70, 0x74ec857e93d828c, 0x789fa6ed3b44d74, 0x116b3c1203c96, 0x74ec857e93d8284, 0xc25ebc3871e280, 0x789fa6ed3b44d64, 0x47a37c3d910b6, 0x116b3c1203cb6, 0xc7322d7a8f48de, 0x74ec857e93d82c4, 0xb509a0ea52e496, 0xc25ebc3871e200, 0x74fdee4681d3e0c, 0x789fa6ed3b44c64, 0x7ffbbd080b2f09a, 0x47a37c3d912b6, 0xd5c937bae506c8, 0x116b3c12038b6, 0xb173c76987625e, 0xc7322d7a8f40de, 0x7591ff36b3a682c, 0x74ec857e93d92c4, 0x72b253bfbfc90c4, 0xb509a0ea52c496, 0x79f2e7b10e6d452, 0xc25ebc3871a200, 0x78c86e951086aac, 0x74fdee4681dbe0c, 0x78c96eb514c602c, 0x789fa6ed3b54c64, 0xc34818b95658e8, 0x7ffbbd080b0f09a, 0x7399f563b1980f2, 0x47a37c3dd12b6, 0xa29e0e28c58880, 0xd5c937baed06c8, 0x788ac23520ac82c, 0x116b3c13038b6, 0xa2c857e83d92b6, 0xb173c769a7625e, 0x608da990122e48, 0xc7322d7acf40de, 0xa3a89269eebefe, 0x7591ff36bba682c, 0xa25ebc2871a200, 0x74ec857e83d92c4, 0x11f62e419f1cfe, 0x72b253bf9fc90c4, 0x7425ebc2871a272, 0xb509a0ee52c496, 0x4ed8555979c8de, 0x79f2e7b18e6d452, 0x6c3580d5915d4d2, 0xc25ebc2871a200, 0, 0x78c86e971086aac}); +typedef Field<uint64_t, 59, 149, StatTable59, &SQR_TABLE_59, &SQR2_TABLE_59, &SQR4_TABLE_59, &SQR8_TABLE_59, &SQR16_TABLE_59, &QRT_TABLE_59, IdTrans, &ID_TRANS, &ID_TRANS> Field59; +#endif + +#ifdef ENABLE_FIELD_INT_60 +// 60 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6> StatTableTRI60; +constexpr StatTableTRI60 SQR_TABLE_TRI60({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x3, 0xc, 0x30, 0xc0, 0x300, 0xc00, 0x3000, 0xc000, 0x30000, 0xc0000, 0x300000, 0xc00000, 0x3000000, 0xc000000, 0x30000000, 0xc0000000, 0x300000000, 0xc00000000, 0x3000000000, 0xc000000000, 0x30000000000, 0xc0000000000, 0x300000000000, 0xc00000000000, 0x3000000000000, 0xc000000000000, 0x30000000000000, 0xc0000000000000, 0x300000000000000, 0xc00000000000000}); +constexpr StatTableTRI60 SQR2_TABLE_TRI60({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x100000000000000, 0x3, 0x30, 0x300, 0x3000, 0x30000, 0x300000, 0x3000000, 0x30000000, 0x300000000, 0x3000000000, 0x30000000000, 0x300000000000, 0x3000000000000, 0x30000000000000, 0x300000000000000, 0x5, 0x50, 0x500, 0x5000, 0x50000, 0x500000, 0x5000000, 0x50000000, 0x500000000, 0x5000000000, 0x50000000000, 0x500000000000, 0x5000000000000, 0x50000000000000, 0x500000000000000, 0xf, 0xf0, 0xf00, 0xf000, 0xf0000, 0xf00000, 0xf000000, 0xf0000000, 0xf00000000, 0xf000000000, 0xf0000000000, 0xf00000000000, 0xf000000000000, 0xf0000000000000, 0xf00000000000000}); +constexpr StatTableTRI60 SQR4_TABLE_TRI60({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x30, 0x300000, 0x3000000000, 0x30000000000000, 0x500, 0x5000000, 0x50000000000, 0x500000000000000, 0xf000, 0xf0000000, 0xf00000000000, 0x11, 0x110000, 0x1100000000, 0x11000000000000, 0x330, 0x3300000, 0x33000000000, 0x330000000000000, 0x5500, 0x55000000, 0x550000000000, 0x50000000000000f, 0xff000, 0xff0000000, 0xff00000000000, 0x101, 0x1010000, 0x10100000000, 0x101000000000000, 0x3030, 0x30300000, 0x303000000000, 0x30000000000005, 0x50500, 0x505000000, 0x5050000000000, 0x5000000000000f0, 0xf0f000, 0xf0f0000000, 0xf0f00000000000, 0x1111, 0x11110000, 0x111100000000, 0x111000000000003, 0x33330, 0x333300000, 0x3333000000000, 0x330000000000055, 0x555500, 0x5555000000, 0x55550000000000, 0x500000000000fff, 0xffff000, 0xffff0000000, 0xffff00000000000}); +constexpr StatTableTRI60 SQR8_TABLE_TRI60({0x1, 0x110000, 0x10100000000, 0x111000000000003, 0x300030, 0x33003300000, 0x30303000000005, 0x330000000555555, 0x50000000500, 0x50000005500000f, 0x5050000f0f000, 0x5000ffff0000fff, 0xf000f000f011, 0xff00ff1010101, 0xf0e11111111111, 0x31, 0x3210000, 0x313100000000, 0x221000000000056, 0x5300530, 0x563056300000, 0x5353530000000f5, 0x63000000faaaaaa, 0xf5000000f500, 0x500000fa500010e, 0xf5f50011e1f000, 0x5010ffef0010ffe, 0x11f011f011f321, 0x10ef10ec1313131, 0x1e2d22222222222, 0x501, 0x55110000, 0x5040100000000, 0x411000000000ffc, 0xf030f030, 0xff33ff3300000, 0xc0c03000001114, 0x330000100555554, 0x11050000110500, 0x50001015500303f, 0x114050333c0f003, 0x5300fcff0300fcf, 0x330f330f330a511, 0x3fc03af4040404, 0x395b44444444444, 0xf531, 0xfa6210000, 0xf5c43100000000, 0x721000000010fa8, 0x11f521f530, 0x10ea73ea6300000, 0x4d4c530000322d7, 0x63000310faaaa9b, 0x321f5000321f500, 0x500313ea505343e, 0x2d4f55677d1f056, 0x310acef5310ace, 0x621a621a62e562e, 0x43bc4cb34c4c4c4, 0x878788888888888}); +constexpr StatTableTRI60 SQR16_TABLE_TRI60({0x1, 0x563055110000, 0x111010233c0f003, 0x1200afffa8baffc, 0x5356030553000c5, 0x7145cf221744a77, 0x5748045489aaaaf, 0x7d52fcee4febdb3, 0x221f633c000a012, 0x41431fb55d4f4c8, 0x7f126132f4be5d5, 0x323da1f43c3a7e0, 0x373b24844474766, 0x6cc378a25584eb, 0x7ef66648aae4aca, 0x33003000031, 0xc0c03fb7c0f1fb, 0x174757777d10536, 0x2116210a52facb3, 0x5316fc100c1fb35, 0x7aae07597d161e1, 0x6752c4decfb6b7f, 0xf590fa78d56bf3, 0x1be67573275f157, 0xe3e0e9e0d61817, 0x25ac0012251ff6c, 0x407de1e40e3a849, 0x7a7264848fdf67e, 0x3bb8ba7d3879348, 0x498941f57060c6c, 0x5000000f0f501, 0x10fa8cfc1213ac0, 0x51a500f5501aab9, 0x73ef9049dcace64, 0x526a202f322f6e7, 0x2789a852500ca93, 0x4d1346684907509, 0x7d02bcfe4febdb2, 0x330a0329aba0521, 0x50a33c66415f5eb, 0x2e99dced402a73d, 0xf78f2f1a2dbcfe, 0x793a675db461a6a, 0x73848cd4c2f25d2, 0x54fa22d244aa9c6, 0xfae22e13e01501, 0x538ead296f222e5, 0x4da65592d2a750a, 0x40f91ebc14fcd2a, 0x5e73ff2f3c21c03, 0x4c72dce55551460, 0x3ffa59f8e5aef0a, 0x30057fa7b802f82, 0x36efe87d58aa6e4, 0x3bc96a196d71957, 0x5a82cfde2ad602f, 0x1f9bce94df9d3bf, 0x43c91d9b6bcabba, 0x2193c1833502ba3, 0xd28f516c1311d3d}); +constexpr StatTableTRI60 QRT_TABLE_TRI60({0x6983c00fe00104a, 0x804570322e054e6, 0x804570322e054e4, 0x15673387e0a4e4, 0x804570322e054e0, 0x100010110, 0x15673387e0a4ec, 0x920d01f34442a70, 0x804570322e054f0, 0x7a8dc0f2e4058f0, 0x100010130, 0x120c01f140462f0, 0x15673387e0a4ac, 0x7bdbb2ca9a4fe5c, 0x920d01f34442af0, 0xe9c6b039ce0c4ac, 0x804570322e055f0, 0xfac8b080ca20c00, 0x7a8dc0f2e405af0, 0x7a8dc4b2e4a59f0, 0x100010530, 0x10000100000, 0x120c01f14046af0, 0x131a02d91c5db6c, 0x15673387e0b4ac, 0x15623387d0b4ac, 0x7bdbb2ca9a4de5c, 0x7ffbbbca0a8ee5c, 0x920d01f34446af0, 0x800000020000000, 0xe9c6b039ce044ac, 0x81130302500f000, 0x804570322e155f0, 0x935b72eb3a48e9c, 0xfac8b080ca00c00, 0x120c016140563c0, 0x7a8dc0f2e445af0, 0x7bcbb3ca8a4ee5c, 0x7a8dc4b2e4259f0, 0xc4000a0300, 0x100110530, 0x11623285c1b19c, 0x10000300000, 0x420890090c3000, 0x120c01f14446af0, 0x68d7b33b9e0b4ac, 0x131a02d9145db6c, 0xe8ccb1e18a56fc0, 0x15673386e0b4ac, 0x7aadc8f2e485af0, 0x15623385d0b4ac, 0x4a0990093c3000, 0x7bdbb2cada4de5c, 0xf9d6b3389e0b4ac, 0x7ffbbbca8a8ee5c, 0xdf6ba38cec84ac, 0x920d01f24446af0, 0x520d01f24446af0, 0x800000000000000, 0}); +typedef FieldTri<uint64_t, 60, 1, StatTableTRI60, &SQR_TABLE_TRI60, &SQR2_TABLE_TRI60, &SQR4_TABLE_TRI60, &SQR8_TABLE_TRI60, &SQR16_TABLE_TRI60, &QRT_TABLE_TRI60, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri60; +#endif + +#ifdef ENABLE_FIELD_INT_61 +// 61 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5> StatTable61; +constexpr StatTable61 SQR_TABLE_61({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x1000000000000000, 0x4e, 0x138, 0x4e0, 0x1380, 0x4e00, 0x13800, 0x4e000, 0x138000, 0x4e0000, 0x1380000, 0x4e00000, 0x13800000, 0x4e000000, 0x138000000, 0x4e0000000, 0x1380000000, 0x4e00000000, 0x13800000000, 0x4e000000000, 0x138000000000, 0x4e0000000000, 0x1380000000000, 0x4e00000000000, 0x13800000000000, 0x4e000000000000, 0x138000000000000, 0x4e0000000000000, 0x1380000000000000, 0xe0000000000004e, 0x180000000000011f}); +constexpr StatTable61 SQR2_TABLE_61({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x100000000000000, 0x1000000000000000, 0x138, 0x1380, 0x13800, 0x138000, 0x1380000, 0x13800000, 0x138000000, 0x1380000000, 0x13800000000, 0x138000000000, 0x1380000000000, 0x13800000000000, 0x138000000000000, 0x1380000000000000, 0x180000000000011f, 0x1054, 0x10540, 0x105400, 0x1054000, 0x10540000, 0x105400000, 0x1054000000, 0x10540000000, 0x105400000000, 0x1054000000000, 0x10540000000000, 0x105400000000000, 0x1054000000000000, 0x540000000000138, 0x14000000000013ce, 0x13d96, 0x13d960, 0x13d9600, 0x13d96000, 0x13d960000, 0x13d9600000, 0x13d96000000, 0x13d960000000, 0x13d9600000000, 0x13d96000000000, 0x13d960000000000, 0x13d9600000000000, 0x1d9600000000011f, 0x196000000000101a, 0x1600000000010004}); +constexpr StatTable61 SQR4_TABLE_61({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x138, 0x1380000, 0x13800000000, 0x138000000000000, 0x10540, 0x105400000, 0x1054000000000, 0x540000000000138, 0x13d9600, 0x13d96000000, 0x13d960000000000, 0x1600000000010004, 0x100111000, 0x1001110000000, 0x11100000000138, 0x10000000013812b8, 0x13812ab8000, 0x13812ab80000000, 0x12ab800000010540, 0x105514114, 0x1055141140000, 0x551411400000138, 0x1140000013d84f6, 0x13d84f72f60, 0x13d84f72f600000, 0x4f72f6000010004, 0xf6000010000010f, 0x1000001010100, 0x10101000138, 0x101010001380000, 0x100013800013938, 0x138000139393800, 0x1393938010540, 0x193938010540011f, 0x180105400104445f, 0x540010444454138, 0x1044445413d9600, 0x445413d96013cae, 0x13d96013caaab96, 0x16013caaab970004, 0x1caaab970011111f, 0xb9700111101100b, 0x11110110010028, 0x11011001002812b8, 0x1001002812aab938, 0x2812aab92b8138, 0x12aab92b81382ec0, 0x192b81382ed1400b, 0x1382ed140105514, 0xed1401055150567, 0x105515056890f6, 0x1515056890f613ce, 0x56890f613d84e58, 0x10f613d84e5db85c, 0x13d84e5db84f6010, 0xe5db84f6000000e, 0x184f600000010123}); +constexpr StatTable61 SQR8_TABLE_61({0x1, 0x100111000, 0x10101000138, 0x1001002812aab938, 0x1000001390478, 0x113916c2d28792b8, 0x1904457c4545aa5f, 0x13aa7f0f280c5e20, 0x1047900101540, 0x13be84504128808e, 0x839d72c6e39c0f1, 0x16a18bbeafc6bac6, 0x7290382d6ea1584, 0x1d7d80a66b181691, 0x19d2aaa6110c5d47, 0x1b613d85f602c96f, 0x3812870738, 0x113dbce704cbbd40, 0xd92856e5392f94b, 0x84f76c3d7c304a3, 0x1a519225fe5ce8cf, 0x1704aca0c7190b8e, 0xb7fb1620ed7d025, 0x12831368539314f6, 0x748fb7c048744be, 0x78cc8029440fcba, 0x10eb05b6015eb730, 0xfd3c38351ebc6bd, 0x1665bcfabbfbe624, 0x136549cb4738e1ec, 0x6db6139d4b707f2, 0x1000057853aeac78, 0x104401500109340, 0x554c25992c8f3d8, 0x192dd4b6c0886747, 0x219c35ac73165fc, 0xdf27daa47ee296b, 0x73ab415a10863d2, 0x1f06884b4f2dc1dd, 0xb56c8c3efd7847f, 0x7a6a82768a4a3f2, 0x8773791c3b9f69f, 0x1e4d128bbd8fa105, 0x16977fb4d8984d86, 0xb9a5106882f60bf, 0xc5102ee91822469, 0xdab44dc3cdf7a0b, 0x18d48e2841f63e4, 0x165b8e4d03de40d4, 0x11a7aec6ef42385a, 0x17064ddd9b5041ea, 0xf89b61f74d1f401, 0x18583a8c57e6cb7f, 0x607279105fda3be, 0x905e9c0d58240c7, 0x1ed3c0319519fa7d, 0xa3227b6d1cc17a1, 0xf6cb7bb2aa84563, 0xdda77eb9b649e97, 0x15480a00ec829caf, 0x62cb6da6128c272}); +constexpr StatTable61 SQR16_TABLE_61({0x1, 0x1c7cd18a3a216933, 0xd201ddad374eb4, 0xee4694049c47289, 0x40db9f51130a1e6, 0x134cab3c67ec43f4, 0x97823873a2fc00f, 0xc08b772e8161a43, 0x128159f3d3611eac, 0x1f002f36181d6c4, 0x9de899abbd8d18f, 0x1a6ecb093fbb558b, 0xa6a1251b5961643, 0x1b285c169fb6616d, 0x9c04f5fcf0a4ce5, 0xd050c0ab89025ad, 0xdab152bf63418d9, 0xad3e33af7686059, 0x1561180155ac0dc8, 0x1d9e862521ab7d29, 0xa21b06e1e7632b5, 0x29b84e35cfc95ac, 0x17a27c78dac90e2c, 0x1312fa5f7b1e4ea2, 0xfe66bf53de6a93d, 0x182041e17dde85e9, 0x1289eb06f1803a2e, 0x129449a509af818c, 0x1f308057c81ab449, 0x419981420870054, 0x19f853b859910eb1, 0x9b422c0e9d60871, 0x9e6aec92bfcfa99, 0x15a788f1748b8f44, 0x1fa9a9c171dd83a1, 0x14096af6c0840cc6, 0x1bbe256976515067, 0x14f853fd9e5c0002, 0xf6256b0235f7a8, 0x37e727448043cf6, 0xbb0f467dd137c3f, 0x2538d574ceec19e, 0x15ff26c652c82188, 0x1c22b1e2a9ed31f3, 0x1f56b4b705c21301, 0x1502df3e9aa51832, 0x89c3dec02a6a543, 0x15eac5a464a4f736, 0x1d5023636fc14fa7, 0x499c5d458f9699e, 0x355b147c1703428, 0x1864a11df3efee51, 0x9af0f612e9c1265, 0x9c613962a1c08d9, 0x1cee6fc68f73b3f7, 0x185720007e663719, 0x101dd90a4502bf06, 0x1569af254da87eb0, 0x1781376276013a90, 0x10d2bf3d5e191483, 0x6215713bdc7d250}); +constexpr StatTable61 QRT_TABLE_61({0x171d34fcdac955d0, 0x12cfc8c049e1c96, 0x12cfc8c049e1c94, 0x71d34fcdac955c2, 0x12cfc8c049e1c90, 0x631c871de564852, 0x71d34fcdac955ca, 0x129fa6407f27300, 0x12cfc8c049e1c80, 0x7094f6fdd0a3b12, 0x631c871de564872, 0xdb28cee59c8256a, 0x71d34fcdac9558a, 0xc8a0be15a915472, 0x129fa6407f27380, 0x12dfcb4058e0b80, 0x12cfc8c049e1d80, 0x117d7f04ad0118, 0x7094f6fdd0a3912, 0x621b576dbe35b6a, 0x631c871de564c72, 0x13c808a013a1ee0, 0xdb28cee59c82d6a, 0x113d79842a0272, 0x71d34fcdac9458a, 0x719776b580b6a98, 0xc8a0be15a917472, 0x6633498d6db760a, 0x129fa6407f23380, 0xbd4ae9e8c3e7560, 0x12dfcb4058e8b80, 0x8000000a, 0x12cfc8c049f1d80, 0x634ce9add3b26ea, 0x117d7f04af0118, 0xda3f19c5d66258a, 0x7094f6fdd0e3912, 0xb87427e85e71560, 0x621b576dbeb5b6a, 0xc8b0b085b8c4e0a, 0x631c871de464c72, 0x1538fc8649458a, 0x13c808a011a1ee0, 0xcddbca6d1cfe360, 0xdb28cee59882d6a, 0xae80f550d1ffff2, 0x113d7984aa0272, 0xda7770f5f195912, 0x71d34fcdbc9458a, 0x137c8a049a1ee0, 0x719776b5a0b6a98, 0xded39a9d236ba78, 0xc8a0be15e917472, 0x6732488ca7ce0a, 0x6633498dedb760a, 0xc0406d0527cb80a, 0x129fa6417f23380, 0x3d4ae9eac3e756a, 0xbd4ae9eac3e7560, 0, 0x12dfcb4458e8b80}); +typedef Field<uint64_t, 61, 39, StatTable61, &SQR_TABLE_61, &SQR2_TABLE_61, &SQR4_TABLE_61, &SQR8_TABLE_61, &SQR16_TABLE_61, &QRT_TABLE_61, IdTrans, &ID_TRANS, &ID_TRANS> Field61; +#endif + +#ifdef ENABLE_FIELD_INT_62 +// 62 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5> StatTable62; +constexpr StatTable62 SQR_TABLE_62({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x1000000000000000, 0x20000001, 0x80000004, 0x200000010, 0x800000040, 0x2000000100, 0x8000000400, 0x20000001000, 0x80000004000, 0x200000010000, 0x800000040000, 0x2000000100000, 0x8000000400000, 0x20000001000000, 0x80000004000000, 0x200000010000000, 0x800000040000000, 0x2000000100000000, 0x440000002, 0x1100000008, 0x4400000020, 0x11000000080, 0x44000000200, 0x110000000800, 0x440000002000, 0x1100000008000, 0x4400000020000, 0x11000000080000, 0x44000000200000, 0x110000000800000, 0x440000002000000, 0x1100000008000000}); +constexpr StatTable62 SQR2_TABLE_62({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x100000000000000, 0x1000000000000000, 0x80000004, 0x800000040, 0x8000000400, 0x80000004000, 0x800000040000, 0x8000000400000, 0x80000004000000, 0x800000040000000, 0x440000002, 0x4400000020, 0x44000000200, 0x440000002000, 0x4400000020000, 0x44000000200000, 0x440000002000000, 0x400000000000001, 0x20000011, 0x200000110, 0x2000001100, 0x20000011000, 0x200000110000, 0x2000001100000, 0x20000011000000, 0x200000110000000, 0x2000001100000000, 0x11100000008, 0x111000000080, 0x1110000000800, 0x11100000008000, 0x111000000080000, 0x1110000000800000, 0x1100000088000004, 0x1000000800000044, 0x8080000444, 0x80800004440, 0x808000044400, 0x8080000444000, 0x80800004440000, 0x808000044400000, 0x80000404000002, 0x800004040000020, 0x40440000202, 0x404400002020, 0x4044000020200, 0x40440000202000, 0x404400002020000}); +constexpr StatTable62 SQR4_TABLE_62({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x80000004, 0x800000040000, 0x440000002, 0x4400000020000, 0x20000011, 0x200000110000, 0x2000001100000000, 0x11100000008000, 0x1000000800000044, 0x8080000444000, 0x800004040000020, 0x40440000202000, 0x400000000000101, 0x20001011000, 0x200010110000000, 0x101110000000800, 0x1100008088000404, 0x80808004044400, 0x80044404000202, 0x444044002020200, 0x440002002001110, 0x20002011101100, 0x20110011000080, 0x1100111000800080, 0x1110080000804400, 0x800080844004440, 0x808400044402000, 0x404400002021, 0x44000000210001, 0x300010110, 0x3000101100000, 0x101118000000c, 0x1118000800c0004, 0x8084c0040446, 0x84c00444460002, 0x4440460020213, 0x404600022130011, 0x2000201120111011, 0x2011301110118000, 0x3011001900008044, 0x1918080044c044, 0x1808004840440064, 0x484c4000646020, 0xc40004040200121, 0x40460001213100, 0x600010111000101, 0x101120001011800, 0x1200018198000404, 0x181910004044800, 0x110004c488000606, 0x4c4808006064400, 0x80046404001312, 0x464044013120200, 0x440112002001190, 0x1120002011901100, 0x20190011004480, 0x1900111044800080, 0x1110480000806400}); +constexpr StatTable62 SQR8_TABLE_62({0x1, 0x400000000000101, 0x44000000210001, 0x40460001213100, 0x404500002021, 0xe40014150200121, 0x80b400145512000, 0x1495c4000646820, 0x8000808c4004445, 0xd0800c8c8440561, 0x1045c80080ad6405, 0x1988b0805419944, 0x190110048480008e, 0x2891049dcc008662, 0x8ac190411026482, 0x241574511233a020, 0x1120002031901110, 0x3040203922110044, 0x1110792020b25580, 0x282a4830647355, 0x2c60001037102032, 0x26e4507065221080, 0x2036c57040579390, 0x3450409552c0cc02, 0x5c4000c66824017, 0x5508ce8e2845301, 0x4934eca8d59343, 0x1c28a918f7c9c0d1, 0xb080581194c8e4, 0x1018495dc440e46a, 0x1ac80985d8604226, 0xc7044545722023, 0x145120003031900, 0x4440003a0200005, 0x134447a19a002514, 0x510e645a31f1135, 0xae4834446175200, 0x264f451435730311, 0x7220c2004155891, 0x2153045891358c65, 0x154154800ca02904, 0x54dc0c88ce92565, 0xdc54bc04d28bc20, 0x8c54b0401283d8b, 0x29088e8109411f28, 0x12d0dc41982620a, 0xc0030c89a712640, 0x1dc8192422907592, 0x145554681022a075, 0x470792013225580, 0x346c6a7130667300, 0x39147d7004b077b2, 0x6c83e2d354461c6, 0xcf6d0046247a030, 0x3221c0f063a45c80, 0x303645fc20539787, 0x21004cf150409b10, 0x1dd380444d78042, 0x1c709df8b7145381, 0x185834a4e8d51327, 0x1420e118b389a4d1, 0xf0c41811b4e8c4}); +constexpr StatTable62 SQR16_TABLE_62({0x1, 0x147095f0731417c5, 0x3189fad107702e11, 0x3d3937fd86a460ab, 0x3ff26c959b47c587, 0x1e2ecbec4bf22bd6, 0x168ebaeceaf71b82, 0x216d6c4471f75c10, 0x1f6d31ccabfaee58, 0x1652ef2066ec0c61, 0x3d62ef6847f808fc, 0x26a33c99ec1b43d4, 0x32f26e79367c91ed, 0x361dcdd0d1e73240, 0xe2d494d081269e2, 0x33d231b9098b6045, 0x3c4e93c22fb78a3, 0x2f655fa56e578df3, 0x3a2b9600532c2609, 0x864e125951bbdb7, 0x2e2fca705bb62c58, 0x28e0629106401eaa, 0x7ac20f0ed6cdc1f, 0x3bd50add28a35850, 0x1a6e5ea19a59ab5d, 0x2add6d1d8c0aaefb, 0x2c3cf9842e6956a3, 0x1906944685f2c7c, 0x925997c95ed1de2, 0xcb9eb5d43c6f2e9, 0x1795f2b48a0fa71d, 0x19de5de41acc2100, 0x2e30c3a8444ef165, 0x29433812a3c4b1cd, 0xcbfa65dcdae6d63, 0x2580f2100e56c068, 0x25ce14544acc08cb, 0x24fa7059a7c87e18, 0x2a01d608b5d57d70, 0x3cefa2f54bdabc51, 0x29225fd40de84dea, 0x2d2276d8df087f20, 0x1a077580d9c5e840, 0x33b71879319b7de1, 0x16017e84617bddf4, 0x2596d6b0bd1a954c, 0x10267caddadbf666, 0x22c43bd90eaa3e05, 0xcaf6704a39c29fc, 0x25a0b38132106551, 0x1a78d1fcfd98f2a2, 0x1924d0b08fe1cc34, 0x3ea0a05c4cb14ee5, 0xa9b505540022072, 0x1e65cd1d5556d710, 0x3682cccd684103f1, 0x20a58fb864d70967, 0x35bfeeacb88f9b9b, 0x3b72dce9c4b09b87, 0x839908c285aaa64, 0x2ed676dc722e9732, 0x3dd67b08dc071450}); +constexpr StatTable62 QRT_TABLE_62({0x30268b6fba455d2c, 0x200000006, 0x200000004, 0x3d67cb6c1fe66c76, 0x200000000, 0x3fc4f1901abfa400, 0x3d67cb6c1fe66c7e, 0x35e79b6c0a66bcbe, 0x200000010, 0x1e9372bc57a9941e, 0x3fc4f1901abfa420, 0x21ec9d424957a5b0, 0x3d67cb6c1fe66c3e, 0x1cb35a6e52f5fb0e, 0x35e79b6c0a66bc3e, 0x215481024c13a730, 0x200000110, 0x1c324a6c52f75b08, 0x1e9372bc57a9961e, 0x3764a9d00f676820, 0x3fc4f1901abfa020, 0x355481020e132730, 0x21ec9d424957adb0, 0x3c43c32c0f34301e, 0x3d67cb6c1fe67c3e, 0x1496122c45259728, 0x1cb35a6e52f5db0e, 0x15e418405b72ec20, 0x35e79b6c0a66fc3e, 0x30268b6e3a445c38, 0x215481024c132730, 0x100010114, 0x200010110, 0, 0x1c324a6c52f55b08, 0x215581044d133776, 0x1e9372bc57ad961e, 0x2155810e4d133766, 0x3764a9d00f6f6820, 0x2157833c4d12323e, 0x3fc4f1901aafa020, 0x1c324a4252f55b58, 0x355481020e332730, 0x28332fc0509d41e, 0x21ec9d424917adb0, 0x215783be4d12332e, 0x3c43c32c0fb4301e, 0x2157822c4d06363e, 0x3d67cb6c1ee67c3e, 0x23f6b9d2484afb78, 0x1496122c47259728, 0x14b8184047648a80, 0x1cb35a6e56f5db0e, 0x3fe4f1901aefa820, 0x15e418405372ec20, 0x3d5fd72c1be276be, 0x35e79b6c1a66fc3e, 0x14b038d24774cf10, 0x30268b6e1a445c38, 0x1d17022e43a7172e, 0x215481020c132730, 0x2157022e4d07372e}); +typedef Field<uint64_t, 62, 536870913, StatTable62, &SQR_TABLE_62, &SQR2_TABLE_62, &SQR4_TABLE_62, &SQR8_TABLE_62, &SQR16_TABLE_62, &QRT_TABLE_62, IdTrans, &ID_TRANS, &ID_TRANS> Field62; +typedef FieldTri<uint64_t, 62, 29, RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5>, &SQR_TABLE_62, &SQR2_TABLE_62, &SQR4_TABLE_62, &SQR8_TABLE_62, &SQR16_TABLE_62, &QRT_TABLE_62, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri62; +#endif + +#ifdef ENABLE_FIELD_INT_63 +// 63 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5> StatTableTRI63; +constexpr StatTableTRI63 SQR_TABLE_TRI63({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x1000000000000000, 0x4000000000000000, 0x6, 0x18, 0x60, 0x180, 0x600, 0x1800, 0x6000, 0x18000, 0x60000, 0x180000, 0x600000, 0x1800000, 0x6000000, 0x18000000, 0x60000000, 0x180000000, 0x600000000, 0x1800000000, 0x6000000000, 0x18000000000, 0x60000000000, 0x180000000000, 0x600000000000, 0x1800000000000, 0x6000000000000, 0x18000000000000, 0x60000000000000, 0x180000000000000, 0x600000000000000, 0x1800000000000000, 0x6000000000000000}); +constexpr StatTableTRI63 SQR2_TABLE_TRI63({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x100000000000000, 0x1000000000000000, 0x6, 0x60, 0x600, 0x6000, 0x60000, 0x600000, 0x6000000, 0x60000000, 0x600000000, 0x6000000000, 0x60000000000, 0x600000000000, 0x6000000000000, 0x60000000000000, 0x600000000000000, 0x6000000000000000, 0x14, 0x140, 0x1400, 0x14000, 0x140000, 0x1400000, 0x14000000, 0x140000000, 0x1400000000, 0x14000000000, 0x140000000000, 0x1400000000000, 0x14000000000000, 0x140000000000000, 0x1400000000000000, 0x4000000000000006, 0x78, 0x780, 0x7800, 0x78000, 0x780000, 0x7800000, 0x78000000, 0x780000000, 0x7800000000, 0x78000000000, 0x780000000000, 0x7800000000000, 0x78000000000000, 0x780000000000000, 0x7800000000000000}); +constexpr StatTableTRI63 SQR4_TABLE_TRI63({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x6, 0x60000, 0x600000000, 0x6000000000000, 0x14, 0x140000, 0x1400000000, 0x14000000000000, 0x78, 0x780000, 0x7800000000, 0x78000000000000, 0x110, 0x1100000, 0x11000000000, 0x110000000000000, 0x660, 0x6600000, 0x66000000000, 0x660000000000000, 0x1540, 0x15400000, 0x154000000000, 0x1540000000000000, 0x7f80, 0x7f800000, 0x7f8000000000, 0x7f80000000000000, 0x10100, 0x101000000, 0x1010000000000, 0x100000000000006, 0x60600, 0x606000000, 0x6060000000000, 0x600000000000014, 0x141400, 0x1414000000, 0x14140000000000, 0x1400000000000078, 0x787800, 0x7878000000, 0x78780000000000, 0x7800000000000110, 0x1111000, 0x11110000000, 0x111100000000000, 0x1000000000000666, 0x6666000, 0x66660000000, 0x666600000000000, 0x6000000000001554, 0x15554000, 0x155540000000, 0x1555400000000000, 0x4000000000007ffe, 0x7fff8000, 0x7fff80000000, 0x7fff800000000000}); +constexpr StatTableTRI63 SQR8_TABLE_TRI63({0x1, 0x110, 0x10100, 0x1111000, 0x100010000, 0x11001100000, 0x1010101000000, 0x111111110000000, 0x100000006, 0x11000000660, 0x1010000060600, 0x111100006666000, 0x1000600060006, 0x110066006600660, 0x106060606060606, 0x1666666666666666, 0x12, 0x1320, 0x121200, 0x13332000, 0x1200120000, 0x132013200000, 0x12121212000000, 0x1333333320000000, 0x120000006c, 0x132000006ac0, 0x121200006c6c00, 0x133320006aaac000, 0x12006c006c006c, 0x13206ac06ac06ac0, 0x126c6c6c6c6c6c6c, 0x4aaaaaaaaaaaaaaa, 0x104, 0x11440, 0x1050400, 0x115544000, 0x10401040000, 0x1144114400000, 0x105050504000000, 0x1555555440000006, 0x10400000618, 0x1144000067980, 0x1050400061e1800, 0x155440067ff98006, 0x104061806180618, 0x1446798679867986, 0x21e1e1e1e1e1e1e, 0x3fffffffffffffec, 0x1248, 0x136c80, 0x125a4800, 0x137fec8000, 0x124812480000, 0x136c936c800000, 0x125a5a5a48000000, 0x7fffffec8000006a, 0x124800006db0, 0x136c80006b6b00, 0x125a48006dddb000, 0x7fec806b006b006a, 0x12486db06db06db0, 0x6ceb6b6b6b6b6b6a, 0x25dddddddddddddc}); +constexpr StatTableTRI63 SQR16_TABLE_TRI63({0x1, 0x10006, 0x100000014, 0x1000600140078, 0x116, 0x1160674, 0x11600001538, 0x116067415387e90, 0x10114, 0x101120678, 0x1011400141510, 0x112066c15687e66, 0x1170338, 0x117054a0a90, 0x1170338152c3f60, 0x54a1fbc41888532, 0x100010110, 0x1000701160660, 0x1010400141546, 0x102060c153e7f92, 0x11601170760, 0x116076301121340, 0x1171258152c6df4, 0x142a78fc131d6a4a, 0x1011500050540, 0x113067b055e1f86, 0x1110440042477e, 0x102261da46f39362, 0x117022e054b0b80, 0x45c09af143a3f72, 0x106721d847ee9ae4, 0x408a833f0a833f0a, 0x100010106, 0x1000701000614, 0x101120014147e, 0x114067814067902, 0x11601171074, 0x116076316066138, 0x117054c152d40e4, 0x33e0a853e0b842a, 0x1011500131278, 0x113066d12126d16, 0x7077c017b681e, 0x76e12736f057056, 0x117022e12493290, 0x45c1ead5f26a912, 0x76518c96bc5efa4, 0xb97397297387286, 0x1700171666, 0x17006516147554, 0x17174a012d3f8a, 0x173872913964814e, 0x160216157534, 0x16026219014b3eb8, 0x16144d1d3902f39c, 0x3964974c65925d30, 0x17163b005d59f8, 0x164974c75837d462, 0x17062a404d28cfa, 0x65854b0a96152d3c, 0x16152c2a5943b390, 0x5854b1be6419dd1e, 0x6045c19c854b1fba}); +constexpr StatTableTRI63 QRT_TABLE_TRI63({0, 0x100010114, 0x100010116, 0x1001701051372, 0x100010112, 0x1000040220, 0x100170105137a, 0x5107703453bba, 0x100010102, 0x101130117155a, 0x1000040200, 0x40000200800, 0x100170105133a, 0x103151a137276d8, 0x5107703453b3a, 0x134e65fc7c222be0, 0x100010002, 0x100030103115a, 0x101130117175a, 0x106052d103f4de2, 0x1000040600, 0x15122707691d3a, 0x40000200000, 0x4530770bc57b3a, 0x100170105033a, 0x103011a131256d8, 0x103151a137256d8, 0x176f29eb55c7a8da, 0x5107703457b3a, 0x130b158b7767d0da, 0x134e65fc7c22abe0, 0x7bcaf59d2f62d3e2, 0x100000002, 0x1001401041260, 0x100030101115a, 0x5107e03443ab8, 0x101130113175a, 0x1043701251b3a, 0x106052d10374de2, 0x134e657d7c232be2, 0x1000140600, 0x106073d103b4be2, 0x15122707491d3a, 0x4438600ac07800, 0x40000600000, 0x176a199c5682d3e0, 0x4530770b457b3a, 0x7bca759c2f62d3e0, 0x100170005033a, 0x6116d02572de2, 0x103011a111256d8, 0x1346656d7c372de2, 0x103151a177256d8, 0x643c600aa07800, 0x176f29eb5dc7a8da, 0x7b4b758b2f67d0da, 0x5107713457b3a, 0x104570776b457b3a, 0x130b158b5767d0da, 0x734e65fc3c22abe0, 0x134e65fc3c22abe0, 0x4000000000000000, 0x7bcaf59daf62d3e2}); +typedef FieldTri<uint64_t, 63, 1, StatTableTRI63, &SQR_TABLE_TRI63, &SQR2_TABLE_TRI63, &SQR4_TABLE_TRI63, &SQR8_TABLE_TRI63, &SQR16_TABLE_TRI63, &QRT_TABLE_TRI63, IdTrans, &ID_TRANS, &ID_TRANS> FieldTri63; +#endif + +#ifdef ENABLE_FIELD_INT_64 +// 64 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5> StatTable64; +constexpr StatTable64 SQR_TABLE_64({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x1000000000000000, 0x4000000000000000, 0x1b, 0x6c, 0x1b0, 0x6c0, 0x1b00, 0x6c00, 0x1b000, 0x6c000, 0x1b0000, 0x6c0000, 0x1b00000, 0x6c00000, 0x1b000000, 0x6c000000, 0x1b0000000, 0x6c0000000, 0x1b00000000, 0x6c00000000, 0x1b000000000, 0x6c000000000, 0x1b0000000000, 0x6c0000000000, 0x1b00000000000, 0x6c00000000000, 0x1b000000000000, 0x6c000000000000, 0x1b0000000000000, 0x6c0000000000000, 0x1b00000000000000, 0x6c00000000000000, 0xb00000000000001b, 0xc00000000000005a}); +constexpr StatTable64 SQR2_TABLE_64({0x1, 0x10, 0x100, 0x1000, 0x10000, 0x100000, 0x1000000, 0x10000000, 0x100000000, 0x1000000000, 0x10000000000, 0x100000000000, 0x1000000000000, 0x10000000000000, 0x100000000000000, 0x1000000000000000, 0x1b, 0x1b0, 0x1b00, 0x1b000, 0x1b0000, 0x1b00000, 0x1b000000, 0x1b0000000, 0x1b00000000, 0x1b000000000, 0x1b0000000000, 0x1b00000000000, 0x1b000000000000, 0x1b0000000000000, 0x1b00000000000000, 0xb00000000000001b, 0x145, 0x1450, 0x14500, 0x145000, 0x1450000, 0x14500000, 0x145000000, 0x1450000000, 0x14500000000, 0x145000000000, 0x1450000000000, 0x14500000000000, 0x145000000000000, 0x1450000000000000, 0x450000000000001b, 0x50000000000001dc, 0x1db7, 0x1db70, 0x1db700, 0x1db7000, 0x1db70000, 0x1db700000, 0x1db7000000, 0x1db70000000, 0x1db700000000, 0x1db7000000000, 0x1db70000000000, 0x1db700000000000, 0x1db7000000000000, 0xdb7000000000001b, 0xb70000000000011f, 0x7000000000001105}); +constexpr StatTable64 SQR4_TABLE_64({0x1, 0x10000, 0x100000000, 0x1000000000000, 0x1b, 0x1b0000, 0x1b00000000, 0x1b000000000000, 0x145, 0x1450000, 0x14500000000, 0x145000000000000, 0x1db7, 0x1db70000, 0x1db700000000, 0x1db7000000000000, 0x11011, 0x110110000, 0x1101100000000, 0x101100000000001b, 0x1ab1ab, 0x1ab1ab0000, 0x1ab1ab00000000, 0xb1ab00000000015e, 0x1514515, 0x15145150000, 0x151451500000000, 0x4515000000001c6b, 0x1c6db6c7, 0x1c6db6c70000, 0x1c6db6c700000000, 0xb6c700000001010f, 0x101000101, 0x1010001010000, 0x10001010000001b, 0x1010000001b1b00, 0x1b1b001b1b, 0x1b1b001b1b0000, 0x1b001b1b00000145, 0x1b1b000001444500, 0x14445014445, 0x144450144450000, 0x4501444500001dac, 0x444500001daab71b, 0x1daab71daab7, 0x1daab71daab70000, 0xb71daab70001110e, 0xaab700011101101f, 0x1110110110111, 0x110110110111001b, 0x10110111001aab1b, 0x111001aab1ab1ab, 0x1aab1ab1ab1aab, 0xab1ab1ab1aab015e, 0xb1ab1aab0150145e, 0x1aab015014514515, 0x150145145145015, 0x1451451450151c70, 0x451450151c71db6b, 0x50151c71db6db6dc, 0x1c71db6db6db71c7, 0xdb6db6db71c6000b, 0xb6db71c60001000f, 0x71c6000100000005}); +constexpr StatTable64 SQR8_TABLE_64({0x1, 0x11011, 0x101000101, 0x1110110110111, 0x100000001001a, 0x10110001100aa1a1, 0x100011a1b1a011a, 0x100baa100bb1aa0a, 0x1a00000144, 0x1ba1ba01505504, 0x1a001b5f4401441a, 0xa0eb1eea544fee41, 0x15e0144001a1ce8, 0xf5ee551fbc9d4f5d, 0x1b4543b0eee81b44, 0xb89a98b89a98b894, 0x10dbc, 0x11d76167c, 0x10cb1bd0cb1bc, 0x1c6b617617606a67, 0xdbc00010da6ad43, 0x167d1d6d105be392, 0xbd170ae2484f0af7, 0x162bc80d36e8d468, 0x1aad58014ae5f0, 0x63df9865e4bbbb5, 0x43fc5a4cbafe0d17, 0xe3d18fd6f8de2666, 0x49e2e5eab134a710, 0x1c78a1664f19bdd8, 0xf0829cea9886f08a, 0x4d8f634d8f625cdd, 0x100514550, 0x1104554401050, 0x15115140114154b, 0x10050551444aec57, 0x4551004b4277f24b, 0xef2afe861bdfb, 0x1d64ceb6c85ed2c9, 0x4975810172576524, 0x73cf4644451101e, 0x4fd1b234005fb6a7, 0x1bddd12e486f9a6f, 0xaa3c6f23ad5e9724, 0xa02b0a9206ef4923, 0x18a08533d5a4e65e, 0x1fc83ef027d0132b, 0x5e54f45f48c9a13c, 0x10deeff7bf8c0, 0x1d21c38d4f8874db, 0x10886029449884cd, 0xfe25b26c0190be86, 0xf5345525adfcb67e, 0xb606f05c0f274ae6, 0x49303a49c3147e89, 0xe3dec1f0cb3467b8, 0xf3dd197b59b91bb7, 0x6e062ec482dfc7e, 0xc24c087e94b8c9c, 0x42e75f2649a63926, 0x4646807e89775aa9, 0xca57e67631079503, 0xf738d302cd26e621, 0xda8702da9702da9d}); +constexpr StatTable64 SQR16_TABLE_64({0x1, 0x15f0144001a114f, 0x1aad43011ba1e5, 0xe34916e80106e21d, 0x11cefef6be466, 0xab943b855d3d776b, 0x1c77b6cf4edf1bd0, 0x46923ddea5ce4e34, 0x5455145e48670f13, 0xfb7d34d8e2b804bb, 0xbbe0dfe164a4d5b4, 0x431d528b1f73a8a2, 0xc259794b79e2607, 0x5945c54c76a8d132, 0xf5cb8b3860386917, 0xb345180ffd7a5551, 0xbaf1bebe1ae4ad02, 0x45562dad588c6260, 0x55b2852b76a728c4, 0xb5908b73d457d739, 0xa5a058173d115951, 0x11e605f10dd49e16, 0xb122096fef2a82a8, 0xfb95933559736ac7, 0x42652cf9ded5daa5, 0xe9a56590d5ab5301, 0xb8cef5ec20abb26f, 0xb50edcd1421d92e0, 0x12ac73f1d2f67094, 0x1c5815d4c184bd2, 0xe227a4ef0cd1165c, 0xe8d4a3a319b07491, 0xb0ef530df44bb042, 0xfbcbf52ff08d7ea3, 0xa0eaea8c7f69bf70, 0xedc22185164a14b1, 0xbfb9f37fc5eb3abc, 0x3712083e323193a, 0xe7bdca1397a3c26c, 0xf2d44dcbd1d02306, 0xa8fcad00bc810b9c, 0x4f7014f9d2186ea, 0x1b4d4ccc40f8060f, 0xe9ecf1e0105dab78, 0xe34e682846de9f1d, 0xace6cd21bf5ef658, 0x10f0cfa8cf3326ff, 0x71a97b1c73b8a63, 0xe1398cba3a3345d1, 0xa439e4c62ecb0615, 0x4bcce9efcca8db40, 0x176e95394759914e, 0xb5c7335e43a80f7f, 0xeb5439d8e177d64d, 0xa6af064a2d733f41, 0x5efc52c7e2f99007, 0x4a6efe65d270460b, 0xfe0ff44f5baa9a6a, 0x104c70edd05ffd6f, 0xf07d029f554aa763, 0x1c3c3cc0aca30a16, 0x7a0a5f6c85237d50, 0x1b862fb6b961ed37, 0xdcd1bd32f8a7d3ba}); +constexpr StatTable64 QRT_TABLE_64({0x19c9369f278adc02, 0x84b2b22ab2383ee4, 0x84b2b22ab2383ee6, 0x9d7b84b495b3e3f6, 0x84b2b22ab2383ee2, 0x37c470b49213f790, 0x9d7b84b495b3e3fe, 0x1000a0105137c, 0x84b2b22ab2383ef2, 0x368e964a8edce1fc, 0x37c470b49213f7b0, 0x19c9368e278fdf4c, 0x9d7b84b495b3e3be, 0x2e4da23cbc7d4570, 0x1000a010513fc, 0x84f35772bac24232, 0x84b2b22ab2383ff2, 0x37c570ba9314e4fc, 0x368e964a8edce3fc, 0xb377c390213cdb0e, 0x37c470b49213f3b0, 0x85ed5a3aa99c24f2, 0x19c9368e278fd74c, 0xaabff0000780000e, 0x9d7b84b495b3f3be, 0x84b6b3dab03038f2, 0x2e4da23cbc7d6570, 0x511ea03494ffc, 0x1000a010553fc, 0xae0c0220343c6c0e, 0x84f35772bac2c232, 0x800000008000000e, 0x84b2b22ab2393ff2, 0xb376c29c202bc97e, 0x37c570ba9316e4fc, 0x9c3062488879e6ce, 0x368e964a8ed8e3fc, 0x41e42c08e47e70, 0xb377c3902134db0e, 0x85b9b108a60f56ce, 0x37c470b49203f3b0, 0x19dd3b6e21f3cb4c, 0x85ed5a3aa9bc24f2, 0x198ddf682c428ac0, 0x19c9368e27cfd74c, 0x4b7c68431ca84b0, 0xaabff0000700000e, 0x8040655489ffefbe, 0x9d7b84b494b3f3be, 0x18c1354e32bfa74c, 0x84b6b3dab23038f2, 0xaaf613cc0f74627e, 0x2e4da23cb87d6570, 0x3248b3d6b3342a8c, 0x511ea0b494ffc, 0xb60813c00e70700e, 0x1000a110553fc, 0x1e0d022a05393ffc, 0xae0c0220143c6c0e, 0xe0c0220143c6c00, 0x84f35772fac2c232, 0xc041e55948fbfdce, 0x800000000000000e, 0}); +typedef Field<uint64_t, 64, 27, StatTable64, &SQR_TABLE_64, &SQR2_TABLE_64, &SQR4_TABLE_64, &SQR8_TABLE_64, &SQR16_TABLE_64, &QRT_TABLE_64, IdTrans, &ID_TRANS, &ID_TRANS> Field64; +#endif +} + +Sketch* ConstructClMul8Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_57 + case 57: return new SketchImpl<Field57>(implementation, 57); +#endif +#ifdef ENABLE_FIELD_INT_58 + case 58: return new SketchImpl<Field58>(implementation, 58); +#endif +#ifdef ENABLE_FIELD_INT_59 + case 59: return new SketchImpl<Field59>(implementation, 59); +#endif +#ifdef ENABLE_FIELD_INT_61 + case 61: return new SketchImpl<Field61>(implementation, 61); +#endif +#ifdef ENABLE_FIELD_INT_62 + case 62: return new SketchImpl<Field62>(implementation, 62); +#endif +#ifdef ENABLE_FIELD_INT_64 + case 64: return new SketchImpl<Field64>(implementation, 64); +#endif + } + return nullptr; +} + +Sketch* ConstructClMulTri8Bytes(int bits, int implementation) { + switch (bits) { +#ifdef ENABLE_FIELD_INT_57 + case 57: return new SketchImpl<FieldTri57>(implementation, 57); +#endif +#ifdef ENABLE_FIELD_INT_58 + case 58: return new SketchImpl<FieldTri58>(implementation, 58); +#endif +#ifdef ENABLE_FIELD_INT_60 + case 60: return new SketchImpl<FieldTri60>(implementation, 60); +#endif +#ifdef ENABLE_FIELD_INT_62 + case 62: return new SketchImpl<FieldTri62>(implementation, 62); +#endif +#ifdef ENABLE_FIELD_INT_63 + case 63: return new SketchImpl<FieldTri63>(implementation, 63); +#endif + } + return nullptr; +} diff --git a/src/minisketch/src/fields/clmul_common_impl.h b/src/minisketch/src/fields/clmul_common_impl.h new file mode 100644 index 0000000000..3d179a1081 --- /dev/null +++ b/src/minisketch/src/fields/clmul_common_impl.h @@ -0,0 +1,170 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _MINISKETCH_FIELDS_CLMUL_COMMON_IMPL_H_ +#define _MINISKETCH_FIELDS_CLMUL_COMMON_IMPL_H_ 1 + +#include <stdint.h> +#include <immintrin.h> + +#include "../int_utils.h" +#include "../lintrans.h" + +namespace { + +// The memory sanitizer in clang < 11 cannot reason through _mm_clmulepi64_si128 calls. +// Disable memory sanitization in the functions using them for those compilers. +#if defined(__clang__) && (__clang_major__ < 11) +# if defined(__has_feature) +# if __has_feature(memory_sanitizer) +# define NO_SANITIZE_MEMORY __attribute__((no_sanitize("memory"))) +# endif +# endif +#endif +#ifndef NO_SANITIZE_MEMORY +# define NO_SANITIZE_MEMORY +#endif + +template<typename I, int BITS, I MOD> NO_SANITIZE_MEMORY I MulWithClMulReduce(I a, I b) +{ + static constexpr I MASK = Mask<BITS, I>(); + + const __m128i MOD128 = _mm_cvtsi64_si128(MOD); + __m128i product = _mm_clmulepi64_si128(_mm_cvtsi64_si128((uint64_t)a), _mm_cvtsi64_si128((uint64_t)b), 0x00); + if (BITS <= 32) { + __m128i high1 = _mm_srli_epi64(product, BITS); + __m128i red1 = _mm_clmulepi64_si128(high1, MOD128, 0x00); + __m128i high2 = _mm_srli_epi64(red1, BITS); + __m128i red2 = _mm_clmulepi64_si128(high2, MOD128, 0x00); + return _mm_cvtsi128_si64(_mm_xor_si128(_mm_xor_si128(product, red1), red2)) & MASK; + } else if (BITS == 64) { + __m128i red1 = _mm_clmulepi64_si128(product, MOD128, 0x01); + __m128i red2 = _mm_clmulepi64_si128(red1, MOD128, 0x01); + return _mm_cvtsi128_si64(_mm_xor_si128(_mm_xor_si128(product, red1), red2)); + } else if ((BITS % 8) == 0) { + __m128i high1 = _mm_srli_si128(product, BITS / 8); + __m128i red1 = _mm_clmulepi64_si128(high1, MOD128, 0x00); + __m128i high2 = _mm_srli_si128(red1, BITS / 8); + __m128i red2 = _mm_clmulepi64_si128(high2, MOD128, 0x00); + return _mm_cvtsi128_si64(_mm_xor_si128(_mm_xor_si128(product, red1), red2)) & MASK; + } else { + __m128i high1 = _mm_or_si128(_mm_srli_epi64(product, BITS), _mm_srli_si128(_mm_slli_epi64(product, 64 - BITS), 8)); + __m128i red1 = _mm_clmulepi64_si128(high1, MOD128, 0x00); + if ((uint64_t(MOD) >> (66 - BITS)) == 0) { + __m128i high2 = _mm_srli_epi64(red1, BITS); + __m128i red2 = _mm_clmulepi64_si128(high2, MOD128, 0x00); + return _mm_cvtsi128_si64(_mm_xor_si128(_mm_xor_si128(product, red1), red2)) & MASK; + } else { + __m128i high2 = _mm_or_si128(_mm_srli_epi64(red1, BITS), _mm_srli_si128(_mm_slli_epi64(red1, 64 - BITS), 8)); + __m128i red2 = _mm_clmulepi64_si128(high2, MOD128, 0x00); + return _mm_cvtsi128_si64(_mm_xor_si128(_mm_xor_si128(product, red1), red2)) & MASK; + } + } +} + +template<typename I, int BITS, int POS> NO_SANITIZE_MEMORY I MulTrinomial(I a, I b) +{ + static constexpr I MASK = Mask<BITS, I>(); + + __m128i product = _mm_clmulepi64_si128(_mm_cvtsi64_si128((uint64_t)a), _mm_cvtsi64_si128((uint64_t)b), 0x00); + if (BITS <= 32) { + __m128i high1 = _mm_srli_epi64(product, BITS); + __m128i red1 = _mm_xor_si128(high1, _mm_slli_epi64(high1, POS)); + if (POS == 1) { + return _mm_cvtsi128_si64(_mm_xor_si128(product, red1)) & MASK; + } else { + __m128i high2 = _mm_srli_epi64(red1, BITS); + __m128i red2 = _mm_xor_si128(high2, _mm_slli_epi64(high2, POS)); + return _mm_cvtsi128_si64(_mm_xor_si128(_mm_xor_si128(product, red1), red2)) & MASK; + } + } else { + __m128i high1 = _mm_or_si128(_mm_srli_epi64(product, BITS), _mm_srli_si128(_mm_slli_epi64(product, 64 - BITS), 8)); + if (BITS + POS <= 66) { + __m128i red1 = _mm_xor_si128(high1, _mm_slli_epi64(high1, POS)); + if (POS == 1) { + return _mm_cvtsi128_si64(_mm_xor_si128(product, red1)) & MASK; + } else if (BITS + POS <= 66) { + __m128i high2 = _mm_srli_epi64(red1, BITS); + __m128i red2 = _mm_xor_si128(high2, _mm_slli_epi64(high2, POS)); + return _mm_cvtsi128_si64(_mm_xor_si128(_mm_xor_si128(product, red1), red2)) & MASK; + } + } else { + const __m128i MOD128 = _mm_cvtsi64_si128(1 + (((uint64_t)1) << POS)); + __m128i red1 = _mm_clmulepi64_si128(high1, MOD128, 0x00); + __m128i high2 = _mm_or_si128(_mm_srli_epi64(red1, BITS), _mm_srli_si128(_mm_slli_epi64(red1, 64 - BITS), 8)); + __m128i red2 = _mm_xor_si128(high2, _mm_slli_epi64(high2, POS)); + return _mm_cvtsi128_si64(_mm_xor_si128(_mm_xor_si128(product, red1), red2)) & MASK; + } + } +} + +/** Implementation of fields that use the SSE clmul intrinsic for multiplication. */ +template<typename I, int B, I MOD, I (*MUL)(I, I), typename F, const F* SQR, const F* SQR2, const F* SQR4, const F* SQR8, const F* SQR16, const F* QRT, typename T, const T* LOAD, const T* SAVE> struct GenField +{ + typedef BitsInt<I, B> O; + typedef LFSR<O, MOD> L; + + static inline constexpr I Sqr1(I a) { return SQR->template Map<O>(a); } + static inline constexpr I Sqr2(I a) { return SQR2->template Map<O>(a); } + static inline constexpr I Sqr4(I a) { return SQR4->template Map<O>(a); } + static inline constexpr I Sqr8(I a) { return SQR8->template Map<O>(a); } + static inline constexpr I Sqr16(I a) { return SQR16->template Map<O>(a); } + +public: + typedef I Elem; + + inline constexpr int Bits() const { return B; } + + inline constexpr Elem Mul2(Elem val) const { return L::Call(val); } + + inline Elem Mul(Elem a, Elem b) const { return MUL(a, b); } + + class Multiplier + { + Elem m_val; + public: + inline constexpr explicit Multiplier(const GenField&, Elem a) : m_val(a) {} + constexpr Elem operator()(Elem a) const { return MUL(m_val, a); } + }; + + /** Compute the square of a. */ + inline constexpr Elem Sqr(Elem val) const { return SQR->template Map<O>(val); } + + /** Compute x such that x^2 + x = a (undefined result if no solution exists). */ + inline constexpr Elem Qrt(Elem val) const { return QRT->template Map<O>(val); } + + /** Compute the inverse of x1. */ + inline Elem Inv(Elem val) const { return InvLadder<I, O, B, MUL, Sqr1, Sqr2, Sqr4, Sqr8, Sqr16>(val); } + + /** Generate a random field element. */ + Elem FromSeed(uint64_t seed) const { + uint64_t k0 = 0x434c4d554c466c64ull; // "CLMULFld" + uint64_t k1 = seed; + uint64_t count = ((uint64_t)B) << 32; + I ret; + do { + ret = O::Mask(I(SipHash(k0, k1, count++))); + } while(ret == 0); + return LOAD->template Map<O>(ret); + } + + Elem Deserialize(BitReader& in) const { return LOAD->template Map<O>(in.Read<B, I>()); } + + void Serialize(BitWriter& out, Elem val) const { out.Write<B, I>(SAVE->template Map<O>(val)); } + + constexpr Elem FromUint64(uint64_t x) const { return LOAD->template Map<O>(O::Mask(I(x))); } + constexpr uint64_t ToUint64(Elem val) const { return uint64_t(SAVE->template Map<O>(val)); } +}; + +template<typename I, int B, I MOD, typename F, const F* SQR, const F* SQR2, const F* SQR4, const F* SQR8, const F* SQR16, const F* QRT, typename T, const T* LOAD, const T* SAVE> +using Field = GenField<I, B, MOD, MulWithClMulReduce<I, B, MOD>, F, SQR, SQR2, SQR4, SQR8, SQR16, QRT, T, LOAD, SAVE>; + +template<typename I, int B, int POS, typename F, const F* SQR, const F* SQR2, const F* SQR4, const F* SQR8, const F* SQR16, const F* QRT, typename T, const T* LOAD, const T* SAVE> +using FieldTri = GenField<I, B, I(1) + (I(1) << POS), MulTrinomial<I, B, POS>, F, SQR, SQR2, SQR4, SQR8, SQR16, QRT, T, LOAD, SAVE>; + +} + +#endif diff --git a/src/minisketch/src/fields/generic_1byte.cpp b/src/minisketch/src/fields/generic_1byte.cpp new file mode 100644 index 0000000000..5ce42dc5f7 --- /dev/null +++ b/src/minisketch/src/fields/generic_1byte.cpp @@ -0,0 +1,112 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_1) + +#include "generic_common_impl.h" + +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_2 +// 2 bit field +typedef RecLinTrans<uint8_t, 2> StatTable2; +typedef RecLinTrans<uint8_t, 2> DynTable2; +constexpr StatTable2 SQR_TABLE_2({0x1, 0x3}); +constexpr StatTable2 QRT_TABLE_2({0x2, 0}); +typedef Field<uint8_t, 2, 3, StatTable2, DynTable2, &SQR_TABLE_2, &QRT_TABLE_2> Field2; +#endif + +#ifdef ENABLE_FIELD_INT_3 +// 3 bit field +typedef RecLinTrans<uint8_t, 3> StatTable3; +typedef RecLinTrans<uint8_t, 3> DynTable3; +constexpr StatTable3 SQR_TABLE_3({0x1, 0x4, 0x6}); +constexpr StatTable3 QRT_TABLE_3({0, 0x4, 0x6}); +typedef Field<uint8_t, 3, 3, StatTable3, DynTable3, &SQR_TABLE_3, &QRT_TABLE_3> Field3; +#endif + +#ifdef ENABLE_FIELD_INT_4 +// 4 bit field +typedef RecLinTrans<uint8_t, 4> StatTable4; +typedef RecLinTrans<uint8_t, 4> DynTable4; +constexpr StatTable4 SQR_TABLE_4({0x1, 0x4, 0x3, 0xc}); +constexpr StatTable4 QRT_TABLE_4({0x6, 0xa, 0x8, 0}); +typedef Field<uint8_t, 4, 3, StatTable4, DynTable4, &SQR_TABLE_4, &QRT_TABLE_4> Field4; +#endif + +#ifdef ENABLE_FIELD_INT_5 +// 5 bit field +typedef RecLinTrans<uint8_t, 5> StatTable5; +typedef RecLinTrans<uint8_t, 3, 2> DynTable5; +constexpr StatTable5 SQR_TABLE_5({0x1, 0x4, 0x10, 0xa, 0xd}); +constexpr StatTable5 QRT_TABLE_5({0x14, 0x8, 0xa, 0, 0xe}); +typedef Field<uint8_t, 5, 5, StatTable5, DynTable5, &SQR_TABLE_5, &QRT_TABLE_5> Field5; +#endif + +#ifdef ENABLE_FIELD_INT_6 +// 6 bit field +typedef RecLinTrans<uint8_t, 6> StatTable6; +typedef RecLinTrans<uint8_t, 3, 3> DynTable6; +constexpr StatTable6 SQR_TABLE_6({0x1, 0x4, 0x10, 0x3, 0xc, 0x30}); +constexpr StatTable6 QRT_TABLE_6({0x3a, 0x26, 0x24, 0x14, 0x20, 0}); +typedef Field<uint8_t, 6, 3, StatTable6, DynTable6, &SQR_TABLE_6, &QRT_TABLE_6> Field6; +#endif + +#ifdef ENABLE_FIELD_INT_7 +// 7 bit field +typedef RecLinTrans<uint8_t, 4, 3> StatTable7; +typedef RecLinTrans<uint8_t, 4, 3> DynTable7; +constexpr StatTable7 SQR_TABLE_7({0x1, 0x4, 0x10, 0x40, 0x6, 0x18, 0x60}); +constexpr StatTable7 QRT_TABLE_7({0, 0x14, 0x16, 0x72, 0x12, 0x40, 0x7a}); +typedef Field<uint8_t, 7, 3, StatTable7, DynTable7, &SQR_TABLE_7, &QRT_TABLE_7> Field7; +#endif + +#ifdef ENABLE_FIELD_INT_8 +// 8 bit field +typedef RecLinTrans<uint8_t, 4, 4> StatTable8; +typedef RecLinTrans<uint8_t, 4, 4> DynTable8; +constexpr StatTable8 SQR_TABLE_8({0x1, 0x4, 0x10, 0x40, 0x1b, 0x6c, 0xab, 0x9a}); +constexpr StatTable8 QRT_TABLE_8({0xbc, 0x2a, 0x28, 0x86, 0x2c, 0xde, 0x8e, 0}); +typedef Field<uint8_t, 8, 27, StatTable8, DynTable8, &SQR_TABLE_8, &QRT_TABLE_8> Field8; +#endif +} + +Sketch* ConstructGeneric1Byte(int bits, int implementation) +{ + switch (bits) { +#ifdef ENABLE_FIELD_INT_2 + case 2: return new SketchImpl<Field2>(implementation, 2); +#endif +#ifdef ENABLE_FIELD_INT_3 + case 3: return new SketchImpl<Field3>(implementation, 3); +#endif +#ifdef ENABLE_FIELD_INT_4 + case 4: return new SketchImpl<Field4>(implementation, 4); +#endif +#ifdef ENABLE_FIELD_INT_5 + case 5: return new SketchImpl<Field5>(implementation, 5); +#endif +#ifdef ENABLE_FIELD_INT_6 + case 6: return new SketchImpl<Field6>(implementation, 6); +#endif +#ifdef ENABLE_FIELD_INT_7 + case 7: return new SketchImpl<Field7>(implementation, 7); +#endif +#ifdef ENABLE_FIELD_INT_8 + case 8: return new SketchImpl<Field8>(implementation, 8); +#endif + default: return nullptr; + } +} diff --git a/src/minisketch/src/fields/generic_2bytes.cpp b/src/minisketch/src/fields/generic_2bytes.cpp new file mode 100644 index 0000000000..12bf3110a6 --- /dev/null +++ b/src/minisketch/src/fields/generic_2bytes.cpp @@ -0,0 +1,124 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_2) + +#include "generic_common_impl.h" + +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_9 +// 9 bit field +typedef RecLinTrans<uint16_t, 5, 4> StatTable9; +typedef RecLinTrans<uint16_t, 3, 3, 3> DynTable9; +constexpr StatTable9 SQR_TABLE_9({0x1, 0x4, 0x10, 0x40, 0x100, 0x6, 0x18, 0x60, 0x180}); +constexpr StatTable9 QRT_TABLE_9({0, 0x4e, 0x4c, 0x1aa, 0x48, 0x22, 0x1a2, 0x100, 0x58}); +typedef Field<uint16_t, 9, 3, StatTable9, DynTable9, &SQR_TABLE_9, &QRT_TABLE_9> Field9; +#endif + +#ifdef ENABLE_FIELD_INT_10 +// 10 bit field +typedef RecLinTrans<uint16_t, 5, 5> StatTable10; +typedef RecLinTrans<uint16_t, 4, 3, 3> DynTable10; +constexpr StatTable10 SQR_TABLE_10({0x1, 0x4, 0x10, 0x40, 0x100, 0x9, 0x24, 0x90, 0x240, 0x112}); +constexpr StatTable10 QRT_TABLE_10({0xec, 0x86, 0x84, 0x30e, 0x80, 0x3c2, 0x306, 0, 0x90, 0x296}); +typedef Field<uint16_t, 10, 9, StatTable10, DynTable10, &SQR_TABLE_10, &QRT_TABLE_10> Field10; +#endif + +#ifdef ENABLE_FIELD_INT_11 +// 11 bit field +typedef RecLinTrans<uint16_t, 6, 5> StatTable11; +typedef RecLinTrans<uint16_t, 4, 4, 3> DynTable11; +constexpr StatTable11 SQR_TABLE_11({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0xa, 0x28, 0xa0, 0x280, 0x205}); +constexpr StatTable11 QRT_TABLE_11({0x734, 0x48, 0x4a, 0x1de, 0x4e, 0x35e, 0x1d6, 0x200, 0x5e, 0, 0x37e}); +typedef Field<uint16_t, 11, 5, StatTable11, DynTable11, &SQR_TABLE_11, &QRT_TABLE_11> Field11; +#endif + +#ifdef ENABLE_FIELD_INT_12 +// 12 bit field +typedef RecLinTrans<uint16_t, 6, 6> StatTable12; +typedef RecLinTrans<uint16_t, 4, 4, 4> DynTable12; +constexpr StatTable12 SQR_TABLE_12({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x9, 0x24, 0x90, 0x240, 0x900, 0x412}); +constexpr StatTable12 QRT_TABLE_12({0x48, 0xc10, 0xc12, 0x208, 0xc16, 0xd82, 0x200, 0x110, 0xc06, 0, 0xda2, 0x5a4}); +typedef Field<uint16_t, 12, 9, StatTable12, DynTable12, &SQR_TABLE_12, &QRT_TABLE_12> Field12; +#endif + +#ifdef ENABLE_FIELD_INT_13 +// 13 bit field +typedef RecLinTrans<uint16_t, 5, 4, 4> StatTable13; +typedef RecLinTrans<uint16_t, 4, 3, 3, 3> DynTable13; +constexpr StatTable13 SQR_TABLE_13({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x36, 0xd8, 0x360, 0xd80, 0x161b, 0x185a}); +constexpr StatTable13 QRT_TABLE_13({0xcfc, 0x1500, 0x1502, 0x382, 0x1506, 0x149c, 0x38a, 0x118, 0x1516, 0, 0x14bc, 0x100e, 0x3ca}); +typedef Field<uint16_t, 13, 27, StatTable13, DynTable13, &SQR_TABLE_13, &QRT_TABLE_13> Field13; +#endif + +#ifdef ENABLE_FIELD_INT_14 +// 14 bit field +typedef RecLinTrans<uint16_t, 5, 5, 4> StatTable14; +typedef RecLinTrans<uint16_t, 4, 4, 3, 3> DynTable14; +constexpr StatTable14 SQR_TABLE_14({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x21, 0x84, 0x210, 0x840, 0x2100, 0x442, 0x1108}); +constexpr StatTable14 QRT_TABLE_14({0x13f2, 0x206, 0x204, 0x3e06, 0x200, 0x1266, 0x3e0e, 0x114, 0x210, 0, 0x1246, 0x2848, 0x3e4e, 0x2258}); +typedef Field<uint16_t, 14, 33, StatTable14, DynTable14, &SQR_TABLE_14, &QRT_TABLE_14> Field14; +#endif + +#ifdef ENABLE_FIELD_INT_15 +// 15 bit field +typedef RecLinTrans<uint16_t, 5, 5, 5> StatTable15; +typedef RecLinTrans<uint16_t, 4, 4, 4, 3> DynTable15; +constexpr StatTable15 SQR_TABLE_15({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x6, 0x18, 0x60, 0x180, 0x600, 0x1800, 0x6000}); +constexpr StatTable15 QRT_TABLE_15({0, 0x114, 0x116, 0x428, 0x112, 0x137a, 0x420, 0x6d62, 0x102, 0x73a, 0x135a, 0x6460, 0x460, 0x4000, 0x6de2}); +typedef Field<uint16_t, 15, 3, StatTable15, DynTable15, &SQR_TABLE_15, &QRT_TABLE_15> Field15; +#endif + +#ifdef ENABLE_FIELD_INT_16 +// 16 bit field +typedef RecLinTrans<uint16_t, 6, 5, 5> StatTable16; +typedef RecLinTrans<uint16_t, 4, 4, 4, 4> DynTable16; +constexpr StatTable16 SQR_TABLE_16({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x2b, 0xac, 0x2b0, 0xac0, 0x2b00, 0xac00, 0xb056, 0xc10e}); +constexpr StatTable16 QRT_TABLE_16({0x732, 0x72b8, 0x72ba, 0x7e96, 0x72be, 0x78b2, 0x7e9e, 0x8cba, 0x72ae, 0xfa24, 0x7892, 0x5892, 0x7ede, 0xbec6, 0x8c3a, 0}); +typedef Field<uint16_t, 16, 43, StatTable16, DynTable16, &SQR_TABLE_16, &QRT_TABLE_16> Field16; +#endif +} + +Sketch* ConstructGeneric2Bytes(int bits, int implementation) +{ + switch (bits) { +#ifdef ENABLE_FIELD_INT_9 + case 9: return new SketchImpl<Field9>(implementation, 9); +#endif +#ifdef ENABLE_FIELD_INT_10 + case 10: return new SketchImpl<Field10>(implementation, 10); +#endif +#ifdef ENABLE_FIELD_INT_11 + case 11: return new SketchImpl<Field11>(implementation, 11); +#endif +#ifdef ENABLE_FIELD_INT_12 + case 12: return new SketchImpl<Field12>(implementation, 12); +#endif +#ifdef ENABLE_FIELD_INT_13 + case 13: return new SketchImpl<Field13>(implementation, 13); +#endif +#ifdef ENABLE_FIELD_INT_14 + case 14: return new SketchImpl<Field14>(implementation, 14); +#endif +#ifdef ENABLE_FIELD_INT_15 + case 15: return new SketchImpl<Field15>(implementation, 15); +#endif +#ifdef ENABLE_FIELD_INT_16 + case 16: return new SketchImpl<Field16>(implementation, 16); +#endif + default: return nullptr; + } +} diff --git a/src/minisketch/src/fields/generic_3bytes.cpp b/src/minisketch/src/fields/generic_3bytes.cpp new file mode 100644 index 0000000000..13e85bd1a1 --- /dev/null +++ b/src/minisketch/src/fields/generic_3bytes.cpp @@ -0,0 +1,124 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_3) + +#include "generic_common_impl.h" + +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_17 +// 17 bit field +typedef RecLinTrans<uint32_t, 6, 6, 5> StatTable17; +typedef RecLinTrans<uint32_t, 4, 4, 3, 3, 3> DynTable17; +constexpr StatTable17 SQR_TABLE_17({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x12, 0x48, 0x120, 0x480, 0x1200, 0x4800, 0x12000, 0x8012}); +constexpr StatTable17 QRT_TABLE_17({0, 0x4c3e, 0x4c3c, 0x1a248, 0x4c38, 0x428, 0x1a240, 0x1b608, 0x4c28, 0x206, 0x408, 0x4000, 0x1a200, 0x18006, 0x1b688, 0x14d2e, 0x4d28}); +typedef Field<uint32_t, 17, 9, StatTable17, DynTable17, &SQR_TABLE_17, &QRT_TABLE_17> Field17; +#endif + +#ifdef ENABLE_FIELD_INT_18 +// 18 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6> StatTable18; +typedef RecLinTrans<uint32_t, 4, 4, 4, 3, 3> DynTable18; +constexpr StatTable18 SQR_TABLE_18({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x9, 0x24, 0x90, 0x240, 0x900, 0x2400, 0x9000, 0x24000, 0x10012}); +constexpr StatTable18 QRT_TABLE_18({0x9208, 0x422, 0x420, 0x8048, 0x424, 0x68b0, 0x8040, 0x30086, 0x434, 0x1040, 0x6890, 0x30ca2, 0x8000, 0x32896, 0x30006, 0, 0x534, 0x20532}); +typedef Field<uint32_t, 18, 9, StatTable18, DynTable18, &SQR_TABLE_18, &QRT_TABLE_18> Field18; +#endif + +#ifdef ENABLE_FIELD_INT_19 +// 19 bit field +typedef RecLinTrans<uint32_t, 5, 5, 5, 4> StatTable19; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 3> DynTable19; +constexpr StatTable19 SQR_TABLE_19({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x4e, 0x138, 0x4e0, 0x1380, 0x4e00, 0x13800, 0x4e000, 0x3804e, 0x6011f}); +constexpr StatTable19 QRT_TABLE_19({0x5d6b0, 0x2f476, 0x2f474, 0x1d6a2, 0x2f470, 0x42a, 0x1d6aa, 0x1060, 0x2f460, 0x19e92, 0x40a, 0x1da98, 0x1d6ea, 0x28c78, 0x10e0, 0xf56a, 0x2f560, 0, 0x19c92}); +typedef Field<uint32_t, 19, 39, StatTable19, DynTable19, &SQR_TABLE_19, &QRT_TABLE_19> Field19; +#endif + +#ifdef ENABLE_FIELD_INT_20 +// 20 bit field +typedef RecLinTrans<uint32_t, 5, 5, 5, 5> StatTable20; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 4> DynTable20; +constexpr StatTable20 SQR_TABLE_20({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x9, 0x24, 0x90, 0x240, 0x900, 0x2400, 0x9000, 0x24000, 0x90000, 0x40012}); +constexpr StatTable20 QRT_TABLE_20({0xc5dea, 0xc0110, 0xc0112, 0xe11de, 0xc0116, 0x24814, 0xe11d6, 0x20080, 0xc0106, 0xfe872, 0x24834, 0xe4106, 0xe1196, 0x1d9a4, 0x20000, 0x31190, 0xc0006, 0, 0xfea72, 0x7ea74}); +typedef Field<uint32_t, 20, 9, StatTable20, DynTable20, &SQR_TABLE_20, &QRT_TABLE_20> Field20; +#endif + +#ifdef ENABLE_FIELD_INT_21 +// 21 bit field +typedef RecLinTrans<uint32_t, 6, 5, 5, 5> StatTable21; +typedef RecLinTrans<uint32_t, 4, 4, 4, 3, 3, 3> DynTable21; +constexpr StatTable21 SQR_TABLE_21({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0xa, 0x28, 0xa0, 0x280, 0xa00, 0x2800, 0xa000, 0x28000, 0xa0000, 0x80005}); +constexpr StatTable21 QRT_TABLE_21({0x1bd5fc, 0xbc196, 0xbc194, 0x74b96, 0xbc190, 0x1048, 0x74b9e, 0x672c8, 0xbc180, 0x4080, 0x1068, 0xc8200, 0x74bde, 0x64280, 0x67248, 0xc4280, 0xbc080, 0x80000, 0x4280, 0, 0x1468}); +typedef Field<uint32_t, 21, 5, StatTable21, DynTable21, &SQR_TABLE_21, &QRT_TABLE_21> Field21; +#endif + +#ifdef ENABLE_FIELD_INT_22 +// 22 bit field +typedef RecLinTrans<uint32_t, 6, 6, 5, 5> StatTable22; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 3, 3> DynTable22; +constexpr StatTable22 SQR_TABLE_22({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x3, 0xc, 0x30, 0xc0, 0x300, 0xc00, 0x3000, 0xc000, 0x30000, 0xc0000, 0x300000}); +constexpr StatTable22 QRT_TABLE_22({0x210d16, 0x104a, 0x1048, 0x4088, 0x104c, 0x200420, 0x4080, 0x492dc, 0x105c, 0x1a67f0, 0x200400, 0x21155c, 0x40c0, 0x20346c, 0x4925c, 0x1af7ac, 0x115c, 0x2274ac, 0x1a65f0, 0x2a65f0, 0x200000, 0}); +typedef Field<uint32_t, 22, 3, StatTable22, DynTable22, &SQR_TABLE_22, &QRT_TABLE_22> Field22; +#endif + +#ifdef ENABLE_FIELD_INT_23 +// 23 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6, 5> StatTable23; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 4, 3> DynTable23; +constexpr StatTable23 SQR_TABLE_23({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x42, 0x108, 0x420, 0x1080, 0x4200, 0x10800, 0x42000, 0x108000, 0x420000, 0x80042, 0x200108}); +constexpr StatTable23 QRT_TABLE_23({0, 0x1040, 0x1042, 0x43056, 0x1046, 0x121d76, 0x4305e, 0x40a0, 0x1056, 0x15176, 0x121d56, 0x7ee1f6, 0x4301e, 0x40000, 0x4020, 0x4f0be, 0x1156, 0x7cf0a0, 0x15376, 0x1ee9e8, 0x121956, 0x3ac9f6, 0x7ee9f6}); +typedef Field<uint32_t, 23, 33, StatTable23, DynTable23, &SQR_TABLE_23, &QRT_TABLE_23> Field23; +#endif + +#ifdef ENABLE_FIELD_INT_24 +// 24 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6, 6> StatTable24; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 4, 4> DynTable24; +constexpr StatTable24 SQR_TABLE_24({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1b, 0x6c, 0x1b0, 0x6c0, 0x1b00, 0x6c00, 0x1b000, 0x6c000, 0x1b0000, 0x6c0000, 0xb0001b, 0xc0005a}); +constexpr StatTable24 QRT_TABLE_24({0x104e, 0xaf42a8, 0xaf42aa, 0xb78186, 0xaf42ae, 0x4090, 0xb7818e, 0x4a37c, 0xaf42be, 0x3688c0, 0x40b0, 0x80080e, 0xb781ce, 0xaf2232, 0x4a3fc, 0x856a82, 0xaf43be, 0x29c970, 0x368ac0, 0x968ace, 0x44b0, 0x77d570, 0x80000e, 0}); +typedef Field<uint32_t, 24, 27, StatTable24, DynTable24, &SQR_TABLE_24, &QRT_TABLE_24> Field24; +#endif +} + +Sketch* ConstructGeneric3Bytes(int bits, int implementation) +{ + switch (bits) { +#ifdef ENABLE_FIELD_INT_17 + case 17: return new SketchImpl<Field17>(implementation, 17); +#endif +#ifdef ENABLE_FIELD_INT_18 + case 18: return new SketchImpl<Field18>(implementation, 18); +#endif +#ifdef ENABLE_FIELD_INT_19 + case 19: return new SketchImpl<Field19>(implementation, 19); +#endif +#ifdef ENABLE_FIELD_INT_20 + case 20: return new SketchImpl<Field20>(implementation, 20); +#endif +#ifdef ENABLE_FIELD_INT_21 + case 21: return new SketchImpl<Field21>(implementation, 21); +#endif +#ifdef ENABLE_FIELD_INT_22 + case 22: return new SketchImpl<Field22>(implementation, 22); +#endif +#ifdef ENABLE_FIELD_INT_23 + case 23: return new SketchImpl<Field23>(implementation, 23); +#endif +#ifdef ENABLE_FIELD_INT_24 + case 24: return new SketchImpl<Field24>(implementation, 24); +#endif + default: return nullptr; + } +} diff --git a/src/minisketch/src/fields/generic_4bytes.cpp b/src/minisketch/src/fields/generic_4bytes.cpp new file mode 100644 index 0000000000..2a26b90521 --- /dev/null +++ b/src/minisketch/src/fields/generic_4bytes.cpp @@ -0,0 +1,124 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_4) + +#include "generic_common_impl.h" + +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_25 +// 25 bit field +typedef RecLinTrans<uint32_t, 5, 5, 5, 5, 5> StatTable25; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 3, 3, 3> DynTable25; +constexpr StatTable25 SQR_TABLE_25({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x12, 0x48, 0x120, 0x480, 0x1200, 0x4800, 0x12000, 0x48000, 0x120000, 0x480000, 0x1200000, 0x800012}); +constexpr StatTable25 QRT_TABLE_25({0, 0x482110, 0x482112, 0x1b3c3e6, 0x482116, 0x4960ae, 0x1b3c3ee, 0x4088, 0x482106, 0x58a726, 0x49608e, 0x5ce52e, 0x1b3c3ae, 0x2006, 0x4008, 0x1c1a8, 0x482006, 0x1e96488, 0x58a526, 0x400000, 0x49648e, 0x1800006, 0x5ced2e, 0xb3d3a8, 0x1b3d3ae}); +typedef Field<uint32_t, 25, 9, StatTable25, DynTable25, &SQR_TABLE_25, &QRT_TABLE_25> Field25; +#endif + +#ifdef ENABLE_FIELD_INT_26 +// 26 bit field +typedef RecLinTrans<uint32_t, 6, 5, 5, 5, 5> StatTable26; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 4, 3, 3> DynTable26; +constexpr StatTable26 SQR_TABLE_26({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x1b, 0x6c, 0x1b0, 0x6c0, 0x1b00, 0x6c00, 0x1b000, 0x6c000, 0x1b0000, 0x6c0000, 0x1b00000, 0x2c0001b, 0x300005a}); +constexpr StatTable26 QRT_TABLE_26({0x217b530, 0x2ae82a8, 0x2ae82aa, 0x2001046, 0x2ae82ae, 0x2de032e, 0x200104e, 0x70c10c, 0x2ae82be, 0x20151f2, 0x2de030e, 0xbc1400, 0x200100e, 0x178570, 0x70c18c, 0x2ae4232, 0x2ae83be, 0x211d742, 0x20153f2, 0x21f54f2, 0x2de070e, 0x5e0700, 0xbc1c00, 0x3abb97e, 0x200000e, 0}); +typedef Field<uint32_t, 26, 27, StatTable26, DynTable26, &SQR_TABLE_26, &QRT_TABLE_26> Field26; +#endif + +#ifdef ENABLE_FIELD_INT_27 +// 27 bit field +typedef RecLinTrans<uint32_t, 6, 6, 5, 5, 5> StatTable27; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 4, 4, 3> DynTable27; +constexpr StatTable27 SQR_TABLE_27({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x4e, 0x138, 0x4e0, 0x1380, 0x4e00, 0x13800, 0x4e000, 0x138000, 0x4e0000, 0x1380000, 0x4e00000, 0x380004e, 0x600011f}); +constexpr StatTable27 QRT_TABLE_27({0x6bf0530, 0x2be4496, 0x2be4494, 0x2bf0522, 0x2be4490, 0x1896cca, 0x2bf052a, 0x408a, 0x2be4480, 0x368ae72, 0x1896cea, 0x18d2ee0, 0x2bf056a, 0x1c76d6a, 0x400a, 0x336e9f8, 0x2be4580, 0x36baf12, 0x368ac72, 0x430360, 0x18968ea, 0x34a6b80, 0x18d26e0, 0xbf1560, 0x2bf156a, 0, 0x1c74d6a}); +typedef Field<uint32_t, 27, 39, StatTable27, DynTable27, &SQR_TABLE_27, &QRT_TABLE_27> Field27; +#endif + +#ifdef ENABLE_FIELD_INT_28 +// 28 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6, 5, 5> StatTable28; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 4, 4, 4> DynTable28; +constexpr StatTable28 SQR_TABLE_28({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x3, 0xc, 0x30, 0xc0, 0x300, 0xc00, 0x3000, 0xc000, 0x30000, 0xc0000, 0x300000, 0xc00000, 0x3000000, 0xc000000}); +constexpr StatTable28 QRT_TABLE_28({0x121d57a, 0x40216, 0x40214, 0x8112578, 0x40210, 0x10110, 0x8112570, 0x12597ec, 0x40200, 0x6983e00, 0x10130, 0x972b99c, 0x8112530, 0x8002000, 0x125976c, 0x815a76c, 0x40300, 0x936b29c, 0x6983c00, 0x97bb8ac, 0x10530, 0x9103000, 0x972b19c, 0xf6384ac, 0x8113530, 0x4113530, 0x8000000, 0}); +typedef Field<uint32_t, 28, 3, StatTable28, DynTable28, &SQR_TABLE_28, &QRT_TABLE_28> Field28; +#endif + +#ifdef ENABLE_FIELD_INT_29 +// 29 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6, 6, 5> StatTable29; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 4, 3, 3, 3> DynTable29; +constexpr StatTable29 SQR_TABLE_29({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0xa, 0x28, 0xa0, 0x280, 0xa00, 0x2800, 0xa000, 0x28000, 0xa0000, 0x280000, 0xa00000, 0x2800000, 0xa000000, 0x8000005}); +constexpr StatTable29 QRT_TABLE_29({0x1b8351dc, 0xb87135e, 0xb87135c, 0xda7b35e, 0xb871358, 0x621a116, 0xda7b356, 0x40200, 0xb871348, 0xc9e2620, 0x621a136, 0x478b16, 0xda7b316, 0x6762e20, 0x40280, 0x6202000, 0xb871248, 0x627a316, 0xc9e2420, 0xcd1ad36, 0x621a536, 0x760e20, 0x478316, 0xa760e20, 0xda7a316, 0x8000000, 0x6760e20, 0, 0x44280}); +typedef Field<uint32_t, 29, 5, StatTable29, DynTable29, &SQR_TABLE_29, &QRT_TABLE_29> Field29; +#endif + +#ifdef ENABLE_FIELD_INT_30 +// 30 bit field +typedef RecLinTrans<uint32_t, 6, 6, 6, 6, 6> StatTable30; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 4, 4, 3, 3> DynTable30; +constexpr StatTable30 SQR_TABLE_30({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x3, 0xc, 0x30, 0xc0, 0x300, 0xc00, 0x3000, 0xc000, 0x30000, 0xc0000, 0x300000, 0xc00000, 0x3000000, 0xc000000, 0x30000000}); +constexpr StatTable30 QRT_TABLE_30({0x2159df4a, 0x109134a, 0x1091348, 0x10114, 0x109134c, 0x3a203420, 0x1011c, 0x20004080, 0x109135c, 0x2005439c, 0x3a203400, 0x100400, 0x1015c, 0x3eb21930, 0x20004000, 0x20504c00, 0x109125c, 0x3b2b276c, 0x2005419c, 0x210450c0, 0x3a203000, 0x3e93186c, 0x100c00, 0x3aa23530, 0x1115c, 0x6b3286c, 0x3eb23930, 0xeb23930, 0x20000000, 0}); +typedef Field<uint32_t, 30, 3, StatTable30, DynTable30, &SQR_TABLE_30, &QRT_TABLE_30> Field30; +#endif + +#ifdef ENABLE_FIELD_INT_31 +// 31 bit field +typedef RecLinTrans<uint32_t, 6, 5, 5, 5, 5, 5> StatTable31; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 4, 4, 4, 3> DynTable31; +constexpr StatTable31 SQR_TABLE_31({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x12, 0x48, 0x120, 0x480, 0x1200, 0x4800, 0x12000, 0x48000, 0x120000, 0x480000, 0x1200000, 0x4800000, 0x12000000, 0x48000000, 0x20000012}); +constexpr StatTable31 QRT_TABLE_31({0, 0x10110, 0x10112, 0x15076e, 0x10116, 0x117130e, 0x150766, 0x4743fa0, 0x10106, 0x1121008, 0x117132e, 0x176b248e, 0x150726, 0x172a2c88, 0x4743f20, 0x7eb81e86, 0x10006, 0x20008, 0x1121208, 0x56b2c8e, 0x117172e, 0x133f1bae, 0x176b2c8e, 0x7f2a0c8e, 0x151726, 0x10000000, 0x172a0c88, 0x60000006, 0x4747f20, 0x3eb89e80, 0x7eb89e86}); +typedef Field<uint32_t, 31, 9, StatTable31, DynTable31, &SQR_TABLE_31, &QRT_TABLE_31> Field31; +#endif + +#ifdef ENABLE_FIELD_INT_32 +// 32 bit field +typedef RecLinTrans<uint32_t, 6, 6, 5, 5, 5, 5> StatTable32; +typedef RecLinTrans<uint32_t, 4, 4, 4, 4, 4, 4, 4, 4> DynTable32; +constexpr StatTable32 SQR_TABLE_32({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x8d, 0x234, 0x8d0, 0x2340, 0x8d00, 0x23400, 0x8d000, 0x234000, 0x8d0000, 0x2340000, 0x8d00000, 0x23400000, 0x8d000000, 0x3400011a, 0xd0000468, 0x40001037}); +constexpr StatTable32 QRT_TABLE_32({0x54fd1264, 0xc26fcd64, 0xc26fcd66, 0x238a7462, 0xc26fcd62, 0x973bccaa, 0x238a746a, 0x77766712, 0xc26fcd72, 0xc1bdd556, 0x973bcc8a, 0x572a094c, 0x238a742a, 0xb693be84, 0x77766792, 0x9555c03e, 0xc26fcc72, 0x568419f8, 0xc1bdd756, 0x96c3d2ca, 0x973bc88a, 0x54861fdc, 0x572a014c, 0xb79badc4, 0x238a642a, 0xb9b99fe0, 0xb6939e84, 0xc519fa86, 0x77762792, 0, 0x9555403e, 0x377627ba}); +typedef Field<uint32_t, 32, 141, StatTable32, DynTable32, &SQR_TABLE_32, &QRT_TABLE_32> Field32; +#endif +} + +Sketch* ConstructGeneric4Bytes(int bits, int implementation) +{ + switch (bits) { +#ifdef ENABLE_FIELD_INT_25 + case 25: return new SketchImpl<Field25>(implementation, 25); +#endif +#ifdef ENABLE_FIELD_INT_26 + case 26: return new SketchImpl<Field26>(implementation, 26); +#endif +#ifdef ENABLE_FIELD_INT_27 + case 27: return new SketchImpl<Field27>(implementation, 27); +#endif +#ifdef ENABLE_FIELD_INT_28 + case 28: return new SketchImpl<Field28>(implementation, 28); +#endif +#ifdef ENABLE_FIELD_INT_29 + case 29: return new SketchImpl<Field29>(implementation, 29); +#endif +#ifdef ENABLE_FIELD_INT_30 + case 30: return new SketchImpl<Field30>(implementation, 30); +#endif +#ifdef ENABLE_FIELD_INT_31 + case 31: return new SketchImpl<Field31>(implementation, 31); +#endif +#ifdef ENABLE_FIELD_INT_32 + case 32: return new SketchImpl<Field32>(implementation, 32); +#endif + default: return nullptr; + } +} diff --git a/src/minisketch/src/fields/generic_5bytes.cpp b/src/minisketch/src/fields/generic_5bytes.cpp new file mode 100644 index 0000000000..b06418184d --- /dev/null +++ b/src/minisketch/src/fields/generic_5bytes.cpp @@ -0,0 +1,124 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_5) + +#include "generic_common_impl.h" + +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_33 +// 33 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 5, 5, 5> StatTable33; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 3, 3, 3> DynTable33; +constexpr StatTable33 SQR_TABLE_33({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x802, 0x2008, 0x8020, 0x20080, 0x80200, 0x200800, 0x802000, 0x2008000, 0x8020000, 0x20080000, 0x80200000, 0x800401, 0x2001004, 0x8004010, 0x20010040, 0x80040100}); +constexpr StatTable33 QRT_TABLE_33({0xba504dd4, 0x1e2798ef2, 0x1e2798ef0, 0x6698a4ec, 0x1e2798ef4, 0x1c7f1bef0, 0x6698a4e4, 0x16da1b384, 0x1e2798ee4, 0x661ca6ec, 0x1c7f1bed0, 0x1483b87a6, 0x6698a4a4, 0x800000, 0x16da1b304, 0x1a185101c, 0x1e2798fe4, 0xaa400954, 0x661ca4ec, 0x667caeec, 0x1c7f1bad0, 0x400800, 0x1483b8fa6, 0, 0x6698b4a4, 0x1c61da4b8, 0x802000, 0x16e5dadec, 0x16da1f304, 0x62fc8eec, 0x1a185901c, 0x1661da5ec, 0x1e2788fe4}); +typedef Field<uint64_t, 33, 1025, StatTable33, DynTable33, &SQR_TABLE_33, &QRT_TABLE_33> Field33; +#endif + +#ifdef ENABLE_FIELD_INT_34 +// 34 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5> StatTable34; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 3, 3> DynTable34; +constexpr StatTable34 SQR_TABLE_34({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x81, 0x204, 0x810, 0x2040, 0x8100, 0x20400, 0x81000, 0x204000, 0x810000, 0x2040000, 0x8100000, 0x20400000, 0x81000000, 0x204000000, 0x10000102, 0x40000408, 0x100001020}); +constexpr StatTable34 QRT_TABLE_34({0x2f973a1f6, 0x40202, 0x40200, 0x348102060, 0x40204, 0x8000420, 0x348102068, 0x1092195c8, 0x40214, 0x3f6881b6e, 0x8000400, 0x3f810383e, 0x348102028, 0x340002068, 0x109219548, 0x24015a774, 0x40314, 0x3f050343e, 0x3f688196e, 0x3f81c3a3a, 0x8000000, 0x24031a560, 0x3f810303e, 0xb08c1a12, 0x348103028, 0xb2881906, 0x340000068, 0, 0x10921d548, 0x2e131e576, 0x240152774, 0x18921d55e, 0x50314, 0x14015271c}); +typedef Field<uint64_t, 34, 129, StatTable34, DynTable34, &SQR_TABLE_34, &QRT_TABLE_34> Field34; +#endif + +#ifdef ENABLE_FIELD_INT_35 +// 35 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5> StatTable35; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 3> DynTable35; +constexpr StatTable35 SQR_TABLE_35({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0xa, 0x28, 0xa0, 0x280, 0xa00, 0x2800, 0xa000, 0x28000, 0xa0000, 0x280000, 0xa00000, 0x2800000, 0xa000000, 0x28000000, 0xa0000000, 0x280000000, 0x200000005}); +constexpr StatTable35 QRT_TABLE_35({0x5c2038114, 0x2bf547ee8, 0x2bf547eea, 0x2bf1074e8, 0x2bf547eee, 0x1883d0736, 0x2bf1074e0, 0x100420, 0x2bf547efe, 0x400800, 0x1883d0716, 0x5e90e4a0, 0x2bf1074a0, 0x4e70ac20, 0x1004a0, 0x2f060c880, 0x2bf547ffe, 0x37d55fffe, 0x400a00, 0x3372573de, 0x1883d0316, 0x700c20, 0x5e90eca0, 0x10604880, 0x2bf1064a0, 0x18f35377e, 0x4e708c20, 0x33f557ffe, 0x1044a0, 0x1bf557ffe, 0x2f0604880, 0x200000000, 0x2bf557ffe, 0, 0x37d57fffe}); +typedef Field<uint64_t, 35, 5, StatTable35, DynTable35, &SQR_TABLE_35, &QRT_TABLE_35> Field35; +#endif + +#ifdef ENABLE_FIELD_INT_36 +// 36 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6> StatTable36; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4> DynTable36; +constexpr StatTable36 SQR_TABLE_36({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x201, 0x804, 0x2010, 0x8040, 0x20100, 0x80400, 0x201000, 0x804000, 0x2010000, 0x8040000, 0x20100000, 0x80400000, 0x201000000, 0x804000000, 0x10000402, 0x40001008, 0x100004020, 0x400010080}); +constexpr StatTable36 QRT_TABLE_36({0x40200, 0x8b0526186, 0x8b0526184, 0x240001000, 0x8b0526180, 0xcb6894d94, 0x240001008, 0xdb6880c22, 0x8b0526190, 0x8000200, 0xcb6894db4, 0x500424836, 0x240001048, 0x406cb2834, 0xdb6880ca2, 0x241200008, 0x8b0526090, 0xdb05021a6, 0x8000000, 0xdb01829b2, 0xcb68949b4, 0x1001000, 0x500424036, 0x106116406, 0x240000048, 0xcb29968a4, 0x406cb0834, 0, 0xdb6884ca2, 0x110010516, 0x241208008, 0x430434520, 0x8b0536090, 0x41208040, 0xdb05221a6, 0xb6884d14}); +typedef Field<uint64_t, 36, 513, StatTable36, DynTable36, &SQR_TABLE_36, &QRT_TABLE_36> Field36; +#endif + +#ifdef ENABLE_FIELD_INT_37 +// 37 bit field +typedef RecLinTrans<uint64_t, 6, 6, 5, 5, 5, 5, 5> StatTable37; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3> DynTable37; +constexpr StatTable37 SQR_TABLE_37({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0xa6, 0x298, 0xa60, 0x2980, 0xa600, 0x29800, 0xa6000, 0x298000, 0xa60000, 0x2980000, 0xa600000, 0x29800000, 0xa6000000, 0x298000000, 0xa60000000, 0x980000053, 0x60000011f, 0x180000047c}); +constexpr StatTable37 QRT_TABLE_37({0xa3c62e7ba, 0xdc7a0c16a, 0xdc7a0c168, 0x12f7484546, 0xdc7a0c16c, 0xa9803a20, 0x12f748454e, 0xda07064a4, 0xdc7a0c17c, 0x123908de8e, 0xa9803a00, 0x122a888a8e, 0x12f748450e, 0x6790add8, 0xda0706424, 0x12e0a0384c, 0xdc7a0c07c, 0xcb28a2c2, 0x123908dc8e, 0xd09f85e86, 0xa9803e00, 0x124d682b6e, 0x122a88828e, 0x1738711a, 0x12f748550e, 0x73035b8, 0x67908dd8, 0xa0702438, 0xda0702424, 0xe0a0b860, 0x12e0a0b84c, 0x1c7a1c060, 0xdc7a1c07c, 0, 0xcb2aa2c2, 0x100000002c, 0x12390cdc8e}); +typedef Field<uint64_t, 37, 83, StatTable37, DynTable37, &SQR_TABLE_37, &QRT_TABLE_37> Field37; +#endif + +#ifdef ENABLE_FIELD_INT_38 +// 38 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 5, 5, 5, 5> StatTable38; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3> DynTable38; +constexpr StatTable38 SQR_TABLE_38({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x63, 0x18c, 0x630, 0x18c0, 0x6300, 0x18c00, 0x63000, 0x18c000, 0x630000, 0x18c0000, 0x6300000, 0x18c00000, 0x63000000, 0x18c000000, 0x630000000, 0x18c0000000, 0x2300000063, 0xc0000014a, 0x3000000528}); +constexpr StatTable38 QRT_TABLE_38({0x34b0ac6430, 0x2223262fa, 0x2223262f8, 0x35554405fe, 0x2223262fc, 0x355514098a, 0x35554405f6, 0x400840, 0x2223262ec, 0x1777726532, 0x35551409aa, 0x15c06fc0, 0x35554405b6, 0x1f5303fec, 0x4008c0, 0x236a21030, 0x2223263ec, 0x1a9008c00, 0x1777726732, 0x3692c60ab6, 0x3555140daa, 0x15556007ee, 0x15c067c0, 0x14a0b030f2, 0x35554415b6, 0x227c06d168, 0x1f5301fec, 0x16c3928fc2, 0x4048c0, 0x3a942c4c0, 0x236a29030, 0x1636a2902e, 0x2223363ec, 0x3a6e898276, 0x1a9028c00, 0x6de74eb2c, 0x1777766732, 0}); +typedef Field<uint64_t, 38, 99, StatTable38, DynTable38, &SQR_TABLE_38, &QRT_TABLE_38> Field38; +#endif + +#ifdef ENABLE_FIELD_INT_39 +// 39 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5, 5> StatTable39; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3> DynTable39; +constexpr StatTable39 SQR_TABLE_39({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x22, 0x88, 0x220, 0x880, 0x2200, 0x8800, 0x22000, 0x88000, 0x220000, 0x880000, 0x2200000, 0x8800000, 0x22000000, 0x88000000, 0x220000000, 0x880000000, 0x2200000000, 0x800000011, 0x2000000044}); +constexpr StatTable39 QRT_TABLE_39({0x66b02a408c, 0x100420, 0x100422, 0x14206080, 0x100426, 0x5dccefab1c, 0x14206088, 0x9fc11e5b6, 0x100436, 0x5466bea62a, 0x5dccefab3c, 0x9aa110536, 0x142060c8, 0x54739ed6e2, 0x9fc11e536, 0xe7a82c080, 0x100536, 0x4002000, 0x5466bea42a, 0x6a4022000, 0x5dccefaf3c, 0x9e8118536, 0x9aa110d36, 0x5680e080, 0x142070c8, 0x7d293c5b6, 0x54739ef6e2, 0x8d680e080, 0x9fc11a536, 0x6d282c080, 0xe7a824080, 0x800000000, 0x110536, 0x2d680e080, 0x4022000, 0, 0x5466baa42a, 0x46b03a44aa, 0x6a40a2000}); +typedef Field<uint64_t, 39, 17, StatTable39, DynTable39, &SQR_TABLE_39, &QRT_TABLE_39> Field39; +#endif + +#ifdef ENABLE_FIELD_INT_40 +// 40 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5, 5> StatTable40; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4> DynTable40; +constexpr StatTable40 SQR_TABLE_40({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x39, 0xe4, 0x390, 0xe40, 0x3900, 0xe400, 0x39000, 0xe4000, 0x390000, 0xe40000, 0x3900000, 0xe400000, 0x39000000, 0xe4000000, 0x390000000, 0xe40000000, 0x3900000000, 0xe400000000, 0x900000004b, 0x400000015e}); +constexpr StatTable40 QRT_TABLE_40({0x624b3cecc, 0xbc5c3f4c6, 0xbc5c3f4c4, 0xde1603e2c, 0xbc5c3f4c0, 0xaabec06cea, 0xde1603e24, 0x6cd9f724c2, 0xbc5c3f4d0, 0xcde1743818, 0xaabec06cca, 0xa138c314ca, 0xde1603e64, 0xaafc00f01a, 0x6cd9f72442, 0xcdca11bb4, 0xbc5c3f5d0, 0xa00002001a, 0xcde1743a18, 0xdf1407b90, 0xaabec068ca, 0xc043b482c8, 0xa138c31cca, 0xcb86977e3c, 0xde1602e64, 0x604596a326, 0xaafc00d01a, 0xcc1c165d0, 0x6cd9f76442, 0x673c94da26, 0xcdca19bb4, 0x67c0940a26, 0xbc5c2f5d0, 0xa4dca19bae, 0xa00000001a, 0x1bc5c2f5d0, 0xcde1703a18, 0, 0xdf1487b90, 0x8df1487b8a}); +typedef Field<uint64_t, 40, 57, StatTable40, DynTable40, &SQR_TABLE_40, &QRT_TABLE_40> Field40; +#endif +} + +Sketch* ConstructGeneric5Bytes(int bits, int implementation) +{ + switch (bits) { +#ifdef ENABLE_FIELD_INT_33 + case 33: return new SketchImpl<Field33>(implementation, 33); +#endif +#ifdef ENABLE_FIELD_INT_34 + case 34: return new SketchImpl<Field34>(implementation, 34); +#endif +#ifdef ENABLE_FIELD_INT_35 + case 35: return new SketchImpl<Field35>(implementation, 35); +#endif +#ifdef ENABLE_FIELD_INT_36 + case 36: return new SketchImpl<Field36>(implementation, 36); +#endif +#ifdef ENABLE_FIELD_INT_37 + case 37: return new SketchImpl<Field37>(implementation, 37); +#endif +#ifdef ENABLE_FIELD_INT_38 + case 38: return new SketchImpl<Field38>(implementation, 38); +#endif +#ifdef ENABLE_FIELD_INT_39 + case 39: return new SketchImpl<Field39>(implementation, 39); +#endif +#ifdef ENABLE_FIELD_INT_40 + case 40: return new SketchImpl<Field40>(implementation, 40); +#endif + default: return nullptr; + } +} diff --git a/src/minisketch/src/fields/generic_6bytes.cpp b/src/minisketch/src/fields/generic_6bytes.cpp new file mode 100644 index 0000000000..becb26e875 --- /dev/null +++ b/src/minisketch/src/fields/generic_6bytes.cpp @@ -0,0 +1,124 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_6) + +#include "generic_common_impl.h" + +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_41 +// 41 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5> StatTable41; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3> DynTable41; +constexpr StatTable41 SQR_TABLE_41({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x12, 0x48, 0x120, 0x480, 0x1200, 0x4800, 0x12000, 0x48000, 0x120000, 0x480000, 0x1200000, 0x4800000, 0x12000000, 0x48000000, 0x120000000, 0x480000000, 0x1200000000, 0x4800000000, 0x12000000000, 0x8000000012}); +constexpr StatTable41 QRT_TABLE_41({0, 0x1599a5e0b0, 0x1599a5e0b2, 0x105c119e0, 0x1599a5e0b6, 0x1a2030452a6, 0x105c119e8, 0x1a307c55b2e, 0x1599a5e0a6, 0x1ee3f47bc8e, 0x1a203045286, 0x400808, 0x105c119a8, 0x1a3038573a6, 0x1a307c55bae, 0x4d2882a520, 0x1599a5e1a6, 0x1ffbaa0b720, 0x1ee3f47be8e, 0x4d68c22528, 0x1a203045686, 0x200006, 0x400008, 0x1b79a21b200, 0x105c109a8, 0x1ef3886a526, 0x1a3038553a6, 0x1b692209200, 0x1a307c51bae, 0x5d99a4e1a6, 0x4d28822520, 0x185e109ae, 0x1599a4e1a6, 0x4e3f43be88, 0x1ffbaa2b720, 0x4000000000, 0x1ee3f43be8e, 0x18000000006, 0x4d68ca2528, 0xa203145680, 0x1a203145686}); +typedef Field<uint64_t, 41, 9, StatTable41, DynTable41, &SQR_TABLE_41, &QRT_TABLE_41> Field41; +#endif + +#ifdef ENABLE_FIELD_INT_42 +// 42 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6> StatTable42; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3> DynTable42; +constexpr StatTable42 SQR_TABLE_42({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x81, 0x204, 0x810, 0x2040, 0x8100, 0x20400, 0x81000, 0x204000, 0x810000, 0x2040000, 0x8100000, 0x20400000, 0x81000000, 0x204000000, 0x810000000, 0x2040000000, 0x8100000000, 0x20400000000, 0x1000000102, 0x4000000408, 0x10000001020}); +constexpr StatTable42 QRT_TABLE_42({0x810200080, 0x120810806, 0x120810804, 0x1068c1a1000, 0x120810800, 0x34005023008, 0x1068c1a1008, 0x800004080, 0x120810810, 0x162818a10, 0x34005023028, 0x42408a14, 0x1068c1a1048, 0x1001040, 0x800004000, 0xb120808906, 0x120810910, 0x34000020068, 0x162818810, 0x68c021400, 0x34005023428, 0x10004000, 0x42408214, 0x162418214, 0x1068c1a0048, 0xb002018116, 0x1003040, 0x10008180448, 0x800000000, 0x62c08b04, 0xb120800906, 0x2408d1a3060, 0x120800910, 0x34401003028, 0x34000000068, 0, 0x162858810, 0xa042058116, 0x68c0a1400, 0x8162858806, 0x34005123428, 0x3068c0a1468}); +typedef Field<uint64_t, 42, 129, StatTable42, DynTable42, &SQR_TABLE_42, &QRT_TABLE_42> Field42; +#endif + +#ifdef ENABLE_FIELD_INT_43 +// 43 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 5, 5, 5, 5, 5> StatTable43; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3> DynTable43; +constexpr StatTable43 SQR_TABLE_43({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0xb2, 0x2c8, 0xb20, 0x2c80, 0xb200, 0x2c800, 0xb2000, 0x2c8000, 0xb20000, 0x2c80000, 0xb200000, 0x2c800000, 0xb2000000, 0x2c8000000, 0xb20000000, 0x2c80000000, 0xb200000000, 0x2c800000000, 0x32000000059, 0x4800000013d, 0x20000000446}); +constexpr StatTable43 QRT_TABLE_43({0x2bccc2d6f6c, 0x4bccc2d6f54, 0x4bccc2d6f56, 0x7cc7bc61df0, 0x4bccc2d6f52, 0x7d13b404b10, 0x7cc7bc61df8, 0x37456e9ac5a, 0x4bccc2d6f42, 0x4e042c6a6, 0x7d13b404b30, 0x4a56de9ef4c, 0x7cc7bc61db8, 0x14bc18d8e, 0x37456e9acda, 0x7c89f84fb1e, 0x4bccc2d6e42, 0x7ffae40d210, 0x4e042c4a6, 0x366f45dd06, 0x7d13b404f30, 0x496fcaf8cca, 0x4a56de9e74c, 0x370b62b6af4, 0x7cc7bc60db8, 0x1498185a8, 0x14bc1ad8e, 0x7e602c46a98, 0x37456e9ecda, 0x36ccc2c6e74, 0x7c89f847b1e, 0x7e27d06d516, 0x4bccc2c6e42, 0x7f93302c396, 0x7ffae42d210, 0x3dd3440706, 0x4e046c4a6, 0x78bbc09da36, 0x366f4ddd06, 0, 0x7d13b504f30, 0x8bbc09da00, 0x496fc8f8cca}); +typedef Field<uint64_t, 43, 89, StatTable43, DynTable43, &SQR_TABLE_43, &QRT_TABLE_43> Field43; +#endif + +#ifdef ENABLE_FIELD_INT_44 +// 44 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5, 5, 5> StatTable44; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4> DynTable44; +constexpr StatTable44 SQR_TABLE_44({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x21, 0x84, 0x210, 0x840, 0x2100, 0x8400, 0x21000, 0x84000, 0x210000, 0x840000, 0x2100000, 0x8400000, 0x21000000, 0x84000000, 0x210000000, 0x840000000, 0x2100000000, 0x8400000000, 0x21000000000, 0x84000000000, 0x10000000042, 0x40000000108}); +constexpr StatTable44 QRT_TABLE_44({0xf05334f4f6e, 0x4002016, 0x4002014, 0xf04350e6246, 0x4002010, 0x4935b379a26, 0xf04350e624e, 0xf84250c228e, 0x4002000, 0xf04300e521e, 0x4935b379a06, 0xb966838dd48, 0xf04350e620e, 0xf7b8b80feda, 0xf84250c220e, 0xf972e097d5e, 0x4002100, 0x8000020000, 0xf04300e501e, 0x430025000, 0x4935b379e06, 0xf976a09dc5e, 0xb966838d548, 0xf84218c029a, 0xf04350e720e, 0x4925f36bf06, 0xf7b8b80deda, 0xb047d3ee758, 0xf84250c620e, 0xf80350e720e, 0xf972e09fd5e, 0x8091825284, 0x4012100, 0x9015063210, 0x8000000000, 0xff31a028c5e, 0xf04300a501e, 0x44340b7100, 0x4300a5000, 0, 0x4935b279e06, 0xa976b2dce18, 0xf976a29dc5e, 0x8935b279e18}); +typedef Field<uint64_t, 44, 33, StatTable44, DynTable44, &SQR_TABLE_44, &QRT_TABLE_44> Field44; +#endif + +#ifdef ENABLE_FIELD_INT_45 +// 45 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5, 5, 5> StatTable45; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3> DynTable45; +constexpr StatTable45 SQR_TABLE_45({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x36, 0xd8, 0x360, 0xd80, 0x3600, 0xd800, 0x36000, 0xd8000, 0x360000, 0xd80000, 0x3600000, 0xd800000, 0x36000000, 0xd8000000, 0x360000000, 0xd80000000, 0x3600000000, 0xd800000000, 0x36000000000, 0xd8000000000, 0x16000000001b, 0x18000000005a}); +constexpr StatTable45 QRT_TABLE_45({0xede34e3e0fc, 0x1554148191aa, 0x1554148191a8, 0x1767be1dc4a6, 0x1554148191ac, 0x26bd4931492, 0x1767be1dc4ae, 0x233ab9c454a, 0x1554148191bc, 0x16939e8bb3dc, 0x26bd49314b2, 0x3c6ca8bac52, 0x1767be1dc4ee, 0x16caa5054c16, 0x233ab9c45ca, 0x14a1649628bc, 0x1554148190bc, 0x3c382881252, 0x16939e8bb1dc, 0x3c7ca0aa160, 0x26bd49310b2, 0x27f40158000, 0x3c6ca8ba452, 0x173fc092853c, 0x1767be1dd4ee, 0x16cbe284f25c, 0x16caa5056c16, 0x155559002f96, 0x233ab9c05ca, 0x26eb8908b32, 0x14a16496a8bc, 0x15440885333c, 0x1554148090bc, 0x17d60702e0, 0x3c3828a1252, 0x54548d10b2, 0x16939e8fb1dc, 0x3ac1e81b1d2, 0x3c7ca02a160, 0x166bd48310bc, 0x26bd48310b2, 0, 0x27f40358000, 0x10000000000e, 0x3c6cacba452}); +typedef Field<uint64_t, 45, 27, StatTable45, DynTable45, &SQR_TABLE_45, &QRT_TABLE_45> Field45; +#endif + +#ifdef ENABLE_FIELD_INT_46 +// 46 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5, 5> StatTable46; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3> DynTable46; +constexpr StatTable46 SQR_TABLE_46({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x3, 0xc, 0x30, 0xc0, 0x300, 0xc00, 0x3000, 0xc000, 0x30000, 0xc0000, 0x300000, 0xc00000, 0x3000000, 0xc000000, 0x30000000, 0xc0000000, 0x300000000, 0xc00000000, 0x3000000000, 0xc000000000, 0x30000000000, 0xc0000000000, 0x300000000000}); +constexpr StatTable46 QRT_TABLE_46({0x211c4fd486ba, 0x100104a, 0x1001048, 0x104d0492d4, 0x100104c, 0x20005040c820, 0x104d0492dc, 0x40008080, 0x100105c, 0x24835068ce00, 0x20005040c800, 0x200000400800, 0x104d04929c, 0x100904325c, 0x40008000, 0x25da9e77daf0, 0x100115c, 0x1184e1696f0, 0x24835068cc00, 0x24825169dd5c, 0x20005040cc00, 0x3ea3241c60c0, 0x200000400000, 0x211c4e5496f0, 0x104d04829c, 0x20005340d86c, 0x100904125c, 0x24835968de5c, 0x4000c000, 0x6400a0c0, 0x25da9e775af0, 0x118cf1687ac, 0x101115c, 0x1ea1745cacc0, 0x1184e1496f0, 0x20181e445af0, 0x2483506ccc00, 0x20240060c0, 0x24825161dd5c, 0x1e21755dbd9c, 0x20005050cc00, 0x26a3746cacc0, 0x3ea3243c60c0, 0xea3243c60c0, 0x200000000000, 0}); +typedef Field<uint64_t, 46, 3, StatTable46, DynTable46, &SQR_TABLE_46, &QRT_TABLE_46> Field46; +#endif + +#ifdef ENABLE_FIELD_INT_47 +// 47 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5> StatTable47; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3> DynTable47; +constexpr StatTable47 SQR_TABLE_47({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x42, 0x108, 0x420, 0x1080, 0x4200, 0x10800, 0x42000, 0x108000, 0x420000, 0x1080000, 0x4200000, 0x10800000, 0x42000000, 0x108000000, 0x420000000, 0x1080000000, 0x4200000000, 0x10800000000, 0x42000000000, 0x108000000000, 0x420000000000, 0x80000000042, 0x200000000108}); +constexpr StatTable47 QRT_TABLE_47({0, 0x1001040, 0x1001042, 0x1047043076, 0x1001046, 0x112471c241e, 0x104704307e, 0x4304e052168, 0x1001056, 0x10004000, 0x112471c243e, 0x172a09c949d6, 0x104704303e, 0x4002020, 0x4304e0521e8, 0x5400e220, 0x1001156, 0x172b08c85080, 0x10004200, 0x41200b0800, 0x112471c203e, 0x172f0cca50a0, 0x172a09c941d6, 0x7eb88a11c1d6, 0x104704203e, 0x1044042020, 0x4000020, 0x42001011156, 0x4304e0561e8, 0x172a28c95880, 0x54006220, 0x112931cc21e, 0x1011156, 0x53670f283e, 0x172b08ca5080, 0x7a80c414a03e, 0x10044200, 0x40000000000, 0x4120030800, 0x1928318801e, 0x112470c203e, 0x799283188000, 0x172f0cea50a0, 0x1eb88a91c1c8, 0x172a098941d6, 0x3ea8cc95e1f6, 0x7eb88a91c1d6}); +typedef Field<uint64_t, 47, 33, StatTable47, DynTable47, &SQR_TABLE_47, &QRT_TABLE_47> Field47; +#endif + +#ifdef ENABLE_FIELD_INT_48 +// 48 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6> StatTable48; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4> DynTable48; +constexpr StatTable48 SQR_TABLE_48({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x2d, 0xb4, 0x2d0, 0xb40, 0x2d00, 0xb400, 0x2d000, 0xb4000, 0x2d0000, 0xb40000, 0x2d00000, 0xb400000, 0x2d000000, 0xb4000000, 0x2d0000000, 0xb40000000, 0x2d00000000, 0xb400000000, 0x2d000000000, 0xb4000000000, 0x2d0000000000, 0xb40000000000, 0xd0000000005a, 0x40000000011f}); +constexpr StatTable48 QRT_TABLE_48({0xc00442c284f0, 0xc16b7fda410a, 0xc16b7fda4108, 0xada3b5c79fbe, 0xc16b7fda410c, 0x16f3c18d5b0, 0xada3b5c79fb6, 0x7090a381f64, 0xc16b7fda411c, 0xcafc15d179f8, 0x16f3c18d590, 0x6630880e534e, 0xada3b5c79ff6, 0xa13dd1f49826, 0x7090a381fe4, 0xb87560f6a74, 0xc16b7fda401c, 0xaaaaffff0012, 0xcafc15d17bf8, 0xaafd15f07bf6, 0x16f3c18d190, 0x60000020000e, 0x6630880e5b4e, 0xcb977fcb401c, 0xada3b5c78ff6, 0x6663420cad0, 0xa13dd1f4b826, 0xc0045fc2f41c, 0x7090a385fe4, 0x6762e24b834, 0xb87560fea74, 0xc6351fed241c, 0xc16b7fdb401c, 0x60065622ea7a, 0xaaaafffd0012, 0xdf9562bea74, 0xcafc15d57bf8, 0x6657ea057bea, 0xaafd15f87bf6, 0xa79329ddaa66, 0x16f3c08d190, 0xa39229f0aa66, 0x60000000000e, 0x175fb4468ad0, 0x6630884e5b4e, 0, 0xcb977f4b401c, 0x2630884e5b40}); +typedef Field<uint64_t, 48, 45, StatTable48, DynTable48, &SQR_TABLE_48, &QRT_TABLE_48> Field48; +#endif +} + +Sketch* ConstructGeneric6Bytes(int bits, int implementation) +{ + switch (bits) { +#ifdef ENABLE_FIELD_INT_41 + case 41: return new SketchImpl<Field41>(implementation, 41); +#endif +#ifdef ENABLE_FIELD_INT_42 + case 42: return new SketchImpl<Field42>(implementation, 42); +#endif +#ifdef ENABLE_FIELD_INT_43 + case 43: return new SketchImpl<Field43>(implementation, 43); +#endif +#ifdef ENABLE_FIELD_INT_44 + case 44: return new SketchImpl<Field44>(implementation, 44); +#endif +#ifdef ENABLE_FIELD_INT_45 + case 45: return new SketchImpl<Field45>(implementation, 45); +#endif +#ifdef ENABLE_FIELD_INT_46 + case 46: return new SketchImpl<Field46>(implementation, 46); +#endif +#ifdef ENABLE_FIELD_INT_47 + case 47: return new SketchImpl<Field47>(implementation, 47); +#endif +#ifdef ENABLE_FIELD_INT_48 + case 48: return new SketchImpl<Field48>(implementation, 48); +#endif + default: return nullptr; + } +} diff --git a/src/minisketch/src/fields/generic_7bytes.cpp b/src/minisketch/src/fields/generic_7bytes.cpp new file mode 100644 index 0000000000..8222f37a64 --- /dev/null +++ b/src/minisketch/src/fields/generic_7bytes.cpp @@ -0,0 +1,124 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_7) + +#include "generic_common_impl.h" + +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_49 +// 49 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 5, 5, 5, 5, 5> StatTable49; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3> DynTable49; +constexpr StatTable49 SQR_TABLE_49({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x402, 0x1008, 0x4020, 0x10080, 0x40200, 0x100800, 0x402000, 0x1008000, 0x4020000, 0x10080000, 0x40200000, 0x100800000, 0x402000000, 0x1008000000, 0x4020000000, 0x10080000000, 0x40200000000, 0x100800000000, 0x402000000000, 0x1008000000000, 0x20000000402, 0x80000001008, 0x200000004020, 0x800000010080}); +constexpr StatTable49 QRT_TABLE_49({0, 0x10004196, 0x10004194, 0x5099461f080, 0x10004190, 0x40840600c20, 0x5099461f088, 0x58a56349cfde, 0x10004180, 0x48641a0c03fe, 0x40840600c00, 0x10084002848, 0x5099461f0c8, 0x4002048, 0x58a56349cf5e, 0x5088460a048, 0x10004080, 0x4c2852624dde, 0x48641a0c01fe, 0x14893129c280, 0x40840600800, 0x1eb23c323ace8, 0x10084002048, 0x48740a09417e, 0x5099461e0c8, 0x40852604d96, 0x4000048, 0x5cad2b29c37e, 0x58a563498f5e, 0x20000200, 0x50884602048, 0x10000000000, 0x10014080, 0x4c2a56624d96, 0x4c2852604dde, 0x1ee2347438ca0, 0x48641a0801fe, 0x480000000048, 0x14893121c280, 0x14091121c080, 0x40840700800, 0x1a5099561e17e, 0x1eb23c303ace8, 0x8740a894136, 0x10084402048, 0x18101c501ace8, 0x48740a89417e, 0x15dace6286f96, 0x5099561e0c8}); +typedef Field<uint64_t, 49, 513, StatTable49, DynTable49, &SQR_TABLE_49, &QRT_TABLE_49> Field49; +#endif + +#ifdef ENABLE_FIELD_INT_50 +// 50 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5, 5, 5, 5> StatTable50; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3> DynTable50; +constexpr StatTable50 SQR_TABLE_50({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x1d, 0x74, 0x1d0, 0x740, 0x1d00, 0x7400, 0x1d000, 0x74000, 0x1d0000, 0x740000, 0x1d00000, 0x7400000, 0x1d000000, 0x74000000, 0x1d0000000, 0x740000000, 0x1d00000000, 0x7400000000, 0x1d000000000, 0x74000000000, 0x1d0000000000, 0x740000000000, 0x1d00000000000, 0x340000000001d, 0x1000000000053}); +constexpr StatTable50 QRT_TABLE_50({0xfbdfa3ae9d4c, 0x38143245a4878, 0x38143245a487a, 0x38527487e7492, 0x38143245a487e, 0x3124c61f56d2a, 0x38527487e749a, 0xfa8c91b087c0, 0x38143245a486e, 0x3eca48c6196be, 0x3124c61f56d0a, 0x380000040080a, 0x38527487e74da, 0x976b2d8b39b4, 0xfa8c91b08740, 0xfa8cd5b02724, 0x38143245a496e, 0x316291dd013fe, 0x3eca48c6194be, 0x10344122064, 0x3124c61f5690a, 0x68c5f006ee40, 0x380000040000a, 0x852749fe64d0, 0x38527487e64da, 0x37ef8e9d0e9da, 0x976b2d8b19b4, 0x37fabd1cef34a, 0xfa8c91b0c740, 0x96282d9159b4, 0xfa8cd5b0a724, 0x464a8249dd0, 0x38143245b496e, 0x37eaa8ddc94be, 0x316291dd213fe, 0x392446035690a, 0x3eca48c6594be, 0x974b258b4964, 0x103441a2064, 0x385a7c87fb4da, 0x3124c61e5690a, 0xeb8ad5d9a724, 0x68c5f026ee40, 0x3724c61e5690a, 0x380000000000a, 0x3a8c5f026ee4a, 0x8527497e64d0, 0, 0x38527497e64da, 0x2fbdfa2ae8d0a}); +typedef Field<uint64_t, 50, 29, StatTable50, DynTable50, &SQR_TABLE_50, &QRT_TABLE_50> Field50; +#endif + +#ifdef ENABLE_FIELD_INT_51 +// 51 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5, 5, 5> StatTable51; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3> DynTable51; +constexpr StatTable51 SQR_TABLE_51({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x96, 0x258, 0x960, 0x2580, 0x9600, 0x25800, 0x96000, 0x258000, 0x960000, 0x2580000, 0x9600000, 0x25800000, 0x96000000, 0x258000000, 0x960000000, 0x2580000000, 0x9600000000, 0x25800000000, 0x96000000000, 0x258000000000, 0x960000000000, 0x2580000000000, 0x160000000004b, 0x580000000012c, 0x6000000000426}); +constexpr StatTable51 QRT_TABLE_51({0x778bf2703d152, 0x2aaaafbff2092, 0x2aaaafbff2090, 0x4d2119c7e7780, 0x2aaaafbff2094, 0x65de1df8ae194, 0x4d2119c7e7788, 0x67d63d7ba262c, 0x2aaaafbff2084, 0x28ff003f4167c, 0x65de1df8ae1b4, 0x658397fb1d034, 0x4d2119c7e77c8, 0x4d7c9284526ba, 0x67d63d7ba26ac, 0x6666333fc0cbe, 0x2aaaafbff2184, 0x295b807ab55ee, 0x28ff003f4147c, 0x2aaabfffe0016, 0x65de1df8ae5b4, 0x209210349d60, 0x658397fb1d834, 0x4d215dc7cf1c8, 0x4d2119c7e67c8, 0x662b2b3d7b4be, 0x4d7c9284506ba, 0x255af00b36e0, 0x67d63d7ba66ac, 0x65de1fb8ac1a6, 0x6666333fc8cbe, 0x662f3b3ded4be, 0x2aaaafbfe2184, 0x663a9dbc3a426, 0x295b807a955ee, 0x4cdc9ec128928, 0x28ff003f0147c, 0x28a0c93cd511c, 0x2aaabfff60016, 0x65d73cf8e78d4, 0x65de1df9ae5b4, 0x4d5eddc44f1c8, 0x209210149d60, 0x357fcc506c8a, 0x658397ff1d834, 0, 0x4d215dcfcf1c8, 0x63f536f5d4554, 0x4d2119d7e67c8, 0x4000000000022, 0x662b2b1d7b4be}); +typedef Field<uint64_t, 51, 75, StatTable51, DynTable51, &SQR_TABLE_51, &QRT_TABLE_51> Field51; +#endif + +#ifdef ENABLE_FIELD_INT_52 +// 52 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5, 5> StatTable52; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4> DynTable52; +constexpr StatTable52 SQR_TABLE_52({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x9, 0x24, 0x90, 0x240, 0x900, 0x2400, 0x9000, 0x24000, 0x90000, 0x240000, 0x900000, 0x2400000, 0x9000000, 0x24000000, 0x90000000, 0x240000000, 0x900000000, 0x2400000000, 0x9000000000, 0x24000000000, 0x90000000000, 0x240000000000, 0x900000000000, 0x2400000000000, 0x9000000000000, 0x4000000000012}); +constexpr StatTable52 QRT_TABLE_52({0xc108165459b0e, 0x10004086, 0x10004084, 0xc00000100104e, 0x10004080, 0x2041810a545b0, 0xc000001001046, 0x1181e055efc0, 0x10004090, 0x40810214390, 0x2041810a54590, 0xc000141019106, 0xc000001001006, 0x10816045ab40, 0x1181e055ef40, 0xc000111015196, 0x10004190, 0xe045c19af44a2, 0x40810214190, 0xe045809ad0532, 0x2041810a54190, 0xdb387a03fe646, 0xc000141019906, 0x2000000800000, 0xc000001000006, 0x2486548199c34, 0x108160458b40, 0x2041808a50534, 0x1181e055af40, 0xc0408312153d6, 0xc00011101d196, 0x21499f0e0eed0, 0x10014190, 0xe15dff9faabe2, 0xe045c19ad44a2, 0xdb787b01ea7d6, 0x40810254190, 0xe484409180532, 0xe045809a50532, 0xc14095164d896, 0x2041810b54190, 0x217dee8fb7a74, 0xdb387a01fe646, 0x441810b54190, 0xc000141419906, 0xc3386e15e7f46, 0x2000000000000, 0x1000141419900, 0xc000000000006, 0, 0x248654a199c34, 0xa48654a199c32}); +typedef Field<uint64_t, 52, 9, StatTable52, DynTable52, &SQR_TABLE_52, &QRT_TABLE_52> Field52; +#endif + +#ifdef ENABLE_FIELD_INT_53 +// 53 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 5> StatTable53; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3> DynTable53; +constexpr StatTable53 SQR_TABLE_53({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x8e, 0x238, 0x8e0, 0x2380, 0x8e00, 0x23800, 0x8e000, 0x238000, 0x8e0000, 0x2380000, 0x8e00000, 0x23800000, 0x8e000000, 0x238000000, 0x8e0000000, 0x2380000000, 0x8e00000000, 0x23800000000, 0x8e000000000, 0x238000000000, 0x8e0000000000, 0x2380000000000, 0x8e00000000000, 0x3800000000047, 0xe00000000011c, 0x18000000000437}); +constexpr StatTable53 QRT_TABLE_53({0xf940b90844076, 0x1f940b90844052, 0x1f940b90844050, 0x9d2a063b43e64, 0x1f940b90844054, 0x936f69323ec14, 0x9d2a063b43e6c, 0xe12270a88898, 0x1f940b90844044, 0x1f917f00bb5a3c, 0x936f69323ec34, 0x1f622df85b46ee, 0x9d2a063b43e2c, 0x9bc65ab040b66, 0xe12270a88818, 0x958330b931986, 0x1f940b90844144, 0x98e2a06e32e0, 0x1f917f00bb583c, 0x1f877970dc1024, 0x936f69323e834, 0x16cc3c9b1558c2, 0x1f622df85b4eee, 0x16de1c3351dae8, 0x9d2a063b42e2c, 0x1fecdc7855f8ee, 0x9bc65ab042b66, 0x933821b1cb6fe, 0xe12270a8c818, 0x1f675958641c0e, 0x958330b939986, 0x9d97e050e960, 0x1f940b90854144, 0x1f820fa0e38adc, 0x98e2a06c32e0, 0x1650f0e358a010, 0x1f917f00bf583c, 0x1643af4b037a3a, 0x1f877970d41024, 0x1ffe2c281d8c16, 0x936f69333e834, 0xf00d50ffccf8, 0x16cc3c9b3558c2, 0x16bc31cbca943a, 0x1f622df81b4eee, 0xa6cbd8007232, 0x16de1c33d1dae8, 0x15d2a062b42e10, 0x9d2a062b42e2c, 0x1aa77896586ca, 0x1fecdc7a55f8ee, 0, 0x9bc65af042b66}); +typedef Field<uint64_t, 53, 71, StatTable53, DynTable53, &SQR_TABLE_53, &QRT_TABLE_53> Field53; +#endif + +#ifdef ENABLE_FIELD_INT_54 +// 54 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 6> StatTable54; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3> DynTable54; +constexpr StatTable54 SQR_TABLE_54({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x201, 0x804, 0x2010, 0x8040, 0x20100, 0x80400, 0x201000, 0x804000, 0x2010000, 0x8040000, 0x20100000, 0x80400000, 0x201000000, 0x804000000, 0x2010000000, 0x8040000000, 0x20100000000, 0x80400000000, 0x201000000000, 0x804000000000, 0x2010000000000, 0x8040000000000, 0x20100000000000, 0x400000000402, 0x1000000001008, 0x4000000004020, 0x10000000010080}); +constexpr StatTable54 QRT_TABLE_54({0x201008000200, 0x26c10916494994, 0x26c10916494996, 0x40008008, 0x26c10916494992, 0x141a2434c12d12, 0x40008000, 0x36c00110594c22, 0x26c10916494982, 0x200000040200, 0x141a2434c12d32, 0x10010816104534, 0x40008040, 0x36da60b01308b2, 0x36c00110594ca2, 0x48200209000, 0x26c10916494882, 0x41b6da2d86106, 0x200000040000, 0x32db2c228965b0, 0x141a2434c12932, 0x9000000200048, 0x10010816104d34, 0x32db68b2832da4, 0x40009040, 0x40045928b4902, 0x36da60b01328b2, 0x1000040000, 0x36c00110590ca2, 0x101b69865a4120, 0x48200201000, 0x22da6434912884, 0x26c10916484882, 0x9000240208008, 0x41b6da2da6106, 0x22c14484c20180, 0x200000000000, 0x4016db29b6812, 0x32db2c228165b0, 0x9008200201048, 0x141a2434d12932, 0x32c36ca2c264b0, 0x9000000000048, 0x140a65b48a2c32, 0x10010816504d34, 0, 0x32db68b2032da4, 0x404490824814, 0x41009040, 0x14da60a4536126, 0x40045908b4902, 0x8000041009008, 0x36da60b41328b2, 0x6db68b2032c12}); +typedef Field<uint64_t, 54, 513, StatTable54, DynTable54, &SQR_TABLE_54, &QRT_TABLE_54> Field54; +#endif + +#ifdef ENABLE_FIELD_INT_55 +// 55 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5> StatTable55; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3> DynTable55; +constexpr StatTable55 SQR_TABLE_55({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x102, 0x408, 0x1020, 0x4080, 0x10200, 0x40800, 0x102000, 0x408000, 0x1020000, 0x4080000, 0x10200000, 0x40800000, 0x102000000, 0x408000000, 0x1020000000, 0x4080000000, 0x10200000000, 0x40800000000, 0x102000000000, 0x408000000000, 0x1020000000000, 0x4080000000000, 0x10200000000000, 0x40800000000000, 0x2000000000102, 0x8000000000408, 0x20000000001020}); +constexpr StatTable55 QRT_TABLE_55({0, 0x121d57b6623fde, 0x121d57b6623fdc, 0x68908340d10e00, 0x121d57b6623fd8, 0x100300510e20, 0x68908340d10e08, 0x10004096, 0x121d57b6623fc8, 0x100010000, 0x100300510e00, 0x7ea8c890a088e8, 0x68908340d10e48, 0x68809540871648, 0x10004016, 0x68808000808068, 0x121d57b6623ec8, 0x68909240d41c48, 0x100010200, 0x6884c170ad0216, 0x100300510a00, 0x68848160a50200, 0x7ea8c890a080e8, 0x7eecbca04ab4b6, 0x68908340d11e48, 0x120c54b62234c8, 0x68809540873648, 0x69929240d61c48, 0x10000016, 0x68808060800000, 0x68808000800068, 0x80000080, 0x121d57b6633ec8, 0x7ea8cb90a18ae8, 0x68909240d61c48, 0x16284090200080, 0x100050200, 0x474302a345e, 0x6884c170a50216, 0x166cbca0cab4de, 0x100300410a00, 0x1000000000000, 0x68848160850200, 0x688cc1f0a50296, 0x7ea8c890e080e8, 0x7e8cc1f0a50280, 0x7eecbca0cab4b6, 0x68000000000068, 0x68908341d11e48, 0x7880954487365e, 0x120c54b42234c8, 0x9929248d61c20, 0x68809544873648, 0x41121208561c20, 0x69929248d61c48}); +typedef Field<uint64_t, 55, 129, StatTable55, DynTable55, &SQR_TABLE_55, &QRT_TABLE_55> Field55; +#endif + +#ifdef ENABLE_FIELD_INT_56 +// 56 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5> StatTable56; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4> DynTable56; +constexpr StatTable56 SQR_TABLE_56({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x95, 0x254, 0x950, 0x2540, 0x9500, 0x25400, 0x95000, 0x254000, 0x950000, 0x2540000, 0x9500000, 0x25400000, 0x95000000, 0x254000000, 0x950000000, 0x2540000000, 0x9500000000, 0x25400000000, 0x95000000000, 0x254000000000, 0x950000000000, 0x2540000000000, 0x9500000000000, 0x25400000000000, 0x95000000000000, 0x5400000000012a, 0x5000000000043d, 0x40000000001061}); +constexpr StatTable56 QRT_TABLE_56({0x10004084, 0xd058f12fd5925e, 0xd058f12fd5925c, 0x41a60b5566d9f0, 0xd058f12fd59258, 0xbda60a142740ba, 0x41a60b5566d9f8, 0xd059f1afc5e688, 0xd058f12fd59248, 0xfc040841615a22, 0xbda60a1427409a, 0xbda60b5426c1ca, 0x41a60b5566d9b8, 0x1a60b4166b950, 0xd059f1afc5e608, 0xfc000041409822, 0xd058f12fd59348, 0xd1ee7a4ef4185c, 0xfc040841615822, 0x9049759b80b4a4, 0xbda60a1427449a, 0xd258e06f301e18, 0xbda60b5426c9ca, 0x6dfeeb3bf6d7d2, 0x41a60b5566c9b8, 0xbdef3ed4ae398a, 0x1a60b41669950, 0xd1ef3f8eeff04c, 0xd059f1afc5a608, 0xbda203340783de, 0xfc000041401822, 0x2dfefbaff2b27a, 0xd058f12fd49348, 0xfdb788a0706776, 0xd1ee7a4ef6185c, 0x2e5de0ae41337a, 0xfc040841655822, 0x41eb17d5ceecf8, 0x9049759b88b4a4, 0x40048874211afc, 0xbda60a1437449a, 0xd04a720f93400c, 0xd258e06f101e18, 0xbc559cf5ac7fce, 0xbda60b5466c9ca, 0x6dc9759b88b4d6, 0x6dfeeb3b76d7d2, 0x92feea7b275af0, 0x41a60b5466c9b8, 0, 0xbdef3ed6ae398a, 0x2811d5edd8ee2a, 0x1a60b45669950, 0xb1a60b5466c9ca, 0xd1ef3f86eff04c, 0xec493582c8f032}); +typedef Field<uint64_t, 56, 149, StatTable56, DynTable56, &SQR_TABLE_56, &QRT_TABLE_56> Field56; +#endif +} + +Sketch* ConstructGeneric7Bytes(int bits, int implementation) +{ + switch (bits) { +#ifdef ENABLE_FIELD_INT_49 + case 49: return new SketchImpl<Field49>(implementation, 49); +#endif +#ifdef ENABLE_FIELD_INT_50 + case 50: return new SketchImpl<Field50>(implementation, 50); +#endif +#ifdef ENABLE_FIELD_INT_51 + case 51: return new SketchImpl<Field51>(implementation, 51); +#endif +#ifdef ENABLE_FIELD_INT_52 + case 52: return new SketchImpl<Field52>(implementation, 52); +#endif +#ifdef ENABLE_FIELD_INT_53 + case 53: return new SketchImpl<Field53>(implementation, 53); +#endif +#ifdef ENABLE_FIELD_INT_54 + case 54: return new SketchImpl<Field54>(implementation, 54); +#endif +#ifdef ENABLE_FIELD_INT_55 + case 55: return new SketchImpl<Field55>(implementation, 55); +#endif +#ifdef ENABLE_FIELD_INT_56 + case 56: return new SketchImpl<Field56>(implementation, 56); +#endif + default: return nullptr; + } +} diff --git a/src/minisketch/src/fields/generic_8bytes.cpp b/src/minisketch/src/fields/generic_8bytes.cpp new file mode 100644 index 0000000000..8bb63e8d3e --- /dev/null +++ b/src/minisketch/src/fields/generic_8bytes.cpp @@ -0,0 +1,124 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +/* This file was substantially auto-generated by doc/gen_params.sage. */ +#include "../fielddefines.h" + +#if defined(ENABLE_FIELD_BYTES_INT_8) + +#include "generic_common_impl.h" + +#include "../lintrans.h" +#include "../sketch_impl.h" + +#endif + +#include "../sketch.h" + +namespace { +#ifdef ENABLE_FIELD_INT_57 +// 57 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5> StatTable57; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3> DynTable57; +constexpr StatTable57 SQR_TABLE_57({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x22, 0x88, 0x220, 0x880, 0x2200, 0x8800, 0x22000, 0x88000, 0x220000, 0x880000, 0x2200000, 0x8800000, 0x22000000, 0x88000000, 0x220000000, 0x880000000, 0x2200000000, 0x8800000000, 0x22000000000, 0x88000000000, 0x220000000000, 0x880000000000, 0x2200000000000, 0x8800000000000, 0x22000000000000, 0x88000000000000, 0x20000000000011, 0x80000000000044}); +constexpr StatTable57 QRT_TABLE_57({0xd0c3a82c902426, 0x232aa54103915e, 0x232aa54103915c, 0x1763e291e61699c, 0x232aa541039158, 0x1f424d678bb15e, 0x1763e291e616994, 0x26fd8122f10d36, 0x232aa541039148, 0x1e0a0206002000, 0x1f424d678bb17e, 0x5d72563f39d7e, 0x1763e291e6169d4, 0x1519beb9d597df4, 0x26fd8122f10db6, 0x150c3a87c90e4aa, 0x232aa541039048, 0x15514891f6179d4, 0x1e0a0206002200, 0x14ec9ba7a94c6aa, 0x1f424d678bb57e, 0x1e0f4286382420, 0x5d72563f3957e, 0x4000080000, 0x1763e291e6179d4, 0x1ac0e804882000, 0x1519beb9d595df4, 0x1f430d6793b57e, 0x26fd8122f14db6, 0x3c68e806882000, 0x150c3a87c9064aa, 0x1484fe18b915e, 0x232aa541029048, 0x14f91eb9b595df4, 0x15514891f6379d4, 0x48f6a82380420, 0x1e0a0206042200, 0x14b1beb99595df4, 0x14ec9ba7a9cc6aa, 0x4cf2a82b00420, 0x1f424d679bb57e, 0x26aa0002000000, 0x1e0f4286182420, 0x173f1039dd17df4, 0x5d72563b3957e, 0x4aa0002000000, 0x4000880000, 0x16d31eb9b595df4, 0x1763e291f6179d4, 0x20000000000000, 0x1ac0e806882000, 0x2caa0002000000, 0x1519beb99595df4, 0, 0x1f430d6f93b57e, 0x73e90d6d93b57e, 0x26fd8132f14db6}); +typedef Field<uint64_t, 57, 17, StatTable57, DynTable57, &SQR_TABLE_57, &QRT_TABLE_57> Field57; +#endif + +#ifdef ENABLE_FIELD_INT_58 +// 58 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5> StatTable58; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3> DynTable58; +constexpr StatTable58 SQR_TABLE_58({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x80001, 0x200004, 0x800010, 0x2000040, 0x8000100, 0x20000400, 0x80001000, 0x200004000, 0x800010000, 0x2000040000, 0x8000100000, 0x20000400000, 0x80001000000, 0x200004000000, 0x800010000000, 0x2000040000000, 0x8000100000000, 0x20000400000000, 0x80001000000000, 0x200004000000000, 0x10000100002, 0x40000400008, 0x100001000020, 0x400004000080, 0x1000010000200, 0x4000040000800, 0x10000100002000, 0x40000400008000, 0x100001000020000}); +constexpr StatTable58 QRT_TABLE_58({0x2450096792a5c5c, 0x610014271011c, 0x610014271011e, 0x1f0cb811314ea88, 0x610014271011a, 0x8000000420, 0x1f0cb811314ea80, 0x265407ad8a20bcc, 0x610014271010a, 0x3d18be98392ebd0, 0x8000000400, 0xc29b930e407056, 0x1f0cb811314eac0, 0x1fcef001154dee8, 0x265407ad8a20b4c, 0xc69b924c61f94a, 0x610014271000a, 0x211006895845190, 0x3d18be98392e9d0, 0x54007accac09cc, 0x8000000000, 0xc08b934e107854, 0xc29b930e407856, 0x275407adc220bcc, 0x1f0cb811314fac0, 0x1f6db815164ea8a, 0x1fcef001154fee8, 0x1b2db801945e396, 0x265407ad8a24b4c, 0x21100ec95865590, 0xc69b924c61794a, 0x273507b1e530ad6, 0x610014270000a, 0x1b4cb835b34e29c, 0x211006895865190, 0x3839bf20d47e016, 0x3d18be98396e9d0, 0x3858bd34f36e01c, 0x54007acca409cc, 0, 0x8000100000, 0xc29a130e507856, 0xc08b934e307854, 0x13253921d448296, 0xc29b930e007856, 0x13c60935f6486bc, 0x275407adca20bcc, 0x3571be8c5e6c9da, 0x1f0cb811214fac0, 0x410014261011c, 0x1f6db815364ea8a, 0x13a50921d1486b6, 0x1fcef001554fee8, 0x64001249245a5c, 0x1b2db801145e396, 0x8610014670200a, 0x265407ac8a24b4c, 0x1a5cbfbdeb0f30c}); +typedef Field<uint64_t, 58, 524289, StatTable58, DynTable58, &SQR_TABLE_58, &QRT_TABLE_58> Field58; +#endif + +#ifdef ENABLE_FIELD_INT_59 +// 59 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5> StatTable59; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3> DynTable59; +constexpr StatTable59 SQR_TABLE_59({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x12a, 0x4a8, 0x12a0, 0x4a80, 0x12a00, 0x4a800, 0x12a000, 0x4a8000, 0x12a0000, 0x4a80000, 0x12a00000, 0x4a800000, 0x12a000000, 0x4a8000000, 0x12a0000000, 0x4a80000000, 0x12a00000000, 0x4a800000000, 0x12a000000000, 0x4a8000000000, 0x12a0000000000, 0x4a80000000000, 0x12a00000000000, 0x4a800000000000, 0x12a000000000000, 0x4a8000000000000, 0x2a000000000012a, 0x28000000000043d, 0x200000000001061}); +constexpr StatTable59 QRT_TABLE_59({0x38d905ab028567a, 0x789fa6ed3b44d72, 0x789fa6ed3b44d70, 0x74ec857e93d828c, 0x789fa6ed3b44d74, 0x116b3c1203c96, 0x74ec857e93d8284, 0xc25ebc3871e280, 0x789fa6ed3b44d64, 0x47a37c3d910b6, 0x116b3c1203cb6, 0xc7322d7a8f48de, 0x74ec857e93d82c4, 0xb509a0ea52e496, 0xc25ebc3871e200, 0x74fdee4681d3e0c, 0x789fa6ed3b44c64, 0x7ffbbd080b2f09a, 0x47a37c3d912b6, 0xd5c937bae506c8, 0x116b3c12038b6, 0xb173c76987625e, 0xc7322d7a8f40de, 0x7591ff36b3a682c, 0x74ec857e93d92c4, 0x72b253bfbfc90c4, 0xb509a0ea52c496, 0x79f2e7b10e6d452, 0xc25ebc3871a200, 0x78c86e951086aac, 0x74fdee4681dbe0c, 0x78c96eb514c602c, 0x789fa6ed3b54c64, 0xc34818b95658e8, 0x7ffbbd080b0f09a, 0x7399f563b1980f2, 0x47a37c3dd12b6, 0xa29e0e28c58880, 0xd5c937baed06c8, 0x788ac23520ac82c, 0x116b3c13038b6, 0xa2c857e83d92b6, 0xb173c769a7625e, 0x608da990122e48, 0xc7322d7acf40de, 0xa3a89269eebefe, 0x7591ff36bba682c, 0xa25ebc2871a200, 0x74ec857e83d92c4, 0x11f62e419f1cfe, 0x72b253bf9fc90c4, 0x7425ebc2871a272, 0xb509a0ee52c496, 0x4ed8555979c8de, 0x79f2e7b18e6d452, 0x6c3580d5915d4d2, 0xc25ebc2871a200, 0, 0x78c86e971086aac}); +typedef Field<uint64_t, 59, 149, StatTable59, DynTable59, &SQR_TABLE_59, &QRT_TABLE_59> Field59; +#endif + +#ifdef ENABLE_FIELD_INT_60 +// 60 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6> StatTable60; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4> DynTable60; +constexpr StatTable60 SQR_TABLE_60({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x3, 0xc, 0x30, 0xc0, 0x300, 0xc00, 0x3000, 0xc000, 0x30000, 0xc0000, 0x300000, 0xc00000, 0x3000000, 0xc000000, 0x30000000, 0xc0000000, 0x300000000, 0xc00000000, 0x3000000000, 0xc000000000, 0x30000000000, 0xc0000000000, 0x300000000000, 0xc00000000000, 0x3000000000000, 0xc000000000000, 0x30000000000000, 0xc0000000000000, 0x300000000000000, 0xc00000000000000}); +constexpr StatTable60 QRT_TABLE_60({0x6983c00fe00104a, 0x804570322e054e6, 0x804570322e054e4, 0x15673387e0a4e4, 0x804570322e054e0, 0x100010110, 0x15673387e0a4ec, 0x920d01f34442a70, 0x804570322e054f0, 0x7a8dc0f2e4058f0, 0x100010130, 0x120c01f140462f0, 0x15673387e0a4ac, 0x7bdbb2ca9a4fe5c, 0x920d01f34442af0, 0xe9c6b039ce0c4ac, 0x804570322e055f0, 0xfac8b080ca20c00, 0x7a8dc0f2e405af0, 0x7a8dc4b2e4a59f0, 0x100010530, 0x10000100000, 0x120c01f14046af0, 0x131a02d91c5db6c, 0x15673387e0b4ac, 0x15623387d0b4ac, 0x7bdbb2ca9a4de5c, 0x7ffbbbca0a8ee5c, 0x920d01f34446af0, 0x800000020000000, 0xe9c6b039ce044ac, 0x81130302500f000, 0x804570322e155f0, 0x935b72eb3a48e9c, 0xfac8b080ca00c00, 0x120c016140563c0, 0x7a8dc0f2e445af0, 0x7bcbb3ca8a4ee5c, 0x7a8dc4b2e4259f0, 0xc4000a0300, 0x100110530, 0x11623285c1b19c, 0x10000300000, 0x420890090c3000, 0x120c01f14446af0, 0x68d7b33b9e0b4ac, 0x131a02d9145db6c, 0xe8ccb1e18a56fc0, 0x15673386e0b4ac, 0x7aadc8f2e485af0, 0x15623385d0b4ac, 0x4a0990093c3000, 0x7bdbb2cada4de5c, 0xf9d6b3389e0b4ac, 0x7ffbbbca8a8ee5c, 0xdf6ba38cec84ac, 0x920d01f24446af0, 0x520d01f24446af0, 0x800000000000000, 0}); +typedef Field<uint64_t, 60, 3, StatTable60, DynTable60, &SQR_TABLE_60, &QRT_TABLE_60> Field60; +#endif + +#ifdef ENABLE_FIELD_INT_61 +// 61 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5> StatTable61; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3> DynTable61; +constexpr StatTable61 SQR_TABLE_61({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x1000000000000000, 0x4e, 0x138, 0x4e0, 0x1380, 0x4e00, 0x13800, 0x4e000, 0x138000, 0x4e0000, 0x1380000, 0x4e00000, 0x13800000, 0x4e000000, 0x138000000, 0x4e0000000, 0x1380000000, 0x4e00000000, 0x13800000000, 0x4e000000000, 0x138000000000, 0x4e0000000000, 0x1380000000000, 0x4e00000000000, 0x13800000000000, 0x4e000000000000, 0x138000000000000, 0x4e0000000000000, 0x1380000000000000, 0xe0000000000004e, 0x180000000000011f}); +constexpr StatTable61 QRT_TABLE_61({0x171d34fcdac955d0, 0x12cfc8c049e1c96, 0x12cfc8c049e1c94, 0x71d34fcdac955c2, 0x12cfc8c049e1c90, 0x631c871de564852, 0x71d34fcdac955ca, 0x129fa6407f27300, 0x12cfc8c049e1c80, 0x7094f6fdd0a3b12, 0x631c871de564872, 0xdb28cee59c8256a, 0x71d34fcdac9558a, 0xc8a0be15a915472, 0x129fa6407f27380, 0x12dfcb4058e0b80, 0x12cfc8c049e1d80, 0x117d7f04ad0118, 0x7094f6fdd0a3912, 0x621b576dbe35b6a, 0x631c871de564c72, 0x13c808a013a1ee0, 0xdb28cee59c82d6a, 0x113d79842a0272, 0x71d34fcdac9458a, 0x719776b580b6a98, 0xc8a0be15a917472, 0x6633498d6db760a, 0x129fa6407f23380, 0xbd4ae9e8c3e7560, 0x12dfcb4058e8b80, 0x8000000a, 0x12cfc8c049f1d80, 0x634ce9add3b26ea, 0x117d7f04af0118, 0xda3f19c5d66258a, 0x7094f6fdd0e3912, 0xb87427e85e71560, 0x621b576dbeb5b6a, 0xc8b0b085b8c4e0a, 0x631c871de464c72, 0x1538fc8649458a, 0x13c808a011a1ee0, 0xcddbca6d1cfe360, 0xdb28cee59882d6a, 0xae80f550d1ffff2, 0x113d7984aa0272, 0xda7770f5f195912, 0x71d34fcdbc9458a, 0x137c8a049a1ee0, 0x719776b5a0b6a98, 0xded39a9d236ba78, 0xc8a0be15e917472, 0x6732488ca7ce0a, 0x6633498dedb760a, 0xc0406d0527cb80a, 0x129fa6417f23380, 0x3d4ae9eac3e756a, 0xbd4ae9eac3e7560, 0, 0x12dfcb4458e8b80}); +typedef Field<uint64_t, 61, 39, StatTable61, DynTable61, &SQR_TABLE_61, &QRT_TABLE_61> Field61; +#endif + +#ifdef ENABLE_FIELD_INT_62 +// 62 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 5> StatTable62; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3> DynTable62; +constexpr StatTable62 SQR_TABLE_62({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x1000000000000000, 0x20000001, 0x80000004, 0x200000010, 0x800000040, 0x2000000100, 0x8000000400, 0x20000001000, 0x80000004000, 0x200000010000, 0x800000040000, 0x2000000100000, 0x8000000400000, 0x20000001000000, 0x80000004000000, 0x200000010000000, 0x800000040000000, 0x2000000100000000, 0x440000002, 0x1100000008, 0x4400000020, 0x11000000080, 0x44000000200, 0x110000000800, 0x440000002000, 0x1100000008000, 0x4400000020000, 0x11000000080000, 0x44000000200000, 0x110000000800000, 0x440000002000000, 0x1100000008000000}); +constexpr StatTable62 QRT_TABLE_62({0x30268b6fba455d2c, 0x200000006, 0x200000004, 0x3d67cb6c1fe66c76, 0x200000000, 0x3fc4f1901abfa400, 0x3d67cb6c1fe66c7e, 0x35e79b6c0a66bcbe, 0x200000010, 0x1e9372bc57a9941e, 0x3fc4f1901abfa420, 0x21ec9d424957a5b0, 0x3d67cb6c1fe66c3e, 0x1cb35a6e52f5fb0e, 0x35e79b6c0a66bc3e, 0x215481024c13a730, 0x200000110, 0x1c324a6c52f75b08, 0x1e9372bc57a9961e, 0x3764a9d00f676820, 0x3fc4f1901abfa020, 0x355481020e132730, 0x21ec9d424957adb0, 0x3c43c32c0f34301e, 0x3d67cb6c1fe67c3e, 0x1496122c45259728, 0x1cb35a6e52f5db0e, 0x15e418405b72ec20, 0x35e79b6c0a66fc3e, 0x30268b6e3a445c38, 0x215481024c132730, 0x100010114, 0x200010110, 0, 0x1c324a6c52f55b08, 0x215581044d133776, 0x1e9372bc57ad961e, 0x2155810e4d133766, 0x3764a9d00f6f6820, 0x2157833c4d12323e, 0x3fc4f1901aafa020, 0x1c324a4252f55b58, 0x355481020e332730, 0x28332fc0509d41e, 0x21ec9d424917adb0, 0x215783be4d12332e, 0x3c43c32c0fb4301e, 0x2157822c4d06363e, 0x3d67cb6c1ee67c3e, 0x23f6b9d2484afb78, 0x1496122c47259728, 0x14b8184047648a80, 0x1cb35a6e56f5db0e, 0x3fe4f1901aefa820, 0x15e418405372ec20, 0x3d5fd72c1be276be, 0x35e79b6c1a66fc3e, 0x14b038d24774cf10, 0x30268b6e1a445c38, 0x1d17022e43a7172e, 0x215481020c132730, 0x2157022e4d07372e}); +typedef Field<uint64_t, 62, 536870913, StatTable62, DynTable62, &SQR_TABLE_62, &QRT_TABLE_62> Field62; +#endif + +#ifdef ENABLE_FIELD_INT_63 +// 63 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5> StatTable63; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 3> DynTable63; +constexpr StatTable63 SQR_TABLE_63({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x1000000000000000, 0x4000000000000000, 0x6, 0x18, 0x60, 0x180, 0x600, 0x1800, 0x6000, 0x18000, 0x60000, 0x180000, 0x600000, 0x1800000, 0x6000000, 0x18000000, 0x60000000, 0x180000000, 0x600000000, 0x1800000000, 0x6000000000, 0x18000000000, 0x60000000000, 0x180000000000, 0x600000000000, 0x1800000000000, 0x6000000000000, 0x18000000000000, 0x60000000000000, 0x180000000000000, 0x600000000000000, 0x1800000000000000, 0x6000000000000000}); +constexpr StatTable63 QRT_TABLE_63({0, 0x100010114, 0x100010116, 0x1001701051372, 0x100010112, 0x1000040220, 0x100170105137a, 0x5107703453bba, 0x100010102, 0x101130117155a, 0x1000040200, 0x40000200800, 0x100170105133a, 0x103151a137276d8, 0x5107703453b3a, 0x134e65fc7c222be0, 0x100010002, 0x100030103115a, 0x101130117175a, 0x106052d103f4de2, 0x1000040600, 0x15122707691d3a, 0x40000200000, 0x4530770bc57b3a, 0x100170105033a, 0x103011a131256d8, 0x103151a137256d8, 0x176f29eb55c7a8da, 0x5107703457b3a, 0x130b158b7767d0da, 0x134e65fc7c22abe0, 0x7bcaf59d2f62d3e2, 0x100000002, 0x1001401041260, 0x100030101115a, 0x5107e03443ab8, 0x101130113175a, 0x1043701251b3a, 0x106052d10374de2, 0x134e657d7c232be2, 0x1000140600, 0x106073d103b4be2, 0x15122707491d3a, 0x4438600ac07800, 0x40000600000, 0x176a199c5682d3e0, 0x4530770b457b3a, 0x7bca759c2f62d3e0, 0x100170005033a, 0x6116d02572de2, 0x103011a111256d8, 0x1346656d7c372de2, 0x103151a177256d8, 0x643c600aa07800, 0x176f29eb5dc7a8da, 0x7b4b758b2f67d0da, 0x5107713457b3a, 0x104570776b457b3a, 0x130b158b5767d0da, 0x734e65fc3c22abe0, 0x134e65fc3c22abe0, 0x4000000000000000, 0x7bcaf59daf62d3e2}); +typedef Field<uint64_t, 63, 3, StatTable63, DynTable63, &SQR_TABLE_63, &QRT_TABLE_63> Field63; +#endif + +#ifdef ENABLE_FIELD_INT_64 +// 64 bit field +typedef RecLinTrans<uint64_t, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5> StatTable64; +typedef RecLinTrans<uint64_t, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4> DynTable64; +constexpr StatTable64 SQR_TABLE_64({0x1, 0x4, 0x10, 0x40, 0x100, 0x400, 0x1000, 0x4000, 0x10000, 0x40000, 0x100000, 0x400000, 0x1000000, 0x4000000, 0x10000000, 0x40000000, 0x100000000, 0x400000000, 0x1000000000, 0x4000000000, 0x10000000000, 0x40000000000, 0x100000000000, 0x400000000000, 0x1000000000000, 0x4000000000000, 0x10000000000000, 0x40000000000000, 0x100000000000000, 0x400000000000000, 0x1000000000000000, 0x4000000000000000, 0x1b, 0x6c, 0x1b0, 0x6c0, 0x1b00, 0x6c00, 0x1b000, 0x6c000, 0x1b0000, 0x6c0000, 0x1b00000, 0x6c00000, 0x1b000000, 0x6c000000, 0x1b0000000, 0x6c0000000, 0x1b00000000, 0x6c00000000, 0x1b000000000, 0x6c000000000, 0x1b0000000000, 0x6c0000000000, 0x1b00000000000, 0x6c00000000000, 0x1b000000000000, 0x6c000000000000, 0x1b0000000000000, 0x6c0000000000000, 0x1b00000000000000, 0x6c00000000000000, 0xb00000000000001b, 0xc00000000000005a}); +constexpr StatTable64 QRT_TABLE_64({0x19c9369f278adc02, 0x84b2b22ab2383ee4, 0x84b2b22ab2383ee6, 0x9d7b84b495b3e3f6, 0x84b2b22ab2383ee2, 0x37c470b49213f790, 0x9d7b84b495b3e3fe, 0x1000a0105137c, 0x84b2b22ab2383ef2, 0x368e964a8edce1fc, 0x37c470b49213f7b0, 0x19c9368e278fdf4c, 0x9d7b84b495b3e3be, 0x2e4da23cbc7d4570, 0x1000a010513fc, 0x84f35772bac24232, 0x84b2b22ab2383ff2, 0x37c570ba9314e4fc, 0x368e964a8edce3fc, 0xb377c390213cdb0e, 0x37c470b49213f3b0, 0x85ed5a3aa99c24f2, 0x19c9368e278fd74c, 0xaabff0000780000e, 0x9d7b84b495b3f3be, 0x84b6b3dab03038f2, 0x2e4da23cbc7d6570, 0x511ea03494ffc, 0x1000a010553fc, 0xae0c0220343c6c0e, 0x84f35772bac2c232, 0x800000008000000e, 0x84b2b22ab2393ff2, 0xb376c29c202bc97e, 0x37c570ba9316e4fc, 0x9c3062488879e6ce, 0x368e964a8ed8e3fc, 0x41e42c08e47e70, 0xb377c3902134db0e, 0x85b9b108a60f56ce, 0x37c470b49203f3b0, 0x19dd3b6e21f3cb4c, 0x85ed5a3aa9bc24f2, 0x198ddf682c428ac0, 0x19c9368e27cfd74c, 0x4b7c68431ca84b0, 0xaabff0000700000e, 0x8040655489ffefbe, 0x9d7b84b494b3f3be, 0x18c1354e32bfa74c, 0x84b6b3dab23038f2, 0xaaf613cc0f74627e, 0x2e4da23cb87d6570, 0x3248b3d6b3342a8c, 0x511ea0b494ffc, 0xb60813c00e70700e, 0x1000a110553fc, 0x1e0d022a05393ffc, 0xae0c0220143c6c0e, 0xe0c0220143c6c00, 0x84f35772fac2c232, 0xc041e55948fbfdce, 0x800000000000000e, 0}); +typedef Field<uint64_t, 64, 27, StatTable64, DynTable64, &SQR_TABLE_64, &QRT_TABLE_64> Field64; +#endif +} + +Sketch* ConstructGeneric8Bytes(int bits, int implementation) +{ + switch (bits) { +#ifdef ENABLE_FIELD_INT_57 + case 57: return new SketchImpl<Field57>(implementation, 57); +#endif +#ifdef ENABLE_FIELD_INT_58 + case 58: return new SketchImpl<Field58>(implementation, 58); +#endif +#ifdef ENABLE_FIELD_INT_59 + case 59: return new SketchImpl<Field59>(implementation, 59); +#endif +#ifdef ENABLE_FIELD_INT_60 + case 60: return new SketchImpl<Field60>(implementation, 60); +#endif +#ifdef ENABLE_FIELD_INT_61 + case 61: return new SketchImpl<Field61>(implementation, 61); +#endif +#ifdef ENABLE_FIELD_INT_62 + case 62: return new SketchImpl<Field62>(implementation, 62); +#endif +#ifdef ENABLE_FIELD_INT_63 + case 63: return new SketchImpl<Field63>(implementation, 63); +#endif +#ifdef ENABLE_FIELD_INT_64 + case 64: return new SketchImpl<Field64>(implementation, 64); +#endif + default: return nullptr; + } +} diff --git a/src/minisketch/src/fields/generic_common_impl.h b/src/minisketch/src/fields/generic_common_impl.h new file mode 100644 index 0000000000..1d1d030b5f --- /dev/null +++ b/src/minisketch/src/fields/generic_common_impl.h @@ -0,0 +1,70 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _MINISKETCH_FIELDS_GENERIC_COMMON_IMPL_H_ +#define _MINISKETCH_FIELDS_GENERIC_COMMON_IMPL_H_ 1 + +#include <stdint.h> + +#include "../int_utils.h" +#include "../lintrans.h" + +namespace { + +/** Generic implementation for fields whose elements can be represented by an integer type. */ +template<typename I, int B, uint32_t MOD, typename F, typename T, const F* SQR, const F* QRT> class Field +{ + typedef BitsInt<I, B> O; + typedef LFSR<O, MOD> L; + +public: + typedef I Elem; + constexpr int Bits() const { return B; } + + constexpr inline Elem Mul2(Elem val) const { return L::Call(val); } + + class Multiplier + { + T table; + public: + explicit Multiplier(const Field&, Elem a) { table.template Build<L::Call>(a); } + constexpr inline Elem operator()(Elem a) const { return table.template Map<O>(a); } + }; + + Elem Mul(Elem a, Elem b) const { return GFMul<I, B, L, O>(a, b); } + + /** Compute the square of a. */ + inline constexpr Elem Sqr(Elem a) const { return SQR->template Map<O>(a); } + + /** Compute x such that x^2 + x = a (undefined result if no solution exists). */ + inline constexpr Elem Qrt(Elem a) const { return QRT->template Map<O>(a); } + + /** Compute the inverse of x1. */ + Elem Inv(Elem a) const { return InvExtGCD<I, O, B, MOD>(a); } + + /** Generate a random field element. */ + Elem FromSeed(uint64_t seed) const { + uint64_t k0 = 0x496e744669656c64ull; // "IntField" + uint64_t k1 = seed; + uint64_t count = ((uint64_t)B) << 32; + Elem ret; + do { + ret = O::Mask(I(SipHash(k0, k1, count++))); + } while(ret == 0); + return ret; + } + + Elem Deserialize(BitReader& in) const { return in.template Read<B, I>(); } + + void Serialize(BitWriter& out, Elem val) const { out.template Write<B, I>(val); } + + constexpr Elem FromUint64(uint64_t x) const { return O::Mask(I(x)); } + constexpr uint64_t ToUint64(Elem val) const { return uint64_t(val); } +}; + +} + +#endif diff --git a/src/minisketch/src/int_utils.h b/src/minisketch/src/int_utils.h new file mode 100644 index 0000000000..62b2c38a29 --- /dev/null +++ b/src/minisketch/src/int_utils.h @@ -0,0 +1,290 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _MINISKETCH_INT_UTILS_H_ +#define _MINISKETCH_INT_UTILS_H_ + +#include <stdlib.h> + +#include <limits> +#include <algorithm> +#include <type_traits> + +#ifdef _MSC_VER +# include <intrin.h> +#endif + +template<int bits> +static constexpr inline uint64_t Rot(uint64_t x) { return (x << bits) | (x >> (64 - bits)); } + +static inline void SipHashRound(uint64_t& v0, uint64_t& v1, uint64_t& v2, uint64_t& v3) { + v0 += v1; v1 = Rot<13>(v1); v1 ^= v0; + v0 = Rot<32>(v0); + v2 += v3; v3 = Rot<16>(v3); v3 ^= v2; + v0 += v3; v3 = Rot<21>(v3); v3 ^= v0; + v2 += v1; v1 = Rot<17>(v1); v1 ^= v2; + v2 = Rot<32>(v2); +} + +inline uint64_t SipHash(uint64_t k0, uint64_t k1, uint64_t data) { + uint64_t v0 = 0x736f6d6570736575ULL ^ k0; + uint64_t v1 = 0x646f72616e646f6dULL ^ k1; + uint64_t v2 = 0x6c7967656e657261ULL ^ k0; + uint64_t v3 = 0x7465646279746573ULL ^ k1 ^ data; + SipHashRound(v0, v1, v2, v3); + SipHashRound(v0, v1, v2, v3); + v0 ^= data; + v3 ^= 0x800000000000000ULL; + SipHashRound(v0, v1, v2, v3); + SipHashRound(v0, v1, v2, v3); + v0 ^= 0x800000000000000ULL; + v2 ^= 0xFF; + SipHashRound(v0, v1, v2, v3); + SipHashRound(v0, v1, v2, v3); + SipHashRound(v0, v1, v2, v3); + SipHashRound(v0, v1, v2, v3); + return v0 ^ v1 ^ v2 ^ v3; +} + +class BitWriter { + unsigned char state = 0; + int offset = 0; + unsigned char* out; + +public: + BitWriter(unsigned char* output) : out(output) {} + + template<int BITS, typename I> + inline void Write(I val) { + int bits = BITS; + if (bits + offset >= 8) { + state |= ((val & ((I(1) << (8 - offset)) - 1)) << offset); + *(out++) = state; + val >>= (8 - offset); + bits -= 8 - offset; + offset = 0; + state = 0; + } + while (bits >= 8) { + *(out++) = val & 255; + val >>= 8; + bits -= 8; + } + state |= ((val & ((I(1) << bits) - 1)) << offset); + offset += bits; + } + + inline void Flush() { + if (offset) { + *(out++) = state; + state = 0; + offset = 0; + } + } +}; + +class BitReader { + unsigned char state = 0; + int offset = 0; + const unsigned char* in; + +public: + BitReader(const unsigned char* input) : in(input) {} + + template<int BITS, typename I> + inline I Read() { + int bits = BITS; + if (offset >= bits) { + I ret = state & ((1 << bits) - 1); + state >>= bits; + offset -= bits; + return ret; + } + I val = state; + int out = offset; + while (out + 8 <= bits) { + val |= ((I(*(in++))) << out); + out += 8; + } + if (out < bits) { + unsigned char c = *(in++); + val |= (c & ((I(1) << (bits - out)) - 1)) << out; + state = c >> (bits - out); + offset = 8 - (bits - out); + } else { + state = 0; + offset = 0; + } + return val; + } +}; + +/** Return a value of type I with its `bits` lowest bits set (bits must be > 0). */ +template<int BITS, typename I> +constexpr inline I Mask() { return ((I((I(-1)) << (std::numeric_limits<I>::digits - BITS))) >> (std::numeric_limits<I>::digits - BITS)); } + +/** Compute the smallest power of two that is larger than val. */ +template<typename I> +static inline int CountBits(I val, int max) { +#ifdef HAVE_CLZ + (void)max; + if (val == 0) return 0; + if (std::numeric_limits<unsigned>::digits >= std::numeric_limits<I>::digits) { + return std::numeric_limits<unsigned>::digits - __builtin_clz(val); + } else if (std::numeric_limits<unsigned long>::digits >= std::numeric_limits<I>::digits) { + return std::numeric_limits<unsigned long>::digits - __builtin_clzl(val); + } else { + return std::numeric_limits<unsigned long long>::digits - __builtin_clzll(val); + } +#elif _MSC_VER + (void)max; + unsigned long index; + unsigned char ret; + if (std::numeric_limits<I>::digits <= 32) { + ret = _BitScanReverse(&index, val); + } else { + ret = _BitScanReverse64(&index, val); + } + if (!ret) return 0; + return index; +#else + while (max && (val >> (max - 1) == 0)) --max; + return max; +#endif +} + +template<typename I, int BITS> +class BitsInt { +private: + static_assert(std::is_integral<I>::value && std::is_unsigned<I>::value, "BitsInt requires an unsigned integer type"); + static_assert(BITS > 0 && BITS <= std::numeric_limits<I>::digits, "BitsInt requires 1 <= Bits <= representation type size"); + + static constexpr I MASK = Mask<BITS, I>(); + +public: + + typedef I Repr; + + static constexpr int SIZE = BITS; + + static void inline Swap(I& a, I& b) { + std::swap(a, b); + } + + static constexpr inline bool IsZero(I a) { return a == 0; } + static constexpr inline I Mask(I val) { return val & MASK; } + static constexpr inline I Shift(I val, int bits) { return ((val << bits) & MASK); } + static constexpr inline I UnsafeShift(I val, int bits) { return (val << bits); } + + template<int Offset, int Count> + static constexpr inline int MidBits(I val) { + static_assert(Count > 0, "BITSInt::MidBits needs Count > 0"); + static_assert(Count + Offset <= BITS, "BitsInt::MidBits overflow of Count+Offset"); + return (val >> Offset) & ((I(1) << Count) - 1); + } + + template<int Count> + static constexpr inline int TopBits(I val) { + static_assert(Count > 0, "BitsInt::TopBits needs Count > 0"); + static_assert(Count <= BITS, "BitsInt::TopBits needs Offset <= BITS"); + return val >> (BITS - Count); + } + + static inline constexpr I CondXorWith(I val, bool cond, I v) { + return val ^ (-I(cond) & v); + } + + template<I MOD> + static inline constexpr I CondXorWith(I val, bool cond) { + return val ^ (-I(cond) & MOD); + } + + static inline int Bits(I val, int max) { return CountBits<I>(val, max); } +}; + +/** Class which implements a stateless LFSR for generic moduli. */ +template<typename F, uint32_t MOD> +struct LFSR { + typedef typename F::Repr I; + /** Shift a value `a` up once, treating it as an `N`-bit LFSR, with pattern `MOD`. */ + static inline constexpr I Call(const I& a) { + return F::template CondXorWith<MOD>(F::Shift(a, 1), F::template TopBits<1>(a)); + } +}; + +/** Helper class for carryless multiplications. */ +template<typename I, int N, typename L, typename F, int K> struct GFMulHelper; +template<typename I, int N, typename L, typename F> struct GFMulHelper<I, N, L, F, 0> +{ + static inline constexpr I Run(const I& a, const I& b) { return I(0); } +}; +template<typename I, int N, typename L, typename F, int K> struct GFMulHelper +{ + static inline constexpr I Run(const I& a, const I& b) { return F::CondXorWith(GFMulHelper<I, N, L, F, K - 1>::Run(L::Call(a), b), F::template MidBits<N - K, 1>(b), a); } +}; + +/** Compute the carry-less multiplication of a and b, with N bits, using L as LFSR type. */ +template<typename I, int N, typename L, typename F> inline constexpr I GFMul(const I& a, const I& b) { return GFMulHelper<I, N, L, F, N>::Run(a, b); } + +/** Compute the inverse of x using an extgcd algorithm. */ +template<typename I, typename F, int BITS, uint32_t MOD> +inline I InvExtGCD(I x) +{ + if (F::IsZero(x)) return x; + I t(0), newt(1); + I r(MOD), newr = x; + int rlen = BITS + 1, newrlen = F::Bits(newr, BITS); + while (newr) { + int q = rlen - newrlen; + r ^= F::Shift(newr, q); + t ^= F::UnsafeShift(newt, q); + rlen = F::Bits(r, rlen - 1); + if (r < newr) { + F::Swap(t, newt); + F::Swap(r, newr); + std::swap(rlen, newrlen); + } + } + return t; +} + +/** Compute the inverse of x1 using an exponentiation ladder. + * + * The `MUL` argument is a multiplication function, `SQR` is a squaring function, and the `SQRi` arguments + * compute x**(2**i). + */ +template<typename I, typename F, int BITS, I (*MUL)(I, I), I (*SQR)(I), I (*SQR2)(I), I(*SQR4)(I), I(*SQR8)(I), I(*SQR16)(I)> +inline I InvLadder(I x1) +{ + static constexpr int INV_EXP = BITS - 1; + I x2 = (INV_EXP >= 2) ? MUL(SQR(x1), x1) : I(); + I x4 = (INV_EXP >= 4) ? MUL(SQR2(x2), x2) : I(); + I x8 = (INV_EXP >= 8) ? MUL(SQR4(x4), x4) : I(); + I x16 = (INV_EXP >= 16) ? MUL(SQR8(x8), x8) : I(); + I x32 = (INV_EXP >= 32) ? MUL(SQR16(x16), x16) : I(); + I r; + if (INV_EXP >= 32) { + r = x32; + } else if (INV_EXP >= 16) { + r = x16; + } else if (INV_EXP >= 8) { + r = x8; + } else if (INV_EXP >= 4) { + r = x4; + } else if (INV_EXP >= 2) { + r = x2; + } else { + r = x1; + } + if (INV_EXP >= 32 && (INV_EXP & 16)) r = MUL(SQR16(r), x16); + if (INV_EXP >= 16 && (INV_EXP & 8)) r = MUL(SQR8(r), x8); + if (INV_EXP >= 8 && (INV_EXP & 4)) r = MUL(SQR4(r), x4); + if (INV_EXP >= 4 && (INV_EXP & 2)) r = MUL(SQR2(r), x2); + if (INV_EXP >= 2 && (INV_EXP & 1)) r = MUL(SQR(r), x1); + return SQR(r); +} + +#endif diff --git a/src/minisketch/src/lintrans.h b/src/minisketch/src/lintrans.h new file mode 100644 index 0000000000..b9d8ea8e1d --- /dev/null +++ b/src/minisketch/src/lintrans.h @@ -0,0 +1,150 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _MINISKETCH_LINTRANS_H_ +#define _MINISKETCH_LINTRANS_H_ + +#include "int_utils.h" + +/** A type to represent integers in the type system. */ +template<int N> struct Num {}; + +/** A Linear N-bit transformation over the field I. */ +template<typename I, int N> class LinTrans { +private: + I table[1 << N]; +public: + LinTrans() = default; + + /* Construct a transformation over 3 to 8 bits, using the images of each bit. */ + constexpr LinTrans(I a, I b) : table{I(0), I(a), I(b), I(a ^ b)} {} + constexpr LinTrans(I a, I b, I c) : table{I(0), I(a), I(b), I(a ^ b), I(c), I(a ^ c), I(b ^ c), I(a ^ b ^ c)} {} + constexpr LinTrans(I a, I b, I c, I d) : table{I(0), I(a), I(b), I(a ^ b), I(c), I(a ^ c), I(b ^ c), I(a ^ b ^ c), I(d), I(a ^ d), I(b ^ d), I(a ^ b ^ d), I(c ^ d), I(a ^ c ^ d), I(b ^ c ^ d), I(a ^ b ^ c ^ d)} {} + constexpr LinTrans(I a, I b, I c, I d, I e) : table{I(0), I(a), I(b), I(a ^ b), I(c), I(a ^ c), I(b ^ c), I(a ^ b ^ c), I(d), I(a ^ d), I(b ^ d), I(a ^ b ^ d), I(c ^ d), I(a ^ c ^ d), I(b ^ c ^ d), I(a ^ b ^ c ^ d), I(e), I(a ^ e), I(b ^ e), I(a ^ b ^ e), I(c ^ e), I(a ^ c ^ e), I(b ^ c ^ e), I(a ^ b ^ c ^ e), I(d ^ e), I(a ^ d ^ e), I(b ^ d ^ e), I(a ^ b ^ d ^ e), I(c ^ d ^ e), I(a ^ c ^ d ^ e), I(b ^ c ^ d ^ e), I(a ^ b ^ c ^ d ^ e)} {} + constexpr LinTrans(I a, I b, I c, I d, I e, I f) : table{I(0), I(a), I(b), I(a ^ b), I(c), I(a ^ c), I(b ^ c), I(a ^ b ^ c), I(d), I(a ^ d), I(b ^ d), I(a ^ b ^ d), I(c ^ d), I(a ^ c ^ d), I(b ^ c ^ d), I(a ^ b ^ c ^ d), I(e), I(a ^ e), I(b ^ e), I(a ^ b ^ e), I(c ^ e), I(a ^ c ^ e), I(b ^ c ^ e), I(a ^ b ^ c ^ e), I(d ^ e), I(a ^ d ^ e), I(b ^ d ^ e), I(a ^ b ^ d ^ e), I(c ^ d ^ e), I(a ^ c ^ d ^ e), I(b ^ c ^ d ^ e), I(a ^ b ^ c ^ d ^ e), I(f), I(a ^ f), I(b^ f), I(a ^ b ^ f), I(c^ f), I(a ^ c ^ f), I(b ^ c ^ f), I(a ^ b ^ c ^ f), I(d ^ f), I(a ^ d ^ f), I(b ^ d ^ f), I(a ^ b ^ d ^ f), I(c ^ d ^ f), I(a ^ c ^ d ^ f), I(b ^ c ^ d ^ f), I(a ^ b ^ c ^ d ^ f), I(e ^ f), I(a ^ e ^ f), I(b ^ e ^ f), I(a ^ b ^ e ^ f), I(c ^ e ^ f), I(a ^ c ^ e ^ f), I(b ^ c ^ e ^ f), I(a ^ b ^ c ^ e ^ f), I(d ^ e ^ f), I(a ^ d ^ e ^ f), I(b ^ d ^ e ^ f), I(a ^ b ^ d ^ e ^ f), I(c ^ d ^ e ^ f), I(a ^ c ^ d ^ e ^ f), I(b ^ c ^ d ^ e ^ f), I(a ^ b ^ c ^ d ^ e ^ f)} {} + constexpr LinTrans(I a, I b, I c, I d, I e, I f, I g) : table{I(0), I(a), I(b), I(a ^ b), I(c), I(a ^ c), I(b ^ c), I(a ^ b ^ c), I(d), I(a ^ d), I(b ^ d), I(a ^ b ^ d), I(c ^ d), I(a ^ c ^ d), I(b ^ c ^ d), I(a ^ b ^ c ^ d), I(e), I(a ^ e), I(b ^ e), I(a ^ b ^ e), I(c ^ e), I(a ^ c ^ e), I(b ^ c ^ e), I(a ^ b ^ c ^ e), I(d ^ e), I(a ^ d ^ e), I(b ^ d ^ e), I(a ^ b ^ d ^ e), I(c ^ d ^ e), I(a ^ c ^ d ^ e), I(b ^ c ^ d ^ e), I(a ^ b ^ c ^ d ^ e), I(f), I(a ^ f), I(b^ f), I(a ^ b ^ f), I(c^ f), I(a ^ c ^ f), I(b ^ c ^ f), I(a ^ b ^ c ^ f), I(d ^ f), I(a ^ d ^ f), I(b ^ d ^ f), I(a ^ b ^ d ^ f), I(c ^ d ^ f), I(a ^ c ^ d ^ f), I(b ^ c ^ d ^ f), I(a ^ b ^ c ^ d ^ f), I(e ^ f), I(a ^ e ^ f), I(b ^ e ^ f), I(a ^ b ^ e ^ f), I(c ^ e ^ f), I(a ^ c ^ e ^ f), I(b ^ c ^ e ^ f), I(a ^ b ^ c ^ e ^ f), I(d ^ e ^ f), I(a ^ d ^ e ^ f), I(b ^ d ^ e ^ f), I(a ^ b ^ d ^ e ^ f), I(c ^ d ^ e ^ f), I(a ^ c ^ d ^ e ^ f), I(b ^ c ^ d ^ e ^ f), I(a ^ b ^ c ^ d ^ e ^ f), I(g), I(a ^ g), I(b ^ g), I(a ^ b ^ g), I(c ^ g), I(a ^ c ^ g), I(b ^ c ^ g), I(a ^ b ^ c ^ g), I(d ^ g), I(a ^ d ^ g), I(b ^ d ^ g), I(a ^ b ^ d ^ g), I(c ^ d ^ g), I(a ^ c ^ d ^ g), I(b ^ c ^ d ^ g), I(a ^ b ^ c ^ d ^ g), I(e ^ g), I(a ^ e ^ g), I(b ^ e ^ g), I(a ^ b ^ e ^ g), I(c ^ e ^ g), I(a ^ c ^ e ^ g), I(b ^ c ^ e ^ g), I(a ^ b ^ c ^ e ^ g), I(d ^ e ^ g), I(a ^ d ^ e ^ g), I(b ^ d ^ e ^ g), I(a ^ b ^ d ^ e ^ g), I(c ^ d ^ e ^ g), I(a ^ c ^ d ^ e ^ g), I(b ^ c ^ d ^ e ^ g), I(a ^ b ^ c ^ d ^ e ^ g), I(f ^ g), I(a ^ f ^ g), I(b^ f ^ g), I(a ^ b ^ f ^ g), I(c^ f ^ g), I(a ^ c ^ f ^ g), I(b ^ c ^ f ^ g), I(a ^ b ^ c ^ f ^ g), I(d ^ f ^ g), I(a ^ d ^ f ^ g), I(b ^ d ^ f ^ g), I(a ^ b ^ d ^ f ^ g), I(c ^ d ^ f ^ g), I(a ^ c ^ d ^ f ^ g), I(b ^ c ^ d ^ f ^ g), I(a ^ b ^ c ^ d ^ f ^ g), I(e ^ f ^ g), I(a ^ e ^ f ^ g), I(b ^ e ^ f ^ g), I(a ^ b ^ e ^ f ^ g), I(c ^ e ^ f ^ g), I(a ^ c ^ e ^ f ^ g), I(b ^ c ^ e ^ f ^ g), I(a ^ b ^ c ^ e ^ f ^ g), I(d ^ e ^ f ^ g), I(a ^ d ^ e ^ f ^ g), I(b ^ d ^ e ^ f ^ g), I(a ^ b ^ d ^ e ^ f ^ g), I(c ^ d ^ e ^ f ^ g), I(a ^ c ^ d ^ e ^ f ^ g), I(b ^ c ^ d ^ e ^ f ^ g), I(a ^ b ^ c ^ d ^ e ^ f ^ g)} {} + constexpr LinTrans(I a, I b, I c, I d, I e, I f, I g, I h) : table{I(0), I(a), I(b), I(a ^ b), I(c), I(a ^ c), I(b ^ c), I(a ^ b ^ c), I(d), I(a ^ d), I(b ^ d), I(a ^ b ^ d), I(c ^ d), I(a ^ c ^ d), I(b ^ c ^ d), I(a ^ b ^ c ^ d), I(e), I(a ^ e), I(b ^ e), I(a ^ b ^ e), I(c ^ e), I(a ^ c ^ e), I(b ^ c ^ e), I(a ^ b ^ c ^ e), I(d ^ e), I(a ^ d ^ e), I(b ^ d ^ e), I(a ^ b ^ d ^ e), I(c ^ d ^ e), I(a ^ c ^ d ^ e), I(b ^ c ^ d ^ e), I(a ^ b ^ c ^ d ^ e), I(f), I(a ^ f), I(b^ f), I(a ^ b ^ f), I(c^ f), I(a ^ c ^ f), I(b ^ c ^ f), I(a ^ b ^ c ^ f), I(d ^ f), I(a ^ d ^ f), I(b ^ d ^ f), I(a ^ b ^ d ^ f), I(c ^ d ^ f), I(a ^ c ^ d ^ f), I(b ^ c ^ d ^ f), I(a ^ b ^ c ^ d ^ f), I(e ^ f), I(a ^ e ^ f), I(b ^ e ^ f), I(a ^ b ^ e ^ f), I(c ^ e ^ f), I(a ^ c ^ e ^ f), I(b ^ c ^ e ^ f), I(a ^ b ^ c ^ e ^ f), I(d ^ e ^ f), I(a ^ d ^ e ^ f), I(b ^ d ^ e ^ f), I(a ^ b ^ d ^ e ^ f), I(c ^ d ^ e ^ f), I(a ^ c ^ d ^ e ^ f), I(b ^ c ^ d ^ e ^ f), I(a ^ b ^ c ^ d ^ e ^ f), I(g), I(a ^ g), I(b ^ g), I(a ^ b ^ g), I(c ^ g), I(a ^ c ^ g), I(b ^ c ^ g), I(a ^ b ^ c ^ g), I(d ^ g), I(a ^ d ^ g), I(b ^ d ^ g), I(a ^ b ^ d ^ g), I(c ^ d ^ g), I(a ^ c ^ d ^ g), I(b ^ c ^ d ^ g), I(a ^ b ^ c ^ d ^ g), I(e ^ g), I(a ^ e ^ g), I(b ^ e ^ g), I(a ^ b ^ e ^ g), I(c ^ e ^ g), I(a ^ c ^ e ^ g), I(b ^ c ^ e ^ g), I(a ^ b ^ c ^ e ^ g), I(d ^ e ^ g), I(a ^ d ^ e ^ g), I(b ^ d ^ e ^ g), I(a ^ b ^ d ^ e ^ g), I(c ^ d ^ e ^ g), I(a ^ c ^ d ^ e ^ g), I(b ^ c ^ d ^ e ^ g), I(a ^ b ^ c ^ d ^ e ^ g), I(f ^ g), I(a ^ f ^ g), I(b^ f ^ g), I(a ^ b ^ f ^ g), I(c^ f ^ g), I(a ^ c ^ f ^ g), I(b ^ c ^ f ^ g), I(a ^ b ^ c ^ f ^ g), I(d ^ f ^ g), I(a ^ d ^ f ^ g), I(b ^ d ^ f ^ g), I(a ^ b ^ d ^ f ^ g), I(c ^ d ^ f ^ g), I(a ^ c ^ d ^ f ^ g), I(b ^ c ^ d ^ f ^ g), I(a ^ b ^ c ^ d ^ f ^ g), I(e ^ f ^ g), I(a ^ e ^ f ^ g), I(b ^ e ^ f ^ g), I(a ^ b ^ e ^ f ^ g), I(c ^ e ^ f ^ g), I(a ^ c ^ e ^ f ^ g), I(b ^ c ^ e ^ f ^ g), I(a ^ b ^ c ^ e ^ f ^ g), I(d ^ e ^ f ^ g), I(a ^ d ^ e ^ f ^ g), I(b ^ d ^ e ^ f ^ g), I(a ^ b ^ d ^ e ^ f ^ g), I(c ^ d ^ e ^ f ^ g), I(a ^ c ^ d ^ e ^ f ^ g), I(b ^ c ^ d ^ e ^ f ^ g), I(a ^ b ^ c ^ d ^ e ^ f ^ g), I(h), I(a ^ h), I(b ^ h), I(a ^ b ^ h), I(c ^ h), I(a ^ c ^ h), I(b ^ c ^ h), I(a ^ b ^ c ^ h), I(d ^ h), I(a ^ d ^ h), I(b ^ d ^ h), I(a ^ b ^ d ^ h), I(c ^ d ^ h), I(a ^ c ^ d ^ h), I(b ^ c ^ d ^ h), I(a ^ b ^ c ^ d ^ h), I(e ^ h), I(a ^ e ^ h), I(b ^ e ^ h), I(a ^ b ^ e ^ h), I(c ^ e ^ h), I(a ^ c ^ e ^ h), I(b ^ c ^ e ^ h), I(a ^ b ^ c ^ e ^ h), I(d ^ e ^ h), I(a ^ d ^ e ^ h), I(b ^ d ^ e ^ h), I(a ^ b ^ d ^ e ^ h), I(c ^ d ^ e ^ h), I(a ^ c ^ d ^ e ^ h), I(b ^ c ^ d ^ e ^ h), I(a ^ b ^ c ^ d ^ e ^ h), I(f ^ h), I(a ^ f ^ h), I(b^ f ^ h), I(a ^ b ^ f ^ h), I(c^ f ^ h), I(a ^ c ^ f ^ h), I(b ^ c ^ f ^ h), I(a ^ b ^ c ^ f ^ h), I(d ^ f ^ h), I(a ^ d ^ f ^ h), I(b ^ d ^ f ^ h), I(a ^ b ^ d ^ f ^ h), I(c ^ d ^ f ^ h), I(a ^ c ^ d ^ f ^ h), I(b ^ c ^ d ^ f ^ h), I(a ^ b ^ c ^ d ^ f ^ h), I(e ^ f ^ h), I(a ^ e ^ f ^ h), I(b ^ e ^ f ^ h), I(a ^ b ^ e ^ f ^ h), I(c ^ e ^ f ^ h), I(a ^ c ^ e ^ f ^ h), I(b ^ c ^ e ^ f ^ h), I(a ^ b ^ c ^ e ^ f ^ h), I(d ^ e ^ f ^ h), I(a ^ d ^ e ^ f ^ h), I(b ^ d ^ e ^ f ^ h), I(a ^ b ^ d ^ e ^ f ^ h), I(c ^ d ^ e ^ f ^ h), I(a ^ c ^ d ^ e ^ f ^ h), I(b ^ c ^ d ^ e ^ f ^ h), I(a ^ b ^ c ^ d ^ e ^ f ^ h), I(g ^ h), I(a ^ g ^ h), I(b ^ g ^ h), I(a ^ b ^ g ^ h), I(c ^ g ^ h), I(a ^ c ^ g ^ h), I(b ^ c ^ g ^ h), I(a ^ b ^ c ^ g ^ h), I(d ^ g ^ h), I(a ^ d ^ g ^ h), I(b ^ d ^ g ^ h), I(a ^ b ^ d ^ g ^ h), I(c ^ d ^ g ^ h), I(a ^ c ^ d ^ g ^ h), I(b ^ c ^ d ^ g ^ h), I(a ^ b ^ c ^ d ^ g ^ h), I(e ^ g ^ h), I(a ^ e ^ g ^ h), I(b ^ e ^ g ^ h), I(a ^ b ^ e ^ g ^ h), I(c ^ e ^ g ^ h), I(a ^ c ^ e ^ g ^ h), I(b ^ c ^ e ^ g ^ h), I(a ^ b ^ c ^ e ^ g ^ h), I(d ^ e ^ g ^ h), I(a ^ d ^ e ^ g ^ h), I(b ^ d ^ e ^ g ^ h), I(a ^ b ^ d ^ e ^ g ^ h), I(c ^ d ^ e ^ g ^ h), I(a ^ c ^ d ^ e ^ g ^ h), I(b ^ c ^ d ^ e ^ g ^ h), I(a ^ b ^ c ^ d ^ e ^ g ^ h), I(f ^ g ^ h), I(a ^ f ^ g ^ h), I(b^ f ^ g ^ h), I(a ^ b ^ f ^ g ^ h), I(c^ f ^ g ^ h), I(a ^ c ^ f ^ g ^ h), I(b ^ c ^ f ^ g ^ h), I(a ^ b ^ c ^ f ^ g ^ h), I(d ^ f ^ g ^ h), I(a ^ d ^ f ^ g ^ h), I(b ^ d ^ f ^ g ^ h), I(a ^ b ^ d ^ f ^ g ^ h), I(c ^ d ^ f ^ g ^ h), I(a ^ c ^ d ^ f ^ g ^ h), I(b ^ c ^ d ^ f ^ g ^ h), I(a ^ b ^ c ^ d ^ f ^ g ^ h), I(e ^ f ^ g ^ h), I(a ^ e ^ f ^ g ^ h), I(b ^ e ^ f ^ g ^ h), I(a ^ b ^ e ^ f ^ g ^ h), I(c ^ e ^ f ^ g ^ h), I(a ^ c ^ e ^ f ^ g ^ h), I(b ^ c ^ e ^ f ^ g ^ h), I(a ^ b ^ c ^ e ^ f ^ g ^ h), I(d ^ e ^ f ^ g ^ h), I(a ^ d ^ e ^ f ^ g ^ h), I(b ^ d ^ e ^ f ^ g ^ h), I(a ^ b ^ d ^ e ^ f ^ g ^ h), I(c ^ d ^ e ^ f ^ g ^ h), I(a ^ c ^ d ^ e ^ f ^ g ^ h), I(b ^ c ^ d ^ e ^ f ^ g ^ h), I(a ^ b ^ c ^ d ^ e ^ f ^ g ^ h)} {} + + /* Construct a transformation over 3 to 8 bits, using a pointer to the bit's images. */ + constexpr LinTrans(const I* p, Num<2>) : LinTrans(I(p[0]), I(p[1])) {} + constexpr LinTrans(const I* p, Num<3>) : LinTrans(I(p[0]), I(p[1]), I(p[2])) {} + constexpr LinTrans(const I* p, Num<4>) : LinTrans(I(p[0]), I(p[1]), I(p[2]), I(p[3])) {} + constexpr LinTrans(const I* p, Num<5>) : LinTrans(I(p[0]), I(p[1]), I(p[2]), I(p[3]), I(p[4])) {} + constexpr LinTrans(const I* p, Num<6>) : LinTrans(I(p[0]), I(p[1]), I(p[2]), I(p[3]), I(p[4]), I(p[5])) {} + constexpr LinTrans(const I* p, Num<7>) : LinTrans(I(p[0]), I(p[1]), I(p[2]), I(p[3]), I(p[4]), I(p[5]), I(p[6])) {} + constexpr LinTrans(const I* p, Num<8>) : LinTrans(I(p[0]), I(p[1]), I(p[2]), I(p[3]), I(p[4]), I(p[5]), I(p[6]), I(p[7])) {} + + template<I (*F)(const I&)> + inline I Build(Num<1>, I a) + { + table[0] = I(); table[1] = a; + return a; + } + + template<I (*F)(const I&)> + inline I Build(Num<2>, I a) + { + I b = F(a); + table[0] = I(); table[1] = a; table[2] = b; table[3] = a ^ b; + return b; + } + + template<I (*F)(const I&)> + inline I Build(Num<3>, I a) + { + I b = F(a), c = F(b); + table[0] = I(); table[1] = a; table[2] = b; table[3] = a ^ b; table[4] = c; table[5] = a ^ c; table[6] = b ^ c; table[7] = a ^ b ^ c; + return c; + } + + template<I (*F)(const I&)> + inline I Build(Num<4>, I a) + { + I b = F(a), c = F(b), d = F(c); + table[0] = I(); table[1] = a; table[2] = b; table[3] = a ^ b; table[4] = c; table[5] = a ^ c; table[6] = b ^ c; table[7] = a ^ b ^ c; + table[8] = d; table[9] = a ^ d; table[10] = b ^ d; table[11] = a ^ b ^ d; table[12] = c ^ d; table[13] = a ^ c ^ d; table[14] = b ^ c ^ d; table[15] = a ^ b ^ c ^ d; + return d; + } + + template<I (*F)(const I&)> + inline I Build(Num<5>, I a) + { + I b = F(a), c = F(b), d = F(c), e = F(d); + table[0] = I(); table[1] = a; table[2] = b; table[3] = a ^ b; table[4] = c; table[5] = a ^ c; table[6] = b ^ c; table[7] = a ^ b ^ c; + table[8] = d; table[9] = a ^ d; table[10] = b ^ d; table[11] = a ^ b ^ d; table[12] = c ^ d; table[13] = a ^ c ^ d; table[14] = b ^ c ^ d; table[15] = a ^ b ^ c ^ d; + table[16] = e; table[17] = a ^ e; table[18] = b ^ e; table[19] = a ^ b ^ e; table[20] = c ^ e; table[21] = a ^ c ^ e; table[22] = b ^ c ^ e; table[23] = a ^ b ^ c ^ e; + table[24] = d ^ e; table[25] = a ^ d ^ e; table[26] = b ^ d ^ e; table[27] = a ^ b ^ d ^ e; table[28] = c ^ d ^ e; table[29] = a ^ c ^ d ^ e; table[30] = b ^ c ^ d ^ e; table[31] = a ^ b ^ c ^ d ^ e; + return e; + } + + template<I (*F)(const I&)> + inline I Build(Num<6>, I a) + { + I b = F(a), c = F(b), d = F(c), e = F(d), f = F(e); + table[0] = I(); table[1] = a; table[2] = b; table[3] = a ^ b; table[4] = c; table[5] = a ^ c; table[6] = b ^ c; table[7] = a ^ b ^ c; + table[8] = d; table[9] = a ^ d; table[10] = b ^ d; table[11] = a ^ b ^ d; table[12] = c ^ d; table[13] = a ^ c ^ d; table[14] = b ^ c ^ d; table[15] = a ^ b ^ c ^ d; + table[16] = e; table[17] = a ^ e; table[18] = b ^ e; table[19] = a ^ b ^ e; table[20] = c ^ e; table[21] = a ^ c ^ e; table[22] = b ^ c ^ e; table[23] = a ^ b ^ c ^ e; + table[24] = d ^ e; table[25] = a ^ d ^ e; table[26] = b ^ d ^ e; table[27] = a ^ b ^ d ^ e; table[28] = c ^ d ^ e; table[29] = a ^ c ^ d ^ e; table[30] = b ^ c ^ d ^ e; table[31] = a ^ b ^ c ^ d ^ e; + table[32] = f; table[33] = a ^ f; table[34] = b ^ f; table[35] = a ^ b ^ f; table[36] = c ^ f; table[37] = a ^ c ^ f; table[38] = b ^ c ^ f; table[39] = a ^ b ^ c ^ f; + table[40] = d ^ f; table[41] = a ^ d ^ f; table[42] = b ^ d ^ f; table[43] = a ^ b ^ d ^ f; table[44] = c ^ d ^ f; table[45] = a ^ c ^ d ^ f; table[46] = b ^ c ^ d ^ f; table[47] = a ^ b ^ c ^ d ^ f; + table[48] = e ^ f; table[49] = a ^ e ^ f; table[50] = b ^ e ^ f; table[51] = a ^ b ^ e ^ f; table[52] = c ^ e ^ f; table[53] = a ^ c ^ e ^ f; table[54] = b ^ c ^ e ^ f; table[55] = a ^ b ^ c ^ e ^ f; + table[56] = d ^ e ^ f; table[57] = a ^ d ^ e ^ f; table[58] = b ^ d ^ e ^ f; table[59] = a ^ b ^ d ^ e ^ f; table[60] = c ^ d ^ e ^ f; table[61] = a ^ c ^ d ^ e ^ f; table[62] = b ^ c ^ d ^ e ^ f; table[63] = a ^ b ^ c ^ d ^ e ^ f; + return f; + } + + template<typename O, int P> + inline I constexpr Map(I a) const { return table[O::template MidBits<P, N>(a)]; } + + template<typename O, int P> + inline I constexpr TopMap(I a) const { static_assert(P + N == O::SIZE, "TopMap inconsistency"); return table[O::template TopBits<N>(a)]; } +}; + + +/** A linear transformation constructed using LinTrans tables for sections of bits. */ +template<typename I, int... N> class RecLinTrans; + +template<typename I, int N> class RecLinTrans<I, N> { + LinTrans<I, N> trans; +public: + static constexpr int BITS = N; + constexpr RecLinTrans(const I* p, Num<BITS>) : trans(p, Num<N>()) {} + constexpr RecLinTrans() = default; + constexpr RecLinTrans(const I (&init)[BITS]) : RecLinTrans(init, Num<BITS>()) {} + + template<typename O, int P = 0> + inline I constexpr Map(I a) const { return trans.template TopMap<O, P>(a); } + + template<I (*F)(const I&)> + inline void Build(I a) { trans.template Build<F>(Num<N>(), a); } +}; + +template<typename I, int N, int... X> class RecLinTrans<I, N, X...> { + LinTrans<I, N> trans; + RecLinTrans<I, X...> rec; +public: + static constexpr int BITS = RecLinTrans<I, X...>::BITS + N; + constexpr RecLinTrans(const I* p, Num<BITS>) : trans(p, Num<N>()), rec(p + N, Num<BITS - N>()) {} + constexpr RecLinTrans() = default; + constexpr RecLinTrans(const I (&init)[BITS]) : RecLinTrans(init, Num<BITS>()) {} + + template<typename O, int P = 0> + inline I constexpr Map(I a) const { return trans.template Map<O, P>(a) ^ rec.template Map<O, P + N>(a); } + + template<I (*F)(const I&)> + inline void Build(I a) { I n = trans.template Build<F>(Num<N>(), a); rec.template Build<F>(F(n)); } +}; + +/** The identity transformation. */ +class IdTrans { +public: + template<typename O, typename I> + inline I constexpr Map(I a) const { return a; } +}; + +/** A singleton for the identity transformation. */ +constexpr IdTrans ID_TRANS{}; + +#endif diff --git a/src/minisketch/src/minisketch.cpp b/src/minisketch/src/minisketch.cpp new file mode 100644 index 0000000000..e9a322f139 --- /dev/null +++ b/src/minisketch/src/minisketch.cpp @@ -0,0 +1,490 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + + +#include <new> + +#define MINISKETCH_BUILD +#ifdef _MINISKETCH_H_ +# error "minisketch.h cannot be included before minisketch.cpp" +#endif +#include "../include/minisketch.h" + +#include "false_positives.h" +#include "fielddefines.h" +#include "sketch.h" + +#ifdef HAVE_CLMUL +# ifdef _MSC_VER +# include <intrin.h> +# else +# include <cpuid.h> +# endif +#endif + +Sketch* ConstructGeneric1Byte(int bits, int implementation); +Sketch* ConstructGeneric2Bytes(int bits, int implementation); +Sketch* ConstructGeneric3Bytes(int bits, int implementation); +Sketch* ConstructGeneric4Bytes(int bits, int implementation); +Sketch* ConstructGeneric5Bytes(int bits, int implementation); +Sketch* ConstructGeneric6Bytes(int bits, int implementation); +Sketch* ConstructGeneric7Bytes(int bits, int implementation); +Sketch* ConstructGeneric8Bytes(int bits, int implementation); + +#ifdef HAVE_CLMUL +Sketch* ConstructClMul1Byte(int bits, int implementation); +Sketch* ConstructClMul2Bytes(int bits, int implementation); +Sketch* ConstructClMul3Bytes(int bits, int implementation); +Sketch* ConstructClMul4Bytes(int bits, int implementation); +Sketch* ConstructClMul5Bytes(int bits, int implementation); +Sketch* ConstructClMul6Bytes(int bits, int implementation); +Sketch* ConstructClMul7Bytes(int bits, int implementation); +Sketch* ConstructClMul8Bytes(int bits, int implementation); +Sketch* ConstructClMulTri1Byte(int bits, int implementation); +Sketch* ConstructClMulTri2Bytes(int bits, int implementation); +Sketch* ConstructClMulTri3Bytes(int bits, int implementation); +Sketch* ConstructClMulTri4Bytes(int bits, int implementation); +Sketch* ConstructClMulTri5Bytes(int bits, int implementation); +Sketch* ConstructClMulTri6Bytes(int bits, int implementation); +Sketch* ConstructClMulTri7Bytes(int bits, int implementation); +Sketch* ConstructClMulTri8Bytes(int bits, int implementation); +#endif + +namespace { + +enum class FieldImpl { + GENERIC = 0, +#ifdef HAVE_CLMUL + CLMUL, + CLMUL_TRI, +#endif +}; + +static inline bool EnableClmul() +{ +#ifdef HAVE_CLMUL +#ifdef _MSC_VER + int regs[4]; + __cpuid(regs, 1); + return (regs[2] & 0x2); +#else + uint32_t eax, ebx, ecx, edx; + return (__get_cpuid(1, &eax, &ebx, &ecx, &edx) && (ecx & 0x2)); +#endif +#else + return false; +#endif +} + +Sketch* Construct(int bits, int impl) +{ + switch (FieldImpl(impl)) { + case FieldImpl::GENERIC: + switch ((bits + 7) / 8) { + case 1: + return ConstructGeneric1Byte(bits, impl); + case 2: + return ConstructGeneric2Bytes(bits, impl); + case 3: + return ConstructGeneric3Bytes(bits, impl); + case 4: + return ConstructGeneric4Bytes(bits, impl); + case 5: + return ConstructGeneric5Bytes(bits, impl); + case 6: + return ConstructGeneric6Bytes(bits, impl); + case 7: + return ConstructGeneric7Bytes(bits, impl); + case 8: + return ConstructGeneric8Bytes(bits, impl); + default: + return nullptr; + } + break; +#ifdef HAVE_CLMUL + case FieldImpl::CLMUL: + if (EnableClmul()) { + switch ((bits + 7) / 8) { + case 1: + return ConstructClMul1Byte(bits, impl); + case 2: + return ConstructClMul2Bytes(bits, impl); + case 3: + return ConstructClMul3Bytes(bits, impl); + case 4: + return ConstructClMul4Bytes(bits, impl); + case 5: + return ConstructClMul5Bytes(bits, impl); + case 6: + return ConstructClMul6Bytes(bits, impl); + case 7: + return ConstructClMul7Bytes(bits, impl); + case 8: + return ConstructClMul8Bytes(bits, impl); + default: + return nullptr; + } + } + break; + case FieldImpl::CLMUL_TRI: + if (EnableClmul()) { + switch ((bits + 7) / 8) { + case 1: + return ConstructClMulTri1Byte(bits, impl); + case 2: + return ConstructClMulTri2Bytes(bits, impl); + case 3: + return ConstructClMulTri3Bytes(bits, impl); + case 4: + return ConstructClMulTri4Bytes(bits, impl); + case 5: + return ConstructClMulTri5Bytes(bits, impl); + case 6: + return ConstructClMulTri6Bytes(bits, impl); + case 7: + return ConstructClMulTri7Bytes(bits, impl); + case 8: + return ConstructClMulTri8Bytes(bits, impl); + default: + return nullptr; + } + } + break; +#endif + } + return nullptr; +} + +} + +extern "C" { + +int minisketch_bits_supported(uint32_t bits) { +#ifdef ENABLE_FIELD_INT_2 + if (bits == 2) return true; +#endif +#ifdef ENABLE_FIELD_INT_3 + if (bits == 3) return true; +#endif +#ifdef ENABLE_FIELD_INT_4 + if (bits == 4) return true; +#endif +#ifdef ENABLE_FIELD_INT_5 + if (bits == 5) return true; +#endif +#ifdef ENABLE_FIELD_INT_6 + if (bits == 6) return true; +#endif +#ifdef ENABLE_FIELD_INT_7 + if (bits == 7) return true; +#endif +#ifdef ENABLE_FIELD_INT_8 + if (bits == 8) return true; +#endif +#ifdef ENABLE_FIELD_INT_9 + if (bits == 9) return true; +#endif +#ifdef ENABLE_FIELD_INT_10 + if (bits == 10) return true; +#endif +#ifdef ENABLE_FIELD_INT_11 + if (bits == 11) return true; +#endif +#ifdef ENABLE_FIELD_INT_12 + if (bits == 12) return true; +#endif +#ifdef ENABLE_FIELD_INT_13 + if (bits == 13) return true; +#endif +#ifdef ENABLE_FIELD_INT_14 + if (bits == 14) return true; +#endif +#ifdef ENABLE_FIELD_INT_15 + if (bits == 15) return true; +#endif +#ifdef ENABLE_FIELD_INT_16 + if (bits == 16) return true; +#endif +#ifdef ENABLE_FIELD_INT_17 + if (bits == 17) return true; +#endif +#ifdef ENABLE_FIELD_INT_18 + if (bits == 18) return true; +#endif +#ifdef ENABLE_FIELD_INT_19 + if (bits == 19) return true; +#endif +#ifdef ENABLE_FIELD_INT_20 + if (bits == 20) return true; +#endif +#ifdef ENABLE_FIELD_INT_21 + if (bits == 21) return true; +#endif +#ifdef ENABLE_FIELD_INT_22 + if (bits == 22) return true; +#endif +#ifdef ENABLE_FIELD_INT_23 + if (bits == 23) return true; +#endif +#ifdef ENABLE_FIELD_INT_24 + if (bits == 24) return true; +#endif +#ifdef ENABLE_FIELD_INT_25 + if (bits == 25) return true; +#endif +#ifdef ENABLE_FIELD_INT_26 + if (bits == 26) return true; +#endif +#ifdef ENABLE_FIELD_INT_27 + if (bits == 27) return true; +#endif +#ifdef ENABLE_FIELD_INT_28 + if (bits == 28) return true; +#endif +#ifdef ENABLE_FIELD_INT_29 + if (bits == 29) return true; +#endif +#ifdef ENABLE_FIELD_INT_30 + if (bits == 30) return true; +#endif +#ifdef ENABLE_FIELD_INT_31 + if (bits == 31) return true; +#endif +#ifdef ENABLE_FIELD_INT_32 + if (bits == 32) return true; +#endif +#ifdef ENABLE_FIELD_INT_33 + if (bits == 33) return true; +#endif +#ifdef ENABLE_FIELD_INT_34 + if (bits == 34) return true; +#endif +#ifdef ENABLE_FIELD_INT_35 + if (bits == 35) return true; +#endif +#ifdef ENABLE_FIELD_INT_36 + if (bits == 36) return true; +#endif +#ifdef ENABLE_FIELD_INT_37 + if (bits == 37) return true; +#endif +#ifdef ENABLE_FIELD_INT_38 + if (bits == 38) return true; +#endif +#ifdef ENABLE_FIELD_INT_39 + if (bits == 39) return true; +#endif +#ifdef ENABLE_FIELD_INT_40 + if (bits == 40) return true; +#endif +#ifdef ENABLE_FIELD_INT_41 + if (bits == 41) return true; +#endif +#ifdef ENABLE_FIELD_INT_42 + if (bits == 42) return true; +#endif +#ifdef ENABLE_FIELD_INT_43 + if (bits == 43) return true; +#endif +#ifdef ENABLE_FIELD_INT_44 + if (bits == 44) return true; +#endif +#ifdef ENABLE_FIELD_INT_45 + if (bits == 45) return true; +#endif +#ifdef ENABLE_FIELD_INT_46 + if (bits == 46) return true; +#endif +#ifdef ENABLE_FIELD_INT_47 + if (bits == 47) return true; +#endif +#ifdef ENABLE_FIELD_INT_48 + if (bits == 48) return true; +#endif +#ifdef ENABLE_FIELD_INT_49 + if (bits == 49) return true; +#endif +#ifdef ENABLE_FIELD_INT_50 + if (bits == 50) return true; +#endif +#ifdef ENABLE_FIELD_INT_51 + if (bits == 51) return true; +#endif +#ifdef ENABLE_FIELD_INT_52 + if (bits == 52) return true; +#endif +#ifdef ENABLE_FIELD_INT_53 + if (bits == 53) return true; +#endif +#ifdef ENABLE_FIELD_INT_54 + if (bits == 54) return true; +#endif +#ifdef ENABLE_FIELD_INT_55 + if (bits == 55) return true; +#endif +#ifdef ENABLE_FIELD_INT_56 + if (bits == 56) return true; +#endif +#ifdef ENABLE_FIELD_INT_57 + if (bits == 57) return true; +#endif +#ifdef ENABLE_FIELD_INT_58 + if (bits == 58) return true; +#endif +#ifdef ENABLE_FIELD_INT_59 + if (bits == 59) return true; +#endif +#ifdef ENABLE_FIELD_INT_60 + if (bits == 60) return true; +#endif +#ifdef ENABLE_FIELD_INT_61 + if (bits == 61) return true; +#endif +#ifdef ENABLE_FIELD_INT_62 + if (bits == 62) return true; +#endif +#ifdef ENABLE_FIELD_INT_63 + if (bits == 63) return true; +#endif +#ifdef ENABLE_FIELD_INT_64 + if (bits == 64) return true; +#endif + return false; +} + +uint32_t minisketch_implementation_max() { + uint32_t ret = 0; +#ifdef HAVE_CLMUL + ret += 2; +#endif + return ret; +} + +int minisketch_implementation_supported(uint32_t bits, uint32_t implementation) { + if (!minisketch_bits_supported(bits) || implementation > minisketch_implementation_max()) { + return 0; + } + try { + Sketch* sketch = Construct(bits, implementation); + if (sketch) { + delete sketch; + return 1; + } + } catch (const std::bad_alloc&) {} + return 0; +} + +minisketch* minisketch_create(uint32_t bits, uint32_t implementation, size_t capacity) { + try { + Sketch* sketch = Construct(bits, implementation); + if (sketch) { + try { + sketch->Init(capacity); + } catch (const std::bad_alloc&) { + delete sketch; + throw; + } + sketch->Ready(); + } + return (minisketch*)sketch; + } catch (const std::bad_alloc&) { + return nullptr; + } +} + +uint32_t minisketch_bits(const minisketch* sketch) { + const Sketch* s = (const Sketch*)sketch; + s->Check(); + return s->Bits(); +} + +size_t minisketch_capacity(const minisketch* sketch) { + const Sketch* s = (const Sketch*)sketch; + s->Check(); + return s->Syndromes(); +} + +uint32_t minisketch_implementation(const minisketch* sketch) { + const Sketch* s = (const Sketch*)sketch; + s->Check(); + return s->Implementation(); +} + +minisketch* minisketch_clone(const minisketch* sketch) { + const Sketch* s = (const Sketch*)sketch; + s->Check(); + Sketch* r = (Sketch*) minisketch_create(s->Bits(), s->Implementation(), s->Syndromes()); + if (r) { + r->Merge(s); + } + return (minisketch*) r; +} + +void minisketch_destroy(minisketch* sketch) { + if (sketch) { + Sketch* s = (Sketch*)sketch; + s->UnReady(); + delete s; + } +} + +size_t minisketch_serialized_size(const minisketch* sketch) { + const Sketch* s = (const Sketch*)sketch; + s->Check(); + size_t bits = s->Bits(); + size_t syndromes = s->Syndromes(); + return (bits * syndromes + 7) / 8; +} + +void minisketch_serialize(const minisketch* sketch, unsigned char* output) { + const Sketch* s = (const Sketch*)sketch; + s->Check(); + s->Serialize(output); +} + +void minisketch_deserialize(minisketch* sketch, const unsigned char* input) { + Sketch* s = (Sketch*)sketch; + s->Check(); + s->Deserialize(input); +} + +void minisketch_add_uint64(minisketch* sketch, uint64_t element) { + Sketch* s = (Sketch*)sketch; + s->Check(); + s->Add(element); +} + +size_t minisketch_merge(minisketch* sketch, const minisketch* other_sketch) { + Sketch* s1 = (Sketch*)sketch; + const Sketch* s2 = (const Sketch*)other_sketch; + s1->Check(); + s2->Check(); + if (s1->Bits() != s2->Bits()) return 0; + if (s1->Implementation() != s2->Implementation()) return 0; + return s1->Merge(s2); +} + +ssize_t minisketch_decode(const minisketch* sketch, size_t max_elements, uint64_t* output) { + const Sketch* s = (const Sketch*)sketch; + s->Check(); + return s->Decode(max_elements, output); +} + +void minisketch_set_seed(minisketch* sketch, uint64_t seed) { + Sketch* s = (Sketch*)sketch; + s->Check(); + s->SetSeed(seed); +} + +size_t minisketch_compute_capacity(uint32_t bits, size_t max_elements, uint32_t fpbits) { + return ComputeCapacity(bits, max_elements, fpbits); +} + +size_t minisketch_compute_max_elements(uint32_t bits, size_t capacity, uint32_t fpbits) { + return ComputeMaxElements(bits, capacity, fpbits); +} + +} diff --git a/src/minisketch/src/sketch.h b/src/minisketch/src/sketch.h new file mode 100644 index 0000000000..3e9bad793d --- /dev/null +++ b/src/minisketch/src/sketch.h @@ -0,0 +1,42 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _MINISKETCH_STATE_H_ +#define _MINISKETCH_STATE_H_ + +#include <stdint.h> +#include <stdlib.h> + +/** Abstract class for internal representation of a minisketch object. */ +class Sketch +{ + uint64_t m_canary; + const int m_implementation; + const int m_bits; + +public: + Sketch(int implementation, int bits) : m_implementation(implementation), m_bits(bits) {} + + void Ready() { m_canary = 0x6d496e536b65LU; } + void Check() const { if (m_canary != 0x6d496e536b65LU) abort(); } + void UnReady() { m_canary = 1; } + int Implementation() const { return m_implementation; } + int Bits() const { return m_bits; } + + virtual ~Sketch() {} + virtual size_t Syndromes() const = 0; + + virtual void Init(int syndromes) = 0; + virtual void Add(uint64_t element) = 0; + virtual void Serialize(unsigned char*) const = 0; + virtual void Deserialize(const unsigned char*) = 0; + virtual size_t Merge(const Sketch* other_sketch) = 0; + virtual void SetSeed(uint64_t seed) = 0; + + virtual int Decode(int max_count, uint64_t* roots) const = 0; +}; + +#endif diff --git a/src/minisketch/src/sketch_impl.h b/src/minisketch/src/sketch_impl.h new file mode 100644 index 0000000000..4547b742f2 --- /dev/null +++ b/src/minisketch/src/sketch_impl.h @@ -0,0 +1,433 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _MINISKETCH_SKETCH_IMPL_H_ +#define _MINISKETCH_SKETCH_IMPL_H_ + +#include <random> + +#include "util.h" +#include "sketch.h" +#include "int_utils.h" + +/** Compute the remainder of a polynomial division of val by mod, putting the result in mod. */ +template<typename F> +void PolyMod(const std::vector<typename F::Elem>& mod, std::vector<typename F::Elem>& val, const F& field) { + size_t modsize = mod.size(); + CHECK_SAFE(modsize > 0 && mod.back() == 1); + if (val.size() < modsize) return; + CHECK_SAFE(val.back() != 0); + while (val.size() >= modsize) { + auto term = val.back(); + val.pop_back(); + if (term != 0) { + typename F::Multiplier mul(field, term); + for (size_t x = 0; x < mod.size() - 1; ++x) { + val[val.size() - modsize + 1 + x] ^= mul(mod[x]); + } + } + } + while (val.size() > 0 && val.back() == 0) val.pop_back(); +} + +/** Compute the quotient of a polynomial division of val by mod, putting the quotient in div and the remainder in val. */ +template<typename F> +void DivMod(const std::vector<typename F::Elem>& mod, std::vector<typename F::Elem>& val, std::vector<typename F::Elem>& div, const F& field) { + size_t modsize = mod.size(); + CHECK_SAFE(mod.size() > 0 && mod.back() == 1); + if (val.size() < mod.size()) { + div.clear(); + return; + } + CHECK_SAFE(val.back() != 0); + div.resize(val.size() - mod.size() + 1); + while (val.size() >= modsize) { + auto term = val.back(); + div[val.size() - modsize] = term; + val.pop_back(); + if (term != 0) { + typename F::Multiplier mul(field, term); + for (size_t x = 0; x < mod.size() - 1; ++x) { + val[val.size() - modsize + 1 + x] ^= mul(mod[x]); + } + } + } +} + +/** Make a polynomial monic. */ +template<typename F> +typename F::Elem MakeMonic(std::vector<typename F::Elem>& a, const F& field) { + CHECK_SAFE(a.back() != 0); + if (a.back() == 1) return 0; + auto inv = field.Inv(a.back()); + typename F::Multiplier mul(field, inv); + a.back() = 1; + for (size_t i = 0; i < a.size() - 1; ++i) { + a[i] = mul(a[i]); + } + return inv; +} + +/** Compute the GCD of two polynomials, putting the result in a. b will be cleared. */ +template<typename F> +void GCD(std::vector<typename F::Elem>& a, std::vector<typename F::Elem>& b, const F& field) { + if (a.size() < b.size()) std::swap(a, b); + while (b.size() > 0) { + if (b.size() == 1) { + a.resize(1); + a[0] = 1; + return; + } + MakeMonic(b, field); + PolyMod(b, a, field); + std::swap(a, b); + } +} + +/** Square a polynomial. */ +template<typename F> +void Sqr(std::vector<typename F::Elem>& poly, const F& field) { + if (poly.size() == 0) return; + poly.resize(poly.size() * 2 - 1); + for (int x = poly.size() - 1; x >= 0; --x) { + poly[x] = (x & 1) ? 0 : field.Sqr(poly[x / 2]); + } +} + +/** Compute the trace map of (param*x) modulo mod, putting the result in out. */ +template<typename F> +void TraceMod(const std::vector<typename F::Elem>& mod, std::vector<typename F::Elem>& out, const typename F::Elem& param, const F& field) { + out.reserve(mod.size() * 2); + out.resize(2); + out[0] = 0; + out[1] = param; + + for (int i = 0; i < field.Bits() - 1; ++i) { + Sqr(out, field); + if (out.size() < 2) out.resize(2); + out[1] = param; + PolyMod(mod, out, field); + } +} + +/** One step of the root finding algorithm; finds roots of stack[pos] and adds them to roots. Stack elements >= pos are destroyed. + * + * It operates on a stack of polynomials. The polynomial operated on is `stack[pos]`, where elements of `stack` with index higher + * than `pos` are used as scratch space. + * + * `stack[pos]` is assumed to be square-free polynomial. If `fully_factorizable` is true, it is also assumed to have no irreducible + * factors of degree higher than 1. + + * This implements the Berlekamp trace algorithm, plus an efficient test to fail fast in + * case the polynomial cannot be fully factored. + */ +template<typename F> +bool RecFindRoots(std::vector<std::vector<typename F::Elem>>& stack, size_t pos, std::vector<typename F::Elem>& roots, bool fully_factorizable, int depth, typename F::Elem randv, const F& field) { + auto& ppoly = stack[pos]; + // We assert ppoly.size() > 1 (instead of just ppoly.size() > 0) to additionally exclude + // constants polynomials because + // - ppoly is not constant initially (this is ensured by FindRoots()), and + // - we never recurse on a constant polynomial. + CHECK_SAFE(ppoly.size() > 1 && ppoly.back() == 1); + /* 1st degree input: constant term is the root. */ + if (ppoly.size() == 2) { + roots.push_back(ppoly[0]); + return true; + } + /* 2nd degree input: use direct quadratic solver. */ + if (ppoly.size() == 3) { + CHECK_RETURN(ppoly[1] != 0, false); // Equations of the form (x^2 + a) have two identical solutions; contradicts square-free assumption. */ + auto input = field.Mul(ppoly[0], field.Sqr(field.Inv(ppoly[1]))); + auto root = field.Qrt(input); + if ((field.Sqr(root) ^ root) != input) { + CHECK_SAFE(!fully_factorizable); + return false; // No root found. + } + auto sol = field.Mul(root, ppoly[1]); + roots.push_back(sol); + roots.push_back(sol ^ ppoly[1]); + return true; + } + /* 3rd degree input and more: recurse further. */ + if (pos + 3 > stack.size()) { + // Allocate memory if necessary. + stack.resize((pos + 3) * 2); + } + auto& poly = stack[pos]; + auto& tmp = stack[pos + 1]; + auto& trace = stack[pos + 2]; + trace.clear(); + tmp.clear(); + for (int iter = 0;; ++iter) { + // Compute the polynomial (trace(x*randv) mod poly(x)) symbolically, + // and put the result in `trace`. + TraceMod(poly, trace, randv, field); + + if (iter >= 1 && !fully_factorizable) { + // If the polynomial cannot be factorized completely (it has an + // irreducible factor of degree higher than 1), we want to avoid + // the case where this is only detected after trying all BITS + // independent split attempts fail (see the assert below). + // + // Observe that if we call y = randv*x, it is true that: + // + // trace = y + y^2 + y^4 + y^8 + ... y^(FIELDSIZE/2) mod poly + // + // Due to the Frobenius endomorphism, this means: + // + // trace^2 = y^2 + y^4 + y^8 + ... + y^FIELDSIZE mod poly + // + // Or, adding them up: + // + // trace + trace^2 = y + y^FIELDSIZE mod poly. + // = randv*x + randv^FIELDSIZE*x^FIELDSIZE + // = randv*x + randv*x^FIELDSIZE + // = randv*(x + x^FIELDSIZE). + // (all mod poly) + // + // x + x^FIELDSIZE is the polynomial which has every field element + // as root once. Whenever x + x^FIELDSIZE is multiple of poly, + // this means it only has unique first degree factors. The same + // holds for its constant multiple randv*(x + x^FIELDSIZE) = + // trace + trace^2. + // + // We use this test to quickly verify whether the polynomial is + // fully factorizable after already having computed a trace. + // We don't invoke it immediately; only when splitting has failed + // at least once, which avoids it for most polynomials that are + // fully factorizable (or at least pushes the test down the + // recursion to factors which are smaller and thus faster). + tmp = trace; + Sqr(tmp, field); + for (size_t i = 0; i < trace.size(); ++i) { + tmp[i] ^= trace[i]; + } + while (tmp.size() && tmp.back() == 0) tmp.pop_back(); + PolyMod(poly, tmp, field); + + // Whenever the test fails, we can immediately abort the root + // finding. Whenever it succeeds, we can remember and pass down + // the information that it is in fact fully factorizable, avoiding + // the need to run the test again. + if (tmp.size() != 0) return false; + fully_factorizable = true; + } + + if (fully_factorizable) { + // Every succesful iteration of this algorithm splits the input + // polynomial further into buckets, each corresponding to a subset + // of 2^(BITS-depth) roots. If after depth splits the degree of + // the polynomial is >= 2^(BITS-depth), something is wrong. + CHECK_RETURN(field.Bits() - depth >= std::numeric_limits<decltype(poly.size())>::digits || + (poly.size() - 2) >> (field.Bits() - depth) == 0, false); + } + + depth++; + // In every iteration we multiply randv by 2. As a result, the set + // of randv values forms a GF(2)-linearly independent basis of splits. + randv = field.Mul2(randv); + tmp = poly; + GCD(trace, tmp, field); + if (trace.size() != poly.size() && trace.size() > 1) break; + } + MakeMonic(trace, field); + DivMod(trace, poly, tmp, field); + // At this point, the stack looks like [... (poly) tmp trace], and we want to recursively + // find roots of trace and tmp (= poly/trace). As we don't care about poly anymore, move + // trace into its position first. + std::swap(poly, trace); + // Now the stack is [... (trace) tmp ...]. First we factor tmp (at pos = pos+1), and then + // we factor trace (at pos = pos). + if (!RecFindRoots(stack, pos + 1, roots, fully_factorizable, depth, randv, field)) return false; + // The stack position pos contains trace, the polynomial with all of poly's roots which (after + // multiplication with randv) have trace 0. This is never the case for irreducible factors + // (which always end up in tmp), so we can set fully_factorizable to true when recursing. + bool ret = RecFindRoots(stack, pos, roots, true, depth, randv, field); + // Because of the above, recursion can never fail here. + CHECK_SAFE(ret); + return ret; +} + +/** Returns the roots of a fully factorizable polynomial + * + * This function assumes that the input polynomial is square-free + * and not the zero polynomial (represented by an empty vector). + * + * In case the square-free polynomial is not fully factorizable, i.e., it + * has fewer roots than its degree, the empty vector is returned. + */ +template<typename F> +std::vector<typename F::Elem> FindRoots(const std::vector<typename F::Elem>& poly, typename F::Elem basis, const F& field) { + std::vector<typename F::Elem> roots; + CHECK_RETURN(poly.size() != 0, {}); + CHECK_RETURN(basis != 0, {}); + if (poly.size() == 1) return roots; // No roots when the polynomial is a constant. + roots.reserve(poly.size() - 1); + std::vector<std::vector<typename F::Elem>> stack = {poly}; + + // Invoke the recursive factorization algorithm. + if (!RecFindRoots(stack, 0, roots, false, 0, basis, field)) { + // Not fully factorizable. + return {}; + } + CHECK_RETURN(poly.size() - 1 == roots.size(), {}); + return roots; +} + +template<typename F> +std::vector<typename F::Elem> BerlekampMassey(const std::vector<typename F::Elem>& syndromes, size_t max_degree, const F& field) { + std::vector<typename F::Multiplier> table; + std::vector<typename F::Elem> current, prev, tmp; + current.reserve(syndromes.size() / 2 + 1); + prev.reserve(syndromes.size() / 2 + 1); + tmp.reserve(syndromes.size() / 2 + 1); + current.resize(1); + current[0] = 1; + prev.resize(1); + prev[0] = 1; + typename F::Elem b = 1, b_inv = 1; + bool b_have_inv = true; + table.reserve(syndromes.size()); + + for (size_t n = 0; n != syndromes.size(); ++n) { + table.emplace_back(field, syndromes[n]); + auto discrepancy = syndromes[n]; + for (size_t i = 1; i < current.size(); ++i) discrepancy ^= table[n - i](current[i]); + if (discrepancy != 0) { + int x = n + 1 - (current.size() - 1) - (prev.size() - 1); + if (!b_have_inv) { + b_inv = field.Inv(b); + b_have_inv = true; + } + bool swap = 2 * (current.size() - 1) <= n; + if (swap) { + if (prev.size() + x - 1 > max_degree) return {}; // We'd exceed maximum degree + tmp = current; + current.resize(prev.size() + x); + } + typename F::Multiplier mul(field, field.Mul(discrepancy, b_inv)); + for (size_t i = 0; i < prev.size(); ++i) current[i + x] ^= mul(prev[i]); + if (swap) { + std::swap(prev, tmp); + b = discrepancy; + b_have_inv = false; + } + } + } + CHECK_RETURN(current.size() && current.back() != 0, {}); + return current; +} + +template<typename F> +std::vector<typename F::Elem> ReconstructAllSyndromes(const std::vector<typename F::Elem>& odd_syndromes, const F& field) { + std::vector<typename F::Elem> all_syndromes; + all_syndromes.resize(odd_syndromes.size() * 2); + for (size_t i = 0; i < odd_syndromes.size(); ++i) { + all_syndromes[i * 2] = odd_syndromes[i]; + all_syndromes[i * 2 + 1] = field.Sqr(all_syndromes[i]); + } + return all_syndromes; +} + +template<typename F> +void AddToOddSyndromes(std::vector<typename F::Elem>& osyndromes, typename F::Elem data, const F& field) { + auto sqr = field.Sqr(data); + typename F::Multiplier mul(field, sqr); + for (auto& osyndrome : osyndromes) { + osyndrome ^= data; + data = mul(data); + } +} + +template<typename F> +std::vector<typename F::Elem> FullDecode(const std::vector<typename F::Elem>& osyndromes, const F& field) { + auto asyndromes = ReconstructAllSyndromes<typename F::Elem>(osyndromes, field); + auto poly = BerlekampMassey(asyndromes, field); + std::reverse(poly.begin(), poly.end()); + return FindRoots(poly, field); +} + +template<typename F> +class SketchImpl final : public Sketch +{ + const F m_field; + std::vector<typename F::Elem> m_syndromes; + typename F::Elem m_basis; + +public: + template<typename... Args> + SketchImpl(int implementation, int bits, const Args&... args) : Sketch(implementation, bits), m_field(args...) { + std::random_device rng; + std::uniform_int_distribution<uint64_t> dist; + m_basis = m_field.FromSeed(dist(rng)); + } + + size_t Syndromes() const override { return m_syndromes.size(); } + void Init(int count) override { m_syndromes.assign(count, 0); } + + void Add(uint64_t val) override + { + auto elem = m_field.FromUint64(val); + AddToOddSyndromes(m_syndromes, elem, m_field); + } + + void Serialize(unsigned char* ptr) const override + { + BitWriter writer(ptr); + for (const auto& val : m_syndromes) { + m_field.Serialize(writer, val); + } + writer.Flush(); + } + + void Deserialize(const unsigned char* ptr) override + { + BitReader reader(ptr); + for (auto& val : m_syndromes) { + val = m_field.Deserialize(reader); + } + } + + int Decode(int max_count, uint64_t* out) const override + { + auto all_syndromes = ReconstructAllSyndromes(m_syndromes, m_field); + auto poly = BerlekampMassey(all_syndromes, max_count, m_field); + if (poly.size() == 0) return -1; + if (poly.size() == 1) return 0; + if ((int)poly.size() > 1 + max_count) return -1; + std::reverse(poly.begin(), poly.end()); + auto roots = FindRoots(poly, m_basis, m_field); + if (roots.size() == 0) return -1; + + for (const auto& root : roots) { + *(out++) = m_field.ToUint64(root); + } + return roots.size(); + } + + size_t Merge(const Sketch* other_sketch) override + { + // Sad cast. This is safe only because the caller code in minisketch.cpp checks + // that implementation and field size match. + const SketchImpl* other = static_cast<const SketchImpl*>(other_sketch); + m_syndromes.resize(std::min(m_syndromes.size(), other->m_syndromes.size())); + for (size_t i = 0; i < m_syndromes.size(); ++i) { + m_syndromes[i] ^= other->m_syndromes[i]; + } + return m_syndromes.size(); + } + + void SetSeed(uint64_t seed) override + { + if (seed == (uint64_t)-1) { + m_basis = 1; + } else { + m_basis = m_field.FromSeed(seed); + } + } +}; + +#endif diff --git a/src/minisketch/src/test.cpp b/src/minisketch/src/test.cpp new file mode 100644 index 0000000000..417937ea5f --- /dev/null +++ b/src/minisketch/src/test.cpp @@ -0,0 +1,316 @@ +/********************************************************************** + * Copyright (c) 2018,2021 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#include <algorithm> +#include <cstdio> +#include <limits> +#include <random> +#include <stdexcept> +#include <vector> + +#include "../include/minisketch.h" +#include "util.h" + +namespace { + +uint64_t Combination(uint64_t n, uint64_t k) { + if (n - k < k) k = n - k; + uint64_t ret = 1; + for (uint64_t i = 1; i <= k; ++i) { + ret = (ret * n) / i; + --n; + } + return ret; +} + +/** Create a vector with Minisketch objects, one for each implementation. */ +std::vector<Minisketch> CreateSketches(uint32_t bits, size_t capacity) { + if (!Minisketch::BitsSupported(bits)) return {}; + std::vector<Minisketch> ret; + for (uint32_t impl = 0; impl <= Minisketch::MaxImplementation(); ++impl) { + if (Minisketch::ImplementationSupported(bits, impl)) { + CHECK(Minisketch::BitsSupported(bits)); + ret.push_back(Minisketch(bits, impl, capacity)); + CHECK((bool)ret.back()); + } else { + // implementation 0 must always work unless field size is disabled + CHECK(impl != 0 || !Minisketch::BitsSupported(bits)); + } + } + return ret; +} + +/** Test properties by exhaustively decoding all 2**(bits*capacity) sketches + * with specified capacity and bits. */ +void TestExhaustive(uint32_t bits, size_t capacity) { + auto sketches = CreateSketches(bits, capacity); + if (sketches.empty()) return; + auto sketches_rebuild = CreateSketches(bits, capacity); + + std::vector<unsigned char> serialized; + std::vector<unsigned char> serialized_empty; + std::vector<uint64_t> counts; //!< counts[i] = number of results with i elements + std::vector<uint64_t> elements_0; //!< Result vector for elements for impl=0 + std::vector<uint64_t> elements_other; //!< Result vector for elements for other impls + std::vector<uint64_t> elements_too_small; //!< Result vector that's too small + + counts.resize(capacity + 1); + serialized.resize(sketches[0].GetSerializedSize()); + serialized_empty.resize(sketches[0].GetSerializedSize()); + + // Iterate over all (bits)-bit sketches with (capacity) syndromes. + for (uint64_t x = 0; (x >> (bits * capacity)) == 0; ++x) { + // Construct the serialization. + for (size_t i = 0; i < serialized.size(); ++i) { + serialized[i] = (x >> (i * 8)) & 0xFF; + } + + // Compute all the solutions + sketches[0].Deserialize(serialized); + elements_0.resize(64); + bool decodable_0 = sketches[0].Decode(elements_0); + std::sort(elements_0.begin(), elements_0.end()); + + // Verify that decoding with other implementations agrees. + for (size_t impl = 1; impl < sketches.size(); ++impl) { + sketches[impl].Deserialize(serialized); + elements_other.resize(64); + bool decodable_other = sketches[impl].Decode(elements_other); + CHECK(decodable_other == decodable_0); + std::sort(elements_other.begin(), elements_other.end()); + CHECK(elements_other == elements_0); + } + + // If there are solutions: + if (decodable_0) { + if (!elements_0.empty()) { + // Decoding with limit one less than the number of elements should fail. + elements_too_small.resize(elements_0.size() - 1); + for (size_t impl = 0; impl < sketches.size(); ++impl) { + CHECK(!sketches[impl].Decode(elements_too_small)); + } + } + + // Reconstruct the sketch from the solutions. + for (size_t impl = 0; impl < sketches.size(); ++impl) { + // Clear the sketch. + sketches_rebuild[impl].Deserialize(serialized_empty); + // Load all decoded elements into it. + for (uint64_t elem : elements_0) { + CHECK(elem != 0); + CHECK(elem >> bits == 0); + sketches_rebuild[impl].Add(elem); + } + // Reserialize the result + auto serialized_rebuild = sketches_rebuild[impl].Serialize(); + // Compare + CHECK(serialized == serialized_rebuild); + // Count it + if (impl == 0 && elements_0.size() <= capacity) ++counts[elements_0.size()]; + } + } + } + + // Verify that the number of decodable sketches with given elements is expected. + uint64_t mask = bits == 64 ? UINT64_MAX : (uint64_t{1} << bits) - 1; + for (uint64_t i = 0; i <= capacity && (i & mask) == i; ++i) { + CHECK(counts[i] == Combination(mask, i)); + } +} + +/** Test properties of sketches with random elements put in. */ +void TestRandomized(uint32_t bits, size_t max_capacity, size_t iter) { + std::random_device rnd; + std::uniform_int_distribution<uint64_t> capacity_dist(0, std::min<uint64_t>(std::numeric_limits<uint64_t>::max() >> (64 - bits), max_capacity)); + std::uniform_int_distribution<uint64_t> element_dist(1, std::numeric_limits<uint64_t>::max() >> (64 - bits)); + std::uniform_int_distribution<uint64_t> rand64(0, std::numeric_limits<uint64_t>::max()); + std::uniform_int_distribution<int64_t> size_offset_dist(-3, 3); + + std::vector<uint64_t> decode_0; + std::vector<uint64_t> decode_other; + std::vector<uint64_t> decode_temp; + std::vector<uint64_t> elements; + + for (size_t i = 0; i < iter; ++i) { + // Determine capacity, and construct Minisketch objects for all implementations. + uint64_t capacity = capacity_dist(rnd); + auto sketches = CreateSketches(bits, capacity); + // Sanity checks + if (sketches.empty()) return; + for (size_t impl = 0; impl < sketches.size(); ++impl) { + CHECK(sketches[impl].GetBits() == bits); + CHECK(sketches[impl].GetCapacity() == capacity); + CHECK(sketches[impl].GetSerializedSize() == sketches[0].GetSerializedSize()); + } + // Determine the number of elements, and create a vector to store them in. + size_t element_count = std::max<int64_t>(0, std::max<int64_t>(0, capacity + size_offset_dist(rnd))); + elements.resize(element_count); + // Add the elements to all sketches + for (size_t j = 0; j < element_count; ++j) { + uint64_t elem = element_dist(rnd); + CHECK(elem != 0); + elements[j] = elem; + for (auto& sketch : sketches) sketch.Add(elem); + } + // Remove pairs of duplicates in elements, as they cancel out. + std::sort(elements.begin(), elements.end()); + size_t real_element_count = element_count; + for (size_t pos = 0; pos + 1 < elements.size(); ++pos) { + if (elements[pos] == elements[pos + 1]) { + real_element_count -= 2; + // Set both elements to 0; afterwards we will move these to the end. + elements[pos] = 0; + elements[pos + 1] = 0; + ++pos; + } + } + if (real_element_count < element_count) { + // Move all introduced zeroes (masking duplicates) to the end. + std::sort(elements.begin(), elements.end(), [](uint64_t a, uint64_t b) { return a != b && (b == 0 || (a != 0 && a < b)); }); + CHECK(elements[real_element_count] == 0); + elements.resize(real_element_count); + } + // Create and compare serializations + auto serialized_0 = sketches[0].Serialize(); + for (size_t impl = 1; impl < sketches.size(); ++impl) { + auto serialized_other = sketches[impl].Serialize(); + CHECK(serialized_other == serialized_0); + } + // Deserialize and reserialize them + for (size_t impl = 0; impl < sketches.size(); ++impl) { + sketches[impl].Deserialize(serialized_0); + auto reserialized = sketches[impl].Serialize(); + CHECK(reserialized == serialized_0); + } + // Decode with limit set to the capacity, and compare results + decode_0.resize(capacity); + bool decodable_0 = sketches[0].Decode(decode_0); + std::sort(decode_0.begin(), decode_0.end()); + for (size_t impl = 1; impl < sketches.size(); ++impl) { + decode_other.resize(capacity); + bool decodable_other = sketches[impl].Decode(decode_other); + CHECK(decodable_other == decodable_0); + std::sort(decode_other.begin(), decode_other.end()); + CHECK(decode_other == decode_0); + } + // If the result is decodable, it should also be decodable with limit + // set to the actual number of elements, and not with one less. + if (decodable_0) { + for (auto& sketch : sketches) { + decode_temp.resize(decode_0.size()); + bool decodable = sketch.Decode(decode_temp); + CHECK(decodable); + std::sort(decode_temp.begin(), decode_temp.end()); + CHECK(decode_temp == decode_0); + if (!decode_0.empty()) { + decode_temp.resize(decode_0.size() - 1); + decodable = sketch.Decode(decode_temp); + CHECK(!decodable); + } + } + } + // If the actual number of elements is not higher than the capacity, the + // result should be decodable, and the result should match what we put in. + if (real_element_count <= capacity) { + CHECK(decodable_0); + CHECK(decode_0 == elements); + } + } +} + +void TestComputeFunctions() { + for (uint32_t bits = 0; bits <= 256; ++bits) { + for (uint32_t fpbits = 0; fpbits <= 512; ++fpbits) { + std::vector<size_t> table_max_elements(1025); + for (size_t capacity = 0; capacity <= 1024; ++capacity) { + table_max_elements[capacity] = minisketch_compute_max_elements(bits, capacity, fpbits); + // Exception for bits==0 + if (bits == 0) CHECK(table_max_elements[capacity] == 0); + // A sketch with capacity N cannot guarantee decoding more than N elements. + CHECK(table_max_elements[capacity] <= capacity); + // When asking for N bits of false positive protection, either no solution exists, or no more than ceil(N / bits) excess capacity should be needed. + if (bits > 0) CHECK(table_max_elements[capacity] == 0 || capacity - table_max_elements[capacity] <= (fpbits + bits - 1) / bits); + // Increasing capacity by one, if there is a solution, should always increment the max_elements by at least one as well. + if (capacity > 0) CHECK(table_max_elements[capacity] == 0 || table_max_elements[capacity] > table_max_elements[capacity - 1]); + } + + std::vector<size_t> table_capacity(513); + for (size_t max_elements = 0; max_elements <= 512; ++max_elements) { + table_capacity[max_elements] = minisketch_compute_capacity(bits, max_elements, fpbits); + // Exception for bits==0 + if (bits == 0) CHECK(table_capacity[max_elements] == 0); + // To be able to decode N elements, capacity needs to be at least N. + if (bits > 0) CHECK(table_capacity[max_elements] >= max_elements); + // A sketch of N bits in total cannot have more than N bits of false positive protection; + if (bits > 0) CHECK(bits * table_capacity[max_elements] >= fpbits); + // When asking for N bits of false positive protection, no more than ceil(N / bits) excess capacity should be needed. + if (bits > 0) CHECK(table_capacity[max_elements] - max_elements <= (fpbits + bits - 1) / bits); + // Increasing max_elements by one can only increment the capacity by 0 or 1. + if (max_elements > 0 && fpbits < 256) CHECK(table_capacity[max_elements] == table_capacity[max_elements - 1] || table_capacity[max_elements] == table_capacity[max_elements - 1] + 1); + // Check round-tripping max_elements->capacity->max_elements (only a lower bound) + CHECK(table_capacity[max_elements] <= 1024); + CHECK(table_max_elements[table_capacity[max_elements]] == 0 || table_max_elements[table_capacity[max_elements]] >= max_elements); + } + + for (size_t capacity = 0; capacity <= 512; ++capacity) { + // Check round-tripping capacity->max_elements->capacity (exact, if it exists) + CHECK(table_max_elements[capacity] <= 512); + CHECK(table_max_elements[capacity] == 0 || table_capacity[table_max_elements[capacity]] == capacity); + } + } + } +} + +} // namespace + +int main(int argc, char** argv) { + uint64_t test_complexity = 4; + if (argc > 1) { + size_t len = 0; + std::string arg{argv[1]}; + try { + test_complexity = 0; + long long complexity = std::stoll(arg, &len); + if (complexity >= 1 && len == arg.size() && ((uint64_t)complexity <= std::numeric_limits<uint64_t>::max() >> 10)) { + test_complexity = complexity; + } + } catch (const std::logic_error&) {} + if (test_complexity == 0) { + fprintf(stderr, "Invalid complexity specified: '%s'\n", arg.c_str()); + return 1; + } + } + +#ifdef MINISKETCH_VERIFY + const char* mode = " in verify mode"; +#else + const char* mode = ""; +#endif + printf("Running libminisketch tests%s with complexity=%llu\n", mode, (unsigned long long)test_complexity); + + TestComputeFunctions(); + + for (unsigned j = 2; j <= 64; ++j) { + TestRandomized(j, 8, (test_complexity << 10) / j); + TestRandomized(j, 128, (test_complexity << 7) / j); + TestRandomized(j, 4096, test_complexity / j); + } + + // Test capacity==0 together with all field sizes, and then + // all combinations of bits and capacity up to a certain bits*capacity, + // depending on test_complexity. + for (int weight = 0; weight <= 40; ++weight) { + for (int bits = 2; weight == 0 ? bits <= 64 : (bits <= 32 && bits <= weight); ++bits) { + int capacity = weight / bits; + if (capacity * bits != weight) continue; + TestExhaustive(bits, capacity); + } + if (weight >= 16 && test_complexity >> (weight - 16) == 0) break; + } + + printf("All tests successful.\n"); + return 0; +} diff --git a/src/minisketch/src/util.h b/src/minisketch/src/util.h new file mode 100644 index 0000000000..fdb3f3a231 --- /dev/null +++ b/src/minisketch/src/util.h @@ -0,0 +1,74 @@ +/********************************************************************** + * Copyright (c) 2018 Pieter Wuille, Greg Maxwell, Gleb Naumenko * + * Distributed under the MIT software license, see the accompanying * + * file LICENSE or http://www.opensource.org/licenses/mit-license.php.* + **********************************************************************/ + +#ifndef _MINISKETCH_UTIL_H_ +#define _MINISKETCH_UTIL_H_ + +#ifdef MINISKETCH_VERIFY +#include <stdio.h> +#endif + +#if !defined(__GNUC_PREREQ) +# if defined(__GNUC__)&&defined(__GNUC_MINOR__) +# define __GNUC_PREREQ(_maj,_min) \ + ((__GNUC__<<16)+__GNUC_MINOR__>=((_maj)<<16)+(_min)) +# else +# define __GNUC_PREREQ(_maj,_min) 0 +# endif +#endif + +#if __GNUC_PREREQ(3, 0) +#define EXPECT(x,c) __builtin_expect((x),(c)) +#else +#define EXPECT(x,c) (x) +#endif + +/* Assertion macros */ + +/** + * Unconditional failure on condition failure. + * Primarily used in testing harnesses. + */ +#define CHECK(cond) do { \ + if (EXPECT(!(cond), 0)) { \ + fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, "Check condition failed: " #cond); \ + abort(); \ + } \ +} while(0) + +/** + * Check macro that does nothing in normal non-verify builds but crashes in verify builds. + * This is used to test conditions at runtime that should always be true, but are either + * expensive to test or in locations where returning on failure would be messy. + */ +#ifdef MINISKETCH_VERIFY +#define CHECK_SAFE(cond) CHECK(cond) +#else +#define CHECK_SAFE(cond) +#endif + +/** + * Check a condition and return on failure in non-verify builds, crash in verify builds. + * Used for inexpensive conditions which believed to be always true in locations where + * a graceful exit is possible. + */ +#ifdef MINISKETCH_VERIFY +#define CHECK_RETURN(cond, rvar) do { \ + if (EXPECT(!(cond), 0)) { \ + fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, "Check condition failed: " #cond); \ + abort(); \ + return rvar; /* Does nothing, but causes compile to warn on incorrect return types. */ \ + } \ +} while(0) +#else +#define CHECK_RETURN(cond, rvar) do { \ + if (EXPECT(!(cond), 0)) { \ + return rvar; \ + } \ +} while(0) +#endif + +#endif diff --git a/src/minisketch/tests/pyminisketch.py b/src/minisketch/tests/pyminisketch.py new file mode 100755 index 0000000000..7a9ea9c1f1 --- /dev/null +++ b/src/minisketch/tests/pyminisketch.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 Pieter Wuille +# Distributed under the MIT software license, see the accompanying +# file LICENSE or http://www.opensource.org/licenses/mit-license.php. + +"""Native Python (slow) reimplementation of libminisketch' algorithms.""" + +import random +import unittest + +# Irreducible polynomials over GF(2) to use (represented as integers). +# +# Most fields can be defined by multiple such polynomials. Minisketch uses the one with the minimal +# number of nonzero coefficients, and tie-breaking by picking the lexicographically first among +# those. +# +# All polynomials for degrees 2 through 64 (inclusive) are given. +GF2_MODULI = [ + None, None, + 2**2 + 2**1 + 1, + 2**3 + 2**1 + 1, + 2**4 + 2**1 + 1, + 2**5 + 2**2 + 1, + 2**6 + 2**1 + 1, + 2**7 + 2**1 + 1, + 2**8 + 2**4 + 2**3 + 2**1 + 1, + 2**9 + 2**1 + 1, + 2**10 + 2**3 + 1, + 2**11 + 2**2 + 1, + 2**12 + 2**3 + 1, + 2**13 + 2**4 + 2**3 + 2**1 + 1, + 2**14 + 2**5 + 1, + 2**15 + 2**1 + 1, + 2**16 + 2**5 + 2**3 + 2**1 + 1, + 2**17 + 2**3 + 1, + 2**18 + 2**3 + 1, + 2**19 + 2**5 + 2**2 + 2**1 + 1, + 2**20 + 2**3 + 1, + 2**21 + 2**2 + 1, + 2**22 + 2**1 + 1, + 2**23 + 2**5 + 1, + 2**24 + 2**4 + 2**3 + 2**1 + 1, + 2**25 + 2**3 + 1, + 2**26 + 2**4 + 2**3 + 2**1 + 1, + 2**27 + 2**5 + 2**2 + 2**1 + 1, + 2**28 + 2**1 + 1, + 2**29 + 2**2 + 1, + 2**30 + 2**1 + 1, + 2**31 + 2**3 + 1, + 2**32 + 2**7 + 2**3 + 2**2 + 1, + 2**33 + 2**10 + 1, + 2**34 + 2**7 + 1, + 2**35 + 2**2 + 1, + 2**36 + 2**9 + 1, + 2**37 + 2**6 + 2**4 + 2**1 + 1, + 2**38 + 2**6 + 2**5 + 2**1 + 1, + 2**39 + 2**4 + 1, + 2**40 + 2**5 + 2**4 + 2**3 + 1, + 2**41 + 2**3 + 1, + 2**42 + 2**7 + 1, + 2**43 + 2**6 + 2**4 + 2**3 + 1, + 2**44 + 2**5 + 1, + 2**45 + 2**4 + 2**3 + 2**1 + 1, + 2**46 + 2**1 + 1, + 2**47 + 2**5 + 1, + 2**48 + 2**5 + 2**3 + 2**2 + 1, + 2**49 + 2**9 + 1, + 2**50 + 2**4 + 2**3 + 2**2 + 1, + 2**51 + 2**6 + 2**3 + 2**1 + 1, + 2**52 + 2**3 + 1, + 2**53 + 2**6 + 2**2 + 2**1 + 1, + 2**54 + 2**9 + 1, + 2**55 + 2**7 + 1, + 2**56 + 2**7 + 2**4 + 2**2 + 1, + 2**57 + 2**4 + 1, + 2**58 + 2**19 + 1, + 2**59 + 2**7 + 2**4 + 2**2 + 1, + 2**60 + 2**1 + 1, + 2**61 + 2**5 + 2**2 + 2**1 + 1, + 2**62 + 2**29 + 1, + 2**63 + 2**1 + 1, + 2**64 + 2**4 + 2**3 + 2**1 + 1 +] + +class GF2Ops: + """Class to perform GF(2^field_size) operations on elements represented as integers. + + Given that elements are represented as integers, addition is simply xor, and not + exposed here. + """ + + def __init__(self, field_size): + """Construct a GF2Ops object for the specified field size.""" + self.field_size = field_size + self._modulus = GF2_MODULI[field_size] + assert self._modulus is not None + + def mul2(self, x): + """Multiply x by 2 in GF(2^field_size).""" + x <<= 1 + if x >> self.field_size: + x ^= self._modulus + return x + + def mul(self, x, y): + """Multiply x by y in GF(2^field_size).""" + ret = 0 + while y: + if y & 1: + ret ^= x + y >>= 1 + x = self.mul2(x) + return ret + + def sqr(self, x): + """Square x in GF(2^field_size).""" + return self.mul(x, x) + + def inv(self, x): + """Compute the inverse of x in GF(2^field_size).""" + assert x != 0 + # Use the extended polynomial Euclidean GCD algorithm on (modulus, x), over GF(2). + # See https://en.wikipedia.org/wiki/Polynomial_greatest_common_divisor. + t1, t2 = 0, 1 + r1, r2 = self._modulus, x + r1l, r2l = self.field_size + 1, r2.bit_length() + while r2: + q = r1l - r2l + r1 ^= r2 << q + t1 ^= t2 << q + r1l = r1.bit_length() + if r1 < r2: + t1, t2 = t2, t1 + r1, r2 = r2, r1 + r1l, r2l = r2l, r1l + assert r1 == 1 + return t1 + +class TestGF2Ops(unittest.TestCase): + """Test class for basic arithmetic properties of GF2Ops.""" + + def field_size_test(self, field_size): + """Test operations for given field_size.""" + + gf = GF2Ops(field_size) + for i in range(100): + x = random.randrange(1 << field_size) + y = random.randrange(1 << field_size) + x2 = gf.mul2(x) + xy = gf.mul(x, y) + self.assertEqual(x2, gf.mul(x, 2)) # mul2(x) == x*2 + self.assertEqual(x2, gf.mul(2, x)) # mul2(x) == 2*x + self.assertEqual(xy == 0, x == 0 or y == 0) + self.assertEqual(xy == x, y == 1 or x == 0) + self.assertEqual(xy == y, x == 1 or y == 0) + self.assertEqual(xy, gf.mul(y, x)) # x*y == y*x + if i < 10: + xp = x + for _ in range(field_size): + xp = gf.sqr(xp) + self.assertEqual(xp, x) # x^(2^field_size) == x + if y != 0: + yi = gf.inv(y) + self.assertEqual(y == yi, y == 1) # y==1/x iff y==1 + self.assertEqual(gf.mul(y, yi), 1) # y*(1/y) == 1 + yii = gf.inv(yi) + self.assertEqual(y, yii) # 1/(1/y) == y + if x != 0: + xi = gf.inv(x) + xyi = gf.inv(xy) + self.assertEqual(xyi, gf.mul(xi, yi)) # (1/x)*(1/y) == 1/(x*y) + + def test(self): + """Run tests.""" + for field_size in range(2, 65): + self.field_size_test(field_size) + +# The operations below operate on polynomials over GF(2^field_size), represented as lists of +# integers: +# +# [a, b, c, ...] = a + b*x + c*x^2 + ... +# +# As an invariant, there are never any trailing zeroes in the list representation. +# +# Examples: +# * [] = 0 +# * [3] = 3 +# * [0, 1] = x +# * [2, 0, 5] = 5*x^2 + 2 + +def poly_monic(poly, gf): + """Return a monic version of the polynomial poly.""" + # Multiply every coefficient with the inverse of the top coefficient. + inv = gf.inv(poly[-1]) + return [gf.mul(inv, v) for v in poly] + +def poly_divmod(poly, mod, gf): + """Return the polynomial (quotient, remainder) of poly divided by mod.""" + assert len(mod) > 0 and mod[-1] == 1 # Require monic mod. + if len(poly) < len(mod): + return ([], poly) + val = list(poly) + div = [0 for _ in range(len(val) - len(mod) + 1)] + while len(val) >= len(mod): + term = val[-1] + div[len(val) - len(mod)] = term + # If the highest coefficient in val is nonzero, subtract a multiple of mod from it. + val.pop() + if term != 0: + for x in range(len(mod) - 1): + val[1 + x - len(mod)] ^= gf.mul(term, mod[x]) + # Prune trailing zero coefficients. + while len(val) > 0 and val[-1] == 0: + val.pop() + return div, val + +def poly_gcd(a, b, gf): + """Return the polynomial GCD of a and b.""" + if len(a) < len(b): + a, b = b, a + # Use Euclid's algorithm to find the GCD of a and b. + # see https://en.wikipedia.org/wiki/Polynomial_greatest_common_divisor#Euclid's_algorithm. + while len(b) > 0: + b = poly_monic(b, gf) + (_, b), a = poly_divmod(a, b, gf), b + return a + +def poly_sqr(poly, gf): + """Return the square of polynomial poly.""" + if len(poly) == 0: + return [] + # In characteristic-2 fields, thanks to Frobenius' endomorphism ((a + b)^2 = a^2 + b^2), + # squaring a polynomial is easy: square all the coefficients and interleave with zeroes. + # E.g., (3 + 5*x + 17*x^2)^2 = 3^2 + (5*x)^2 + (17*x^2)^2. + # See https://en.wikipedia.org/wiki/Frobenius_endomorphism. + return [0 if i & 1 else gf.sqr(poly[i // 2]) for i in range(2 * len(poly) - 1)] + +def poly_tracemod(poly, param, gf): + """Compute y + y^2 + y^4 + ... + y^(2^(field_size-1)) mod poly, where y = param*x.""" + out = [0, param] + for _ in range(gf.field_size - 1): + # In each loop iteration i, we start with out = y + y^2 + ... + y^(2^i). By squaring that we + # transform it into out = y^2 + y^4 + ... + y^(2^(i+1)). + out = poly_sqr(out, gf) + # Thus, we just need to add y again to it to get out = y + ... + y^(2^(i+1)). + while len(out) < 2: + out.append(0) + out[1] = param + # Finally take a modulus to keep the intermediary polynomials small. + _, out = poly_divmod(out, poly, gf) + return out + +def poly_frobeniusmod(poly, gf): + """Compute x^(2^field_size) mod poly.""" + out = [0, 1] + for _ in range(gf.field_size): + _, out = poly_divmod(poly_sqr(out, gf), poly, gf) + return out + +def poly_find_roots(poly, gf): + """Find the roots of poly if fully factorizable with unique roots, [] otherwise.""" + assert len(poly) > 0 + # If the polynomial is constant (and nonzero), it has no roots. + if len(poly) == 1: + return [] + # Make the polynomial monic (which doesn't change its roots). + poly = poly_monic(poly, gf) + # If the polynomial is of the form x+a, return a. + if len(poly) == 2: + return [poly[0]] + # Otherwise, first test that poly can be completely factored into unique roots. The polynomial + # x^(2^fieldsize)-x has every field element once as root. Thus we want to know that that is a + # multiple of poly. Compute x^(field_size) mod poly, which needs to equal x if that is the case + # (unless poly has degree <= 1, but that case is handled above). + if poly_frobeniusmod(poly, gf) != [0, 1]: + return [] + + def rec_split(poly, randv): + """Recursively split poly using the Berlekamp trace algorithm.""" + # See https://hal.archives-ouvertes.fr/hal-00626997/document. + assert len(poly) > 1 and poly[-1] == 1 # Require a monic poly. + # If poly is of the form x+a, its root is a. + if len(poly) == 2: + return [poly[0]] + # Try consecutive randomization factors randv, until one is found that factors poly. + while True: + # Compute the trace of (randv*x) mod poly. This is a polynomial that maps half of the + # domain to 0, and the other half to 1. Which half that is is controlled by randv. + # By taking it modulo poly, we only add a multiple of poly. Thus the result has at least + # the shared roots of the trace polynomial and poly still, but may have others. + trace = poly_tracemod(poly, randv, gf) + # Using the set {2^i*a for i=0..fieldsize-1} gives optimally independent randv values + # (no more than fieldsize are ever needed). + randv = gf.mul2(randv) + # Now take the GCD of this trace polynomial with poly. The result is a polynomial + # that only has the shared roots of the trace polynomial and poly as roots. + gcd = poly_gcd(trace, poly, gf) + # If the result has a degree higher than 1, and lower than that of poly, we found a + # useful factorization. + if len(gcd) != len(poly) and len(gcd) > 1: + break + # Otherwise, continue with another randv. + # Find the actual factors: the monic version of the GCD above, and poly divided by it. + factor1 = poly_monic(gcd, gf) + factor2, _ = poly_divmod(poly, gcd, gf) + # Recurse. + return rec_split(factor1, randv) + rec_split(factor2, randv) + + # Invoke the recursive splitting with a random initial factor, and sort the results. + return sorted(rec_split(poly, random.randrange(1, 1 << gf.field_size))) + +class TestPolyFindRoots(unittest.TestCase): + """Test class for poly_find_roots.""" + + def field_size_test(self, field_size): + """Run tests for given field_size.""" + gf = GF2Ops(field_size) + for test_size in [0, 1, 2, 3, 10]: + roots = [random.randrange(1 << field_size) for _ in range(test_size)] + roots_set = set(roots) + # Construct a polynomial with all elements of roots as roots (with multiplicity). + poly = [1] + for root in roots: + new_poly = [0] + poly + for n, c in enumerate(poly): + new_poly[n] ^= gf.mul(c, root) + poly = new_poly + # Invoke the root finding algorithm. + found_roots = poly_find_roots(poly, gf) + # The result must match the input, unless any roots were repeated. + if len(roots) == len(roots_set): + self.assertEqual(found_roots, sorted(roots)) + else: + self.assertEqual(found_roots, []) + + def test(self): + """Run tests.""" + for field_size in range(2, 65): + self.field_size_test(field_size) + +def berlekamp_massey(syndromes, gf): + """Implement the Berlekamp-Massey algorithm. + + Takes as input a sequence of GF(2^field_size) elements, and returns the shortest LSFR + that generates it, represented as a polynomial. + """ + # See https://en.wikipedia.org/wiki/Berlekamp%E2%80%93Massey_algorithm. + current = [1] + prev = [1] + b_inv = 1 + for n, discrepancy in enumerate(syndromes): + # Compute discrepancy + for i in range(1, len(current)): + discrepancy ^= gf.mul(syndromes[n - i], current[i]) + + # Correct if discrepancy is nonzero. + if discrepancy: + x = n + 1 - (len(current) - 1) - (len(prev) - 1) + if 2 * (len(current) - 1) <= n: + tmp = list(current) + current.extend(0 for _ in range(len(prev) + x - len(current))) + mul = gf.mul(discrepancy, b_inv) + for i, v in enumerate(prev): + current[i + x] ^= gf.mul(mul, v) + prev = tmp + b_inv = gf.inv(discrepancy) + else: + mul = gf.mul(discrepancy, b_inv) + for i, v in enumerate(prev): + current[i + x] ^= gf.mul(mul, v) + return current + +class Minisketch: + """A Minisketch sketch. + + This represents a sketch of a certain capacity, with elements of a certain bit size. + """ + + def __init__(self, field_size, capacity): + """Initialize an empty sketch with the specified field_size size and capacity.""" + self.field_size = field_size + self.capacity = capacity + self.odd_syndromes = [0] * capacity + self.gf = GF2Ops(field_size) + + def add(self, element): + """Add an element to this sketch. 1 <= element < 2**field_size.""" + sqr = self.gf.sqr(element) + for pos in range(self.capacity): + self.odd_syndromes[pos] ^= element + element = self.gf.mul(sqr, element) + + def serialized_size(self): + """Compute how many bytes a serialization of this sketch will be in size.""" + return (self.capacity * self.field_size + 7) // 8 + + def serialize(self): + """Serialize this sketch to bytes.""" + val = 0 + for i in range(self.capacity): + val |= self.odd_syndromes[i] << (self.field_size * i) + return val.to_bytes(self.serialized_size(), 'little') + + def deserialize(self, byte_data): + """Deserialize a byte array into this sketch, overwriting its contents.""" + assert len(byte_data) == self.serialized_size() + val = int.from_bytes(byte_data, 'little') + for i in range(self.capacity): + self.odd_syndromes[i] = (val >> (self.field_size * i)) & ((1 << self.field_size) - 1) + + def clone(self): + """Return a clone of this sketch.""" + ret = Minisketch(self.field_size, self.capacity) + ret.odd_syndromes = list(self.odd_syndromes) + ret.gf = self.gf + return ret + + def merge(self, other): + """Merge a sketch with another sketch. Corresponds to XOR'ing their serializations.""" + assert self.capacity == other.capacity + assert self.field_size == other.field_size + for i in range(self.capacity): + self.odd_syndromes[i] ^= other.odd_syndromes[i] + + def decode(self, max_count=None): + """Decode the contents of this sketch. + + Returns either a list of elements or None if undecodable. + """ + # We know the odd syndromes s1=x+y+..., s3=x^3+y^3+..., s5=..., and reconstruct the even + # syndromes from this: + # * s2 = x^2+y^2+.... = (x+y+...)^2 = s1^2 + # * s4 = x^4+y^4+.... = (x^2+y^2+...)^2 = s2^2 + # * s6 = x^6+y^6+.... = (x^3+y^3+...)^2 = s3^2 + all_syndromes = [0 for _ in range(2 * len(self.odd_syndromes))] + for i in range(len(self.odd_syndromes)): + all_syndromes[i * 2] = self.odd_syndromes[i] + all_syndromes[i * 2 + 1] = self.gf.sqr(all_syndromes[i]) + # Given the syndromes, find the polynomial that generates them. + poly = berlekamp_massey(all_syndromes, self.gf) + # Deal with failure and trivial cases. + if len(poly) == 0: + return None + if len(poly) == 1: + return [] + if max_count is not None and len(poly) > 1 + max_count: + return None + # If the polynomial can be factored into (1-m1*x)*(1-m2*x)*...*(1-mn*x), then {m1,m2,...,mn} + # is our set. As each factor (1-m*x) has 1/m as root, we're really just looking for the + # inverses of the roots. We find these by reversing the order of the coefficients, and + # finding the roots. + roots = poly_find_roots(list(reversed(poly)), self.gf) + if len(roots) == 0: + return None + return roots + +class TestMinisketch(unittest.TestCase): + """Test class for Minisketch.""" + + @classmethod + def construct_data(cls, field_size, num_a_only, num_b_only, num_both): + """Construct two random lists of elements in [1..2**field_size-1]. + + Each list will have unique elements that don't appear in the other (num_a_only in the first + and num_b_only in the second), and num_both elements will appear in both.""" + sample = [] + # Simulate random.sample here (which doesn't work with ranges over 2**63). + for _ in range(num_a_only + num_b_only + num_both): + while True: + r = random.randrange(1, 1 << field_size) + if r not in sample: + sample.append(r) + break + full_a = sample[:num_a_only + num_both] + full_b = sample[num_a_only:] + random.shuffle(full_a) + random.shuffle(full_b) + return full_a, full_b + + def field_size_capacity_test(self, field_size, capacity): + """Test Minisketch methods for a specific field and capacity.""" + used_capacity = random.randrange(capacity + 1) + num_a = random.randrange(used_capacity + 1) + num_both = random.randrange(min(2 * capacity, (1 << field_size) - 1 - used_capacity) + 1) + full_a, full_b = self.construct_data(field_size, num_a, used_capacity - num_a, num_both) + sketch_a = Minisketch(field_size, capacity) + sketch_b = Minisketch(field_size, capacity) + for v in full_a: + sketch_a.add(v) + for v in full_b: + sketch_b.add(v) + sketch_combined = sketch_a.clone() + sketch_b_ser = sketch_b.serialize() + sketch_b_received = Minisketch(field_size, capacity) + sketch_b_received.deserialize(sketch_b_ser) + sketch_combined.merge(sketch_b_received) + decode = sketch_combined.decode() + self.assertEqual(decode, sorted(set(full_a) ^ set(full_b))) + + def test(self): + """Run tests.""" + for field_size in range(2, 65): + for capacity in [0, 1, 2, 5, 10, field_size]: + self.field_size_capacity_test(field_size, min(capacity, (1 << field_size) - 1)) + +if __name__ == '__main__': + unittest.main() diff --git a/src/net.cpp b/src/net.cpp index ad558dd598..2d9e69b6fb 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -10,6 +10,7 @@ #include <net.h> #include <addrdb.h> +#include <addrman.h> #include <banman.h> #include <clientversion.h> #include <compat.h> @@ -230,9 +231,27 @@ std::optional<CAddress> GetLocalAddrForPeer(CNode *pnode) return std::nullopt; } +/** + * If an IPv6 address belongs to the address range used by the CJDNS network and + * the CJDNS network is reachable (-cjdnsreachable config is set), then change + * the type from NET_IPV6 to NET_CJDNS. + * @param[in] service Address to potentially convert. + * @return a copy of `service` either unmodified or changed to CJDNS. + */ +CService MaybeFlipIPv6toCJDNS(const CService& service) +{ + CService ret{service}; + if (ret.m_net == NET_IPV6 && ret.m_addr[0] == 0xfc && IsReachable(NET_CJDNS)) { + ret.m_net = NET_CJDNS; + } + return ret; +} + // learn a new local address -bool AddLocal(const CService& addr, int nScore) +bool AddLocal(const CService& addr_, int nScore) { + CService addr{MaybeFlipIPv6toCJDNS(addr_)}; + if (!addr.IsRoutable()) return false; @@ -308,8 +327,8 @@ bool IsLocal(const CService& addr) CNode* CConnman::FindNode(const CNetAddr& ip) { - LOCK(cs_vNodes); - for (CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (CNode* pnode : m_nodes) { if (static_cast<CNetAddr>(pnode->addr) == ip) { return pnode; } @@ -319,8 +338,8 @@ CNode* CConnman::FindNode(const CNetAddr& ip) CNode* CConnman::FindNode(const CSubNet& subNet) { - LOCK(cs_vNodes); - for (CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (CNode* pnode : m_nodes) { if (subNet.Match(static_cast<CNetAddr>(pnode->addr))) { return pnode; } @@ -330,8 +349,8 @@ CNode* CConnman::FindNode(const CSubNet& subNet) CNode* CConnman::FindNode(const std::string& addrName) { - LOCK(cs_vNodes); - for (CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (CNode* pnode : m_nodes) { if (pnode->m_addr_name == addrName) { return pnode; } @@ -341,8 +360,8 @@ CNode* CConnman::FindNode(const std::string& addrName) CNode* CConnman::FindNode(const CService& addr) { - LOCK(cs_vNodes); - for (CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (CNode* pnode : m_nodes) { if (static_cast<CService>(pnode->addr) == addr) { return pnode; } @@ -357,8 +376,8 @@ bool CConnman::AlreadyConnectedToAddress(const CAddress& addr) bool CConnman::CheckIncomingNonce(uint64_t nonce) { - LOCK(cs_vNodes); - for (const CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (const CNode* pnode : m_nodes) { if (!pnode->fSuccessfullyConnected && !pnode->IsInboundConn() && pnode->GetLocalNonce() == nonce) return false; } @@ -409,14 +428,15 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo if (pszDest) { std::vector<CService> resolved; if (Lookup(pszDest, resolved, default_port, fNameLookup && !HaveNameProxy(), 256) && !resolved.empty()) { - addrConnect = CAddress(resolved[GetRand(resolved.size())], NODE_NONE); + const CService rnd{resolved[GetRand(resolved.size())]}; + addrConnect = CAddress{MaybeFlipIPv6toCJDNS(rnd), NODE_NONE}; if (!addrConnect.IsValid()) { LogPrint(BCLog::NET, "Resolver returned invalid address %s for %s\n", addrConnect.ToString(), pszDest); return nullptr; } // It is possible that we already have a connection to the IP/port pszDest resolved to. // In that case, drop the connection that was just created. - LOCK(cs_vNodes); + LOCK(m_nodes_mutex); CNode* pnode = FindNode(static_cast<CService>(addrConnect)); if (pnode) { LogPrintf("Failed to open new connection, already connected\n"); @@ -626,25 +646,26 @@ bool CNode::ReceiveMsgBytes(Span<const uint8_t> msg_bytes, bool& complete) if (m_deserializer->Complete()) { // decompose a transport agnostic CNetMessage from the deserializer - uint32_t out_err_raw_size{0}; - std::optional<CNetMessage> result{m_deserializer->GetMessage(time, out_err_raw_size)}; - if (!result) { + bool reject_message{false}; + CNetMessage msg = m_deserializer->GetMessage(time, reject_message); + if (reject_message) { // Message deserialization failed. Drop the message but don't disconnect the peer. // store the size of the corrupt message - mapRecvBytesPerMsgCmd.find(NET_MESSAGE_COMMAND_OTHER)->second += out_err_raw_size; + mapRecvBytesPerMsgCmd.at(NET_MESSAGE_COMMAND_OTHER) += msg.m_raw_message_size; continue; } - //store received bytes per message command - //to prevent a memory DOS, only allow valid commands - mapMsgCmdSize::iterator i = mapRecvBytesPerMsgCmd.find(result->m_command); - if (i == mapRecvBytesPerMsgCmd.end()) + // Store received bytes per message command + // to prevent a memory DOS, only allow valid commands + auto i = mapRecvBytesPerMsgCmd.find(msg.m_command); + if (i == mapRecvBytesPerMsgCmd.end()) { i = mapRecvBytesPerMsgCmd.find(NET_MESSAGE_COMMAND_OTHER); + } assert(i != mapRecvBytesPerMsgCmd.end()); - i->second += result->m_raw_message_size; + i->second += msg.m_raw_message_size; // push the message to the process queue, - vRecvMsg.push_back(std::move(*result)); + vRecvMsg.push_back(std::move(msg)); complete = true; } @@ -718,16 +739,18 @@ const uint256& V1TransportDeserializer::GetMessageHash() const return data_hash; } -std::optional<CNetMessage> V1TransportDeserializer::GetMessage(const std::chrono::microseconds time, uint32_t& out_err_raw_size) +CNetMessage V1TransportDeserializer::GetMessage(const std::chrono::microseconds time, bool& reject_message) { + // Initialize out parameter + reject_message = false; // decompose a single CNetMessage from the TransportDeserializer - std::optional<CNetMessage> msg(std::move(vRecv)); + CNetMessage msg(std::move(vRecv)); // store command string, time, and sizes - msg->m_command = hdr.GetCommand(); - msg->m_time = time; - msg->m_message_size = hdr.nMessageSize; - msg->m_raw_message_size = hdr.nMessageSize + CMessageHeader::HEADER_SIZE; + msg.m_command = hdr.GetCommand(); + msg.m_time = time; + msg.m_message_size = hdr.nMessageSize; + msg.m_raw_message_size = hdr.nMessageSize + CMessageHeader::HEADER_SIZE; uint256 hash = GetMessageHash(); @@ -737,17 +760,15 @@ std::optional<CNetMessage> V1TransportDeserializer::GetMessage(const std::chrono // Check checksum and header command string if (memcmp(hash.begin(), hdr.pchChecksum, CMessageHeader::CHECKSUM_SIZE) != 0) { LogPrint(BCLog::NET, "Header error: Wrong checksum (%s, %u bytes), expected %s was %s, peer=%d\n", - SanitizeString(msg->m_command), msg->m_message_size, - HexStr(Span<uint8_t>(hash.begin(), hash.begin() + CMessageHeader::CHECKSUM_SIZE)), + SanitizeString(msg.m_command), msg.m_message_size, + HexStr(Span{hash}.first(CMessageHeader::CHECKSUM_SIZE)), HexStr(hdr.pchChecksum), m_node_id); - out_err_raw_size = msg->m_raw_message_size; - msg = std::nullopt; + reject_message = true; } else if (!hdr.IsCommandValid()) { LogPrint(BCLog::NET, "Header error: Invalid message type (%s, %u bytes), peer=%d\n", - SanitizeString(hdr.GetCommand()), msg->m_message_size, m_node_id); - out_err_raw_size = msg->m_raw_message_size; - msg.reset(); + SanitizeString(hdr.GetCommand()), msg.m_message_size, m_node_id); + reject_message = true; } // Always reset the network deserializer (prepare for the next message) @@ -1036,8 +1057,8 @@ bool CConnman::AttemptToEvictConnection() std::vector<NodeEvictionCandidate> vEvictionCandidates; { - LOCK(cs_vNodes); - for (const CNode* node : vNodes) { + LOCK(m_nodes_mutex); + for (const CNode* node : m_nodes) { if (node->HasPermission(NetPermissionFlags::NoBan)) continue; if (!node->IsInboundConn()) @@ -1064,8 +1085,8 @@ bool CConnman::AttemptToEvictConnection() if (!node_id_to_evict) { return false; } - LOCK(cs_vNodes); - for (CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (CNode* pnode : m_nodes) { if (pnode->GetId() == *node_id_to_evict) { LogPrint(BCLog::NET, "selected %s connection for eviction peer=%d; disconnecting\n", pnode->ConnectionTypeAsString(), pnode->GetId()); pnode->fDisconnect = true; @@ -1091,9 +1112,11 @@ void CConnman::AcceptConnection(const ListenSocket& hListenSocket) { if (!addr.SetSockAddr((const struct sockaddr*)&sockaddr)) { LogPrintf("Warning: Unknown socket family\n"); + } else { + addr = CAddress{MaybeFlipIPv6toCJDNS(addr), NODE_NONE}; } - const CAddress addr_bind = GetBindAddress(hSocket); + const CAddress addr_bind{MaybeFlipIPv6toCJDNS(GetBindAddress(hSocket)), NODE_NONE}; NetPermissionFlags permissionFlags = NetPermissionFlags::None; hListenSocket.AddSocketPermissionFlags(permissionFlags); @@ -1119,8 +1142,8 @@ void CConnman::CreateNodeFromAcceptedSocket(SOCKET hSocket, } { - LOCK(cs_vNodes); - for (const CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (const CNode* pnode : m_nodes) { if (pnode->IsInboundConn()) nInbound++; } } @@ -1188,8 +1211,8 @@ void CConnman::CreateNodeFromAcceptedSocket(SOCKET hSocket, LogPrint(BCLog::NET, "connection from %s accepted\n", addr.ToString()); { - LOCK(cs_vNodes); - vNodes.push_back(pnode); + LOCK(m_nodes_mutex); + m_nodes.push_back(pnode); } // We received a new connection, harvest entropy from the time (and our peer count) @@ -1216,8 +1239,8 @@ bool CConnman::AddConnection(const std::string& address, ConnectionType conn_typ } // no default case, so the compiler can warn about missing cases // Count existing connections - int existing_connections = WITH_LOCK(cs_vNodes, - return std::count_if(vNodes.begin(), vNodes.end(), [conn_type](CNode* node) { return node->m_conn_type == conn_type; });); + int existing_connections = WITH_LOCK(m_nodes_mutex, + return std::count_if(m_nodes.begin(), m_nodes.end(), [conn_type](CNode* node) { return node->m_conn_type == conn_type; });); // Max connections of specified type already exist if (max_connections != std::nullopt && existing_connections >= max_connections) return false; @@ -1233,11 +1256,11 @@ bool CConnman::AddConnection(const std::string& address, ConnectionType conn_typ void CConnman::DisconnectNodes() { { - LOCK(cs_vNodes); + LOCK(m_nodes_mutex); if (!fNetworkActive) { // Disconnect any connected nodes - for (CNode* pnode : vNodes) { + for (CNode* pnode : m_nodes) { if (!pnode->fDisconnect) { LogPrint(BCLog::NET, "Network not active, dropping peer=%d\n", pnode->GetId()); pnode->fDisconnect = true; @@ -1246,13 +1269,13 @@ void CConnman::DisconnectNodes() } // Disconnect unused nodes - std::vector<CNode*> vNodesCopy = vNodes; - for (CNode* pnode : vNodesCopy) + std::vector<CNode*> nodes_copy = m_nodes; + for (CNode* pnode : nodes_copy) { if (pnode->fDisconnect) { - // remove from vNodes - vNodes.erase(remove(vNodes.begin(), vNodes.end(), pnode), vNodes.end()); + // remove from m_nodes + m_nodes.erase(remove(m_nodes.begin(), m_nodes.end(), pnode), m_nodes.end()); // release outbound grant (if any) pnode->grantOutbound.Release(); @@ -1262,18 +1285,18 @@ void CConnman::DisconnectNodes() // hold in disconnected pool until all refs are released pnode->Release(); - vNodesDisconnected.push_back(pnode); + m_nodes_disconnected.push_back(pnode); } } } { // Delete disconnected nodes - std::list<CNode*> vNodesDisconnectedCopy = vNodesDisconnected; - for (CNode* pnode : vNodesDisconnectedCopy) + std::list<CNode*> nodes_disconnected_copy = m_nodes_disconnected; + for (CNode* pnode : nodes_disconnected_copy) { // Destroy the object only after other threads have stopped using it. if (pnode->GetRefCount() <= 0) { - vNodesDisconnected.remove(pnode); + m_nodes_disconnected.remove(pnode); DeleteNode(pnode); } } @@ -1282,15 +1305,15 @@ void CConnman::DisconnectNodes() void CConnman::NotifyNumConnectionsChanged() { - size_t vNodesSize; + size_t nodes_size; { - LOCK(cs_vNodes); - vNodesSize = vNodes.size(); + LOCK(m_nodes_mutex); + nodes_size = m_nodes.size(); } - if(vNodesSize != nPrevNodeCount) { - nPrevNodeCount = vNodesSize; + if(nodes_size != nPrevNodeCount) { + nPrevNodeCount = nodes_size; if (m_client_interface) { - m_client_interface->NotifyNumConnectionsChanged(vNodesSize); + m_client_interface->NotifyNumConnectionsChanged(nodes_size); } } } @@ -1331,46 +1354,45 @@ bool CConnman::InactivityCheck(const CNode& node) const return false; } -bool CConnman::GenerateSelectSet(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_set, std::set<SOCKET> &error_set) +bool CConnman::GenerateSelectSet(const std::vector<CNode*>& nodes, + std::set<SOCKET>& recv_set, + std::set<SOCKET>& send_set, + std::set<SOCKET>& error_set) { for (const ListenSocket& hListenSocket : vhListenSocket) { recv_set.insert(hListenSocket.socket); } - { - LOCK(cs_vNodes); - for (CNode* pnode : vNodes) + for (CNode* pnode : nodes) { + // Implement the following logic: + // * If there is data to send, select() for sending data. As this only + // happens when optimistic write failed, we choose to first drain the + // write buffer in this case before receiving more. This avoids + // needlessly queueing received data, if the remote peer is not themselves + // receiving data. This means properly utilizing TCP flow control signalling. + // * Otherwise, if there is space left in the receive buffer, select() for + // receiving data. + // * Hand off all complete messages to the processor, to be handled without + // blocking here. + + bool select_recv = !pnode->fPauseRecv; + bool select_send; { - // Implement the following logic: - // * If there is data to send, select() for sending data. As this only - // happens when optimistic write failed, we choose to first drain the - // write buffer in this case before receiving more. This avoids - // needlessly queueing received data, if the remote peer is not themselves - // receiving data. This means properly utilizing TCP flow control signalling. - // * Otherwise, if there is space left in the receive buffer, select() for - // receiving data. - // * Hand off all complete messages to the processor, to be handled without - // blocking here. - - bool select_recv = !pnode->fPauseRecv; - bool select_send; - { - LOCK(pnode->cs_vSend); - select_send = !pnode->vSendMsg.empty(); - } + LOCK(pnode->cs_vSend); + select_send = !pnode->vSendMsg.empty(); + } - LOCK(pnode->cs_hSocket); - if (pnode->hSocket == INVALID_SOCKET) - continue; + LOCK(pnode->cs_hSocket); + if (pnode->hSocket == INVALID_SOCKET) + continue; - error_set.insert(pnode->hSocket); - if (select_send) { - send_set.insert(pnode->hSocket); - continue; - } - if (select_recv) { - recv_set.insert(pnode->hSocket); - } + error_set.insert(pnode->hSocket); + if (select_send) { + send_set.insert(pnode->hSocket); + continue; + } + if (select_recv) { + recv_set.insert(pnode->hSocket); } } @@ -1378,10 +1400,13 @@ bool CConnman::GenerateSelectSet(std::set<SOCKET> &recv_set, std::set<SOCKET> &s } #ifdef USE_POLL -void CConnman::SocketEvents(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_set, std::set<SOCKET> &error_set) +void CConnman::SocketEvents(const std::vector<CNode*>& nodes, + std::set<SOCKET>& recv_set, + std::set<SOCKET>& send_set, + std::set<SOCKET>& error_set) { std::set<SOCKET> recv_select_set, send_select_set, error_select_set; - if (!GenerateSelectSet(recv_select_set, send_select_set, error_select_set)) { + if (!GenerateSelectSet(nodes, recv_select_set, send_select_set, error_select_set)) { interruptNet.sleep_for(std::chrono::milliseconds(SELECT_TIMEOUT_MILLISECONDS)); return; } @@ -1420,10 +1445,13 @@ void CConnman::SocketEvents(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_s } } #else -void CConnman::SocketEvents(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_set, std::set<SOCKET> &error_set) +void CConnman::SocketEvents(const std::vector<CNode*>& nodes, + std::set<SOCKET>& recv_set, + std::set<SOCKET>& send_set, + std::set<SOCKET>& error_set) { std::set<SOCKET> recv_select_set, send_select_set, error_select_set; - if (!GenerateSelectSet(recv_select_set, send_select_set, error_select_set)) { + if (!GenerateSelectSet(nodes, recv_select_set, send_select_set, error_select_set)) { interruptNet.sleep_for(std::chrono::milliseconds(SELECT_TIMEOUT_MILLISECONDS)); return; } @@ -1497,34 +1525,33 @@ void CConnman::SocketEvents(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_s void CConnman::SocketHandler() { - std::set<SOCKET> recv_set, send_set, error_set; - SocketEvents(recv_set, send_set, error_set); - - if (interruptNet) return; + std::set<SOCKET> recv_set; + std::set<SOCKET> send_set; + std::set<SOCKET> error_set; - // - // Accept new connections - // - for (const ListenSocket& hListenSocket : vhListenSocket) { - if (hListenSocket.socket != INVALID_SOCKET && recv_set.count(hListenSocket.socket) > 0) - { - AcceptConnection(hListenSocket); - } - } + const NodesSnapshot snap{*this, /*shuffle=*/false}; - // - // Service each socket - // - std::vector<CNode*> vNodesCopy; - { - LOCK(cs_vNodes); - vNodesCopy = vNodes; - for (CNode* pnode : vNodesCopy) - pnode->AddRef(); + // Check for the readiness of the already connected sockets and the + // listening sockets in one call ("readiness" as in poll(2) or + // select(2)). If none are ready, wait for a short while and return + // empty sets. + SocketEvents(snap.Nodes(), recv_set, send_set, error_set); + + // Service (send/receive) each of the already connected nodes. + SocketHandlerConnected(snap.Nodes(), recv_set, send_set, error_set); } - for (CNode* pnode : vNodesCopy) - { + + // Accept new connections from listening sockets. + SocketHandlerListening(recv_set); +} + +void CConnman::SocketHandlerConnected(const std::vector<CNode*>& nodes, + const std::set<SOCKET>& recv_set, + const std::set<SOCKET>& send_set, + const std::set<SOCKET>& error_set) +{ + for (CNode* pnode : nodes) { if (interruptNet) return; @@ -1556,8 +1583,9 @@ void CConnman::SocketHandler() if (nBytes > 0) { bool notify = false; - if (!pnode->ReceiveMsgBytes(Span<const uint8_t>(pchBuf, nBytes), notify)) + if (!pnode->ReceiveMsgBytes({pchBuf, (size_t)nBytes}, notify)) { pnode->CloseSocketDisconnect(); + } RecordBytesRecv(nBytes); if (notify) { size_t nSizeAdded = 0; @@ -1606,10 +1634,17 @@ void CConnman::SocketHandler() if (InactivityCheck(*pnode)) pnode->fDisconnect = true; } - { - LOCK(cs_vNodes); - for (CNode* pnode : vNodesCopy) - pnode->Release(); +} + +void CConnman::SocketHandlerListening(const std::set<SOCKET>& recv_set) +{ + for (const ListenSocket& listen_socket : vhListenSocket) { + if (interruptNet) { + return; + } + if (recv_set.count(listen_socket.socket) > 0) { + AcceptConnection(listen_socket); + } } } @@ -1683,8 +1718,8 @@ void CConnman::ThreadDNSAddressSeed() int nRelevant = 0; { - LOCK(cs_vNodes); - for (const CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (const CNode* pnode : m_nodes) { if (pnode->fSuccessfullyConnected && pnode->IsFullOutboundConn()) ++nRelevant; } } @@ -1792,8 +1827,8 @@ int CConnman::GetExtraFullOutboundCount() const { int full_outbound_peers = 0; { - LOCK(cs_vNodes); - for (const CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (const CNode* pnode : m_nodes) { if (pnode->fSuccessfullyConnected && !pnode->fDisconnect && pnode->IsFullOutboundConn()) { ++full_outbound_peers; } @@ -1806,8 +1841,8 @@ int CConnman::GetExtraBlockRelayCount() const { int block_relay_peers = 0; { - LOCK(cs_vNodes); - for (const CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (const CNode* pnode : m_nodes) { if (pnode->fSuccessfullyConnected && !pnode->fDisconnect && pnode->IsBlockOnlyConn()) { ++block_relay_peers; } @@ -1878,8 +1913,8 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // Checking !dnsseed is cheaper before locking 2 mutexes. if (!add_fixed_seeds_now && !dnsseed) { - LOCK2(m_addr_fetches_mutex, cs_vAddedNodes); - if (m_addr_fetches.empty() && vAddedNodes.empty()) { + LOCK2(m_addr_fetches_mutex, m_added_nodes_mutex); + if (m_addr_fetches.empty() && m_added_nodes.empty()) { add_fixed_seeds_now = true; LogPrintf("Adding fixed seeds as -dnsseed=0, -addnode is not provided and all -seednode(s) attempted\n"); } @@ -1904,8 +1939,8 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) std::set<std::vector<unsigned char> > setConnected; { - LOCK(cs_vNodes); - for (const CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (const CNode* pnode : m_nodes) { if (pnode->IsFullOutboundConn()) nOutboundFullRelay++; if (pnode->IsBlockOnlyConn()) nOutboundBlockRelay++; @@ -2093,8 +2128,8 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) std::vector<CAddress> CConnman::GetCurrentBlockRelayOnlyConns() const { std::vector<CAddress> ret; - LOCK(cs_vNodes); - for (const CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (const CNode* pnode : m_nodes) { if (pnode->IsBlockOnlyConn()) { ret.push_back(pnode->addr); } @@ -2109,9 +2144,9 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const std::list<std::string> lAddresses(0); { - LOCK(cs_vAddedNodes); - ret.reserve(vAddedNodes.size()); - std::copy(vAddedNodes.cbegin(), vAddedNodes.cend(), std::back_inserter(lAddresses)); + LOCK(m_added_nodes_mutex); + ret.reserve(m_added_nodes.size()); + std::copy(m_added_nodes.cbegin(), m_added_nodes.cend(), std::back_inserter(lAddresses)); } @@ -2119,8 +2154,8 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const std::map<CService, bool> mapConnected; std::map<std::string, std::pair<bool, CService>> mapConnectedByName; { - LOCK(cs_vNodes); - for (const CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (const CNode* pnode : m_nodes) { if (pnode->addr.IsValid()) { mapConnected[pnode->addr] = pnode->IsInboundConn(); } @@ -2216,57 +2251,42 @@ void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai m_msgproc->InitializeNode(pnode); { - LOCK(cs_vNodes); - vNodes.push_back(pnode); + LOCK(m_nodes_mutex); + m_nodes.push_back(pnode); } } void CConnman::ThreadMessageHandler() { SetSyscallSandboxPolicy(SyscallSandboxPolicy::MESSAGE_HANDLER); - FastRandomContext rng; while (!flagInterruptMsgProc) { - std::vector<CNode*> vNodesCopy; - { - LOCK(cs_vNodes); - vNodesCopy = vNodes; - for (CNode* pnode : vNodesCopy) { - pnode->AddRef(); - } - } - bool fMoreWork = false; - // Randomize the order in which we process messages from/to our peers. - // This prevents attacks in which an attacker exploits having multiple - // consecutive connections in the vNodes list. - Shuffle(vNodesCopy.begin(), vNodesCopy.end(), rng); - - for (CNode* pnode : vNodesCopy) { - if (pnode->fDisconnect) - continue; + // Randomize the order in which we process messages from/to our peers. + // This prevents attacks in which an attacker exploits having multiple + // consecutive connections in the m_nodes list. + const NodesSnapshot snap{*this, /*shuffle=*/true}; - // Receive messages - bool fMoreNodeWork = m_msgproc->ProcessMessages(pnode, flagInterruptMsgProc); - fMoreWork |= (fMoreNodeWork && !pnode->fPauseSend); - if (flagInterruptMsgProc) - return; - // Send messages - { - LOCK(pnode->cs_sendProcessing); - m_msgproc->SendMessages(pnode); - } + for (CNode* pnode : snap.Nodes()) { + if (pnode->fDisconnect) + continue; - if (flagInterruptMsgProc) - return; - } + // Receive messages + bool fMoreNodeWork = m_msgproc->ProcessMessages(pnode, flagInterruptMsgProc); + fMoreWork |= (fMoreNodeWork && !pnode->fPauseSend); + if (flagInterruptMsgProc) + return; + // Send messages + { + LOCK(pnode->cs_sendProcessing); + m_msgproc->SendMessages(pnode); + } - { - LOCK(cs_vNodes); - for (CNode* pnode : vNodesCopy) - pnode->Release(); + if (flagInterruptMsgProc) + return; + } } WAIT_LOCK(mutexMsgProc, lock); @@ -2459,7 +2479,10 @@ NodeId CConnman::GetNewNodeId() } -bool CConnman::Bind(const CService &addr, unsigned int flags, NetPermissionFlags permissions) { +bool CConnman::Bind(const CService& addr_, unsigned int flags, NetPermissionFlags permissions) +{ + const CService addr{MaybeFlipIPv6toCJDNS(addr_)}; + if (!(flags & BF_EXPLICIT) && !IsReachable(addr)) { return false; } @@ -2673,7 +2696,7 @@ void CConnman::StopNodes() // Delete peer connections. std::vector<CNode*> nodes; - WITH_LOCK(cs_vNodes, nodes.swap(vNodes)); + WITH_LOCK(m_nodes_mutex, nodes.swap(m_nodes)); for (CNode* pnode : nodes) { pnode->CloseSocketDisconnect(); DeleteNode(pnode); @@ -2688,10 +2711,10 @@ void CConnman::StopNodes() } } - for (CNode* pnode : vNodesDisconnected) { + for (CNode* pnode : m_nodes_disconnected) { DeleteNode(pnode); } - vNodesDisconnected.clear(); + m_nodes_disconnected.clear(); vhListenSocket.clear(); semOutbound.reset(); semAddnode.reset(); @@ -2764,21 +2787,21 @@ std::vector<CAddress> CConnman::GetAddresses(CNode& requestor, size_t max_addres bool CConnman::AddNode(const std::string& strNode) { - LOCK(cs_vAddedNodes); - for (const std::string& it : vAddedNodes) { + LOCK(m_added_nodes_mutex); + for (const std::string& it : m_added_nodes) { if (strNode == it) return false; } - vAddedNodes.push_back(strNode); + m_added_nodes.push_back(strNode); return true; } bool CConnman::RemoveAddedNode(const std::string& strNode) { - LOCK(cs_vAddedNodes); - for(std::vector<std::string>::iterator it = vAddedNodes.begin(); it != vAddedNodes.end(); ++it) { + LOCK(m_added_nodes_mutex); + for(std::vector<std::string>::iterator it = m_added_nodes.begin(); it != m_added_nodes.end(); ++it) { if (strNode == *it) { - vAddedNodes.erase(it); + m_added_nodes.erase(it); return true; } } @@ -2787,12 +2810,12 @@ bool CConnman::RemoveAddedNode(const std::string& strNode) size_t CConnman::GetNodeCount(ConnectionDirection flags) const { - LOCK(cs_vNodes); + LOCK(m_nodes_mutex); if (flags == ConnectionDirection::Both) // Shortcut if we want total - return vNodes.size(); + return m_nodes.size(); int nNum = 0; - for (const auto& pnode : vNodes) { + for (const auto& pnode : m_nodes) { if (flags & (pnode->IsInboundConn() ? ConnectionDirection::In : ConnectionDirection::Out)) { nNum++; } @@ -2804,9 +2827,9 @@ size_t CConnman::GetNodeCount(ConnectionDirection flags) const void CConnman::GetNodeStats(std::vector<CNodeStats>& vstats) const { vstats.clear(); - LOCK(cs_vNodes); - vstats.reserve(vNodes.size()); - for (CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + vstats.reserve(m_nodes.size()); + for (CNode* pnode : m_nodes) { vstats.emplace_back(); pnode->CopyStats(vstats.back()); vstats.back().m_mapped_as = pnode->addr.GetMappedAS(addrman.GetAsmap()); @@ -2815,7 +2838,7 @@ void CConnman::GetNodeStats(std::vector<CNodeStats>& vstats) const bool CConnman::DisconnectNode(const std::string& strNode) { - LOCK(cs_vNodes); + LOCK(m_nodes_mutex); if (CNode* pnode = FindNode(strNode)) { LogPrint(BCLog::NET, "disconnect by address%s matched peer=%d; disconnecting\n", (fLogIPs ? strprintf("=%s", strNode) : ""), pnode->GetId()); pnode->fDisconnect = true; @@ -2827,8 +2850,8 @@ bool CConnman::DisconnectNode(const std::string& strNode) bool CConnman::DisconnectNode(const CSubNet& subnet) { bool disconnected = false; - LOCK(cs_vNodes); - for (CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for (CNode* pnode : m_nodes) { if (subnet.Match(pnode->addr)) { LogPrint(BCLog::NET, "disconnect by subnet%s matched peer=%d; disconnecting\n", (fLogIPs ? strprintf("=%s", subnet.ToString()) : ""), pnode->GetId()); pnode->fDisconnect = true; @@ -2845,8 +2868,8 @@ bool CConnman::DisconnectNode(const CNetAddr& addr) bool CConnman::DisconnectNode(NodeId id) { - LOCK(cs_vNodes); - for(CNode* pnode : vNodes) { + LOCK(m_nodes_mutex); + for(CNode* pnode : m_nodes) { if (id == pnode->GetId()) { LogPrint(BCLog::NET, "disconnect by id peer=%d; disconnecting\n", pnode->GetId()); pnode->fDisconnect = true; @@ -2858,7 +2881,6 @@ bool CConnman::DisconnectNode(NodeId id) void CConnman::RecordBytesRecv(uint64_t bytes) { - LOCK(cs_totalBytesRecv); nTotalBytesRecv += bytes; } @@ -2935,7 +2957,6 @@ uint64_t CConnman::GetOutboundTargetBytesLeft() const uint64_t CConnman::GetTotalBytesRecv() const { - LOCK(cs_totalBytesRecv); return nTotalBytesRecv; } @@ -2980,7 +3001,7 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, SOCKET hSocketIn, const LogPrint(BCLog::NET, "Added connection peer=%d\n", id); } - m_deserializer = std::make_unique<V1TransportDeserializer>(V1TransportDeserializer(Params(), GetId(), SER_NETWORK, INIT_PROTO_VERSION)); + m_deserializer = std::make_unique<V1TransportDeserializer>(V1TransportDeserializer(Params(), id, SER_NETWORK, INIT_PROTO_VERSION)); m_serializer = std::make_unique<V1TransportSerializer>(V1TransportSerializer()); } @@ -2999,7 +3020,7 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) size_t nMessageSize = msg.data.size(); LogPrint(BCLog::NET, "sending %s (%d bytes) peer=%d\n", msg.m_type, nMessageSize, pnode->GetId()); if (gArgs.GetBoolArg("-capturemessages", false)) { - CaptureMessage(pnode->addr, msg.m_type, msg.data, /* incoming */ false); + CaptureMessage(pnode->addr, msg.m_type, msg.data, /*is_incoming=*/false); } TRACE6(net, outbound_message, @@ -3038,8 +3059,8 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) bool CConnman::ForNode(NodeId id, std::function<bool(CNode* pnode)> func) { CNode* found = nullptr; - LOCK(cs_vNodes); - for (auto&& pnode : vNodes) { + LOCK(m_nodes_mutex); + for (auto&& pnode : m_nodes) { if(pnode->GetId() == id) { found = pnode; break; @@ -6,7 +6,6 @@ #ifndef BITCOIN_NET_H #define BITCOIN_NET_H -#include <addrman.h> #include <chainparams.h> #include <common/bloom.h> #include <compat.h> @@ -37,9 +36,10 @@ #include <thread> #include <vector> -class CScheduler; -class CNode; +class AddrMan; class BanMan; +class CNode; +class CScheduler; struct bilingual_str; /** Default for -whitelistrelay. */ @@ -70,7 +70,7 @@ static const bool DEFAULT_LISTEN = true; /** The maximum number of peer connections to maintain. */ static const unsigned int DEFAULT_MAX_PEER_CONNECTIONS = 125; /** The default for -maxuploadtarget. 0 = Unlimited */ -static constexpr uint64_t DEFAULT_MAX_UPLOAD_TARGET = 0; +static const std::string DEFAULT_MAX_UPLOAD_TARGET{"0M"}; /** Default for blocks only*/ static const bool DEFAULT_BLOCKSONLY = false; /** -peertimeout default */ @@ -308,7 +308,7 @@ public: /** read and deserialize data, advances msg_bytes data pointer */ virtual int Read(Span<const uint8_t>& msg_bytes) = 0; // decomposes a message from the context - virtual std::optional<CNetMessage> GetMessage(std::chrono::microseconds time, uint32_t& out_err) = 0; + virtual CNetMessage GetMessage(std::chrono::microseconds time, bool& reject_message) = 0; virtual ~TransportDeserializer() {} }; @@ -372,7 +372,7 @@ public: } return ret; } - std::optional<CNetMessage> GetMessage(std::chrono::microseconds time, uint32_t& out_err_raw_size) override; + CNetMessage GetMessage(std::chrono::microseconds time, bool& reject_message) override; }; /** The TransportSerializer prepares messages for the network transport @@ -791,8 +791,8 @@ public: } vWhitelistedRange = connOptions.vWhitelistedRange; { - LOCK(cs_vAddedNodes); - vAddedNodes = connOptions.m_added_nodes; + LOCK(m_added_nodes_mutex); + m_added_nodes = connOptions.m_added_nodes; } m_onion_binds = connOptions.onion_binds; } @@ -823,8 +823,8 @@ public: using NodeFn = std::function<void(CNode*)>; void ForEachNode(const NodeFn& func) { - LOCK(cs_vNodes); - for (auto&& node : vNodes) { + LOCK(m_nodes_mutex); + for (auto&& node : m_nodes) { if (NodeFullyConnected(node)) func(node); } @@ -832,8 +832,8 @@ public: void ForEachNode(const NodeFn& func) const { - LOCK(cs_vNodes); - for (auto&& node : vNodes) { + LOCK(m_nodes_mutex); + for (auto&& node : m_nodes) { if (NodeFullyConnected(node)) func(node); } @@ -968,7 +968,7 @@ private: /** * Create a `CNode` object from a socket that has just been accepted and add the node to - * the `vNodes` member. + * the `m_nodes` member. * @param[in] hSocket Connected socket to communicate with the peer. * @param[in] permissionFlags The peer's permissions. * @param[in] addr_bind The address and port at our side of the connection. @@ -983,9 +983,57 @@ private: void NotifyNumConnectionsChanged(); /** Return true if the peer is inactive and should be disconnected. */ bool InactivityCheck(const CNode& node) const; - bool GenerateSelectSet(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_set, std::set<SOCKET> &error_set); - void SocketEvents(std::set<SOCKET> &recv_set, std::set<SOCKET> &send_set, std::set<SOCKET> &error_set); + + /** + * Generate a collection of sockets to check for IO readiness. + * @param[in] nodes Select from these nodes' sockets. + * @param[out] recv_set Sockets to check for read readiness. + * @param[out] send_set Sockets to check for write readiness. + * @param[out] error_set Sockets to check for errors. + * @return true if at least one socket is to be checked (the returned set is not empty) + */ + bool GenerateSelectSet(const std::vector<CNode*>& nodes, + std::set<SOCKET>& recv_set, + std::set<SOCKET>& send_set, + std::set<SOCKET>& error_set); + + /** + * Check which sockets are ready for IO. + * @param[in] nodes Select from these nodes' sockets. + * @param[out] recv_set Sockets which are ready for read. + * @param[out] send_set Sockets which are ready for write. + * @param[out] error_set Sockets which have errors. + * This calls `GenerateSelectSet()` to gather a list of sockets to check. + */ + void SocketEvents(const std::vector<CNode*>& nodes, + std::set<SOCKET>& recv_set, + std::set<SOCKET>& send_set, + std::set<SOCKET>& error_set); + + /** + * Check connected and listening sockets for IO readiness and process them accordingly. + */ void SocketHandler(); + + /** + * Do the read/write for connected sockets that are ready for IO. + * @param[in] nodes Nodes to process. The socket of each node is checked against + * `recv_set`, `send_set` and `error_set`. + * @param[in] recv_set Sockets that are ready for read. + * @param[in] send_set Sockets that are ready for send. + * @param[in] error_set Sockets that have an exceptional condition (error). + */ + void SocketHandlerConnected(const std::vector<CNode*>& nodes, + const std::set<SOCKET>& recv_set, + const std::set<SOCKET>& send_set, + const std::set<SOCKET>& error_set); + + /** + * Accept incoming connections, one from each read-ready listening socket. + * @param[in] recv_set Sockets that are ready for read. + */ + void SocketHandlerListening(const std::set<SOCKET>& recv_set); + void ThreadSocketHandler(); void ThreadDNSAddressSeed(); @@ -1026,9 +1074,8 @@ private: static bool NodeFullyConnected(const CNode* pnode); // Network usage totals - mutable RecursiveMutex cs_totalBytesRecv; mutable RecursiveMutex cs_totalBytesSent; - uint64_t nTotalBytesRecv GUARDED_BY(cs_totalBytesRecv) {0}; + std::atomic<uint64_t> nTotalBytesRecv{0}; uint64_t nTotalBytesSent GUARDED_BY(cs_totalBytesSent) {0}; // outbound limit & stats @@ -1051,12 +1098,12 @@ private: bool fAddressesInitialized{false}; AddrMan& addrman; std::deque<std::string> m_addr_fetches GUARDED_BY(m_addr_fetches_mutex); - RecursiveMutex m_addr_fetches_mutex; - std::vector<std::string> vAddedNodes GUARDED_BY(cs_vAddedNodes); - mutable RecursiveMutex cs_vAddedNodes; - std::vector<CNode*> vNodes GUARDED_BY(cs_vNodes); - std::list<CNode*> vNodesDisconnected; - mutable RecursiveMutex cs_vNodes; + Mutex m_addr_fetches_mutex; + std::vector<std::string> m_added_nodes GUARDED_BY(m_added_nodes_mutex); + mutable Mutex m_added_nodes_mutex; + std::vector<CNode*> m_nodes GUARDED_BY(m_nodes_mutex); + std::list<CNode*> m_nodes_disconnected; + mutable RecursiveMutex m_nodes_mutex; std::atomic<NodeId> nLastNodeId{0}; unsigned int nPrevNodeCount{0}; @@ -1177,6 +1224,43 @@ private: */ std::vector<CService> m_onion_binds; + /** + * RAII helper to atomically create a copy of `m_nodes` and add a reference + * to each of the nodes. The nodes are released when this object is destroyed. + */ + class NodesSnapshot + { + public: + explicit NodesSnapshot(const CConnman& connman, bool shuffle) + { + { + LOCK(connman.m_nodes_mutex); + m_nodes_copy = connman.m_nodes; + for (auto& node : m_nodes_copy) { + node->AddRef(); + } + } + if (shuffle) { + Shuffle(m_nodes_copy.begin(), m_nodes_copy.end(), FastRandomContext{}); + } + } + + ~NodesSnapshot() + { + for (auto& node : m_nodes_copy) { + node->Release(); + } + } + + const std::vector<CNode*>& Nodes() const + { + return m_nodes_copy; + } + + private: + std::vector<CNode*> m_nodes_copy; + }; + friend struct CConnmanTest; friend struct ConnmanTestMsg; }; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 9f3aa5b4a3..f352246b0d 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -461,9 +461,9 @@ private: bool AlreadyHaveTx(const GenTxid& gtxid) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** - * Filter for transactions that were recently rejected by - * AcceptToMemoryPool. These are not rerequested until the chain tip - * changes, at which point the entire filter is reset. + * Filter for transactions that were recently rejected by the mempool. + * These are not rerequested until the chain tip changes, at which point + * the entire filter is reset. * * Without this filter we'd be re-requesting txs from each of our peers, * increasing bandwidth consumption considerably. For instance, with 100 @@ -1409,6 +1409,7 @@ bool PeerManagerImpl::MaybePunishNodeForTx(NodeId nodeid, const TxValidationStat case TxValidationResult::TX_WITNESS_STRIPPED: case TxValidationResult::TX_CONFLICT: case TxValidationResult::TX_MEMPOOL_POLICY: + case TxValidationResult::TX_NO_MEMPOOL: break; } if (message != "") { @@ -1814,7 +1815,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& if (!ReadRawBlockFromDisk(block_data, pindex, m_chainparams.MessageStart())) { assert(!"cannot load block from disk"); } - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, MakeSpan(block_data))); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, Span{block_data})); // Don't set pblock as we've sent the block } else { // Send block from disk @@ -2242,7 +2243,7 @@ void PeerManagerImpl::ProcessOrphanTx(std::set<uint256>& orphan_work_set) const auto [porphanTx, from_peer] = m_orphanage.GetTx(orphanHash); if (porphanTx == nullptr) continue; - const MempoolAcceptResult result = AcceptToMemoryPool(m_chainman.ActiveChainstate(), m_mempool, porphanTx, false /* bypass_limits */); + const MempoolAcceptResult result = m_chainman.ProcessTransaction(porphanTx); const TxValidationState& state = result.m_state; if (result.m_result_type == MempoolAcceptResult::ResultType::VALID) { @@ -2299,8 +2300,6 @@ void PeerManagerImpl::ProcessOrphanTx(std::set<uint256>& orphan_work_set) break; } } - CChainState& active_chainstate = m_chainman.ActiveChainstate(); - m_mempool.check(active_chainstate.CoinsTip(), active_chainstate.m_chain.Height() + 1); } bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& peer, @@ -2684,7 +2683,11 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, int64_t nTimeOffset = nTime - GetTime(); pfrom.nTimeOffset = nTimeOffset; - AddTimeData(pfrom.addr, nTimeOffset); + if (!pfrom.IsInboundConn()) { + // Don't use timedata samples from inbound peers to make it + // harder for others to tamper with our adjusted time. + AddTimeData(pfrom.addr, nTimeOffset); + } // If the peer is old enough to have the old alert system, send it the final alert. if (greatest_common_version <= 70012) { @@ -3207,6 +3210,11 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } + // Stop processing the transaction early if we are still in IBD since we don't + // have enough information to validate it yet. Sending unsolicited transactions + // is not considered a protocol violation, so don't punish the peer. + if (m_chainman.ActiveChainstate().IsInitialBlockDownload()) return; + CTransactionRef ptx; vRecv >> ptx; const CTransaction& tx = *ptx; @@ -3259,12 +3267,10 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } - const MempoolAcceptResult result = AcceptToMemoryPool(m_chainman.ActiveChainstate(), m_mempool, ptx, false /* bypass_limits */); + const MempoolAcceptResult result = m_chainman.ProcessTransaction(ptx); const TxValidationState& state = result.m_state; if (result.m_result_type == MempoolAcceptResult::ResultType::VALID) { - CChainState& active_chainstate = m_chainman.ActiveChainstate(); - m_mempool.check(active_chainstate.CoinsTip(), active_chainstate.m_chain.Height() + 1); // As this version of the transaction was acceptable, we can forget about any // requests for it. m_txrequest.ForgetTxHash(tx.GetHash()); @@ -3383,8 +3389,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } // If a tx has been detected by m_recent_rejects, we will have reached - // this point and the tx will have been ignored. Because we haven't run - // the tx through AcceptToMemoryPool, we won't have computed a DoS + // this point and the tx will have been ignored. Because we haven't + // submitted the tx to our mempool, we won't have computed a DoS // score for it or determined exactly why we consider it invalid. // // This means we won't penalize any peer subsequently relaying a DoSy @@ -4108,7 +4114,7 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt ); if (gArgs.GetBoolArg("-capturemessages", false)) { - CaptureMessage(pfrom->addr, msg.m_command, MakeUCharSpan(msg.m_recv), /* incoming */ true); + CaptureMessage(pfrom->addr, msg.m_command, MakeUCharSpan(msg.m_recv), /*is_incoming=*/true); } msg.SetVersion(pfrom->GetCommonVersion()); diff --git a/src/netaddress.cpp b/src/netaddress.cpp index f9fff5a6d5..f6bc7fbb94 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -196,7 +196,7 @@ static void Checksum(Span<const uint8_t> addr_pubkey, uint8_t (&checksum)[CHECKS SHA3_256 hasher; - hasher.Write(MakeSpan(prefix).first(prefix_len)); + hasher.Write(Span{prefix}.first(prefix_len)); hasher.Write(addr_pubkey); hasher.Write(VERSION); @@ -303,7 +303,7 @@ CNetAddr::CNetAddr(const struct in_addr& ipv4Addr) CNetAddr::CNetAddr(const struct in6_addr& ipv6Addr, const uint32_t scope) { - SetLegacyIPv6(Span<const uint8_t>(reinterpret_cast<const uint8_t*>(&ipv6Addr), sizeof(ipv6Addr))); + SetLegacyIPv6({reinterpret_cast<const uint8_t*>(&ipv6Addr), sizeof(ipv6Addr)}); m_scope_id = scope; } @@ -663,7 +663,7 @@ bool CNetAddr::GetInAddr(struct in_addr* pipv4Addr) const } /** - * Try to get our IPv6 address. + * Try to get our IPv6 (or CJDNS) address. * * @param[out] pipv6Addr The in6_addr struct to which to copy. * @@ -674,7 +674,7 @@ bool CNetAddr::GetInAddr(struct in_addr* pipv4Addr) const */ bool CNetAddr::GetIn6Addr(struct in6_addr* pipv6Addr) const { - if (!IsIPv6()) { + if (!IsIPv6() && !IsCJDNS()) { return false; } assert(sizeof(*pipv6Addr) == m_addr.size()); @@ -693,13 +693,13 @@ uint32_t CNetAddr::GetLinkedIPv4() const return ReadBE32(m_addr.data()); } else if (IsRFC6052() || IsRFC6145()) { // mapped IPv4, SIIT translated IPv4: the IPv4 address is the last 4 bytes of the address - return ReadBE32(MakeSpan(m_addr).last(ADDR_IPV4_SIZE).data()); + return ReadBE32(Span{m_addr}.last(ADDR_IPV4_SIZE).data()); } else if (IsRFC3964()) { // 6to4 tunneled IPv4: the IPv4 address is in bytes 2-6 - return ReadBE32(MakeSpan(m_addr).subspan(2, ADDR_IPV4_SIZE).data()); + return ReadBE32(Span{m_addr}.subspan(2, ADDR_IPV4_SIZE).data()); } else if (IsRFC4380()) { // Teredo tunneled IPv4: the IPv4 address is in the last 4 bytes of the address, but bitflipped - return ~ReadBE32(MakeSpan(m_addr).last(ADDR_IPV4_SIZE).data()); + return ~ReadBE32(Span{m_addr}.last(ADDR_IPV4_SIZE).data()); } assert(false); } @@ -794,8 +794,14 @@ std::vector<unsigned char> CNetAddr::GetGroup(const std::vector<bool> &asmap) co vchRet.push_back((ipv4 >> 24) & 0xFF); vchRet.push_back((ipv4 >> 16) & 0xFF); return vchRet; - } else if (IsTor() || IsI2P() || IsCJDNS()) { + } else if (IsTor() || IsI2P()) { nBits = 4; + } else if (IsCJDNS()) { + // Treat in the same way as Tor and I2P because the address in all of + // them is "random" bytes (derived from a public key). However in CJDNS + // the first byte is a constant 0xfc, so the random bytes come after it. + // Thus skip the constant 8 bits at the start. + nBits = 12; } else if (IsHeNet()) { // for he.net, use /36 groups nBits = 36; @@ -892,6 +898,11 @@ int CNetAddr::GetReachabilityFrom(const CNetAddr *paddrPartner) const case NET_I2P: return REACH_PRIVATE; default: return REACH_DEFAULT; } + case NET_CJDNS: + switch (ourNet) { + case NET_CJDNS: return REACH_PRIVATE; + default: return REACH_DEFAULT; + } case NET_TEREDO: switch(ourNet) { default: return REACH_DEFAULT; @@ -993,7 +1004,7 @@ bool CService::GetSockAddr(struct sockaddr* paddr, socklen_t *addrlen) const paddrin->sin_port = htons(port); return true; } - if (IsIPv6()) { + if (IsIPv6() || IsCJDNS()) { if (*addrlen < (socklen_t)sizeof(struct sockaddr_in6)) return false; *addrlen = sizeof(struct sockaddr_in6); diff --git a/src/netaddress.h b/src/netaddress.h index 57eb8bc72f..c8af4a9605 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -224,7 +224,7 @@ public: */ bool IsRelayable() const { - return IsIPv4() || IsIPv6() || IsTor() || IsI2P(); + return IsIPv4() || IsIPv6() || IsTor() || IsI2P() || IsCJDNS(); } /** @@ -385,6 +385,12 @@ private: /** * Unserialize from a pre-ADDRv2/BIP155 format from an array. + * + * This function is only called from UnserializeV1Stream() and is a wrapper + * for SetLegacyIPv6(); however, we keep it for symmetry with + * SerializeV1Array() to have pairs of ser/unser functions and to make clear + * that if one is altered, a corresponding reverse modification should be + * applied to the other. */ void UnserializeV1Array(uint8_t (&arr)[V1_SERIALIZATION_SIZE]) { @@ -427,7 +433,7 @@ private: if (SetNetFromBIP155Network(bip155_net, address_size)) { m_addr.resize(address_size); - s >> MakeSpan(m_addr); + s >> Span{m_addr}; if (m_net != NET_IPV6) { return; @@ -550,6 +556,7 @@ public: } friend class CServiceHash; + friend CService MaybeFlipIPv6toCJDNS(const CService& service); }; class CServiceHash diff --git a/src/netbase.cpp b/src/netbase.cpp index 64d17189a6..3cb12f1abc 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -96,6 +96,9 @@ enum Network ParseNetwork(const std::string& net_in) { if (net == "i2p") { return NET_I2P; } + if (net == "cjdns") { + return NET_CJDNS; + } return NET_UNROUTABLE; } @@ -120,7 +123,7 @@ std::vector<std::string> GetNetworkNames(bool append_unroutable) std::vector<std::string> names; for (int n = 0; n < NET_MAX; ++n) { const enum Network network{static_cast<Network>(n)}; - if (network == NET_UNROUTABLE || network == NET_CJDNS || network == NET_INTERNAL) continue; + if (network == NET_UNROUTABLE || network == NET_INTERNAL) continue; names.emplace_back(GetNetworkName(network)); } if (append_unroutable) { @@ -679,14 +682,11 @@ bool LookupSubNet(const std::string& strSubnet, CSubNet& ret, DNSLookupFn dns_lo return false; } size_t slash = strSubnet.find_last_of('/'); - std::vector<CNetAddr> vIP; + CNetAddr network; std::string strAddress = strSubnet.substr(0, slash); - // TODO: Use LookupHost(const std::string&, CNetAddr&, bool) instead to just get - // one CNetAddr. - if (LookupHost(strAddress, vIP, 1, false, dns_lookup_function)) + if (LookupHost(strAddress, network, false, dns_lookup_function)) { - CNetAddr network = vIP[0]; if (slash != strSubnet.npos) { std::string strNetmask = strSubnet.substr(slash + 1); @@ -698,14 +698,15 @@ bool LookupSubNet(const std::string& strSubnet, CSubNet& ret, DNSLookupFn dns_lo } else // If not a valid number, try full netmask syntax { + CNetAddr netmask; // Never allow lookup for netmask - if (LookupHost(strNetmask, vIP, 1, false, dns_lookup_function)) { - ret = CSubNet(network, vIP[0]); + if (LookupHost(strNetmask, netmask, false, dns_lookup_function)) { + ret = CSubNet(network, netmask); return ret.IsValid(); } } } - else + else // Single IP subnet (<ipv4>/32 or <ipv6>/128) { ret = CSubNet(network); return ret.IsValid(); diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 192caf7994..075842ef73 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -67,6 +67,17 @@ using interfaces::WalletClient; namespace node { namespace { +#ifdef ENABLE_EXTERNAL_SIGNER +class ExternalSignerImpl : public interfaces::ExternalSigner +{ +public: + ExternalSignerImpl(::ExternalSigner signer) : m_signer(std::move(signer)) {} + std::string getName() override { return m_signer.m_name; } +private: + ::ExternalSigner m_signer; +}; +#endif + class NodeImpl : public Node { private: @@ -172,14 +183,18 @@ public: } return false; } - std::vector<ExternalSigner> externalSigners() override + std::vector<std::unique_ptr<interfaces::ExternalSigner>> listExternalSigners() override { #ifdef ENABLE_EXTERNAL_SIGNER std::vector<ExternalSigner> signers = {}; const std::string command = gArgs.GetArg("-signer", ""); - if (command == "") return signers; + if (command == "") return {}; ExternalSigner::Enumerate(command, signers, Params().NetworkIDString()); - return signers; + std::vector<std::unique_ptr<interfaces::ExternalSigner>> result; + for (auto& signer : signers) { + result.emplace_back(std::make_unique<ExternalSignerImpl>(std::move(signer))); + } + return result; #else // This result is indistinguishable from a successful call that returns // no signers. For the current GUI this doesn't matter, because the wallet @@ -261,6 +276,10 @@ public: LOCK(::cs_main); return chainman().ActiveChainstate().CoinsTip().GetCoin(output, coin); } + TransactionError broadcastTransaction(CTransactionRef tx, CAmount max_tx_fee, std::string& err_string) override + { + return BroadcastTransaction(*m_context, std::move(tx), err_string, max_tx_fee, /*relay=*/ true, /*wait_callback=*/ false); + } WalletClient& walletClient() override { return *Assert(m_context->wallet_client); @@ -281,6 +300,10 @@ public: { return MakeHandler(::uiInterface.ShowProgress_connect(fn)); } + std::unique_ptr<Handler> handleInitWallet(InitWalletFn fn) override + { + return MakeHandler(::uiInterface.InitWallet_connect(fn)); + } std::unique_ptr<Handler> handleNotifyNumConnectionsChanged(NotifyNumConnectionsChangedFn fn) override { return MakeHandler(::uiInterface.NotifyNumConnectionsChanged_connect(fn)); @@ -517,8 +540,11 @@ public: const CBlockIndex* block2 = m_node.chainman->m_blockman.LookupBlockIndex(block_hash2); const CBlockIndex* ancestor = block1 && block2 ? LastCommonAncestor(block1, block2) : nullptr; // Using & instead of && below to avoid short circuiting and leaving - // output uninitialized. - return FillBlock(ancestor, ancestor_out, lock, active) & FillBlock(block1, block1_out, lock, active) & FillBlock(block2, block2_out, lock, active); + // output uninitialized. Cast bool to int to avoid -Wbitwise-instead-of-logical + // compiler warnings. + return int{FillBlock(ancestor, ancestor_out, lock, active)} & + int{FillBlock(block1, block1_out, lock, active)} & + int{FillBlock(block2, block2_out, lock, active)}; } void findCoins(std::map<COutPoint, Coin>& coins) override { return FindCoins(m_node, coins); } double guessVerificationProgress(const uint256& block_hash) override @@ -630,7 +656,6 @@ public: return chainman().ActiveChainstate().IsInitialBlockDownload(); } bool shutdownRequested() override { return ShutdownRequested(); } - int64_t getAdjustedTime() override { return GetAdjustedTime(); } void initMessage(const std::string& message) override { ::uiInterface.InitMessage(message); } void initWarning(const bilingual_str& message) override { InitWarning(message); } void initError(const bilingual_str& message) override { InitError(message); } @@ -698,12 +723,6 @@ public: notifications.transactionAddedToMempool(entry.GetSharedTx(), 0 /* mempool_sequence */); } } - bool isTaprootActive() override - { - LOCK(::cs_main); - const CBlockIndex* tip = Assert(m_node.chainman)->ActiveChain().Tip(); - return DeploymentActiveAfter(tip, Params().GetConsensus(), Consensus::DEPLOYMENT_TAPROOT); - } NodeContext& m_node; }; } // namespace diff --git a/src/miner.cpp b/src/node/miner.cpp index 1ef246cd14..291a6e1d10 100644 --- a/src/miner.cpp +++ b/src/node/miner.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <miner.h> +#include <node/miner.h> #include <chain.h> #include <chainparams.h> @@ -21,6 +21,7 @@ #include <timedata.h> #include <util/moneystr.h> #include <util/system.h> +#include <validation.h> #include <algorithm> #include <utility> @@ -28,14 +29,16 @@ int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev) { int64_t nOldTime = pblock->nTime; - int64_t nNewTime = std::max(pindexPrev->GetMedianTimePast()+1, GetAdjustedTime()); + int64_t nNewTime = std::max(pindexPrev->GetMedianTimePast() + 1, GetAdjustedTime()); - if (nOldTime < nNewTime) + if (nOldTime < nNewTime) { pblock->nTime = nNewTime; + } // Updating time can change work required on testnet: - if (consensusParams.fPowAllowMinDifficultyBlocks) + if (consensusParams.fPowAllowMinDifficultyBlocks) { pblock->nBits = GetNextWorkRequired(pindexPrev, pblock, consensusParams); + } return nNewTime - nOldTime; } @@ -52,7 +55,8 @@ void RegenerateCommitments(CBlock& block, ChainstateManager& chainman) block.hashMerkleRoot = BlockMerkleRoot(block); } -BlockAssembler::Options::Options() { +BlockAssembler::Options::Options() +{ blockMinFeeRate = CFeeRate(DEFAULT_BLOCK_MIN_TX_FEE); nBlockMaxWeight = DEFAULT_BLOCK_MAX_WEIGHT; } @@ -107,8 +111,9 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc pblocktemplate.reset(new CBlockTemplate()); - if(!pblocktemplate.get()) + if (!pblocktemplate.get()) { return nullptr; + } CBlock* const pblock = &pblocktemplate->block; // pointer for convenience // Add dummy coinbase tx as first transaction @@ -124,15 +129,12 @@ std::unique_ptr<CBlockTemplate> BlockAssembler::CreateNewBlock(const CScript& sc pblock->nVersion = g_versionbitscache.ComputeBlockVersion(pindexPrev, chainparams.GetConsensus()); // -regtest only: allow overriding block.nVersion with // -blockversion=N to test forking scenarios - if (chainparams.MineBlocksOnDemand()) + if (chainparams.MineBlocksOnDemand()) { pblock->nVersion = gArgs.GetIntArg("-blockversion", pblock->nVersion); + } pblock->nTime = GetAdjustedTime(); - const int64_t nMedianTimePast = pindexPrev->GetMedianTimePast(); - - nLockTimeCutoff = (STANDARD_LOCKTIME_VERIFY_FLAGS & LOCKTIME_MEDIAN_TIME_PAST) - ? nMedianTimePast - : pblock->GetBlockTime(); + m_lock_time_cutoff = pindexPrev->GetMedianTimePast(); // Decide whether to include witness transactions // This is only needed in case the witness softfork activation is reverted @@ -192,8 +194,7 @@ void BlockAssembler::onlyUnconfirmed(CTxMemPool::setEntries& testSet) // Only test txs not already in the block if (inBlock.count(*iit)) { testSet.erase(iit++); - } - else { + } else { iit++; } } @@ -202,10 +203,12 @@ void BlockAssembler::onlyUnconfirmed(CTxMemPool::setEntries& testSet) bool BlockAssembler::TestPackage(uint64_t packageSize, int64_t packageSigOpsCost) const { // TODO: switch to weight-based accounting for packages instead of vsize-based accounting. - if (nBlockWeight + WITNESS_SCALE_FACTOR * packageSize >= nBlockMaxWeight) + if (nBlockWeight + WITNESS_SCALE_FACTOR * packageSize >= nBlockMaxWeight) { return false; - if (nBlockSigOpsCost + packageSigOpsCost >= MAX_BLOCK_SIGOPS_COST) + } + if (nBlockSigOpsCost + packageSigOpsCost >= MAX_BLOCK_SIGOPS_COST) { return false; + } return true; } @@ -216,10 +219,12 @@ bool BlockAssembler::TestPackage(uint64_t packageSize, int64_t packageSigOpsCost bool BlockAssembler::TestPackageTransactions(const CTxMemPool::setEntries& package) const { for (CTxMemPool::txiter it : package) { - if (!IsFinalTx(it->GetTx(), nHeight, nLockTimeCutoff)) + if (!IsFinalTx(it->GetTx(), nHeight, m_lock_time_cutoff)) { return false; - if (!fIncludeWitness && it->GetTx().HasWitness()) + } + if (!fIncludeWitness && it->GetTx().HasWitness()) { return false; + } } return true; } @@ -252,8 +257,9 @@ int BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& already m_mempool.CalculateDescendants(it, descendants); // Insert all descendants (not yet in block) into the modified set for (CTxMemPool::txiter desc : descendants) { - if (alreadyAdded.count(desc)) + if (alreadyAdded.count(desc)) { continue; + } ++nDescendantsUpdated; modtxiter mit = mapModifiedTx.find(desc); if (mit == mapModifiedTx.end()) { @@ -279,7 +285,7 @@ int BlockAssembler::UpdatePackagesForAdded(const CTxMemPool::setEntries& already // guaranteed to fail again, but as a belt-and-suspenders check we put it in // failedTx and avoid re-evaluation, since the re-evaluation would be using // cached size/sigops/fee values that are not actually correct. -bool BlockAssembler::SkipMapTxEntry(CTxMemPool::txiter it, indexed_modified_transaction_set &mapModifiedTx, CTxMemPool::setEntries &failedTx) +bool BlockAssembler::SkipMapTxEntry(CTxMemPool::txiter it, indexed_modified_transaction_set& mapModifiedTx, CTxMemPool::setEntries& failedTx) { assert(it != m_mempool.mapTx.end()); return mapModifiedTx.count(it) || inBlock.count(it) || failedTx.count(it); @@ -306,7 +312,7 @@ void BlockAssembler::SortForBlock(const CTxMemPool::setEntries& package, std::ve // Each time through the loop, we compare the best transaction in // mapModifiedTxs with the next transaction in the mempool to decide what // transaction package to work on next. -void BlockAssembler::addPackageTxs(int &nPackagesSelected, int &nDescendantsUpdated) +void BlockAssembler::addPackageTxs(int& nPackagesSelected, int& nDescendantsUpdated) { // mapModifiedTx will store sorted packages after they are modified // because some of their txs are already in the block @@ -422,7 +428,7 @@ void BlockAssembler::addPackageTxs(int &nPackagesSelected, int &nDescendantsUpda std::vector<CTxMemPool::txiter> sortedEntries; SortForBlock(ancestors, sortedEntries); - for (size_t i=0; i<sortedEntries.size(); ++i) { + for (size_t i = 0; i < sortedEntries.size(); ++i) { AddToBlock(sortedEntries[i]); // Erase from the modified set, if present mapModifiedTx.erase(sortedEntries[i]); @@ -439,13 +445,12 @@ void IncrementExtraNonce(CBlock* pblock, const CBlockIndex* pindexPrev, unsigned { // Update nExtraNonce static uint256 hashPrevBlock; - if (hashPrevBlock != pblock->hashPrevBlock) - { + if (hashPrevBlock != pblock->hashPrevBlock) { nExtraNonce = 0; hashPrevBlock = pblock->hashPrevBlock; } ++nExtraNonce; - unsigned int nHeight = pindexPrev->nHeight+1; // Height first in coinbase required for block.version=2 + unsigned int nHeight = pindexPrev->nHeight + 1; // Height first in coinbase required for block.version=2 CMutableTransaction txCoinbase(*pblock->vtx[0]); txCoinbase.vin[0].scriptSig = (CScript() << nHeight << CScriptNum(nExtraNonce)); assert(txCoinbase.vin[0].scriptSig.size() <= 100); diff --git a/src/miner.h b/src/node/miner.h index 10a80f4392..e50db731b7 100644 --- a/src/miner.h +++ b/src/node/miner.h @@ -3,20 +3,20 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_MINER_H -#define BITCOIN_MINER_H +#ifndef BITCOIN_NODE_MINER_H +#define BITCOIN_NODE_MINER_H #include <primitives/block.h> #include <txmempool.h> -#include <validation.h> #include <memory> #include <optional> #include <stdint.h> -#include <boost/multi_index_container.hpp> #include <boost/multi_index/ordered_index.hpp> +#include <boost/multi_index_container.hpp> +class ChainstateManager; class CBlockIndex; class CChainParams; class CScript; @@ -80,10 +80,11 @@ struct modifiedentry_iter { // This is sufficient to sort an ancestor package in an order that is valid // to appear in a block. struct CompareTxIterByAncestorCount { - bool operator()(const CTxMemPool::txiter &a, const CTxMemPool::txiter &b) const + bool operator()(const CTxMemPool::txiter& a, const CTxMemPool::txiter& b) const { - if (a->GetCountWithAncestors() != b->GetCountWithAncestors()) + if (a->GetCountWithAncestors() != b->GetCountWithAncestors()) { return a->GetCountWithAncestors() < b->GetCountWithAncestors(); + } return CompareIteratorByHash()(a, b); } }; @@ -143,7 +144,8 @@ private: // Chain context for the block int nHeight; - int64_t nLockTimeCutoff; + int64_t m_lock_time_cutoff; + const CChainParams& chainparams; const CTxMemPool& m_mempool; CChainState& m_chainstate; @@ -205,4 +207,4 @@ int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParam /** Update an old GenerateCoinbaseCommitment from CreateNewBlock after the block txs have changed */ void RegenerateCommitments(CBlock& block, ChainstateManager& chainman); -#endif // BITCOIN_MINER_H +#endif // BITCOIN_NODE_MINER_H diff --git a/src/node/minisketchwrapper.cpp b/src/node/minisketchwrapper.cpp new file mode 100644 index 0000000000..572df63463 --- /dev/null +++ b/src/node/minisketchwrapper.cpp @@ -0,0 +1,77 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/minisketchwrapper.h> + +#include <logging.h> +#include <util/time.h> + +#include <minisketch.h> + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <optional> +#include <utility> +#include <vector> + +namespace { + +static constexpr uint32_t BITS = 32; + +uint32_t FindBestImplementation() +{ + std::optional<std::pair<int64_t, uint32_t>> best; + + uint32_t max_impl = Minisketch::MaxImplementation(); + for (uint32_t impl = 0; impl <= max_impl; ++impl) { + std::vector<int64_t> benches; + uint64_t offset = 0; + /* Run a little benchmark with capacity 32, adding 184 entries, and decoding 11 of them once. */ + for (int b = 0; b < 11; ++b) { + if (!Minisketch::ImplementationSupported(BITS, impl)) break; + Minisketch sketch(BITS, impl, 32); + auto start = GetTimeMicros(); + for (uint64_t e = 0; e < 100; ++e) { + sketch.Add(e*1337 + b*13337 + offset); + } + for (uint64_t e = 0; e < 84; ++e) { + sketch.Add(e*1337 + b*13337 + offset); + } + offset += (*sketch.Decode(32))[0]; + auto stop = GetTimeMicros(); + benches.push_back(stop - start); + } + /* Remember which implementation has the best median benchmark time. */ + if (!benches.empty()) { + std::sort(benches.begin(), benches.end()); + if (!best || best->first > benches[5]) { + best = std::make_pair(benches[5], impl); + } + } + } + assert(best.has_value()); + LogPrintf("Using Minisketch implementation number %i\n", best->second); + return best->second; +} + +uint32_t Minisketch32Implementation() +{ + // Fast compute-once idiom. + static uint32_t best = FindBestImplementation(); + return best; +} + +} // namespace + + +Minisketch MakeMinisketch32(size_t capacity) +{ + return Minisketch(BITS, Minisketch32Implementation(), capacity); +} + +Minisketch MakeMinisketch32FP(size_t max_elements, uint32_t fpbits) +{ + return Minisketch::CreateFP(BITS, Minisketch32Implementation(), max_elements, fpbits); +} diff --git a/src/node/minisketchwrapper.h b/src/node/minisketchwrapper.h new file mode 100644 index 0000000000..a8aef68d01 --- /dev/null +++ b/src/node/minisketchwrapper.h @@ -0,0 +1,18 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_MINISKETCHWRAPPER_H +#define BITCOIN_NODE_MINISKETCHWRAPPER_H + +#include <minisketch.h> + +#include <cstddef> +#include <cstdint> + +/** Wrapper around Minisketch::Minisketch(32, implementation, capacity). */ +Minisketch MakeMinisketch32(size_t capacity); +/** Wrapper around Minisketch::CreateFP. */ +Minisketch MakeMinisketch32FP(size_t max_elements, uint32_t fpbits); + +#endif // BITCOIN_NODE_MINISKETCHWRAPPER_H diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index 2a7bcc057f..33b8e9351c 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -70,8 +70,7 @@ TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef t if (max_tx_fee > 0) { // First, call ATMP with test_accept and check the fee. If ATMP // fails here, return error immediately. - const MempoolAcceptResult result = AcceptToMemoryPool(node.chainman->ActiveChainstate(), *node.mempool, tx, false /* bypass_limits */, - true /* test_accept */); + const MempoolAcceptResult result = node.chainman->ProcessTransaction(tx, /*test_accept=*/ true); if (result.m_result_type != MempoolAcceptResult::ResultType::VALID) { return HandleATMPError(result.m_state, err_string); } else if (result.m_base_fees.value() > max_tx_fee) { @@ -79,8 +78,7 @@ TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef t } } // Try to submit the transaction to the mempool. - const MempoolAcceptResult result = AcceptToMemoryPool(node.chainman->ActiveChainstate(), *node.mempool, tx, false /* bypass_limits */, - false /* test_accept */); + const MempoolAcceptResult result = node.chainman->ProcessTransaction(tx, /*test_accept=*/ false); if (result.m_result_type != MempoolAcceptResult::ResultType::VALID) { return HandleATMPError(result.m_state, err_string); } diff --git a/src/node/ui_interface.cpp b/src/node/ui_interface.cpp index 8d3665975d..29fa16d8be 100644 --- a/src/node/ui_interface.cpp +++ b/src/node/ui_interface.cpp @@ -15,6 +15,7 @@ struct UISignals { boost::signals2::signal<CClientUIInterface::ThreadSafeMessageBoxSig, boost::signals2::optional_last_value<bool>> ThreadSafeMessageBox; boost::signals2::signal<CClientUIInterface::ThreadSafeQuestionSig, boost::signals2::optional_last_value<bool>> ThreadSafeQuestion; boost::signals2::signal<CClientUIInterface::InitMessageSig> InitMessage; + boost::signals2::signal<CClientUIInterface::InitWalletSig> InitWallet; boost::signals2::signal<CClientUIInterface::NotifyNumConnectionsChangedSig> NotifyNumConnectionsChanged; boost::signals2::signal<CClientUIInterface::NotifyNetworkActiveChangedSig> NotifyNetworkActiveChanged; boost::signals2::signal<CClientUIInterface::NotifyAlertChangedSig> NotifyAlertChanged; @@ -34,6 +35,7 @@ static UISignals g_ui_signals; ADD_SIGNALS_IMPL_WRAPPER(ThreadSafeMessageBox); ADD_SIGNALS_IMPL_WRAPPER(ThreadSafeQuestion); ADD_SIGNALS_IMPL_WRAPPER(InitMessage); +ADD_SIGNALS_IMPL_WRAPPER(InitWallet); ADD_SIGNALS_IMPL_WRAPPER(NotifyNumConnectionsChanged); ADD_SIGNALS_IMPL_WRAPPER(NotifyNetworkActiveChanged); ADD_SIGNALS_IMPL_WRAPPER(NotifyAlertChanged); @@ -45,6 +47,7 @@ ADD_SIGNALS_IMPL_WRAPPER(BannedListChanged); bool CClientUIInterface::ThreadSafeMessageBox(const bilingual_str& message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeMessageBox(message, caption, style).value_or(false);} bool CClientUIInterface::ThreadSafeQuestion(const bilingual_str& message, const std::string& non_interactive_message, const std::string& caption, unsigned int style) { return g_ui_signals.ThreadSafeQuestion(message, non_interactive_message, caption, style).value_or(false);} void CClientUIInterface::InitMessage(const std::string& message) { return g_ui_signals.InitMessage(message); } +void CClientUIInterface::InitWallet() { return g_ui_signals.InitWallet(); } void CClientUIInterface::NotifyNumConnectionsChanged(int newNumConnections) { return g_ui_signals.NotifyNumConnectionsChanged(newNumConnections); } void CClientUIInterface::NotifyNetworkActiveChanged(bool networkActive) { return g_ui_signals.NotifyNetworkActiveChanged(networkActive); } void CClientUIInterface::NotifyAlertChanged() { return g_ui_signals.NotifyAlertChanged(); } diff --git a/src/node/ui_interface.h b/src/node/ui_interface.h index d574ab879f..f969bcde21 100644 --- a/src/node/ui_interface.h +++ b/src/node/ui_interface.h @@ -82,6 +82,9 @@ public: /** Progress message during initialization. */ ADD_SIGNALS_DECL_WRAPPER(InitMessage, void, const std::string& message); + /** Wallet client created. */ + ADD_SIGNALS_DECL_WRAPPER(InitWallet, void, ); + /** Number of network connections changed. */ ADD_SIGNALS_DECL_WRAPPER(NotifyNumConnectionsChanged, void, int newNumConnections); diff --git a/src/policy/feerate.cpp b/src/policy/feerate.cpp index 25b9282b4e..ce149067b7 100644 --- a/src/policy/feerate.cpp +++ b/src/policy/feerate.cpp @@ -7,6 +7,8 @@ #include <tinyformat.h> +#include <cmath> + CFeeRate::CFeeRate(const CAmount& nFeePaid, uint32_t num_bytes) { const int64_t nSize{num_bytes}; @@ -22,7 +24,9 @@ CAmount CFeeRate::GetFee(uint32_t num_bytes) const { const int64_t nSize{num_bytes}; - CAmount nFee = nSatoshisPerK * nSize / 1000; + // Be explicit that we're converting from a double to int64_t (CAmount) here. + // We've previously had issues with the silent double->int64_t conversion. + CAmount nFee{static_cast<CAmount>(std::ceil(nSatoshisPerK * nSize / 1000.0))}; if (nFee == 0 && nSize != 0) { if (nSatoshisPerK > 0) nFee = CAmount(1); diff --git a/src/policy/feerate.h b/src/policy/feerate.h index b16f3f8251..13c7ec2002 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -48,6 +48,7 @@ public: CFeeRate(const CAmount& nFeePaid, uint32_t num_bytes); /** * Return the fee in satoshis for the given size in bytes. + * If the calculated fee would have fractional satoshis, then the returned fee will always be rounded up to the nearest satoshi. */ CAmount GetFee(uint32_t num_bytes) const; /** @@ -66,4 +67,4 @@ public: SERIALIZE_METHODS(CFeeRate, obj) { READWRITE(obj.nSatoshisPerK); } }; -#endif // BITCOIN_POLICY_FEERATE_H +#endif // BITCOIN_POLICY_FEERATE_H diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index d8c21bd833..36cf786bd5 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -493,6 +493,12 @@ void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHe bool CBlockPolicyEstimator::removeTx(uint256 hash, bool inBlock) { LOCK(m_cs_fee_estimator); + return _removeTx(hash, inBlock); +} + +bool CBlockPolicyEstimator::_removeTx(const uint256& hash, bool inBlock) +{ + AssertLockHeld(m_cs_fee_estimator); std::map<uint256, TxStatsInfo>::iterator pos = mapMemPoolTxs.find(hash); if (pos != mapMemPoolTxs.end()) { feeStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock); @@ -576,7 +582,8 @@ void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, boo bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry) { - if (!removeTx(entry->GetTx().GetHash(), true)) { + AssertLockHeld(m_cs_fee_estimator); + if (!_removeTx(entry->GetTx().GetHash(), true)) { // This transaction wasn't being tracked for fee estimation return false; } @@ -985,7 +992,7 @@ void CBlockPolicyEstimator::FlushUnconfirmed() { // Remove every entry in mapMemPoolTxs while (!mapMemPoolTxs.empty()) { auto mi = mapMemPoolTxs.begin(); - removeTx(mi->first, false); // this calls erase() on mapMemPoolTxs + _removeTx(mi->first, false); // this calls erase() on mapMemPoolTxs } int64_t endclear = GetTimeMicros(); LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %gs\n", num_entries, (endclear - startclear)*0.000001); diff --git a/src/policy/fees.h b/src/policy/fees.h index 27f9120c64..37a7051045 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -186,47 +186,59 @@ public: /** Process all the transactions that have been included in a block */ void processBlock(unsigned int nBlockHeight, - std::vector<const CTxMemPoolEntry*>& entries); + std::vector<const CTxMemPoolEntry*>& entries) + EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** Process a transaction accepted to the mempool*/ - void processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate); + void processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate) + EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** Remove a transaction from the mempool tracking stats*/ - bool removeTx(uint256 hash, bool inBlock); + bool removeTx(uint256 hash, bool inBlock) + EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** DEPRECATED. Return a feerate estimate */ - CFeeRate estimateFee(int confTarget) const; + CFeeRate estimateFee(int confTarget) const + EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** Estimate feerate needed to get be included in a block within confTarget * blocks. If no answer can be given at confTarget, return an estimate at * the closest target where one can be given. 'conservative' estimates are * valid over longer time horizons also. */ - CFeeRate estimateSmartFee(int confTarget, FeeCalculation *feeCalc, bool conservative) const; + CFeeRate estimateSmartFee(int confTarget, FeeCalculation *feeCalc, bool conservative) const + EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** Return a specific fee estimate calculation with a given success * threshold and time horizon, and optionally return detailed data about * calculation */ - CFeeRate estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult *result = nullptr) const; + CFeeRate estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, + EstimationResult* result = nullptr) const + EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** Write estimation data to a file */ - bool Write(CAutoFile& fileout) const; + bool Write(CAutoFile& fileout) const + EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** Read estimation data from a file */ - bool Read(CAutoFile& filein); + bool Read(CAutoFile& filein) + EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** Empty mempool transactions on shutdown to record failure to confirm for txs still in mempool */ - void FlushUnconfirmed(); + void FlushUnconfirmed() + EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** Calculation of highest target that estimates are tracked for */ - unsigned int HighestTargetTracked(FeeEstimateHorizon horizon) const; + unsigned int HighestTargetTracked(FeeEstimateHorizon horizon) const + EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** Drop still unconfirmed transactions and record current estimations, if the fee estimation file is present. */ - void Flush(); + void Flush() + EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); private: - mutable RecursiveMutex m_cs_fee_estimator; + mutable Mutex m_cs_fee_estimator; unsigned int nBestSeenHeight GUARDED_BY(m_cs_fee_estimator); unsigned int firstRecordedHeight GUARDED_BY(m_cs_fee_estimator); @@ -267,6 +279,10 @@ private: unsigned int HistoricalBlockSpan() const EXCLUSIVE_LOCKS_REQUIRED(m_cs_fee_estimator); /** Calculation of highest target that reasonable estimate can be provided for */ unsigned int MaxUsableEstimate() const EXCLUSIVE_LOCKS_REQUIRED(m_cs_fee_estimator); + + /** A non-thread-safe helper for the removeTx function */ + bool _removeTx(const uint256& hash, bool inBlock) + EXCLUSIVE_LOCKS_REQUIRED(m_cs_fee_estimator); }; class FeeFilterRounder diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index fced397e51..5d0bb68ec6 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -161,13 +161,13 @@ bool IsStandardTx(const CTransaction& tx, bool permit_bare_multisig, const CFeeR * * Note that only the non-witness portion of the transaction is checked here. */ -bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs, bool taproot_active) +bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) { - if (tx.IsCoinBase()) + if (tx.IsCoinBase()) { return true; // Coinbases don't use vin normally + } - for (unsigned int i = 0; i < tx.vin.size(); i++) - { + for (unsigned int i = 0; i < tx.vin.size(); i++) { const CTxOut& prev = mapInputs.AccessCoin(tx.vin[i].prevout).out; std::vector<std::vector<unsigned char> > vSolutions; @@ -189,9 +189,6 @@ bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs, if (subscript.GetSigOpCount(true) > MAX_P2SH_SIGOPS) { return false; } - } else if (whichType == TxoutType::WITNESS_V1_TAPROOT) { - // Don't allow Taproot spends unless Taproot is active. - if (!taproot_active) return false; } } @@ -254,7 +251,7 @@ bool IsWitnessStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) // - No annexes if (witnessversion == 1 && witnessprogram.size() == WITNESS_V1_TAPROOT_SIZE && !p2sh) { // Taproot spend (non-P2SH-wrapped, version 1, witness program size 32; see BIP 341) - auto stack = MakeSpan(tx.vin[i].scriptWitness.stack); + Span stack{tx.vin[i].scriptWitness.stack}; if (stack.size() >= 2 && !stack.back().empty() && stack.back()[0] == ANNEX_TAG) { // Annexes are nonstandard as long as no semantics are defined for them. return false; diff --git a/src/policy/policy.h b/src/policy/policy.h index f2a3f35546..f6ac6500f6 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -105,10 +105,9 @@ bool IsStandardTx(const CTransaction& tx, bool permit_bare_multisig, const CFeeR /** * Check for standard transaction types * @param[in] mapInputs Map of previous transactions that have outputs we're spending -* @param[in] taproot_active Whether or taproot consensus rules are active (used to decide whether spends of them are permitted) * @return True if all inputs (scriptSigs) use only standard transaction forms */ -bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs, bool taproot_active); +bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs); /** * Check if the transaction is over standard P2WSH resources limit: * 3600bytes witnessScript size, 80bytes per witness stack element, 100 witness stack elements diff --git a/src/psbt.cpp b/src/psbt.cpp index 5445bc8aa1..6585766807 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -223,7 +223,7 @@ void UpdatePSBTOutput(const SigningProvider& provider, PartiallySignedTransactio // Construct a would-be spend of this output, to update sigdata with. // Note that ProduceSignature is used to fill in metadata (not actual signatures), // so provider does not need to provide any private keys (it can be a HidingSigningProvider). - MutableTransactionSignatureCreator creator(&tx, /* index */ 0, out.nValue, SIGHASH_ALL); + MutableTransactionSignatureCreator creator(&tx, /*input_idx=*/0, out.nValue, SIGHASH_ALL); ProduceSignature(provider, creator, out.scriptPubKey, sigdata); // Put redeem_script, witness_script, key paths, into PSBTOutput. @@ -247,7 +247,7 @@ PrecomputedTransactionData PrecomputePSBTData(const PartiallySignedTransaction& return txdata; } -bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index, const PrecomputedTransactionData* txdata, int sighash, SignatureData* out_sigdata) +bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index, const PrecomputedTransactionData* txdata, int sighash, SignatureData* out_sigdata, bool finalize) { PSBTInput& input = psbt.inputs.at(index); const CMutableTransaction& tx = *psbt.tx; @@ -295,6 +295,10 @@ bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& } // Verify that a witness signature was produced in case one was required. if (require_witness_sig && !sigdata.witness) return false; + + // If we are not finalizing, set sigdata.complete to false to not set the scriptWitness + if (!finalize && sigdata.complete) sigdata.complete = false; + input.FromSignatureData(sigdata); // If we have a witness signature, put a witness UTXO. @@ -324,7 +328,7 @@ bool FinalizePSBT(PartiallySignedTransaction& psbtx) bool complete = true; const PrecomputedTransactionData txdata = PrecomputePSBTData(psbtx); for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { - complete &= SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, &txdata, SIGHASH_ALL); + complete &= SignPSBTInput(DUMMY_SIGNING_PROVIDER, psbtx, i, &txdata, SIGHASH_ALL, nullptr, true); } return complete; diff --git a/src/psbt.h b/src/psbt.h index f6b82b43de..1171ecf1dd 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -81,7 +81,7 @@ struct PSBTInput if (final_script_sig.empty() && final_script_witness.IsNull()) { // Write any partial signatures for (auto sig_pair : partial_sigs) { - SerializeToVector(s, PSBT_IN_PARTIAL_SIG, MakeSpan(sig_pair.second.first)); + SerializeToVector(s, PSBT_IN_PARTIAL_SIG, Span{sig_pair.second.first}); s << sig_pair.second.second; } @@ -578,7 +578,7 @@ bool PSBTInputSigned(const PSBTInput& input); * txdata should be the output of PrecomputePSBTData (which can be shared across * multiple SignPSBTInput calls). If it is nullptr, a dummy signature will be created. **/ -bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index, const PrecomputedTransactionData* txdata, int sighash = SIGHASH_ALL, SignatureData* out_sigdata = nullptr); +bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& psbt, int index, const PrecomputedTransactionData* txdata, int sighash = SIGHASH_ALL, SignatureData* out_sigdata = nullptr, bool finalize = true); /** Counts the unsigned inputs of a PSBT. */ size_t CountPSBTUnsignedInputs(const PartiallySignedTransaction& psbt); diff --git a/src/pubkey.h b/src/pubkey.h index 861a2cf500..2453c92d92 100644 --- a/src/pubkey.h +++ b/src/pubkey.h @@ -141,7 +141,7 @@ public: template <typename Stream> void Unserialize(Stream& s) { - unsigned int len = ::ReadCompactSize(s); + const unsigned int len(::ReadCompactSize(s)); if (len <= SIZE) { s.read((char*)vch, len); if (len != size()) { @@ -149,9 +149,7 @@ public: } } else { // invalid pubkey, skip available data - char dummy; - while (len--) - s.read(&dummy, 1); + s.ignore(len); Invalidate(); } } @@ -159,13 +157,13 @@ public: //! Get the KeyID of this public key (hash of its serialization) CKeyID GetID() const { - return CKeyID(Hash160(MakeSpan(vch).first(size()))); + return CKeyID(Hash160(Span{vch}.first(size()))); } //! Get the 256-bit hash of this public key. uint256 GetHash() const { - return Hash(MakeSpan(vch).first(size())); + return Hash(Span{vch}.first(size())); } /* @@ -230,8 +228,8 @@ public: XOnlyPubKey& operator=(const XOnlyPubKey&) = default; /** Determine if this pubkey is fully valid. This is true for approximately 50% of all - * possible 32-byte arrays. If false, VerifySchnorr and CreatePayToContract will always - * fail. */ + * possible 32-byte arrays. If false, VerifySchnorr, CheckTapTweak and CreateTapTweak + * will always fail. */ bool IsFullyValid() const; /** Test whether this is the 0 key (the result of default construction). This implies @@ -242,7 +240,7 @@ public: explicit XOnlyPubKey(Span<const unsigned char> bytes); /** Construct an x-only pubkey from a normal pubkey. */ - explicit XOnlyPubKey(const CPubKey& pubkey) : XOnlyPubKey(Span<const unsigned char>(pubkey.begin() + 1, pubkey.begin() + 33)) {} + explicit XOnlyPubKey(const CPubKey& pubkey) : XOnlyPubKey(Span{pubkey}.subspan(1, 32)) {} /** Verify a Schnorr signature against this public key. * diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 3f412d8f19..922aac531f 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -273,7 +273,6 @@ void BitcoinApplication::createSplashScreen(const NetworkStyle *networkStyle) // We don't hold a direct pointer to the splash screen after creation, but the splash // screen will take care of deleting itself when finish() happens. m_splash->show(); - connect(this, &BitcoinApplication::requestedInitialize, m_splash, &SplashScreen::handleLoadWallet); connect(this, &BitcoinApplication::splashFinished, m_splash, &SplashScreen::finish); connect(this, &BitcoinApplication::requestedShutdown, m_splash, &QWidget::close); } diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index b68ce39b53..81a1d88d20 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -870,7 +870,7 @@ void BitcoinGUI::showHelpMessageClicked() #ifdef ENABLE_WALLET void BitcoinGUI::openClicked() { - OpenURIDialog dlg(this); + OpenURIDialog dlg(platformStyle, this); if(dlg.exec()) { Q_EMIT receivedURI(dlg.getURI()); diff --git a/src/qt/createwalletdialog.cpp b/src/qt/createwalletdialog.cpp index f9a61c3e60..eba70331f8 100644 --- a/src/qt/createwalletdialog.cpp +++ b/src/qt/createwalletdialog.cpp @@ -6,7 +6,7 @@ #include <config/bitcoin-config.h> #endif -#include <external_signer.h> +#include <interfaces/node.h> #include <qt/createwalletdialog.h> #include <qt/forms/ui_createwalletdialog.h> @@ -113,7 +113,7 @@ CreateWalletDialog::~CreateWalletDialog() delete ui; } -void CreateWalletDialog::setSigners(const std::vector<ExternalSigner>& signers) +void CreateWalletDialog::setSigners(const std::vector<std::unique_ptr<interfaces::ExternalSigner>>& signers) { m_has_signers = !signers.empty(); if (m_has_signers) { @@ -126,7 +126,7 @@ void CreateWalletDialog::setSigners(const std::vector<ExternalSigner>& signers) ui->blank_wallet_checkbox->setChecked(false); ui->disable_privkeys_checkbox->setEnabled(false); ui->disable_privkeys_checkbox->setChecked(true); - const std::string label = signers[0].m_name; + const std::string label = signers[0]->getName(); ui->wallet_name_line_edit->setText(QString::fromStdString(label)); ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(true); } else { diff --git a/src/qt/createwalletdialog.h b/src/qt/createwalletdialog.h index fc13cc44eb..63a5e012d8 100644 --- a/src/qt/createwalletdialog.h +++ b/src/qt/createwalletdialog.h @@ -7,7 +7,12 @@ #include <QDialog> +#include <memory> + +namespace interfaces { class ExternalSigner; +} // namespace interfaces + class WalletModel; namespace Ui { @@ -24,7 +29,7 @@ public: explicit CreateWalletDialog(QWidget* parent); virtual ~CreateWalletDialog(); - void setSigners(const std::vector<ExternalSigner>& signers); + void setSigners(const std::vector<std::unique_ptr<interfaces::ExternalSigner>>& signers); QString walletName() const; bool isEncryptWalletChecked() const; diff --git a/src/qt/forms/openuridialog.ui b/src/qt/forms/openuridialog.ui index 1b7291ab9d..97399e59a2 100644 --- a/src/qt/forms/openuridialog.ui +++ b/src/qt/forms/openuridialog.ui @@ -30,6 +30,27 @@ </property> </widget> </item> + <item> + <widget class="QToolButton" name="pasteButton"> + <property name="toolTip"> + <string extracomment="Tooltip text for button that allows you to paste an address that is in your clipboard.">Paste address from clipboard</string> + </property> + <property name="text"> + <string/> + </property> + <property name="icon"> + <iconset resource="../bitcoin.qrc"> + <normaloff>:/icons/editpaste</normaloff>:/icons/editpaste + </iconset> + </property> + <property name="iconSize"> + <size> + <width>22</width> + <height>22</height> + </size> + </property> + </widget> + </item> </layout> </item> <item> diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index 1c22124616..6b3a4630a3 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -33,7 +33,7 @@ <string>Automatically start %1 after logging in to the system.</string> </property> <property name="text"> - <string>Start %1 on system &login</string> + <string>&Start %1 on system login</string> </property> </widget> </item> @@ -192,7 +192,7 @@ <string extracomment="Tooltip text for Options window setting that enables the RPC server.">This allows you or a third party tool to communicate with the node through command-line and JSON-RPC commands.</string> </property> <property name="text"> - <string extracomment="An Options window setting to enable the RPC server.">Enable RPC &server</string> + <string extracomment="An Options window setting to enable the RPC server.">Enable R&PC server</string> </property> </widget> </item> diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index 211f3f506d..364962ee9a 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -428,6 +428,15 @@ namespace GUIUtil */ void ShowModalDialogAndDeleteOnClose(QDialog* dialog); + inline bool IsEscapeOrBack(int key) + { + if (key == Qt::Key_Escape) return true; +#ifdef Q_OS_ANDROID + if (key == Qt::Key_Back) return true; +#endif // Q_OS_ANDROID + return false; + } + } // namespace GUIUtil #endif // BITCOIN_QT_GUIUTIL_H diff --git a/src/qt/openuridialog.cpp b/src/qt/openuridialog.cpp index 10bf82d532..a68eee718e 100644 --- a/src/qt/openuridialog.cpp +++ b/src/qt/openuridialog.cpp @@ -6,15 +6,20 @@ #include <qt/forms/ui_openuridialog.h> #include <qt/guiutil.h> +#include <qt/platformstyle.h> #include <qt/sendcoinsrecipient.h> +#include <QAbstractButton> +#include <QLineEdit> #include <QUrl> -OpenURIDialog::OpenURIDialog(QWidget *parent) : - QDialog(parent, GUIUtil::dialog_flags), - ui(new Ui::OpenURIDialog) +OpenURIDialog::OpenURIDialog(const PlatformStyle* platformStyle, QWidget* parent) : QDialog(parent, GUIUtil::dialog_flags), + ui(new Ui::OpenURIDialog), + m_platform_style(platformStyle) { ui->setupUi(this); + ui->pasteButton->setIcon(m_platform_style->SingleColorIcon(":/icons/editpaste")); + QObject::connect(ui->pasteButton, &QAbstractButton::clicked, ui->uriEdit, &QLineEdit::paste); GUIUtil::handleCloseWindowShortcut(this); } @@ -32,11 +37,19 @@ QString OpenURIDialog::getURI() void OpenURIDialog::accept() { SendCoinsRecipient rcp; - if(GUIUtil::parseBitcoinURI(getURI(), &rcp)) - { + if (GUIUtil::parseBitcoinURI(getURI(), &rcp)) { /* Only accept value URIs */ QDialog::accept(); } else { ui->uriEdit->setValid(false); } } + +void OpenURIDialog::changeEvent(QEvent* e) +{ + if (e->type() == QEvent::PaletteChange) { + ui->pasteButton->setIcon(m_platform_style->SingleColorIcon(":/icons/editpaste")); + } + + QDialog::changeEvent(e); +} diff --git a/src/qt/openuridialog.h b/src/qt/openuridialog.h index efe4b86f37..f3a8b0ba22 100644 --- a/src/qt/openuridialog.h +++ b/src/qt/openuridialog.h @@ -7,6 +7,8 @@ #include <QDialog> +class PlatformStyle; + namespace Ui { class OpenURIDialog; } @@ -16,16 +18,19 @@ class OpenURIDialog : public QDialog Q_OBJECT public: - explicit OpenURIDialog(QWidget *parent); + explicit OpenURIDialog(const PlatformStyle* platformStyle, QWidget* parent); ~OpenURIDialog(); QString getURI(); protected Q_SLOTS: void accept() override; + void changeEvent(QEvent* e) override; private: - Ui::OpenURIDialog *ui; + Ui::OpenURIDialog* ui; + + const PlatformStyle* m_platform_style; }; #endif // BITCOIN_QT_OPENURIDIALOG_H diff --git a/src/qt/psbtoperationsdialog.cpp b/src/qt/psbtoperationsdialog.cpp index 34d56e5506..3e598bfab9 100644 --- a/src/qt/psbtoperationsdialog.cpp +++ b/src/qt/psbtoperationsdialog.cpp @@ -110,8 +110,8 @@ void PSBTOperationsDialog::broadcastTransaction() CTransactionRef tx = MakeTransactionRef(mtx); std::string err_string; - TransactionError error = BroadcastTransaction( - *m_client_model->node().context(), tx, err_string, DEFAULT_MAX_RAW_TX_FEE_RATE.GetFeePerK(), /* relay */ true, /* await_callback */ false); + TransactionError error = + m_client_model->node().broadcastTransaction(tx, DEFAULT_MAX_RAW_TX_FEE_RATE.GetFeePerK(), err_string); if (error == TransactionError::OK) { showStatus(tr("Transaction broadcast successfully! Transaction ID: %1") diff --git a/src/qt/res/animation/makespinner.sh b/src/qt/res/animation/makespinner.sh index 83142f5034..647ee1aed8 100755 --- a/src/qt/res/animation/makespinner.sh +++ b/src/qt/res/animation/makespinner.sh @@ -5,10 +5,10 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C -FRAMEDIR=$(dirname $0) +FRAMEDIR=$(dirname "$0") for i in {0..35} do - frame=$(printf "%03d" $i) + frame=$(printf "%03d" "$i") angle=$((i * 10)) - convert $FRAMEDIR/../src/spinner.png -background "rgba(0,0,0,0.0)" -distort SRT $angle $FRAMEDIR/spinner-$frame.png + convert "${FRAMEDIR}/../src/spinner.png" -background "rgba(0,0,0,0.0)" -distort SRT $angle "${FRAMEDIR}/spinner-${frame}.png" done diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 0c3332ab76..1c19fb4b27 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -14,6 +14,7 @@ #include <netbase.h> #include <qt/bantablemodel.h> #include <qt/clientmodel.h> +#include <qt/guiutil.h> #include <qt/peertablesortproxy.h> #include <qt/platformstyle.h> #include <qt/walletmodel.h> @@ -866,7 +867,11 @@ void RPCConsole::clear(bool keep_prompt) } // Set default style sheet +#ifdef Q_OS_MAC + QFontInfo fixedFontInfo(GUIUtil::fixedPitchFont(/*use_embedded_font=*/true)); +#else QFontInfo fixedFontInfo(GUIUtil::fixedPitchFont()); +#endif ui->messagesWidget->document()->setDefaultStyleSheet( QString( "table { }" @@ -906,8 +911,7 @@ void RPCConsole::clear(bool keep_prompt) void RPCConsole::keyPressEvent(QKeyEvent *event) { - if(windowType() != Qt::Widget && event->key() == Qt::Key_Escape) - { + if (windowType() != Qt::Widget && GUIUtil::IsEscapeOrBack(event->key())) { close(); } } diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index 61b52fd08a..85703b3350 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -194,6 +194,7 @@ void SplashScreen::subscribeToCoreSignals() // Connect signals to client m_handler_init_message = m_node->handleInitMessage(std::bind(InitMessage, this, std::placeholders::_1)); m_handler_show_progress = m_node->handleShowProgress(std::bind(ShowProgress, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); + m_handler_init_wallet = m_node->handleInitWallet([this]() { handleLoadWallet(); }); } void SplashScreen::handleLoadWallet() diff --git a/src/qt/splashscreen.h b/src/qt/splashscreen.h index 386039291c..8a5875d2a6 100644 --- a/src/qt/splashscreen.h +++ b/src/qt/splashscreen.h @@ -66,6 +66,7 @@ private: bool m_shutdown = false; std::unique_ptr<interfaces::Handler> m_handler_init_message; std::unique_ptr<interfaces::Handler> m_handler_show_progress; + std::unique_ptr<interfaces::Handler> m_handler_init_wallet; std::unique_ptr<interfaces::Handler> m_handler_load_wallet; std::list<std::unique_ptr<interfaces::Wallet>> m_connected_wallets; std::list<std::unique_ptr<interfaces::Handler>> m_connected_wallet_handlers; diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index ede0e4cf9e..39cc1c1e1d 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -63,7 +63,7 @@ void TestAddAddressesToSendBook(interfaces::Node& node) auto wallet_client = interfaces::MakeWalletClient(*test.m_node.chain, *Assert(test.m_node.args)); test.m_node.wallet_client = wallet_client.get(); node.setContext(&test.m_node); - const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", gArgs, CreateMockWalletDatabase()); wallet->LoadWallet(); wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); { diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index b26cddf4ae..e7a3d724bb 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -69,7 +69,7 @@ int main(int argc, char* argv[]) #if defined(WIN32) if (getenv("QT_QPA_PLATFORM") == nullptr) _putenv_s("QT_QPA_PLATFORM", "minimal"); #else - setenv("QT_QPA_PLATFORM", "minimal", /* overwrite */ 0); + setenv("QT_QPA_PLATFORM", "minimal", 0 /* overwrite */); #endif // Don't remove this, it's needed to access diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index badaf019b0..93d6aa805f 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -141,7 +141,7 @@ void TestGUI(interfaces::Node& node) auto wallet_client = interfaces::MakeWalletClient(*test.m_node.chain, *Assert(test.m_node.args)); test.m_node.wallet_client = wallet_client.get(); node.setContext(&test.m_node); - const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", gArgs, CreateMockWalletDatabase()); wallet->LoadWallet(); wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); { diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index a0ad59f12a..b9a9fcf3d1 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -280,9 +280,9 @@ void CreateWalletActivity::create() { m_create_wallet_dialog = new CreateWalletDialog(m_parent_widget); - std::vector<ExternalSigner> signers; + std::vector<std::unique_ptr<interfaces::ExternalSigner>> signers; try { - signers = node().externalSigners(); + signers = node().listExternalSigners(); } catch (const std::runtime_error& e) { QMessageBox::critical(nullptr, tr("Can't list signers"), e.what()); } diff --git a/src/randomenv.h b/src/randomenv.h index 46cea6f6f2..746516b79b 100644 --- a/src/randomenv.h +++ b/src/randomenv.h @@ -14,4 +14,4 @@ void RandAddDynamicEnv(CSHA512& hasher); /** Gather non-cryptographic environment data that does not change over time. */ void RandAddStaticEnv(CSHA512& hasher); -#endif +#endif // BITCOIN_RANDOMENV_H diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index ccfef3a7c7..2011ffb4fd 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -15,10 +15,12 @@ #include <core_io.h> #include <deploymentinfo.h> #include <deploymentstatus.h> +#include <fs.h> #include <hash.h> #include <index/blockfilterindex.h> #include <index/coinstatsindex.h> #include <node/blockstorage.h> +#include <logging/timer.h> #include <node/coinstats.h> #include <node/context.h> #include <node/utxo_snapshot.h> @@ -2211,7 +2213,11 @@ static RPCHelpMan savemempool() return RPCHelpMan{"savemempool", "\nDumps the mempool to disk. It will fail until the previous dump is fully loaded.\n", {}, - RPCResult{RPCResult::Type::NONE, "", ""}, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR, "filename", "the directory and file where the mempool was saved"}, + }}, RPCExamples{ HelpExampleCli("savemempool", "") + HelpExampleRpc("savemempool", "") @@ -2220,6 +2226,8 @@ static RPCHelpMan savemempool() { const CTxMemPool& mempool = EnsureAnyMemPool(request.context); + const NodeContext& node = EnsureAnyNodeContext(request.context); + if (!mempool.IsLoaded()) { throw JSONRPCError(RPC_MISC_ERROR, "The mempool was not loaded yet"); } @@ -2228,7 +2236,10 @@ static RPCHelpMan savemempool() throw JSONRPCError(RPC_MISC_ERROR, "Unable to dump mempool to disk"); } - return NullUniValue; + UniValue ret(UniValue::VOBJ); + ret.pushKV("filename", fs::path((node.args->GetDataDirNet() / "mempool.dat")).u8string()); + + return ret; }, }; } @@ -2296,6 +2307,9 @@ public: static RPCHelpMan scantxoutset() { + // scriptPubKey corresponding to mainnet address 12cbQLTFMXRnSzktFkuoG3eHoMeFtpTu3S + const std::string EXAMPLE_DESCRIPTOR_RAW = "raw(76a91411b366edfc0a8b66feebae5c2e25a7b6a5d1cf3188ac)#fm24fxxy"; + return RPCHelpMan{"scantxoutset", "\nScans the unspent transaction output set for entries that match certain output descriptors.\n" "Examples of output descriptors are:\n" @@ -2353,7 +2367,14 @@ static RPCHelpMan scantxoutset() {RPCResult::Type::STR_AMOUNT, "total_amount", "The total amount of all found unspent outputs in " + CURRENCY_UNIT}, }}, }, - RPCExamples{""}, + RPCExamples{ + HelpExampleCli("scantxoutset", "start \'[\"" + EXAMPLE_DESCRIPTOR_RAW + "\"]\'") + + HelpExampleCli("scantxoutset", "status") + + HelpExampleCli("scantxoutset", "abort") + + HelpExampleRpc("scantxoutset", "\"start\", [\"" + EXAMPLE_DESCRIPTOR_RAW + "\"]") + + HelpExampleRpc("scantxoutset", "\"status\"") + + HelpExampleRpc("scantxoutset", "\"abort\"") + }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VARR}); @@ -2556,6 +2577,8 @@ static RPCHelpMan dumptxoutset() {RPCResult::Type::STR_HEX, "base_hash", "the hash of the base of the snapshot"}, {RPCResult::Type::NUM, "base_height", "the height of the base of the snapshot"}, {RPCResult::Type::STR, "path", "the absolute path that the snapshot was written to"}, + {RPCResult::Type::STR_HEX, "txoutset_hash", "the hash of the UTXO set contents"}, + {RPCResult::Type::NUM, "nchaintx", "the number of transactions in the chain up to and including the base block"}, } }, RPCExamples{ @@ -2578,7 +2601,8 @@ static RPCHelpMan dumptxoutset() FILE* file{fsbridge::fopen(temppath, "wb")}; CAutoFile afile{file, SER_DISK, CLIENT_VERSION}; NodeContext& node = EnsureAnyNodeContext(request.context); - UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), afile); + UniValue result = CreateUTXOSnapshot( + node, node.chainman->ActiveChainstate(), afile, path, temppath); fs::rename(temppath, path); result.pushKV("path", path.u8string()); @@ -2587,10 +2611,15 @@ static RPCHelpMan dumptxoutset() }; } -UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFile& afile) +UniValue CreateUTXOSnapshot( + NodeContext& node, + CChainState& chainstate, + CAutoFile& afile, + const fs::path& path, + const fs::path& temppath) { std::unique_ptr<CCoinsViewCursor> pcursor; - CCoinsStats stats{CoinStatsHashType::NONE}; + CCoinsStats stats{CoinStatsHashType::HASH_SERIALIZED}; CBlockIndex* tip; { @@ -2619,6 +2648,10 @@ UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFil CHECK_NONFATAL(tip); } + LOG_TIME_SECONDS(strprintf("writing UTXO snapshot at height %s (%s) to file %s (via %s)", + tip->nHeight, tip->GetBlockHash().ToString(), + fs::PathToString(path), fs::PathToString(temppath))); + SnapshotMetadata metadata{tip->GetBlockHash(), stats.coins_count, tip->nChainTx}; afile << metadata; @@ -2644,7 +2677,11 @@ UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFil result.pushKV("coins_written", stats.coins_count); result.pushKV("base_hash", tip->GetBlockHash().ToString()); result.pushKV("base_height", tip->nHeight); - + result.pushKV("path", path.u8string()); + result.pushKV("txoutset_hash", stats.hashSerialized.ToString()); + // Cast required because univalue doesn't have serialization specified for + // `unsigned int`, nChainTx's type. + result.pushKV("nchaintx", uint64_t{tip->nChainTx}); return result; } diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index d9c6761f47..0ad68a7fd6 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -7,6 +7,7 @@ #include <consensus/amount.h> #include <core_io.h> +#include <fs.h> #include <streams.h> #include <sync.h> @@ -65,6 +66,11 @@ CBlockPolicyEstimator& EnsureAnyFeeEstimator(const std::any& context); * Helper to create UTXO snapshots given a chainstate and a file handle. * @return a UniValue map containing metadata about the snapshot. */ -UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFile& afile); - -#endif +UniValue CreateUTXOSnapshot( + NodeContext& node, + CChainState& chainstate, + CAutoFile& afile, + const fs::path& path, + const fs::path& tmppath); + +#endif // BITCOIN_RPC_BLOCKCHAIN_H diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 93e49cb9a8..90fbd823a4 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -115,6 +115,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "walletcreatefundedpsbt", 4, "bip32derivs" }, { "walletprocesspsbt", 1, "sign" }, { "walletprocesspsbt", 3, "bip32derivs" }, + { "walletprocesspsbt", 4, "finalize" }, { "createpsbt", 0, "inputs" }, { "createpsbt", 1, "outputs" }, { "createpsbt", 2, "locktime" }, diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 518c41d12a..83eab32779 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -13,9 +13,9 @@ #include <deploymentinfo.h> #include <deploymentstatus.h> #include <key_io.h> -#include <miner.h> #include <net.h> #include <node/context.h> +#include <node/miner.h> #include <policy/fees.h> #include <pow.h> #include <rpc/blockchain.h> @@ -1010,7 +1010,7 @@ static RPCHelpMan submitblock() bool new_block; auto sc = std::make_shared<submitblock_StateCatcher>(block.GetHash()); RegisterSharedValidationInterface(sc); - bool accepted = chainman.ProcessNewBlock(Params(), blockptr, /* fForceProcessing */ true, /* fNewBlock */ &new_block); + bool accepted = chainman.ProcessNewBlock(Params(), blockptr, /*force_processing=*/true, /*new_block=*/&new_block); UnregisterSharedValidationInterface(sc); if (!new_block && accepted) { return "duplicate"; @@ -1121,10 +1121,10 @@ static RPCHelpMan estimatesmartfee() UniValue errors(UniValue::VARR); FeeCalculation feeCalc; CFeeRate feeRate{fee_estimator.estimateSmartFee(conf_target, &feeCalc, conservative)}; - CFeeRate min_mempool_feerate{mempool.GetMinFee(gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000)}; - CFeeRate min_relay_feerate{::minRelayTxFee}; - feeRate = std::max({feeRate, min_mempool_feerate, min_relay_feerate}); if (feeRate != CFeeRate(0)) { + CFeeRate min_mempool_feerate{mempool.GetMinFee(gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000)}; + CFeeRate min_relay_feerate{::minRelayTxFee}; + feeRate = std::max({feeRate, min_mempool_feerate, min_relay_feerate}); result.pushKV("feerate", ValueFromAmount(feeRate.GetFeePerK())); } else { errors.push_back("Insufficient data or no feerate found"); diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 39bd9c6091..2c05fc39fd 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -36,32 +36,38 @@ static RPCHelpMan validateaddress() { - return RPCHelpMan{"validateaddress", - "\nReturn information about the given bitcoin address.\n", - { - {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address to validate"}, - }, - RPCResult{ - RPCResult::Type::OBJ, "", "", + return RPCHelpMan{ + "validateaddress", + "\nReturn information about the given bitcoin address.\n", + { + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address to validate"}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::BOOL, "isvalid", "If the address is valid or not"}, + {RPCResult::Type::STR, "address", /* optional */ true, "The bitcoin address validated"}, + {RPCResult::Type::STR_HEX, "scriptPubKey", /* optional */ true, "The hex-encoded scriptPubKey generated by the address"}, + {RPCResult::Type::BOOL, "isscript", /* optional */ true, "If the key is a script"}, + {RPCResult::Type::BOOL, "iswitness", /* optional */ true, "If the address is a witness address"}, + {RPCResult::Type::NUM, "witness_version", /* optional */ true, "The version number of the witness program"}, + {RPCResult::Type::STR_HEX, "witness_program", /* optional */ true, "The hex value of the witness program"}, + {RPCResult::Type::STR, "error", /* optional */ true, "Error message, if any"}, + {RPCResult::Type::ARR, "error_locations", /*optional=*/true, "Indices of likely error locations in address, if known (e.g. Bech32 errors)", { - {RPCResult::Type::BOOL, "isvalid", "If the address is valid or not"}, - {RPCResult::Type::STR, "address", /* optional */ true, "The bitcoin address validated"}, - {RPCResult::Type::STR_HEX, "scriptPubKey", /* optional */ true, "The hex-encoded scriptPubKey generated by the address"}, - {RPCResult::Type::BOOL, "isscript", /* optional */ true, "If the key is a script"}, - {RPCResult::Type::BOOL, "iswitness", /* optional */ true, "If the address is a witness address"}, - {RPCResult::Type::NUM, "witness_version", /* optional */ true, "The version number of the witness program"}, - {RPCResult::Type::STR_HEX, "witness_program", /* optional */ true, "The hex value of the witness program"}, - {RPCResult::Type::STR, "error", /* optional */ true, "Error message, if any"}, - } - }, - RPCExamples{ - HelpExampleCli("validateaddress", "\"" + EXAMPLE_ADDRESS[0] + "\"") + - HelpExampleRpc("validateaddress", "\"" + EXAMPLE_ADDRESS[0] + "\"") - }, + {RPCResult::Type::NUM, "index", "index of a potential error"}, + }}, + } + }, + RPCExamples{ + HelpExampleCli("validateaddress", "\"" + EXAMPLE_ADDRESS[0] + "\"") + + HelpExampleRpc("validateaddress", "\"" + EXAMPLE_ADDRESS[0] + "\"") + }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { std::string error_msg; - CTxDestination dest = DecodeDestination(request.params[0].get_str(), error_msg); + std::vector<int> error_locations; + CTxDestination dest = DecodeDestination(request.params[0].get_str(), error_msg, &error_locations); const bool isValid = IsValidDestination(dest); CHECK_NONFATAL(isValid == error_msg.empty()); @@ -77,6 +83,9 @@ static RPCHelpMan validateaddress() UniValue detail = DescribeAddress(dest); ret.pushKVs(detail); } else { + UniValue error_indices(UniValue::VARR); + for (int i : error_locations) error_indices.push_back(i); + ret.pushKV("error_locations", error_indices); ret.pushKV("error", error_msg); } @@ -550,7 +559,7 @@ static RPCHelpMan getmemoryinfo() #ifdef HAVE_MALLOC_INFO return RPCMallocInfo(); #else - throw JSONRPCError(RPC_INVALID_PARAMETER, "mallocinfo is only available when compiled with glibc 2.10+"); + throw JSONRPCError(RPC_INVALID_PARAMETER, "mallocinfo mode not available"); #endif } else { throw JSONRPCError(RPC_INVALID_PARAMETER, "unknown mode " + mode); diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index a9bee33c5d..021e6ae320 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -4,6 +4,7 @@ #include <rpc/server.h> +#include <addrman.h> #include <banman.h> #include <chainparams.h> #include <clientversion.h> @@ -105,82 +106,83 @@ static RPCHelpMan ping() static RPCHelpMan getpeerinfo() { - return RPCHelpMan{"getpeerinfo", - "\nReturns data about each connected network node as a json array of objects.\n", - {}, - RPCResult{ - RPCResult::Type::ARR, "", "", + return RPCHelpMan{ + "getpeerinfo", + "\nReturns data about each connected network node as a json array of objects.\n", + {}, + RPCResult{ + RPCResult::Type::ARR, "", "", + { + {RPCResult::Type::OBJ, "", "", + { { - {RPCResult::Type::OBJ, "", "", - { - { - {RPCResult::Type::NUM, "id", "Peer index"}, - {RPCResult::Type::STR, "addr", "(host:port) The IP address and port of the peer"}, - {RPCResult::Type::STR, "addrbind", /* optional */ true, "(ip:port) Bind address of the connection to the peer"}, - {RPCResult::Type::STR, "addrlocal", /* optional */ true, "(ip:port) Local address as reported by the peer"}, - {RPCResult::Type::STR, "network", "Network (" + Join(GetNetworkNames(/* append_unroutable */ true), ", ") + ")"}, - {RPCResult::Type::NUM, "mapped_as", /* optional */ true, "The AS in the BGP route to the peer used for diversifying\n" - "peer selection (only available if the asmap config flag is set)"}, - {RPCResult::Type::STR_HEX, "services", "The services offered"}, - {RPCResult::Type::ARR, "servicesnames", "the services offered, in human-readable form", - { - {RPCResult::Type::STR, "SERVICE_NAME", "the service name if it is recognised"} - }}, - {RPCResult::Type::BOOL, "relaytxes", "Whether peer has asked us to relay transactions to it"}, - {RPCResult::Type::NUM_TIME, "lastsend", "The " + UNIX_EPOCH_TIME + " of the last send"}, - {RPCResult::Type::NUM_TIME, "lastrecv", "The " + UNIX_EPOCH_TIME + " of the last receive"}, - {RPCResult::Type::NUM_TIME, "last_transaction", "The " + UNIX_EPOCH_TIME + " of the last valid transaction received from this peer"}, - {RPCResult::Type::NUM_TIME, "last_block", "The " + UNIX_EPOCH_TIME + " of the last block received from this peer"}, - {RPCResult::Type::NUM, "bytessent", "The total bytes sent"}, - {RPCResult::Type::NUM, "bytesrecv", "The total bytes received"}, - {RPCResult::Type::NUM_TIME, "conntime", "The " + UNIX_EPOCH_TIME + " of the connection"}, - {RPCResult::Type::NUM, "timeoffset", "The time offset in seconds"}, - {RPCResult::Type::NUM, "pingtime", /* optional */ true, "ping time (if available)"}, - {RPCResult::Type::NUM, "minping", /* optional */ true, "minimum observed ping time (if any at all)"}, - {RPCResult::Type::NUM, "pingwait", /* optional */ true, "ping wait (if non-zero)"}, - {RPCResult::Type::NUM, "version", "The peer version, such as 70001"}, - {RPCResult::Type::STR, "subver", "The string version"}, - {RPCResult::Type::BOOL, "inbound", "Inbound (true) or Outbound (false)"}, - {RPCResult::Type::BOOL, "bip152_hb_to", "Whether we selected peer as (compact blocks) high-bandwidth peer"}, - {RPCResult::Type::BOOL, "bip152_hb_from", "Whether peer selected us as (compact blocks) high-bandwidth peer"}, - {RPCResult::Type::NUM, "startingheight", "The starting height (block) of the peer"}, - {RPCResult::Type::NUM, "synced_headers", "The last header we have in common with this peer"}, - {RPCResult::Type::NUM, "synced_blocks", "The last block we have in common with this peer"}, - {RPCResult::Type::ARR, "inflight", "", - { - {RPCResult::Type::NUM, "n", "The heights of blocks we're currently asking from this peer"}, - }}, - {RPCResult::Type::BOOL, "addr_relay_enabled", "Whether we participate in address relay with this peer"}, - {RPCResult::Type::NUM, "addr_processed", "The total number of addresses processed, excluding those dropped due to rate limiting"}, - {RPCResult::Type::NUM, "addr_rate_limited", "The total number of addresses dropped due to rate limiting"}, - {RPCResult::Type::ARR, "permissions", "Any special permissions that have been granted to this peer", - { - {RPCResult::Type::STR, "permission_type", Join(NET_PERMISSIONS_DOC, ",\n") + ".\n"}, - }}, - {RPCResult::Type::NUM, "minfeefilter", "The minimum fee rate for transactions this peer accepts"}, - {RPCResult::Type::OBJ_DYN, "bytessent_per_msg", "", - { - {RPCResult::Type::NUM, "msg", "The total bytes sent aggregated by message type\n" - "When a message type is not listed in this json object, the bytes sent are 0.\n" - "Only known message types can appear as keys in the object."} - }}, - {RPCResult::Type::OBJ_DYN, "bytesrecv_per_msg", "", - { - {RPCResult::Type::NUM, "msg", "The total bytes received aggregated by message type\n" - "When a message type is not listed in this json object, the bytes received are 0.\n" - "Only known message types can appear as keys in the object and all bytes received\n" - "of unknown message types are listed under '"+NET_MESSAGE_COMMAND_OTHER+"'."} - }}, - {RPCResult::Type::STR, "connection_type", "Type of connection: \n" + Join(CONNECTION_TYPE_DOC, ",\n") + ".\n" - "Please note this output is unlikely to be stable in upcoming releases as we iterate to\n" - "best capture connection behaviors."}, - }}, + {RPCResult::Type::NUM, "id", "Peer index"}, + {RPCResult::Type::STR, "addr", "(host:port) The IP address and port of the peer"}, + {RPCResult::Type::STR, "addrbind", /* optional */ true, "(ip:port) Bind address of the connection to the peer"}, + {RPCResult::Type::STR, "addrlocal", /* optional */ true, "(ip:port) Local address as reported by the peer"}, + {RPCResult::Type::STR, "network", "Network (" + Join(GetNetworkNames(/* append_unroutable */ true), ", ") + ")"}, + {RPCResult::Type::NUM, "mapped_as", /* optional */ true, "The AS in the BGP route to the peer used for diversifying\n" + "peer selection (only available if the asmap config flag is set)"}, + {RPCResult::Type::STR_HEX, "services", "The services offered"}, + {RPCResult::Type::ARR, "servicesnames", "the services offered, in human-readable form", + { + {RPCResult::Type::STR, "SERVICE_NAME", "the service name if it is recognised"} }}, - }, - RPCExamples{ - HelpExampleCli("getpeerinfo", "") + {RPCResult::Type::BOOL, "relaytxes", "Whether peer has asked us to relay transactions to it"}, + {RPCResult::Type::NUM_TIME, "lastsend", "The " + UNIX_EPOCH_TIME + " of the last send"}, + {RPCResult::Type::NUM_TIME, "lastrecv", "The " + UNIX_EPOCH_TIME + " of the last receive"}, + {RPCResult::Type::NUM_TIME, "last_transaction", "The " + UNIX_EPOCH_TIME + " of the last valid transaction received from this peer"}, + {RPCResult::Type::NUM_TIME, "last_block", "The " + UNIX_EPOCH_TIME + " of the last block received from this peer"}, + {RPCResult::Type::NUM, "bytessent", "The total bytes sent"}, + {RPCResult::Type::NUM, "bytesrecv", "The total bytes received"}, + {RPCResult::Type::NUM_TIME, "conntime", "The " + UNIX_EPOCH_TIME + " of the connection"}, + {RPCResult::Type::NUM, "timeoffset", "The time offset in seconds"}, + {RPCResult::Type::NUM, "pingtime", /* optional */ true, "ping time (if available)"}, + {RPCResult::Type::NUM, "minping", /* optional */ true, "minimum observed ping time (if any at all)"}, + {RPCResult::Type::NUM, "pingwait", /* optional */ true, "ping wait (if non-zero)"}, + {RPCResult::Type::NUM, "version", "The peer version, such as 70001"}, + {RPCResult::Type::STR, "subver", "The string version"}, + {RPCResult::Type::BOOL, "inbound", "Inbound (true) or Outbound (false)"}, + {RPCResult::Type::BOOL, "bip152_hb_to", "Whether we selected peer as (compact blocks) high-bandwidth peer"}, + {RPCResult::Type::BOOL, "bip152_hb_from", "Whether peer selected us as (compact blocks) high-bandwidth peer"}, + {RPCResult::Type::NUM, "startingheight", /*optional=*/true, "The starting height (block) of the peer"}, + {RPCResult::Type::NUM, "synced_headers", /*optional=*/true, "The last header we have in common with this peer"}, + {RPCResult::Type::NUM, "synced_blocks", /*optional=*/true, "The last block we have in common with this peer"}, + {RPCResult::Type::ARR, "inflight", /*optional=*/true, "", + { + {RPCResult::Type::NUM, "n", "The heights of blocks we're currently asking from this peer"}, + }}, + {RPCResult::Type::BOOL, "addr_relay_enabled", /*optional=*/true, "Whether we participate in address relay with this peer"}, + {RPCResult::Type::NUM, "addr_processed", /*optional=*/true, "The total number of addresses processed, excluding those dropped due to rate limiting"}, + {RPCResult::Type::NUM, "addr_rate_limited", /*optional=*/true, "The total number of addresses dropped due to rate limiting"}, + {RPCResult::Type::ARR, "permissions", "Any special permissions that have been granted to this peer", + { + {RPCResult::Type::STR, "permission_type", Join(NET_PERMISSIONS_DOC, ",\n") + ".\n"}, + }}, + {RPCResult::Type::NUM, "minfeefilter", "The minimum fee rate for transactions this peer accepts"}, + {RPCResult::Type::OBJ_DYN, "bytessent_per_msg", "", + { + {RPCResult::Type::NUM, "msg", "The total bytes sent aggregated by message type\n" + "When a message type is not listed in this json object, the bytes sent are 0.\n" + "Only known message types can appear as keys in the object."} + }}, + {RPCResult::Type::OBJ_DYN, "bytesrecv_per_msg", "", + { + {RPCResult::Type::NUM, "msg", "The total bytes received aggregated by message type\n" + "When a message type is not listed in this json object, the bytes received are 0.\n" + "Only known message types can appear as keys in the object and all bytes received\n" + "of unknown message types are listed under '"+NET_MESSAGE_COMMAND_OTHER+"'."} + }}, + {RPCResult::Type::STR, "connection_type", "Type of connection: \n" + Join(CONNECTION_TYPE_DOC, ",\n") + ".\n" + "Please note this output is unlikely to be stable in upcoming releases as we iterate to\n" + "best capture connection behaviors."}, + }}, + }}, + }, + RPCExamples{ + HelpExampleCli("getpeerinfo", "") + HelpExampleRpc("getpeerinfo", "") - }, + }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { NodeContext& node = EnsureAnyNodeContext(request.context); @@ -566,7 +568,7 @@ static UniValue GetNetworksInfo() UniValue networks(UniValue::VARR); for (int n = 0; n < NET_MAX; ++n) { enum Network network = static_cast<enum Network>(n); - if (network == NET_UNROUTABLE || network == NET_CJDNS || network == NET_INTERNAL) continue; + if (network == NET_UNROUTABLE || network == NET_INTERNAL) continue; proxyType proxy; UniValue obj(UniValue::VOBJ); GetProxy(network, proxy); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 483717aa7a..bd56e584e1 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -69,6 +69,43 @@ static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& } } +static std::vector<RPCArg> CreateTxDoc() +{ + return { + {"inputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "The inputs", + { + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + { + {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, + {"vout", RPCArg::Type::NUM, RPCArg::Optional::NO, "The output number"}, + {"sequence", RPCArg::Type::NUM, RPCArg::DefaultHint{"depends on the value of the 'replaceable' and 'locktime' arguments"}, "The sequence number"}, + }, + }, + }, + }, + {"outputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "The outputs (key-value pairs), where none of the keys are duplicated.\n" + "That is, each address can only appear once and there can only be one 'data' object.\n" + "For compatibility reasons, a dictionary, which holds the key-value pairs directly, is also\n" + " accepted as second parameter.", + { + {"", RPCArg::Type::OBJ_USER_KEYS, RPCArg::Optional::OMITTED, "", + { + {"address", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "A key-value pair. The key (string) is the bitcoin address, the value (float or string) is the amount in " + CURRENCY_UNIT}, + }, + }, + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + { + {"data", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "A key-value pair. The key must be \"data\", the value is hex-encoded data"}, + }, + }, + }, + }, + {"locktime", RPCArg::Type::NUM, RPCArg::Default{0}, "Raw locktime. Non-0 value also locktime-activates inputs"}, + {"replaceable", RPCArg::Type::BOOL, RPCArg::Default{false}, "Marks this transaction as BIP125-replaceable.\n" + "Allows this transaction to be replaced by a transaction with higher fees. If provided, it is an error if explicit sequence numbers are incompatible."}, + }; +} + static RPCHelpMan getrawtransaction() { return RPCHelpMan{ @@ -375,39 +412,7 @@ static RPCHelpMan createrawtransaction() "Returns hex-encoded raw transaction.\n" "Note that the transaction's inputs are not signed, and\n" "it is not stored in the wallet or transmitted to the network.\n", - { - {"inputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "The inputs", - { - {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", - { - {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, - {"vout", RPCArg::Type::NUM, RPCArg::Optional::NO, "The output number"}, - {"sequence", RPCArg::Type::NUM, RPCArg::DefaultHint{"depends on the value of the 'replaceable' and 'locktime' arguments"}, "The sequence number"}, - }, - }, - }, - }, - {"outputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "The outputs (key-value pairs), where none of the keys are duplicated.\n" - "That is, each address can only appear once and there can only be one 'data' object.\n" - "For compatibility reasons, a dictionary, which holds the key-value pairs directly, is also\n" - " accepted as second parameter.", - { - {"", RPCArg::Type::OBJ_USER_KEYS, RPCArg::Optional::OMITTED, "", - { - {"address", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "A key-value pair. The key (string) is the bitcoin address, the value (float or string) is the amount in " + CURRENCY_UNIT}, - }, - }, - {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", - { - {"data", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "A key-value pair. The key must be \"data\", the value is hex-encoded data"}, - }, - }, - }, - }, - {"locktime", RPCArg::Type::NUM, RPCArg::Default{0}, "Raw locktime. Non-0 value also locktime-activates inputs"}, - {"replaceable", RPCArg::Type::BOOL, RPCArg::Default{false}, "Marks this transaction as BIP125-replaceable.\n" - " Allows this transaction to be replaced by a transaction with higher fees. If provided, it is an error if explicit sequence numbers are incompatible."}, - }, + CreateTxDoc(), RPCResult{ RPCResult::Type::STR_HEX, "transaction", "hex string of the transaction" }, @@ -535,32 +540,35 @@ static std::string GetAllOutputTypes() static RPCHelpMan decodescript() { - return RPCHelpMan{"decodescript", - "\nDecode a hex-encoded script.\n", - { - {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "the hex-encoded script"}, - }, - RPCResult{ - RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR, "asm", "Script public key"}, - {RPCResult::Type::STR, "type", "The output type (e.g. "+GetAllOutputTypes()+")"}, - {RPCResult::Type::STR, "address", /* optional */ true, "The Bitcoin address (only if a well-defined address exists)"}, - {RPCResult::Type::STR, "p2sh", /* optional */ true, "address of P2SH script wrapping this redeem script (not returned if the script is already a P2SH)"}, - {RPCResult::Type::OBJ, "segwit", /* optional */ true, "Result of a witness script public key wrapping this redeem script (not returned if the script is a P2SH or witness)", - { - {RPCResult::Type::STR, "asm", "String representation of the script public key"}, - {RPCResult::Type::STR_HEX, "hex", "Hex string of the script public key"}, - {RPCResult::Type::STR, "type", "The type of the script public key (e.g. witness_v0_keyhash or witness_v0_scripthash)"}, - {RPCResult::Type::STR, "address", /* optional */ true, "The Bitcoin address (only if a well-defined address exists)"}, - {RPCResult::Type::STR, "p2sh-segwit", "address of the P2SH script wrapping this witness redeem script"}, - }}, - } - }, - RPCExamples{ - HelpExampleCli("decodescript", "\"hexstring\"") - + HelpExampleRpc("decodescript", "\"hexstring\"") - }, + return RPCHelpMan{ + "decodescript", + "\nDecode a hex-encoded script.\n", + { + {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "the hex-encoded script"}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR, "asm", "Script public key"}, + {RPCResult::Type::STR, "type", "The output type (e.g. " + GetAllOutputTypes() + ")"}, + {RPCResult::Type::STR, "address", /* optional */ true, "The Bitcoin address (only if a well-defined address exists)"}, + {RPCResult::Type::STR, "p2sh", /*optional=*/true, + "address of P2SH script wrapping this redeem script (not returned for types that should not be wrapped)"}, + {RPCResult::Type::OBJ, "segwit", /*optional=*/true, + "Result of a witness script public key wrapping this redeem script (not returned for types that should not be wrapped)", + { + {RPCResult::Type::STR, "asm", "String representation of the script public key"}, + {RPCResult::Type::STR_HEX, "hex", "Hex string of the script public key"}, + {RPCResult::Type::STR, "type", "The type of the script public key (e.g. witness_v0_keyhash or witness_v0_scripthash)"}, + {RPCResult::Type::STR, "address", /* optional */ true, "The Bitcoin address (only if a well-defined address exists)"}, + {RPCResult::Type::STR, "p2sh-segwit", "address of the P2SH script wrapping this witness redeem script"}, + }}, + }, + }, + RPCExamples{ + HelpExampleCli("decodescript", "\"hexstring\"") + + HelpExampleRpc("decodescript", "\"hexstring\"") + }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { RPCTypeCheck(request.params, {UniValue::VSTR}); @@ -575,27 +583,71 @@ static RPCHelpMan decodescript() } ScriptPubKeyToUniv(script, r, /* include_hex */ false); - UniValue type; - type = find_value(r, "type"); + std::vector<std::vector<unsigned char>> solutions_data; + const TxoutType which_type{Solver(script, solutions_data)}; + + const bool can_wrap{[&] { + switch (which_type) { + case TxoutType::MULTISIG: + case TxoutType::NONSTANDARD: + case TxoutType::PUBKEY: + case TxoutType::PUBKEYHASH: + case TxoutType::WITNESS_V0_KEYHASH: + case TxoutType::WITNESS_V0_SCRIPTHASH: + // Can be wrapped if the checks below pass + break; + case TxoutType::NULL_DATA: + case TxoutType::SCRIPTHASH: + case TxoutType::WITNESS_UNKNOWN: + case TxoutType::WITNESS_V1_TAPROOT: + // Should not be wrapped + return false; + } // no default case, so the compiler can warn about missing cases + if (!script.HasValidOps() || script.IsUnspendable()) { + return false; + } + for (CScript::const_iterator it{script.begin()}; it != script.end();) { + opcodetype op; + CHECK_NONFATAL(script.GetOp(it, op)); + if (op == OP_CHECKSIGADD || IsOpSuccess(op)) { + return false; + } + } + return true; + }()}; - if (type.isStr() && type.get_str() != "scripthash") { - // P2SH cannot be wrapped in a P2SH. If this script is already a P2SH, - // don't return the address for a P2SH of the P2SH. + if (can_wrap) { r.pushKV("p2sh", EncodeDestination(ScriptHash(script))); // P2SH and witness programs cannot be wrapped in P2WSH, if this script // is a witness program, don't return addresses for a segwit programs. - if (type.get_str() == "pubkey" || type.get_str() == "pubkeyhash" || type.get_str() == "multisig" || type.get_str() == "nonstandard") { - std::vector<std::vector<unsigned char>> solutions_data; - TxoutType which_type = Solver(script, solutions_data); + const bool can_wrap_P2WSH{[&] { + switch (which_type) { + case TxoutType::MULTISIG: + case TxoutType::PUBKEY: // Uncompressed pubkeys cannot be used with segwit checksigs. // If the script contains an uncompressed pubkey, skip encoding of a segwit program. - if ((which_type == TxoutType::PUBKEY) || (which_type == TxoutType::MULTISIG)) { for (const auto& solution : solutions_data) { if ((solution.size() != 1) && !CPubKey(solution).IsCompressed()) { - return r; + return false; } } - } + return true; + case TxoutType::NONSTANDARD: + case TxoutType::PUBKEYHASH: + // Can be P2WSH wrapped + return true; + case TxoutType::NULL_DATA: + case TxoutType::SCRIPTHASH: + case TxoutType::WITNESS_UNKNOWN: + case TxoutType::WITNESS_V0_KEYHASH: + case TxoutType::WITNESS_V0_SCRIPTHASH: + case TxoutType::WITNESS_V1_TAPROOT: + // Should not be wrapped + return false; + } // no default case, so the compiler can warn about missing cases + CHECK_NONFATAL(false); + }()}; + if (can_wrap_P2WSH) { UniValue sr(UniValue::VOBJ); CScript segwitScr; if (which_type == TxoutType::PUBKEY) { @@ -604,7 +656,6 @@ static RPCHelpMan decodescript() segwitScr = GetScriptForDestination(WitnessV0KeyHash(uint160{solutions_data[0]})); } else { // Scripts that are not fit for P2WPKH are encoded as P2WSH. - // Newer segwit program versions should be considered when then become available. segwitScr = GetScriptForDestination(WitnessV0ScriptHash(script)); } ScriptPubKeyToUniv(segwitScr, sr, /* include_hex */ true); @@ -946,12 +997,13 @@ static RPCHelpMan testmempoolaccept() NodeContext& node = EnsureAnyNodeContext(request.context); CTxMemPool& mempool = EnsureMemPool(node); - CChainState& chainstate = EnsureChainman(node).ActiveChainstate(); + ChainstateManager& chainman = EnsureChainman(node); + CChainState& chainstate = chainman.ActiveChainstate(); const PackageMempoolAcceptResult package_result = [&] { LOCK(::cs_main); if (txns.size() > 1) return ProcessNewPackage(chainstate, mempool, txns, /* test_accept */ true); return PackageMempoolAcceptResult(txns[0]->GetWitnessHash(), - AcceptToMemoryPool(chainstate, mempool, txns[0], /* bypass_limits */ false, /* test_accept*/ true)); + chainman.ProcessTransaction(txns[0], /*test_accept=*/ true)); }(); UniValue rpc_result(UniValue::VARR); @@ -977,7 +1029,7 @@ static RPCHelpMan testmempoolaccept() if (tx_result.m_result_type == MempoolAcceptResult::ResultType::VALID) { const CAmount fee = tx_result.m_base_fees.value(); // Check that fee does not exceed maximum fee - const int64_t virtual_size = GetVirtualTransactionSize(*tx); + const int64_t virtual_size = tx_result.m_vsize.value(); const CAmount max_raw_tx_fee = max_raw_tx_fee_rate.GetFee(virtual_size); if (max_raw_tx_fee && fee > max_raw_tx_fee) { result_inner.pushKV("allowed", false); @@ -1434,39 +1486,7 @@ static RPCHelpMan createpsbt() return RPCHelpMan{"createpsbt", "\nCreates a transaction in the Partially Signed Transaction format.\n" "Implements the Creator role.\n", - { - {"inputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "The json objects", - { - {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", - { - {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, - {"vout", RPCArg::Type::NUM, RPCArg::Optional::NO, "The output number"}, - {"sequence", RPCArg::Type::NUM, RPCArg::DefaultHint{"depends on the value of the 'replaceable' and 'locktime' arguments"}, "The sequence number"}, - }, - }, - }, - }, - {"outputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "The outputs (key-value pairs), where none of the keys are duplicated.\n" - "That is, each address can only appear once and there can only be one 'data' object.\n" - "For compatibility reasons, a dictionary, which holds the key-value pairs directly, is also\n" - " accepted as second parameter.", - { - {"", RPCArg::Type::OBJ_USER_KEYS, RPCArg::Optional::OMITTED, "", - { - {"address", RPCArg::Type::AMOUNT, RPCArg::Optional::NO, "A key-value pair. The key (string) is the bitcoin address, the value (float or string) is the amount in " + CURRENCY_UNIT}, - }, - }, - {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", - { - {"data", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "A key-value pair. The key must be \"data\", the value is hex-encoded data"}, - }, - }, - }, - }, - {"locktime", RPCArg::Type::NUM, RPCArg::Default{0}, "Raw locktime. Non-0 value also locktime-activates inputs"}, - {"replaceable", RPCArg::Type::BOOL, RPCArg::Default{false}, "Marks this transaction as BIP125 replaceable.\n" - " Allows this transaction to be replaced by a transaction with higher fees. If provided, it is an error if explicit sequence numbers are incompatible."}, - }, + CreateTxDoc(), RPCResult{ RPCResult::Type::STR, "", "The resulting raw transaction (base64-encoded string)" }, @@ -1618,7 +1638,7 @@ static RPCHelpMan utxoupdatepsbt() } } // We don't actually need private keys further on; hide them as a precaution. - HidingSigningProvider public_provider(&provider, /* nosign */ true, /* nobip32derivs */ false); + HidingSigningProvider public_provider(&provider, /*hide_secret=*/true, /*hide_origin=*/false); // Fetch previous transactions (inputs): CCoinsView viewDummy; @@ -1657,7 +1677,7 @@ static RPCHelpMan utxoupdatepsbt() // Update script/keypath information using descriptor data. // Note that SignPSBTInput does a lot more than just constructing ECDSA signatures // we don't actually care about those here, in fact. - SignPSBTInput(public_provider, psbtx, i, &txdata, /* sighash_type */ 1); + SignPSBTInput(public_provider, psbtx, i, &txdata, /*sighash=*/1); } // Update script/keypath information using descriptor data. diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 2d7f5f2894..683cf88af8 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -317,7 +317,7 @@ public: UniValue obj(UniValue::VOBJ); obj.pushKV("iswitness", true); obj.pushKV("witness_version", (int)id.version); - obj.pushKV("witness_program", HexStr(Span<const unsigned char>(id.program, id.length))); + obj.pushKV("witness_program", HexStr({id.program, id.length})); return obj; } }; diff --git a/src/scheduler.h b/src/scheduler.h index 9eec8c0fa0..be878661db 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -146,4 +146,4 @@ public: size_t CallbacksPending(); }; -#endif +#endif // BITCOIN_SCHEDULER_H diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 621a1b9fd6..30f929664f 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -631,7 +631,7 @@ public: out.origins.emplace(entry.first.GetID(), std::make_pair<CPubKey, KeyOriginInfo>(CPubKey(entry.first), std::move(entry.second))); } - output_scripts = MakeScripts(pubkeys, MakeSpan(subscripts), out); + output_scripts = MakeScripts(pubkeys, Span{subscripts}, out); return true; } @@ -851,6 +851,7 @@ protected: builder.Finalize(xpk); WitnessV1Taproot output = builder.GetOutput(); out.tr_spenddata[output].Merge(builder.GetSpendData()); + out.pubkeys.emplace(keys[0].GetID(), keys[0]); return Vector(GetScriptForDestination(output)); } bool ToStringSubScriptHelper(const SigningProvider* arg, std::string& ret, const StringType type, const DescriptorCache* cache = nullptr) const override @@ -973,10 +974,10 @@ std::unique_ptr<PubkeyProvider> ParsePubkeyInner(uint32_t key_exp_index, const S } KeyPath path; DeriveType type = DeriveType::NO; - if (split.back() == MakeSpan("*").first(1)) { + if (split.back() == Span{"*"}.first(1)) { split.pop_back(); type = DeriveType::UNHARDENED; - } else if (split.back() == MakeSpan("*'").first(2) || split.back() == MakeSpan("*h").first(2)) { + } else if (split.back() == Span{"*'"}.first(2) || split.back() == Span{"*h"}.first(2)) { split.pop_back(); type = DeriveType::HARDENED; } @@ -1251,7 +1252,7 @@ std::unique_ptr<PubkeyProvider> InferXOnlyPubkey(const XOnlyPubKey& xkey, ParseS std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptContext ctx, const SigningProvider& provider) { if (ctx == ParseScriptContext::P2TR && script.size() == 34 && script[0] == 32 && script[33] == OP_CHECKSIG) { - XOnlyPubKey key{Span<const unsigned char>{script.data() + 1, script.data() + 33}}; + XOnlyPubKey key{Span{script}.subspan(1, 32)}; return std::make_unique<PKDescriptor>(InferXOnlyPubkey(key, ctx, provider)); } diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index eafa9840d7..6433ba1b58 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1483,7 +1483,7 @@ template void PrecomputedTransactionData::Init(const CMutableTransaction& txTo, template PrecomputedTransactionData::PrecomputedTransactionData(const CTransaction& txTo); template PrecomputedTransactionData::PrecomputedTransactionData(const CMutableTransaction& txTo); -static const CHashWriter HASHER_TAPSIGHASH = TaggedHash("TapSighash"); +const CHashWriter HASHER_TAPSIGHASH = TaggedHash("TapSighash"); const CHashWriter HASHER_TAPLEAF = TaggedHash("TapLeaf"); const CHashWriter HASHER_TAPBRANCH = TaggedHash("TapBranch"); @@ -1858,7 +1858,7 @@ uint256 ComputeTaprootMerkleRoot(Span<const unsigned char> control, const uint25 uint256 k = tapleaf_hash; for (int i = 0; i < path_len; ++i) { CHashWriter ss_branch{HASHER_TAPBRANCH}; - Span<const unsigned char> node(control.data() + TAPROOT_CONTROL_BASE_SIZE + TAPROOT_CONTROL_NODE_SIZE * i, TAPROOT_CONTROL_NODE_SIZE); + Span node{Span{control}.subspan(TAPROOT_CONTROL_BASE_SIZE + TAPROOT_CONTROL_NODE_SIZE * i, TAPROOT_CONTROL_NODE_SIZE)}; if (std::lexicographical_compare(k.begin(), k.end(), node.begin(), node.end())) { ss_branch << k << node; } else { @@ -1874,7 +1874,7 @@ static bool VerifyTaprootCommitment(const std::vector<unsigned char>& control, c assert(control.size() >= TAPROOT_CONTROL_BASE_SIZE); assert(program.size() >= uint256::size()); //! The internal pubkey (x-only, so no Y coordinate parity). - const XOnlyPubKey p{Span<const unsigned char>{control.data() + 1, control.data() + TAPROOT_CONTROL_BASE_SIZE}}; + const XOnlyPubKey p{Span{control}.subspan(1, TAPROOT_CONTROL_BASE_SIZE - 1)}; //! The output pubkey (taken from the scriptPubKey). const XOnlyPubKey q{program}; // Compute the Merkle root from the leaf and the provided path. @@ -1886,7 +1886,7 @@ static bool VerifyTaprootCommitment(const std::vector<unsigned char>& control, c static bool VerifyWitnessProgram(const CScriptWitness& witness, int witversion, const std::vector<unsigned char>& program, unsigned int flags, const BaseSignatureChecker& checker, ScriptError* serror, bool is_p2sh) { CScript exec_script; //!< Actually executed script (last stack item in P2WSH; implied P2PKH script in P2WPKH; leaf script in P2TR) - Span<const valtype> stack{witness.stack}; + Span stack{witness.stack}; ScriptExecutionData execdata; if (witversion == 0) { diff --git a/src/script/interpreter.h b/src/script/interpreter.h index ab49e84577..513eaaf94c 100644 --- a/src/script/interpreter.h +++ b/src/script/interpreter.h @@ -229,6 +229,7 @@ static constexpr size_t TAPROOT_CONTROL_NODE_SIZE = 32; static constexpr size_t TAPROOT_CONTROL_MAX_NODE_COUNT = 128; static constexpr size_t TAPROOT_CONTROL_MAX_SIZE = TAPROOT_CONTROL_BASE_SIZE + TAPROOT_CONTROL_NODE_SIZE * TAPROOT_CONTROL_MAX_NODE_COUNT; +extern const CHashWriter HASHER_TAPSIGHASH; //!< Hasher with tag "TapSighash" pre-fed to it. extern const CHashWriter HASHER_TAPLEAF; //!< Hasher with tag "TapLeaf" pre-fed to it. extern const CHashWriter HASHER_TAPBRANCH; //!< Hasher with tag "TapBranch" pre-fed to it. diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 4cb2125747..3f7060879c 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -17,16 +17,16 @@ typedef std::vector<unsigned char> valtype; -MutableTransactionSignatureCreator::MutableTransactionSignatureCreator(const CMutableTransaction* txToIn, unsigned int nInIn, const CAmount& amountIn, int nHashTypeIn) - : txTo(txToIn), nIn(nInIn), nHashType(nHashTypeIn), amount(amountIn), checker(txTo, nIn, amountIn, MissingDataBehavior::FAIL), +MutableTransactionSignatureCreator::MutableTransactionSignatureCreator(const CMutableTransaction* tx, unsigned int input_idx, const CAmount& amount, int hash_type) + : txTo{tx}, nIn{input_idx}, nHashType{hash_type}, amount{amount}, checker{txTo, nIn, amount, MissingDataBehavior::FAIL}, m_txdata(nullptr) { } -MutableTransactionSignatureCreator::MutableTransactionSignatureCreator(const CMutableTransaction* txToIn, unsigned int nInIn, const CAmount& amountIn, const PrecomputedTransactionData* txdata, int nHashTypeIn) - : txTo(txToIn), nIn(nInIn), nHashType(nHashTypeIn), amount(amountIn), - checker(txdata ? MutableTransactionSignatureChecker(txTo, nIn, amount, *txdata, MissingDataBehavior::FAIL) : - MutableTransactionSignatureChecker(txTo, nIn, amount, MissingDataBehavior::FAIL)), +MutableTransactionSignatureCreator::MutableTransactionSignatureCreator(const CMutableTransaction* tx, unsigned int input_idx, const CAmount& amount, const PrecomputedTransactionData* txdata, int hash_type) + : txTo{tx}, nIn{input_idx}, nHashType{hash_type}, amount{amount}, + checker{txdata ? MutableTransactionSignatureChecker{txTo, nIn, amount, *txdata, MissingDataBehavior::FAIL} : + MutableTransactionSignatureChecker{txTo, nIn, amount, MissingDataBehavior::FAIL}}, m_txdata(txdata) { } @@ -81,7 +81,8 @@ bool MutableTransactionSignatureCreator::CreateSchnorrSig(const SigningProvider& uint256 hash; if (!SignatureHashSchnorr(hash, execdata, *txTo, nIn, nHashType, sigversion, *m_txdata, MissingDataBehavior::FAIL)) return false; sig.resize(64); - if (!key.SignSchnorr(hash, sig, merkle_root, nullptr)) return false; + // Use uint256{} as aux_rnd for now. + if (!key.SignSchnorr(hash, sig, merkle_root, {})) return false; if (nHashType) sig.push_back(nHashType); return true; } @@ -166,7 +167,7 @@ static bool SignTaprootScript(const SigningProvider& provider, const BaseSignatu // <xonly pubkey> OP_CHECKSIG if (script.size() == 34 && script[33] == OP_CHECKSIG && script[0] == 0x20) { - XOnlyPubKey pubkey(MakeSpan(script).subspan(1, 32)); + XOnlyPubKey pubkey{Span{script}.subspan(1, 32)}; std::vector<unsigned char> sig; if (CreateTaprootScriptSig(creator, sigdata, provider, sig, pubkey, leaf_hash, sigversion)) { result = Vector(std::move(sig)); diff --git a/src/script/sign.h b/src/script/sign.h index 6d3479c143..50525af332 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -45,8 +45,8 @@ class MutableTransactionSignatureCreator : public BaseSignatureCreator { const PrecomputedTransactionData* m_txdata; public: - MutableTransactionSignatureCreator(const CMutableTransaction* txToIn, unsigned int nInIn, const CAmount& amountIn, int nHashTypeIn); - MutableTransactionSignatureCreator(const CMutableTransaction* txToIn, unsigned int nInIn, const CAmount& amountIn, const PrecomputedTransactionData* txdata, int nHashTypeIn); + MutableTransactionSignatureCreator(const CMutableTransaction* tx, unsigned int input_idx, const CAmount& amount, int hash_type); + MutableTransactionSignatureCreator(const CMutableTransaction* tx, unsigned int input_idx, const CAmount& amount, const PrecomputedTransactionData* txdata, int hash_type); const BaseSignatureChecker& Checker() const override { return checker; } bool CreateSig(const SigningProvider& provider, std::vector<unsigned char>& vchSig, const CKeyID& keyid, const CScript& scriptCode, SigVersion sigversion) const override; bool CreateSchnorrSig(const SigningProvider& provider, std::vector<unsigned char>& sig, const XOnlyPubKey& pubkey, const uint256* leaf_hash, const uint256* merkle_root, SigVersion sigversion) const override; @@ -149,7 +149,7 @@ void SerializeHDKeypaths(Stream& s, const std::map<CPubKey, KeyOriginInfo>& hd_k if (!keypath_pair.first.IsValid()) { throw std::ios_base::failure("Invalid CPubKey being serialized"); } - SerializeToVector(s, type, MakeSpan(keypath_pair.first)); + SerializeToVector(s, type, Span{keypath_pair.first}); WriteCompactSize(s, (keypath_pair.second.path.size() + 1) * sizeof(uint32_t)); s << keypath_pair.second.fingerprint; for (const auto& path : keypath_pair.second.path) { diff --git a/src/script/signingprovider.cpp b/src/script/signingprovider.cpp index b80fbe22ce..17f97fa30c 100644 --- a/src/script/signingprovider.cpp +++ b/src/script/signingprovider.cpp @@ -190,8 +190,8 @@ bool FillableSigningProvider::GetCScript(const CScriptID &hash, CScript& redeemS CKeyID GetKeyForDestination(const SigningProvider& store, const CTxDestination& dest) { - // Only supports destinations which map to single public keys, i.e. P2PKH, - // P2WPKH, and P2SH-P2WPKH. + // Only supports destinations which map to single public keys: + // P2PKH, P2WPKH, P2SH-P2WPKH, P2TR if (auto id = std::get_if<PKHash>(&dest)) { return ToKeyID(*id); } @@ -208,5 +208,15 @@ CKeyID GetKeyForDestination(const SigningProvider& store, const CTxDestination& } } } + if (auto output_key = std::get_if<WitnessV1Taproot>(&dest)) { + TaprootSpendData spenddata; + CPubKey pub; + if (store.GetTaprootSpendData(*output_key, spenddata) + && !spenddata.internal_key.IsNull() + && spenddata.merkle_root.IsNull() + && store.GetPubKeyByXOnly(spenddata.internal_key, pub)) { + return pub.GetID(); + } + } return CKeyID(); } diff --git a/src/shutdown.h b/src/shutdown.h index ff56c6bd87..9df601d478 100644 --- a/src/shutdown.h +++ b/src/shutdown.h @@ -32,4 +32,4 @@ bool ShutdownRequested(); */ void WaitForShutdown(); -#endif +#endif // BITCOIN_SHUTDOWN_H diff --git a/src/signet.cpp b/src/signet.cpp index aafd1999ee..6366851790 100644 --- a/src/signet.cpp +++ b/src/signet.cpp @@ -38,7 +38,7 @@ static bool FetchAndClearCommitmentSection(const Span<const uint8_t> header, CSc std::vector<uint8_t> pushdata; while (witness_commitment.GetOp(pc, opcode, pushdata)) { if (pushdata.size() > 0) { - if (!found_header && pushdata.size() > (size_t) header.size() && Span<const uint8_t>(pushdata.data(), header.size()) == header) { + if (!found_header && pushdata.size() > (size_t)header.size() && Span{pushdata}.first(header.size()) == header) { // pushdata only counts if it has the header _and_ some data result.insert(result.end(), pushdata.begin() + header.size(), pushdata.end()); pushdata.erase(pushdata.begin() + header.size(), pushdata.end()); @@ -98,7 +98,7 @@ std::optional<SignetTxs> SignetTxs::Create(const CBlock& block, const CScript& c // no signet solution -- allow this to support OP_TRUE as trivial block challenge } else { try { - VectorReader v(SER_NETWORK, INIT_PROTO_VERSION, signet_solution, 0); + SpanReader v{SER_NETWORK, INIT_PROTO_VERSION, signet_solution}; v >> tx_spending.vin[0].scriptSig; v >> tx_spending.vin[0].scriptWitness.stack; if (!v.empty()) return std::nullopt; // extraneous data encountered diff --git a/src/span.h b/src/span.h index 830164514b..6703e889c5 100644 --- a/src/span.h +++ b/src/span.h @@ -30,7 +30,11 @@ /** A Span is an object that can refer to a contiguous sequence of objects. * - * It implements a subset of C++20's std::span. + * This file implements a subset of C++20's std::span. It can be considered + * temporary compatibility code until C++20 and is designed to be a + * self-contained abstraction without depending on other project files. For this + * reason, Clang lifetimebound is defined here instead of including + * <attributes.h>, which also defines it. * * Things to be aware of when writing code that deals with Spans: * @@ -60,7 +64,7 @@ * types that expose a data() and size() member function), functions that * accept a Span as input parameter can be called with any compatible * range-like object. For example, this works: -* + * * void Foo(Span<const int> arg); * * Foo(std::vector<int>{1, 2, 3}); // Works @@ -180,6 +184,7 @@ public: return m_data[m_size - 1]; } constexpr std::size_t size() const noexcept { return m_size; } + constexpr std::size_t size_bytes() const noexcept { return sizeof(C) * m_size; } constexpr bool empty() const noexcept { return size() == 0; } CONSTEXPR_IF_NOT_DEBUG C& operator[](std::size_t pos) const noexcept { @@ -217,13 +222,15 @@ public: template <typename O> friend class Span; }; -// MakeSpan helps constructing a Span of the right type automatically. -/** MakeSpan for arrays: */ -template <typename A, int N> Span<A> constexpr MakeSpan(A (&a)[N]) { return Span<A>(a, N); } -/** MakeSpan for temporaries / rvalue references, only supporting const output. */ -template <typename V> constexpr auto MakeSpan(V&& v SPAN_ATTR_LIFETIMEBOUND) -> typename std::enable_if<!std::is_lvalue_reference<V>::value, Span<const typename std::remove_pointer<decltype(v.data())>::type>>::type { return std::forward<V>(v); } -/** MakeSpan for (lvalue) references, supporting mutable output. */ -template <typename V> constexpr auto MakeSpan(V& v SPAN_ATTR_LIFETIMEBOUND) -> Span<typename std::remove_pointer<decltype(v.data())>::type> { return v; } +// Deduction guides for Span +// For the pointer/size based and iterator based constructor: +template <typename T, typename EndOrSize> Span(T*, EndOrSize) -> Span<T>; +// For the array constructor: +template <typename T, std::size_t N> Span(T (&)[N]) -> Span<T>; +// For the temporaries/rvalue references constructor, only supporting const output. +template <typename T> Span(T&&) -> Span<std::enable_if_t<!std::is_lvalue_reference_v<T>, const std::remove_pointer_t<decltype(std::declval<T&&>().data())>>>; +// For (lvalue) references, supporting mutable output. +template <typename T> Span(T&) -> Span<std::remove_pointer_t<decltype(std::declval<T&>().data())>>; /** Pop the last element off a span, and return a reference to that element. */ template <typename T> @@ -236,16 +243,40 @@ T& SpanPopBack(Span<T>& span) return back; } +// From C++20 as_bytes and as_writeable_bytes +template <typename T> +Span<const std::byte> AsBytes(Span<T> s) noexcept +{ + return {reinterpret_cast<const std::byte*>(s.data()), s.size_bytes()}; +} +template <typename T> +Span<std::byte> AsWritableBytes(Span<T> s) noexcept +{ + return {reinterpret_cast<std::byte*>(s.data()), s.size_bytes()}; +} + +template <typename V> +Span<const std::byte> MakeByteSpan(V&& v) noexcept +{ + return AsBytes(Span{std::forward<V>(v)}); +} +template <typename V> +Span<std::byte> MakeWritableByteSpan(V&& v) noexcept +{ + return AsWritableBytes(Span{std::forward<V>(v)}); +} + // Helper functions to safely cast to unsigned char pointers. inline unsigned char* UCharCast(char* c) { return (unsigned char*)c; } inline unsigned char* UCharCast(unsigned char* c) { return c; } inline const unsigned char* UCharCast(const char* c) { return (unsigned char*)c; } inline const unsigned char* UCharCast(const unsigned char* c) { return c; } +inline const unsigned char* UCharCast(const std::byte* c) { return reinterpret_cast<const unsigned char*>(c); } // Helper function to safely convert a Span to a Span<[const] unsigned char>. template <typename T> constexpr auto UCharSpanCast(Span<T> s) -> Span<typename std::remove_pointer<decltype(UCharCast(s.data()))>::type> { return {UCharCast(s.data()), s.size()}; } -/** Like MakeSpan, but for (const) unsigned char member types only. Only works for (un)signed char containers. */ -template <typename V> constexpr auto MakeUCharSpan(V&& v) -> decltype(UCharSpanCast(MakeSpan(std::forward<V>(v)))) { return UCharSpanCast(MakeSpan(std::forward<V>(v))); } +/** Like the Span constructor, but for (const) unsigned char member types only. Only works for (un)signed char containers. */ +template <typename V> constexpr auto MakeUCharSpan(V&& v) -> decltype(UCharSpanCast(Span{std::forward<V>(v)})) { return UCharSpanCast(Span{std::forward<V>(v)}); } -#endif +#endif // BITCOIN_SPAN_H diff --git a/src/streams.h b/src/streams.h index 31407287ae..98b99b1a62 100644 --- a/src/streams.h +++ b/src/streams.h @@ -128,15 +128,14 @@ private: size_t nPos; }; -/** Minimal stream for reading from an existing vector by reference +/** Minimal stream for reading from an existing byte array by Span. */ -class VectorReader +class SpanReader { private: const int m_type; const int m_version; - const std::vector<unsigned char>& m_data; - size_t m_pos = 0; + Span<const unsigned char> m_data; public: @@ -144,30 +143,12 @@ public: * @param[in] type Serialization Type * @param[in] version Serialization Version (including any flags) * @param[in] data Referenced byte vector to overwrite/append - * @param[in] pos Starting position. Vector index where reads should start. */ - VectorReader(int type, int version, const std::vector<unsigned char>& data, size_t pos) - : m_type(type), m_version(version), m_data(data), m_pos(pos) - { - if (m_pos > m_data.size()) { - throw std::ios_base::failure("VectorReader(...): end of data (m_pos > m_data.size())"); - } - } - - /** - * (other params same as above) - * @param[in] args A list of items to deserialize starting at pos. - */ - template <typename... Args> - VectorReader(int type, int version, const std::vector<unsigned char>& data, size_t pos, - Args&&... args) - : VectorReader(type, version, data, pos) - { - ::UnserializeMany(*this, std::forward<Args>(args)...); - } + SpanReader(int type, int version, Span<const unsigned char> data) + : m_type(type), m_version(version), m_data(data) {} template<typename T> - VectorReader& operator>>(T&& obj) + SpanReader& operator>>(T&& obj) { // Unserialize from this stream ::Unserialize(*this, obj); @@ -177,8 +158,8 @@ public: int GetVersion() const { return m_version; } int GetType() const { return m_type; } - size_t size() const { return m_data.size() - m_pos; } - bool empty() const { return m_data.size() == m_pos; } + size_t size() const { return m_data.size(); } + bool empty() const { return m_data.empty(); } void read(char* dst, size_t n) { @@ -187,12 +168,11 @@ public: } // Read from the beginning of the buffer - size_t pos_next = m_pos + n; - if (pos_next > m_data.size()) { - throw std::ios_base::failure("VectorReader::read(): end of data"); + if (n > m_data.size()) { + throw std::ios_base::failure("SpanReader::read(): end of data"); } - memcpy(dst, m_data.data() + m_pos, n); - m_pos = pos_next; + memcpy(dst, m_data.data(), n); + m_data = m_data.subspan(n); } }; @@ -226,7 +206,7 @@ public: : nType{nTypeIn}, nVersion{nVersionIn} {} - explicit CDataStream(Span<const uint8_t> sp, int nTypeIn, int nVersionIn) + explicit CDataStream(Span<const value_type> sp, int nTypeIn, int nVersionIn) : vch(sp.data(), sp.data() + sp.size()), nType{nTypeIn}, nVersion{nVersionIn} {} @@ -254,17 +234,17 @@ public: iterator end() { return vch.end(); } size_type size() const { return vch.size() - nReadPos; } bool empty() const { return vch.size() == nReadPos; } - void resize(size_type n, value_type c=0) { vch.resize(n + nReadPos, c); } + void resize(size_type n, value_type c = value_type{}) { vch.resize(n + nReadPos, c); } void reserve(size_type n) { vch.reserve(n + nReadPos); } const_reference operator[](size_type pos) const { return vch[pos + nReadPos]; } reference operator[](size_type pos) { return vch[pos + nReadPos]; } void clear() { vch.clear(); nReadPos = 0; } - iterator insert(iterator it, const uint8_t x) { return vch.insert(it, x); } - void insert(iterator it, size_type n, const uint8_t x) { vch.insert(it, n, x); } + iterator insert(iterator it, const value_type x) { return vch.insert(it, x); } + void insert(iterator it, size_type n, const value_type x) { vch.insert(it, n, x); } value_type* data() { return vch.data() + nReadPos; } const value_type* data() const { return vch.data() + nReadPos; } - void insert(iterator it, std::vector<uint8_t>::const_iterator first, std::vector<uint8_t>::const_iterator last) + void insert(iterator it, std::vector<value_type>::const_iterator first, std::vector<value_type>::const_iterator last) { if (last == first) return; assert(last - first > 0); @@ -278,7 +258,7 @@ public: vch.insert(it, first, last); } - void insert(iterator it, const char* first, const char* last) + void insert(iterator it, const value_type* first, const value_type* last) { if (last == first) return; assert(last - first > 0); diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index eabc11c467..31f30d0379 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -22,80 +22,20 @@ using namespace std::literals; -class AddrManSerializationMock : public AddrMan -{ -public: - virtual void Serialize(CDataStream& s) const = 0; - - AddrManSerializationMock() - : AddrMan(/* asmap */ std::vector<bool>(), /* deterministic */ true, /* consistency_check_ratio */ 100) - {} -}; - -class AddrManUncorrupted : public AddrManSerializationMock -{ -public: - void Serialize(CDataStream& s) const override - { - AddrMan::Serialize(s); - } -}; - -class AddrManCorrupted : public AddrManSerializationMock -{ -public: - void Serialize(CDataStream& s) const override - { - // Produces corrupt output that claims addrman has 20 addrs when it only has one addr. - unsigned char nVersion = 1; - s << nVersion; - s << ((unsigned char)32); - s << uint256::ONE; - s << 10; // nNew - s << 10; // nTried - - int nUBuckets = ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30); - s << nUBuckets; - - CService serv; - BOOST_CHECK(Lookup("252.1.1.1", serv, 7777, false)); - CAddress addr = CAddress(serv, NODE_NONE); - CNetAddr resolved; - BOOST_CHECK(LookupHost("252.2.2.2", resolved, false)); - AddrInfo info = AddrInfo(addr, resolved); - s << info; - } -}; - -static CDataStream AddrmanToStream(const AddrManSerializationMock& _addrman) -{ - CDataStream ssPeersIn(SER_DISK, CLIENT_VERSION); - ssPeersIn << Params().MessageStart(); - ssPeersIn << _addrman; - std::string str = ssPeersIn.str(); - std::vector<unsigned char> vchData(str.begin(), str.end()); - return CDataStream(vchData, SER_DISK, CLIENT_VERSION); -} - class AddrManTest : public AddrMan { -private: - bool deterministic; public: - explicit AddrManTest(bool makeDeterministic = true, - std::vector<bool> asmap = std::vector<bool>()) - : AddrMan(asmap, makeDeterministic, /* consistency_check_ratio */ 100) - { - deterministic = makeDeterministic; - } + explicit AddrManTest(std::vector<bool> asmap = std::vector<bool>()) + : AddrMan(asmap, /*deterministic=*/true, /*consistency_check_ratio=*/100) + {} - AddrInfo* Find(const CService& addr, int* pnId = nullptr) + AddrInfo* Find(const CService& addr) { LOCK(m_impl->cs); - return m_impl->Find(addr, pnId); + return m_impl->Find(addr); } - AddrInfo* Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId = nullptr) + AddrInfo* Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId) { LOCK(m_impl->cs); return m_impl->Create(addr, addrSource, pnId); @@ -125,14 +65,14 @@ public: // Simulates connection failure so that we can test eviction of offline nodes void SimConnFail(const CService& addr) { - int64_t nLastSuccess = 1; - // Set last good connection in the deep past. - Good(addr, nLastSuccess); - - bool count_failure = false; - int64_t nLastTry = GetAdjustedTime()-61; - Attempt(addr, count_failure, nLastTry); - } + int64_t nLastSuccess = 1; + // Set last good connection in the deep past. + Good(addr, nLastSuccess); + + bool count_failure = false; + int64_t nLastTry = GetAdjustedTime() - 61; + Attempt(addr, count_failure, nLastTry); + } }; static CNetAddr ResolveIP(const std::string& ip) @@ -150,7 +90,8 @@ static CService ResolveService(const std::string& ip, uint16_t port = 0) } -static std::vector<bool> FromBytes(const unsigned char* source, int vector_size) { +static std::vector<bool> FromBytes(const unsigned char* source, int vector_size) +{ std::vector<bool> result(vector_size); for (int byte_i = 0; byte_i < vector_size / 8; ++byte_i) { unsigned char cur_byte = source[byte_i]; @@ -306,15 +247,15 @@ BOOST_AUTO_TEST_CASE(addrman_new_collisions) BOOST_CHECK_EQUAL(addrman.size(), num_addrs); - while (num_addrs < 22) { // Magic number! 250.1.1.1 - 250.1.1.22 do not collide with deterministic key = 1 + while (num_addrs < 22) { // Magic number! 250.1.1.1 - 250.1.1.22 do not collide with deterministic key = 1 CService addr = ResolveService("250.1.1." + ToString(++num_addrs)); BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); - //Test: No collision in new table yet. + // Test: No collision in new table yet. BOOST_CHECK_EQUAL(addrman.size(), num_addrs); } - //Test: new table collision! + // Test: new table collision! CService addr1 = ResolveService("250.1.1." + ToString(++num_addrs)); uint32_t collisions{1}; BOOST_CHECK(addrman.Add({CAddress(addr1, NODE_NONE)}, source)); @@ -335,16 +276,16 @@ BOOST_AUTO_TEST_CASE(addrman_tried_collisions) BOOST_CHECK_EQUAL(addrman.size(), num_addrs); - while (num_addrs < 64) { // Magic number! 250.1.1.1 - 250.1.1.64 do not collide with deterministic key = 1 + while (num_addrs < 64) { // Magic number! 250.1.1.1 - 250.1.1.64 do not collide with deterministic key = 1 CService addr = ResolveService("250.1.1." + ToString(++num_addrs)); BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); addrman.Good(CAddress(addr, NODE_NONE)); - //Test: No collision in tried table yet. + // Test: No collision in tried table yet. BOOST_CHECK_EQUAL(addrman.size(), num_addrs); } - //Test: tried table collision! + // Test: tried table collision! CService addr1 = ResolveService("250.1.1." + ToString(++num_addrs)); uint32_t collisions{1}; BOOST_CHECK(!addrman.Add({CAddress(addr1, NODE_NONE)}, source)); @@ -435,7 +376,7 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) // Test: Sanity check, GetAddr should never return anything if addrman // is empty. BOOST_CHECK_EQUAL(addrman.size(), 0U); - std::vector<CAddress> vAddr1 = addrman.GetAddr(/* max_addresses */ 0, /* max_pct */ 0, /* network */ std::nullopt); + std::vector<CAddress> vAddr1 = addrman.GetAddr(/*max_addresses=*/0, /*max_pct=*/0, /*network=*/std::nullopt); BOOST_CHECK_EQUAL(vAddr1.size(), 0U); CAddress addr1 = CAddress(ResolveService("250.250.2.1", 8333), NODE_NONE); @@ -455,15 +396,15 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) BOOST_CHECK(addrman.Add({addr1, addr3, addr5}, source1)); BOOST_CHECK(addrman.Add({addr2, addr4}, source2)); - BOOST_CHECK_EQUAL(addrman.GetAddr(/* max_addresses */ 0, /* max_pct */ 0, /* network */ std::nullopt).size(), 5U); + BOOST_CHECK_EQUAL(addrman.GetAddr(/*max_addresses=*/0, /*max_pct=*/0, /*network=*/std::nullopt).size(), 5U); // Net processing asks for 23% of addresses. 23% of 5 is 1 rounded down. - BOOST_CHECK_EQUAL(addrman.GetAddr(/* max_addresses */ 2500, /* max_pct */ 23, /* network */ std::nullopt).size(), 1U); + BOOST_CHECK_EQUAL(addrman.GetAddr(/*max_addresses=*/2500, /*max_pct=*/23, /*network=*/std::nullopt).size(), 1U); // Test: Ensure GetAddr works with new and tried addresses. addrman.Good(CAddress(addr1, NODE_NONE)); addrman.Good(CAddress(addr2, NODE_NONE)); - BOOST_CHECK_EQUAL(addrman.GetAddr(/* max_addresses */ 0, /* max_pct */ 0, /* network */ std::nullopt).size(), 5U); - BOOST_CHECK_EQUAL(addrman.GetAddr(/* max_addresses */ 2500, /* max_pct */ 23, /* network */ std::nullopt).size(), 1U); + BOOST_CHECK_EQUAL(addrman.GetAddr(/*max_addresses=*/0, /*max_pct=*/0, /*network=*/std::nullopt).size(), 5U); + BOOST_CHECK_EQUAL(addrman.GetAddr(/*max_addresses=*/2500, /*max_pct=*/23, /*network=*/std::nullopt).size(), 1U); // Test: Ensure GetAddr still returns 23% when addrman has many addrs. for (unsigned int i = 1; i < (8 * 256); i++) { @@ -478,7 +419,7 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) if (i % 8 == 0) addrman.Good(addr); } - std::vector<CAddress> vAddr = addrman.GetAddr(/* max_addresses */ 2500, /* max_pct */ 23, /* network */ std::nullopt); + std::vector<CAddress> vAddr = addrman.GetAddr(/*max_addresses=*/2500, /*max_pct=*/23, /*network=*/std::nullopt); size_t percent23 = (addrman.size() * 23) / 100; BOOST_CHECK_EQUAL(vAddr.size(), percent23); @@ -753,15 +694,14 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) // Test: IP addresses in the different source /16 prefixes sometimes map to NO MORE // than 1 bucket. BOOST_CHECK(buckets.size() == 1); - } BOOST_AUTO_TEST_CASE(addrman_serialization) { std::vector<bool> asmap1 = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); - auto addrman_asmap1 = std::make_unique<AddrManTest>(true, asmap1); - auto addrman_asmap1_dup = std::make_unique<AddrManTest>(true, asmap1); + auto addrman_asmap1 = std::make_unique<AddrManTest>(asmap1); + auto addrman_asmap1_dup = std::make_unique<AddrManTest>(asmap1); auto addrman_noasmap = std::make_unique<AddrManTest>(); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); @@ -792,7 +732,7 @@ BOOST_AUTO_TEST_CASE(addrman_serialization) BOOST_CHECK(bucketAndEntry_asmap1.second != bucketAndEntry_noasmap.second); // deserializing non-asmaped peers.dat to asmaped addrman - addrman_asmap1 = std::make_unique<AddrManTest>(true, asmap1); + addrman_asmap1 = std::make_unique<AddrManTest>(asmap1); addrman_noasmap = std::make_unique<AddrManTest>(); addrman_noasmap->Add({addr}, default_source); stream << *addrman_noasmap; @@ -804,7 +744,7 @@ BOOST_AUTO_TEST_CASE(addrman_serialization) BOOST_CHECK(bucketAndEntry_asmap1_deser.second == bucketAndEntry_asmap1_dup.second); // used to map to different buckets, now maps to the same bucket. - addrman_asmap1 = std::make_unique<AddrManTest>(true, asmap1); + addrman_asmap1 = std::make_unique<AddrManTest>(asmap1); addrman_noasmap = std::make_unique<AddrManTest>(); CAddress addr1 = CAddress(ResolveService("250.1.1.1"), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.2.1.1"), NODE_NONE); @@ -874,7 +814,7 @@ BOOST_AUTO_TEST_CASE(addrman_selecttriedcollision) // Add twenty two addresses. CNetAddr source = ResolveIP("252.2.2.2"); for (unsigned int i = 1; i < 23; i++) { - CService addr = ResolveService("250.1.1."+ToString(i)); + CService addr = ResolveService("250.1.1." + ToString(i)); BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); addrman.Good(addr); @@ -885,13 +825,12 @@ BOOST_AUTO_TEST_CASE(addrman_selecttriedcollision) // Ensure Good handles duplicates well. for (unsigned int i = 1; i < 23; i++) { - CService addr = ResolveService("250.1.1."+ToString(i)); + CService addr = ResolveService("250.1.1." + ToString(i)); addrman.Good(addr); BOOST_CHECK(addrman.size() == 22); BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } - } BOOST_AUTO_TEST_CASE(addrman_noevict) @@ -901,7 +840,7 @@ BOOST_AUTO_TEST_CASE(addrman_noevict) // Add 35 addresses. CNetAddr source = ResolveIP("252.2.2.2"); for (unsigned int i = 1; i < 36; i++) { - CService addr = ResolveService("250.1.1."+ToString(i)); + CService addr = ResolveService("250.1.1." + ToString(i)); BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); addrman.Good(addr); @@ -924,7 +863,7 @@ BOOST_AUTO_TEST_CASE(addrman_noevict) // Lets create two collisions. for (unsigned int i = 37; i < 59; i++) { - CService addr = ResolveService("250.1.1."+ToString(i)); + CService addr = ResolveService("250.1.1." + ToString(i)); BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); addrman.Good(addr); @@ -962,7 +901,7 @@ BOOST_AUTO_TEST_CASE(addrman_evictionworks) // Add 35 addresses CNetAddr source = ResolveIP("252.2.2.2"); for (unsigned int i = 1; i < 36; i++) { - CService addr = ResolveService("250.1.1."+ToString(i)); + CService addr = ResolveService("250.1.1." + ToString(i)); BOOST_CHECK(addrman.Add({CAddress(addr, NODE_NONE)}, source)); addrman.Good(addr); @@ -1004,9 +943,18 @@ BOOST_AUTO_TEST_CASE(addrman_evictionworks) BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } +static CDataStream AddrmanToStream(const AddrMan& addrman) +{ + CDataStream ssPeersIn(SER_DISK, CLIENT_VERSION); + ssPeersIn << Params().MessageStart(); + ssPeersIn << addrman; + return ssPeersIn; +} + BOOST_AUTO_TEST_CASE(load_addrman) { - AddrManUncorrupted addrmanUncorrupted; + AddrMan addrman{/*asmap=*/ std::vector<bool>(), /*deterministic=*/ true, + /*consistency_check_ratio=*/ 100}; CService addr1, addr2, addr3; BOOST_CHECK(Lookup("250.7.1.1", addr1, 8333, false)); @@ -1019,13 +967,13 @@ BOOST_AUTO_TEST_CASE(load_addrman) CService source; BOOST_CHECK(Lookup("252.5.1.1", source, 8333, false)); std::vector<CAddress> addresses{CAddress(addr1, NODE_NONE), CAddress(addr2, NODE_NONE), CAddress(addr3, NODE_NONE)}; - BOOST_CHECK(addrmanUncorrupted.Add(addresses, source)); - BOOST_CHECK(addrmanUncorrupted.size() == 3); + BOOST_CHECK(addrman.Add(addresses, source)); + BOOST_CHECK(addrman.size() == 3); // Test that the de-serialization does not throw an exception. - CDataStream ssPeers1 = AddrmanToStream(addrmanUncorrupted); + CDataStream ssPeers1 = AddrmanToStream(addrman); bool exceptionThrown = false; - AddrMan addrman1(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); + AddrMan addrman1(/*asmap=*/std::vector<bool>(), /*deterministic=*/false, /*consistency_check_ratio=*/100); BOOST_CHECK(addrman1.size() == 0); try { @@ -1040,23 +988,47 @@ BOOST_AUTO_TEST_CASE(load_addrman) BOOST_CHECK(exceptionThrown == false); // Test that ReadFromStream creates an addrman with the correct number of addrs. - CDataStream ssPeers2 = AddrmanToStream(addrmanUncorrupted); + CDataStream ssPeers2 = AddrmanToStream(addrman); - AddrMan addrman2(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); + AddrMan addrman2(/*asmap=*/std::vector<bool>(), /*deterministic=*/false, /*consistency_check_ratio=*/100); BOOST_CHECK(addrman2.size() == 0); ReadFromStream(addrman2, ssPeers2); BOOST_CHECK(addrman2.size() == 3); } +// Produce a corrupt peers.dat that claims 20 addrs when it only has one addr. +static CDataStream MakeCorruptPeersDat() +{ + CDataStream s(SER_DISK, CLIENT_VERSION); + s << ::Params().MessageStart(); + + unsigned char nVersion = 1; + s << nVersion; + s << ((unsigned char)32); + s << uint256::ONE; + s << 10; // nNew + s << 10; // nTried + + int nUBuckets = ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30); + s << nUBuckets; + + CService serv; + BOOST_CHECK(Lookup("252.1.1.1", serv, 7777, false)); + CAddress addr = CAddress(serv, NODE_NONE); + CNetAddr resolved; + BOOST_CHECK(LookupHost("252.2.2.2", resolved, false)); + AddrInfo info = AddrInfo(addr, resolved); + s << info; + + return s; +} BOOST_AUTO_TEST_CASE(load_addrman_corrupted) { - AddrManCorrupted addrmanCorrupted; - - // Test that the de-serialization of corrupted addrman throws an exception. - CDataStream ssPeers1 = AddrmanToStream(addrmanCorrupted); + // Test that the de-serialization of corrupted peers.dat throws an exception. + CDataStream ssPeers1 = MakeCorruptPeersDat(); bool exceptionThrown = false; - AddrMan addrman1(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); + AddrMan addrman1(/*asmap=*/std::vector<bool>(), /*deterministic=*/false, /*consistency_check_ratio=*/100); BOOST_CHECK(addrman1.size() == 0); try { unsigned char pchMsgTmp[4]; @@ -1070,9 +1042,9 @@ BOOST_AUTO_TEST_CASE(load_addrman_corrupted) BOOST_CHECK(exceptionThrown); // Test that ReadFromStream fails if peers.dat is corrupt - CDataStream ssPeers2 = AddrmanToStream(addrmanCorrupted); + CDataStream ssPeers2 = MakeCorruptPeersDat(); - AddrMan addrman2(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); + AddrMan addrman2(/*asmap=*/std::vector<bool>(), /*deterministic=*/false, /*consistency_check_ratio=*/100); BOOST_CHECK(addrman2.size() == 0); BOOST_CHECK_THROW(ReadFromStream(addrman2, ssPeers2), std::ios_base::failure); } diff --git a/src/test/amount_tests.cpp b/src/test/amount_tests.cpp index 114fe3907c..aa23d71671 100644 --- a/src/test/amount_tests.cpp +++ b/src/test/amount_tests.cpp @@ -48,13 +48,13 @@ BOOST_AUTO_TEST_CASE(GetFeeTest) BOOST_CHECK_EQUAL(feeRate.GetFee(9e3), CAmount(-9e3)); feeRate = CFeeRate(123); - // Truncates the result, if not integer + // Rounds up the result, if not integer BOOST_CHECK_EQUAL(feeRate.GetFee(0), CAmount(0)); BOOST_CHECK_EQUAL(feeRate.GetFee(8), CAmount(1)); // Special case: returns 1 instead of 0 - BOOST_CHECK_EQUAL(feeRate.GetFee(9), CAmount(1)); - BOOST_CHECK_EQUAL(feeRate.GetFee(121), CAmount(14)); - BOOST_CHECK_EQUAL(feeRate.GetFee(122), CAmount(15)); - BOOST_CHECK_EQUAL(feeRate.GetFee(999), CAmount(122)); + BOOST_CHECK_EQUAL(feeRate.GetFee(9), CAmount(2)); + BOOST_CHECK_EQUAL(feeRate.GetFee(121), CAmount(15)); + BOOST_CHECK_EQUAL(feeRate.GetFee(122), CAmount(16)); + BOOST_CHECK_EQUAL(feeRate.GetFee(999), CAmount(123)); BOOST_CHECK_EQUAL(feeRate.GetFee(1e3), CAmount(123)); BOOST_CHECK_EQUAL(feeRate.GetFee(9e3), CAmount(1107)); diff --git a/src/test/base64_tests.cpp b/src/test/base64_tests.cpp index 9d1dfd46f1..c5fce7bec0 100644 --- a/src/test/base64_tests.cpp +++ b/src/test/base64_tests.cpp @@ -23,6 +23,16 @@ BOOST_AUTO_TEST_CASE(base64_testvectors) BOOST_CHECK_EQUAL(strDec, vstrIn[i]); } + { + const std::vector<uint8_t> in_u{0xff, 0x01, 0xff}; + const std::vector<std::byte> in_b{std::byte{0xff}, std::byte{0x01}, std::byte{0xff}}; + const std::string in_s{"\xff\x01\xff"}; + const std::string out_exp{"/wH/"}; + BOOST_CHECK_EQUAL(EncodeBase64(in_u), out_exp); + BOOST_CHECK_EQUAL(EncodeBase64(in_b), out_exp); + BOOST_CHECK_EQUAL(EncodeBase64(in_s), out_exp); + } + // Decoding strings with embedded NUL characters should fail bool failure; (void)DecodeBase64("invalid\0"s, &failure); diff --git a/src/test/bech32_tests.cpp b/src/test/bech32_tests.cpp index c0344b3cbb..51a1d1199e 100644 --- a/src/test/bech32_tests.cpp +++ b/src/test/bech32_tests.cpp @@ -1,4 +1,5 @@ // Copyright (c) 2017 Pieter Wuille +// Copyright (c) 2021 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -68,10 +69,38 @@ BOOST_AUTO_TEST_CASE(bech32_testvectors_invalid) "1qzzfhee", "a12UEL5L", "A12uEL5L", + "abcdef1qpzrz9x8gf2tvdw0s3jn54khce6mua7lmqqqxw", + "test1zg69w7y6hn0aqy352euf40x77qddq3dc", }; + static const std::pair<std::string, std::vector<int>> ERRORS[] = { + {"Invalid character or mixed case", {0}}, + {"Invalid character or mixed case", {0}}, + {"Invalid character or mixed case", {0}}, + {"Bech32 string too long", {90}}, + {"Missing separator", {}}, + {"Invalid separator position", {0}}, + {"Invalid Base 32 character", {2}}, + {"Invalid separator position", {2}}, + {"Invalid character or mixed case", {8}}, + {"Invalid checksum", {}}, // The checksum is calculated using the uppercase form so the entire string is invalid, not just a few characters + {"Invalid separator position", {0}}, + {"Invalid separator position", {0}}, + {"Invalid character or mixed case", {3, 4, 5, 7}}, + {"Invalid character or mixed case", {3}}, + {"Invalid Bech32 checksum", {11}}, + {"Invalid Bech32 checksum", {9, 16}}, + }; + static_assert(std::size(CASES) == std::size(ERRORS), "Bech32 CASES and ERRORS should have the same length"); + + int i = 0; for (const std::string& str : CASES) { + const auto& err = ERRORS[i]; const auto dec = bech32::Decode(str); BOOST_CHECK(dec.encoding == bech32::Encoding::INVALID); + auto [error, error_locations] = bech32::LocateErrors(str); + BOOST_CHECK_EQUAL(err.first, error); + BOOST_CHECK(err.second == error_locations); + i++; } } @@ -91,11 +120,39 @@ BOOST_AUTO_TEST_CASE(bech32m_testvectors_invalid) "au1s5cgom", "M1VUXWEZ", "16plkw9", - "1p2gdwpf" + "1p2gdwpf", + "abcdef1l7aum6echk45nj2s0wdvt2fg8x9yrzpqzd3ryx", + "test1zg69v7y60n00qy352euf40x77qcusag6", }; + static const std::pair<std::string, std::vector<int>> ERRORS[] = { + {"Invalid character or mixed case", {0}}, + {"Invalid character or mixed case", {0}}, + {"Invalid character or mixed case", {0}}, + {"Bech32 string too long", {90}}, + {"Missing separator", {}}, + {"Invalid separator position", {0}}, + {"Invalid Base 32 character", {2}}, + {"Invalid Base 32 character", {3}}, + {"Invalid separator position", {2}}, + {"Invalid Base 32 character", {8}}, + {"Invalid Base 32 character", {7}}, + {"Invalid checksum", {}}, + {"Invalid separator position", {0}}, + {"Invalid separator position", {0}}, + {"Invalid Bech32m checksum", {21}}, + {"Invalid Bech32m checksum", {13, 32}}, + }; + static_assert(std::size(CASES) == std::size(ERRORS), "Bech32m CASES and ERRORS should have the same length"); + + int i = 0; for (const std::string& str : CASES) { + const auto& err = ERRORS[i]; const auto dec = bech32::Decode(str); BOOST_CHECK(dec.encoding == bech32::Encoding::INVALID); + auto [error, error_locations] = bech32::LocateErrors(str); + BOOST_CHECK_EQUAL(err.first, error); + BOOST_CHECK(err.second == error_locations); + i++; } } diff --git a/src/test/bip32_tests.cpp b/src/test/bip32_tests.cpp index a89868e1ef..0fa6b7784f 100644 --- a/src/test/bip32_tests.cpp +++ b/src/test/bip32_tests.cpp @@ -124,7 +124,7 @@ void RunTest(const TestVector &test) { std::vector<unsigned char> seed = ParseHex(test.strHexMaster); CExtKey key; CExtPubKey pubkey; - key.SetSeed(seed.data(), seed.size()); + key.SetSeed(seed); pubkey = key.Neuter(); for (const TestDerivation &derive : test.vDerive) { unsigned char data[74]; diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index 2eb653e9ec..7d50def509 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -6,7 +6,7 @@ #include <chainparams.h> #include <consensus/validation.h> #include <index/blockfilterindex.h> -#include <miner.h> +#include <node/miner.h> #include <pow.h> #include <script/standard.h> #include <test/util/blockfilter.h> diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index 64c6d7f634..4d34407ca8 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -18,7 +18,17 @@ #include <utility> #include <vector> -BOOST_FIXTURE_TEST_SUITE(checkqueue_tests, TestingSetup) +/** + * Identical to TestingSetup but excludes lock contention logging, as some of + * these tests are designed to be heavily contested to trigger race conditions + * or other issues. + */ +struct NoLockLoggingTestingSetup : public TestingSetup { + NoLockLoggingTestingSetup() + : TestingSetup{CBaseChainParams::MAIN, /*extra_args=*/{"-debugexclude=lock"}} {} +}; + +BOOST_FIXTURE_TEST_SUITE(checkqueue_tests, NoLockLoggingTestingSetup) static const unsigned int QUEUE_BATCH_SIZE = 128; static const int SCRIPT_CHECK_THREADS = 3; diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 06db3b846e..91218511bd 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -269,7 +269,7 @@ BOOST_AUTO_TEST_CASE(coins_cache_simulation_test) CCoinsViewTest base; SimulationTest(&base, false); - CCoinsViewDB db_base{"test", /*nCacheSize*/ 1 << 23, /*fMemory*/ true, /*fWipe*/ false}; + CCoinsViewDB db_base{"test", /*nCacheSize=*/1 << 23, /*fMemory=*/true, /*fWipe=*/false}; SimulationTest(&db_base, true); } diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 1483bd3cb3..bedef5de37 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -574,10 +574,10 @@ BOOST_AUTO_TEST_CASE(hkdf_hmac_sha256_l32_tests) { // Use rfc5869 test vectors but truncated to 32 bytes (our implementation only support length 32) TestHKDF_SHA256_32( - /* IKM */ "0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b", - /* salt */ "000102030405060708090a0b0c", - /* info */ "f0f1f2f3f4f5f6f7f8f9", - /* expected OKM */ "3cb25f25faacd57a90434f64d0362f2a2d2d0a90cf1a5a4c5db02d56ecc4c5bf"); + /*ikm_hex=*/"0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b", + /*salt_hex=*/"000102030405060708090a0b0c", + /*info_hex=*/"f0f1f2f3f4f5f6f7f8f9", + /*okm_check_hex=*/"3cb25f25faacd57a90434f64d0362f2a2d2d0a90cf1a5a4c5db02d56ecc4c5bf"); TestHKDF_SHA256_32( "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f", "606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeaf", @@ -770,8 +770,8 @@ static void TestSHA3_256(const std::string& input, const std::string& output) int s1 = InsecureRandRange(in_bytes.size() + 1); int s2 = InsecureRandRange(in_bytes.size() + 1 - s1); int s3 = in_bytes.size() - s1 - s2; - sha.Write(MakeSpan(in_bytes).first(s1)).Write(MakeSpan(in_bytes).subspan(s1, s2)); - sha.Write(MakeSpan(in_bytes).last(s3)).Finalize(out); + sha.Write(Span{in_bytes}.first(s1)).Write(Span{in_bytes}.subspan(s1, s2)); + sha.Write(Span{in_bytes}.last(s3)).Finalize(out); BOOST_CHECK(std::equal(std::begin(out_bytes), std::end(out_bytes), out)); } diff --git a/src/test/data/bip341_wallet_vectors.json b/src/test/data/bip341_wallet_vectors.json new file mode 100644 index 0000000000..11261b00ba --- /dev/null +++ b/src/test/data/bip341_wallet_vectors.json @@ -0,0 +1,452 @@ +{ + "version": 1, + "scriptPubKey": [ + { + "given": { + "internalPubkey": "d6889cb081036e0faefa3a35157ad71086b123b2b144b649798b494c300a961d", + "scriptTree": null + }, + "intermediary": { + "merkleRoot": null, + "tweak": "b86e7be8f39bab32a6f2c0443abbc210f0edac0e2c53d501b36b64437d9c6c70", + "tweakedPubkey": "53a1f6e454df1aa2776a2814a721372d6258050de330b3c6d10ee8f4e0dda343" + }, + "expected": { + "scriptPubKey": "512053a1f6e454df1aa2776a2814a721372d6258050de330b3c6d10ee8f4e0dda343", + "bip350Address": "bc1p2wsldez5mud2yam29q22wgfh9439spgduvct83k3pm50fcxa5dps59h4z5" + } + }, + { + "given": { + "internalPubkey": "187791b6f712a8ea41c8ecdd0ee77fab3e85263b37e1ec18a3651926b3a6cf27", + "scriptTree": { + "id": 0, + "script": "20d85a959b0290bf19bb89ed43c916be835475d013da4b362117393e25a48229b8ac", + "leafVersion": 192 + } + }, + "intermediary": { + "leafHashes": [ + "5b75adecf53548f3ec6ad7d78383bf84cc57b55a3127c72b9a2481752dd88b21" + ], + "merkleRoot": "5b75adecf53548f3ec6ad7d78383bf84cc57b55a3127c72b9a2481752dd88b21", + "tweak": "cbd8679ba636c1110ea247542cfbd964131a6be84f873f7f3b62a777528ed001", + "tweakedPubkey": "147c9c57132f6e7ecddba9800bb0c4449251c92a1e60371ee77557b6620f3ea3" + }, + "expected": { + "scriptPubKey": "5120147c9c57132f6e7ecddba9800bb0c4449251c92a1e60371ee77557b6620f3ea3", + "bip350Address": "bc1pz37fc4cn9ah8anwm4xqqhvxygjf9rjf2resrw8h8w4tmvcs0863sa2e586", + "scriptPathControlBlocks": [ + "c1187791b6f712a8ea41c8ecdd0ee77fab3e85263b37e1ec18a3651926b3a6cf27" + ] + } + }, + { + "given": { + "internalPubkey": "93478e9488f956df2396be2ce6c5cced75f900dfa18e7dabd2428aae78451820", + "scriptTree": { + "id": 0, + "script": "20b617298552a72ade070667e86ca63b8f5789a9fe8731ef91202a91c9f3459007ac", + "leafVersion": 192 + } + }, + "intermediary": { + "leafHashes": [ + "c525714a7f49c28aedbbba78c005931a81c234b2f6c99a73e4d06082adc8bf2b" + ], + "merkleRoot": "c525714a7f49c28aedbbba78c005931a81c234b2f6c99a73e4d06082adc8bf2b", + "tweak": "6af9e28dbf9d6aaf027696e2598a5b3d056f5fd2355a7fd5a37a0e5008132d30", + "tweakedPubkey": "e4d810fd50586274face62b8a807eb9719cef49c04177cc6b76a9a4251d5450e" + }, + "expected": { + "scriptPubKey": "5120e4d810fd50586274face62b8a807eb9719cef49c04177cc6b76a9a4251d5450e", + "bip350Address": "bc1punvppl2stp38f7kwv2u2spltjuvuaayuqsthe34hd2dyy5w4g58qqfuag5", + "scriptPathControlBlocks": [ + "c093478e9488f956df2396be2ce6c5cced75f900dfa18e7dabd2428aae78451820" + ] + } + }, + { + "given": { + "internalPubkey": "ee4fe085983462a184015d1f782d6a5f8b9c2b60130aff050ce221ecf3786592", + "scriptTree": [ + { + "id": 0, + "script": "20387671353e273264c495656e27e39ba899ea8fee3bb69fb2a680e22093447d48ac", + "leafVersion": 192 + }, + { + "id": 1, + "script": "06424950333431", + "leafVersion": 250 + } + ] + }, + "intermediary": { + "leafHashes": [ + "8ad69ec7cf41c2a4001fd1f738bf1e505ce2277acdcaa63fe4765192497f47a7", + "f224a923cd0021ab202ab139cc56802ddb92dcfc172b9212261a539df79a112a" + ], + "merkleRoot": "6c2dc106ab816b73f9d07e3cd1ef2c8c1256f519748e0813e4edd2405d277bef", + "tweak": "9e0517edc8259bb3359255400b23ca9507f2a91cd1e4250ba068b4eafceba4a9", + "tweakedPubkey": "712447206d7a5238acc7ff53fbe94a3b64539ad291c7cdbc490b7577e4b17df5" + }, + "expected": { + "scriptPubKey": "5120712447206d7a5238acc7ff53fbe94a3b64539ad291c7cdbc490b7577e4b17df5", + "bip350Address": "bc1pwyjywgrd0ffr3tx8laflh6228dj98xkjj8rum0zfpd6h0e930h6saqxrrm", + "scriptPathControlBlocks": [ + "c0ee4fe085983462a184015d1f782d6a5f8b9c2b60130aff050ce221ecf3786592f224a923cd0021ab202ab139cc56802ddb92dcfc172b9212261a539df79a112a", + "faee4fe085983462a184015d1f782d6a5f8b9c2b60130aff050ce221ecf37865928ad69ec7cf41c2a4001fd1f738bf1e505ce2277acdcaa63fe4765192497f47a7" + ] + } + }, + { + "given": { + "internalPubkey": "f9f400803e683727b14f463836e1e78e1c64417638aa066919291a225f0e8dd8", + "scriptTree": [ + { + "id": 0, + "script": "2044b178d64c32c4a05cc4f4d1407268f764c940d20ce97abfd44db5c3592b72fdac", + "leafVersion": 192 + }, + { + "id": 1, + "script": "07546170726f6f74", + "leafVersion": 192 + } + ] + }, + "intermediary": { + "leafHashes": [ + "64512fecdb5afa04f98839b50e6f0cb7b1e539bf6f205f67934083cdcc3c8d89", + "2cb2b90daa543b544161530c925f285b06196940d6085ca9474d41dc3822c5cb" + ], + "merkleRoot": "ab179431c28d3b68fb798957faf5497d69c883c6fb1e1cd9f81483d87bac90cc", + "tweak": "639f0281b7ac49e742cd25b7f188657626da1ad169209078e2761cefd91fd65e", + "tweakedPubkey": "77e30a5522dd9f894c3f8b8bd4c4b2cf82ca7da8a3ea6a239655c39c050ab220" + }, + "expected": { + "scriptPubKey": "512077e30a5522dd9f894c3f8b8bd4c4b2cf82ca7da8a3ea6a239655c39c050ab220", + "bip350Address": "bc1pwl3s54fzmk0cjnpl3w9af39je7pv5ldg504x5guk2hpecpg2kgsqaqstjq", + "scriptPathControlBlocks": [ + "c1f9f400803e683727b14f463836e1e78e1c64417638aa066919291a225f0e8dd82cb2b90daa543b544161530c925f285b06196940d6085ca9474d41dc3822c5cb", + "c1f9f400803e683727b14f463836e1e78e1c64417638aa066919291a225f0e8dd864512fecdb5afa04f98839b50e6f0cb7b1e539bf6f205f67934083cdcc3c8d89" + ] + } + }, + { + "given": { + "internalPubkey": "e0dfe2300b0dd746a3f8674dfd4525623639042569d829c7f0eed9602d263e6f", + "scriptTree": [ + { + "id": 0, + "script": "2072ea6adcf1d371dea8fba1035a09f3d24ed5a059799bae114084130ee5898e69ac", + "leafVersion": 192 + }, + [ + { + "id": 1, + "script": "202352d137f2f3ab38d1eaa976758873377fa5ebb817372c71e2c542313d4abda8ac", + "leafVersion": 192 + }, + { + "id": 2, + "script": "207337c0dd4253cb86f2c43a2351aadd82cccb12a172cd120452b9bb8324f2186aac", + "leafVersion": 192 + } + ] + ] + }, + "intermediary": { + "leafHashes": [ + "2645a02e0aac1fe69d69755733a9b7621b694bb5b5cde2bbfc94066ed62b9817", + "ba982a91d4fc552163cb1c0da03676102d5b7a014304c01f0c77b2b8e888de1c", + "9e31407bffa15fefbf5090b149d53959ecdf3f62b1246780238c24501d5ceaf6" + ], + "merkleRoot": "ccbd66c6f7e8fdab47b3a486f59d28262be857f30d4773f2d5ea47f7761ce0e2", + "tweak": "b57bfa183d28eeb6ad688ddaabb265b4a41fbf68e5fed2c72c74de70d5a786f4", + "tweakedPubkey": "91b64d5324723a985170e4dc5a0f84c041804f2cd12660fa5dec09fc21783605" + }, + "expected": { + "scriptPubKey": "512091b64d5324723a985170e4dc5a0f84c041804f2cd12660fa5dec09fc21783605", + "bip350Address": "bc1pjxmy65eywgafs5tsunw95ruycpqcqnev6ynxp7jaasylcgtcxczs6n332e", + "scriptPathControlBlocks": [ + "c0e0dfe2300b0dd746a3f8674dfd4525623639042569d829c7f0eed9602d263e6fffe578e9ea769027e4f5a3de40732f75a88a6353a09d767ddeb66accef85e553", + "c0e0dfe2300b0dd746a3f8674dfd4525623639042569d829c7f0eed9602d263e6f9e31407bffa15fefbf5090b149d53959ecdf3f62b1246780238c24501d5ceaf62645a02e0aac1fe69d69755733a9b7621b694bb5b5cde2bbfc94066ed62b9817", + "c0e0dfe2300b0dd746a3f8674dfd4525623639042569d829c7f0eed9602d263e6fba982a91d4fc552163cb1c0da03676102d5b7a014304c01f0c77b2b8e888de1c2645a02e0aac1fe69d69755733a9b7621b694bb5b5cde2bbfc94066ed62b9817" + ] + } + }, + { + "given": { + "internalPubkey": "55adf4e8967fbd2e29f20ac896e60c3b0f1d5b0efa9d34941b5958c7b0a0312d", + "scriptTree": [ + { + "id": 0, + "script": "2071981521ad9fc9036687364118fb6ccd2035b96a423c59c5430e98310a11abe2ac", + "leafVersion": 192 + }, + [ + { + "id": 1, + "script": "20d5094d2dbe9b76e2c245a2b89b6006888952e2faa6a149ae318d69e520617748ac", + "leafVersion": 192 + }, + { + "id": 2, + "script": "20c440b462ad48c7a77f94cd4532d8f2119dcebbd7c9764557e62726419b08ad4cac", + "leafVersion": 192 + } + ] + ] + }, + "intermediary": { + "leafHashes": [ + "f154e8e8e17c31d3462d7132589ed29353c6fafdb884c5a6e04ea938834f0d9d", + "737ed1fe30bc42b8022d717b44f0d93516617af64a64753b7a06bf16b26cd711", + "d7485025fceb78b9ed667db36ed8b8dc7b1f0b307ac167fa516fe4352b9f4ef7" + ], + "merkleRoot": "2f6b2c5397b6d68ca18e09a3f05161668ffe93a988582d55c6f07bd5b3329def", + "tweak": "6579138e7976dc13b6a92f7bfd5a2fc7684f5ea42419d43368301470f3b74ed9", + "tweakedPubkey": "75169f4001aa68f15bbed28b218df1d0a62cbbcf1188c6665110c293c907b831" + }, + "expected": { + "scriptPubKey": "512075169f4001aa68f15bbed28b218df1d0a62cbbcf1188c6665110c293c907b831", + "bip350Address": "bc1pw5tf7sqp4f50zka7629jrr036znzew70zxyvvej3zrpf8jg8hqcssyuewe", + "scriptPathControlBlocks": [ + "c155adf4e8967fbd2e29f20ac896e60c3b0f1d5b0efa9d34941b5958c7b0a0312d3cd369a528b326bc9d2133cbd2ac21451acb31681a410434672c8e34fe757e91", + "c155adf4e8967fbd2e29f20ac896e60c3b0f1d5b0efa9d34941b5958c7b0a0312dd7485025fceb78b9ed667db36ed8b8dc7b1f0b307ac167fa516fe4352b9f4ef7f154e8e8e17c31d3462d7132589ed29353c6fafdb884c5a6e04ea938834f0d9d", + "c155adf4e8967fbd2e29f20ac896e60c3b0f1d5b0efa9d34941b5958c7b0a0312d737ed1fe30bc42b8022d717b44f0d93516617af64a64753b7a06bf16b26cd711f154e8e8e17c31d3462d7132589ed29353c6fafdb884c5a6e04ea938834f0d9d" + ] + } + } + ], + "keyPathSpending": [ + { + "given": { + "rawUnsignedTx": "02000000097de20cbff686da83a54981d2b9bab3586f4ca7e48f57f5b55963115f3b334e9c010000000000000000d7b7cab57b1393ace2d064f4d4a2cb8af6def61273e127517d44759b6dafdd990000000000fffffffff8e1f583384333689228c5d28eac13366be082dc57441760d957275419a418420000000000fffffffff0689180aa63b30cb162a73c6d2a38b7eeda2a83ece74310fda0843ad604853b0100000000feffffffaa5202bdf6d8ccd2ee0f0202afbbb7461d9264a25e5bfd3c5a52ee1239e0ba6c0000000000feffffff956149bdc66faa968eb2be2d2faa29718acbfe3941215893a2a3446d32acd050000000000000000000e664b9773b88c09c32cb70a2a3e4da0ced63b7ba3b22f848531bbb1d5d5f4c94010000000000000000e9aa6b8e6c9de67619e6a3924ae25696bb7b694bb677a632a74ef7eadfd4eabf0000000000ffffffffa778eb6a263dc090464cd125c466b5a99667720b1c110468831d058aa1b82af10100000000ffffffff0200ca9a3b000000001976a91406afd46bcdfd22ef94ac122aa11f241244a37ecc88ac807840cb0000000020ac9a87f5594be208f8532db38cff670c450ed2fea8fcdefcc9a663f78bab962b0065cd1d", + "utxosSpent": [ + { + "scriptPubKey": "512053a1f6e454df1aa2776a2814a721372d6258050de330b3c6d10ee8f4e0dda343", + "amountSats": 420000000 + }, + { + "scriptPubKey": "5120147c9c57132f6e7ecddba9800bb0c4449251c92a1e60371ee77557b6620f3ea3", + "amountSats": 462000000 + }, + { + "scriptPubKey": "76a914751e76e8199196d454941c45d1b3a323f1433bd688ac", + "amountSats": 294000000 + }, + { + "scriptPubKey": "5120e4d810fd50586274face62b8a807eb9719cef49c04177cc6b76a9a4251d5450e", + "amountSats": 504000000 + }, + { + "scriptPubKey": "512091b64d5324723a985170e4dc5a0f84c041804f2cd12660fa5dec09fc21783605", + "amountSats": 630000000 + }, + { + "scriptPubKey": "00147dd65592d0ab2fe0d0257d571abf032cd9db93dc", + "amountSats": 378000000 + }, + { + "scriptPubKey": "512075169f4001aa68f15bbed28b218df1d0a62cbbcf1188c6665110c293c907b831", + "amountSats": 672000000 + }, + { + "scriptPubKey": "5120712447206d7a5238acc7ff53fbe94a3b64539ad291c7cdbc490b7577e4b17df5", + "amountSats": 546000000 + }, + { + "scriptPubKey": "512077e30a5522dd9f894c3f8b8bd4c4b2cf82ca7da8a3ea6a239655c39c050ab220", + "amountSats": 588000000 + } + ] + }, + "intermediary": { + "hashAmounts": "58a6964a4f5f8f0b642ded0a8a553be7622a719da71d1f5befcefcdee8e0fde6", + "hashOutputs": "a2e6dab7c1f0dcd297c8d61647fd17d821541ea69c3cc37dcbad7f90d4eb4bc5", + "hashPrevouts": "e3b33bb4ef3a52ad1fffb555c0d82828eb22737036eaeb02a235d82b909c4c3f", + "hashScriptPubkeys": "23ad0f61ad2bca5ba6a7693f50fce988e17c3780bf2b1e720cfbb38fbdd52e21", + "hashSequences": "18959c7221ab5ce9e26c3cd67b22c24f8baa54bac281d8e6b05e400e6c3a957e" + }, + "inputSpending": [ + { + "given": { + "txinIndex": 0, + "internalPrivkey": "6b973d88838f27366ed61c9ad6367663045cb456e28335c109e30717ae0c6baa", + "merkleRoot": null, + "hashType": 3 + }, + "intermediary": { + "internalPubkey": "d6889cb081036e0faefa3a35157ad71086b123b2b144b649798b494c300a961d", + "tweak": "b86e7be8f39bab32a6f2c0443abbc210f0edac0e2c53d501b36b64437d9c6c70", + "tweakedPrivkey": "2405b971772ad26915c8dcdf10f238753a9b837e5f8e6a86fd7c0cce5b7296d9", + "sigMsg": "0003020000000065cd1de3b33bb4ef3a52ad1fffb555c0d82828eb22737036eaeb02a235d82b909c4c3f58a6964a4f5f8f0b642ded0a8a553be7622a719da71d1f5befcefcdee8e0fde623ad0f61ad2bca5ba6a7693f50fce988e17c3780bf2b1e720cfbb38fbdd52e2118959c7221ab5ce9e26c3cd67b22c24f8baa54bac281d8e6b05e400e6c3a957e0000000000d0418f0e9a36245b9a50ec87f8bf5be5bcae434337b87139c3a5b1f56e33cba0", + "precomputedUsed": [ + "hashAmounts", + "hashPrevouts", + "hashScriptPubkeys", + "hashSequences" + ], + "sigHash": "2514a6272f85cfa0f45eb907fcb0d121b808ed37c6ea160a5a9046ed5526d555" + }, + "expected": { + "witness": [ + "ed7c1647cb97379e76892be0cacff57ec4a7102aa24296ca39af7541246d8ff14d38958d4cc1e2e478e4d4a764bbfd835b16d4e314b72937b29833060b87276c03" + ] + } + }, + { + "given": { + "txinIndex": 1, + "internalPrivkey": "1e4da49f6aaf4e5cd175fe08a32bb5cb4863d963921255f33d3bc31e1343907f", + "merkleRoot": "5b75adecf53548f3ec6ad7d78383bf84cc57b55a3127c72b9a2481752dd88b21", + "hashType": 131 + }, + "intermediary": { + "internalPubkey": "187791b6f712a8ea41c8ecdd0ee77fab3e85263b37e1ec18a3651926b3a6cf27", + "tweak": "cbd8679ba636c1110ea247542cfbd964131a6be84f873f7f3b62a777528ed001", + "tweakedPrivkey": "ea260c3b10e60f6de018455cd0278f2f5b7e454be1999572789e6a9565d26080", + "sigMsg": "0083020000000065cd1d00d7b7cab57b1393ace2d064f4d4a2cb8af6def61273e127517d44759b6dafdd9900000000808f891b00000000225120147c9c57132f6e7ecddba9800bb0c4449251c92a1e60371ee77557b6620f3ea3ffffffffffcef8fb4ca7efc5433f591ecfc57391811ce1e186a3793024def5c884cba51d", + "precomputedUsed": [], + "sigHash": "325a644af47e8a5a2591cda0ab0723978537318f10e6a63d4eed783b96a71a4d" + }, + "expected": { + "witness": [ + "052aedffc554b41f52b521071793a6b88d6dbca9dba94cf34c83696de0c1ec35ca9c5ed4ab28059bd606a4f3a657eec0bb96661d42921b5f50a95ad33675b54f83" + ] + } + }, + { + "given": { + "txinIndex": 3, + "internalPrivkey": "d3c7af07da2d54f7a7735d3d0fc4f0a73164db638b2f2f7c43f711f6d4aa7e64", + "merkleRoot": "c525714a7f49c28aedbbba78c005931a81c234b2f6c99a73e4d06082adc8bf2b", + "hashType": 1 + }, + "intermediary": { + "internalPubkey": "93478e9488f956df2396be2ce6c5cced75f900dfa18e7dabd2428aae78451820", + "tweak": "6af9e28dbf9d6aaf027696e2598a5b3d056f5fd2355a7fd5a37a0e5008132d30", + "tweakedPrivkey": "97323385e57015b75b0339a549c56a948eb961555973f0951f555ae6039ef00d", + "sigMsg": "0001020000000065cd1de3b33bb4ef3a52ad1fffb555c0d82828eb22737036eaeb02a235d82b909c4c3f58a6964a4f5f8f0b642ded0a8a553be7622a719da71d1f5befcefcdee8e0fde623ad0f61ad2bca5ba6a7693f50fce988e17c3780bf2b1e720cfbb38fbdd52e2118959c7221ab5ce9e26c3cd67b22c24f8baa54bac281d8e6b05e400e6c3a957ea2e6dab7c1f0dcd297c8d61647fd17d821541ea69c3cc37dcbad7f90d4eb4bc50003000000", + "precomputedUsed": [ + "hashAmounts", + "hashOutputs", + "hashPrevouts", + "hashScriptPubkeys", + "hashSequences" + ], + "sigHash": "bf013ea93474aa67815b1b6cc441d23b64fa310911d991e713cd34c7f5d46669" + }, + "expected": { + "witness": [ + "ff45f742a876139946a149ab4d9185574b98dc919d2eb6754f8abaa59d18b025637a3aa043b91817739554f4ed2026cf8022dbd83e351ce1fabc272841d2510a01" + ] + } + }, + { + "given": { + "txinIndex": 4, + "internalPrivkey": "f36bb07a11e469ce941d16b63b11b9b9120a84d9d87cff2c84a8d4affb438f4e", + "merkleRoot": "ccbd66c6f7e8fdab47b3a486f59d28262be857f30d4773f2d5ea47f7761ce0e2", + "hashType": 0 + }, + "intermediary": { + "internalPubkey": "e0dfe2300b0dd746a3f8674dfd4525623639042569d829c7f0eed9602d263e6f", + "tweak": "b57bfa183d28eeb6ad688ddaabb265b4a41fbf68e5fed2c72c74de70d5a786f4", + "tweakedPrivkey": "a8e7aa924f0d58854185a490e6c41f6efb7b675c0f3331b7f14b549400b4d501", + "sigMsg": "0000020000000065cd1de3b33bb4ef3a52ad1fffb555c0d82828eb22737036eaeb02a235d82b909c4c3f58a6964a4f5f8f0b642ded0a8a553be7622a719da71d1f5befcefcdee8e0fde623ad0f61ad2bca5ba6a7693f50fce988e17c3780bf2b1e720cfbb38fbdd52e2118959c7221ab5ce9e26c3cd67b22c24f8baa54bac281d8e6b05e400e6c3a957ea2e6dab7c1f0dcd297c8d61647fd17d821541ea69c3cc37dcbad7f90d4eb4bc50004000000", + "precomputedUsed": [ + "hashAmounts", + "hashOutputs", + "hashPrevouts", + "hashScriptPubkeys", + "hashSequences" + ], + "sigHash": "4f900a0bae3f1446fd48490c2958b5a023228f01661cda3496a11da502a7f7ef" + }, + "expected": { + "witness": [ + "b4010dd48a617db09926f729e79c33ae0b4e94b79f04a1ae93ede6315eb3669de185a17d2b0ac9ee09fd4c64b678a0b61a0a86fa888a273c8511be83bfd6810f" + ] + } + }, + { + "given": { + "txinIndex": 6, + "internalPrivkey": "415cfe9c15d9cea27d8104d5517c06e9de48e2f986b695e4f5ffebf230e725d8", + "merkleRoot": "2f6b2c5397b6d68ca18e09a3f05161668ffe93a988582d55c6f07bd5b3329def", + "hashType": 2 + }, + "intermediary": { + "internalPubkey": "55adf4e8967fbd2e29f20ac896e60c3b0f1d5b0efa9d34941b5958c7b0a0312d", + "tweak": "6579138e7976dc13b6a92f7bfd5a2fc7684f5ea42419d43368301470f3b74ed9", + "tweakedPrivkey": "241c14f2639d0d7139282aa6abde28dd8a067baa9d633e4e7230287ec2d02901", + "sigMsg": "0002020000000065cd1de3b33bb4ef3a52ad1fffb555c0d82828eb22737036eaeb02a235d82b909c4c3f58a6964a4f5f8f0b642ded0a8a553be7622a719da71d1f5befcefcdee8e0fde623ad0f61ad2bca5ba6a7693f50fce988e17c3780bf2b1e720cfbb38fbdd52e2118959c7221ab5ce9e26c3cd67b22c24f8baa54bac281d8e6b05e400e6c3a957e0006000000", + "precomputedUsed": [ + "hashAmounts", + "hashPrevouts", + "hashScriptPubkeys", + "hashSequences" + ], + "sigHash": "15f25c298eb5cdc7eb1d638dd2d45c97c4c59dcaec6679cfc16ad84f30876b85" + }, + "expected": { + "witness": [ + "a3785919a2ce3c4ce26f298c3d51619bc474ae24014bcdd31328cd8cfbab2eff3395fa0a16fe5f486d12f22a9cedded5ae74feb4bbe5351346508c5405bcfee002" + ] + } + }, + { + "given": { + "txinIndex": 7, + "internalPrivkey": "c7b0e81f0a9a0b0499e112279d718cca98e79a12e2f137c72ae5b213aad0d103", + "merkleRoot": "6c2dc106ab816b73f9d07e3cd1ef2c8c1256f519748e0813e4edd2405d277bef", + "hashType": 130 + }, + "intermediary": { + "internalPubkey": "ee4fe085983462a184015d1f782d6a5f8b9c2b60130aff050ce221ecf3786592", + "tweak": "9e0517edc8259bb3359255400b23ca9507f2a91cd1e4250ba068b4eafceba4a9", + "tweakedPrivkey": "65b6000cd2bfa6b7cf736767a8955760e62b6649058cbc970b7c0871d786346b", + "sigMsg": "0082020000000065cd1d00e9aa6b8e6c9de67619e6a3924ae25696bb7b694bb677a632a74ef7eadfd4eabf00000000804c8b2000000000225120712447206d7a5238acc7ff53fbe94a3b64539ad291c7cdbc490b7577e4b17df5ffffffff", + "precomputedUsed": [], + "sigHash": "cd292de50313804dabe4685e83f923d2969577191a3e1d2882220dca88cbeb10" + }, + "expected": { + "witness": [ + "ea0c6ba90763c2d3a296ad82ba45881abb4f426b3f87af162dd24d5109edc1cdd11915095ba47c3a9963dc1e6c432939872bc49212fe34c632cd3ab9fed429c482" + ] + } + }, + { + "given": { + "txinIndex": 8, + "internalPrivkey": "77863416be0d0665e517e1c375fd6f75839544eca553675ef7fdf4949518ebaa", + "merkleRoot": "ab179431c28d3b68fb798957faf5497d69c883c6fb1e1cd9f81483d87bac90cc", + "hashType": 129 + }, + "intermediary": { + "internalPubkey": "f9f400803e683727b14f463836e1e78e1c64417638aa066919291a225f0e8dd8", + "tweak": "639f0281b7ac49e742cd25b7f188657626da1ad169209078e2761cefd91fd65e", + "tweakedPrivkey": "ec18ce6af99f43815db543f47b8af5ff5df3b2cb7315c955aa4a86e8143d2bf5", + "sigMsg": "0081020000000065cd1da2e6dab7c1f0dcd297c8d61647fd17d821541ea69c3cc37dcbad7f90d4eb4bc500a778eb6a263dc090464cd125c466b5a99667720b1c110468831d058aa1b82af101000000002b0c230000000022512077e30a5522dd9f894c3f8b8bd4c4b2cf82ca7da8a3ea6a239655c39c050ab220ffffffff", + "precomputedUsed": [ + "hashOutputs" + ], + "sigHash": "cccb739eca6c13a8a89e6e5cd317ffe55669bbda23f2fd37b0f18755e008edd2" + }, + "expected": { + "witness": [ + "bbc9584a11074e83bc8c6759ec55401f0ae7b03ef290c3139814f545b58a9f8127258000874f44bc46db7646322107d4d86aec8e73b8719a61fff761d75b5dd981" + ] + } + } + ], + "auxiliary": { + "fullySignedTx": "020000000001097de20cbff686da83a54981d2b9bab3586f4ca7e48f57f5b55963115f3b334e9c010000000000000000d7b7cab57b1393ace2d064f4d4a2cb8af6def61273e127517d44759b6dafdd990000000000fffffffff8e1f583384333689228c5d28eac13366be082dc57441760d957275419a41842000000006b4830450221008f3b8f8f0537c420654d2283673a761b7ee2ea3c130753103e08ce79201cf32a022079e7ab904a1980ef1c5890b648c8783f4d10103dd62f740d13daa79e298d50c201210279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798fffffffff0689180aa63b30cb162a73c6d2a38b7eeda2a83ece74310fda0843ad604853b0100000000feffffffaa5202bdf6d8ccd2ee0f0202afbbb7461d9264a25e5bfd3c5a52ee1239e0ba6c0000000000feffffff956149bdc66faa968eb2be2d2faa29718acbfe3941215893a2a3446d32acd050000000000000000000e664b9773b88c09c32cb70a2a3e4da0ced63b7ba3b22f848531bbb1d5d5f4c94010000000000000000e9aa6b8e6c9de67619e6a3924ae25696bb7b694bb677a632a74ef7eadfd4eabf0000000000ffffffffa778eb6a263dc090464cd125c466b5a99667720b1c110468831d058aa1b82af10100000000ffffffff0200ca9a3b000000001976a91406afd46bcdfd22ef94ac122aa11f241244a37ecc88ac807840cb0000000020ac9a87f5594be208f8532db38cff670c450ed2fea8fcdefcc9a663f78bab962b0141ed7c1647cb97379e76892be0cacff57ec4a7102aa24296ca39af7541246d8ff14d38958d4cc1e2e478e4d4a764bbfd835b16d4e314b72937b29833060b87276c030141052aedffc554b41f52b521071793a6b88d6dbca9dba94cf34c83696de0c1ec35ca9c5ed4ab28059bd606a4f3a657eec0bb96661d42921b5f50a95ad33675b54f83000141ff45f742a876139946a149ab4d9185574b98dc919d2eb6754f8abaa59d18b025637a3aa043b91817739554f4ed2026cf8022dbd83e351ce1fabc272841d2510a010140b4010dd48a617db09926f729e79c33ae0b4e94b79f04a1ae93ede6315eb3669de185a17d2b0ac9ee09fd4c64b678a0b61a0a86fa888a273c8511be83bfd6810f0247304402202b795e4de72646d76eab3f0ab27dfa30b810e856ff3a46c9a702df53bb0d8cc302203ccc4d822edab5f35caddb10af1be93583526ccfbade4b4ead350781e2f8adcd012102f9308a019258c31049344f85f89d5229b531c845836f99b08601f113bce036f90141a3785919a2ce3c4ce26f298c3d51619bc474ae24014bcdd31328cd8cfbab2eff3395fa0a16fe5f486d12f22a9cedded5ae74feb4bbe5351346508c5405bcfee0020141ea0c6ba90763c2d3a296ad82ba45881abb4f426b3f87af162dd24d5109edc1cdd11915095ba47c3a9963dc1e6c432939872bc49212fe34c632cd3ab9fed429c4820141bbc9584a11074e83bc8c6759ec55401f0ae7b03ef290c3139814f545b58a9f8127258000874f44bc46db7646322107d4d86aec8e73b8719a61fff761d75b5dd9810065cd1d" + } + } + ] +} diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 668ff150ee..ef18ed1385 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -59,7 +59,7 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) // Mock an outbound peer CAddress addr1(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode1(id++, ServiceFlags(NODE_NETWORK | NODE_WITNESS), INVALID_SOCKET, addr1, /* nKeyedNetGroupIn */ 0, /* nLocalHostNonceIn */ 0, CAddress(), /* pszDest */ "", ConnectionType::OUTBOUND_FULL_RELAY, /* inbound_onion */ false); + CNode dummyNode1(id++, ServiceFlags(NODE_NETWORK | NODE_WITNESS), INVALID_SOCKET, addr1, /*nKeyedNetGroupIn=*/0, /*nLocalHostNonceIn=*/0, CAddress(), /*addrNameIn=*/"", ConnectionType::OUTBOUND_FULL_RELAY, /*inbound_onion=*/false); dummyNode1.SetCommonVersion(PROTOCOL_VERSION); peerLogic->InitializeNode(&dummyNode1); @@ -105,10 +105,10 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) peerLogic->FinalizeNode(dummyNode1); } -static void AddRandomOutboundPeer(std::vector<CNode*>& vNodes, PeerManager& peerLogic, ConnmanTestMsg& connman) +static void AddRandomOutboundPeer(std::vector<CNode*>& vNodes, PeerManager& peerLogic, ConnmanTestMsg& connman, ConnectionType connType) { CAddress addr(ip(g_insecure_rand_ctx.randbits(32)), NODE_NONE); - vNodes.emplace_back(new CNode(id++, ServiceFlags(NODE_NETWORK | NODE_WITNESS), INVALID_SOCKET, addr, /* nKeyedNetGroupIn */ 0, /* nLocalHostNonceIn */ 0, CAddress(), /* pszDest */ "", ConnectionType::OUTBOUND_FULL_RELAY, /* inbound_onion */ false)); + vNodes.emplace_back(new CNode(id++, ServiceFlags(NODE_NETWORK | NODE_WITNESS), INVALID_SOCKET, addr, /*nKeyedNetGroupIn=*/0, /*nLocalHostNonceIn=*/0, CAddress(), /*addrNameIn=*/"", connType, /*inbound_onion=*/false)); CNode &node = *vNodes.back(); node.SetCommonVersion(PROTOCOL_VERSION); @@ -136,7 +136,7 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) // Mock some outbound peers for (int i = 0; i < max_outbound_full_relay; ++i) { - AddRandomOutboundPeer(vNodes, *peerLogic, *connman); + AddRandomOutboundPeer(vNodes, *peerLogic, *connman, ConnectionType::OUTBOUND_FULL_RELAY); } peerLogic->CheckForStaleTipAndEvictPeers(); @@ -161,7 +161,7 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) // If we add one more peer, something should get marked for eviction // on the next check (since we're mocking the time to be in the future, the // required time connected check should be satisfied). - AddRandomOutboundPeer(vNodes, *peerLogic, *connman); + AddRandomOutboundPeer(vNodes, *peerLogic, *connman, ConnectionType::OUTBOUND_FULL_RELAY); peerLogic->CheckForStaleTipAndEvictPeers(); for (int i = 0; i < max_outbound_full_relay; ++i) { @@ -190,6 +190,68 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) connman->ClearTestNodes(); } +BOOST_AUTO_TEST_CASE(block_relay_only_eviction) +{ + const CChainParams& chainparams = Params(); + auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman); + auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, nullptr, + *m_node.chainman, *m_node.mempool, false); + + constexpr int max_outbound_block_relay{MAX_BLOCK_RELAY_ONLY_CONNECTIONS}; + constexpr int64_t MINIMUM_CONNECT_TIME{30}; + CConnman::Options options; + options.nMaxConnections = DEFAULT_MAX_PEER_CONNECTIONS; + options.m_max_outbound_full_relay = MAX_OUTBOUND_FULL_RELAY_CONNECTIONS; + options.m_max_outbound_block_relay = max_outbound_block_relay; + + connman->Init(options); + std::vector<CNode*> vNodes; + + // Add block-relay-only peers up to the limit + for (int i = 0; i < max_outbound_block_relay; ++i) { + AddRandomOutboundPeer(vNodes, *peerLogic, *connman, ConnectionType::BLOCK_RELAY); + } + peerLogic->CheckForStaleTipAndEvictPeers(); + + for (int i = 0; i < max_outbound_block_relay; ++i) { + BOOST_CHECK(vNodes[i]->fDisconnect == false); + } + + // Add an extra block-relay-only peer breaking the limit (mocks logic in ThreadOpenConnections) + AddRandomOutboundPeer(vNodes, *peerLogic, *connman, ConnectionType::BLOCK_RELAY); + peerLogic->CheckForStaleTipAndEvictPeers(); + + // The extra peer should only get marked for eviction after MINIMUM_CONNECT_TIME + for (int i = 0; i < max_outbound_block_relay; ++i) { + BOOST_CHECK(vNodes[i]->fDisconnect == false); + } + BOOST_CHECK(vNodes.back()->fDisconnect == false); + + SetMockTime(GetTime() + MINIMUM_CONNECT_TIME + 1); + peerLogic->CheckForStaleTipAndEvictPeers(); + for (int i = 0; i < max_outbound_block_relay; ++i) { + BOOST_CHECK(vNodes[i]->fDisconnect == false); + } + BOOST_CHECK(vNodes.back()->fDisconnect == true); + + // Update the last block time for the extra peer, + // and check that the next youngest peer gets evicted. + vNodes.back()->fDisconnect = false; + vNodes.back()->nLastBlockTime = GetTime(); + + peerLogic->CheckForStaleTipAndEvictPeers(); + for (int i = 0; i < max_outbound_block_relay - 1; ++i) { + BOOST_CHECK(vNodes[i]->fDisconnect == false); + } + BOOST_CHECK(vNodes[max_outbound_block_relay - 1]->fDisconnect == true); + BOOST_CHECK(vNodes.back()->fDisconnect == false); + + for (const CNode* node : vNodes) { + peerLogic->FinalizeNode(*node); + } + connman->ClearTestNodes(); +} + BOOST_AUTO_TEST_CASE(peer_discouragement) { const CChainParams& chainparams = Params(); @@ -212,14 +274,14 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) std::array<CNode*, 3> nodes; banman->ClearBanned(); - nodes[0] = new CNode{id++, NODE_NETWORK, INVALID_SOCKET, addr[0], /* nKeyedNetGroupIn */ 0, - /* nLocalHostNonceIn */ 0, CAddress(), /* pszDest */ "", - ConnectionType::INBOUND, /* inbound_onion */ false}; + nodes[0] = new CNode{id++, NODE_NETWORK, INVALID_SOCKET, addr[0], /*nKeyedNetGroupIn=*/0, + /*nLocalHostNonceIn=*/0, CAddress(), /*addrNameIn=*/"", + ConnectionType::INBOUND, /*inbound_onion=*/false}; nodes[0]->SetCommonVersion(PROTOCOL_VERSION); peerLogic->InitializeNode(nodes[0]); nodes[0]->fSuccessfullyConnected = true; connman->AddTestNode(*nodes[0]); - peerLogic->Misbehaving(nodes[0]->GetId(), DISCOURAGEMENT_THRESHOLD, /* message */ ""); // Should be discouraged + peerLogic->Misbehaving(nodes[0]->GetId(), DISCOURAGEMENT_THRESHOLD, /*message=*/""); // Should be discouraged { LOCK(nodes[0]->cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(nodes[0])); @@ -228,14 +290,14 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) BOOST_CHECK(nodes[0]->fDisconnect); BOOST_CHECK(!banman->IsDiscouraged(other_addr)); // Different address, not discouraged - nodes[1] = new CNode{id++, NODE_NETWORK, INVALID_SOCKET, addr[1], /* nKeyedNetGroupIn */ 1, - /* nLocalHostNonceIn */ 1, CAddress(), /* pszDest */ "", - ConnectionType::INBOUND, /* inbound_onion */ false}; + nodes[1] = new CNode{id++, NODE_NETWORK, INVALID_SOCKET, addr[1], /*nKeyedNetGroupIn=*/1, + /*nLocalHostNonceIn=*/1, CAddress(), /*addrNameIn=*/"", + ConnectionType::INBOUND, /*inbound_onion=*/false}; nodes[1]->SetCommonVersion(PROTOCOL_VERSION); peerLogic->InitializeNode(nodes[1]); nodes[1]->fSuccessfullyConnected = true; connman->AddTestNode(*nodes[1]); - peerLogic->Misbehaving(nodes[1]->GetId(), DISCOURAGEMENT_THRESHOLD - 1, /* message */ ""); + peerLogic->Misbehaving(nodes[1]->GetId(), DISCOURAGEMENT_THRESHOLD - 1, /*message=*/""); { LOCK(nodes[1]->cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(nodes[1])); @@ -246,7 +308,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) // [1] is not discouraged/disconnected yet. BOOST_CHECK(!banman->IsDiscouraged(addr[1])); BOOST_CHECK(!nodes[1]->fDisconnect); - peerLogic->Misbehaving(nodes[1]->GetId(), 1, /* message */ ""); // [1] reaches discouragement threshold + peerLogic->Misbehaving(nodes[1]->GetId(), 1, /*message=*/""); // [1] reaches discouragement threshold { LOCK(nodes[1]->cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(nodes[1])); @@ -259,14 +321,14 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) // Make sure non-IP peers are discouraged and disconnected properly. - nodes[2] = new CNode{id++, NODE_NETWORK, INVALID_SOCKET, addr[2], /* nKeyedNetGroupIn */ 1, - /* nLocalHostNonceIn */ 1, CAddress(), /* pszDest */ "", - ConnectionType::OUTBOUND_FULL_RELAY, /* inbound_onion */ false}; + nodes[2] = new CNode{id++, NODE_NETWORK, INVALID_SOCKET, addr[2], /*nKeyedNetGroupIn=*/1, + /*nLocalHostNonceIn=*/1, CAddress(), /*addrNameIn=*/"", + ConnectionType::OUTBOUND_FULL_RELAY, /*inbound_onion=*/false}; nodes[2]->SetCommonVersion(PROTOCOL_VERSION); peerLogic->InitializeNode(nodes[2]); nodes[2]->fSuccessfullyConnected = true; connman->AddTestNode(*nodes[2]); - peerLogic->Misbehaving(nodes[2]->GetId(), DISCOURAGEMENT_THRESHOLD, /* message */ ""); + peerLogic->Misbehaving(nodes[2]->GetId(), DISCOURAGEMENT_THRESHOLD, /*message=*/""); { LOCK(nodes[2]->cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(nodes[2])); @@ -297,12 +359,12 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) SetMockTime(nStartTime); // Overrides future calls to GetTime() CAddress addr(ip(0xa0b0c001), NODE_NONE); - CNode dummyNode(id++, NODE_NETWORK, INVALID_SOCKET, addr, /* nKeyedNetGroupIn */ 4, /* nLocalHostNonceIn */ 4, CAddress(), /* pszDest */ "", ConnectionType::INBOUND, /* inbound_onion */ false); + CNode dummyNode(id++, NODE_NETWORK, INVALID_SOCKET, addr, /*nKeyedNetGroupIn=*/4, /*nLocalHostNonceIn=*/4, CAddress(), /*addrNameIn=*/"", ConnectionType::INBOUND, /*inbound_onion=*/false); dummyNode.SetCommonVersion(PROTOCOL_VERSION); peerLogic->InitializeNode(&dummyNode); dummyNode.fSuccessfullyConnected = true; - peerLogic->Misbehaving(dummyNode.GetId(), DISCOURAGEMENT_THRESHOLD, /* message */ ""); + peerLogic->Misbehaving(dummyNode.GetId(), DISCOURAGEMENT_THRESHOLD, /*message=*/""); { LOCK(dummyNode.cs_sendProcessing); BOOST_CHECK(peerLogic->SendMessages(&dummyNode)); @@ -334,7 +396,7 @@ static void MakeNewKeyWithFastRandomContext(CKey& key) { std::vector<unsigned char> keydata; keydata = g_insecure_rand_ctx.randbytes(32); - key.Set(keydata.data(), keydata.data() + keydata.size(), /*fCompressedIn*/ true); + key.Set(keydata.data(), keydata.data() + keydata.size(), /*fCompressedIn=*/true); assert(key.IsValid()); } diff --git a/src/test/fuzz/addrman.cpp b/src/test/fuzz/addrman.cpp index c6df6a0e61..9c85c20e2b 100644 --- a/src/test/fuzz/addrman.cpp +++ b/src/test/fuzz/addrman.cpp @@ -29,7 +29,7 @@ FUZZ_TARGET_INIT(data_stream_addr_man, initialize_addrman) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; CDataStream data_stream = ConsumeDataStream(fuzzed_data_provider); - AddrMan addr_man(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + AddrMan addr_man(/*asmap=*/std::vector<bool>(), /*deterministic=*/false, /*consistency_check_ratio=*/0); try { ReadFromStream(addr_man, data_stream); } catch (const std::exception&) { @@ -113,7 +113,7 @@ class AddrManDeterministic : public AddrMan { public: explicit AddrManDeterministic(std::vector<bool> asmap, FuzzedDataProvider& fuzzed_data_provider) - : AddrMan(std::move(asmap), /* deterministic */ true, /* consistency_check_ratio */ 0) + : AddrMan(std::move(asmap), /*deterministic=*/true, /*consistency_check_ratio=*/0) { WITH_LOCK(m_impl->cs, m_impl->insecure_rand = FastRandomContext{ConsumeUInt256(fuzzed_data_provider)}); } @@ -236,7 +236,7 @@ FUZZ_TARGET_INIT(addrman, initialize_addrman) } } AddrManDeterministic& addr_man = *addr_man_ptr; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, [&] { @@ -247,7 +247,7 @@ FUZZ_TARGET_INIT(addrman, initialize_addrman) }, [&] { std::vector<CAddress> addresses; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const std::optional<CAddress> opt_address = ConsumeDeserializable<CAddress>(fuzzed_data_provider); if (!opt_address) { break; @@ -286,9 +286,9 @@ FUZZ_TARGET_INIT(addrman, initialize_addrman) } const AddrMan& const_addr_man{addr_man}; (void)const_addr_man.GetAddr( - /* max_addresses */ fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096), - /* max_pct */ fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096), - /* network */ std::nullopt); + /*max_addresses=*/fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096), + /*max_pct=*/fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096), + /*network=*/std::nullopt); (void)const_addr_man.Select(fuzzed_data_provider.ConsumeBool()); (void)const_addr_man.size(); CDataStream data_stream(SER_NETWORK, PROTOCOL_VERSION); diff --git a/src/test/fuzz/asmap.cpp b/src/test/fuzz/asmap.cpp index d402f8632c..c5e9c56049 100644 --- a/src/test/fuzz/asmap.cpp +++ b/src/test/fuzz/asmap.cpp @@ -49,7 +49,7 @@ FUZZ_TARGET(asmap) CNetAddr net_addr; if (ipv6) { assert(addr_size == ADDR_IPV6_SIZE); - net_addr.SetLegacyIPv6(Span<const uint8_t>(addr_data, addr_size)); + net_addr.SetLegacyIPv6({addr_data, addr_size}); } else { assert(addr_size == ADDR_IPV4_SIZE); in_addr ipv4; diff --git a/src/test/fuzz/autofile.cpp b/src/test/fuzz/autofile.cpp index 479342e4be..0cc2d12d29 100644 --- a/src/test/fuzz/autofile.cpp +++ b/src/test/fuzz/autofile.cpp @@ -19,7 +19,7 @@ FUZZ_TARGET(autofile) FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); CAutoFile auto_file = fuzzed_auto_file_provider.open(); - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, [&] { diff --git a/src/test/fuzz/banman.cpp b/src/test/fuzz/banman.cpp index fbba25c404..b2969ecdc0 100644 --- a/src/test/fuzz/banman.cpp +++ b/src/test/fuzz/banman.cpp @@ -58,7 +58,7 @@ FUZZ_TARGET_INIT(banman, initialize_banman) } { - BanMan ban_man{banlist_file, /* client_interface */ nullptr, /* default_ban_time */ ConsumeBanTimeOffset(fuzzed_data_provider)}; + BanMan ban_man{banlist_file, /*client_interface=*/nullptr, /*default_ban_time=*/ConsumeBanTimeOffset(fuzzed_data_provider)}; // The complexity is O(N^2), where N is the input size, because each call // might call DumpBanlist (or other methods that are at least linear // complexity of the input size). @@ -105,7 +105,7 @@ FUZZ_TARGET_INIT(banman, initialize_banman) SetMockTime(ConsumeTime(fuzzed_data_provider)); banmap_t banmap; ban_man.GetBanned(banmap); - BanMan ban_man_read{banlist_file, /* client_interface */ nullptr, /* default_ban_time */ 0}; + BanMan ban_man_read{banlist_file, /*client_interface=*/nullptr, /*default_ban_time=*/0}; banmap_t banmap_read; ban_man_read.GetBanned(banmap_read); assert(banmap == banmap_read); diff --git a/src/test/fuzz/bloom_filter.cpp b/src/test/fuzz/bloom_filter.cpp index 746591a176..3e303ecc0f 100644 --- a/src/test/fuzz/bloom_filter.cpp +++ b/src/test/fuzz/bloom_filter.cpp @@ -24,7 +24,7 @@ FUZZ_TARGET(bloom_filter) 1.0 / fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(1, std::numeric_limits<unsigned int>::max()), fuzzed_data_provider.ConsumeIntegral<unsigned int>(), static_cast<unsigned char>(fuzzed_data_provider.PickValueInArray({BLOOM_UPDATE_NONE, BLOOM_UPDATE_ALL, BLOOM_UPDATE_P2PUBKEY_ONLY, BLOOM_UPDATE_MASK}))}; - while (fuzzed_data_provider.remaining_bytes() > 0) { + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes() > 0, 10000) { CallOneOf( fuzzed_data_provider, [&] { diff --git a/src/test/fuzz/buffered_file.cpp b/src/test/fuzz/buffered_file.cpp index ed72260d10..c3c2e4050f 100644 --- a/src/test/fuzz/buffered_file.cpp +++ b/src/test/fuzz/buffered_file.cpp @@ -29,7 +29,7 @@ FUZZ_TARGET(buffered_file) } if (opt_buffered_file && fuzzed_file != nullptr) { bool setpos_fail = false; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, [&] { diff --git a/src/test/fuzz/chain.cpp b/src/test/fuzz/chain.cpp index 9f7074b423..0e12a55408 100644 --- a/src/test/fuzz/chain.cpp +++ b/src/test/fuzz/chain.cpp @@ -35,7 +35,7 @@ FUZZ_TARGET(chain) (void)CDiskBlockIndex{*disk_block_index}; (void)disk_block_index->BuildSkip(); - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const BlockStatus block_status = fuzzed_data_provider.PickValueInArray({ BlockStatus::BLOCK_VALID_UNKNOWN, BlockStatus::BLOCK_VALID_RESERVED, diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index 87e70861fa..2f33598348 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -51,7 +51,7 @@ FUZZ_TARGET_INIT(coins_view, initialize_coins_view) COutPoint random_out_point; Coin random_coin; CMutableTransaction random_mutable_transaction; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, [&] { @@ -114,7 +114,7 @@ FUZZ_TARGET_INIT(coins_view, initialize_coins_view) }, [&] { CCoinsMap coins_map; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CCoinsCacheEntry coins_cache_entry; coins_cache_entry.flags = fuzzed_data_provider.ConsumeIntegral<unsigned char>(); if (fuzzed_data_provider.ConsumeBool()) { @@ -221,8 +221,7 @@ FUZZ_TARGET_INIT(coins_view, initialize_coins_view) assert(expected_code_path); }, [&] { - (void)AreInputsStandard(CTransaction{random_mutable_transaction}, coins_view_cache, false); - (void)AreInputsStandard(CTransaction{random_mutable_transaction}, coins_view_cache, true); + (void)AreInputsStandard(CTransaction{random_mutable_transaction}, coins_view_cache); }, [&] { TxValidationState state; diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index d381345a0d..f87b6f1503 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <addrman.h> #include <chainparams.h> #include <chainparamsbase.h> #include <net.h> @@ -25,13 +26,13 @@ FUZZ_TARGET_INIT(connman, initialize_connman) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; SetMockTime(ConsumeTime(fuzzed_data_provider)); - AddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + AddrMan addrman(/*asmap=*/std::vector<bool>(), /*deterministic=*/false, /*consistency_check_ratio=*/0); CConnman connman{fuzzed_data_provider.ConsumeIntegral<uint64_t>(), fuzzed_data_provider.ConsumeIntegral<uint64_t>(), addrman, fuzzed_data_provider.ConsumeBool()}; CNetAddr random_netaddr; CNode random_node = ConsumeNode(fuzzed_data_provider); CSubNet random_subnet; std::string random_string; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, [&] { @@ -69,15 +70,15 @@ FUZZ_TARGET_INIT(connman, initialize_connman) }, [&] { (void)connman.GetAddresses( - /* max_addresses */ fuzzed_data_provider.ConsumeIntegral<size_t>(), - /* max_pct */ fuzzed_data_provider.ConsumeIntegral<size_t>(), - /* network */ std::nullopt); + /*max_addresses=*/fuzzed_data_provider.ConsumeIntegral<size_t>(), + /*max_pct=*/fuzzed_data_provider.ConsumeIntegral<size_t>(), + /*network=*/std::nullopt); }, [&] { (void)connman.GetAddresses( - /* requestor */ random_node, - /* max_addresses */ fuzzed_data_provider.ConsumeIntegral<size_t>(), - /* max_pct */ fuzzed_data_provider.ConsumeIntegral<size_t>()); + /*requestor=*/random_node, + /*max_addresses=*/fuzzed_data_provider.ConsumeIntegral<size_t>(), + /*max_pct=*/fuzzed_data_provider.ConsumeIntegral<size_t>()); }, [&] { (void)connman.GetDeterministicRandomizer(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); diff --git a/src/test/fuzz/crypto_aes256.cpp b/src/test/fuzz/crypto_aes256.cpp index ccabd1f7dc..0937026fdd 100644 --- a/src/test/fuzz/crypto_aes256.cpp +++ b/src/test/fuzz/crypto_aes256.cpp @@ -19,7 +19,7 @@ FUZZ_TARGET(crypto_aes256) AES256Encrypt encrypt{key.data()}; AES256Decrypt decrypt{key.data()}; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const std::vector<uint8_t> plaintext = ConsumeFixedLengthByteVector(fuzzed_data_provider, AES_BLOCKSIZE); std::vector<uint8_t> ciphertext(AES_BLOCKSIZE); encrypt.Encrypt(ciphertext.data(), plaintext.data()); diff --git a/src/test/fuzz/crypto_aes256cbc.cpp b/src/test/fuzz/crypto_aes256cbc.cpp index 6d4138e546..5fe67bd4da 100644 --- a/src/test/fuzz/crypto_aes256cbc.cpp +++ b/src/test/fuzz/crypto_aes256cbc.cpp @@ -21,7 +21,7 @@ FUZZ_TARGET(crypto_aes256cbc) AES256CBCEncrypt encrypt{key.data(), iv.data(), pad}; AES256CBCDecrypt decrypt{key.data(), iv.data(), pad}; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const std::vector<uint8_t> plaintext = ConsumeRandomLengthByteVector(fuzzed_data_provider); std::vector<uint8_t> ciphertext(plaintext.size() + AES_BLOCKSIZE); const int encrypt_ret = encrypt.Encrypt(plaintext.data(), plaintext.size(), ciphertext.data()); diff --git a/src/test/fuzz/crypto_chacha20.cpp b/src/test/fuzz/crypto_chacha20.cpp index 8adfa92420..3f552a8cda 100644 --- a/src/test/fuzz/crypto_chacha20.cpp +++ b/src/test/fuzz/crypto_chacha20.cpp @@ -19,7 +19,7 @@ FUZZ_TARGET(crypto_chacha20) const std::vector<unsigned char> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, fuzzed_data_provider.ConsumeIntegralInRange<size_t>(16, 32)); chacha20 = ChaCha20{key.data(), key.size()}; } - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, [&] { diff --git a/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp b/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp index bb4ef22158..5e60b0f25b 100644 --- a/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp +++ b/src/test/fuzz/crypto_chacha20_poly1305_aead.cpp @@ -28,7 +28,7 @@ FUZZ_TARGET(crypto_chacha20_poly1305_aead) std::vector<uint8_t> in(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + POLY1305_TAGLEN, 0); std::vector<uint8_t> out(buffer_size + CHACHA20_POLY1305_AEAD_AAD_LEN + POLY1305_TAGLEN, 0); bool is_encrypt = fuzzed_data_provider.ConsumeBool(); - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, [&] { diff --git a/src/test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp b/src/test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp index 8cb9c55283..24bcc03dfd 100644 --- a/src/test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp +++ b/src/test/fuzz/crypto_hkdf_hmac_sha256_l32.cpp @@ -18,7 +18,7 @@ FUZZ_TARGET(crypto_hkdf_hmac_sha256_l32) const std::vector<uint8_t> initial_key_material = ConsumeRandomLengthByteVector(fuzzed_data_provider); CHKDF_HMAC_SHA256_L32 hkdf_hmac_sha256_l32(initial_key_material.data(), initial_key_material.size(), fuzzed_data_provider.ConsumeRandomLengthString(1024)); - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { std::vector<uint8_t> out(32); hkdf_hmac_sha256_l32.Expand32(fuzzed_data_provider.ConsumeRandomLengthString(128), out.data()); } diff --git a/src/test/fuzz/cuckoocache.cpp b/src/test/fuzz/cuckoocache.cpp index a522c837ef..019afe1c47 100644 --- a/src/test/fuzz/cuckoocache.cpp +++ b/src/test/fuzz/cuckoocache.cpp @@ -37,7 +37,7 @@ FUZZ_TARGET(cuckoocache) } else { cuckoo_cache.setup(fuzzed_data_provider.ConsumeIntegralInRange<uint32_t>(0, 4096)); } - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { if (fuzzed_data_provider.ConsumeBool()) { cuckoo_cache.insert(fuzzed_data_provider.ConsumeBool()); } else { diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index a9325fa738..48574d71cc 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -189,7 +189,7 @@ FUZZ_TARGET_DESERIALIZE(blockmerkleroot, { BlockMerkleRoot(block, &mutated); }) FUZZ_TARGET_DESERIALIZE(addrman_deserialize, { - AddrMan am(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + AddrMan am(/*asmap=*/std::vector<bool>(), /*deterministic=*/false, /*consistency_check_ratio=*/0); DeserializeFromFuzzingInput(buffer, am); }) FUZZ_TARGET_DESERIALIZE(blockheader_deserialize, { diff --git a/src/test/fuzz/fees.cpp b/src/test/fuzz/fees.cpp index b5a07c7ba3..bcab66842c 100644 --- a/src/test/fuzz/fees.cpp +++ b/src/test/fuzz/fees.cpp @@ -18,7 +18,7 @@ FUZZ_TARGET(fees) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const CFeeRate minimal_incremental_fee{ConsumeMoney(fuzzed_data_provider)}; FeeFilterRounder fee_filter_rounder{minimal_incremental_fee}; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const CAmount current_minimum_fee = ConsumeMoney(fuzzed_data_provider); const CAmount rounded_fee = fee_filter_rounder.round(current_minimum_fee); assert(MoneyRange(rounded_fee)); diff --git a/src/test/fuzz/golomb_rice.cpp b/src/test/fuzz/golomb_rice.cpp index c99bf940c7..746347ac95 100644 --- a/src/test/fuzz/golomb_rice.cpp +++ b/src/test/fuzz/golomb_rice.cpp @@ -82,8 +82,8 @@ FUZZ_TARGET(golomb_rice) std::vector<uint64_t> decoded_deltas; { - VectorReader stream{SER_NETWORK, 0, golomb_rice_data, 0}; - BitStreamReader<VectorReader> bitreader(stream); + SpanReader stream{SER_NETWORK, 0, golomb_rice_data}; + BitStreamReader<SpanReader> bitreader{stream}; const uint32_t n = static_cast<uint32_t>(ReadCompactSize(stream)); for (uint32_t i = 0; i < n; ++i) { decoded_deltas.push_back(GolombRiceDecode(bitreader, BASIC_FILTER_P)); @@ -94,14 +94,14 @@ FUZZ_TARGET(golomb_rice) { const std::vector<uint8_t> random_bytes = ConsumeRandomLengthByteVector(fuzzed_data_provider, 1024); - VectorReader stream{SER_NETWORK, 0, random_bytes, 0}; + SpanReader stream{SER_NETWORK, 0, random_bytes}; uint32_t n; try { n = static_cast<uint32_t>(ReadCompactSize(stream)); } catch (const std::ios_base::failure&) { return; } - BitStreamReader<VectorReader> bitreader(stream); + BitStreamReader<SpanReader> bitreader{stream}; for (uint32_t i = 0; i < std::min<uint32_t>(n, 1024); ++i) { try { (void)GolombRiceDecode(bitreader, BASIC_FILTER_P); diff --git a/src/test/fuzz/merkleblock.cpp b/src/test/fuzz/merkleblock.cpp index 1eefd4c521..6271367a9c 100644 --- a/src/test/fuzz/merkleblock.cpp +++ b/src/test/fuzz/merkleblock.cpp @@ -34,7 +34,7 @@ FUZZ_TARGET(merkleblock) if (fuzzed_data_provider.ConsumeBool()) { merkle_block = CMerkleBlock{*opt_block, bloom_filter}; } else if (fuzzed_data_provider.ConsumeBool()) { - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { txids.insert(ConsumeUInt256(fuzzed_data_provider)); } merkle_block = CMerkleBlock{*opt_block, txids}; diff --git a/src/test/fuzz/minisketch.cpp b/src/test/fuzz/minisketch.cpp new file mode 100644 index 0000000000..93954bd3cf --- /dev/null +++ b/src/test/fuzz/minisketch.cpp @@ -0,0 +1,64 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <minisketch.h> +#include <node/minisketchwrapper.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <util/check.h> + +#include <map> +#include <numeric> + +FUZZ_TARGET(minisketch) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + const auto capacity{fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 200)}; + Minisketch sketch_a{Assert(MakeMinisketch32(capacity))}; + Minisketch sketch_b{Assert(MakeMinisketch32(capacity))}; + + // Fill two sets and keep the difference in a map + std::map<uint32_t, bool> diff; + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) + { + const auto entry{fuzzed_data_provider.ConsumeIntegralInRange<uint32_t>(1, std::numeric_limits<uint32_t>::max() - 1)}; + const auto KeepDiff{[&] { + bool& mut{diff[entry]}; + mut = !mut; + }}; + CallOneOf( + fuzzed_data_provider, + [&] { + sketch_a.Add(entry); + KeepDiff(); + }, + [&] { + sketch_b.Add(entry); + KeepDiff(); + }, + [&] { + sketch_a.Add(entry); + sketch_b.Add(entry); + }); + } + const auto num_diff{std::accumulate(diff.begin(), diff.end(), size_t{0}, [](auto n, const auto& e) { return n + e.second; })}; + + Minisketch sketch_ar{MakeMinisketch32(capacity)}; + Minisketch sketch_br{MakeMinisketch32(capacity)}; + sketch_ar.Deserialize(sketch_a.Serialize()); + sketch_br.Deserialize(sketch_b.Serialize()); + + Minisketch sketch_diff{std::move(fuzzed_data_provider.ConsumeBool() ? sketch_a : sketch_ar)}; + sketch_diff.Merge(fuzzed_data_provider.ConsumeBool() ? sketch_b : sketch_br); + + if (capacity >= num_diff) { + const auto max_elements{fuzzed_data_provider.ConsumeIntegralInRange<size_t>(num_diff, capacity)}; + const auto dec{*Assert(sketch_diff.Decode(max_elements))}; + Assert(dec.size() == num_diff); + for (auto d : dec) { + Assert(diff.at(d)); + } + } +} diff --git a/src/test/fuzz/net.cpp b/src/test/fuzz/net.cpp index bd1bb79d0e..fb11ea36ce 100644 --- a/src/test/fuzz/net.cpp +++ b/src/test/fuzz/net.cpp @@ -32,7 +32,7 @@ FUZZ_TARGET_INIT(net, initialize_net) SetMockTime(ConsumeTime(fuzzed_data_provider)); CNode node{ConsumeNode(fuzzed_data_provider)}; node.SetCommonVersion(fuzzed_data_provider.ConsumeIntegral<int>()); - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, [&] { diff --git a/src/test/fuzz/netbase_dns_lookup.cpp b/src/test/fuzz/netbase_dns_lookup.cpp index cf2fa33744..d01d413cff 100644 --- a/src/test/fuzz/netbase_dns_lookup.cpp +++ b/src/test/fuzz/netbase_dns_lookup.cpp @@ -22,7 +22,7 @@ FUZZ_TARGET(netbase_dns_lookup) auto fuzzed_dns_lookup_function = [&](const std::string&, bool) { std::vector<CNetAddr> resolved_addresses; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { resolved_addresses.push_back(ConsumeNetAddr(fuzzed_data_provider)); } return resolved_addresses; diff --git a/src/test/fuzz/node_eviction.cpp b/src/test/fuzz/node_eviction.cpp index a3f71426fa..64031fde42 100644 --- a/src/test/fuzz/node_eviction.cpp +++ b/src/test/fuzz/node_eviction.cpp @@ -18,20 +18,20 @@ FUZZ_TARGET(node_eviction) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; std::vector<NodeEvictionCandidate> eviction_candidates; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { eviction_candidates.push_back({ - /* id */ fuzzed_data_provider.ConsumeIntegral<NodeId>(), - /* nTimeConnected */ fuzzed_data_provider.ConsumeIntegral<int64_t>(), - /* m_min_ping_time */ std::chrono::microseconds{fuzzed_data_provider.ConsumeIntegral<int64_t>()}, - /* nLastBlockTime */ fuzzed_data_provider.ConsumeIntegral<int64_t>(), - /* nLastTXTime */ fuzzed_data_provider.ConsumeIntegral<int64_t>(), - /* fRelevantServices */ fuzzed_data_provider.ConsumeBool(), - /* fRelayTxes */ fuzzed_data_provider.ConsumeBool(), - /* fBloomFilter */ fuzzed_data_provider.ConsumeBool(), - /* nKeyedNetGroup */ fuzzed_data_provider.ConsumeIntegral<uint64_t>(), - /* prefer_evict */ fuzzed_data_provider.ConsumeBool(), - /* m_is_local */ fuzzed_data_provider.ConsumeBool(), - /* m_network */ fuzzed_data_provider.PickValueInArray(ALL_NETWORKS), + /*id=*/fuzzed_data_provider.ConsumeIntegral<NodeId>(), + /*nTimeConnected=*/fuzzed_data_provider.ConsumeIntegral<int64_t>(), + /*m_min_ping_time=*/std::chrono::microseconds{fuzzed_data_provider.ConsumeIntegral<int64_t>()}, + /*nLastBlockTime=*/fuzzed_data_provider.ConsumeIntegral<int64_t>(), + /*nLastTXTime=*/fuzzed_data_provider.ConsumeIntegral<int64_t>(), + /*fRelevantServices=*/fuzzed_data_provider.ConsumeBool(), + /*fRelayTxes=*/fuzzed_data_provider.ConsumeBool(), + /*fBloomFilter=*/fuzzed_data_provider.ConsumeBool(), + /*nKeyedNetGroup=*/fuzzed_data_provider.ConsumeIntegral<uint64_t>(), + /*prefer_evict=*/fuzzed_data_provider.ConsumeBool(), + /*m_is_local=*/fuzzed_data_provider.ConsumeBool(), + /*m_network=*/fuzzed_data_provider.PickValueInArray(ALL_NETWORKS), }); } // Make a copy since eviction_candidates may be in some valid but otherwise diff --git a/src/test/fuzz/p2p_transport_serialization.cpp b/src/test/fuzz/p2p_transport_serialization.cpp index edee5aeef7..29b7223c90 100644 --- a/src/test/fuzz/p2p_transport_serialization.cpp +++ b/src/test/fuzz/p2p_transport_serialization.cpp @@ -68,18 +68,16 @@ FUZZ_TARGET_INIT(p2p_transport_serialization, initialize_p2p_transport_serializa } if (deserializer.Complete()) { const std::chrono::microseconds m_time{std::numeric_limits<int64_t>::max()}; - uint32_t out_err_raw_size{0}; - std::optional<CNetMessage> result{deserializer.GetMessage(m_time, out_err_raw_size)}; - if (result) { - assert(result->m_command.size() <= CMessageHeader::COMMAND_SIZE); - assert(result->m_raw_message_size <= mutable_msg_bytes.size()); - assert(result->m_raw_message_size == CMessageHeader::HEADER_SIZE + result->m_message_size); - assert(result->m_time == m_time); + bool reject_message{false}; + CNetMessage msg = deserializer.GetMessage(m_time, reject_message); + assert(msg.m_command.size() <= CMessageHeader::COMMAND_SIZE); + assert(msg.m_raw_message_size <= mutable_msg_bytes.size()); + assert(msg.m_raw_message_size == CMessageHeader::HEADER_SIZE + msg.m_message_size); + assert(msg.m_time == m_time); - std::vector<unsigned char> header; - auto msg = CNetMsgMaker{result->m_recv.GetVersion()}.Make(result->m_command, MakeUCharSpan(result->m_recv)); - serializer.prepareForTransport(msg, header); - } + std::vector<unsigned char> header; + auto msg2 = CNetMsgMaker{msg.m_recv.GetVersion()}.Make(msg.m_command, MakeUCharSpan(msg.m_recv)); + serializer.prepareForTransport(msg2, header); } } } diff --git a/src/test/fuzz/policy_estimator.cpp b/src/test/fuzz/policy_estimator.cpp index 116b7a71d9..e4d95f72a0 100644 --- a/src/test/fuzz/policy_estimator.cpp +++ b/src/test/fuzz/policy_estimator.cpp @@ -24,7 +24,7 @@ FUZZ_TARGET_INIT(policy_estimator, initialize_policy_estimator) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); CBlockPolicyEstimator block_policy_estimator; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, [&] { @@ -35,12 +35,12 @@ FUZZ_TARGET_INIT(policy_estimator, initialize_policy_estimator) const CTransaction tx{*mtx}; block_policy_estimator.processTransaction(ConsumeTxMemPoolEntry(fuzzed_data_provider, tx), fuzzed_data_provider.ConsumeBool()); if (fuzzed_data_provider.ConsumeBool()) { - (void)block_policy_estimator.removeTx(tx.GetHash(), /* inBlock */ fuzzed_data_provider.ConsumeBool()); + (void)block_policy_estimator.removeTx(tx.GetHash(), /*inBlock=*/fuzzed_data_provider.ConsumeBool()); } }, [&] { std::vector<CTxMemPoolEntry> mempool_entries; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (!mtx) { break; @@ -56,7 +56,7 @@ FUZZ_TARGET_INIT(policy_estimator, initialize_policy_estimator) block_policy_estimator.processBlock(fuzzed_data_provider.ConsumeIntegral<unsigned int>(), ptrs); }, [&] { - (void)block_policy_estimator.removeTx(ConsumeUInt256(fuzzed_data_provider), /* inBlock */ fuzzed_data_provider.ConsumeBool()); + (void)block_policy_estimator.removeTx(ConsumeUInt256(fuzzed_data_provider), /*inBlock=*/fuzzed_data_provider.ConsumeBool()); }, [&] { block_policy_estimator.FlushUnconfirmed(); diff --git a/src/test/fuzz/pow.cpp b/src/test/fuzz/pow.cpp index 47b4323e81..1123c8c170 100644 --- a/src/test/fuzz/pow.cpp +++ b/src/test/fuzz/pow.cpp @@ -27,7 +27,7 @@ FUZZ_TARGET_INIT(pow, initialize_pow) std::vector<CBlockIndex> blocks; const uint32_t fixed_time = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); const uint32_t fixed_bits = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); - while (fuzzed_data_provider.remaining_bytes() > 0) { + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes() > 0, 10000) { const std::optional<CBlockHeader> block_header = ConsumeDeserializable<CBlockHeader>(fuzzed_data_provider); if (!block_header) { continue; diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 7b99193ad0..94a71859e9 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -83,7 +83,7 @@ void fuzz_target(FuzzBufferType buffer, const std::string& LIMIT_TO_MESSAGE_TYPE p2p_node.fSuccessfullyConnected = successfully_connected; connman.AddTestNode(p2p_node); g_setup->m_node.peerman->InitializeNode(&p2p_node); - FillNode(fuzzed_data_provider, p2p_node, /* init_version */ successfully_connected); + FillNode(fuzzed_data_provider, p2p_node, /*init_version=*/successfully_connected); const auto mock_time = ConsumeTime(fuzzed_data_provider); SetMockTime(mock_time); diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index 11b236c9bd..21a959315e 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -50,12 +50,12 @@ FUZZ_TARGET_INIT(process_messages, initialize_process_messages) p2p_node.fSuccessfullyConnected = successfully_connected; p2p_node.fPauseSend = false; g_setup->m_node.peerman->InitializeNode(&p2p_node); - FillNode(fuzzed_data_provider, p2p_node, /* init_version */ successfully_connected); + FillNode(fuzzed_data_provider, p2p_node, /*init_version=*/successfully_connected); connman.AddTestNode(p2p_node); } - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const std::string random_message_type{fuzzed_data_provider.ConsumeBytesAsString(CMessageHeader::COMMAND_SIZE).c_str()}; const auto mock_time = ConsumeTime(fuzzed_data_provider); diff --git a/src/test/fuzz/rbf.cpp b/src/test/fuzz/rbf.cpp index 26c89a70c3..990bce5f6c 100644 --- a/src/test/fuzz/rbf.cpp +++ b/src/test/fuzz/rbf.cpp @@ -24,7 +24,7 @@ FUZZ_TARGET(rbf) return; } CTxMemPool pool; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const std::optional<CMutableTransaction> another_mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); if (!another_mtx) { break; diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 9195cc4873..790421d60c 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -41,13 +41,17 @@ struct RPCFuzzTestingSetup : public TestingSetup { { } - UniValue CallRPC(const std::string& rpc_method, const std::vector<std::string>& arguments) + void CallRPC(const std::string& rpc_method, const std::vector<std::string>& arguments) { JSONRPCRequest request; request.context = &m_node; request.strMethod = rpc_method; - request.params = RPCConvertValues(rpc_method, arguments); - return tableRPC.execute(request); + try { + request.params = RPCConvertValues(rpc_method, arguments); + } catch (const std::runtime_error&) { + return; + } + tableRPC.execute(request); } std::vector<std::string> GetRPCCommands() const @@ -294,7 +298,7 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) std::string ConsumeArrayRPCArgument(FuzzedDataProvider& fuzzed_data_provider) { std::vector<std::string> scalar_arguments; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) { scalar_arguments.push_back(ConsumeScalarRPCArgument(fuzzed_data_provider)); } return "[\"" + Join(scalar_arguments, "\",\"") + "\"]"; @@ -348,12 +352,16 @@ FUZZ_TARGET_INIT(rpc, initialize_rpc) return; } std::vector<std::string> arguments; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) { arguments.push_back(ConsumeRPCArgument(fuzzed_data_provider)); } try { rpc_testing_setup->CallRPC(rpc_command, arguments); - } catch (const UniValue&) { - } catch (const std::runtime_error&) { + } catch (const UniValue& json_rpc_error) { + const std::string error_msg{find_value(json_rpc_error, "message").get_str()}; + if (error_msg.find("Internal bug detected") != std::string::npos) { + // Only allow the intentional internal bug + assert(error_msg.find("trigger_internal_bug") != std::string::npos); + } } } diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index 74c576322a..eb170aab76 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -41,9 +41,7 @@ void initialize_script() FUZZ_TARGET_INIT(script, initialize_script) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - const std::optional<CScript> script_opt = ConsumeDeserializable<CScript>(fuzzed_data_provider); - if (!script_opt) return; - const CScript script{*script_opt}; + const CScript script{ConsumeScript(fuzzed_data_provider)}; CompressedScript compressed; if (CompressScript(script, compressed)) { @@ -166,7 +164,7 @@ FUZZ_TARGET_INIT(script, initialize_script) const std::string encoded_dest{EncodeDestination(tx_destination_1)}; const UniValue json_dest{DescribeAddress(tx_destination_1)}; Assert(tx_destination_1 == DecodeDestination(encoded_dest)); - (void)GetKeyForDestination(/* store */ {}, tx_destination_1); + (void)GetKeyForDestination(/*store=*/{}, tx_destination_1); const CScript dest{GetScriptForDestination(tx_destination_1)}; const bool valid{IsValidDestination(tx_destination_1)}; Assert(dest.empty() != valid); diff --git a/src/test/fuzz/script_assets_test_minimizer.cpp b/src/test/fuzz/script_assets_test_minimizer.cpp index a80338b965..00a3bed12f 100644 --- a/src/test/fuzz/script_assets_test_minimizer.cpp +++ b/src/test/fuzz/script_assets_test_minimizer.cpp @@ -54,7 +54,7 @@ CMutableTransaction TxFromHex(const std::string& str) { CMutableTransaction tx; try { - VectorReader(SER_DISK, SERIALIZE_TRANSACTION_NO_WITNESS, CheckedParseHex(str), 0) >> tx; + SpanReader{SER_DISK, SERIALIZE_TRANSACTION_NO_WITNESS, CheckedParseHex(str)} >> tx; } catch (const std::ios_base::failure&) { throw std::runtime_error("Tx deserialization failure"); } @@ -68,7 +68,7 @@ std::vector<CTxOut> TxOutsFromJSON(const UniValue& univalue) for (size_t i = 0; i < univalue.size(); ++i) { CTxOut txout; try { - VectorReader(SER_DISK, 0, CheckedParseHex(univalue[i].get_str()), 0) >> txout; + SpanReader{SER_DISK, 0, CheckedParseHex(univalue[i].get_str())} >> txout; } catch (const std::ios_base::failure&) { throw std::runtime_error("Prevout invalid format"); } @@ -190,7 +190,7 @@ void test_init() static ECCVerifyHandle handle; } -FUZZ_TARGET_INIT_HIDDEN(script_assets_test_minimizer, test_init, /* hidden */ true) +FUZZ_TARGET_INIT_HIDDEN(script_assets_test_minimizer, test_init, /*hidden=*/true) { if (buffer.size() < 2 || buffer.back() != '\n' || buffer[buffer.size() - 2] != ',') return; const std::string str((const char*)buffer.data(), buffer.size() - 2); diff --git a/src/test/fuzz/script_descriptor_cache.cpp b/src/test/fuzz/script_descriptor_cache.cpp index 6ce13d5679..a90ad5e8ed 100644 --- a/src/test/fuzz/script_descriptor_cache.cpp +++ b/src/test/fuzz/script_descriptor_cache.cpp @@ -17,7 +17,7 @@ FUZZ_TARGET(script_descriptor_cache) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); DescriptorCache descriptor_cache; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const std::vector<uint8_t> code = fuzzed_data_provider.ConsumeBytes<uint8_t>(BIP32_EXTKEY_SIZE); if (code.size() == BIP32_EXTKEY_SIZE) { CExtPubKey xpub; diff --git a/src/test/fuzz/script_ops.cpp b/src/test/fuzz/script_ops.cpp index 4bc709ed35..12247679f2 100644 --- a/src/test/fuzz/script_ops.cpp +++ b/src/test/fuzz/script_ops.cpp @@ -15,7 +15,7 @@ FUZZ_TARGET(script_ops) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); CScript script_mut = ConsumeScript(fuzzed_data_provider); - while (fuzzed_data_provider.remaining_bytes() > 0) { + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes() > 0, 1000000) { CallOneOf( fuzzed_data_provider, [&] { diff --git a/src/test/fuzz/script_sign.cpp b/src/test/fuzz/script_sign.cpp index 684324c36e..79380bd9c9 100644 --- a/src/test/fuzz/script_sign.cpp +++ b/src/test/fuzz/script_sign.cpp @@ -48,7 +48,7 @@ FUZZ_TARGET_INIT(script_sign, initialize_script_sign) { std::map<CPubKey, KeyOriginInfo> hd_keypaths; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const std::optional<CPubKey> pub_key = ConsumeDeserializable<CPubKey>(fuzzed_data_provider); if (!pub_key) { break; @@ -125,7 +125,7 @@ FUZZ_TARGET_INIT(script_sign, initialize_script_sign) (void)signature_creator.CreateSig(provider, vch_sig, address, ConsumeScript(fuzzed_data_provider), fuzzed_data_provider.PickValueInArray({SigVersion::BASE, SigVersion::WITNESS_V0})); } std::map<COutPoint, Coin> coins; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { const std::optional<COutPoint> outpoint = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); if (!outpoint) { break; diff --git a/src/test/fuzz/scriptnum_ops.cpp b/src/test/fuzz/scriptnum_ops.cpp index 62ed50d13f..0681aaf949 100644 --- a/src/test/fuzz/scriptnum_ops.cpp +++ b/src/test/fuzz/scriptnum_ops.cpp @@ -28,7 +28,7 @@ FUZZ_TARGET(scriptnum_ops) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); CScriptNum script_num = ConsumeScriptNum(fuzzed_data_provider); - while (fuzzed_data_provider.remaining_bytes() > 0) { + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes() > 0, 1000000) { CallOneOf( fuzzed_data_provider, [&] { diff --git a/src/test/fuzz/signature_checker.cpp b/src/test/fuzz/signature_checker.cpp index 6b86c8889d..c3f416632d 100644 --- a/src/test/fuzz/signature_checker.cpp +++ b/src/test/fuzz/signature_checker.cpp @@ -58,8 +58,8 @@ FUZZ_TARGET_INIT(signature_checker, initialize_signature_checker) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const unsigned int flags = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); const SigVersion sig_version = fuzzed_data_provider.PickValueInArray({SigVersion::BASE, SigVersion::WITNESS_V0}); - const auto script_1 = ConsumeScript(fuzzed_data_provider, 65536); - const auto script_2 = ConsumeScript(fuzzed_data_provider, 65536); + const auto script_1{ConsumeScript(fuzzed_data_provider)}; + const auto script_2{ConsumeScript(fuzzed_data_provider)}; std::vector<std::vector<unsigned char>> stack; (void)EvalScript(stack, script_1, flags, FuzzedSignatureChecker(fuzzed_data_provider), sig_version, nullptr); if (!IsValidFlagCombination(flags)) { diff --git a/src/test/fuzz/torcontrol.cpp b/src/test/fuzz/torcontrol.cpp index a97d3962bf..a78715f769 100644 --- a/src/test/fuzz/torcontrol.cpp +++ b/src/test/fuzz/torcontrol.cpp @@ -44,7 +44,7 @@ FUZZ_TARGET_INIT(torcontrol, initialize_torcontrol) FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; TorController tor_controller; - while (fuzzed_data_provider.ConsumeBool()) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { TorControlReply tor_control_reply; CallOneOf( fuzzed_data_provider, diff --git a/src/test/fuzz/transaction.cpp b/src/test/fuzz/transaction.cpp index a21e5cea0c..389da6f5d7 100644 --- a/src/test/fuzz/transaction.cpp +++ b/src/test/fuzz/transaction.cpp @@ -98,11 +98,10 @@ FUZZ_TARGET_INIT(transaction, initialize_transaction) CCoinsView coins_view; const CCoinsViewCache coins_view_cache(&coins_view); - (void)AreInputsStandard(tx, coins_view_cache, false); - (void)AreInputsStandard(tx, coins_view_cache, true); + (void)AreInputsStandard(tx, coins_view_cache); (void)IsWitnessStandard(tx, coins_view_cache); UniValue u(UniValue::VOBJ); - TxToUniv(tx, /* hashBlock */ uint256::ZERO, u); - TxToUniv(tx, /* hashBlock */ uint256::ONE, u); + TxToUniv(tx, /*hashBlock=*/uint256::ZERO, u); + TxToUniv(tx, /*hashBlock=*/uint256::ONE, u); } diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index 17b5ef88b9..c32c965ab0 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/validation.h> -#include <miner.h> +#include <node/miner.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> @@ -85,15 +85,15 @@ void Finish(FuzzedDataProvider& fuzzed_data_provider, MockedTxPool& tx_pool, CCh { BlockAssembler::Options options; options.nBlockMaxWeight = fuzzed_data_provider.ConsumeIntegralInRange(0U, MAX_BLOCK_WEIGHT); - options.blockMinFeeRate = CFeeRate{ConsumeMoney(fuzzed_data_provider, /* max */ COIN)}; - auto assembler = BlockAssembler{chainstate, *static_cast<CTxMemPool*>(&tx_pool), ::Params(), options}; + options.blockMinFeeRate = CFeeRate{ConsumeMoney(fuzzed_data_provider, /*max=*/COIN)}; + auto assembler = BlockAssembler{chainstate, *static_cast<CTxMemPool*>(&tx_pool), chainstate.m_params, options}; auto block_template = assembler.CreateNewBlock(CScript{} << OP_TRUE); Assert(block_template->block.vtx.size() >= 1); } const auto info_all = tx_pool.infoAll(); if (!info_all.empty()) { const auto& tx_to_remove = *PickValue(fuzzed_data_provider, info_all).tx; - WITH_LOCK(tx_pool.cs, tx_pool.removeRecursive(tx_to_remove, /* dummy */ MemPoolRemovalReason::BLOCK)); + WITH_LOCK(tx_pool.cs, tx_pool.removeRecursive(tx_to_remove, MemPoolRemovalReason::BLOCK /* dummy */)); std::vector<uint256> all_txids; tx_pool.queryHashes(all_txids); assert(all_txids.size() < info_all.size()); @@ -131,7 +131,7 @@ FUZZ_TARGET_INIT(tx_pool_standard, initialize_tx_pool) // The sum of the values of all spendable outpoints constexpr CAmount SUPPLY_TOTAL{COINBASE_MATURITY * 50 * COIN}; - CTxMemPool tx_pool_{/* estimator */ nullptr, /* check_ratio */ 1}; + CTxMemPool tx_pool_{/*estimator=*/nullptr, /*check_ratio=*/1}; MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(&tx_pool_); // Helper to query an amount @@ -224,13 +224,13 @@ FUZZ_TARGET_INIT(tx_pool_standard, initialize_tx_pool) // Make sure ProcessNewPackage on one transaction works and always fully validates the transaction. // The result is not guaranteed to be the same as what is returned by ATMP. const auto result_package = WITH_LOCK(::cs_main, - return ProcessNewPackage(node.chainman->ActiveChainstate(), tx_pool, {tx}, true)); + return ProcessNewPackage(chainstate, tx_pool, {tx}, true)); auto it = result_package.m_tx_results.find(tx->GetWitnessHash()); Assert(it != result_package.m_tx_results.end()); Assert(it->second.m_result_type == MempoolAcceptResult::ResultType::VALID || it->second.m_result_type == MempoolAcceptResult::ResultType::INVALID); - const auto res = WITH_LOCK(::cs_main, return AcceptToMemoryPool(chainstate, tx_pool, tx, bypass_limits)); + const auto res = WITH_LOCK(::cs_main, return AcceptToMemoryPool(tx_pool, chainstate, tx, GetTime(), bypass_limits, /* test_accept= */ false)); const bool accepted = res.m_result_type == MempoolAcceptResult::ResultType::VALID; SyncWithValidationInterfaceQueue(); UnregisterSharedValidationInterface(txr); @@ -267,10 +267,10 @@ FUZZ_TARGET_INIT(tx_pool_standard, initialize_tx_pool) // Outpoints that no longer count toward the total supply std::set<COutPoint> consumed_supply; for (const auto& removed_tx : removed) { - insert_tx(/* created_by_tx */ {consumed_erased}, /* consumed_by_tx */ {outpoints_supply}, /* tx */ *removed_tx); + insert_tx(/*created_by_tx=*/{consumed_erased}, /*consumed_by_tx=*/{outpoints_supply}, /*tx=*/*removed_tx); } for (const auto& added_tx : added) { - insert_tx(/* created_by_tx */ {outpoints_supply, outpoints_rbf}, /* consumed_by_tx */ {consumed_supply}, /* tx */ *added_tx); + insert_tx(/*created_by_tx=*/{outpoints_supply, outpoints_rbf}, /*consumed_by_tx=*/{consumed_supply}, /*tx=*/*added_tx); } for (const auto& p : consumed_erased) { Assert(outpoints_supply.erase(p) == 1); @@ -303,7 +303,7 @@ FUZZ_TARGET_INIT(tx_pool, initialize_tx_pool) txids.push_back(ConsumeUInt256(fuzzed_data_provider)); } - CTxMemPool tx_pool_{/* estimator */ nullptr, /* check_ratio */ 1}; + CTxMemPool tx_pool_{/*estimator=*/nullptr, /*check_ratio=*/1}; MockedTxPool& tx_pool = *static_cast<MockedTxPool*>(&tx_pool_); LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 300) @@ -330,7 +330,7 @@ FUZZ_TARGET_INIT(tx_pool, initialize_tx_pool) const auto tx = MakeTransactionRef(mut_tx); const bool bypass_limits = fuzzed_data_provider.ConsumeBool(); ::fRequireStandard = fuzzed_data_provider.ConsumeBool(); - const auto res = WITH_LOCK(::cs_main, return AcceptToMemoryPool(node.chainman->ActiveChainstate(), tx_pool, tx, bypass_limits)); + const auto res = WITH_LOCK(::cs_main, return AcceptToMemoryPool(tx_pool, chainstate, tx, GetTime(), bypass_limits, /* test_accept= */ false)); const bool accepted = res.m_result_type == MempoolAcceptResult::ResultType::VALID; if (accepted) { txids.push_back(tx->GetHash()); diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index d83d2924bb..ae5f7a379e 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -267,7 +267,7 @@ CMutableTransaction ConsumeTransaction(FuzzedDataProvider& fuzzed_data_provider, const auto amount = fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(-10, 50 * COIN + 10); const auto script_pk = p2wsh_op_true ? P2WSH_OP_TRUE : - ConsumeScript(fuzzed_data_provider, /* max_length */ 128, /* maybe_p2wsh */ true); + ConsumeScript(fuzzed_data_provider, /*maybe_p2wsh=*/true); tx_mut.vout.emplace_back(amount, script_pk); } return tx_mut; @@ -283,10 +283,63 @@ CScriptWitness ConsumeScriptWitness(FuzzedDataProvider& fuzzed_data_provider, co return ret; } -CScript ConsumeScript(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length, const bool maybe_p2wsh) noexcept +CScript ConsumeScript(FuzzedDataProvider& fuzzed_data_provider, const bool maybe_p2wsh) noexcept { - const std::vector<uint8_t> b = ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length); - CScript r_script{b.begin(), b.end()}; + CScript r_script{}; + { + // Keep a buffer of bytes to allow the fuzz engine to produce smaller + // inputs to generate CScripts with repeated data. + static constexpr unsigned MAX_BUFFER_SZ{128}; + std::vector<uint8_t> buffer(MAX_BUFFER_SZ, uint8_t{'a'}); + while (fuzzed_data_provider.ConsumeBool()) { + CallOneOf( + fuzzed_data_provider, + [&] { + // Insert byte vector directly to allow malformed or unparsable scripts + r_script.insert(r_script.end(), buffer.begin(), buffer.begin() + fuzzed_data_provider.ConsumeIntegralInRange(0U, MAX_BUFFER_SZ)); + }, + [&] { + // Push a byte vector from the buffer + r_script << std::vector<uint8_t>{buffer.begin(), buffer.begin() + fuzzed_data_provider.ConsumeIntegralInRange(0U, MAX_BUFFER_SZ)}; + }, + [&] { + // Push multisig + // There is a special case for this to aid the fuzz engine + // navigate the highly structured multisig format. + r_script << fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(0, 22); + int num_data{fuzzed_data_provider.ConsumeIntegralInRange(1, 22)}; + std::vector<uint8_t> pubkey_comp{buffer.begin(), buffer.begin() + CPubKey::COMPRESSED_SIZE}; + pubkey_comp.front() = fuzzed_data_provider.ConsumeIntegralInRange(2, 3); // Set first byte for GetLen() to pass + std::vector<uint8_t> pubkey_uncomp{buffer.begin(), buffer.begin() + CPubKey::SIZE}; + pubkey_uncomp.front() = fuzzed_data_provider.ConsumeIntegralInRange(4, 7); // Set first byte for GetLen() to pass + while (num_data--) { + auto& pubkey{fuzzed_data_provider.ConsumeBool() ? pubkey_uncomp : pubkey_comp}; + if (fuzzed_data_provider.ConsumeBool()) { + pubkey.back() = num_data; // Make each pubkey different + } + r_script << pubkey; + } + r_script << fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(0, 22); + }, + [&] { + // Mutate the buffer + const auto vec{ConsumeRandomLengthByteVector(fuzzed_data_provider, /*max_length=*/MAX_BUFFER_SZ)}; + std::copy(vec.begin(), vec.end(), buffer.begin()); + }, + [&] { + // Push an integral + r_script << fuzzed_data_provider.ConsumeIntegral<int64_t>(); + }, + [&] { + // Push an opcode + r_script << ConsumeOpcodeType(fuzzed_data_provider); + }, + [&] { + // Push a scriptnum + r_script << ConsumeScriptNum(fuzzed_data_provider); + }); + } + } if (maybe_p2wsh && fuzzed_data_provider.ConsumeBool()) { uint256 script_hash; CSHA256().Write(r_script.data(), r_script.size()).Finalize(script_hash.begin()); diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 1bc6f1db45..40aaeac63f 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -132,7 +132,7 @@ template <typename WeakEnumType, size_t size> [[nodiscard]] CScriptWitness ConsumeScriptWitness(FuzzedDataProvider& fuzzed_data_provider, const size_t max_stack_elem_size = 32) noexcept; -[[nodiscard]] CScript ConsumeScript(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length = std::nullopt, const bool maybe_p2wsh = false) noexcept; +[[nodiscard]] CScript ConsumeScript(FuzzedDataProvider& fuzzed_data_provider, const bool maybe_p2wsh = false) noexcept; [[nodiscard]] uint32_t ConsumeSequence(FuzzedDataProvider& fuzzed_data_provider) noexcept; diff --git a/src/test/fuzz/utxo_snapshot.cpp b/src/test/fuzz/utxo_snapshot.cpp index 8d2a06f11a..1b9f0c8a02 100644 --- a/src/test/fuzz/utxo_snapshot.cpp +++ b/src/test/fuzz/utxo_snapshot.cpp @@ -38,7 +38,7 @@ FUZZ_TARGET_INIT(utxo_snapshot, initialize_chain) { CAutoFile outfile{fsbridge::fopen(snapshot_path, "wb"), SER_DISK, CLIENT_VERSION}; const auto file_data{ConsumeRandomLengthByteVector(fuzzed_data_provider)}; - outfile << Span<const uint8_t>{file_data}; + outfile << Span{file_data}; } const auto ActivateFuzzedSnapshot{[&] { @@ -49,7 +49,7 @@ FUZZ_TARGET_INIT(utxo_snapshot, initialize_chain) } catch (const std::ios_base::failure&) { return false; } - return chainman.ActivateSnapshot(infile, metadata, /* in_memory */ true); + return chainman.ActivateSnapshot(infile, metadata, /*in_memory=*/true); }}; if (fuzzed_data_provider.ConsumeBool()) { diff --git a/src/test/fuzz/versionbits.cpp b/src/test/fuzz/versionbits.cpp index 73a7d24971..cf95c0b9bf 100644 --- a/src/test/fuzz/versionbits.cpp +++ b/src/test/fuzz/versionbits.cpp @@ -199,7 +199,7 @@ FUZZ_TARGET_INIT(versionbits, initialize) const uint32_t signalling_mask = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); // mine prior periods - while (fuzzed_data_provider.remaining_bytes() > 0) { + while (fuzzed_data_provider.remaining_bytes() > 0) { // early exit; no need for LIMITED_WHILE // all blocks in these periods either do or don't signal bool signal = fuzzed_data_provider.ConsumeBool(); for (int b = 0; b < period; ++b) { diff --git a/src/test/key_io_tests.cpp b/src/test/key_io_tests.cpp index 8629d13840..02268dbcf5 100644 --- a/src/test/key_io_tests.cpp +++ b/src/test/key_io_tests.cpp @@ -46,7 +46,7 @@ BOOST_AUTO_TEST_CASE(key_io_valid_parse) privkey = DecodeSecret(exp_base58string); BOOST_CHECK_MESSAGE(privkey.IsValid(), "!IsValid:" + strTest); BOOST_CHECK_MESSAGE(privkey.IsCompressed() == isCompressed, "compressed mismatch:" + strTest); - BOOST_CHECK_MESSAGE(privkey.size() == exp_payload.size() && std::equal(privkey.begin(), privkey.end(), exp_payload.begin()), "key mismatch:" + strTest); + BOOST_CHECK_MESSAGE(Span{privkey} == Span{exp_payload}, "key mismatch:" + strTest); // Private key must be invalid public key destination = DecodeDestination(exp_base58string); diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index b915982d98..2769dde367 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -321,7 +321,7 @@ BOOST_AUTO_TEST_CASE(bip340_test_vectors) key.Set(sec.begin(), sec.end(), true); XOnlyPubKey pubkey(key.GetPubKey()); BOOST_CHECK(std::equal(pubkey.begin(), pubkey.end(), pub.begin(), pub.end())); - bool ok = key.SignSchnorr(msg256, sig64, nullptr, &aux256); + bool ok = key.SignSchnorr(msg256, sig64, nullptr, aux256); BOOST_CHECK(ok); BOOST_CHECK(std::vector<unsigned char>(sig64, sig64 + 64) == sig); // Verify those signatures for good measure. @@ -337,7 +337,7 @@ BOOST_AUTO_TEST_CASE(bip340_test_vectors) BOOST_CHECK(tweaked); XOnlyPubKey tweaked_key = tweaked->first; aux256 = InsecureRand256(); - bool ok = key.SignSchnorr(msg256, sig64, &merkle_root, &aux256); + bool ok = key.SignSchnorr(msg256, sig64, &merkle_root, aux256); BOOST_CHECK(ok); BOOST_CHECK(tweaked_key.VerifySchnorr(msg256, sig64)); } diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index b3497b8ef8..005752d508 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -602,7 +602,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) // // [tx1] // - CTransactionRef tx1 = make_tx(/* output_values */ {10 * COIN}); + CTransactionRef tx1 = make_tx(/*output_values=*/{10 * COIN}); pool.addUnchecked(entry.Fee(10000LL).FromTx(tx1)); // Ancestors / descendants should be 1 / 1 (itself / itself) @@ -614,7 +614,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) // // [tx1].0 <- [tx2] // - CTransactionRef tx2 = make_tx(/* output_values */ {495 * CENT, 5 * COIN}, /* inputs */ {tx1}); + CTransactionRef tx2 = make_tx(/*output_values=*/{495 * CENT, 5 * COIN}, /*inputs=*/{tx1}); pool.addUnchecked(entry.Fee(10000LL).FromTx(tx2)); // Ancestors / descendants should be: @@ -633,7 +633,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) // // [tx1].0 <- [tx2].0 <- [tx3] // - CTransactionRef tx3 = make_tx(/* output_values */ {290 * CENT, 200 * CENT}, /* inputs */ {tx2}); + CTransactionRef tx3 = make_tx(/*output_values=*/{290 * CENT, 200 * CENT}, /*inputs=*/{tx2}); pool.addUnchecked(entry.Fee(10000LL).FromTx(tx3)); // Ancestors / descendants should be: @@ -658,7 +658,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) // | // \---1 <- [tx4] // - CTransactionRef tx4 = make_tx(/* output_values */ {290 * CENT, 250 * CENT}, /* inputs */ {tx2}, /* input_indices */ {1}); + CTransactionRef tx4 = make_tx(/*output_values=*/{290 * CENT, 250 * CENT}, /*inputs=*/{tx2}, /*input_indices=*/{1}); pool.addUnchecked(entry.Fee(10000LL).FromTx(tx4)); // Ancestors / descendants should be: @@ -694,14 +694,14 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) CAmount v = 5 * COIN; for (uint64_t i = 0; i < 5; i++) { CTransactionRef& tyi = *ty[i]; - tyi = make_tx(/* output_values */ {v}, /* inputs */ i > 0 ? std::vector<CTransactionRef>{*ty[i - 1]} : std::vector<CTransactionRef>{}); + tyi = make_tx(/*output_values=*/{v}, /*inputs=*/i > 0 ? std::vector<CTransactionRef>{*ty[i - 1]} : std::vector<CTransactionRef>{}); v -= 50 * CENT; pool.addUnchecked(entry.Fee(10000LL).FromTx(tyi)); pool.GetTransactionAncestry(tyi->GetHash(), ancestors, descendants); BOOST_CHECK_EQUAL(ancestors, i+1); BOOST_CHECK_EQUAL(descendants, i+1); } - CTransactionRef ty6 = make_tx(/* output_values */ {5 * COIN}, /* inputs */ {tx3, ty5}); + CTransactionRef ty6 = make_tx(/*output_values=*/{5 * COIN}, /*inputs=*/{tx3, ty5}); pool.addUnchecked(entry.Fee(10000LL).FromTx(ty6)); // Ancestors / descendants should be: @@ -755,10 +755,10 @@ BOOST_AUTO_TEST_CASE(MempoolAncestryTests) // \---1 <- [tc].0 --<--/ // CTransactionRef ta, tb, tc, td; - ta = make_tx(/* output_values */ {10 * COIN}); - tb = make_tx(/* output_values */ {5 * COIN, 3 * COIN}, /* inputs */ {ta}); - tc = make_tx(/* output_values */ {2 * COIN}, /* inputs */ {tb}, /* input_indices */ {1}); - td = make_tx(/* output_values */ {6 * COIN}, /* inputs */ {tb, tc}, /* input_indices */ {0, 0}); + ta = make_tx(/*output_values=*/{10 * COIN}); + tb = make_tx(/*output_values=*/{5 * COIN, 3 * COIN}, /*inputs=*/ {ta}); + tc = make_tx(/*output_values=*/{2 * COIN}, /*inputs=*/{tb}, /*input_indices=*/{1}); + td = make_tx(/*output_values=*/{6 * COIN}, /*inputs=*/{tb, tc}, /*input_indices=*/{0, 0}); pool.clear(); pool.addUnchecked(entry.Fee(10000LL).FromTx(ta)); pool.addUnchecked(entry.Fee(10000LL).FromTx(tb)); diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 7f44dcf20e..bdc6ff6130 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -7,7 +7,7 @@ #include <consensus/consensus.h> #include <consensus/merkle.h> #include <consensus/tx_verify.h> -#include <miner.h> +#include <node/miner.h> #include <policy/policy.h> #include <script/standard.h> #include <txmempool.h> diff --git a/src/test/minisketch_tests.cpp b/src/test/minisketch_tests.cpp new file mode 100644 index 0000000000..f7dd18923b --- /dev/null +++ b/src/test/minisketch_tests.cpp @@ -0,0 +1,49 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <minisketch.h> +#include <node/minisketchwrapper.h> +#include <random.h> +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +#include <utility> + +BOOST_AUTO_TEST_SUITE(minisketch_tests) + +BOOST_AUTO_TEST_CASE(minisketch_test) +{ + for (int i = 0; i < 100; ++i) { + uint32_t errors = 0 + InsecureRandRange(11); + uint32_t start_a = 1 + InsecureRandRange(1000000000); + uint32_t a_not_b = InsecureRandRange(errors + 1); + uint32_t b_not_a = errors - a_not_b; + uint32_t both = InsecureRandRange(10000); + uint32_t end_a = start_a + a_not_b + both; + uint32_t start_b = start_a + a_not_b; + uint32_t end_b = start_b + both + b_not_a; + + Minisketch sketch_a = MakeMinisketch32(10); + for (uint32_t a = start_a; a < end_a; ++a) sketch_a.Add(a); + Minisketch sketch_b = MakeMinisketch32(10); + for (uint32_t b = start_b; b < end_b; ++b) sketch_b.Add(b); + + Minisketch sketch_ar = MakeMinisketch32(10); + Minisketch sketch_br = MakeMinisketch32(10); + sketch_ar.Deserialize(sketch_a.Serialize()); + sketch_br.Deserialize(sketch_b.Serialize()); + + Minisketch sketch_c = std::move(sketch_ar); + sketch_c.Merge(sketch_br); + auto dec = sketch_c.Decode(errors); + BOOST_CHECK(dec.has_value()); + auto sols = std::move(*dec); + std::sort(sols.begin(), sols.end()); + for (uint32_t i = 0; i < a_not_b; ++i) BOOST_CHECK_EQUAL(sols[i], start_a + i); + for (uint32_t i = 0; i < b_not_a; ++i) BOOST_CHECK_EQUAL(sols[i + a_not_b], start_b + both + i); + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/net_peer_eviction_tests.cpp b/src/test/net_peer_eviction_tests.cpp index 5eb280b498..9470ed814d 100644 --- a/src/test/net_peer_eviction_tests.cpp +++ b/src/test/net_peer_eviction_tests.cpp @@ -72,8 +72,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_is_local = false; c.m_network = NET_IPV4; }, - /* protected_peer_ids */ {0, 1, 2, 3, 4, 5}, - /* unprotected_peer_ids */ {6, 7, 8, 9, 10, 11}, + /*protected_peer_ids=*/{0, 1, 2, 3, 4, 5}, + /*unprotected_peer_ids=*/{6, 7, 8, 9, 10, 11}, random_context)); // Verify in the opposite direction. @@ -83,8 +83,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_is_local = false; c.m_network = NET_IPV6; }, - /* protected_peer_ids */ {6, 7, 8, 9, 10, 11}, - /* unprotected_peer_ids */ {0, 1, 2, 3, 4, 5}, + /*protected_peer_ids=*/{6, 7, 8, 9, 10, 11}, + /*unprotected_peer_ids=*/{0, 1, 2, 3, 4, 5}, random_context)); // Test protection of onion, localhost, and I2P peers... @@ -96,8 +96,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_is_local = false; c.m_network = (c.id == 3 || c.id == 8 || c.id == 9) ? NET_ONION : NET_IPV4; }, - /* protected_peer_ids */ {3, 8, 9}, - /* unprotected_peer_ids */ {}, + /*protected_peer_ids=*/{3, 8, 9}, + /*unprotected_peer_ids=*/{}, random_context)); // Expect 1/4 onion peers and 1/4 of the other peers to be protected, @@ -108,8 +108,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_is_local = false; c.m_network = (c.id == 3 || c.id > 7) ? NET_ONION : NET_IPV6; }, - /* protected_peer_ids */ {0, 1, 2, 3, 8, 9}, - /* unprotected_peer_ids */ {4, 5, 6, 7, 10, 11}, + /*protected_peer_ids=*/{0, 1, 2, 3, 8, 9}, + /*unprotected_peer_ids=*/{4, 5, 6, 7, 10, 11}, random_context)); // Expect 1/4 localhost peers to be protected from eviction, @@ -119,8 +119,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_is_local = (c.id == 1 || c.id == 9 || c.id == 11); c.m_network = NET_IPV4; }, - /* protected_peer_ids */ {1, 9, 11}, - /* unprotected_peer_ids */ {}, + /*protected_peer_ids=*/{1, 9, 11}, + /*unprotected_peer_ids=*/{}, random_context)); // Expect 1/4 localhost peers and 1/4 of the other peers to be protected, @@ -131,8 +131,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_is_local = (c.id > 6); c.m_network = NET_IPV6; }, - /* protected_peer_ids */ {0, 1, 2, 7, 8, 9}, - /* unprotected_peer_ids */ {3, 4, 5, 6, 10, 11}, + /*protected_peer_ids=*/{0, 1, 2, 7, 8, 9}, + /*unprotected_peer_ids=*/{3, 4, 5, 6, 10, 11}, random_context)); // Expect 1/4 I2P peers to be protected from eviction, @@ -142,8 +142,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_is_local = false; c.m_network = (c.id == 2 || c.id == 7 || c.id == 10) ? NET_I2P : NET_IPV4; }, - /* protected_peer_ids */ {2, 7, 10}, - /* unprotected_peer_ids */ {}, + /*protected_peer_ids=*/{2, 7, 10}, + /*unprotected_peer_ids=*/{}, random_context)); // Expect 1/4 I2P peers and 1/4 of the other peers to be protected, @@ -154,8 +154,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_is_local = false; c.m_network = (c.id == 4 || c.id > 8) ? NET_I2P : NET_IPV6; }, - /* protected_peer_ids */ {0, 1, 2, 4, 9, 10}, - /* unprotected_peer_ids */ {3, 5, 6, 7, 8, 11}, + /*protected_peer_ids=*/{0, 1, 2, 4, 9, 10}, + /*unprotected_peer_ids=*/{3, 5, 6, 7, 8, 11}, random_context)); // Tests with 2 networks... @@ -169,8 +169,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_is_local = (c.id == 4); c.m_network = (c.id == 3) ? NET_ONION : NET_IPV4; }, - /* protected_peer_ids */ {0, 4}, - /* unprotected_peer_ids */ {1, 2}, + /*protected_peer_ids=*/{0, 4}, + /*unprotected_peer_ids=*/{1, 2}, random_context)); // Combined test: expect having 1 localhost and 1 onion peer out of 7 to @@ -182,8 +182,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_is_local = (c.id == 6); c.m_network = (c.id == 5) ? NET_ONION : NET_IPV4; }, - /* protected_peer_ids */ {0, 1, 6}, - /* unprotected_peer_ids */ {2, 3, 4, 5}, + /*protected_peer_ids=*/{0, 1, 6}, + /*unprotected_peer_ids=*/{2, 3, 4, 5}, random_context)); // Combined test: expect having 1 localhost and 1 onion peer out of 8 to @@ -195,8 +195,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_is_local = (c.id == 6); c.m_network = (c.id == 5) ? NET_ONION : NET_IPV4; }, - /* protected_peer_ids */ {0, 1, 5, 6}, - /* unprotected_peer_ids */ {2, 3, 4, 7}, + /*protected_peer_ids=*/{0, 1, 5, 6}, + /*unprotected_peer_ids=*/{2, 3, 4, 7}, random_context)); // Combined test: expect having 3 localhost and 3 onion peers out of 12 to @@ -208,8 +208,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_is_local = (c.id == 6 || c.id == 9 || c.id == 11); c.m_network = (c.id == 7 || c.id == 8 || c.id == 10) ? NET_ONION : NET_IPV6; }, - /* protected_peer_ids */ {0, 1, 2, 6, 7, 9}, - /* unprotected_peer_ids */ {3, 4, 5, 8, 10, 11}, + /*protected_peer_ids=*/{0, 1, 2, 6, 7, 9}, + /*unprotected_peer_ids=*/{3, 4, 5, 8, 10, 11}, random_context)); // Combined test: expect having 4 localhost and 1 onion peer out of 12 to @@ -220,8 +220,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_is_local = (c.id > 4 && c.id < 9); c.m_network = (c.id == 10) ? NET_ONION : NET_IPV4; }, - /* protected_peer_ids */ {0, 1, 2, 5, 6, 10}, - /* unprotected_peer_ids */ {3, 4, 7, 8, 9, 11}, + /*protected_peer_ids=*/{0, 1, 2, 5, 6, 10}, + /*unprotected_peer_ids=*/{3, 4, 7, 8, 9, 11}, random_context)); // Combined test: expect having 4 localhost and 2 onion peers out of 16 to @@ -232,8 +232,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_is_local = (c.id == 6 || c.id == 9 || c.id == 11 || c.id == 12); c.m_network = (c.id == 8 || c.id == 10) ? NET_ONION : NET_IPV6; }, - /* protected_peer_ids */ {0, 1, 2, 3, 6, 8, 9, 10}, - /* unprotected_peer_ids */ {4, 5, 7, 11, 12, 13, 14, 15}, + /*protected_peer_ids=*/{0, 1, 2, 3, 6, 8, 9, 10}, + /*unprotected_peer_ids=*/{4, 5, 7, 11, 12, 13, 14, 15}, random_context)); // Combined test: expect having 5 localhost and 1 onion peer out of 16 to @@ -245,8 +245,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_is_local = (c.id > 10); c.m_network = (c.id == 10) ? NET_ONION : NET_IPV4; }, - /* protected_peer_ids */ {0, 1, 2, 3, 10, 11, 12, 13}, - /* unprotected_peer_ids */ {4, 5, 6, 7, 8, 9, 14, 15}, + /*protected_peer_ids=*/{0, 1, 2, 3, 10, 11, 12, 13}, + /*unprotected_peer_ids=*/{4, 5, 6, 7, 8, 9, 14, 15}, random_context)); // Combined test: expect having 1 localhost and 4 onion peers out of 16 to @@ -258,8 +258,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_is_local = (c.id == 15); c.m_network = (c.id > 6 && c.id < 11) ? NET_ONION : NET_IPV6; }, - /* protected_peer_ids */ {0, 1, 2, 3, 7, 8, 9, 15}, - /* unprotected_peer_ids */ {5, 6, 10, 11, 12, 13, 14}, + /*protected_peer_ids=*/{0, 1, 2, 3, 7, 8, 9, 15}, + /*unprotected_peer_ids=*/{5, 6, 10, 11, 12, 13, 14}, random_context)); // Combined test: expect having 2 onion and 4 I2P out of 12 peers to protect @@ -277,8 +277,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_network = NET_IPV4; } }, - /* protected_peer_ids */ {0, 1, 2, 6, 8, 10}, - /* unprotected_peer_ids */ {3, 4, 5, 7, 9, 11}, + /*protected_peer_ids=*/{0, 1, 2, 6, 8, 10}, + /*unprotected_peer_ids=*/{3, 4, 5, 7, 9, 11}, random_context)); // Tests with 3 networks... @@ -298,8 +298,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_network = NET_IPV6; } }, - /* protected_peer_ids */ {0, 4}, - /* unprotected_peer_ids */ {1, 2}, + /*protected_peer_ids=*/{0, 4}, + /*unprotected_peer_ids=*/{1, 2}, random_context)); // Combined test: expect having 1 localhost, 1 I2P and 1 onion peer out of 7 @@ -317,8 +317,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_network = NET_IPV6; } }, - /* protected_peer_ids */ {0, 1, 6}, - /* unprotected_peer_ids */ {2, 3, 4, 5}, + /*protected_peer_ids=*/{0, 1, 6}, + /*unprotected_peer_ids=*/{2, 3, 4, 5}, random_context)); // Combined test: expect having 1 localhost, 1 I2P and 1 onion peer out of 8 @@ -336,8 +336,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_network = NET_IPV6; } }, - /* protected_peer_ids */ {0, 1, 5, 6}, - /* unprotected_peer_ids */ {2, 3, 4, 7}, + /*protected_peer_ids=*/{0, 1, 5, 6}, + /*unprotected_peer_ids=*/{2, 3, 4, 7}, random_context)); // Combined test: expect having 4 localhost, 2 I2P, and 2 onion peers out of @@ -355,8 +355,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_network = NET_IPV4; } }, - /* protected_peer_ids */ {0, 1, 2, 3, 6, 7, 9, 11}, - /* unprotected_peer_ids */ {4, 5, 8, 10, 12, 13, 14, 15}, + /*protected_peer_ids=*/{0, 1, 2, 3, 6, 7, 9, 11}, + /*unprotected_peer_ids=*/{4, 5, 8, 10, 12, 13, 14, 15}, random_context)); // Combined test: expect having 1 localhost, 8 I2P and 1 onion peer out of @@ -374,8 +374,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_network = NET_IPV6; } }, - /* protected_peer_ids */ {0, 1, 2, 3, 4, 5, 12, 15, 16, 17, 18, 23}, - /* unprotected_peer_ids */ {6, 7, 8, 9, 10, 11, 13, 14, 19, 20, 21, 22}, + /*protected_peer_ids=*/{0, 1, 2, 3, 4, 5, 12, 15, 16, 17, 18, 23}, + /*unprotected_peer_ids=*/{6, 7, 8, 9, 10, 11, 13, 14, 19, 20, 21, 22}, random_context)); // Combined test: expect having 1 localhost, 3 I2P and 6 onion peers out of @@ -393,8 +393,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_network = NET_IPV4; } }, - /* protected_peer_ids */ {0, 1, 2, 3, 4, 5, 12, 14, 15, 17, 18, 19}, - /* unprotected_peer_ids */ {6, 7, 8, 9, 10, 11, 13, 16, 20, 21, 22, 23}, + /*protected_peer_ids=*/{0, 1, 2, 3, 4, 5, 12, 14, 15, 17, 18, 19}, + /*unprotected_peer_ids=*/{6, 7, 8, 9, 10, 11, 13, 16, 20, 21, 22, 23}, random_context)); // Combined test: expect having 1 localhost, 7 I2P and 4 onion peers out of @@ -412,8 +412,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_network = NET_IPV6; } }, - /* protected_peer_ids */ {0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 17, 18}, - /* unprotected_peer_ids */ {6, 7, 8, 9, 10, 11, 16, 19, 20, 21, 22, 23}, + /*protected_peer_ids=*/{0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 17, 18}, + /*unprotected_peer_ids=*/{6, 7, 8, 9, 10, 11, 16, 19, 20, 21, 22, 23}, random_context)); // Combined test: expect having 8 localhost, 4 I2P, and 3 onion peers out of @@ -431,8 +431,8 @@ BOOST_AUTO_TEST_CASE(peer_protection_test) c.m_network = NET_IPV4; } }, - /* protected_peer_ids */ {0, 1, 2, 3, 4, 5, 7, 8, 11, 12, 16, 17}, - /* unprotected_peer_ids */ {6, 9, 10, 13, 14, 15, 18, 19, 20, 21, 22, 23}, + /*protected_peer_ids=*/{0, 1, 2, 3, 4, 5, 7, 8, 11, 12, 16, 17}, + /*unprotected_peer_ids=*/{6, 9, 10, 13, 14, 15, 18, 19, 20, 21, 22, 23}, random_context)); } diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 29938d4ede..d0f0e7d50f 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -386,9 +386,9 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s.SetVersion(s.GetVersion() | ADDRV2_FORMAT); // Valid IPv4. - s << MakeSpan(ParseHex("01" // network type (IPv4) - "04" // address length - "01020304")); // address + s << Span{ParseHex("01" // network type (IPv4) + "04" // address length + "01020304")}; // address s >> addr; BOOST_CHECK(addr.IsValid()); BOOST_CHECK(addr.IsIPv4()); @@ -397,35 +397,35 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) BOOST_REQUIRE(s.empty()); // Invalid IPv4, valid length but address itself is shorter. - s << MakeSpan(ParseHex("01" // network type (IPv4) - "04" // address length - "0102")); // address + s << Span{ParseHex("01" // network type (IPv4) + "04" // address length + "0102")}; // address BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure, HasReason("end of data")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); // Invalid IPv4, with bogus length. - s << MakeSpan(ParseHex("01" // network type (IPv4) - "05" // address length - "01020304")); // address + s << Span{ParseHex("01" // network type (IPv4) + "05" // address length + "01020304")}; // address BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure, HasReason("BIP155 IPv4 address with length 5 (should be 4)")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); // Invalid IPv4, with extreme length. - s << MakeSpan(ParseHex("01" // network type (IPv4) - "fd0102" // address length (513 as CompactSize) - "01020304")); // address + s << Span{ParseHex("01" // network type (IPv4) + "fd0102" // address length (513 as CompactSize) + "01020304")}; // address BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure, HasReason("Address too long: 513 > 512")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); // Valid IPv6. - s << MakeSpan(ParseHex("02" // network type (IPv6) - "10" // address length - "0102030405060708090a0b0c0d0e0f10")); // address + s << Span{ParseHex("02" // network type (IPv6) + "10" // address length + "0102030405060708090a0b0c0d0e0f10")}; // address s >> addr; BOOST_CHECK(addr.IsValid()); BOOST_CHECK(addr.IsIPv6()); @@ -434,10 +434,10 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) BOOST_REQUIRE(s.empty()); // Valid IPv6, contains embedded "internal". - s << MakeSpan(ParseHex( + s << Span{ParseHex( "02" // network type (IPv6) "10" // address length - "fd6b88c08724ca978112ca1bbdcafac2")); // address: 0xfd + sha256("bitcoin")[0:5] + + "fd6b88c08724ca978112ca1bbdcafac2")}; // address: 0xfd + sha256("bitcoin")[0:5] + // sha256(name)[0:10] s >> addr; BOOST_CHECK(addr.IsInternal()); @@ -446,44 +446,44 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) BOOST_REQUIRE(s.empty()); // Invalid IPv6, with bogus length. - s << MakeSpan(ParseHex("02" // network type (IPv6) - "04" // address length - "00")); // address + s << Span{ParseHex("02" // network type (IPv6) + "04" // address length + "00")}; // address BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure, HasReason("BIP155 IPv6 address with length 4 (should be 16)")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); // Invalid IPv6, contains embedded IPv4. - s << MakeSpan(ParseHex("02" // network type (IPv6) - "10" // address length - "00000000000000000000ffff01020304")); // address + s << Span{ParseHex("02" // network type (IPv6) + "10" // address length + "00000000000000000000ffff01020304")}; // address s >> addr; BOOST_CHECK(!addr.IsValid()); BOOST_REQUIRE(s.empty()); // Invalid IPv6, contains embedded TORv2. - s << MakeSpan(ParseHex("02" // network type (IPv6) - "10" // address length - "fd87d87eeb430102030405060708090a")); // address + s << Span{ParseHex("02" // network type (IPv6) + "10" // address length + "fd87d87eeb430102030405060708090a")}; // address s >> addr; BOOST_CHECK(!addr.IsValid()); BOOST_REQUIRE(s.empty()); // TORv2, no longer supported. - s << MakeSpan(ParseHex("03" // network type (TORv2) - "0a" // address length - "f1f2f3f4f5f6f7f8f9fa")); // address + s << Span{ParseHex("03" // network type (TORv2) + "0a" // address length + "f1f2f3f4f5f6f7f8f9fa")}; // address s >> addr; BOOST_CHECK(!addr.IsValid()); BOOST_REQUIRE(s.empty()); // Valid TORv3. - s << MakeSpan(ParseHex("04" // network type (TORv3) - "20" // address length - "79bcc625184b05194975c28b66b66b04" // address - "69f7f6556fb1ac3189a79b40dda32f1f" - )); + s << Span{ParseHex("04" // network type (TORv3) + "20" // address length + "79bcc625184b05194975c28b66b66b04" // address + "69f7f6556fb1ac3189a79b40dda32f1f" + )}; s >> addr; BOOST_CHECK(addr.IsValid()); BOOST_CHECK(addr.IsTor()); @@ -493,20 +493,20 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) BOOST_REQUIRE(s.empty()); // Invalid TORv3, with bogus length. - s << MakeSpan(ParseHex("04" // network type (TORv3) - "00" // address length - "00" // address - )); + s << Span{ParseHex("04" // network type (TORv3) + "00" // address length + "00" // address + )}; BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure, HasReason("BIP155 TORv3 address with length 0 (should be 32)")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); // Valid I2P. - s << MakeSpan(ParseHex("05" // network type (I2P) - "20" // address length - "a2894dabaec08c0051a481a6dac88b64" // address - "f98232ae42d4b6fd2fa81952dfe36a87")); + s << Span{ParseHex("05" // network type (I2P) + "20" // address length + "a2894dabaec08c0051a481a6dac88b64" // address + "f98232ae42d4b6fd2fa81952dfe36a87")}; s >> addr; BOOST_CHECK(addr.IsValid()); BOOST_CHECK(addr.IsI2P()); @@ -516,20 +516,20 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) BOOST_REQUIRE(s.empty()); // Invalid I2P, with bogus length. - s << MakeSpan(ParseHex("05" // network type (I2P) - "03" // address length - "00" // address - )); + s << Span{ParseHex("05" // network type (I2P) + "03" // address length + "00" // address + )}; BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure, HasReason("BIP155 I2P address with length 3 (should be 32)")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); // Valid CJDNS. - s << MakeSpan(ParseHex("06" // network type (CJDNS) - "10" // address length - "fc000001000200030004000500060007" // address - )); + s << Span{ParseHex("06" // network type (CJDNS) + "10" // address length + "fc000001000200030004000500060007" // address + )}; s >> addr; BOOST_CHECK(addr.IsValid()); BOOST_CHECK(addr.IsCJDNS()); @@ -538,49 +538,49 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) BOOST_REQUIRE(s.empty()); // Invalid CJDNS, wrong prefix. - s << MakeSpan(ParseHex("06" // network type (CJDNS) - "10" // address length - "aa000001000200030004000500060007" // address - )); + s << Span{ParseHex("06" // network type (CJDNS) + "10" // address length + "aa000001000200030004000500060007" // address + )}; s >> addr; BOOST_CHECK(addr.IsCJDNS()); BOOST_CHECK(!addr.IsValid()); BOOST_REQUIRE(s.empty()); // Invalid CJDNS, with bogus length. - s << MakeSpan(ParseHex("06" // network type (CJDNS) - "01" // address length - "00" // address - )); + s << Span{ParseHex("06" // network type (CJDNS) + "01" // address length + "00" // address + )}; BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure, HasReason("BIP155 CJDNS address with length 1 (should be 16)")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); // Unknown, with extreme length. - s << MakeSpan(ParseHex("aa" // network type (unknown) - "fe00000002" // address length (CompactSize's MAX_SIZE) - "01020304050607" // address - )); + s << Span{ParseHex("aa" // network type (unknown) + "fe00000002" // address length (CompactSize's MAX_SIZE) + "01020304050607" // address + )}; BOOST_CHECK_EXCEPTION(s >> addr, std::ios_base::failure, HasReason("Address too long: 33554432 > 512")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); // Unknown, with reasonable length. - s << MakeSpan(ParseHex("aa" // network type (unknown) - "04" // address length - "01020304" // address - )); + s << Span{ParseHex("aa" // network type (unknown) + "04" // address length + "01020304" // address + )}; s >> addr; BOOST_CHECK(!addr.IsValid()); BOOST_REQUIRE(s.empty()); // Unknown, with zero length. - s << MakeSpan(ParseHex("aa" // network type (unknown) - "00" // address length - "" // address - )); + s << Span{ParseHex("aa" // network type (unknown) + "00" // address length + "" // address + )}; s >> addr; BOOST_CHECK(!addr.IsValid()); BOOST_REQUIRE(s.empty()); @@ -607,7 +607,7 @@ BOOST_AUTO_TEST_CASE(ipv4_peer_with_ipv6_addrMe_test) in_addr ipv4AddrPeer; ipv4AddrPeer.s_addr = 0xa0b0c001; CAddress addr = CAddress(CService(ipv4AddrPeer, 7777), NODE_NETWORK); - std::unique_ptr<CNode> pnode = std::make_unique<CNode>(0, NODE_NETWORK, INVALID_SOCKET, addr, /* nKeyedNetGroupIn */ 0, /* nLocalHostNonceIn */ 0, CAddress{}, /* pszDest */ std::string{}, ConnectionType::OUTBOUND_FULL_RELAY, /* inbound_onion */ false); + std::unique_ptr<CNode> pnode = std::make_unique<CNode>(0, NODE_NETWORK, INVALID_SOCKET, addr, /*nKeyedNetGroupIn=*/0, /*nLocalHostNonceIn=*/0, CAddress{}, /*pszDest=*/std::string{}, ConnectionType::OUTBOUND_FULL_RELAY, /*inbound_onion=*/false); pnode->fSuccessfullyConnected.store(true); // the peer claims to be reaching us via IPv6 diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 687d2f6747..b6d7496cc7 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -339,11 +339,13 @@ BOOST_AUTO_TEST_CASE(netbase_parsenetwork) BOOST_CHECK_EQUAL(ParseNetwork("ipv6"), NET_IPV6); BOOST_CHECK_EQUAL(ParseNetwork("onion"), NET_ONION); BOOST_CHECK_EQUAL(ParseNetwork("tor"), NET_ONION); + BOOST_CHECK_EQUAL(ParseNetwork("cjdns"), NET_CJDNS); BOOST_CHECK_EQUAL(ParseNetwork("IPv4"), NET_IPV4); BOOST_CHECK_EQUAL(ParseNetwork("IPv6"), NET_IPV6); BOOST_CHECK_EQUAL(ParseNetwork("ONION"), NET_ONION); BOOST_CHECK_EQUAL(ParseNetwork("TOR"), NET_ONION); + BOOST_CHECK_EQUAL(ParseNetwork("CJDNS"), NET_CJDNS); BOOST_CHECK_EQUAL(ParseNetwork(":)"), NET_UNROUTABLE); BOOST_CHECK_EQUAL(ParseNetwork("tÖr"), NET_UNROUTABLE); diff --git a/src/test/scheduler_tests.cpp b/src/test/scheduler_tests.cpp index d57c000b92..fed941247c 100644 --- a/src/test/scheduler_tests.cpp +++ b/src/test/scheduler_tests.cpp @@ -44,7 +44,7 @@ BOOST_AUTO_TEST_CASE(manythreads) std::mutex counterMutex[10]; int counter[10] = { 0 }; - FastRandomContext rng{/* fDeterministic */ true}; + FastRandomContext rng{/*fDeterministic=*/true}; auto zeroToNine = [](FastRandomContext& rc) -> int { return rc.randrange(10); }; // [0, 9] auto randomMsec = [](FastRandomContext& rc) -> int { return -11 + (int)rc.randrange(1012); }; // [-11, 1000] auto randomDelta = [](FastRandomContext& rc) -> int { return -1000 + (int)rc.randrange(2001); }; // [-1000, 1000] diff --git a/src/test/script_p2sh_tests.cpp b/src/test/script_p2sh_tests.cpp index d8a44a65dd..17b3359624 100644 --- a/src/test/script_p2sh_tests.cpp +++ b/src/test/script_p2sh_tests.cpp @@ -343,7 +343,7 @@ BOOST_AUTO_TEST_CASE(AreInputsStandard) txTo.vin[3].scriptSig << OP_11 << OP_11 << std::vector<unsigned char>(oneAndTwo.begin(), oneAndTwo.end()); txTo.vin[4].scriptSig << std::vector<unsigned char>(fifteenSigops.begin(), fifteenSigops.end()); - BOOST_CHECK(::AreInputsStandard(CTransaction(txTo), coins, false)); + BOOST_CHECK(::AreInputsStandard(CTransaction(txTo), coins)); // 22 P2SH sigops for all inputs (1 for vin[0], 6 for vin[3], 15 for vin[4] BOOST_CHECK_EQUAL(GetP2SHSigOpCount(CTransaction(txTo), coins), 22U); @@ -356,7 +356,7 @@ BOOST_AUTO_TEST_CASE(AreInputsStandard) txToNonStd1.vin[0].prevout.hash = txFrom.GetHash(); txToNonStd1.vin[0].scriptSig << std::vector<unsigned char>(sixteenSigops.begin(), sixteenSigops.end()); - BOOST_CHECK(!::AreInputsStandard(CTransaction(txToNonStd1), coins, false)); + BOOST_CHECK(!::AreInputsStandard(CTransaction(txToNonStd1), coins)); BOOST_CHECK_EQUAL(GetP2SHSigOpCount(CTransaction(txToNonStd1), coins), 16U); CMutableTransaction txToNonStd2; @@ -368,7 +368,7 @@ BOOST_AUTO_TEST_CASE(AreInputsStandard) txToNonStd2.vin[0].prevout.hash = txFrom.GetHash(); txToNonStd2.vin[0].scriptSig << std::vector<unsigned char>(twentySigops.begin(), twentySigops.end()); - BOOST_CHECK(!::AreInputsStandard(CTransaction(txToNonStd2), coins, false)); + BOOST_CHECK(!::AreInputsStandard(CTransaction(txToNonStd2), coins)); BOOST_CHECK_EQUAL(GetP2SHSigOpCount(CTransaction(txToNonStd2), coins), 20U); } diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index bf8ff5f5e2..5a5cc6ab29 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -2,6 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <test/data/bip341_wallet_vectors.json.h> + #include <key.h> #include <key_io.h> #include <script/script.h> @@ -12,6 +14,8 @@ #include <boost/test/unit_test.hpp> +#include <univalue.h> + BOOST_FIXTURE_TEST_SUITE(script_standard_tests, BasicTestingSetup) @@ -385,4 +389,46 @@ BOOST_AUTO_TEST_CASE(script_standard_taproot_builder) BOOST_CHECK_EQUAL(EncodeDestination(builder.GetOutput()), "bc1pj6gaw944fy0xpmzzu45ugqde4rz7mqj5kj0tg8kmr5f0pjq8vnaqgynnge"); } +BOOST_AUTO_TEST_CASE(bip341_spk_test_vectors) +{ + using control_set = decltype(TaprootSpendData::scripts)::mapped_type; + + UniValue tests; + tests.read((const char*)json_tests::bip341_wallet_vectors, sizeof(json_tests::bip341_wallet_vectors)); + + const auto& vectors = tests["scriptPubKey"]; + + for (const auto& vec : vectors.getValues()) { + TaprootBuilder spktest; + std::map<std::pair<CScript, int>, int> scriptposes; + std::function<void (const UniValue&, int)> parse_tree = [&](const UniValue& node, int depth) { + if (node.isNull()) return; + if (node.isObject()) { + auto script_bytes = ParseHex(node["script"].get_str()); + CScript script(script_bytes.begin(), script_bytes.end()); + int idx = node["id"].get_int(); + int leaf_version = node["leafVersion"].get_int(); + scriptposes[{script, leaf_version}] = idx; + spktest.Add(depth, script, leaf_version); + } else { + parse_tree(node[0], depth + 1); + parse_tree(node[1], depth + 1); + } + }; + parse_tree(vec["given"]["scriptTree"], 0); + spktest.Finalize(XOnlyPubKey(ParseHex(vec["given"]["internalPubkey"].get_str()))); + BOOST_CHECK_EQUAL(HexStr(GetScriptForDestination(spktest.GetOutput())), vec["expected"]["scriptPubKey"].get_str()); + BOOST_CHECK_EQUAL(EncodeDestination(spktest.GetOutput()), vec["expected"]["bip350Address"].get_str()); + auto spend_data = spktest.GetSpendData(); + BOOST_CHECK_EQUAL(vec["intermediary"]["merkleRoot"].isNull(), spend_data.merkle_root.IsNull()); + if (!spend_data.merkle_root.IsNull()) { + BOOST_CHECK_EQUAL(vec["intermediary"]["merkleRoot"].get_str(), HexStr(spend_data.merkle_root)); + } + BOOST_CHECK_EQUAL(spend_data.scripts.size(), scriptposes.size()); + for (const auto& scriptpos : scriptposes) { + BOOST_CHECK(spend_data.scripts[scriptpos.first] == control_set{ParseHex(vec["expected"]["scriptPathControlBlocks"][scriptpos.second].get_str())}); + } + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 2c39cbffb9..f1304dfc82 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <test/data/script_tests.json.h> +#include <test/data/bip341_wallet_vectors.json.h> #include <core_io.h> #include <fs.h> @@ -283,7 +284,7 @@ public: CScript scriptPubKey = script; if (wm == WitnessMode::PKH) { uint160 hash; - CHash160().Write(MakeSpan(script).subspan(1)).Finalize(hash); + CHash160().Write(Span{script}.subspan(1)).Finalize(hash); script = CScript() << OP_DUP << OP_HASH160 << ToByteVector(hash) << OP_EQUALVERIFY << OP_CHECKSIG; scriptPubKey = CScript() << witnessversion << ToByteVector(hash); } else if (wm == WitnessMode::SH) { @@ -1472,7 +1473,7 @@ BOOST_AUTO_TEST_CASE(script_HasValidOps) static CMutableTransaction TxFromHex(const std::string& str) { CMutableTransaction tx; - VectorReader(SER_DISK, SERIALIZE_TRANSACTION_NO_WITNESS, ParseHex(str), 0) >> tx; + SpanReader{SER_DISK, SERIALIZE_TRANSACTION_NO_WITNESS, ParseHex(str)} >> tx; return tx; } @@ -1482,7 +1483,7 @@ static std::vector<CTxOut> TxOutsFromJSON(const UniValue& univalue) std::vector<CTxOut> prevouts; for (size_t i = 0; i < univalue.size(); ++i) { CTxOut txout; - VectorReader(SER_DISK, 0, ParseHex(univalue[i].get_str()), 0) >> txout; + SpanReader{SER_DISK, 0, ParseHex(univalue[i].get_str())} >> txout; prevouts.push_back(std::move(txout)); } return prevouts; @@ -1743,4 +1744,79 @@ BOOST_AUTO_TEST_CASE(script_assets_test) file.close(); } +BOOST_AUTO_TEST_CASE(bip341_keypath_test_vectors) +{ + UniValue tests; + tests.read((const char*)json_tests::bip341_wallet_vectors, sizeof(json_tests::bip341_wallet_vectors)); + + const auto& vectors = tests["keyPathSpending"]; + + for (const auto& vec : vectors.getValues()) { + auto txhex = ParseHex(vec["given"]["rawUnsignedTx"].get_str()); + CMutableTransaction tx; + SpanReader{SER_NETWORK, PROTOCOL_VERSION, txhex} >> tx; + std::vector<CTxOut> utxos; + for (const auto& utxo_spent : vec["given"]["utxosSpent"].getValues()) { + auto script_bytes = ParseHex(utxo_spent["scriptPubKey"].get_str()); + CScript script{script_bytes.begin(), script_bytes.end()}; + CAmount amount{utxo_spent["amountSats"].get_int()}; + utxos.emplace_back(amount, script); + } + + PrecomputedTransactionData txdata; + txdata.Init(tx, std::vector<CTxOut>{utxos}, true); + + BOOST_CHECK(txdata.m_bip341_taproot_ready); + BOOST_CHECK_EQUAL(HexStr(txdata.m_spent_amounts_single_hash), vec["intermediary"]["hashAmounts"].get_str()); + BOOST_CHECK_EQUAL(HexStr(txdata.m_outputs_single_hash), vec["intermediary"]["hashOutputs"].get_str()); + BOOST_CHECK_EQUAL(HexStr(txdata.m_prevouts_single_hash), vec["intermediary"]["hashPrevouts"].get_str()); + BOOST_CHECK_EQUAL(HexStr(txdata.m_spent_scripts_single_hash), vec["intermediary"]["hashScriptPubkeys"].get_str()); + BOOST_CHECK_EQUAL(HexStr(txdata.m_sequences_single_hash), vec["intermediary"]["hashSequences"].get_str()); + + for (const auto& input : vec["inputSpending"].getValues()) { + int txinpos = input["given"]["txinIndex"].get_int(); + int hashtype = input["given"]["hashType"].get_int(); + + // Load key. + auto privkey = ParseHex(input["given"]["internalPrivkey"].get_str()); + CKey key; + key.Set(privkey.begin(), privkey.end(), true); + + // Load Merkle root. + uint256 merkle_root; + if (!input["given"]["merkleRoot"].isNull()) { + merkle_root = uint256{ParseHex(input["given"]["merkleRoot"].get_str())}; + } + + // Compute and verify (internal) public key. + XOnlyPubKey pubkey{key.GetPubKey()}; + BOOST_CHECK_EQUAL(HexStr(pubkey), input["intermediary"]["internalPubkey"].get_str()); + + // Sign and verify signature. + FlatSigningProvider provider; + provider.keys[key.GetPubKey().GetID()] = key; + MutableTransactionSignatureCreator creator(&tx, txinpos, utxos[txinpos].nValue, &txdata, hashtype); + std::vector<unsigned char> signature; + BOOST_CHECK(creator.CreateSchnorrSig(provider, signature, pubkey, nullptr, &merkle_root, SigVersion::TAPROOT)); + BOOST_CHECK_EQUAL(HexStr(signature), input["expected"]["witness"][0].get_str()); + + // We can't observe the tweak used inside the signing logic, so verify by recomputing it. + BOOST_CHECK_EQUAL(HexStr(pubkey.ComputeTapTweakHash(merkle_root.IsNull() ? nullptr : &merkle_root)), input["intermediary"]["tweak"].get_str()); + + // We can't observe the sighash used inside the signing logic, so verify by recomputing it. + ScriptExecutionData sed; + sed.m_annex_init = true; + sed.m_annex_present = false; + uint256 sighash; + BOOST_CHECK(SignatureHashSchnorr(sighash, sed, tx, txinpos, hashtype, SigVersion::TAPROOT, txdata, MissingDataBehavior::FAIL)); + BOOST_CHECK_EQUAL(HexStr(sighash), input["intermediary"]["sigHash"].get_str()); + + // To verify the sigmsg, hash the expected sigmsg, and compare it with the (expected) sighash. + BOOST_CHECK_EQUAL(HexStr((CHashWriter(HASHER_TAPSIGHASH) << Span{ParseHex(input["intermediary"]["sigMsg"].get_str())}).GetSHA256()), input["intermediary"]["sigHash"].get_str()); + } + + } + +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/scriptnum10.h b/src/test/scriptnum10.h index 352797f18d..3937366f01 100644 --- a/src/test/scriptnum10.h +++ b/src/test/scriptnum10.h @@ -179,4 +179,4 @@ private: }; -#endif // BITCOIN_TEST_BIGNUM_H +#endif // BITCOIN_TEST_SCRIPTNUM10_H diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index acd0151e1a..8d44e92f97 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -71,7 +71,7 @@ BOOST_AUTO_TEST_CASE(streams_vector_reader) { std::vector<unsigned char> vch = {1, 255, 3, 4, 5, 6}; - VectorReader reader(SER_NETWORK, INIT_PROTO_VERSION, vch, 0); + SpanReader reader{SER_NETWORK, INIT_PROTO_VERSION, vch}; BOOST_CHECK_EQUAL(reader.size(), 6U); BOOST_CHECK(!reader.empty()); @@ -101,7 +101,7 @@ BOOST_AUTO_TEST_CASE(streams_vector_reader) BOOST_CHECK_THROW(reader >> d, std::ios_base::failure); // Read a 4 bytes as a signed int from the beginning of the buffer. - VectorReader new_reader(SER_NETWORK, INIT_PROTO_VERSION, vch, 0); + SpanReader new_reader{SER_NETWORK, INIT_PROTO_VERSION, vch}; new_reader >> d; BOOST_CHECK_EQUAL(d, 67370753); // 1,255,3,4 in little-endian base-256 BOOST_CHECK_EQUAL(new_reader.size(), 2U); @@ -115,7 +115,7 @@ BOOST_AUTO_TEST_CASE(streams_vector_reader) BOOST_AUTO_TEST_CASE(streams_vector_reader_rvalue) { std::vector<uint8_t> data{0x82, 0xa7, 0x31}; - VectorReader reader(SER_NETWORK, INIT_PROTO_VERSION, data, /* pos= */ 0); + SpanReader reader{SER_NETWORK, INIT_PROTO_VERSION, data}; uint32_t varint = 0; // Deserialize into r-value reader >> VARINT(varint); @@ -172,7 +172,7 @@ BOOST_AUTO_TEST_CASE(streams_serializedata_xor) ds.Xor(key); BOOST_CHECK_EQUAL( std::string(expected_xor.begin(), expected_xor.end()), - std::string(ds.begin(), ds.end())); + ds.str()); in.push_back('\x0f'); in.push_back('\xf0'); @@ -189,7 +189,7 @@ BOOST_AUTO_TEST_CASE(streams_serializedata_xor) ds.Xor(key); BOOST_CHECK_EQUAL( std::string(expected_xor.begin(), expected_xor.end()), - std::string(ds.begin(), ds.end())); + ds.str()); // Multi character key @@ -210,12 +210,14 @@ BOOST_AUTO_TEST_CASE(streams_serializedata_xor) ds.Xor(key); BOOST_CHECK_EQUAL( std::string(expected_xor.begin(), expected_xor.end()), - std::string(ds.begin(), ds.end())); + ds.str()); } BOOST_AUTO_TEST_CASE(streams_buffered_file) { - FILE* file = fsbridge::fopen("streams_test_tmp", "w+b"); + fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp"; + FILE* file = fsbridge::fopen(streams_test_filename, "w+b"); + // The value at each offset is the offset. for (uint8_t j = 0; j < 40; ++j) { fwrite(&j, 1, 1, file); @@ -343,7 +345,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) // We can explicitly close the file, or the destructor will do it. bf.fclose(); - fs::remove("streams_test_tmp"); + fs::remove(streams_test_filename); } BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) @@ -351,8 +353,9 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) // Make this test deterministic. SeedInsecureRand(SeedRand::ZEROS); + fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp"; for (int rep = 0; rep < 50; ++rep) { - FILE* file = fsbridge::fopen("streams_test_tmp", "w+b"); + FILE* file = fsbridge::fopen(streams_test_filename, "w+b"); size_t fileSize = InsecureRandRange(256); for (uint8_t i = 0; i < fileSize; ++i) { fwrite(&i, 1, 1, file); @@ -453,7 +456,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) maxPos = currentPos; } } - fs::remove("streams_test_tmp"); + fs::remove(streams_test_filename); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index c813fbea32..6b0614ed97 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -249,26 +249,26 @@ BOOST_AUTO_TEST_CASE(tx_valid) BOOST_ERROR("Bad test flags: " << strTest); } - BOOST_CHECK_MESSAGE(CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, ~verify_flags, txdata, strTest, /* expect_valid */ true), + BOOST_CHECK_MESSAGE(CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, ~verify_flags, txdata, strTest, /*expect_valid=*/true), "Tx unexpectedly failed: " << strTest); // Backwards compatibility of script verification flags: Removing any flag(s) should not invalidate a valid transaction for (const auto& [name, flag] : mapFlagNames) { // Removing individual flags unsigned int flags = TrimFlags(~(verify_flags | flag)); - if (!CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, flags, txdata, strTest, /* expect_valid */ true)) { + if (!CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, flags, txdata, strTest, /*expect_valid=*/true)) { BOOST_ERROR("Tx unexpectedly failed with flag " << name << " unset: " << strTest); } // Removing random combinations of flags flags = TrimFlags(~(verify_flags | (unsigned int)InsecureRandBits(mapFlagNames.size()))); - if (!CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, flags, txdata, strTest, /* expect_valid */ true)) { + if (!CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, flags, txdata, strTest, /*expect_valid=*/true)) { BOOST_ERROR("Tx unexpectedly failed with random flags " << ToString(flags) << ": " << strTest); } } // Check that flags are maximal: transaction should fail if any unset flags are set. for (auto flags_excluding_one : ExcludeIndividualFlags(verify_flags)) { - if (!CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, ~flags_excluding_one, txdata, strTest, /* expect_valid */ false)) { + if (!CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, ~flags_excluding_one, txdata, strTest, /*expect_valid=*/false)) { BOOST_ERROR("Too many flags unset: " << strTest); } } @@ -340,26 +340,26 @@ BOOST_AUTO_TEST_CASE(tx_invalid) } // Not using FillFlags() in the main test, in order to detect invalid verifyFlags combination - BOOST_CHECK_MESSAGE(CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, verify_flags, txdata, strTest, /* expect_valid */ false), + BOOST_CHECK_MESSAGE(CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, verify_flags, txdata, strTest, /*expect_valid=*/false), "Tx unexpectedly passed: " << strTest); // Backwards compatibility of script verification flags: Adding any flag(s) should not validate an invalid transaction for (const auto& [name, flag] : mapFlagNames) { unsigned int flags = FillFlags(verify_flags | flag); // Adding individual flags - if (!CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, flags, txdata, strTest, /* expect_valid */ false)) { + if (!CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, flags, txdata, strTest, /*expect_valid=*/false)) { BOOST_ERROR("Tx unexpectedly passed with flag " << name << " set: " << strTest); } // Adding random combinations of flags flags = FillFlags(verify_flags | (unsigned int)InsecureRandBits(mapFlagNames.size())); - if (!CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, flags, txdata, strTest, /* expect_valid */ false)) { + if (!CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, flags, txdata, strTest, /*expect_valid=*/false)) { BOOST_ERROR("Tx unexpectedly passed with random flags " << name << ": " << strTest); } } // Check that flags are minimal: transaction should succeed if any set flags are unset. for (auto flags_excluding_one : ExcludeIndividualFlags(verify_flags)) { - if (!CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, flags_excluding_one, txdata, strTest, /* expect_valid */ true)) { + if (!CheckTxScripts(tx, mapprevOutScriptPubKeys, mapprevOutValues, flags_excluding_one, txdata, strTest, /*expect_valid=*/true)) { BOOST_ERROR("Too many flags set: " << strTest); } } @@ -406,7 +406,7 @@ BOOST_AUTO_TEST_CASE(test_Get) t1.vout[0].nValue = 90*CENT; t1.vout[0].scriptPubKey << OP_1; - BOOST_CHECK(AreInputsStandard(CTransaction(t1), coins, false)); + BOOST_CHECK(AreInputsStandard(CTransaction(t1), coins)); } static void CreateCreditAndSpend(const FillableSigningProvider& keystore, const CScript& outscript, CTransactionRef& output, CMutableTransaction& input, bool success = true) @@ -810,10 +810,10 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) // nDustThreshold = 182 * 3702 / 1000 dustRelayFee = CFeeRate(3702); // dust: - t.vout[0].nValue = 673 - 1; + t.vout[0].nValue = 674 - 1; CheckIsNotStandard(t, "dust"); // not dust: - t.vout[0].nValue = 673; + t.vout[0].nValue = 674; CheckIsStandard(t); dustRelayFee = CFeeRate(DUST_RELAY_TX_FEE); diff --git a/src/test/txpackage_tests.cpp b/src/test/txpackage_tests.cpp new file mode 100644 index 0000000000..2193e21780 --- /dev/null +++ b/src/test/txpackage_tests.cpp @@ -0,0 +1,117 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <consensus/validation.h> +#include <key_io.h> +#include <policy/packages.h> +#include <policy/policy.h> +#include <primitives/transaction.h> +#include <script/script.h> +#include <script/standard.h> +#include <test/util/setup_common.h> +#include <validation.h> + +#include <boost/test/unit_test.hpp> + +BOOST_AUTO_TEST_SUITE(txpackage_tests) + +// Create placeholder transactions that have no meaning. +inline CTransactionRef create_placeholder_tx(size_t num_inputs, size_t num_outputs) +{ + CMutableTransaction mtx = CMutableTransaction(); + mtx.vin.resize(num_inputs); + mtx.vout.resize(num_outputs); + auto random_script = CScript() << ToByteVector(InsecureRand256()) << ToByteVector(InsecureRand256()); + for (size_t i{0}; i < num_inputs; ++i) { + mtx.vin[i].prevout.hash = InsecureRand256(); + mtx.vin[i].prevout.n = 0; + mtx.vin[i].scriptSig = random_script; + } + for (size_t o{0}; o < num_outputs; ++o) { + mtx.vout[o].nValue = 1 * CENT; + mtx.vout[o].scriptPubKey = random_script; + } + return MakeTransactionRef(mtx); +} + +BOOST_FIXTURE_TEST_CASE(package_sanitization_tests, TestChain100Setup) +{ + // Packages can't have more than 25 transactions. + Package package_too_many; + package_too_many.reserve(MAX_PACKAGE_COUNT + 1); + for (size_t i{0}; i < MAX_PACKAGE_COUNT + 1; ++i) { + package_too_many.emplace_back(create_placeholder_tx(1, 1)); + } + PackageValidationState state_too_many; + BOOST_CHECK(!CheckPackage(package_too_many, state_too_many)); + BOOST_CHECK_EQUAL(state_too_many.GetResult(), PackageValidationResult::PCKG_POLICY); + BOOST_CHECK_EQUAL(state_too_many.GetRejectReason(), "package-too-many-transactions"); + + // Packages can't have a total size of more than 101KvB. + CTransactionRef large_ptx = create_placeholder_tx(150, 150); + Package package_too_large; + auto size_large = GetVirtualTransactionSize(*large_ptx); + size_t total_size{0}; + while (total_size <= MAX_PACKAGE_SIZE * 1000) { + package_too_large.push_back(large_ptx); + total_size += size_large; + } + BOOST_CHECK(package_too_large.size() <= MAX_PACKAGE_COUNT); + PackageValidationState state_too_large; + BOOST_CHECK(!CheckPackage(package_too_large, state_too_large)); + BOOST_CHECK_EQUAL(state_too_large.GetResult(), PackageValidationResult::PCKG_POLICY); + BOOST_CHECK_EQUAL(state_too_large.GetRejectReason(), "package-too-large"); +} + +BOOST_FIXTURE_TEST_CASE(package_validation_tests, TestChain100Setup) +{ + LOCK(cs_main); + unsigned int initialPoolSize = m_node.mempool->size(); + + // Parent and Child Package + CKey parent_key; + parent_key.MakeNewKey(true); + CScript parent_locking_script = GetScriptForDestination(PKHash(parent_key.GetPubKey())); + auto mtx_parent = CreateValidMempoolTransaction(/*input_transaction=*/ m_coinbase_txns[0], /*input_vout=*/0, + /*input_height=*/ 0, /*input_signing_key=*/coinbaseKey, + /*output_destination=*/ parent_locking_script, + /*output_amount=*/ CAmount(49 * COIN), /*submit=*/false); + CTransactionRef tx_parent = MakeTransactionRef(mtx_parent); + + CKey child_key; + child_key.MakeNewKey(true); + CScript child_locking_script = GetScriptForDestination(PKHash(child_key.GetPubKey())); + auto mtx_child = CreateValidMempoolTransaction(/*input_transaction=*/ tx_parent, /*input_vout=*/0, + /*input_height=*/ 101, /*input_signing_key=*/parent_key, + /*output_destination=*/child_locking_script, + /*output_amount=*/ CAmount(48 * COIN), /*submit=*/false); + CTransactionRef tx_child = MakeTransactionRef(mtx_child); + const auto result_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, {tx_parent, tx_child}, /*test_accept=*/true); + BOOST_CHECK_MESSAGE(result_parent_child.m_state.IsValid(), + "Package validation unexpectedly failed: " << result_parent_child.m_state.GetRejectReason()); + auto it_parent = result_parent_child.m_tx_results.find(tx_parent->GetWitnessHash()); + auto it_child = result_parent_child.m_tx_results.find(tx_child->GetWitnessHash()); + BOOST_CHECK(it_parent != result_parent_child.m_tx_results.end()); + BOOST_CHECK_MESSAGE(it_parent->second.m_state.IsValid(), + "Package validation unexpectedly failed: " << it_parent->second.m_state.GetRejectReason()); + BOOST_CHECK(it_child != result_parent_child.m_tx_results.end()); + BOOST_CHECK_MESSAGE(it_child->second.m_state.IsValid(), + "Package validation unexpectedly failed: " << it_child->second.m_state.GetRejectReason()); + + + // A single, giant transaction submitted through ProcessNewPackage fails on single tx policy. + CTransactionRef giant_ptx = create_placeholder_tx(999, 999); + BOOST_CHECK(GetVirtualTransactionSize(*giant_ptx) > MAX_PACKAGE_SIZE * 1000); + auto result_single_large = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, {giant_ptx}, /*test_accept=*/true); + BOOST_CHECK(result_single_large.m_state.IsInvalid()); + BOOST_CHECK_EQUAL(result_single_large.m_state.GetResult(), PackageValidationResult::PCKG_TX); + BOOST_CHECK_EQUAL(result_single_large.m_state.GetRejectReason(), "transaction failed"); + auto it_giant_tx = result_single_large.m_tx_results.find(giant_ptx->GetWitnessHash()); + BOOST_CHECK(it_giant_tx != result_single_large.m_tx_results.end()); + BOOST_CHECK_EQUAL(it_giant_tx->second.m_state.GetRejectReason(), "tx-size"); + + // Check that mempool size hasn't changed. + BOOST_CHECK_EQUAL(m_node.mempool->size(), initialPoolSize); +} +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidation_tests.cpp b/src/test/txvalidation_tests.cpp index ade9e210f2..c71ab01af4 100644 --- a/src/test/txvalidation_tests.cpp +++ b/src/test/txvalidation_tests.cpp @@ -37,8 +37,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_reject_coinbase, TestChain100Setup) LOCK(cs_main); unsigned int initialPoolSize = m_node.mempool->size(); - const MempoolAcceptResult result = AcceptToMemoryPool(m_node.chainman->ActiveChainstate(), *m_node.mempool, MakeTransactionRef(coinbaseTx), - true /* bypass_limits */); + const MempoolAcceptResult result = m_node.chainman->ProcessTransaction(MakeTransactionRef(coinbaseTx)); BOOST_CHECK(result.m_result_type == MempoolAcceptResult::ResultType::INVALID); @@ -50,99 +49,4 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_reject_coinbase, TestChain100Setup) BOOST_CHECK_EQUAL(result.m_state.GetRejectReason(), "coinbase"); BOOST_CHECK(result.m_state.GetResult() == TxValidationResult::TX_CONSENSUS); } - -// Create placeholder transactions that have no meaning. -inline CTransactionRef create_placeholder_tx(size_t num_inputs, size_t num_outputs) -{ - CMutableTransaction mtx = CMutableTransaction(); - mtx.vin.resize(num_inputs); - mtx.vout.resize(num_outputs); - auto random_script = CScript() << ToByteVector(InsecureRand256()) << ToByteVector(InsecureRand256()); - for (size_t i{0}; i < num_inputs; ++i) { - mtx.vin[i].prevout.hash = InsecureRand256(); - mtx.vin[i].prevout.n = 0; - mtx.vin[i].scriptSig = random_script; - } - for (size_t o{0}; o < num_outputs; ++o) { - mtx.vout[o].nValue = 1 * CENT; - mtx.vout[o].scriptPubKey = random_script; - } - return MakeTransactionRef(mtx); -} - -BOOST_FIXTURE_TEST_CASE(package_tests, TestChain100Setup) -{ - LOCK(cs_main); - unsigned int initialPoolSize = m_node.mempool->size(); - - // Parent and Child Package - CKey parent_key; - parent_key.MakeNewKey(true); - CScript parent_locking_script = GetScriptForDestination(PKHash(parent_key.GetPubKey())); - auto mtx_parent = CreateValidMempoolTransaction(/* input_transaction */ m_coinbase_txns[0], /* vout */ 0, - /* input_height */ 0, /* input_signing_key */ coinbaseKey, - /* output_destination */ parent_locking_script, - /* output_amount */ CAmount(49 * COIN), /* submit */ false); - CTransactionRef tx_parent = MakeTransactionRef(mtx_parent); - - CKey child_key; - child_key.MakeNewKey(true); - CScript child_locking_script = GetScriptForDestination(PKHash(child_key.GetPubKey())); - auto mtx_child = CreateValidMempoolTransaction(/* input_transaction */ tx_parent, /* vout */ 0, - /* input_height */ 101, /* input_signing_key */ parent_key, - /* output_destination */ child_locking_script, - /* output_amount */ CAmount(48 * COIN), /* submit */ false); - CTransactionRef tx_child = MakeTransactionRef(mtx_child); - const auto result_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, {tx_parent, tx_child}, /* test_accept */ true); - BOOST_CHECK_MESSAGE(result_parent_child.m_state.IsValid(), - "Package validation unexpectedly failed: " << result_parent_child.m_state.GetRejectReason()); - auto it_parent = result_parent_child.m_tx_results.find(tx_parent->GetWitnessHash()); - auto it_child = result_parent_child.m_tx_results.find(tx_child->GetWitnessHash()); - BOOST_CHECK(it_parent != result_parent_child.m_tx_results.end()); - BOOST_CHECK_MESSAGE(it_parent->second.m_state.IsValid(), - "Package validation unexpectedly failed: " << it_parent->second.m_state.GetRejectReason()); - BOOST_CHECK(it_child != result_parent_child.m_tx_results.end()); - BOOST_CHECK_MESSAGE(it_child->second.m_state.IsValid(), - "Package validation unexpectedly failed: " << it_child->second.m_state.GetRejectReason()); - - // Packages can't have more than 25 transactions. - Package package_too_many; - package_too_many.reserve(MAX_PACKAGE_COUNT + 1); - for (size_t i{0}; i < MAX_PACKAGE_COUNT + 1; ++i) { - package_too_many.emplace_back(create_placeholder_tx(1, 1)); - } - auto result_too_many = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_too_many, /* test_accept */ true); - BOOST_CHECK(result_too_many.m_state.IsInvalid()); - BOOST_CHECK_EQUAL(result_too_many.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); - BOOST_CHECK_EQUAL(result_too_many.m_state.GetRejectReason(), "package-too-many-transactions"); - - // Packages can't have a total size of more than 101KvB. - CTransactionRef large_ptx = create_placeholder_tx(150, 150); - Package package_too_large; - auto size_large = GetVirtualTransactionSize(*large_ptx); - size_t total_size{0}; - while (total_size <= MAX_PACKAGE_SIZE * 1000) { - package_too_large.push_back(large_ptx); - total_size += size_large; - } - BOOST_CHECK(package_too_large.size() <= MAX_PACKAGE_COUNT); - auto result_too_large = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_too_large, /* test_accept */ true); - BOOST_CHECK(result_too_large.m_state.IsInvalid()); - BOOST_CHECK_EQUAL(result_too_large.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); - BOOST_CHECK_EQUAL(result_too_large.m_state.GetRejectReason(), "package-too-large"); - - // A single, giant transaction submitted through ProcessNewPackage fails on single tx policy. - CTransactionRef giant_ptx = create_placeholder_tx(999, 999); - BOOST_CHECK(GetVirtualTransactionSize(*giant_ptx) > MAX_PACKAGE_SIZE * 1000); - auto result_single_large = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, {giant_ptx}, /* test_accept */ true); - BOOST_CHECK(result_single_large.m_state.IsInvalid()); - BOOST_CHECK_EQUAL(result_single_large.m_state.GetResult(), PackageValidationResult::PCKG_TX); - BOOST_CHECK_EQUAL(result_single_large.m_state.GetRejectReason(), "transaction failed"); - auto it_giant_tx = result_single_large.m_tx_results.find(giant_ptx->GetWitnessHash()); - BOOST_CHECK(it_giant_tx != result_single_large.m_tx_results.end()); - BOOST_CHECK_EQUAL(it_giant_tx->second.m_state.GetRejectReason(), "tx-size"); - - // Check that mempool size hasn't changed. - BOOST_CHECK_EQUAL(m_node.mempool->size(), initialPoolSize); -} BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index afb3ad0cfd..8be64531c4 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -36,8 +36,7 @@ BOOST_FIXTURE_TEST_CASE(tx_mempool_block_doublespend, Dersig100Setup) const auto ToMemPool = [this](const CMutableTransaction& tx) { LOCK(cs_main); - const MempoolAcceptResult result = AcceptToMemoryPool(m_node.chainman->ActiveChainstate(), *m_node.mempool, MakeTransactionRef(tx), - true /* bypass_limits */); + const MempoolAcceptResult result = m_node.chainman->ProcessTransaction(MakeTransactionRef(tx)); return result.m_result_type == MempoolAcceptResult::ResultType::VALID; }; diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h index e95573022c..a9092bd0ef 100644 --- a/src/test/util/chainstate.h +++ b/src/test/util/chainstate.h @@ -34,7 +34,8 @@ CreateAndActivateUTXOSnapshot(NodeContext& node, const fs::path root, F malleati FILE* outfile{fsbridge::fopen(snapshot_path, "wb")}; CAutoFile auto_outfile{outfile, SER_DISK, CLIENT_VERSION}; - UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), auto_outfile); + UniValue result = CreateUTXOSnapshot( + node, node.chainman->ActiveChainstate(), auto_outfile, snapshot_path, snapshot_path); BOOST_TEST_MESSAGE( "Wrote UTXO snapshot to " << fs::PathToString(snapshot_path.make_preferred()) << ": " << result.write()); @@ -47,7 +48,7 @@ CreateAndActivateUTXOSnapshot(NodeContext& node, const fs::path root, F malleati malleation(auto_infile, metadata); - return node.chainman->ActivateSnapshot(auto_infile, metadata, /*in_memory*/ true); + return node.chainman->ActivateSnapshot(auto_infile, metadata, /*in_memory=*/true); } diff --git a/src/test/util/mining.cpp b/src/test/util/mining.cpp index f6a11bc02e..8b9069bea3 100644 --- a/src/test/util/mining.cpp +++ b/src/test/util/mining.cpp @@ -7,8 +7,8 @@ #include <chainparams.h> #include <consensus/merkle.h> #include <key_io.h> -#include <miner.h> #include <node/context.h> +#include <node/miner.h> #include <pow.h> #include <script/standard.h> #include <test/util/script.h> diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index 28d7967078..696fd902f8 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -46,18 +46,18 @@ std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(int n_candida std::vector<NodeEvictionCandidate> candidates; for (int id = 0; id < n_candidates; ++id) { candidates.push_back({ - /* id */ id, - /* nTimeConnected */ static_cast<int64_t>(random_context.randrange(100)), - /* m_min_ping_time */ std::chrono::microseconds{random_context.randrange(100)}, - /* nLastBlockTime */ static_cast<int64_t>(random_context.randrange(100)), - /* nLastTXTime */ static_cast<int64_t>(random_context.randrange(100)), - /* fRelevantServices */ random_context.randbool(), - /* fRelayTxes */ random_context.randbool(), - /* fBloomFilter */ random_context.randbool(), - /* nKeyedNetGroup */ random_context.randrange(100), - /* prefer_evict */ random_context.randbool(), - /* m_is_local */ random_context.randbool(), - /* m_network */ ALL_NETWORKS[random_context.randrange(ALL_NETWORKS.size())], + /*id=*/id, + /*nTimeConnected=*/static_cast<int64_t>(random_context.randrange(100)), + /*m_min_ping_time=*/std::chrono::microseconds{random_context.randrange(100)}, + /*nLastBlockTime=*/static_cast<int64_t>(random_context.randrange(100)), + /*nLastTXTime=*/static_cast<int64_t>(random_context.randrange(100)), + /*fRelevantServices=*/random_context.randbool(), + /*fRelayTxes=*/random_context.randbool(), + /*fBloomFilter=*/random_context.randbool(), + /*nKeyedNetGroup=*/random_context.randrange(100), + /*prefer_evict=*/random_context.randbool(), + /*m_is_local=*/random_context.randbool(), + /*m_network=*/ALL_NETWORKS[random_context.randrange(ALL_NETWORKS.size())], }); } return candidates; diff --git a/src/test/util/net.h b/src/test/util/net.h index d89fc34b75..2de6e712a2 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -25,16 +25,16 @@ struct ConnmanTestMsg : public CConnman { void AddTestNode(CNode& node) { - LOCK(cs_vNodes); - vNodes.push_back(&node); + LOCK(m_nodes_mutex); + m_nodes.push_back(&node); } void ClearTestNodes() { - LOCK(cs_vNodes); - for (CNode* node : vNodes) { + LOCK(m_nodes_mutex); + for (CNode* node : m_nodes) { delete node; } - vNodes.clear(); + m_nodes.clear(); } void ProcessMessagesOnce(CNode& node) { m_msgproc->ProcessMessages(&node, flagInterruptMsgProc); } diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index a3c7564d76..f5cc88f4ce 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -13,9 +13,9 @@ #include <crypto/sha256.h> #include <init.h> #include <interfaces/chain.h> -#include <miner.h> #include <net.h> #include <net_processing.h> +#include <node/miner.h> #include <noui.h> #include <policy/fees.h> #include <pow.h> @@ -179,7 +179,7 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const m_node.chainman->InitializeChainstate(m_node.mempool.get()); m_node.chainman->ActiveChainstate().InitCoinsDB( - /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); assert(!m_node.chainman->ActiveChainstate().CanFlushToDisk()); m_node.chainman->ActiveChainstate().InitCoinsCache(1 << 23); assert(m_node.chainman->ActiveChainstate().CanFlushToDisk()); @@ -192,7 +192,7 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString())); } - m_node.addrman = std::make_unique<AddrMan>(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + m_node.addrman = std::make_unique<AddrMan>(/*asmap=*/std::vector<bool>(), /*deterministic=*/false, /*consistency_check_ratio=*/0); m_node.banman = std::make_unique<BanMan>(m_args.GetDataDirBase() / "banlist", nullptr, DEFAULT_MISBEHAVING_BANTIME); m_node.connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman); // Deterministic randomness for tests. m_node.peerman = PeerManager::make(chainparams, *m_node.connman, *m_node.addrman, @@ -315,7 +315,7 @@ CMutableTransaction TestChain100Setup::CreateValidMempoolTransaction(CTransactio // If submit=true, add transaction to the mempool. if (submit) { LOCK(cs_main); - const MempoolAcceptResult result = AcceptToMemoryPool(m_node.chainman->ActiveChainstate(), *m_node.mempool.get(), MakeTransactionRef(mempool_txn), /* bypass_limits */ false); + const MempoolAcceptResult result = m_node.chainman->ProcessTransaction(MakeTransactionRef(mempool_txn)); assert(result.m_result_type == MempoolAcceptResult::ResultType::VALID); } diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 7518cdb042..4f2ccb6ebb 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -56,7 +56,7 @@ void Seed(FastRandomContext& ctx); static inline void SeedInsecureRand(SeedRand seed = SeedRand::SEED) { if (seed == SeedRand::ZEROS) { - g_insecure_rand_ctx = FastRandomContext(/* deterministic */ true); + g_insecure_rand_ctx = FastRandomContext(/*fDeterministic=*/true); } else { Seed(g_insecure_rand_ctx); } @@ -227,4 +227,4 @@ private: const std::string m_reason; }; -#endif +#endif // BITCOIN_TEST_UTIL_SETUP_COMMON_H diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index b1300d06ba..9540cead24 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -142,21 +142,32 @@ BOOST_AUTO_TEST_CASE(util_HexStr) "04678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5f"); BOOST_CHECK_EQUAL( - HexStr(Span<const unsigned char>( - ParseHex_expected + sizeof(ParseHex_expected), - ParseHex_expected + sizeof(ParseHex_expected))), + HexStr(Span{ParseHex_expected}.last(0)), ""); BOOST_CHECK_EQUAL( - HexStr(Span<const unsigned char>(ParseHex_expected, ParseHex_expected)), + HexStr(Span{ParseHex_expected}.first(0)), ""); - std::vector<unsigned char> ParseHex_vec(ParseHex_expected, ParseHex_expected + 5); + { + const std::vector<char> in_s{ParseHex_expected, ParseHex_expected + 5}; + const Span<const uint8_t> in_u{MakeUCharSpan(in_s)}; + const Span<const std::byte> in_b{MakeByteSpan(in_s)}; + const std::string out_exp{"04678afdb0"}; + + BOOST_CHECK_EQUAL(HexStr(in_u), out_exp); + BOOST_CHECK_EQUAL(HexStr(in_s), out_exp); + BOOST_CHECK_EQUAL(HexStr(in_b), out_exp); + } +} - BOOST_CHECK_EQUAL( - HexStr(ParseHex_vec), - "04678afdb0" - ); +BOOST_AUTO_TEST_CASE(span_write_bytes) +{ + std::array mut_arr{uint8_t{0xaa}, uint8_t{0xbb}}; + const auto mut_bytes{MakeWritableByteSpan(mut_arr)}; + mut_bytes[1] = std::byte{0x11}; + BOOST_CHECK_EQUAL(mut_arr.at(0), 0xaa); + BOOST_CHECK_EQUAL(mut_arr.at(1), 0x11); } BOOST_AUTO_TEST_CASE(util_Join) @@ -2456,4 +2467,52 @@ BOOST_AUTO_TEST_CASE(remove_prefix) BOOST_CHECK_EQUAL(RemovePrefix("", ""), ""); } +BOOST_AUTO_TEST_CASE(util_ParseByteUnits) +{ + auto noop = ByteUnit::NOOP; + + // no multiplier + BOOST_CHECK_EQUAL(ParseByteUnits("1", noop).value(), 1); + BOOST_CHECK_EQUAL(ParseByteUnits("0", noop).value(), 0); + + BOOST_CHECK_EQUAL(ParseByteUnits("1k", noop).value(), 1000ULL); + BOOST_CHECK_EQUAL(ParseByteUnits("1K", noop).value(), 1ULL << 10); + + BOOST_CHECK_EQUAL(ParseByteUnits("2m", noop).value(), 2'000'000ULL); + BOOST_CHECK_EQUAL(ParseByteUnits("2M", noop).value(), 2ULL << 20); + + BOOST_CHECK_EQUAL(ParseByteUnits("3g", noop).value(), 3'000'000'000ULL); + BOOST_CHECK_EQUAL(ParseByteUnits("3G", noop).value(), 3ULL << 30); + + BOOST_CHECK_EQUAL(ParseByteUnits("4t", noop).value(), 4'000'000'000'000ULL); + BOOST_CHECK_EQUAL(ParseByteUnits("4T", noop).value(), 4ULL << 40); + + // check default multiplier + BOOST_CHECK_EQUAL(ParseByteUnits("5", ByteUnit::K).value(), 5ULL << 10); + + // NaN + BOOST_CHECK(!ParseByteUnits("", noop)); + BOOST_CHECK(!ParseByteUnits("foo", noop)); + + // whitespace + BOOST_CHECK(!ParseByteUnits("123m ", noop)); + BOOST_CHECK(!ParseByteUnits(" 123m", noop)); + + // no +- + BOOST_CHECK(!ParseByteUnits("-123m", noop)); + BOOST_CHECK(!ParseByteUnits("+123m", noop)); + + // zero padding + BOOST_CHECK_EQUAL(ParseByteUnits("020M", noop).value(), 20ULL << 20); + + // fractions not allowed + BOOST_CHECK(!ParseByteUnits("0.5T", noop)); + + // overflow + BOOST_CHECK(!ParseByteUnits("18446744073709551615g", noop)); + + // invalid unit + BOOST_CHECK(!ParseByteUnits("1x", noop)); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 8f4ff6815b..3efa74fcc3 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -7,7 +7,7 @@ #include <chainparams.h> #include <consensus/merkle.h> #include <consensus/validation.h> -#include <miner.h> +#include <node/miner.h> #include <pow.h> #include <random.h> #include <script/standard.h> @@ -222,7 +222,7 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) { bool ignored; auto ProcessBlock = [&](std::shared_ptr<const CBlock> block) -> bool { - return Assert(m_node.chainman)->ProcessNewBlock(Params(), block, /* fForceProcessing */ true, /* fNewBlock */ &ignored); + return Assert(m_node.chainman)->ProcessNewBlock(Params(), block, /*force_processing=*/true, /*new_block=*/&ignored); }; // Process all mined blocks @@ -273,7 +273,7 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) { LOCK(cs_main); for (const auto& tx : txs) { - const MempoolAcceptResult result = AcceptToMemoryPool(m_node.chainman->ActiveChainstate(), *m_node.mempool, tx, false /* bypass_limits */); + const MempoolAcceptResult result = m_node.chainman->ProcessTransaction(tx); BOOST_REQUIRE(result.m_result_type == MempoolAcceptResult::ResultType::VALID); } } diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 9bb08f774f..b890ae4931 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -41,7 +41,7 @@ BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) CChainState& c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool)); c1.InitCoinsDB( - /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); // Add a coin to the in-memory cache, upsize once, then downsize. diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index be9e05a65e..a1f70e7e70 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -39,7 +39,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) CChainState& c1 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(&mempool)); chainstates.push_back(&c1); c1.InitCoinsDB( - /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); BOOST_CHECK(!manager.IsSnapshotActive()); @@ -68,7 +68,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) BOOST_CHECK_EQUAL(manager.SnapshotBlockhash().value(), snapshot_blockhash); c2.InitCoinsDB( - /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); WITH_LOCK(::cs_main, c2.InitCoinsCache(1 << 23)); // Unlike c1, which doesn't have any blocks. Gets us different tip, height. c2.LoadGenesisBlock(); @@ -118,7 +118,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) CChainState& c1 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool)); chainstates.push_back(&c1); c1.InitCoinsDB( - /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); { LOCK(::cs_main); @@ -136,7 +136,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) CChainState& c2 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool, GetRandHash())); chainstates.push_back(&c2); c2.InitCoinsDB( - /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); + /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); { LOCK(::cs_main); diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp index 9136c497ea..b4daceb72c 100644 --- a/src/test/validation_flush_tests.cpp +++ b/src/test/validation_flush_tests.cpp @@ -21,7 +21,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) CTxMemPool mempool; BlockManager blockman{}; CChainState chainstate{&mempool, blockman, *Assert(m_node.chainman)}; - chainstate.InitCoinsDB(/*cache_size_bytes*/ 1 << 10, /*in_memory*/ true, /*should_wipe*/ false); + chainstate.InitCoinsDB(/*cache_size_bytes=*/1 << 10, /*in_memory=*/true, /*should_wipe=*/false); WITH_LOCK(::cs_main, chainstate.InitCoinsCache(1 << 10)); constexpr bool is_64_bit = sizeof(void*) == 8; @@ -56,7 +56,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) // Without any coins in the cache, we shouldn't need to flush. BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes=*/0), CoinsCacheSizeState::OK); // If the initial memory allocations of cacheCoins don't match these common @@ -71,7 +71,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) } BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes=*/0), CoinsCacheSizeState::CRITICAL); BOOST_TEST_MESSAGE("Exiting cache flush tests early due to unsupported arch"); @@ -92,7 +92,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) print_view_mem_usage(view); BOOST_CHECK_EQUAL(view.AccessCoin(res).DynamicMemoryUsage(), COIN_SIZE); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes=*/0), CoinsCacheSizeState::OK); } @@ -100,26 +100,26 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate) for (int i{0}; i < 4; ++i) { add_coin(view); print_view_mem_usage(view); - if (chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0) == + if (chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes=*/0) == CoinsCacheSizeState::CRITICAL) { break; } } BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 0), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes=*/0), CoinsCacheSizeState::CRITICAL); // Passing non-zero max mempool usage should allow us more headroom. BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes=*/1 << 10), CoinsCacheSizeState::OK); for (int i{0}; i < 3; ++i) { add_coin(view); print_view_mem_usage(view); BOOST_CHECK_EQUAL( - chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes*/ 1 << 10), + chainstate.GetCoinsCacheSizeState(MAX_COINS_CACHE_BYTES, /*max_mempool_size_bytes=*/1 << 10), CoinsCacheSizeState::OK); } diff --git a/src/threadinterrupt.h b/src/threadinterrupt.h index 2665f8a5be..42470c70ee 100644 --- a/src/threadinterrupt.h +++ b/src/threadinterrupt.h @@ -33,4 +33,4 @@ private: std::atomic<bool> flag; }; -#endif //BITCOIN_THREADINTERRUPT_H +#endif // BITCOIN_THREADINTERRUPT_H diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 55618a5c57..776981b7f7 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -22,19 +22,17 @@ #include <deque> #include <functional> #include <set> -#include <stdlib.h> #include <vector> -#include <boost/signals2/signal.hpp> -#include <boost/algorithm/string/split.hpp> #include <boost/algorithm/string/classification.hpp> #include <boost/algorithm/string/replace.hpp> +#include <boost/algorithm/string/split.hpp> -#include <event2/bufferevent.h> #include <event2/buffer.h> -#include <event2/util.h> +#include <event2/bufferevent.h> #include <event2/event.h> #include <event2/thread.h> +#include <event2/util.h> /** Default control port */ const std::string DEFAULT_TOR_CONTROL = "127.0.0.1:9051"; @@ -277,9 +275,15 @@ std::map<std::string,std::string> ParseTorReplyMapping(const std::string &s) if (j == 3 && value[i] > '3') { j--; } - escaped_value.push_back(strtol(value.substr(i, j).c_str(), nullptr, 8)); + const auto end{i + j}; + uint8_t val{0}; + while (i < end) { + val *= 8; + val += value[i++] - '0'; + } + escaped_value.push_back(char(val)); // Account for automatic incrementing at loop end - i += j - 1; + --i; } else { escaped_value.push_back(value[i]); } diff --git a/src/torcontrol.h b/src/torcontrol.h index 7258f27cb6..8fc852f843 100644 --- a/src/torcontrol.h +++ b/src/torcontrol.h @@ -157,4 +157,4 @@ public: static void reconnect_cb(evutil_socket_t fd, short what, void *arg); }; -#endif /* BITCOIN_TORCONTROL_H */ +#endif // BITCOIN_TORCONTROL_H diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 9bc2377c63..fcfc27d38e 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -5,6 +5,7 @@ #include <txmempool.h> +#include <chain.h> #include <coins.h> #include <consensus/consensus.h> #include <consensus/tx_verify.h> @@ -16,12 +17,80 @@ #include <util/moneystr.h> #include <util/system.h> #include <util/time.h> -#include <validation.h> #include <validationinterface.h> #include <cmath> #include <optional> +// Helpers for modifying CTxMemPool::mapTx, which is a boost multi_index. +struct update_descendant_state +{ + update_descendant_state(int64_t _modifySize, CAmount _modifyFee, int64_t _modifyCount) : + modifySize(_modifySize), modifyFee(_modifyFee), modifyCount(_modifyCount) + {} + + void operator() (CTxMemPoolEntry &e) + { e.UpdateDescendantState(modifySize, modifyFee, modifyCount); } + + private: + int64_t modifySize; + CAmount modifyFee; + int64_t modifyCount; +}; + +struct update_ancestor_state +{ + update_ancestor_state(int64_t _modifySize, CAmount _modifyFee, int64_t _modifyCount, int64_t _modifySigOpsCost) : + modifySize(_modifySize), modifyFee(_modifyFee), modifyCount(_modifyCount), modifySigOpsCost(_modifySigOpsCost) + {} + + void operator() (CTxMemPoolEntry &e) + { e.UpdateAncestorState(modifySize, modifyFee, modifyCount, modifySigOpsCost); } + + private: + int64_t modifySize; + CAmount modifyFee; + int64_t modifyCount; + int64_t modifySigOpsCost; +}; + +struct update_fee_delta +{ + explicit update_fee_delta(int64_t _feeDelta) : feeDelta(_feeDelta) { } + + void operator() (CTxMemPoolEntry &e) { e.UpdateFeeDelta(feeDelta); } + +private: + int64_t feeDelta; +}; + +struct update_lock_points +{ + explicit update_lock_points(const LockPoints& _lp) : lp(_lp) { } + + void operator() (CTxMemPoolEntry &e) { e.UpdateLockPoints(lp); } + +private: + const LockPoints& lp; +}; + +bool TestLockPointValidity(CChain& active_chain, const LockPoints& lp) +{ + AssertLockHeld(cs_main); + // If there are relative lock times then the maxInputBlock will be set + // If there are no relative lock times, the LockPoints don't depend on the chain + if (lp.maxInputBlock) { + // Check whether active_chain is an extension of the block at which the LockPoints + // calculation was valid. If not LockPoints are no longer valid + if (!active_chain.Contains(lp.maxInputBlock)) { + return false; + } + } + + // LockPoints still valid + return true; +} + CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& tx, CAmount fee, int64_t time, unsigned int entry_height, bool spends_coinbase, int64_t sigops_cost, LockPoints lp) @@ -277,7 +346,7 @@ bool CTxMemPool::CalculateMemPoolAncestors(const CTxMemPoolEntry &entry, void CTxMemPool::UpdateAncestorsOf(bool add, txiter it, setEntries &setAncestors) { - CTxMemPoolEntry::Parents parents = it->GetMemPoolParents(); + const CTxMemPoolEntry::Parents& parents = it->GetMemPoolParentsConst(); // add or remove this tx as a child of each parent for (const CTxMemPoolEntry& parent : parents) { UpdateChild(mapTx.iterator_to(parent), it, add); @@ -564,44 +633,27 @@ void CTxMemPool::removeRecursive(const CTransaction &origTx, MemPoolRemovalReaso RemoveStaged(setAllRemoves, false, reason); } -void CTxMemPool::removeForReorg(CChainState& active_chainstate, int flags) +void CTxMemPool::removeForReorg(CChain& chain, std::function<bool(txiter)> check_final_and_mature) { // Remove transactions spending a coinbase which are now immature and no-longer-final transactions AssertLockHeld(cs); + AssertLockHeld(::cs_main); + setEntries txToRemove; for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { - const CTransaction& tx = it->GetTx(); - LockPoints lp = it->GetLockPoints(); - bool validLP = TestLockPointValidity(active_chainstate.m_chain, &lp); - CCoinsViewMemPool view_mempool(&active_chainstate.CoinsTip(), *this); - if (!CheckFinalTx(active_chainstate.m_chain.Tip(), tx, flags) - || !CheckSequenceLocks(active_chainstate.m_chain.Tip(), view_mempool, tx, flags, &lp, validLP)) { - // Note if CheckSequenceLocks fails the LockPoints may still be invalid - // So it's critical that we remove the tx and not depend on the LockPoints. - txToRemove.insert(it); - } else if (it->GetSpendsCoinbase()) { - for (const CTxIn& txin : tx.vin) { - indexed_transaction_set::const_iterator it2 = mapTx.find(txin.prevout.hash); - if (it2 != mapTx.end()) - continue; - const Coin &coin = active_chainstate.CoinsTip().AccessCoin(txin.prevout); - if (m_check_ratio != 0) assert(!coin.IsSpent()); - unsigned int nMemPoolHeight = active_chainstate.m_chain.Tip()->nHeight + 1; - if (coin.IsSpent() || (coin.IsCoinBase() && ((signed long)nMemPoolHeight) - coin.nHeight < COINBASE_MATURITY)) { - txToRemove.insert(it); - break; - } - } - } - if (!validLP) { - mapTx.modify(it, update_lock_points(lp)); - } + if (check_final_and_mature(it)) txToRemove.insert(it); } setEntries setAllRemoves; for (txiter it : txToRemove) { CalculateDescendants(it, setAllRemoves); } RemoveStaged(setAllRemoves, false, MemPoolRemovalReason::REORG); + for (indexed_transaction_set::const_iterator it = mapTx.begin(); it != mapTx.end(); it++) { + const LockPoints lp{it->GetLockPoints()}; + if (!TestLockPointValidity(chain, lp)) { + mapTx.modify(it, update_lock_points(lp)); + } + } } void CTxMemPool::removeConflicts(const CTransaction &tx) diff --git a/src/txmempool.h b/src/txmempool.h index 90b2aee371..f87ecc9cd0 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -25,12 +25,13 @@ #include <util/epochguard.h> #include <util/hasher.h> -#include <boost/multi_index_container.hpp> #include <boost/multi_index/hashed_index.hpp> #include <boost/multi_index/ordered_index.hpp> #include <boost/multi_index/sequenced_index.hpp> +#include <boost/multi_index_container.hpp> class CBlockIndex; +class CChain; class CChainState; extern RecursiveMutex cs_main; @@ -49,6 +50,11 @@ struct LockPoints { CBlockIndex* maxInputBlock{nullptr}; }; +/** + * Test whether the LockPoints height and time are still valid on the current chain + */ +bool TestLockPointValidity(CChain& active_chain, const LockPoints& lp) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + struct CompareIteratorByHash { // SFINAE for T where T is either a pointer type (e.g., a txiter) or a reference_wrapper<T> // (e.g. a wrapped CTxMemPoolEntry&) @@ -159,58 +165,6 @@ public: mutable Epoch::Marker m_epoch_marker; //!< epoch when last touched, useful for graph algorithms }; -// Helpers for modifying CTxMemPool::mapTx, which is a boost multi_index. -struct update_descendant_state -{ - update_descendant_state(int64_t _modifySize, CAmount _modifyFee, int64_t _modifyCount) : - modifySize(_modifySize), modifyFee(_modifyFee), modifyCount(_modifyCount) - {} - - void operator() (CTxMemPoolEntry &e) - { e.UpdateDescendantState(modifySize, modifyFee, modifyCount); } - - private: - int64_t modifySize; - CAmount modifyFee; - int64_t modifyCount; -}; - -struct update_ancestor_state -{ - update_ancestor_state(int64_t _modifySize, CAmount _modifyFee, int64_t _modifyCount, int64_t _modifySigOpsCost) : - modifySize(_modifySize), modifyFee(_modifyFee), modifyCount(_modifyCount), modifySigOpsCost(_modifySigOpsCost) - {} - - void operator() (CTxMemPoolEntry &e) - { e.UpdateAncestorState(modifySize, modifyFee, modifyCount, modifySigOpsCost); } - - private: - int64_t modifySize; - CAmount modifyFee; - int64_t modifyCount; - int64_t modifySigOpsCost; -}; - -struct update_fee_delta -{ - explicit update_fee_delta(int64_t _feeDelta) : feeDelta(_feeDelta) { } - - void operator() (CTxMemPoolEntry &e) { e.UpdateFeeDelta(feeDelta); } - -private: - int64_t feeDelta; -}; - -struct update_lock_points -{ - explicit update_lock_points(const LockPoints& _lp) : lp(_lp) { } - - void operator() (CTxMemPoolEntry &e) { e.UpdateLockPoints(lp); } - -private: - const LockPoints& lp; -}; - // extracts a transaction hash from CTxMemPoolEntry or CTransactionRef struct mempoolentry_txid { @@ -635,7 +589,10 @@ public: void addUnchecked(const CTxMemPoolEntry& entry, setEntries& setAncestors, bool validFeeEstimate = true) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main); void removeRecursive(const CTransaction& tx, MemPoolRemovalReason reason) EXCLUSIVE_LOCKS_REQUIRED(cs); - void removeForReorg(CChainState& active_chainstate, int flags) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main); + /** After reorg, check if mempool entries are now non-final, premature coinbase spends, or have + * invalid lockpoints. Update lockpoints and remove entries (and descendants of entries) that + * are no longer valid. */ + void removeForReorg(CChain& chain, std::function<bool(txiter)> check_final_and_mature) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main); void removeConflicts(const CTransaction& tx) EXCLUSIVE_LOCKS_REQUIRED(cs); void removeForBlock(const std::vector<CTransactionRef>& vtx, unsigned int nBlockHeight) EXCLUSIVE_LOCKS_REQUIRED(cs); diff --git a/src/util/error.cpp b/src/util/error.cpp index 48c81693f3..d019ba018d 100644 --- a/src/util/error.cpp +++ b/src/util/error.cpp @@ -20,9 +20,9 @@ bilingual_str TransactionErrorString(const TransactionError err) case TransactionError::P2P_DISABLED: return Untranslated("Peer-to-peer functionality missing or disabled"); case TransactionError::MEMPOOL_REJECTED: - return Untranslated("Transaction rejected by AcceptToMemoryPool"); + return Untranslated("Transaction rejected by mempool"); case TransactionError::MEMPOOL_ERROR: - return Untranslated("AcceptToMemoryPool failed"); + return Untranslated("Mempool internal error"); case TransactionError::INVALID_PSBT: return Untranslated("PSBT is not well-formed"); case TransactionError::PSBT_MISMATCH: diff --git a/src/util/overloaded.h b/src/util/overloaded.h new file mode 100644 index 0000000000..6be7453f81 --- /dev/null +++ b/src/util/overloaded.h @@ -0,0 +1,22 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_OVERLOADED_H +#define BITCOIN_UTIL_OVERLOADED_H + +namespace util { +//! Overloaded helper for std::visit. This helper and std::visit in general are +//! useful to write code that switches on a variant type. Unlike if/else-if and +//! switch/case statements, std::visit will trigger compile errors if there are +//! unhandled cases. +//! +//! Implementation comes from and example usage can be found at +//! https://en.cppreference.com/w/cpp/utility/variant/visit#Example +template<class... Ts> struct Overloaded : Ts... { using Ts::operator()...; }; + +//! Explicit deduction guide (not needed as of C++20) +template<class... Ts> Overloaded(Ts...) -> Overloaded<Ts...>; +} // namespace util + +#endif // BITCOIN_UTIL_OVERLOADED_H diff --git a/src/util/readwritefile.h b/src/util/readwritefile.h index 1dab874b38..a59d0be131 100644 --- a/src/util/readwritefile.h +++ b/src/util/readwritefile.h @@ -25,4 +25,4 @@ std::pair<bool,std::string> ReadBinaryFile(const fs::path &filename, size_t maxs */ bool WriteBinaryFile(const fs::path &filename, const std::string &data); -#endif /* BITCOIN_UTIL_READWRITEFILE_H */ +#endif // BITCOIN_UTIL_READWRITEFILE_H diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index 15bd07b374..430f1963ea 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -11,6 +11,7 @@ #include <algorithm> #include <cstdlib> #include <cstring> +#include <limits> #include <optional> static const std::string CHARS_ALPHA_NUM = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; @@ -138,11 +139,6 @@ std::string EncodeBase64(Span<const unsigned char> input) return str; } -std::string EncodeBase64(const std::string& str) -{ - return EncodeBase64(MakeUCharSpan(str)); -} - std::vector<unsigned char> DecodeBase64(const char* p, bool* pf_invalid) { static const int decode64_table[256] = @@ -526,3 +522,48 @@ std::string HexStr(const Span<const uint8_t> s) assert(it == rv.end()); return rv; } + +std::optional<uint64_t> ParseByteUnits(const std::string& str, ByteUnit default_multiplier) +{ + if (str.empty()) { + return std::nullopt; + } + auto multiplier = default_multiplier; + char unit = str.back(); + switch (unit) { + case 'k': + multiplier = ByteUnit::k; + break; + case 'K': + multiplier = ByteUnit::K; + break; + case 'm': + multiplier = ByteUnit::m; + break; + case 'M': + multiplier = ByteUnit::M; + break; + case 'g': + multiplier = ByteUnit::g; + break; + case 'G': + multiplier = ByteUnit::G; + break; + case 't': + multiplier = ByteUnit::t; + break; + case 'T': + multiplier = ByteUnit::T; + break; + default: + unit = 0; + break; + } + + uint64_t unit_amount = static_cast<uint64_t>(multiplier); + auto parsed_num = ToIntegral<uint64_t>(unit ? str.substr(0, str.size() - 1) : str); + if (!parsed_num || parsed_num > std::numeric_limits<uint64_t>::max() / unit_amount) { // check overflow + return std::nullopt; + } + return *parsed_num * unit_amount; +} diff --git a/src/util/strencodings.h b/src/util/strencodings.h index eedb5ec2f8..08a5465de1 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -30,6 +30,23 @@ enum SafeChars }; /** + * Used by ParseByteUnits() + * Lowercase base 1000 + * Uppercase base 1024 +*/ +enum class ByteUnit : uint64_t { + NOOP = 1ULL, + k = 1000ULL, + K = 1024ULL, + m = 1'000'000ULL, + M = 1ULL << 20, + g = 1'000'000'000ULL, + G = 1ULL << 30, + t = 1'000'000'000'000ULL, + T = 1ULL << 40, +}; + +/** * Remove unsafe chars. Safe chars chosen to allow simple messages/URLs/email * addresses, but avoid anything even possibly remotely dangerous like & or > * @param[in] str The string to sanitize @@ -50,7 +67,8 @@ bool IsHexNumber(const std::string& str); std::vector<unsigned char> DecodeBase64(const char* p, bool* pf_invalid = nullptr); std::string DecodeBase64(const std::string& str, bool* pf_invalid = nullptr); std::string EncodeBase64(Span<const unsigned char> input); -std::string EncodeBase64(const std::string& str); +inline std::string EncodeBase64(Span<const std::byte> input) { return EncodeBase64(MakeUCharSpan(input)); } +inline std::string EncodeBase64(const std::string& str) { return EncodeBase64(MakeUCharSpan(str)); } std::vector<unsigned char> DecodeBase32(const char* p, bool* pf_invalid = nullptr); std::string DecodeBase32(const std::string& str, bool* pf_invalid = nullptr); @@ -189,6 +207,7 @@ std::optional<T> ToIntegral(const std::string& str) */ std::string HexStr(const Span<const uint8_t> s); inline std::string HexStr(const Span<const char> s) { return HexStr(MakeUCharSpan(s)); } +inline std::string HexStr(const Span<const std::byte> s) { return HexStr(MakeUCharSpan(s)); } /** * Format a paragraph of text to a fixed width, adding spaces for @@ -305,4 +324,17 @@ std::string ToUpper(const std::string& str); */ std::string Capitalize(std::string str); +/** + * Parse a string with suffix unit [k|K|m|M|g|G|t|T]. + * Must be a whole integer, fractions not allowed (0.5t), no whitespace or +- + * Lowercase units are 1000 base. Uppercase units are 1024 base. + * Examples: 2m,27M,19g,41T + * + * @param[in] str the string to convert into bytes + * @param[in] default_multiplier if no unit is found in str use this unit + * @returns optional uint64_t bytes from str or nullopt + * if ToIntegral is false, str is empty, trailing whitespace or overflow + */ +std::optional<uint64_t> ParseByteUnits(const std::string& str, ByteUnit default_multiplier); + #endif // BITCOIN_UTIL_STRENCODINGS_H diff --git a/src/util/string.h b/src/util/string.h index 5617e4acc1..07c87cfcda 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -103,4 +103,4 @@ template <typename T1, size_t PREFIX_LEN> std::equal(std::begin(prefix), std::end(prefix), std::begin(obj)); } -#endif // BITCOIN_UTIL_STRENCODINGS_H +#endif // BITCOIN_UTIL_STRING_H diff --git a/src/util/syscall_sandbox.cpp b/src/util/syscall_sandbox.cpp index bc69df44f4..6e1cc9b457 100644 --- a/src/util/syscall_sandbox.cpp +++ b/src/util/syscall_sandbox.cpp @@ -581,7 +581,7 @@ public: allowed_syscalls.insert(__NR_fdatasync); // synchronize a file's in-core state with storage device allowed_syscalls.insert(__NR_flock); // apply or remove an advisory lock on an open file allowed_syscalls.insert(__NR_fstat); // get file status - allowed_syscalls.insert(__NR_newfstatat); // get file status + allowed_syscalls.insert(__NR_fstatfs); // get file system status allowed_syscalls.insert(__NR_fsync); // synchronize a file's in-core state with storage device allowed_syscalls.insert(__NR_ftruncate); // truncate a file to a specified length allowed_syscalls.insert(__NR_getcwd); // get current working directory @@ -589,6 +589,7 @@ public: allowed_syscalls.insert(__NR_getdents64); // get directory entries allowed_syscalls.insert(__NR_lstat); // get file status allowed_syscalls.insert(__NR_mkdir); // create a directory + allowed_syscalls.insert(__NR_newfstatat); // get file status allowed_syscalls.insert(__NR_open); // open and possibly create a file allowed_syscalls.insert(__NR_openat); // open and possibly create a file allowed_syscalls.insert(__NR_readlink); // read value of a symbolic link diff --git a/src/util/trace.h b/src/util/trace.h index 9c92cb10e7..bb901e05da 100644 --- a/src/util/trace.h +++ b/src/util/trace.h @@ -42,4 +42,4 @@ #endif -#endif /* BITCOIN_UTIL_TRACE_H */ +#endif // BITCOIN_UTIL_TRACE_H diff --git a/src/validation.cpp b/src/validation.cpp index 207cdc8233..91ef961964 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -212,24 +212,6 @@ bool CheckFinalTx(const CBlockIndex* active_chain_tip, const CTransaction &tx, i return IsFinalTx(tx, nBlockHeight, nBlockTime); } -bool TestLockPointValidity(CChain& active_chain, const LockPoints* lp) -{ - AssertLockHeld(cs_main); - assert(lp); - // If there are relative lock times then the maxInputBlock will be set - // If there are no relative lock times, the LockPoints don't depend on the chain - if (lp->maxInputBlock) { - // Check whether active_chain is an extension of the block at which the LockPoints - // calculation was valid. If not LockPoints are no longer valid - if (!active_chain.Contains(lp->maxInputBlock)) { - return false; - } - } - - // LockPoints still valid - return true; -} - bool CheckSequenceLocks(CBlockIndex* tip, const CCoinsView& coins_view, const CTransaction& tx, @@ -349,8 +331,8 @@ void CChainState::MaybeUpdateMempoolForReorg( while (it != disconnectpool.queuedTx.get<insertion_order>().rend()) { // ignore validation errors in resurrected transactions if (!fAddToMempool || (*it)->IsCoinBase() || - AcceptToMemoryPool( - *this, *m_mempool, *it, true /* bypass_limits */).m_result_type != + AcceptToMemoryPool(*m_mempool, *this, *it, GetTime(), + /* bypass_limits= */true, /* test_accept= */ false).m_result_type != MempoolAcceptResult::ResultType::VALID) { // If the transaction doesn't make it in to the mempool, remove any // transactions that depend on it (which would now be orphans). @@ -368,8 +350,39 @@ void CChainState::MaybeUpdateMempoolForReorg( // the disconnectpool that were added back and cleans up the mempool state. m_mempool->UpdateTransactionsFromBlock(vHashUpdate); + const auto check_final_and_mature = [this, flags=STANDARD_LOCKTIME_VERIFY_FLAGS](CTxMemPool::txiter it) + EXCLUSIVE_LOCKS_REQUIRED(m_mempool->cs, ::cs_main) { + bool should_remove = false; + AssertLockHeld(m_mempool->cs); + AssertLockHeld(::cs_main); + const CTransaction& tx = it->GetTx(); + LockPoints lp = it->GetLockPoints(); + const bool validLP{TestLockPointValidity(m_chain, lp)}; + CCoinsViewMemPool view_mempool(&CoinsTip(), *m_mempool); + if (!CheckFinalTx(m_chain.Tip(), tx, flags) + || !CheckSequenceLocks(m_chain.Tip(), view_mempool, tx, flags, &lp, validLP)) { + // Note if CheckSequenceLocks fails the LockPoints may still be invalid + // So it's critical that we remove the tx and not depend on the LockPoints. + should_remove = true; + } else if (it->GetSpendsCoinbase()) { + for (const CTxIn& txin : tx.vin) { + auto it2 = m_mempool->mapTx.find(txin.prevout.hash); + if (it2 != m_mempool->mapTx.end()) + continue; + const Coin &coin = CoinsTip().AccessCoin(txin.prevout); + assert(!coin.IsSpent()); + const auto mempool_spend_height{m_chain.Tip()->nHeight + 1}; + if (coin.IsSpent() || (coin.IsCoinBase() && mempool_spend_height - coin.nHeight < COINBASE_MATURITY)) { + should_remove = true; + break; + } + } + } + return should_remove; + }; + // We also need to remove any now-immature transactions - m_mempool->removeForReorg(*this, STANDARD_LOCKTIME_VERIFY_FLAGS); + m_mempool->removeForReorg(m_chain, check_final_and_mature); // Re-limit mempool size, in case we added any transactions LimitMempoolSize( *m_mempool, @@ -450,7 +463,36 @@ public: /** Whether we allow transactions to replace mempool transactions by BIP125 rules. If false, * any transaction spending the same inputs as a transaction in the mempool is considered * a conflict. */ - const bool m_allow_bip125_replacement{true}; + const bool m_allow_bip125_replacement; + + /** Parameters for single transaction mempool validation. */ + static ATMPArgs SingleAccept(const CChainParams& chainparams, int64_t accept_time, + bool bypass_limits, std::vector<COutPoint>& coins_to_uncache, + bool test_accept) { + return ATMPArgs{/* m_chainparams */ chainparams, + /* m_accept_time */ accept_time, + /* m_bypass_limits */ bypass_limits, + /* m_coins_to_uncache */ coins_to_uncache, + /* m_test_accept */ test_accept, + /* m_allow_bip125_replacement */ true, + }; + } + + /** Parameters for test package mempool validation through testmempoolaccept. */ + static ATMPArgs PackageTestAccept(const CChainParams& chainparams, int64_t accept_time, + std::vector<COutPoint>& coins_to_uncache) { + return ATMPArgs{/* m_chainparams */ chainparams, + /* m_accept_time */ accept_time, + /* m_bypass_limits */ false, + /* m_coins_to_uncache */ coins_to_uncache, + /* m_test_accept */ true, + /* m_allow_bip125_replacement */ false, + }; + } + + // No default ctor to avoid exposing details to clients and allowing the possibility of + // mixing up the order of the arguments. Use static functions above instead. + ATMPArgs() = delete; }; // Single transaction acceptance @@ -468,13 +510,29 @@ private: // of checking a given transaction. struct Workspace { explicit Workspace(const CTransactionRef& ptx) : m_ptx(ptx), m_hash(ptx->GetHash()) {} + /** Txids of mempool transactions that this transaction directly conflicts with. */ std::set<uint256> m_conflicts; + /** Iterators to mempool entries that this transaction directly conflicts with. */ + CTxMemPool::setEntries m_iters_conflicting; + /** Iterators to all mempool entries that would be replaced by this transaction, including + * those it directly conflicts with and their descendants. */ CTxMemPool::setEntries m_all_conflicting; + /** All mempool ancestors of this transaction. */ CTxMemPool::setEntries m_ancestors; + /** Mempool entry constructed for this transaction. Constructed in PreChecks() but not + * inserted into the mempool until Finalize(). */ std::unique_ptr<CTxMemPoolEntry> m_entry; + /** Pointers to the transactions that have been removed from the mempool and replaced by + * this transaction, used to return to the MemPoolAccept caller. Only populated if + * validation is successful and the original transactions are removed. */ std::list<CTransactionRef> m_replaced_transactions; + /** Virtual size of the transaction as used by the mempool, calculated using serialized size + * of the transaction and sigops. */ + int64_t m_vsize; + /** Fees paid by this transaction: total input amounts subtracted by total output amounts. */ CAmount m_base_fees; + /** Base fees + any fee delta set by the user with prioritisetransaction. */ CAmount m_modified_fees; /** Total modified fees of all transactions being replaced. */ CAmount m_conflicting_fees{0}; @@ -482,8 +540,12 @@ private: size_t m_conflicting_size{0}; const CTransactionRef& m_ptx; + /** Txid. */ const uint256& m_hash; TxValidationState m_state; + /** A temporary cache containing serialized transaction data for signature verification. + * Reused across PolicyScriptChecks and ConsensusScriptChecks. */ + PrecomputedTransactionData m_precomputed_txdata; }; // Run the policy checks on a given transaction, excluding any script checks. @@ -492,15 +554,23 @@ private: // only tests that are fast should be done here (to avoid CPU DoS). bool PreChecks(ATMPArgs& args, Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); + // Run checks for mempool replace-by-fee. + bool ReplacementChecks(Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); + + // Enforce package mempool ancestor/descendant limits (distinct from individual + // ancestor/descendant limits done in PreChecks). + bool PackageMempoolChecks(const std::vector<CTransactionRef>& txns, + PackageValidationState& package_state) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); + // Run the script checks using our policy flags. As this can be slow, we should // only invoke this on transactions that have otherwise passed policy checks. - bool PolicyScriptChecks(const ATMPArgs& args, Workspace& ws, PrecomputedTransactionData& txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); + bool PolicyScriptChecks(const ATMPArgs& args, Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); // Re-run the script checks, using consensus flags, and try to cache the // result in the scriptcache. This should be done after // PolicyScriptChecks(). This requires that all inputs either be in our // utxo set or in the mempool. - bool ConsensusScriptChecks(const ATMPArgs& args, Workspace& ws, PrecomputedTransactionData &txdata) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); + bool ConsensusScriptChecks(const ATMPArgs& args, Workspace& ws) EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); // Try to add the transaction to the mempool, removing any conflicts first. // Returns true if the transaction is in the mempool after any size @@ -536,6 +606,9 @@ private: // in-mempool conflicts; see below). size_t m_limit_descendants; size_t m_limit_descendant_size; + + /** Whether the transaction(s) would replace any mempool transactions. If so, RBF rules apply. */ + bool m_rbf{false}; }; bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) @@ -551,13 +624,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // Alias what we need out of ws TxValidationState& state = ws.m_state; - std::set<uint256>& setConflicts = ws.m_conflicts; - CTxMemPool::setEntries& allConflicting = ws.m_all_conflicting; - CTxMemPool::setEntries& setAncestors = ws.m_ancestors; std::unique_ptr<CTxMemPoolEntry>& entry = ws.m_entry; - CAmount& nModifiedFees = ws.m_modified_fees; - CAmount& nConflictingFees = ws.m_conflicting_fees; - size_t& nConflictingSize = ws.m_conflicting_size; if (!CheckTransaction(tx, state)) { return false; // state filled in by CheckTransaction @@ -603,7 +670,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // Transaction conflicts with a mempool tx, but we're not allowing replacements. return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "bip125-replacement-disallowed"); } - if (!setConflicts.count(ptxConflicting->GetHash())) + if (!ws.m_conflicts.count(ptxConflicting->GetHash())) { // Transactions that don't explicitly signal replaceability are // *not* replaceable with the current logic, even if one of their @@ -616,7 +683,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "txn-mempool-conflict"); } - setConflicts.insert(ptxConflicting->GetHash()); + ws.m_conflicts.insert(ptxConflicting->GetHash()); } } } @@ -656,6 +723,8 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // to coins_to_uncache) m_view.SetBackend(m_dummy); + assert(m_active_chainstate.m_blockman.LookupBlockIndex(m_view.GetBestBlock()) == m_active_chainstate.m_chain.Tip()); + // Only accept BIP68 sequence locked transactions that can be mined in the next // block; we don't want our mempool filled up with transactions that can't // be mined yet. @@ -664,13 +733,13 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) if (!CheckSequenceLocks(m_active_chainstate.m_chain.Tip(), m_view, tx, STANDARD_LOCKTIME_VERIFY_FLAGS, &lp)) return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-BIP68-final"); - if (!Consensus::CheckTxInputs(tx, state, m_view, m_active_chainstate.m_blockman.GetSpendHeight(m_view), ws.m_base_fees)) { + // The mempool holds txs for the next block, so pass height+1 to CheckTxInputs + if (!Consensus::CheckTxInputs(tx, state, m_view, m_active_chainstate.m_chain.Height() + 1, ws.m_base_fees)) { return false; // state filled in by CheckTxInputs } // Check for non-standard pay-to-script-hash in inputs - const bool taproot_active = DeploymentActiveAfter(m_active_chainstate.m_chain.Tip(), args.m_chainparams.GetConsensus(), Consensus::DEPLOYMENT_TAPROOT); - if (fRequireStandard && !AreInputsStandard(tx, m_view, taproot_active)) { + if (fRequireStandard && !AreInputsStandard(tx, m_view)) { return state.Invalid(TxValidationResult::TX_INPUTS_NOT_STANDARD, "bad-txns-nonstandard-inputs"); } @@ -680,9 +749,9 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) int64_t nSigOpsCost = GetTransactionSigOpCost(tx, m_view, STANDARD_SCRIPT_VERIFY_FLAGS); - // nModifiedFees includes any fee deltas from PrioritiseTransaction - nModifiedFees = ws.m_base_fees; - m_pool.ApplyDelta(hash, nModifiedFees); + // ws.m_modified_fees includes any fee deltas from PrioritiseTransaction + ws.m_modified_fees = ws.m_base_fees; + m_pool.ApplyDelta(hash, ws.m_modified_fees); // Keep track of transactions that spend a coinbase, which we re-scan // during reorgs to ensure COINBASE_MATURITY is still met. @@ -697,7 +766,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) entry.reset(new CTxMemPoolEntry(ptx, ws.m_base_fees, nAcceptTime, m_active_chainstate.m_chain.Height(), fSpendsCoinbase, nSigOpsCost, lp)); - unsigned int nSize = entry->GetTxSize(); + ws.m_vsize = entry->GetTxSize(); if (nSigOpsCost > MAX_STANDARD_TX_SIGOPS_COST) return state.Invalid(TxValidationResult::TX_NOT_STANDARD, "bad-txns-too-many-sigops", @@ -705,11 +774,11 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // No transactions are allowed below minRelayTxFee except from disconnected // blocks - if (!bypass_limits && !CheckFeeRate(nSize, nModifiedFees, state)) return false; + if (!bypass_limits && !CheckFeeRate(ws.m_vsize, ws.m_modified_fees, state)) return false; - const CTxMemPool::setEntries setIterConflicting = m_pool.GetIterSet(setConflicts); + ws.m_iters_conflicting = m_pool.GetIterSet(ws.m_conflicts); // Calculate in-mempool ancestors, up to a limit. - if (setConflicts.size() == 1) { + if (ws.m_conflicts.size() == 1) { // In general, when we receive an RBF transaction with mempool conflicts, we want to know whether we // would meet the chain limits after the conflicts have been removed. However, there isn't a practical // way to do this short of calculating the ancestor and descendant sets with an overlay cache of @@ -737,16 +806,16 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // the ancestor limits should be the same for both our new transaction and any conflicts). // We don't bother incrementing m_limit_descendants by the full removal count as that limit never comes // into force here (as we're only adding a single transaction). - assert(setIterConflicting.size() == 1); - CTxMemPool::txiter conflict = *setIterConflicting.begin(); + assert(ws.m_iters_conflicting.size() == 1); + CTxMemPool::txiter conflict = *ws.m_iters_conflicting.begin(); m_limit_descendants += 1; m_limit_descendant_size += conflict->GetSizeWithDescendants(); } std::string errString; - if (!m_pool.CalculateMemPoolAncestors(*entry, setAncestors, m_limit_ancestors, m_limit_ancestor_size, m_limit_descendants, m_limit_descendant_size, errString)) { - setAncestors.clear(); + if (!m_pool.CalculateMemPoolAncestors(*entry, ws.m_ancestors, m_limit_ancestors, m_limit_ancestor_size, m_limit_descendants, m_limit_descendant_size, errString)) { + ws.m_ancestors.clear(); // If CalculateMemPoolAncestors fails second time, we want the original error string. std::string dummy_err_string; // Contracting/payment channels CPFP carve-out: @@ -760,60 +829,85 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // to be secure by simply only having two immediately-spendable // outputs - one for each counterparty. For more info on the uses for // this, see https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2018-November/016518.html - if (nSize > EXTRA_DESCENDANT_TX_SIZE_LIMIT || - !m_pool.CalculateMemPoolAncestors(*entry, setAncestors, 2, m_limit_ancestor_size, m_limit_descendants + 1, m_limit_descendant_size + EXTRA_DESCENDANT_TX_SIZE_LIMIT, dummy_err_string)) { + if (ws.m_vsize > EXTRA_DESCENDANT_TX_SIZE_LIMIT || + !m_pool.CalculateMemPoolAncestors(*entry, ws.m_ancestors, 2, m_limit_ancestor_size, m_limit_descendants + 1, m_limit_descendant_size + EXTRA_DESCENDANT_TX_SIZE_LIMIT, dummy_err_string)) { return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "too-long-mempool-chain", errString); } } // A transaction that spends outputs that would be replaced by it is invalid. Now // that we have the set of all ancestors we can detect this - // pathological case by making sure setConflicts and setAncestors don't + // pathological case by making sure ws.m_conflicts and ws.m_ancestors don't // intersect. - if (const auto err_string{EntriesAndTxidsDisjoint(setAncestors, setConflicts, hash)}) { + if (const auto err_string{EntriesAndTxidsDisjoint(ws.m_ancestors, ws.m_conflicts, hash)}) { // We classify this as a consensus error because a transaction depending on something it // conflicts with would be inconsistent. return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-spends-conflicting-tx", *err_string); } + m_rbf = !ws.m_conflicts.empty(); + return true; +} - if (!setConflicts.empty()) { - CFeeRate newFeeRate(nModifiedFees, nSize); - // It's possible that the replacement pays more fees than its direct conflicts but not more - // than all conflicts (i.e. the direct conflicts have high-fee descendants). However, if the - // replacement doesn't pay more fees than its direct conflicts, then we can be sure it's not - // more economically rational to mine. Before we go digging through the mempool for all - // transactions that would need to be removed (direct conflicts and all descendants), check - // that the replacement transaction pays more than its direct conflicts. - if (const auto err_string{PaysMoreThanConflicts(setIterConflicting, newFeeRate, hash)}) { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", *err_string); - } +bool MemPoolAccept::ReplacementChecks(Workspace& ws) +{ + AssertLockHeld(cs_main); + AssertLockHeld(m_pool.cs); - // Calculate all conflicting entries and enforce BIP125 Rule #5. - if (const auto err_string{GetEntriesForConflicts(tx, m_pool, setIterConflicting, allConflicting)}) { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, - "too many potential replacements", *err_string); - } - // Enforce BIP125 Rule #2. - if (const auto err_string{HasNoNewUnconfirmed(tx, m_pool, setIterConflicting)}) { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, - "replacement-adds-unconfirmed", *err_string); - } + const CTransaction& tx = *ws.m_ptx; + const uint256& hash = ws.m_hash; + TxValidationState& state = ws.m_state; - // Check if it's economically rational to mine this transaction rather than the ones it - // replaces and pays for its own relay fees. Enforce BIP125 Rules #3 and #4. - for (CTxMemPool::txiter it : allConflicting) { - nConflictingFees += it->GetModifiedFee(); - nConflictingSize += it->GetTxSize(); - } - if (const auto err_string{PaysForRBF(nConflictingFees, nModifiedFees, nSize, ::incrementalRelayFee, hash)}) { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", *err_string); - } + CFeeRate newFeeRate(ws.m_modified_fees, ws.m_vsize); + // It's possible that the replacement pays more fees than its direct conflicts but not more + // than all conflicts (i.e. the direct conflicts have high-fee descendants). However, if the + // replacement doesn't pay more fees than its direct conflicts, then we can be sure it's not + // more economically rational to mine. Before we go digging through the mempool for all + // transactions that would need to be removed (direct conflicts and all descendants), check + // that the replacement transaction pays more than its direct conflicts. + if (const auto err_string{PaysMoreThanConflicts(ws.m_iters_conflicting, newFeeRate, hash)}) { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", *err_string); + } + + // Calculate all conflicting entries and enforce BIP125 Rule #5. + if (const auto err_string{GetEntriesForConflicts(tx, m_pool, ws.m_iters_conflicting, ws.m_all_conflicting)}) { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, + "too many potential replacements", *err_string); + } + // Enforce BIP125 Rule #2. + if (const auto err_string{HasNoNewUnconfirmed(tx, m_pool, ws.m_iters_conflicting)}) { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, + "replacement-adds-unconfirmed", *err_string); + } + // Check if it's economically rational to mine this transaction rather than the ones it + // replaces and pays for its own relay fees. Enforce BIP125 Rules #3 and #4. + for (CTxMemPool::txiter it : ws.m_all_conflicting) { + ws.m_conflicting_fees += it->GetModifiedFee(); + ws.m_conflicting_size += it->GetTxSize(); + } + if (const auto err_string{PaysForRBF(ws.m_conflicting_fees, ws.m_modified_fees, ws.m_vsize, + ::incrementalRelayFee, hash)}) { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", *err_string); } return true; } -bool MemPoolAccept::PolicyScriptChecks(const ATMPArgs& args, Workspace& ws, PrecomputedTransactionData& txdata) +bool MemPoolAccept::PackageMempoolChecks(const std::vector<CTransactionRef>& txns, + PackageValidationState& package_state) +{ + AssertLockHeld(cs_main); + AssertLockHeld(m_pool.cs); + + std::string err_string; + if (!m_pool.CheckPackageLimits(txns, m_limit_ancestors, m_limit_ancestor_size, m_limit_descendants, + m_limit_descendant_size, err_string)) { + // This is a package-wide error, separate from an individual transaction error. + return package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package-mempool-limits", err_string); + } + return true; +} + +bool MemPoolAccept::PolicyScriptChecks(const ATMPArgs& args, Workspace& ws) { const CTransaction& tx = *ws.m_ptx; TxValidationState& state = ws.m_state; @@ -822,13 +916,13 @@ bool MemPoolAccept::PolicyScriptChecks(const ATMPArgs& args, Workspace& ws, Prec // Check input scripts and signatures. // This is done last to help prevent CPU exhaustion denial-of-service attacks. - if (!CheckInputScripts(tx, state, m_view, scriptVerifyFlags, true, false, txdata)) { + if (!CheckInputScripts(tx, state, m_view, scriptVerifyFlags, true, false, ws.m_precomputed_txdata)) { // SCRIPT_VERIFY_CLEANSTACK requires SCRIPT_VERIFY_WITNESS, so we // need to turn both off, and compare against just turning off CLEANSTACK // to see if the failure is specifically due to witness validation. TxValidationState state_dummy; // Want reported failures to be from first CheckInputScripts - if (!tx.HasWitness() && CheckInputScripts(tx, state_dummy, m_view, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true, false, txdata) && - !CheckInputScripts(tx, state_dummy, m_view, scriptVerifyFlags & ~SCRIPT_VERIFY_CLEANSTACK, true, false, txdata)) { + if (!tx.HasWitness() && CheckInputScripts(tx, state_dummy, m_view, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true, false, ws.m_precomputed_txdata) && + !CheckInputScripts(tx, state_dummy, m_view, scriptVerifyFlags & ~SCRIPT_VERIFY_CLEANSTACK, true, false, ws.m_precomputed_txdata)) { // Only the witness is missing, so the transaction itself may be fine. state.Invalid(TxValidationResult::TX_WITNESS_STRIPPED, state.GetRejectReason(), state.GetDebugMessage()); @@ -839,7 +933,7 @@ bool MemPoolAccept::PolicyScriptChecks(const ATMPArgs& args, Workspace& ws, Prec return true; } -bool MemPoolAccept::ConsensusScriptChecks(const ATMPArgs& args, Workspace& ws, PrecomputedTransactionData& txdata) +bool MemPoolAccept::ConsensusScriptChecks(const ATMPArgs& args, Workspace& ws) { const CTransaction& tx = *ws.m_ptx; const uint256& hash = ws.m_hash; @@ -862,9 +956,10 @@ bool MemPoolAccept::ConsensusScriptChecks(const ATMPArgs& args, Workspace& ws, P // invalid blocks (using TestBlockValidity), however allowing such // transactions into the mempool can be exploited as a DoS attack. unsigned int currentBlockScriptVerifyFlags = GetBlockScriptFlags(m_active_chainstate.m_chain.Tip(), chainparams.GetConsensus()); - if (!CheckInputsFromMempoolAndCache(tx, state, m_view, m_pool, currentBlockScriptVerifyFlags, txdata, m_active_chainstate.CoinsTip())) { - return error("%s: BUG! PLEASE REPORT THIS! CheckInputScripts failed against latest-block but not STANDARD flags %s, %s", - __func__, hash.ToString(), state.ToString()); + if (!CheckInputsFromMempoolAndCache(tx, state, m_view, m_pool, currentBlockScriptVerifyFlags, + ws.m_precomputed_txdata, m_active_chainstate.CoinsTip())) { + LogPrintf("BUG! PLEASE REPORT THIS! CheckInputScripts failed against latest-block but not STANDARD flags %s, %s\n", hash.ToString(), state.ToString()); + return Assume(false); } return true; @@ -877,24 +972,19 @@ bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) TxValidationState& state = ws.m_state; const bool bypass_limits = args.m_bypass_limits; - CTxMemPool::setEntries& allConflicting = ws.m_all_conflicting; - CTxMemPool::setEntries& setAncestors = ws.m_ancestors; - const CAmount& nModifiedFees = ws.m_modified_fees; - const CAmount& nConflictingFees = ws.m_conflicting_fees; - const size_t& nConflictingSize = ws.m_conflicting_size; std::unique_ptr<CTxMemPoolEntry>& entry = ws.m_entry; // Remove conflicting transactions from the mempool - for (CTxMemPool::txiter it : allConflicting) + for (CTxMemPool::txiter it : ws.m_all_conflicting) { LogPrint(BCLog::MEMPOOL, "replacing tx %s with %s for %s additional fees, %d delta bytes\n", it->GetTx().GetHash().ToString(), hash.ToString(), - FormatMoney(nModifiedFees - nConflictingFees), - (int)entry->GetTxSize() - (int)nConflictingSize); + FormatMoney(ws.m_modified_fees - ws.m_conflicting_fees), + (int)entry->GetTxSize() - (int)ws.m_conflicting_size); ws.m_replaced_transactions.push_back(it->GetSharedTx()); } - m_pool.RemoveStaged(allConflicting, false, MemPoolRemovalReason::REPLACED); + m_pool.RemoveStaged(ws.m_all_conflicting, false, MemPoolRemovalReason::REPLACED); // This transaction should only count for fee estimation if: // - it's not being re-added during a reorg which bypasses typical mempool fee limits @@ -903,7 +993,7 @@ bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) bool validForFeeEstimation = !bypass_limits && IsCurrentForFeeEstimation(m_active_chainstate) && m_pool.HasNoInputsOf(tx); // Store transaction in memory - m_pool.addUnchecked(*entry, setAncestors, validForFeeEstimation); + m_pool.addUnchecked(*entry, ws.m_ancestors, validForFeeEstimation); // trim mempool and check if tx was trimmed if (!bypass_limits) { @@ -923,26 +1013,24 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef if (!PreChecks(args, ws)) return MempoolAcceptResult::Failure(ws.m_state); - // Only compute the precomputed transaction data if we need to verify - // scripts (ie, other policy checks pass). We perform the inexpensive - // checks first and avoid hashing and signature verification unless those - // checks pass, to mitigate CPU exhaustion denial-of-service attacks. - PrecomputedTransactionData txdata; + if (m_rbf && !ReplacementChecks(ws)) return MempoolAcceptResult::Failure(ws.m_state); - if (!PolicyScriptChecks(args, ws, txdata)) return MempoolAcceptResult::Failure(ws.m_state); + // Perform the inexpensive checks first and avoid hashing and signature verification unless + // those checks pass, to mitigate CPU exhaustion denial-of-service attacks. + if (!PolicyScriptChecks(args, ws)) return MempoolAcceptResult::Failure(ws.m_state); - if (!ConsensusScriptChecks(args, ws, txdata)) return MempoolAcceptResult::Failure(ws.m_state); + if (!ConsensusScriptChecks(args, ws)) return MempoolAcceptResult::Failure(ws.m_state); // Tx was accepted, but not added if (args.m_test_accept) { - return MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_base_fees); + return MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, ws.m_base_fees); } if (!Finalize(args, ws)) return MempoolAcceptResult::Failure(ws.m_state); GetMainSignals().TransactionAddedToMempool(ptx, m_pool.GetAndIncrementSequence()); - return MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_base_fees); + return MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, ws.m_base_fees); } PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std::vector<CTransactionRef>& txns, ATMPArgs& args) @@ -981,18 +1069,12 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: // because it's unnecessary. Also, CPFP carve out can increase the limit for individual // transactions, but this exemption is not extended to packages in CheckPackageLimits(). std::string err_string; - if (txns.size() > 1 && - !m_pool.CheckPackageLimits(txns, m_limit_ancestors, m_limit_ancestor_size, m_limit_descendants, - m_limit_descendant_size, err_string)) { - // All transactions must have individually passed mempool ancestor and descendant limits - // inside of PreChecks(), so this is separate from an individual transaction error. - package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package-mempool-limits", err_string); + if (txns.size() > 1 && !PackageMempoolChecks(txns, package_state)) { return PackageMempoolAcceptResult(package_state, std::move(results)); } for (Workspace& ws : workspaces) { - PrecomputedTransactionData txdata; - if (!PolicyScriptChecks(args, ws, txdata)) { + if (!PolicyScriptChecks(args, ws)) { // Exit early to avoid doing pointless work. Update the failed tx result; the rest are unfinished. package_state.Invalid(PackageValidationResult::PCKG_TX, "transaction failed"); results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Failure(ws.m_state)); @@ -1002,7 +1084,8 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: // When test_accept=true, transactions that pass PolicyScriptChecks are valid because there are // no further mempool checks (passing PolicyScriptChecks implies passing ConsensusScriptChecks). results.emplace(ws.m_ptx->GetWitnessHash(), - MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_base_fees)); + MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), + ws.m_vsize, ws.m_base_fees)); } } @@ -1011,17 +1094,13 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: } // anon namespace -/** (try to) add transaction to memory pool with a specified acceptance time **/ -static MempoolAcceptResult AcceptToMemoryPoolWithTime(const CChainParams& chainparams, CTxMemPool& pool, - CChainState& active_chainstate, - const CTransactionRef &tx, int64_t nAcceptTime, - bool bypass_limits, bool test_accept) - EXCLUSIVE_LOCKS_REQUIRED(cs_main) +MempoolAcceptResult AcceptToMemoryPool(CTxMemPool& pool, CChainState& active_chainstate, const CTransactionRef& tx, + int64_t accept_time, bool bypass_limits, bool test_accept) + EXCLUSIVE_LOCKS_REQUIRED(cs_main) { + const CChainParams& chainparams{active_chainstate.m_params}; std::vector<COutPoint> coins_to_uncache; - MemPoolAccept::ATMPArgs args { chainparams, nAcceptTime, bypass_limits, coins_to_uncache, - test_accept, /* m_allow_bip125_replacement */ true }; - + auto args = MemPoolAccept::ATMPArgs::SingleAccept(chainparams, accept_time, bypass_limits, coins_to_uncache, test_accept); const MempoolAcceptResult result = MemPoolAccept(pool, active_chainstate).AcceptSingleTransaction(tx, args); if (result.m_result_type != MempoolAcceptResult::ResultType::VALID) { // Remove coins that were not present in the coins cache before calling @@ -1038,12 +1117,6 @@ static MempoolAcceptResult AcceptToMemoryPoolWithTime(const CChainParams& chainp return result; } -MempoolAcceptResult AcceptToMemoryPool(CChainState& active_chainstate, CTxMemPool& pool, const CTransactionRef& tx, - bool bypass_limits, bool test_accept) -{ - return AcceptToMemoryPoolWithTime(Params(), pool, active_chainstate, tx, GetTime(), bypass_limits, test_accept); -} - PackageMempoolAcceptResult ProcessNewPackage(CChainState& active_chainstate, CTxMemPool& pool, const Package& package, bool test_accept) { @@ -1054,8 +1127,7 @@ PackageMempoolAcceptResult ProcessNewPackage(CChainState& active_chainstate, CTx std::vector<COutPoint> coins_to_uncache; const CChainParams& chainparams = Params(); - MemPoolAccept::ATMPArgs args { chainparams, GetTime(), /* bypass_limits */ false, coins_to_uncache, - test_accept, /* m_allow_bip125_replacement */ false }; + auto args = MemPoolAccept::ATMPArgs::PackageTestAccept(chainparams, GetTime(), coins_to_uncache); const PackageMempoolAcceptResult result = MemPoolAccept(pool, active_chainstate).AcceptMultipleTransactions(package, args); // Uncache coins pertaining to transactions that were not submitted to the mempool. @@ -1097,8 +1169,8 @@ CChainState::CChainState( ChainstateManager& chainman, std::optional<uint256> from_snapshot_blockhash) : m_mempool(mempool), - m_params(::Params()), m_blockman(blockman), + m_params(::Params()), m_chainman(chainman), m_from_snapshot_blockhash(from_snapshot_blockhash) {} @@ -1242,14 +1314,6 @@ bool CScriptCheck::operator()() { return VerifyScript(scriptSig, m_tx_out.scriptPubKey, witness, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, m_tx_out.nValue, cacheStore, *txdata), &error); } -int BlockManager::GetSpendHeight(const CCoinsViewCache& inputs) -{ - AssertLockHeld(cs_main); - CBlockIndex* pindexPrev = LookupBlockIndex(inputs.GetBestBlock()); - return pindexPrev->nHeight + 1; -} - - static CuckooCache::cache<uint256, SignatureCacheHasher> g_scriptExecutionCache; static CSHA256 g_scriptExecutionCacheHasher; @@ -1421,6 +1485,7 @@ int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out) * When FAILED is returned, view is left in an indeterminate state. */ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view) { + AssertLockHeld(::cs_main); bool fClean = true; CBlockUndo blockUndo; @@ -1658,8 +1723,6 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, // can be duplicated to remove the ability to spend the first instance -- even after // being sent to another address. // See BIP30, CVE-2012-1909, and http://r6.ca/blog/20120206T005236Z.html for more information. - // This logic is not necessary for memory pool transactions, as AcceptToMemoryPool - // already refuses previously-known transaction ids entirely. // This rule was originally applied to all blocks with a timestamp after March 15, 2012, 0:00 UTC. // Now that the whole chain is irreversibly beyond that time it is applied to all blocks except the // two in the chain that violate it. This prevents exploiting the issue against nodes during their @@ -1981,7 +2044,7 @@ bool CChainState::FlushStateToDisk( fDoFullFlush = (mode == FlushStateMode::ALWAYS) || fCacheLarge || fCacheCritical || fPeriodicFlush || fFlushForPrune; // Write blocks and block index to disk. if (fDoFullFlush || fPeriodicWrite) { - // Depend on nMinDiskSpace to ensure we can write block index + // Ensure we can write block index if (!CheckDiskSpace(gArgs.GetBlocksDirPath())) { return AbortNode(state, "Disk space is too low!", _("Disk space is too low!")); } @@ -2039,6 +2102,13 @@ bool CChainState::FlushStateToDisk( nLastFlush = nNow; full_flush_completed = true; } + TRACE6(utxocache, flush, + (int64_t)(GetTimeMicros() - nNow.count()), // in microseconds (µs) + (u_int32_t)mode, + (u_int64_t)coins_count, + (u_int64_t)coins_mem_usage, + (bool)fFlushForPrune, + (bool)fDoFullFlush); } if (full_flush_completed) { // Update best block in wallet (so we can detect restored wallets). @@ -3423,6 +3493,19 @@ bool ChainstateManager::ProcessNewBlock(const CChainParams& chainparams, const s return true; } +MempoolAcceptResult ChainstateManager::ProcessTransaction(const CTransactionRef& tx, bool test_accept) +{ + CChainState& active_chainstate = ActiveChainstate(); + if (!active_chainstate.m_mempool) { + TxValidationState state; + state.Invalid(TxValidationResult::TX_NO_MEMPOOL, "no-mempool"); + return MempoolAcceptResult::Failure(state); + } + auto result = AcceptToMemoryPool(*active_chainstate.m_mempool, active_chainstate, tx, GetTime(), /* bypass_limits= */ false, test_accept); + active_chainstate.m_mempool->check(active_chainstate.CoinsTip(), active_chainstate.m_chain.Height() + 1); + return result; +} + bool TestBlockValidity(BlockValidationState& state, const CChainParams& chainparams, CChainState& chainstate, @@ -4448,7 +4531,6 @@ static const uint64_t MEMPOOL_DUMP_VERSION = 1; bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mockable_fopen_function) { - const CChainParams& chainparams = Params(); int64_t nExpiryTimeout = gArgs.GetIntArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60; FILE* filestr{mockable_fopen_function(gArgs.GetDataDirNet() / "mempool.dat", "rb")}; CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); @@ -4486,8 +4568,8 @@ bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mocka } if (nTime > nNow - nExpiryTimeout) { LOCK(cs_main); - if (AcceptToMemoryPoolWithTime(chainparams, pool, active_chainstate, tx, nTime, false /* bypass_limits */, - false /* test_accept */).m_result_type == MempoolAcceptResult::ResultType::VALID) { + if (AcceptToMemoryPool(pool, active_chainstate, tx, nTime, /* bypass_limits= */ false, + /* test_accept= */ false).m_result_type == MempoolAcceptResult::ResultType::VALID) { ++count; } else { // mempool may contain the transaction already, e.g. from diff --git a/src/validation.h b/src/validation.h index 4da8ec8d24..2609538ece 100644 --- a/src/validation.h +++ b/src/validation.h @@ -158,14 +158,16 @@ struct MempoolAcceptResult { // The following fields are only present when m_result_type = ResultType::VALID /** Mempool transactions replaced by the tx per BIP 125 rules. */ const std::optional<std::list<CTransactionRef>> m_replaced_transactions; + /** Virtual size as used by the mempool, calculated using serialized size and sigops. */ + const std::optional<int64_t> m_vsize; /** Raw base fees in satoshis. */ const std::optional<CAmount> m_base_fees; static MempoolAcceptResult Failure(TxValidationState state) { return MempoolAcceptResult(state); } - static MempoolAcceptResult Success(std::list<CTransactionRef>&& replaced_txns, CAmount fees) { - return MempoolAcceptResult(std::move(replaced_txns), fees); + static MempoolAcceptResult Success(std::list<CTransactionRef>&& replaced_txns, int64_t vsize, CAmount fees) { + return MempoolAcceptResult(std::move(replaced_txns), vsize, fees); } // Private constructors. Use static methods MempoolAcceptResult::Success, etc. to construct. @@ -177,9 +179,9 @@ private: } /** Constructor for success case */ - explicit MempoolAcceptResult(std::list<CTransactionRef>&& replaced_txns, CAmount fees) + explicit MempoolAcceptResult(std::list<CTransactionRef>&& replaced_txns, int64_t vsize, CAmount fees) : m_result_type(ResultType::VALID), - m_replaced_transactions(std::move(replaced_txns)), m_base_fees(fees) {} + m_replaced_transactions(std::move(replaced_txns)), m_vsize{vsize}, m_base_fees(fees) {} }; /** @@ -206,12 +208,23 @@ struct PackageMempoolAcceptResult }; /** - * (Try to) add a transaction to the memory pool. - * @param[in] bypass_limits When true, don't enforce mempool fee limits. - * @param[in] test_accept When true, run validation checks but don't submit to mempool. + * Try to add a transaction to the mempool. This is an internal function and is exposed only for testing. + * Client code should use ChainstateManager::ProcessTransaction() + * + * @param[in] pool Reference to the node's mempool. + * @param[in] active_chainstate Reference to the active chainstate. + * @param[in] tx The transaction to submit for mempool acceptance. + * @param[in] accept_time The timestamp for adding the transaction to the mempool. Usually + * the current system time, but may be different. + * It is also used to determine when the entry expires. + * @param[in] bypass_limits When true, don't enforce mempool fee and capacity limits. + * @param[in] test_accept When true, run validation checks but don't submit to mempool. + * + * @returns a MempoolAcceptResult indicating whether the transaction was accepted/rejected with reason. */ -MempoolAcceptResult AcceptToMemoryPool(CChainState& active_chainstate, CTxMemPool& pool, const CTransactionRef& tx, - bool bypass_limits, bool test_accept=false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); +MempoolAcceptResult AcceptToMemoryPool(CTxMemPool& pool, CChainState& active_chainstate, const CTransactionRef& tx, + int64_t accept_time, bool bypass_limits, bool test_accept) + EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** * Atomically test acceptance of a package. If the package only contains one tx, package rules still @@ -241,11 +254,6 @@ PackageMempoolAcceptResult ProcessNewPackage(CChainState& active_chainstate, CTx bool CheckFinalTx(const CBlockIndex* active_chain_tip, const CTransaction &tx, int flags = -1) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** - * Test whether the LockPoints height and time are still valid on the current chain - */ -bool TestLockPointValidity(CChain& active_chain, const LockPoints* lp) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - -/** * Check if transaction will be BIP68 final in the next block to be created on top of tip. * @param[in] tip Chain tip to check tx sequence locks against. For example, * the tip of the current active chain. @@ -470,13 +478,6 @@ public: //! Returns last CBlockIndex* that is a checkpoint CBlockIndex* GetLastCheckpoint(const CCheckpointData& data) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - /** - * Return the spend height, which is one more than the inputs.GetBestBlock(). - * While checking, GetBestBlock() refers to the parent block. (protected by cs_main) - * This is also true for mempool checks. - */ - int GetSpendHeight(const CCoinsViewCache& inputs) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - ~BlockManager() { Unload(); } @@ -572,8 +573,6 @@ protected: //! Only the active chainstate has a mempool. CTxMemPool* m_mempool; - const CChainParams& m_params; - //! Manages the UTXO set, which is a reflection of the contents of `m_chain`. std::unique_ptr<CoinsViews> m_coins_views; @@ -582,6 +581,9 @@ public: //! CChainState instances. BlockManager& m_blockman; + /** Chain parameters for this chainstate */ + const CChainParams& m_params; + //! The chainstate manager that owns this chainstate. The reference is //! necessary so that this instance can check whether it is the active //! chainstate within deeply nested method calls. @@ -716,7 +718,8 @@ public: bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) EXCLUSIVE_LOCKS_REQUIRED(cs_main); // Block (dis)connection on a given view: - DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view); + DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); bool ConnectBlock(const CBlock& block, BlockValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, bool fJustCheck = false) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -994,6 +997,15 @@ public: */ bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, BlockValidationState& state, const CChainParams& chainparams, const CBlockIndex** ppindex = nullptr) LOCKS_EXCLUDED(cs_main); + /** + * Try to add a transaction to the memory pool. + * + * @param[in] tx The transaction to submit for mempool acceptance. + * @param[in] test_accept When true, run validation checks but don't submit to mempool. + */ + [[nodiscard]] MempoolAcceptResult ProcessTransaction(const CTransactionRef& tx, bool test_accept=false) + EXCLUSIVE_LOCKS_REQUIRED(cs_main); + //! Load the block tree and coins database from disk, initializing state if we're running with -reindex bool LoadBlockIndex() EXCLUSIVE_LOCKS_REQUIRED(cs_main); diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h index b666a8e73a..7d0f80518a 100644 --- a/src/wallet/bdb.h +++ b/src/wallet/bdb.h @@ -113,7 +113,7 @@ public: */ bool Rewrite(const char* pszSkip=nullptr) override; - /** Indicate the a new database user has began using the database. */ + /** Indicate that a new database user has begun using the database. */ void AddRef() override; /** Indicate that database user has stopped using the database and that it could be flushed or closed. */ void RemoveRef() override; diff --git a/src/wallet/context.h b/src/wallet/context.h index a382fb9021..dbd172e88e 100644 --- a/src/wallet/context.h +++ b/src/wallet/context.h @@ -34,6 +34,8 @@ using LoadWalletFn = std::function<void(std::unique_ptr<interfaces::Wallet> wall struct WalletContext { interfaces::Chain* chain{nullptr}; ArgsManager* args{nullptr}; // Currently a raw pointer because the memory is not managed by this struct + // It is unsafe to lock this after locking a CWallet::cs_wallet mutex because + // this could introduce inconsistent lock ordering and cause deadlocks. Mutex wallets_mutex; std::vector<std::shared_ptr<CWallet>> wallets GUARDED_BY(wallets_mutex); std::list<LoadWalletFn> wallet_load_fns GUARDED_BY(wallets_mutex); diff --git a/src/wallet/dump.cpp b/src/wallet/dump.cpp index 08d94b76d9..e50b4ca5f7 100644 --- a/src/wallet/dump.cpp +++ b/src/wallet/dump.cpp @@ -191,7 +191,7 @@ bool CreateFromDump(const std::string& name, const fs::path& wallet_path, biling // dummy chain interface bool ret = true; - std::shared_ptr<CWallet> wallet(new CWallet(nullptr /* chain */, name, std::move(database)), WalletToolReleaseWallet); + std::shared_ptr<CWallet> wallet(new CWallet(nullptr /* chain */, name, gArgs, std::move(database)), WalletToolReleaseWallet); { LOCK(wallet->cs_wallet); DBErrors load_wallet_ret = wallet->LoadWallet(); diff --git a/src/wallet/external_signer_scriptpubkeyman.cpp b/src/wallet/external_signer_scriptpubkeyman.cpp index efef1ec754..6a73efb472 100644 --- a/src/wallet/external_signer_scriptpubkeyman.cpp +++ b/src/wallet/external_signer_scriptpubkeyman.cpp @@ -60,10 +60,10 @@ bool ExternalSignerScriptPubKeyMan::DisplayAddress(const CScript scriptPubKey, c } // If sign is true, transaction must previously have been filled -TransactionError ExternalSignerScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type, bool sign, bool bip32derivs, int* n_signed) const +TransactionError ExternalSignerScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type, bool sign, bool bip32derivs, int* n_signed, bool finalize) const { if (!sign) { - return DescriptorScriptPubKeyMan::FillPSBT(psbt, txdata, sighash_type, false, bip32derivs, n_signed); + return DescriptorScriptPubKeyMan::FillPSBT(psbt, txdata, sighash_type, false, bip32derivs, n_signed, finalize); } // Already complete if every input is now signed @@ -79,6 +79,6 @@ TransactionError ExternalSignerScriptPubKeyMan::FillPSBT(PartiallySignedTransact tfm::format(std::cerr, "Failed to sign: %s\n", strFailReason); return TransactionError::EXTERNAL_SIGNER_FAILED; } - FinalizePSBT(psbt); // This won't work in a multisig setup + if (finalize) FinalizePSBT(psbt); // This won't work in a multisig setup return TransactionError::OK; } diff --git a/src/wallet/external_signer_scriptpubkeyman.h b/src/wallet/external_signer_scriptpubkeyman.h index 61df3d0015..53d65d9e46 100644 --- a/src/wallet/external_signer_scriptpubkeyman.h +++ b/src/wallet/external_signer_scriptpubkeyman.h @@ -28,6 +28,6 @@ class ExternalSignerScriptPubKeyMan : public DescriptorScriptPubKeyMan bool DisplayAddress(const CScript scriptPubKey, const ExternalSigner &signer) const; - TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr) const override; + TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const override; }; #endif // BITCOIN_WALLET_EXTERNAL_SIGNER_SCRIPTPUBKEYMAN_H diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 7a5526a4cb..4ff049170e 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -43,9 +43,9 @@ const WalletInitInterface& g_wallet_init_interface = WalletInit(); void WalletInit::AddWalletOptions(ArgsManager& argsman) const { - argsman.AddArg("-addresstype", strprintf("What type of addresses to use (\"legacy\", \"p2sh-segwit\", or \"bech32\", default: \"%s\")", FormatOutputType(DEFAULT_ADDRESS_TYPE)), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); + argsman.AddArg("-addresstype", strprintf("What type of addresses to use (\"legacy\", \"p2sh-segwit\", \"bech32\", or \"bech32m\", default: \"%s\")", FormatOutputType(DEFAULT_ADDRESS_TYPE)), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); argsman.AddArg("-avoidpartialspends", strprintf("Group outputs by address, selecting many (possibly all) or none, instead of selecting on a per-output basis. Privacy is improved as addresses are mostly swept with fewer transactions and outputs are aggregated in clean change addresses. It may result in higher fees due to less optimal coin selection caused by this added limitation and possibly a larger-than-necessary number of inputs being used. Always enabled for wallets with \"avoid_reuse\" enabled, otherwise default: %u.", DEFAULT_AVOIDPARTIALSPENDS), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); - argsman.AddArg("-changetype", "What type of change to use (\"legacy\", \"p2sh-segwit\", or \"bech32\"). Default is same as -addresstype, except when -addresstype=p2sh-segwit a native segwit output is used when sending to a native segwit address)", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); + argsman.AddArg("-changetype", "What type of change to use (\"legacy\", \"p2sh-segwit\", \"bech32\", or \"bech32m\"). Default is same as -addresstype, except when -addresstype=p2sh-segwit a native segwit output is used when sending to a native segwit address)", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); argsman.AddArg("-consolidatefeerate=<amt>", strprintf("The maximum feerate (in %s/kvB) at which transaction building may use more inputs than strictly necessary so that the wallet's UTXO pool can be reduced (default: %s).", CURRENCY_UNIT, FormatMoney(DEFAULT_CONSOLIDATE_FEERATE)), ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); argsman.AddArg("-disablewallet", "Do not load the wallet and disable wallet RPC calls", ArgsManager::ALLOW_ANY, OptionsCategory::WALLET); argsman.AddArg("-discardfee=<amt>", strprintf("The fee rate (in %s/kvB) that indicates your tolerance for discarding change by adding it to the fee (default: %s). " diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index 57f1a6a67a..6489e2a5fc 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -82,7 +82,10 @@ WalletTx MakeWalletTx(CWallet& wallet, const CWalletTx& wtx) WalletTxStatus MakeWalletTxStatus(const CWallet& wallet, const CWalletTx& wtx) { WalletTxStatus result; - result.block_height = wtx.m_confirm.block_height > 0 ? wtx.m_confirm.block_height : std::numeric_limits<int>::max(); + result.block_height = + wtx.state<TxStateConfirmed>() ? wtx.state<TxStateConfirmed>()->confirmed_block_height : + wtx.state<TxStateConflicted>() ? wtx.state<TxStateConflicted>()->conflicting_block_height : + std::numeric_limits<int>::max(); result.blocks_to_maturity = wallet.GetTxBlocksToMaturity(wtx); result.depth_in_main_chain = wallet.GetTxDepthInMainChain(wtx); result.time_received = wtx.nTimeReceived; diff --git a/src/wallet/ismine.h b/src/wallet/ismine.h index 38ed7e7770..8605547cf2 100644 --- a/src/wallet/ismine.h +++ b/src/wallet/ismine.h @@ -8,8 +8,9 @@ #include <script/standard.h> -#include <stdint.h> #include <bitset> +#include <cstdint> +#include <type_traits> class CWallet; class CScript; @@ -35,8 +36,7 @@ class CScript; * ISMINE_USED: the scriptPubKey corresponds to a used address owned by the wallet user. * */ -enum isminetype : unsigned int -{ +enum isminetype : unsigned int { ISMINE_NO = 0, ISMINE_WATCH_ONLY = 1 << 0, ISMINE_SPENDABLE = 1 << 1, @@ -46,7 +46,7 @@ enum isminetype : unsigned int ISMINE_ENUM_ELEMENTS, }; /** used for bitflags of isminetype */ -typedef uint8_t isminefilter; +using isminefilter = std::underlying_type<isminetype>::type; /** * Cachable amount subdivided into watchonly and spendable parts. diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpc/backup.cpp index 1f13b80f3e..7e06ff0e4f 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpc/backup.cpp @@ -17,7 +17,7 @@ #include <util/system.h> #include <util/time.h> #include <util/translation.h> -#include <wallet/rpcwallet.h> +#include <wallet/rpc/util.h> #include <wallet/wallet.h> #include <stdint.h> @@ -35,7 +35,7 @@ std::string static EncodeDumpString(const std::string &str) { std::stringstream ret; for (const unsigned char c : str) { if (c <= 32 || c >= 128 || c == '%') { - ret << '%' << HexStr(Span<const unsigned char>(&c, 1)); + ret << '%' << HexStr({&c, 1}); } else { ret << c; } @@ -57,7 +57,7 @@ static std::string DecodeDumpString(const std::string &str) { return ret.str(); } -static bool GetWalletAddressesForKey(LegacyScriptPubKeyMan* spk_man, const CWallet& wallet, const CKeyID& keyid, std::string& strAddr, std::string& strLabel) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +static bool GetWalletAddressesForKey(const LegacyScriptPubKeyMan* spk_man, const CWallet& wallet, const CKeyID& keyid, std::string& strAddr, std::string& strLabel) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { bool fLabelFound = false; CKey key; @@ -369,11 +369,9 @@ RPCHelpMan importprunedfunds() unsigned int txnIndex = vIndex[it - vMatch.begin()]; - CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, height, merkleBlock.header.GetHash(), txnIndex); - CTransactionRef tx_ref = MakeTransactionRef(tx); if (pwallet->IsMine(*tx_ref)) { - pwallet->AddToWallet(std::move(tx_ref), confirm); + pwallet->AddToWallet(std::move(tx_ref), TxStateConfirmed{merkleBlock.header.GetHash(), height, static_cast<int>(txnIndex)}); return NullUniValue; } @@ -681,10 +679,10 @@ RPCHelpMan dumpprivkey() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; - LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet); + const LegacyScriptPubKeyMan& spk_man = EnsureConstLegacyScriptPubKeyMan(*pwallet); LOCK2(pwallet->cs_wallet, spk_man.cs_KeyStore); @@ -731,11 +729,11 @@ RPCHelpMan dumpwallet() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; - CWallet& wallet = *pwallet; - LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(wallet); + const CWallet& wallet = *pwallet; + const LegacyScriptPubKeyMan& spk_man = EnsureConstLegacyScriptPubKeyMan(wallet); // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now @@ -797,7 +795,7 @@ RPCHelpMan dumpwallet() CKey seed; if (spk_man.GetKey(seed_id, seed)) { CExtKey masterKey; - masterKey.SetSeed(seed.begin(), seed.size()); + masterKey.SetSeed(seed); file << "# extended private masterkey: " << EncodeExtKey(masterKey) << "\n\n"; } @@ -809,6 +807,9 @@ RPCHelpMan dumpwallet() std::string strLabel; CKey key; if (spk_man.GetKey(keyid, key)) { + CKeyMetadata metadata; + const auto it{spk_man.mapKeyMetadata.find(keyid)}; + if (it != spk_man.mapKeyMetadata.end()) metadata = it->second; file << strprintf("%s %s ", EncodeSecret(key), strTime); if (GetWalletAddressesForKey(&spk_man, wallet, keyid, strAddr, strLabel)) { file << strprintf("label=%s", strLabel); @@ -816,12 +817,12 @@ RPCHelpMan dumpwallet() file << "hdseed=1"; } else if (mapKeyPool.count(keyid)) { file << "reserve=1"; - } else if (spk_man.mapKeyMetadata[keyid].hdKeypath == "s") { + } else if (metadata.hdKeypath == "s") { file << "inactivehdseed=1"; } else { file << "change=1"; } - file << strprintf(" # addr=%s%s\n", strAddr, (spk_man.mapKeyMetadata[keyid].has_key_origin ? " hdkeypath="+WriteHDKeypath(spk_man.mapKeyMetadata[keyid].key_origin.path) : "")); + file << strprintf(" # addr=%s%s\n", strAddr, (metadata.has_key_origin ? " hdkeypath="+WriteHDKeypath(metadata.key_origin.path) : "")); } } file << "\n"; @@ -1345,6 +1346,11 @@ RPCHelpMan importmulti() { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(mainRequest); if (!pwallet) return NullUniValue; + CWallet& wallet{*pwallet}; + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + wallet.BlockUntilSyncedToCurrentChain(); RPCTypeCheck(mainRequest.params, {UniValue::VARR, UniValue::VOBJ}); @@ -1545,18 +1551,6 @@ static UniValue ProcessDescriptorImport(CWallet& wallet, const UniValue& data, c } } - // Taproot descriptors cannot be imported if Taproot is not yet active. - // Check if this is a Taproot descriptor - CTxDestination dest; - ExtractDestination(scripts[0], dest); - if (std::holds_alternative<WitnessV1Taproot>(dest)) { - // Check if Taproot is active - if (!wallet.chain().isTaprootActive()) { - // Taproot is not active, raise an error - throw JSONRPCError(RPC_WALLET_ERROR, "Cannot import tr() descriptor when Taproot is not active"); - } - } - // If private keys are enabled, check some things. if (!wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { if (keys.keys.empty()) { @@ -1660,6 +1654,11 @@ RPCHelpMan importdescriptors() { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(main_request); if (!pwallet) return NullUniValue; + CWallet& wallet{*pwallet}; + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + wallet.BlockUntilSyncedToCurrentChain(); // Make sure wallet is a descriptor wallet if (!pwallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { @@ -1819,11 +1818,10 @@ RPCHelpMan listdescriptors() } spk.pushKV("desc", descriptor); spk.pushKV("timestamp", wallet_descriptor.creation_time); - const bool active = active_spk_mans.count(desc_spk_man) != 0; - spk.pushKV("active", active); - const auto& type = wallet_descriptor.descriptor->GetOutputType(); - if (active && type) { - spk.pushKV("internal", wallet->GetScriptPubKeyMan(*type, true) == desc_spk_man); + spk.pushKV("active", active_spk_mans.count(desc_spk_man) != 0); + const auto internal = wallet->IsInternalScriptPubKeyMan(desc_spk_man); + if (internal.has_value()) { + spk.pushKV("internal", *internal); } if (wallet_descriptor.descriptor->IsRange()) { UniValue range(UniValue::VARR); @@ -1843,3 +1841,99 @@ RPCHelpMan listdescriptors() }, }; } + +RPCHelpMan backupwallet() +{ + return RPCHelpMan{"backupwallet", + "\nSafely copies current wallet file to destination, which can be a directory or a path with filename.\n", + { + {"destination", RPCArg::Type::STR, RPCArg::Optional::NO, "The destination directory or file"}, + }, + RPCResult{RPCResult::Type::NONE, "", ""}, + RPCExamples{ + HelpExampleCli("backupwallet", "\"backup.dat\"") + + HelpExampleRpc("backupwallet", "\"backup.dat\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); + if (!pwallet) return NullUniValue; + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + pwallet->BlockUntilSyncedToCurrentChain(); + + LOCK(pwallet->cs_wallet); + + std::string strDest = request.params[0].get_str(); + if (!pwallet->BackupWallet(strDest)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Error: Wallet backup failed!"); + } + + return NullUniValue; +}, + }; +} + + +RPCHelpMan restorewallet() +{ + return RPCHelpMan{ + "restorewallet", + "\nRestore and loads a wallet from backup.\n", + { + {"wallet_name", RPCArg::Type::STR, RPCArg::Optional::NO, "The name that will be applied to the restored wallet"}, + {"backup_file", RPCArg::Type::STR, RPCArg::Optional::NO, "The backup file that will be used to restore the wallet."}, + {"load_on_startup", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED_NAMED_ARG, "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR, "name", "The wallet name if restored successfully."}, + {RPCResult::Type::STR, "warning", "Warning message if wallet was not loaded cleanly."}, + } + }, + RPCExamples{ + HelpExampleCli("restorewallet", "\"testwallet\" \"home\\backups\\backup-file.bak\"") + + HelpExampleRpc("restorewallet", "\"testwallet\" \"home\\backups\\backup-file.bak\"") + + HelpExampleCliNamed("restorewallet", {{"wallet_name", "testwallet"}, {"backup_file", "home\\backups\\backup-file.bak\""}, {"load_on_startup", true}}) + + HelpExampleRpcNamed("restorewallet", {{"wallet_name", "testwallet"}, {"backup_file", "home\\backups\\backup-file.bak\""}, {"load_on_startup", true}}) + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + + WalletContext& context = EnsureWalletContext(request.context); + + auto backup_file = fs::u8path(request.params[1].get_str()); + + if (!fs::exists(backup_file)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Backup file does not exist"); + } + + std::string wallet_name = request.params[0].get_str(); + + const fs::path wallet_path = fsbridge::AbsPathJoin(GetWalletDir(), fs::u8path(wallet_name)); + + if (fs::exists(wallet_path)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Wallet name already exists."); + } + + if (!TryCreateDirectories(wallet_path)) { + throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Failed to create database path '%s'. Database already exists.", wallet_path.u8string())); + } + + auto wallet_file = wallet_path / "wallet.dat"; + + fs::copy_file(backup_file, wallet_file, fs::copy_option::fail_if_exists); + + auto [wallet, warnings] = LoadWalletHelper(context, request.params[2], wallet_name); + + UniValue obj(UniValue::VOBJ); + obj.pushKV("name", wallet->GetName()); + obj.pushKV("warning", Join(warnings, Untranslated("\n")).original); + + return obj; + +}, + }; +} diff --git a/src/wallet/rpc/encrypt.cpp b/src/wallet/rpc/encrypt.cpp new file mode 100644 index 0000000000..e659f434a3 --- /dev/null +++ b/src/wallet/rpc/encrypt.cpp @@ -0,0 +1,248 @@ +// Copyright (c) 2011-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <rpc/util.h> +#include <wallet/rpc/util.h> +#include <wallet/wallet.h> + + +RPCHelpMan walletpassphrase() +{ + return RPCHelpMan{"walletpassphrase", + "\nStores the wallet decryption key in memory for 'timeout' seconds.\n" + "This is needed prior to performing transactions related to private keys such as sending bitcoins\n" + "\nNote:\n" + "Issuing the walletpassphrase command while the wallet is already unlocked will set a new unlock\n" + "time that overrides the old one.\n", + { + {"passphrase", RPCArg::Type::STR, RPCArg::Optional::NO, "The wallet passphrase"}, + {"timeout", RPCArg::Type::NUM, RPCArg::Optional::NO, "The time to keep the decryption key in seconds; capped at 100000000 (~3 years)."}, + }, + RPCResult{RPCResult::Type::NONE, "", ""}, + RPCExamples{ + "\nUnlock the wallet for 60 seconds\n" + + HelpExampleCli("walletpassphrase", "\"my pass phrase\" 60") + + "\nLock the wallet again (before 60 seconds)\n" + + HelpExampleCli("walletlock", "") + + "\nAs a JSON-RPC call\n" + + HelpExampleRpc("walletpassphrase", "\"my pass phrase\", 60") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); + if (!wallet) return NullUniValue; + CWallet* const pwallet = wallet.get(); + + int64_t nSleepTime; + int64_t relock_time; + // Prevent concurrent calls to walletpassphrase with the same wallet. + LOCK(pwallet->m_unlock_mutex); + { + LOCK(pwallet->cs_wallet); + + if (!pwallet->IsCrypted()) { + throw JSONRPCError(RPC_WALLET_WRONG_ENC_STATE, "Error: running with an unencrypted wallet, but walletpassphrase was called."); + } + + // Note that the walletpassphrase is stored in request.params[0] which is not mlock()ed + SecureString strWalletPass; + strWalletPass.reserve(100); + // TODO: get rid of this .c_str() by implementing SecureString::operator=(std::string) + // Alternately, find a way to make request.params[0] mlock()'d to begin with. + strWalletPass = request.params[0].get_str().c_str(); + + // Get the timeout + nSleepTime = request.params[1].get_int64(); + // Timeout cannot be negative, otherwise it will relock immediately + if (nSleepTime < 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Timeout cannot be negative."); + } + // Clamp timeout + constexpr int64_t MAX_SLEEP_TIME = 100000000; // larger values trigger a macos/libevent bug? + if (nSleepTime > MAX_SLEEP_TIME) { + nSleepTime = MAX_SLEEP_TIME; + } + + if (strWalletPass.empty()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "passphrase can not be empty"); + } + + if (!pwallet->Unlock(strWalletPass)) { + throw JSONRPCError(RPC_WALLET_PASSPHRASE_INCORRECT, "Error: The wallet passphrase entered was incorrect."); + } + + pwallet->TopUpKeyPool(); + + pwallet->nRelockTime = GetTime() + nSleepTime; + relock_time = pwallet->nRelockTime; + } + + // rpcRunLater must be called without cs_wallet held otherwise a deadlock + // can occur. The deadlock would happen when RPCRunLater removes the + // previous timer (and waits for the callback to finish if already running) + // and the callback locks cs_wallet. + AssertLockNotHeld(wallet->cs_wallet); + // Keep a weak pointer to the wallet so that it is possible to unload the + // wallet before the following callback is called. If a valid shared pointer + // is acquired in the callback then the wallet is still loaded. + std::weak_ptr<CWallet> weak_wallet = wallet; + pwallet->chain().rpcRunLater(strprintf("lockwallet(%s)", pwallet->GetName()), [weak_wallet, relock_time] { + if (auto shared_wallet = weak_wallet.lock()) { + LOCK(shared_wallet->cs_wallet); + // Skip if this is not the most recent rpcRunLater callback. + if (shared_wallet->nRelockTime != relock_time) return; + shared_wallet->Lock(); + shared_wallet->nRelockTime = 0; + } + }, nSleepTime); + + return NullUniValue; +}, + }; +} + + +RPCHelpMan walletpassphrasechange() +{ + return RPCHelpMan{"walletpassphrasechange", + "\nChanges the wallet passphrase from 'oldpassphrase' to 'newpassphrase'.\n", + { + {"oldpassphrase", RPCArg::Type::STR, RPCArg::Optional::NO, "The current passphrase"}, + {"newpassphrase", RPCArg::Type::STR, RPCArg::Optional::NO, "The new passphrase"}, + }, + RPCResult{RPCResult::Type::NONE, "", ""}, + RPCExamples{ + HelpExampleCli("walletpassphrasechange", "\"old one\" \"new one\"") + + HelpExampleRpc("walletpassphrasechange", "\"old one\", \"new one\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + if (!pwallet) return NullUniValue; + + LOCK(pwallet->cs_wallet); + + if (!pwallet->IsCrypted()) { + throw JSONRPCError(RPC_WALLET_WRONG_ENC_STATE, "Error: running with an unencrypted wallet, but walletpassphrasechange was called."); + } + + // TODO: get rid of these .c_str() calls by implementing SecureString::operator=(std::string) + // Alternately, find a way to make request.params[0] mlock()'d to begin with. + SecureString strOldWalletPass; + strOldWalletPass.reserve(100); + strOldWalletPass = request.params[0].get_str().c_str(); + + SecureString strNewWalletPass; + strNewWalletPass.reserve(100); + strNewWalletPass = request.params[1].get_str().c_str(); + + if (strOldWalletPass.empty() || strNewWalletPass.empty()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "passphrase can not be empty"); + } + + if (!pwallet->ChangeWalletPassphrase(strOldWalletPass, strNewWalletPass)) { + throw JSONRPCError(RPC_WALLET_PASSPHRASE_INCORRECT, "Error: The wallet passphrase entered was incorrect."); + } + + return NullUniValue; +}, + }; +} + + +RPCHelpMan walletlock() +{ + return RPCHelpMan{"walletlock", + "\nRemoves the wallet encryption key from memory, locking the wallet.\n" + "After calling this method, you will need to call walletpassphrase again\n" + "before being able to call any methods which require the wallet to be unlocked.\n", + {}, + RPCResult{RPCResult::Type::NONE, "", ""}, + RPCExamples{ + "\nSet the passphrase for 2 minutes to perform a transaction\n" + + HelpExampleCli("walletpassphrase", "\"my pass phrase\" 120") + + "\nPerform a send (requires passphrase set)\n" + + HelpExampleCli("sendtoaddress", "\"" + EXAMPLE_ADDRESS[0] + "\" 1.0") + + "\nClear the passphrase since we are done before 2 minutes is up\n" + + HelpExampleCli("walletlock", "") + + "\nAs a JSON-RPC call\n" + + HelpExampleRpc("walletlock", "") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + if (!pwallet) return NullUniValue; + + LOCK(pwallet->cs_wallet); + + if (!pwallet->IsCrypted()) { + throw JSONRPCError(RPC_WALLET_WRONG_ENC_STATE, "Error: running with an unencrypted wallet, but walletlock was called."); + } + + pwallet->Lock(); + pwallet->nRelockTime = 0; + + return NullUniValue; +}, + }; +} + + +RPCHelpMan encryptwallet() +{ + return RPCHelpMan{"encryptwallet", + "\nEncrypts the wallet with 'passphrase'. This is for first time encryption.\n" + "After this, any calls that interact with private keys such as sending or signing \n" + "will require the passphrase to be set prior the making these calls.\n" + "Use the walletpassphrase call for this, and then walletlock call.\n" + "If the wallet is already encrypted, use the walletpassphrasechange call.\n", + { + {"passphrase", RPCArg::Type::STR, RPCArg::Optional::NO, "The pass phrase to encrypt the wallet with. It must be at least 1 character, but should be long."}, + }, + RPCResult{RPCResult::Type::STR, "", "A string with further instructions"}, + RPCExamples{ + "\nEncrypt your wallet\n" + + HelpExampleCli("encryptwallet", "\"my pass phrase\"") + + "\nNow set the passphrase to use the wallet, such as for signing or sending bitcoin\n" + + HelpExampleCli("walletpassphrase", "\"my pass phrase\"") + + "\nNow we can do something like sign\n" + + HelpExampleCli("signmessage", "\"address\" \"test message\"") + + "\nNow lock the wallet again by removing the passphrase\n" + + HelpExampleCli("walletlock", "") + + "\nAs a JSON-RPC call\n" + + HelpExampleRpc("encryptwallet", "\"my pass phrase\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + if (!pwallet) return NullUniValue; + + LOCK(pwallet->cs_wallet); + + if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { + throw JSONRPCError(RPC_WALLET_ENCRYPTION_FAILED, "Error: wallet does not contain private keys, nothing to encrypt."); + } + + if (pwallet->IsCrypted()) { + throw JSONRPCError(RPC_WALLET_WRONG_ENC_STATE, "Error: running with an encrypted wallet, but encryptwallet was called."); + } + + // TODO: get rid of this .c_str() by implementing SecureString::operator=(std::string) + // Alternately, find a way to make request.params[0] mlock()'d to begin with. + SecureString strWalletPass; + strWalletPass.reserve(100); + strWalletPass = request.params[0].get_str().c_str(); + + if (strWalletPass.empty()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "passphrase can not be empty"); + } + + if (!pwallet->EncryptWallet(strWalletPass)) { + throw JSONRPCError(RPC_WALLET_ENCRYPTION_FAILED, "Error: Failed to encrypt the wallet."); + } + + return "wallet encrypted; The keypool has been flushed and a new HD seed was generated (if you are using HD). You need to make a new backup."; +}, + }; +} diff --git a/src/wallet/rpc/signmessage.cpp b/src/wallet/rpc/signmessage.cpp new file mode 100644 index 0000000000..bb8d7fc13f --- /dev/null +++ b/src/wallet/rpc/signmessage.cpp @@ -0,0 +1,68 @@ +// Copyright (c) 2011-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <key_io.h> +#include <rpc/util.h> +#include <util/message.h> +#include <wallet/rpc/util.h> +#include <wallet/wallet.h> + +#include <univalue.h> + +RPCHelpMan signmessage() +{ + return RPCHelpMan{"signmessage", + "\nSign a message with the private key of an address" + + HELP_REQUIRING_PASSPHRASE, + { + {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address to use for the private key."}, + {"message", RPCArg::Type::STR, RPCArg::Optional::NO, "The message to create a signature of."}, + }, + RPCResult{ + RPCResult::Type::STR, "signature", "The signature of the message encoded in base 64" + }, + RPCExamples{ + "\nUnlock the wallet for 30 seconds\n" + + HelpExampleCli("walletpassphrase", "\"mypassphrase\" 30") + + "\nCreate the signature\n" + + HelpExampleCli("signmessage", "\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\" \"my message\"") + + "\nVerify the signature\n" + + HelpExampleCli("verifymessage", "\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\" \"signature\" \"my message\"") + + "\nAs a JSON-RPC call\n" + + HelpExampleRpc("signmessage", "\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\", \"my message\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); + if (!pwallet) return NullUniValue; + + LOCK(pwallet->cs_wallet); + + EnsureWalletIsUnlocked(*pwallet); + + std::string strAddress = request.params[0].get_str(); + std::string strMessage = request.params[1].get_str(); + + CTxDestination dest = DecodeDestination(strAddress); + if (!IsValidDestination(dest)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + + const PKHash* pkhash = std::get_if<PKHash>(&dest); + if (!pkhash) { + throw JSONRPCError(RPC_TYPE_ERROR, "Address does not refer to key"); + } + + std::string signature; + SigningResult err = pwallet->SignMessage(strMessage, *pkhash, signature); + if (err == SigningResult::SIGNING_FAILED) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, SigningResultString(err)); + } else if (err != SigningResult::OK) { + throw JSONRPCError(RPC_WALLET_ERROR, SigningResultString(err)); + } + + return signature; + }, + }; +} diff --git a/src/wallet/rpc/util.cpp b/src/wallet/rpc/util.cpp new file mode 100644 index 0000000000..e2126b7236 --- /dev/null +++ b/src/wallet/rpc/util.cpp @@ -0,0 +1,154 @@ +// Copyright (c) 2011-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <wallet/rpc/util.h> + +#include <rpc/util.h> +#include <util/translation.h> +#include <util/url.h> +#include <wallet/context.h> +#include <wallet/wallet.h> + +#include <univalue.h> + +static const std::string WALLET_ENDPOINT_BASE = "/wallet/"; +const std::string HELP_REQUIRING_PASSPHRASE{"\nRequires wallet passphrase to be set with walletpassphrase call if wallet is encrypted.\n"}; + +bool GetAvoidReuseFlag(const CWallet& wallet, const UniValue& param) { + bool can_avoid_reuse = wallet.IsWalletFlagSet(WALLET_FLAG_AVOID_REUSE); + bool avoid_reuse = param.isNull() ? can_avoid_reuse : param.get_bool(); + + if (avoid_reuse && !can_avoid_reuse) { + throw JSONRPCError(RPC_WALLET_ERROR, "wallet does not have the \"avoid reuse\" feature enabled"); + } + + return avoid_reuse; +} + +/** Used by RPC commands that have an include_watchonly parameter. + * We default to true for watchonly wallets if include_watchonly isn't + * explicitly set. + */ +bool ParseIncludeWatchonly(const UniValue& include_watchonly, const CWallet& wallet) +{ + if (include_watchonly.isNull()) { + // if include_watchonly isn't explicitly set, then check if we have a watchonly wallet + return wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); + } + + // otherwise return whatever include_watchonly was set to + return include_watchonly.get_bool(); +} + +bool GetWalletNameFromJSONRPCRequest(const JSONRPCRequest& request, std::string& wallet_name) +{ + if (URL_DECODE && request.URI.substr(0, WALLET_ENDPOINT_BASE.size()) == WALLET_ENDPOINT_BASE) { + // wallet endpoint was used + wallet_name = URL_DECODE(request.URI.substr(WALLET_ENDPOINT_BASE.size())); + return true; + } + return false; +} + +std::shared_ptr<CWallet> GetWalletForJSONRPCRequest(const JSONRPCRequest& request) +{ + CHECK_NONFATAL(request.mode == JSONRPCRequest::EXECUTE); + WalletContext& context = EnsureWalletContext(request.context); + + std::string wallet_name; + if (GetWalletNameFromJSONRPCRequest(request, wallet_name)) { + const std::shared_ptr<CWallet> pwallet = GetWallet(context, wallet_name); + if (!pwallet) throw JSONRPCError(RPC_WALLET_NOT_FOUND, "Requested wallet does not exist or is not loaded"); + return pwallet; + } + + std::vector<std::shared_ptr<CWallet>> wallets = GetWallets(context); + if (wallets.size() == 1) { + return wallets[0]; + } + + if (wallets.empty()) { + throw JSONRPCError( + RPC_WALLET_NOT_FOUND, "No wallet is loaded. Load a wallet using loadwallet or create a new one with createwallet. (Note: A default wallet is no longer automatically created)"); + } + throw JSONRPCError(RPC_WALLET_NOT_SPECIFIED, + "Wallet file not specified (must request wallet RPC through /wallet/<filename> uri-path)."); +} + +void EnsureWalletIsUnlocked(const CWallet& wallet) +{ + if (wallet.IsLocked()) { + throw JSONRPCError(RPC_WALLET_UNLOCK_NEEDED, "Error: Please enter the wallet passphrase with walletpassphrase first."); + } +} + +WalletContext& EnsureWalletContext(const std::any& context) +{ + auto wallet_context = util::AnyPtr<WalletContext>(context); + if (!wallet_context) { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Wallet context not found"); + } + return *wallet_context; +} + +// also_create should only be set to true only when the RPC is expected to add things to a blank wallet and make it no longer blank +LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet, bool also_create) +{ + LegacyScriptPubKeyMan* spk_man = wallet.GetLegacyScriptPubKeyMan(); + if (!spk_man && also_create) { + spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); + } + if (!spk_man) { + throw JSONRPCError(RPC_WALLET_ERROR, "This type of wallet does not support this command"); + } + return *spk_man; +} + +const LegacyScriptPubKeyMan& EnsureConstLegacyScriptPubKeyMan(const CWallet& wallet) +{ + const LegacyScriptPubKeyMan* spk_man = wallet.GetLegacyScriptPubKeyMan(); + if (!spk_man) { + throw JSONRPCError(RPC_WALLET_ERROR, "This type of wallet does not support this command"); + } + return *spk_man; +} + +std::string LabelFromValue(const UniValue& value) +{ + std::string label = value.get_str(); + if (label == "*") + throw JSONRPCError(RPC_WALLET_INVALID_LABEL_NAME, "Invalid label name"); + return label; +} + +std::tuple<std::shared_ptr<CWallet>, std::vector<bilingual_str>> LoadWalletHelper(WalletContext& context, UniValue load_on_start_param, const std::string wallet_name) +{ + DatabaseOptions options; + DatabaseStatus status; + options.require_existing = true; + bilingual_str error; + std::vector<bilingual_str> warnings; + std::optional<bool> load_on_start = load_on_start_param.isNull() ? std::nullopt : std::optional<bool>(load_on_start_param.get_bool()); + std::shared_ptr<CWallet> const wallet = LoadWallet(context, wallet_name, load_on_start, options, status, error, warnings); + + if (!wallet) { + // Map bad format to not found, since bad format is returned when the + // wallet directory exists, but doesn't contain a data file. + RPCErrorCode code = RPC_WALLET_ERROR; + switch (status) { + case DatabaseStatus::FAILED_NOT_FOUND: + case DatabaseStatus::FAILED_BAD_FORMAT: + code = RPC_WALLET_NOT_FOUND; + break; + case DatabaseStatus::FAILED_ALREADY_LOADED: + code = RPC_WALLET_ALREADY_LOADED; + break; + default: // RPC_WALLET_ERROR is returned for all other cases. + break; + } + throw JSONRPCError(code, error.original); + } + + return { wallet, warnings }; +} diff --git a/src/wallet/rpc/util.h b/src/wallet/rpc/util.h new file mode 100644 index 0000000000..a1fa4d49b1 --- /dev/null +++ b/src/wallet/rpc/util.h @@ -0,0 +1,42 @@ +// Copyright (c) 2017-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_WALLET_RPC_UTIL_H +#define BITCOIN_WALLET_RPC_UTIL_H + +#include <any> +#include <memory> +#include <string> +#include <vector> + +struct bilingual_str; +class CWallet; +class JSONRPCRequest; +class LegacyScriptPubKeyMan; +class UniValue; +struct WalletContext; + +extern const std::string HELP_REQUIRING_PASSPHRASE; + +/** + * Figures out what wallet, if any, to use for a JSONRPCRequest. + * + * @param[in] request JSONRPCRequest that wishes to access a wallet + * @return nullptr if no wallet should be used, or a pointer to the CWallet + */ +std::shared_ptr<CWallet> GetWalletForJSONRPCRequest(const JSONRPCRequest& request); +bool GetWalletNameFromJSONRPCRequest(const JSONRPCRequest& request, std::string& wallet_name); + +void EnsureWalletIsUnlocked(const CWallet&); +WalletContext& EnsureWalletContext(const std::any& context); +LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet, bool also_create = false); +const LegacyScriptPubKeyMan& EnsureConstLegacyScriptPubKeyMan(const CWallet& wallet); + +bool GetAvoidReuseFlag(const CWallet& wallet, const UniValue& param); +bool ParseIncludeWatchonly(const UniValue& include_watchonly, const CWallet& wallet); +std::string LabelFromValue(const UniValue& value); + +std::tuple<std::shared_ptr<CWallet>, std::vector<bilingual_str>> LoadWalletHelper(WalletContext& context, UniValue load_on_start_param, const std::string wallet_name); + +#endif // BITCOIN_WALLET_RPC_UTIL_H diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 86bfa10d88..73949f26cb 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -20,7 +20,6 @@ #include <script/sign.h> #include <util/bip32.h> #include <util/fees.h> -#include <util/message.h> // For MessageSign() #include <util/moneystr.h> #include <util/string.h> #include <util/system.h> @@ -33,6 +32,7 @@ #include <wallet/load.h> #include <wallet/receive.h> #include <wallet/rpcwallet.h> +#include <wallet/rpc/util.h> #include <wallet/spend.h> #include <wallet/wallet.h> #include <wallet/walletdb.h> @@ -47,36 +47,6 @@ using interfaces::FoundBlock; -static const std::string WALLET_ENDPOINT_BASE = "/wallet/"; -static const std::string HELP_REQUIRING_PASSPHRASE{"\nRequires wallet passphrase to be set with walletpassphrase call if wallet is encrypted.\n"}; - -static inline bool GetAvoidReuseFlag(const CWallet& wallet, const UniValue& param) { - bool can_avoid_reuse = wallet.IsWalletFlagSet(WALLET_FLAG_AVOID_REUSE); - bool avoid_reuse = param.isNull() ? can_avoid_reuse : param.get_bool(); - - if (avoid_reuse && !can_avoid_reuse) { - throw JSONRPCError(RPC_WALLET_ERROR, "wallet does not have the \"avoid reuse\" feature enabled"); - } - - return avoid_reuse; -} - - -/** Used by RPC commands that have an include_watchonly parameter. - * We default to true for watchonly wallets if include_watchonly isn't - * explicitly set. - */ -static bool ParseIncludeWatchonly(const UniValue& include_watchonly, const CWallet& wallet) -{ - if (include_watchonly.isNull()) { - // if include_watchonly isn't explicitly set, then check if we have a watchonly wallet - return wallet.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS); - } - - // otherwise return whatever include_watchonly was set to - return include_watchonly.get_bool(); -} - /** Checks if a CKey is in the given CWallet compressed or otherwise*/ bool HaveKey(const SigningProvider& wallet, const CKey& key) @@ -86,70 +56,6 @@ bool HaveKey(const SigningProvider& wallet, const CKey& key) return wallet.HaveKey(key.GetPubKey().GetID()) || wallet.HaveKey(key2.GetPubKey().GetID()); } -bool GetWalletNameFromJSONRPCRequest(const JSONRPCRequest& request, std::string& wallet_name) -{ - if (URL_DECODE && request.URI.substr(0, WALLET_ENDPOINT_BASE.size()) == WALLET_ENDPOINT_BASE) { - // wallet endpoint was used - wallet_name = URL_DECODE(request.URI.substr(WALLET_ENDPOINT_BASE.size())); - return true; - } - return false; -} - -std::shared_ptr<CWallet> GetWalletForJSONRPCRequest(const JSONRPCRequest& request) -{ - CHECK_NONFATAL(request.mode == JSONRPCRequest::EXECUTE); - WalletContext& context = EnsureWalletContext(request.context); - - std::string wallet_name; - if (GetWalletNameFromJSONRPCRequest(request, wallet_name)) { - const std::shared_ptr<CWallet> pwallet = GetWallet(context, wallet_name); - if (!pwallet) throw JSONRPCError(RPC_WALLET_NOT_FOUND, "Requested wallet does not exist or is not loaded"); - return pwallet; - } - - std::vector<std::shared_ptr<CWallet>> wallets = GetWallets(context); - if (wallets.size() == 1) { - return wallets[0]; - } - - if (wallets.empty()) { - throw JSONRPCError( - RPC_WALLET_NOT_FOUND, "No wallet is loaded. Load a wallet using loadwallet or create a new one with createwallet. (Note: A default wallet is no longer automatically created)"); - } - throw JSONRPCError(RPC_WALLET_NOT_SPECIFIED, - "Wallet file not specified (must request wallet RPC through /wallet/<filename> uri-path)."); -} - -void EnsureWalletIsUnlocked(const CWallet& wallet) -{ - if (wallet.IsLocked()) { - throw JSONRPCError(RPC_WALLET_UNLOCK_NEEDED, "Error: Please enter the wallet passphrase with walletpassphrase first."); - } -} - -WalletContext& EnsureWalletContext(const std::any& context) -{ - auto wallet_context = util::AnyPtr<WalletContext>(context); - if (!wallet_context) { - throw JSONRPCError(RPC_INTERNAL_ERROR, "Wallet context not found"); - } - return *wallet_context; -} - -// also_create should only be set to true only when the RPC is expected to add things to a blank wallet and make it no longer blank -LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet, bool also_create) -{ - LegacyScriptPubKeyMan* spk_man = wallet.GetLegacyScriptPubKeyMan(); - if (!spk_man && also_create) { - spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); - } - if (!spk_man) { - throw JSONRPCError(RPC_WALLET_ERROR, "This type of wallet does not support this command"); - } - return *spk_man; -} - static void WalletTxToJSON(const CWallet& wallet, const CWalletTx& wtx, UniValue& entry) { interfaces::Chain& chain = wallet.chain(); @@ -157,13 +63,13 @@ static void WalletTxToJSON(const CWallet& wallet, const CWalletTx& wtx, UniValue entry.pushKV("confirmations", confirms); if (wtx.IsCoinBase()) entry.pushKV("generated", true); - if (confirms > 0) + if (auto* conf = wtx.state<TxStateConfirmed>()) { - entry.pushKV("blockhash", wtx.m_confirm.hashBlock.GetHex()); - entry.pushKV("blockheight", wtx.m_confirm.block_height); - entry.pushKV("blockindex", wtx.m_confirm.nIndex); + entry.pushKV("blockhash", conf->confirmed_block_hash.GetHex()); + entry.pushKV("blockheight", conf->confirmed_block_height); + entry.pushKV("blockindex", conf->position_in_block); int64_t block_time; - CHECK_NONFATAL(chain.findBlock(wtx.m_confirm.hashBlock, FoundBlock().time(block_time))); + CHECK_NONFATAL(chain.findBlock(conf->confirmed_block_hash, FoundBlock().time(block_time))); entry.pushKV("blocktime", block_time); } else { entry.pushKV("trusted", CachedTxIsTrusted(wallet, wtx)); @@ -175,7 +81,7 @@ static void WalletTxToJSON(const CWallet& wallet, const CWalletTx& wtx, UniValue conflicts.push_back(conflict.GetHex()); entry.pushKV("walletconflicts", conflicts); entry.pushKV("time", wtx.GetTxTime()); - entry.pushKV("timereceived", (int64_t)wtx.nTimeReceived); + entry.pushKV("timereceived", int64_t{wtx.nTimeReceived}); // Add opt-in RBF status std::string rbfStatus = "no"; @@ -192,13 +98,6 @@ static void WalletTxToJSON(const CWallet& wallet, const CWalletTx& wtx, UniValue entry.pushKV(item.first, item.second); } -static std::string LabelFromValue(const UniValue& value) -{ - std::string label = value.get_str(); - if (label == "*") - throw JSONRPCError(RPC_WALLET_INVALID_LABEL_NAME, "Invalid label name"); - return label; -} /** * Update coin control with fee estimation based on the given parameters @@ -245,7 +144,7 @@ static RPCHelpMan getnewaddress() "so payments received with the address will be associated with 'label'.\n", { {"label", RPCArg::Type::STR, RPCArg::Default{""}, "The label name for the address to be linked to. It can also be set to the empty string \"\" to represent the default label. The label does not need to exist, it will be created if there is no label by the given name."}, - {"address_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -addresstype"}, "The address type to use. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, + {"address_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -addresstype"}, "The address type to use. Options are \"legacy\", \"p2sh-segwit\", \"bech32\", and \"bech32m\"."}, }, RPCResult{ RPCResult::Type::STR, "address", "The new bitcoin address" @@ -298,7 +197,7 @@ static RPCHelpMan getrawchangeaddress() "\nReturns a new Bitcoin address, for receiving change.\n" "This is for use with raw transactions, NOT normal use.\n", { - {"address_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The address type to use. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, + {"address_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The address type to use. Options are \"legacy\", \"p2sh-segwit\", \"bech32\", and \"bech32m\"."}, }, RPCResult{ RPCResult::Type::STR, "address", "The address" @@ -603,63 +502,6 @@ static RPCHelpMan listaddressgroupings() }; } -static RPCHelpMan signmessage() -{ - return RPCHelpMan{"signmessage", - "\nSign a message with the private key of an address" + - HELP_REQUIRING_PASSPHRASE, - { - {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The bitcoin address to use for the private key."}, - {"message", RPCArg::Type::STR, RPCArg::Optional::NO, "The message to create a signature of."}, - }, - RPCResult{ - RPCResult::Type::STR, "signature", "The signature of the message encoded in base 64" - }, - RPCExamples{ - "\nUnlock the wallet for 30 seconds\n" - + HelpExampleCli("walletpassphrase", "\"mypassphrase\" 30") + - "\nCreate the signature\n" - + HelpExampleCli("signmessage", "\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\" \"my message\"") + - "\nVerify the signature\n" - + HelpExampleCli("verifymessage", "\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\" \"signature\" \"my message\"") + - "\nAs a JSON-RPC call\n" - + HelpExampleRpc("signmessage", "\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XX\", \"my message\"") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; - - LOCK(pwallet->cs_wallet); - - EnsureWalletIsUnlocked(*pwallet); - - std::string strAddress = request.params[0].get_str(); - std::string strMessage = request.params[1].get_str(); - - CTxDestination dest = DecodeDestination(strAddress); - if (!IsValidDestination(dest)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); - } - - const PKHash* pkhash = std::get_if<PKHash>(&dest); - if (!pkhash) { - throw JSONRPCError(RPC_TYPE_ERROR, "Address does not refer to key"); - } - - std::string signature; - SigningResult err = pwallet->SignMessage(strMessage, *pkhash, signature); - if (err == SigningResult::SIGNING_FAILED) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, SigningResultString(err)); - } else if (err != SigningResult::OK){ - throw JSONRPCError(RPC_WALLET_ERROR, SigningResultString(err)); - } - - return signature; -}, - }; -} - static CAmount GetReceived(const CWallet& wallet, const UniValue& params, bool by_label) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { std::set<CTxDestination> address_set; @@ -1814,41 +1656,6 @@ static RPCHelpMan abandontransaction() }; } - -static RPCHelpMan backupwallet() -{ - return RPCHelpMan{"backupwallet", - "\nSafely copies current wallet file to destination, which can be a directory or a path with filename.\n", - { - {"destination", RPCArg::Type::STR, RPCArg::Optional::NO, "The destination directory or file"}, - }, - RPCResult{RPCResult::Type::NONE, "", ""}, - RPCExamples{ - HelpExampleCli("backupwallet", "\"backup.dat\"") - + HelpExampleRpc("backupwallet", "\"backup.dat\"") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - const std::shared_ptr<const CWallet> pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; - - // Make sure the results are valid at least up to the most recent block - // the user could have gotten from another RPC command prior to now - pwallet->BlockUntilSyncedToCurrentChain(); - - LOCK(pwallet->cs_wallet); - - std::string strDest = request.params[0].get_str(); - if (!pwallet->BackupWallet(strDest)) { - throw JSONRPCError(RPC_WALLET_ERROR, "Error: Wallet backup failed!"); - } - - return NullUniValue; -}, - }; -} - - static RPCHelpMan keypoolrefill() { return RPCHelpMan{"keypoolrefill", @@ -1920,247 +1727,6 @@ static RPCHelpMan newkeypool() }; } - -static RPCHelpMan walletpassphrase() -{ - return RPCHelpMan{"walletpassphrase", - "\nStores the wallet decryption key in memory for 'timeout' seconds.\n" - "This is needed prior to performing transactions related to private keys such as sending bitcoins\n" - "\nNote:\n" - "Issuing the walletpassphrase command while the wallet is already unlocked will set a new unlock\n" - "time that overrides the old one.\n", - { - {"passphrase", RPCArg::Type::STR, RPCArg::Optional::NO, "The wallet passphrase"}, - {"timeout", RPCArg::Type::NUM, RPCArg::Optional::NO, "The time to keep the decryption key in seconds; capped at 100000000 (~3 years)."}, - }, - RPCResult{RPCResult::Type::NONE, "", ""}, - RPCExamples{ - "\nUnlock the wallet for 60 seconds\n" - + HelpExampleCli("walletpassphrase", "\"my pass phrase\" 60") + - "\nLock the wallet again (before 60 seconds)\n" - + HelpExampleCli("walletlock", "") + - "\nAs a JSON-RPC call\n" - + HelpExampleRpc("walletpassphrase", "\"my pass phrase\", 60") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - std::shared_ptr<CWallet> const wallet = GetWalletForJSONRPCRequest(request); - if (!wallet) return NullUniValue; - CWallet* const pwallet = wallet.get(); - - int64_t nSleepTime; - int64_t relock_time; - // Prevent concurrent calls to walletpassphrase with the same wallet. - LOCK(pwallet->m_unlock_mutex); - { - LOCK(pwallet->cs_wallet); - - if (!pwallet->IsCrypted()) { - throw JSONRPCError(RPC_WALLET_WRONG_ENC_STATE, "Error: running with an unencrypted wallet, but walletpassphrase was called."); - } - - // Note that the walletpassphrase is stored in request.params[0] which is not mlock()ed - SecureString strWalletPass; - strWalletPass.reserve(100); - // TODO: get rid of this .c_str() by implementing SecureString::operator=(std::string) - // Alternately, find a way to make request.params[0] mlock()'d to begin with. - strWalletPass = request.params[0].get_str().c_str(); - - // Get the timeout - nSleepTime = request.params[1].get_int64(); - // Timeout cannot be negative, otherwise it will relock immediately - if (nSleepTime < 0) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Timeout cannot be negative."); - } - // Clamp timeout - constexpr int64_t MAX_SLEEP_TIME = 100000000; // larger values trigger a macos/libevent bug? - if (nSleepTime > MAX_SLEEP_TIME) { - nSleepTime = MAX_SLEEP_TIME; - } - - if (strWalletPass.empty()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "passphrase can not be empty"); - } - - if (!pwallet->Unlock(strWalletPass)) { - throw JSONRPCError(RPC_WALLET_PASSPHRASE_INCORRECT, "Error: The wallet passphrase entered was incorrect."); - } - - pwallet->TopUpKeyPool(); - - pwallet->nRelockTime = GetTime() + nSleepTime; - relock_time = pwallet->nRelockTime; - } - - // rpcRunLater must be called without cs_wallet held otherwise a deadlock - // can occur. The deadlock would happen when RPCRunLater removes the - // previous timer (and waits for the callback to finish if already running) - // and the callback locks cs_wallet. - AssertLockNotHeld(wallet->cs_wallet); - // Keep a weak pointer to the wallet so that it is possible to unload the - // wallet before the following callback is called. If a valid shared pointer - // is acquired in the callback then the wallet is still loaded. - std::weak_ptr<CWallet> weak_wallet = wallet; - pwallet->chain().rpcRunLater(strprintf("lockwallet(%s)", pwallet->GetName()), [weak_wallet, relock_time] { - if (auto shared_wallet = weak_wallet.lock()) { - LOCK(shared_wallet->cs_wallet); - // Skip if this is not the most recent rpcRunLater callback. - if (shared_wallet->nRelockTime != relock_time) return; - shared_wallet->Lock(); - shared_wallet->nRelockTime = 0; - } - }, nSleepTime); - - return NullUniValue; -}, - }; -} - - -static RPCHelpMan walletpassphrasechange() -{ - return RPCHelpMan{"walletpassphrasechange", - "\nChanges the wallet passphrase from 'oldpassphrase' to 'newpassphrase'.\n", - { - {"oldpassphrase", RPCArg::Type::STR, RPCArg::Optional::NO, "The current passphrase"}, - {"newpassphrase", RPCArg::Type::STR, RPCArg::Optional::NO, "The new passphrase"}, - }, - RPCResult{RPCResult::Type::NONE, "", ""}, - RPCExamples{ - HelpExampleCli("walletpassphrasechange", "\"old one\" \"new one\"") - + HelpExampleRpc("walletpassphrasechange", "\"old one\", \"new one\"") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; - - LOCK(pwallet->cs_wallet); - - if (!pwallet->IsCrypted()) { - throw JSONRPCError(RPC_WALLET_WRONG_ENC_STATE, "Error: running with an unencrypted wallet, but walletpassphrasechange was called."); - } - - // TODO: get rid of these .c_str() calls by implementing SecureString::operator=(std::string) - // Alternately, find a way to make request.params[0] mlock()'d to begin with. - SecureString strOldWalletPass; - strOldWalletPass.reserve(100); - strOldWalletPass = request.params[0].get_str().c_str(); - - SecureString strNewWalletPass; - strNewWalletPass.reserve(100); - strNewWalletPass = request.params[1].get_str().c_str(); - - if (strOldWalletPass.empty() || strNewWalletPass.empty()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "passphrase can not be empty"); - } - - if (!pwallet->ChangeWalletPassphrase(strOldWalletPass, strNewWalletPass)) { - throw JSONRPCError(RPC_WALLET_PASSPHRASE_INCORRECT, "Error: The wallet passphrase entered was incorrect."); - } - - return NullUniValue; -}, - }; -} - - -static RPCHelpMan walletlock() -{ - return RPCHelpMan{"walletlock", - "\nRemoves the wallet encryption key from memory, locking the wallet.\n" - "After calling this method, you will need to call walletpassphrase again\n" - "before being able to call any methods which require the wallet to be unlocked.\n", - {}, - RPCResult{RPCResult::Type::NONE, "", ""}, - RPCExamples{ - "\nSet the passphrase for 2 minutes to perform a transaction\n" - + HelpExampleCli("walletpassphrase", "\"my pass phrase\" 120") + - "\nPerform a send (requires passphrase set)\n" - + HelpExampleCli("sendtoaddress", "\"" + EXAMPLE_ADDRESS[0] + "\" 1.0") + - "\nClear the passphrase since we are done before 2 minutes is up\n" - + HelpExampleCli("walletlock", "") + - "\nAs a JSON-RPC call\n" - + HelpExampleRpc("walletlock", "") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; - - LOCK(pwallet->cs_wallet); - - if (!pwallet->IsCrypted()) { - throw JSONRPCError(RPC_WALLET_WRONG_ENC_STATE, "Error: running with an unencrypted wallet, but walletlock was called."); - } - - pwallet->Lock(); - pwallet->nRelockTime = 0; - - return NullUniValue; -}, - }; -} - - -static RPCHelpMan encryptwallet() -{ - return RPCHelpMan{"encryptwallet", - "\nEncrypts the wallet with 'passphrase'. This is for first time encryption.\n" - "After this, any calls that interact with private keys such as sending or signing \n" - "will require the passphrase to be set prior the making these calls.\n" - "Use the walletpassphrase call for this, and then walletlock call.\n" - "If the wallet is already encrypted, use the walletpassphrasechange call.\n", - { - {"passphrase", RPCArg::Type::STR, RPCArg::Optional::NO, "The pass phrase to encrypt the wallet with. It must be at least 1 character, but should be long."}, - }, - RPCResult{RPCResult::Type::STR, "", "A string with further instructions"}, - RPCExamples{ - "\nEncrypt your wallet\n" - + HelpExampleCli("encryptwallet", "\"my pass phrase\"") + - "\nNow set the passphrase to use the wallet, such as for signing or sending bitcoin\n" - + HelpExampleCli("walletpassphrase", "\"my pass phrase\"") + - "\nNow we can do something like sign\n" - + HelpExampleCli("signmessage", "\"address\" \"test message\"") + - "\nNow lock the wallet again by removing the passphrase\n" - + HelpExampleCli("walletlock", "") + - "\nAs a JSON-RPC call\n" - + HelpExampleRpc("encryptwallet", "\"my pass phrase\"") - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); - if (!pwallet) return NullUniValue; - - LOCK(pwallet->cs_wallet); - - if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { - throw JSONRPCError(RPC_WALLET_ENCRYPTION_FAILED, "Error: wallet does not contain private keys, nothing to encrypt."); - } - - if (pwallet->IsCrypted()) { - throw JSONRPCError(RPC_WALLET_WRONG_ENC_STATE, "Error: running with an encrypted wallet, but encryptwallet was called."); - } - - // TODO: get rid of this .c_str() by implementing SecureString::operator=(std::string) - // Alternately, find a way to make request.params[0] mlock()'d to begin with. - SecureString strWalletPass; - strWalletPass.reserve(100); - strWalletPass = request.params[0].get_str().c_str(); - - if (strWalletPass.empty()) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "passphrase can not be empty"); - } - - if (!pwallet->EncryptWallet(strWalletPass)) { - throw JSONRPCError(RPC_WALLET_ENCRYPTION_FAILED, "Error: Failed to encrypt the wallet."); - } - - return "wallet encrypted; The keypool has been flushed and a new HD seed was generated (if you are using HD). You need to make a new backup."; -}, - }; -} - static RPCHelpMan lockunspent() { return RPCHelpMan{"lockunspent", @@ -2513,7 +2079,6 @@ static RPCHelpMan getwalletinfo() size_t kpExternalSize = pwallet->KeypoolCountExternalKeys(); const auto bal = GetBalance(*pwallet); - int64_t kp_oldest = pwallet->GetOldestKeyPoolTime(); obj.pushKV("walletname", pwallet->GetName()); obj.pushKV("walletversion", pwallet->GetVersion()); obj.pushKV("format", pwallet->GetDatabase().Format()); @@ -2521,8 +2086,9 @@ static RPCHelpMan getwalletinfo() obj.pushKV("unconfirmed_balance", ValueFromAmount(bal.m_mine_untrusted_pending)); obj.pushKV("immature_balance", ValueFromAmount(bal.m_mine_immature)); obj.pushKV("txcount", (int)pwallet->mapWallet.size()); - if (kp_oldest > 0) { - obj.pushKV("keypoololdest", kp_oldest); + const auto kp_oldest = pwallet->GetOldestKeyPoolTime(); + if (kp_oldest.has_value()) { + obj.pushKV("keypoololdest", kp_oldest.value()); } obj.pushKV("keypoolsize", (int64_t)kpExternalSize); @@ -2625,37 +2191,6 @@ static RPCHelpMan listwallets() }; } -static std::tuple<std::shared_ptr<CWallet>, std::vector<bilingual_str>> LoadWalletHelper(WalletContext& context, UniValue load_on_start_param, const std::string wallet_name) -{ - DatabaseOptions options; - DatabaseStatus status; - options.require_existing = true; - bilingual_str error; - std::vector<bilingual_str> warnings; - std::optional<bool> load_on_start = load_on_start_param.isNull() ? std::nullopt : std::optional<bool>(load_on_start_param.get_bool()); - std::shared_ptr<CWallet> const wallet = LoadWallet(context, wallet_name, load_on_start, options, status, error, warnings); - - if (!wallet) { - // Map bad format to not found, since bad format is returned when the - // wallet directory exists, but doesn't contain a data file. - RPCErrorCode code = RPC_WALLET_ERROR; - switch (status) { - case DatabaseStatus::FAILED_NOT_FOUND: - case DatabaseStatus::FAILED_BAD_FORMAT: - code = RPC_WALLET_NOT_FOUND; - break; - case DatabaseStatus::FAILED_ALREADY_LOADED: - code = RPC_WALLET_ALREADY_LOADED; - break; - default: // RPC_WALLET_ERROR is returned for all other cases. - break; - } - throw JSONRPCError(code, error.original); - } - - return { wallet, warnings }; -} - static RPCHelpMan loadwallet() { return RPCHelpMan{"loadwallet", @@ -2855,68 +2390,6 @@ static RPCHelpMan createwallet() }; } -static RPCHelpMan restorewallet() -{ - return RPCHelpMan{ - "restorewallet", - "\nRestore and loads a wallet from backup.\n", - { - {"wallet_name", RPCArg::Type::STR, RPCArg::Optional::NO, "The name that will be applied to the restored wallet"}, - {"backup_file", RPCArg::Type::STR, RPCArg::Optional::NO, "The backup file that will be used to restore the wallet."}, - {"load_on_startup", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED_NAMED_ARG, "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, - }, - RPCResult{ - RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR, "name", "The wallet name if restored successfully."}, - {RPCResult::Type::STR, "warning", "Warning message if wallet was not loaded cleanly."}, - } - }, - RPCExamples{ - HelpExampleCli("restorewallet", "\"testwallet\" \"home\\backups\\backup-file.bak\"") - + HelpExampleRpc("restorewallet", "\"testwallet\" \"home\\backups\\backup-file.bak\"") - + HelpExampleCliNamed("restorewallet", {{"wallet_name", "testwallet"}, {"backup_file", "home\\backups\\backup-file.bak\""}, {"load_on_startup", true}}) - + HelpExampleRpcNamed("restorewallet", {{"wallet_name", "testwallet"}, {"backup_file", "home\\backups\\backup-file.bak\""}, {"load_on_startup", true}}) - }, - [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue -{ - - WalletContext& context = EnsureWalletContext(request.context); - - auto backup_file = fs::u8path(request.params[1].get_str()); - - if (!fs::exists(backup_file)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Backup file does not exist"); - } - - std::string wallet_name = request.params[0].get_str(); - - const fs::path wallet_path = fsbridge::AbsPathJoin(GetWalletDir(), fs::u8path(wallet_name)); - - if (fs::exists(wallet_path)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Wallet name already exists."); - } - - if (!TryCreateDirectories(wallet_path)) { - throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Failed to create database path '%s'. Database already exists.", wallet_path.u8string())); - } - - auto wallet_file = wallet_path / "wallet.dat"; - - fs::copy_file(backup_file, wallet_file, fs::copy_option::fail_if_exists); - - auto [wallet, warnings] = LoadWalletHelper(context, request.params[2], wallet_name); - - UniValue obj(UniValue::VOBJ); - obj.pushKV("name", wallet->GetName()); - obj.pushKV("warning", Join(warnings, Untranslated("\n")).original); - - return obj; - -}, - }; -} - static RPCHelpMan unloadwallet() { return RPCHelpMan{"unloadwallet", @@ -3211,7 +2684,7 @@ static std::vector<RPCArg> FundTxDoc() {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, std::string() + "The fee estimate mode, must be one of (case insensitive):\n" " \"" + FeeModes("\"\n\"") + "\""}, - {"replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125 replaceable.\n" + {"replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125-replaceable.\n" "Allows this transaction to be replaced by a transaction with higher fees"}, {"solving_data", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "Keys and scripts needed for producing a final transaction with a dummy signature.\n" "Used for fee estimation during coin selection.", @@ -3831,6 +3304,11 @@ static RPCHelpMan rescanblockchain() { std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return NullUniValue; + CWallet& wallet{*pwallet}; + + // Make sure the results are valid at least up to the most recent block + // the user could have gotten from another RPC command prior to now + wallet.BlockUntilSyncedToCurrentChain(); WalletRescanReserver reserver(*pwallet); if (!reserver.reserve()) { @@ -4098,8 +3576,12 @@ RPCHelpMan getaddressinfo() ret.pushKV("solvable", false); } + const auto& spk_mans = pwallet->GetScriptPubKeyMans(scriptPubKey); + // In most cases there is only one matching ScriptPubKey manager and we can't resolve ambiguity in a better way + ScriptPubKeyMan* spk_man{nullptr}; + if (spk_mans.size()) spk_man = *spk_mans.begin(); - DescriptorScriptPubKeyMan* desc_spk_man = dynamic_cast<DescriptorScriptPubKeyMan*>(pwallet->GetScriptPubKeyMan(scriptPubKey)); + DescriptorScriptPubKeyMan* desc_spk_man = dynamic_cast<DescriptorScriptPubKeyMan*>(spk_man); if (desc_spk_man) { std::string desc_str; if (desc_spk_man->GetDescriptorString(desc_str, /* priv */ false)) { @@ -4114,7 +3596,6 @@ RPCHelpMan getaddressinfo() ret.pushKV("ischange", ScriptIsChange(*pwallet, scriptPubKey)); - ScriptPubKeyMan* spk_man = pwallet->GetScriptPubKeyMan(scriptPubKey); if (spk_man) { if (const std::unique_ptr<CKeyMetadata> meta = spk_man->GetMetadata(dest)) { ret.pushKV("timestamp", meta->nCreateTime); @@ -4542,6 +4023,7 @@ static RPCHelpMan walletprocesspsbt() " \"NONE|ANYONECANPAY\"\n" " \"SINGLE|ANYONECANPAY\""}, {"bip32derivs", RPCArg::Type::BOOL, RPCArg::Default{true}, "Include BIP 32 derivation paths for public keys if we know them"}, + {"finalize", RPCArg::Type::BOOL, RPCArg::Default{true}, "Also finalize inputs if possible"}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -4563,7 +4045,7 @@ static RPCHelpMan walletprocesspsbt() // the user could have gotten from another RPC command prior to now wallet.BlockUntilSyncedToCurrentChain(); - RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VBOOL, UniValue::VSTR}); + RPCTypeCheck(request.params, {UniValue::VSTR}); // Unserialize the transaction PartiallySignedTransaction psbtx; @@ -4578,11 +4060,12 @@ static RPCHelpMan walletprocesspsbt() // Fill transaction with our data and also sign bool sign = request.params[1].isNull() ? true : request.params[1].get_bool(); bool bip32derivs = request.params[3].isNull() ? true : request.params[3].get_bool(); + bool finalize = request.params[4].isNull() ? true : request.params[4].get_bool(); bool complete = true; if (sign) EnsureWalletIsUnlocked(*pwallet); - const TransactionError err{wallet.FillPSBT(psbtx, complete, nHashType, sign, bip32derivs)}; + const TransactionError err{wallet.FillPSBT(psbtx, complete, nHashType, sign, bip32derivs, nullptr, finalize)}; if (err != TransactionError::OK) { throw JSONRPCTransactionError(err); } @@ -4851,6 +4334,15 @@ RPCHelpMan removeprunedfunds(); RPCHelpMan importmulti(); RPCHelpMan importdescriptors(); RPCHelpMan listdescriptors(); +RPCHelpMan signmessage(); +RPCHelpMan backupwallet(); +RPCHelpMan restorewallet(); + +// encryption +RPCHelpMan walletpassphrase(); +RPCHelpMan walletpassphrasechange(); +RPCHelpMan walletlock(); +RPCHelpMan encryptwallet(); Span<const CRPCCommand> GetWalletRPCCommands() { @@ -4926,5 +4418,5 @@ static const CRPCCommand commands[] = { "wallet", &walletprocesspsbt, }, }; // clang-format on - return MakeSpan(commands); + return commands; } diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index 8b88ffe8ed..2c3d413cb0 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -7,34 +7,10 @@ #include <span.h> -#include <any> -#include <memory> -#include <string> -#include <vector> - class CRPCCommand; -class CWallet; -class JSONRPCRequest; -class LegacyScriptPubKeyMan; -class UniValue; -class CTransaction; -struct PartiallySignedTransaction; -struct WalletContext; Span<const CRPCCommand> GetWalletRPCCommands(); -/** - * Figures out what wallet, if any, to use for a JSONRPCRequest. - * - * @param[in] request JSONRPCRequest that wishes to access a wallet - * @return nullptr if no wallet should be used, or a pointer to the CWallet - */ -std::shared_ptr<CWallet> GetWalletForJSONRPCRequest(const JSONRPCRequest& request); - -void EnsureWalletIsUnlocked(const CWallet&); -WalletContext& EnsureWalletContext(const std::any& context); -LegacyScriptPubKeyMan& EnsureLegacyScriptPubKeyMan(CWallet& wallet, bool also_create = false); - RPCHelpMan getaddressinfo(); RPCHelpMan signrawtransactionwithwallet(); -#endif //BITCOIN_WALLET_RPCWALLET_H +#endif // BITCOIN_WALLET_RPCWALLET_H diff --git a/src/wallet/salvage.cpp b/src/wallet/salvage.cpp index 4151099c1f..241d77c9de 100644 --- a/src/wallet/salvage.cpp +++ b/src/wallet/salvage.cpp @@ -133,7 +133,7 @@ bool RecoverDatabaseFile(const fs::path& file_path, bilingual_str& error, std::v } DbTxn* ptxn = env->TxnBegin(); - CWallet dummyWallet(nullptr, "", CreateDummyWalletDatabase()); + CWallet dummyWallet(nullptr, "", gArgs, CreateDummyWalletDatabase()); for (KeyValPair& row : salvagedData) { /* Filter for only private key type KV pairs to be added to the salvaged wallet */ diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 619ebc8b4f..1769429efe 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -354,15 +354,22 @@ bool LegacyScriptPubKeyMan::TopUpInactiveHDChain(const CKeyID seed_id, int64_t i return true; } -void LegacyScriptPubKeyMan::MarkUnusedAddresses(const CScript& script) +std::vector<WalletDestination> LegacyScriptPubKeyMan::MarkUnusedAddresses(const CScript& script) { LOCK(cs_KeyStore); + std::vector<WalletDestination> result; // extract addresses and check if they match with an unused keypool key for (const auto& keyid : GetAffectedKeys(script, *this)) { std::map<CKeyID, int64_t>::const_iterator mi = m_pool_key_to_index.find(keyid); if (mi != m_pool_key_to_index.end()) { WalletLogPrintf("%s: Detected a used keypool key, mark all keypool keys up to this key as used\n", __func__); - MarkReserveKeysAsUsed(mi->second); + for (const auto& keypool : MarkReserveKeysAsUsed(mi->second)) { + // derive all possible destinations as any of them could have been used + for (const auto& type : LEGACY_OUTPUT_TYPES) { + const auto& dest = GetDestinationForKey(keypool.vchPubKey, type); + result.push_back({dest, keypool.fInternal}); + } + } if (!TopUp()) { WalletLogPrintf("%s: Topping up keypool failed (locked wallet)\n", __func__); @@ -384,6 +391,8 @@ void LegacyScriptPubKeyMan::MarkUnusedAddresses(const CScript& script) } } } + + return result; } void LegacyScriptPubKeyMan::UpgradeKeyMetadata() @@ -400,7 +409,7 @@ void LegacyScriptPubKeyMan::UpgradeKeyMetadata() CKey key; GetKey(meta.hd_seed_id, key); CExtKey masterKey; - masterKey.SetSeed(key.begin(), key.size()); + masterKey.SetSeed(key); // Add to map CKeyID master_id = masterKey.key.GetPubKey().GetID(); std::copy(master_id.begin(), master_id.begin() + 4, meta.key_origin.fingerprint); @@ -528,7 +537,7 @@ static int64_t GetOldestKeyTimeInPool(const std::set<int64_t>& setKeyPool, Walle return keypool.nTime; } -int64_t LegacyScriptPubKeyMan::GetOldestKeyPoolTime() const +std::optional<int64_t> LegacyScriptPubKeyMan::GetOldestKeyPoolTime() const { LOCK(cs_KeyStore); @@ -610,7 +619,7 @@ SigningResult LegacyScriptPubKeyMan::SignMessage(const std::string& message, con return SigningResult::SIGNING_FAILED; } -TransactionError LegacyScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbtx, const PrecomputedTransactionData& txdata, int sighash_type, bool sign, bool bip32derivs, int* n_signed) const +TransactionError LegacyScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbtx, const PrecomputedTransactionData& txdata, int sighash_type, bool sign, bool bip32derivs, int* n_signed, bool finalize) const { if (n_signed) { *n_signed = 0; @@ -639,7 +648,7 @@ TransactionError LegacyScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psb } SignatureData sigdata; input.FillSignatureData(sigdata); - SignPSBTInput(HidingSigningProvider(this, !sign, !bip32derivs), psbtx, i, &txdata, sighash_type); + SignPSBTInput(HidingSigningProvider(this, !sign, !bip32derivs), psbtx, i, &txdata, sighash_type, nullptr, finalize); bool signed_one = PSBTInputSigned(input); if (n_signed && (signed_one || !sign)) { @@ -1085,7 +1094,7 @@ void LegacyScriptPubKeyMan::DeriveNewChildKey(WalletBatch &batch, CKeyMetadata& if (!GetKey(hd_chain.seed_id, seed)) throw std::runtime_error(std::string(__func__) + ": seed not found"); - masterKey.SetSeed(seed.begin(), seed.size()); + masterKey.SetSeed(seed); // derive m/0' // use hardened derivation (child keys >= 0x80000000 are hardened after bip32) @@ -1427,7 +1436,7 @@ void LegacyScriptPubKeyMan::LearnAllRelatedScripts(const CPubKey& key) LearnRelatedScripts(key, OutputType::P2SH_SEGWIT); } -void LegacyScriptPubKeyMan::MarkReserveKeysAsUsed(int64_t keypool_id) +std::vector<CKeyPool> LegacyScriptPubKeyMan::MarkReserveKeysAsUsed(int64_t keypool_id) { AssertLockHeld(cs_KeyStore); bool internal = setInternalKeyPool.count(keypool_id); @@ -1435,6 +1444,7 @@ void LegacyScriptPubKeyMan::MarkReserveKeysAsUsed(int64_t keypool_id) std::set<int64_t> *setKeyPool = internal ? &setInternalKeyPool : (set_pre_split_keypool.empty() ? &setExternalKeyPool : &set_pre_split_keypool); auto it = setKeyPool->begin(); + std::vector<CKeyPool> result; WalletBatch batch(m_storage.GetDatabase()); while (it != std::end(*setKeyPool)) { const int64_t& index = *(it); @@ -1448,7 +1458,10 @@ void LegacyScriptPubKeyMan::MarkReserveKeysAsUsed(int64_t keypool_id) batch.ErasePool(index); WalletLogPrintf("keypool index %d removed\n", index); it = setKeyPool->erase(it); + result.push_back(std::move(keypool)); } + + return result; } std::vector<CKeyID> GetAffectedKeys(const CScript& spk, const SigningProvider& provider) @@ -1820,19 +1833,32 @@ bool DescriptorScriptPubKeyMan::TopUp(unsigned int size) return true; } -void DescriptorScriptPubKeyMan::MarkUnusedAddresses(const CScript& script) +std::vector<WalletDestination> DescriptorScriptPubKeyMan::MarkUnusedAddresses(const CScript& script) { LOCK(cs_desc_man); + std::vector<WalletDestination> result; if (IsMine(script)) { int32_t index = m_map_script_pub_keys[script]; if (index >= m_wallet_descriptor.next_index) { WalletLogPrintf("%s: Detected a used keypool item at index %d, mark all keypool items up to this item as used\n", __func__, index); - m_wallet_descriptor.next_index = index + 1; + auto out_keys = std::make_unique<FlatSigningProvider>(); + std::vector<CScript> scripts_temp; + while (index >= m_wallet_descriptor.next_index) { + if (!m_wallet_descriptor.descriptor->ExpandFromCache(m_wallet_descriptor.next_index, m_wallet_descriptor.cache, scripts_temp, *out_keys)) { + throw std::runtime_error(std::string(__func__) + ": Unable to expand descriptor from cache"); + } + CTxDestination dest; + ExtractDestination(scripts_temp[0], dest); + result.push_back({dest, std::nullopt}); + m_wallet_descriptor.next_index++; + } } if (!TopUp()) { WalletLogPrintf("%s: Topping up keypool failed (locked wallet)\n", __func__); } } + + return result; } void DescriptorScriptPubKeyMan::AddDescriptorKey(const CKey& key, const CPubKey &pubkey) @@ -1876,12 +1902,6 @@ bool DescriptorScriptPubKeyMan::AddDescriptorKeyWithDB(WalletBatch& batch, const bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_key, OutputType addr_type, bool internal) { - if (addr_type == OutputType::BECH32M) { - // Don't allow setting up taproot descriptors yet - // TODO: Allow setting up taproot descriptors - return false; - } - LOCK(cs_desc_man); assert(m_storage.IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); @@ -1911,7 +1931,10 @@ bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_ desc_prefix = "wpkh(" + xpub + "/84'"; break; } - case OutputType::BECH32M: assert(false); // TODO: Setup taproot descriptor + case OutputType::BECH32M: { + desc_prefix = "tr(" + xpub + "/86'"; + break; + } } // no default case, so the compiler can warn about missing cases assert(!desc_prefix.empty()); @@ -1970,11 +1993,10 @@ bool DescriptorScriptPubKeyMan::HavePrivateKeys() const return m_map_keys.size() > 0 || m_map_crypted_keys.size() > 0; } -int64_t DescriptorScriptPubKeyMan::GetOldestKeyPoolTime() const +std::optional<int64_t> DescriptorScriptPubKeyMan::GetOldestKeyPoolTime() const { // This is only used for getwalletinfo output and isn't relevant to descriptor wallets. - // The magic number 0 indicates that it shouldn't be displayed so that's what we return. - return 0; + return std::nullopt; } @@ -2078,7 +2100,7 @@ SigningResult DescriptorScriptPubKeyMan::SignMessage(const std::string& message, return SigningResult::OK; } -TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbtx, const PrecomputedTransactionData& txdata, int sighash_type, bool sign, bool bip32derivs, int* n_signed) const +TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& psbtx, const PrecomputedTransactionData& txdata, int sighash_type, bool sign, bool bip32derivs, int* n_signed, bool finalize) const { if (n_signed) { *n_signed = 0; @@ -2128,7 +2150,7 @@ TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& } } - SignPSBTInput(HidingSigningProvider(keys.get(), !sign, !bip32derivs), psbtx, i, &txdata, sighash_type); + SignPSBTInput(HidingSigningProvider(keys.get(), !sign, !bip32derivs), psbtx, i, &txdata, sighash_type, nullptr, finalize); bool signed_one = PSBTInputSigned(input); if (n_signed && (signed_one || !sign)) { diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index ef74638751..ebe064fa0a 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -19,6 +19,7 @@ #include <boost/signals2/signal.hpp> +#include <optional> #include <unordered_map> enum class OutputType; @@ -148,6 +149,12 @@ public: } }; +struct WalletDestination +{ + CTxDestination dest; + std::optional<bool> internal; +}; + /* * A class implementing ScriptPubKeyMan manages some (or all) scriptPubKeys used in a wallet. * It contains the scripts and keys related to the scriptPubKeys it manages. @@ -180,8 +187,14 @@ public: */ virtual bool TopUp(unsigned int size = 0) { return false; } - //! Mark unused addresses as being used - virtual void MarkUnusedAddresses(const CScript& script) {} + /** Mark unused addresses as being used + * Affects all keys up to and including the one determined by provided script. + * + * @param script determines the last key to mark as used + * + * @return All of the addresses affected + */ + virtual std::vector<WalletDestination> MarkUnusedAddresses(const CScript& script) { return {}; } /** Sets up the key generation stuff, i.e. generates new HD seeds and sets them as active. * Returns false if already setup or setup fails, true if setup is successful @@ -203,7 +216,7 @@ public: //! The action to do when the DB needs rewrite virtual void RewriteDB() {} - virtual int64_t GetOldestKeyPoolTime() const { return GetTime(); } + virtual std::optional<int64_t> GetOldestKeyPoolTime() const { return GetTime(); } virtual unsigned int GetKeyPoolSize() const { return 0; } @@ -223,7 +236,7 @@ public: /** Sign a message with the given script */ virtual SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const { return SigningResult::SIGNING_FAILED; }; /** Adds script and derivation path information to a PSBT, and optionally signs it. */ - virtual TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr) const { return TransactionError::INVALID_PSBT; } + virtual TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const { return TransactionError::INVALID_PSBT; } virtual uint256 GetID() const { return uint256(); } @@ -356,7 +369,7 @@ public: bool TopUp(unsigned int size = 0) override; - void MarkUnusedAddresses(const CScript& script) override; + std::vector<WalletDestination> MarkUnusedAddresses(const CScript& script) override; //! Upgrade stored CKeyMetadata objects to store key origin info as KeyOriginInfo void UpgradeKeyMetadata(); @@ -371,7 +384,7 @@ public: void RewriteDB() override; - int64_t GetOldestKeyPoolTime() const override; + std::optional<int64_t> GetOldestKeyPoolTime() const override; size_t KeypoolCountExternalKeys() const; unsigned int GetKeyPoolSize() const override; @@ -387,7 +400,7 @@ public: bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, bilingual_str>& input_errors) const override; SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const override; - TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr) const override; + TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const override; uint256 GetID() const override; @@ -481,9 +494,13 @@ public: void LearnAllRelatedScripts(const CPubKey& key); /** - * Marks all keys in the keypool up to and including reserve_key as used. + * Marks all keys in the keypool up to and including the provided key as used. + * + * @param keypool_id determines the last key to mark as used + * + * @return All affected keys */ - void MarkReserveKeysAsUsed(int64_t keypool_id) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); + std::vector<CKeyPool> MarkReserveKeysAsUsed(int64_t keypool_id) EXCLUSIVE_LOCKS_REQUIRED(cs_KeyStore); const std::map<CKeyID, int64_t>& GetAllReserveKeys() const { return m_pool_key_to_index; } std::set<CKeyID> GetKeys() const override; @@ -563,7 +580,7 @@ public: // (with or without private keys), the "keypool" is a single xpub. bool TopUp(unsigned int size = 0) override; - void MarkUnusedAddresses(const CScript& script) override; + std::vector<WalletDestination> MarkUnusedAddresses(const CScript& script) override; bool IsHDEnabled() const override; @@ -577,7 +594,7 @@ public: bool HavePrivateKeys() const override; - int64_t GetOldestKeyPoolTime() const override; + std::optional<int64_t> GetOldestKeyPoolTime() const override; unsigned int GetKeyPoolSize() const override; int64_t GetTimeFirstKey() const override; @@ -592,7 +609,7 @@ public: bool SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, Coin>& coins, int sighash, std::map<int, bilingual_str>& input_errors) const override; SigningResult SignMessage(const std::string& message, const PKHash& pkhash, std::string& str_sig) const override; - TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr) const override; + TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const override; uint256 GetID() const override; diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 8606924bb3..0acd8a811f 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -74,7 +74,7 @@ static void add_coin(std::vector<COutput>& coins, CWallet& wallet, const CAmount uint256 txid = tx.GetHash(); LOCK(wallet.cs_wallet); - auto ret = wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(txid), std::forward_as_tuple(MakeTransactionRef(std::move(tx)))); + auto ret = wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(txid), std::forward_as_tuple(MakeTransactionRef(std::move(tx)), TxStateInactive{})); assert(ret.second); CWalletTx& wtx = (*ret.first).second; if (fIsFromMe) @@ -133,7 +133,7 @@ inline std::vector<OutputGroup>& KnapsackGroupOutputs(const std::vector<COutput> /* long_term_feerate= */ CFeeRate(0), /* discard_feerate= */ CFeeRate(0), /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); static std::vector<OutputGroup> static_groups; - static_groups = GroupOutputs(wallet, coins, coin_selection_params, filter, /* positive_only */false); + static_groups = GroupOutputs(wallet, coins, coin_selection_params, filter, /*positive_only=*/false); return static_groups; } @@ -273,7 +273,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) /* long_term_feerate= */ CFeeRate(1000), /* discard_feerate= */ CFeeRate(1000), /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); { - std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", CreateMockWalletDatabase()); + std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()); wallet->LoadWallet(); LOCK(wallet->cs_wallet); wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -297,7 +297,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) } { - std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", CreateMockWalletDatabase()); + std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()); wallet->LoadWallet(); LOCK(wallet->cs_wallet); wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -320,7 +320,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) BOOST_AUTO_TEST_CASE(knapsack_solver_test) { - std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", CreateMockWalletDatabase()); + std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()); wallet->LoadWallet(); LOCK(wallet->cs_wallet); wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -601,7 +601,7 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) BOOST_AUTO_TEST_CASE(ApproximateBestSubset) { - std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", CreateMockWalletDatabase()); + std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()); wallet->LoadWallet(); LOCK(wallet->cs_wallet); wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -624,7 +624,7 @@ BOOST_AUTO_TEST_CASE(ApproximateBestSubset) // Tests that with the ideal conditions, the coin selector will always be able to find a solution that can pay the target value BOOST_AUTO_TEST_CASE(SelectCoins_test) { - std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", CreateMockWalletDatabase()); + std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()); wallet->LoadWallet(); LOCK(wallet->cs_wallet); wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -733,7 +733,7 @@ BOOST_AUTO_TEST_CASE(waste_test) add_coin(1 * COIN, 1, selection, fee, fee); add_coin(2 * COIN, 2, selection, fee, fee); const CAmount exact_target{in_amt - fee * 2}; - BOOST_CHECK_EQUAL(0, GetSelectionWaste(selection, /* change_cost */ 0, exact_target)); + BOOST_CHECK_EQUAL(0, GetSelectionWaste(selection, /*change_cost=*/0, exact_target)); selection.clear(); // No Waste when (fee - long_term_fee) == (-cost_of_change), and no excess diff --git a/src/wallet/test/fuzz/notifications.cpp b/src/wallet/test/fuzz/notifications.cpp new file mode 100644 index 0000000000..0601c492cd --- /dev/null +++ b/src/wallet/test/fuzz/notifications.cpp @@ -0,0 +1,170 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/setup_common.h> +#include <util/translation.h> +#include <wallet/context.h> +#include <wallet/receive.h> +#include <wallet/wallet.h> +#include <wallet/walletdb.h> +#include <wallet/walletutil.h> + +#include <cassert> +#include <cstdint> +#include <string> +#include <vector> + +namespace { +const TestingSetup* g_setup; + +void initialize_setup() +{ + static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); + g_setup = testing_setup.get(); +} + +/** + * Wraps a descriptor wallet for fuzzing. The constructor writes the sqlite db + * to disk, the destructor deletes it. + */ +struct FuzzedWallet { + ArgsManager args; + WalletContext context; + std::shared_ptr<CWallet> wallet; + FuzzedWallet(const std::string& name) + { + context.args = &args; + context.chain = g_setup->m_node.chain.get(); + + DatabaseOptions options; + options.require_create = true; + options.create_flags = WALLET_FLAG_DESCRIPTORS; + const std::optional<bool> load_on_start; + gArgs.ForceSetArg("-keypool", "0"); // Avoid timeout in TopUp() + + DatabaseStatus status; + bilingual_str error; + std::vector<bilingual_str> warnings; + wallet = CreateWallet(context, name, load_on_start, options, status, error, warnings); + assert(wallet); + assert(error.empty()); + assert(warnings.empty()); + assert(wallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); + } + ~FuzzedWallet() + { + const auto name{wallet->GetName()}; + std::vector<bilingual_str> warnings; + std::optional<bool> load_on_start; + assert(RemoveWallet(context, wallet, load_on_start, warnings)); + assert(warnings.empty()); + UnloadWallet(std::move(wallet)); + fs::remove_all(GetWalletDir() / name); + } + CScript GetScriptPubKey(FuzzedDataProvider& fuzzed_data_provider) + { + auto type{fuzzed_data_provider.PickValueInArray(OUTPUT_TYPES)}; + CTxDestination dest; + bilingual_str error; + if (fuzzed_data_provider.ConsumeBool()) { + assert(wallet->GetNewDestination(type, "", dest, error)); + } else { + assert(wallet->GetNewChangeDestination(type, dest, error)); + } + assert(error.empty()); + return GetScriptForDestination(dest); + } +}; + +FUZZ_TARGET_INIT(wallet_notifications, initialize_setup) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + // The total amount, to be distributed to the wallets a and b in txs + // without fee. Thus, the balance of the wallets should always equal the + // total amount. + const auto total_amount{ConsumeMoney(fuzzed_data_provider)}; + FuzzedWallet a{"fuzzed_wallet_a"}; + FuzzedWallet b{"fuzzed_wallet_b"}; + + // Keep track of all coins in this test. + // Each tuple in the chain represents the coins and the block created with + // those coins. Once the block is mined, the next tuple will have an empty + // block and the freshly mined coins. + using Coins = std::set<std::tuple<CAmount, COutPoint>>; + std::vector<std::tuple<Coins, CBlock>> chain; + { + // Add the initial entry + chain.emplace_back(); + auto& [coins, block]{chain.back()}; + coins.emplace(total_amount, COutPoint{uint256::ONE, 1}); + } + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 200) + { + CallOneOf( + fuzzed_data_provider, + [&] { + auto& [coins_orig, block]{chain.back()}; + // Copy the coins for this block and consume all of them + Coins coins = coins_orig; + while (!coins.empty()) { + // Create a new tx + CMutableTransaction tx{}; + // Add some coins as inputs to it + auto num_inputs{fuzzed_data_provider.ConsumeIntegralInRange<int>(1, coins.size())}; + CAmount in{0}; + while (num_inputs-- > 0) { + const auto& [coin_amt, coin_outpoint]{*coins.begin()}; + in += coin_amt; + tx.vin.emplace_back(coin_outpoint); + coins.erase(coins.begin()); + } + // Create some outputs spending all inputs, without fee + LIMITED_WHILE(in > 0 && fuzzed_data_provider.ConsumeBool(), 100) + { + const auto out_value{ConsumeMoney(fuzzed_data_provider, in)}; + in -= out_value; + auto& wallet{fuzzed_data_provider.ConsumeBool() ? a : b}; + tx.vout.emplace_back(out_value, wallet.GetScriptPubKey(fuzzed_data_provider)); + } + // Spend the remaining input value, if any + auto& wallet{fuzzed_data_provider.ConsumeBool() ? a : b}; + tx.vout.emplace_back(in, wallet.GetScriptPubKey(fuzzed_data_provider)); + // Add tx to block + block.vtx.emplace_back(MakeTransactionRef(tx)); + } + // Mine block + a.wallet->blockConnected(block, chain.size()); + b.wallet->blockConnected(block, chain.size()); + // Store the coins for the next block + Coins coins_new; + for (const auto& tx : block.vtx) { + uint32_t i{0}; + for (const auto& out : tx->vout) { + coins_new.emplace(out.nValue, COutPoint{tx->GetHash(), i++}); + } + } + chain.emplace_back(coins_new, CBlock{}); + }, + [&] { + if (chain.size() <= 1) return; // The first entry can't be removed + auto& [coins, block]{chain.back()}; + if (block.vtx.empty()) return; // Can only disconnect if the block was submitted first + // Disconnect block + a.wallet->blockDisconnected(block, chain.size() - 1); + b.wallet->blockDisconnected(block, chain.size() - 1); + chain.pop_back(); + }); + auto& [coins, first_block]{chain.front()}; + if (!first_block.vtx.empty()) { + // Only check balance when at least one block was submitted + const auto bal_a{GetBalance(*a.wallet).m_mine_trusted}; + const auto bal_b{GetBalance(*b.wallet).m_mine_trusted}; + assert(total_amount == bal_a + bal_b); + } + } +} +} // namespace diff --git a/src/wallet/test/ismine_tests.cpp b/src/wallet/test/ismine_tests.cpp index 5d25885bd4..dda202d55e 100644 --- a/src/wallet/test/ismine_tests.cpp +++ b/src/wallet/test/ismine_tests.cpp @@ -34,7 +34,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK compressed { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForRawPubKey(pubkeys[0]); @@ -51,7 +51,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK uncompressed { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForRawPubKey(uncompressedPubkey); @@ -68,7 +68,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PKH compressed { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForDestination(PKHash(pubkeys[0])); @@ -85,7 +85,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PKH uncompressed { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForDestination(PKHash(uncompressedPubkey)); @@ -102,7 +102,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2SH { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -126,7 +126,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2SH inside P2SH (invalid) { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -144,7 +144,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2SH inside P2WSH (invalid) { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -162,7 +162,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH inside P2WSH (invalid) { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -178,7 +178,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2WSH inside P2WSH (invalid) { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -196,7 +196,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH compressed { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -211,7 +211,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH uncompressed { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); @@ -230,7 +230,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // scriptPubKey multisig { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -261,7 +261,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2SH multisig { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); @@ -282,7 +282,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig with compressed keys { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -308,7 +308,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig with uncompressed key { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); @@ -334,7 +334,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig wrapped in P2SH { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -361,7 +361,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // OP_RETURN { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -375,7 +375,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // witness unspendable { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -389,7 +389,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // witness unknown { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -403,7 +403,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // Nonstandard { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index dd24fa2c19..7bc2bb5583 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -34,12 +34,12 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) CDataStream s_prev_tx1(ParseHex("0200000000010158e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd7501000000171600145f275f436b09a8cc9a2eb2a2f528485c68a56323feffffff02d8231f1b0100000017a914aed962d6654f9a2b36608eb9d64d2b260db4f1118700c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88702483045022100a22edcc6e5bc511af4cc4ae0de0fcd75c7e04d8c1c3a8aa9d820ed4b967384ec02200642963597b9b1bc22c75e9f3e117284a962188bf5e8a74c895089046a20ad770121035509a48eb623e10aace8bfd0212fdb8a8e5af3c94b0b133b95e114cab89e4f7965000000"), SER_NETWORK, PROTOCOL_VERSION); CTransactionRef prev_tx1; s_prev_tx1 >> prev_tx1; - m_wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(prev_tx1->GetHash()), std::forward_as_tuple(prev_tx1)); + m_wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(prev_tx1->GetHash()), std::forward_as_tuple(prev_tx1, TxStateInactive{})); CDataStream s_prev_tx2(ParseHex("0200000001aad73931018bd25f84ae400b68848be09db706eac2ac18298babee71ab656f8b0000000048473044022058f6fc7c6a33e1b31548d481c826c015bd30135aad42cd67790dab66d2ad243b02204a1ced2604c6735b6393e5b41691dd78b00f0c5942fb9f751856faa938157dba01feffffff0280f0fa020000000017a9140fb9463421696b82c833af241c78c17ddbde493487d0f20a270100000017a91429ca74f8a08f81999428185c97b5d852e4063f618765000000"), SER_NETWORK, PROTOCOL_VERSION); CTransactionRef prev_tx2; s_prev_tx2 >> prev_tx2; - m_wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(prev_tx2->GetHash()), std::forward_as_tuple(prev_tx2)); + m_wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(prev_tx2->GetHash()), std::forward_as_tuple(prev_tx2, TxStateInactive{})); // Import descriptors for keys and scripts import_descriptor(m_wallet, "sh(multi(2,xprv9s21ZrQH143K2LE7W4Xf3jATf9jECxSb7wj91ZnmY4qEJrS66Qru9RFqq8xbkgT32ya6HqYJweFdJUEDf5Q6JFV7jMiUws7kQfe6Tv4RbfN/0h/0h/0h,xprv9s21ZrQH143K2LE7W4Xf3jATf9jECxSb7wj91ZnmY4qEJrS66Qru9RFqq8xbkgT32ya6HqYJweFdJUEDf5Q6JFV7jMiUws7kQfe6Tv4RbfN/0h/0h/1h))"); diff --git a/src/wallet/test/scriptpubkeyman_tests.cpp b/src/wallet/test/scriptpubkeyman_tests.cpp index 347a436429..0e78855ced 100644 --- a/src/wallet/test/scriptpubkeyman_tests.cpp +++ b/src/wallet/test/scriptpubkeyman_tests.cpp @@ -17,7 +17,7 @@ BOOST_FIXTURE_TEST_SUITE(scriptpubkeyman_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(CanProvide) { // Set up wallet and keyman variables. - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); LegacyScriptPubKeyMan& keyman = *wallet.GetOrCreateLegacyScriptPubKeyMan(); // Make a 1 of 2 multisig script diff --git a/src/wallet/test/spend_tests.cpp b/src/wallet/test/spend_tests.cpp index d88d8eabdb..926f28686d 100644 --- a/src/wallet/test/spend_tests.cpp +++ b/src/wallet/test/spend_tests.cpp @@ -17,7 +17,7 @@ BOOST_FIXTURE_TEST_SUITE(spend_tests, WalletTestingSetup) BOOST_FIXTURE_TEST_CASE(SubtractFee, TestChain100Setup) { CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); - auto wallet = CreateSyncedWallet(*m_node.chain, m_node.chainman->ActiveChain(), coinbaseKey); + auto wallet = CreateSyncedWallet(*m_node.chain, m_node.chainman->ActiveChain(), m_args, coinbaseKey); // Check that a subtract-from-recipient transaction slightly less than the // coinbase input amount does not create a change output (because it would diff --git a/src/wallet/test/util.cpp b/src/wallet/test/util.cpp index 2990fc8f8d..93a3404d2c 100644 --- a/src/wallet/test/util.cpp +++ b/src/wallet/test/util.cpp @@ -15,9 +15,9 @@ #include <memory> -std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cchain, const CKey& key) +std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cchain, ArgsManager& args, const CKey& key) { - auto wallet = std::make_unique<CWallet>(&chain, "", CreateMockWalletDatabase()); + auto wallet = std::make_unique<CWallet>(&chain, "", args, CreateMockWalletDatabase()); { LOCK2(wallet->cs_wallet, ::cs_main); wallet->SetLastBlockProcessed(cchain.Height(), cchain.Tip()->GetBlockHash()); diff --git a/src/wallet/test/util.h b/src/wallet/test/util.h index 288c111571..3adb82b85f 100644 --- a/src/wallet/test/util.h +++ b/src/wallet/test/util.h @@ -7,6 +7,7 @@ #include <memory> +class ArgsManager; class CChain; class CKey; class CWallet; @@ -14,6 +15,6 @@ namespace interfaces { class Chain; } // namespace interfaces -std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cchain, const CKey& key); +std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cchain, ArgsManager& args, const CKey& key); #endif // BITCOIN_WALLET_TEST_UTIL_H diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index 313606cc28..762702522e 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -8,7 +8,7 @@ WalletTestingSetup::WalletTestingSetup(const std::string& chainName) : TestingSetup(chainName), - m_wallet(m_node.chain.get(), "", CreateMockWalletDatabase()) + m_wallet(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()) { m_wallet.LoadWallet(); m_chain_notifications_handler = m_node.chain->handleNotifications({ &m_wallet, [](CWallet*) {} }); diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 7a658c10a2..8cb3cede9b 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -98,7 +98,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions fails to read an unknown start block. { - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -118,7 +118,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions picks up transactions in both the old // and new block files. { - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -145,7 +145,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions only picks transactions in the new block // file. { - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -171,7 +171,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions scans no blocks. { - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -208,7 +208,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) // before the missing block, and success for a key whose creation time is // after. { - const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); wallet->SetupLegacyScriptPubKeyMan(); WITH_LOCK(wallet->cs_wallet, wallet->SetLastBlockProcessed(newTip->nHeight, newTip->GetBlockHash())); WalletContext context; @@ -274,7 +274,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) { WalletContext context; context.args = &gArgs; - const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); { auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); @@ -296,7 +296,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // Call importwallet RPC and verify all blocks with timestamps >= BLOCK_TIME // were scanned, and no prior blocks were scanned. { - const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); LOCK(wallet->cs_wallet); wallet->SetupLegacyScriptPubKeyMan(); @@ -329,8 +329,8 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // debit functions. BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) { - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); - CWalletTx wtx(m_coinbase_txns.back()); + CWallet wallet(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); + CWalletTx wtx{m_coinbase_txns.back(), TxStateConfirmed{m_node.chainman->ActiveChain().Tip()->GetBlockHash(), m_node.chainman->ActiveChain().Height(), /*position_in_block=*/0}}; LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -338,9 +338,6 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) wallet.SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); - CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash(), 0); - wtx.m_confirm = confirm; - // Call GetImmatureCredit() once before adding the key to the wallet to // cache the current immature credit amount, which is 0. BOOST_CHECK_EQUAL(CachedTxGetImmatureCredit(wallet, wtx), 0); @@ -355,7 +352,7 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) static int64_t AddTx(ChainstateManager& chainman, CWallet& wallet, uint32_t lockTime, int64_t mockTime, int64_t blockTime) { CMutableTransaction tx; - CWalletTx::Confirmation confirm; + TxState state = TxStateInactive{}; tx.nLockTime = lockTime; SetMockTime(mockTime); CBlockIndex* block = nullptr; @@ -367,13 +364,13 @@ static int64_t AddTx(ChainstateManager& chainman, CWallet& wallet, uint32_t lock block = inserted.first->second; block->nTime = blockTime; block->phashBlock = &hash; - confirm = {CWalletTx::Status::CONFIRMED, block->nHeight, hash, 0}; + state = TxStateConfirmed{hash, block->nHeight, /*position_in_block=*/0}; } - - // If transaction is already in map, to avoid inconsistencies, unconfirmation - // is needed before confirm again with different block. - return wallet.AddToWallet(MakeTransactionRef(tx), confirm, [&](CWalletTx& wtx, bool /* new_tx */) { - wtx.setUnconfirmed(); + return wallet.AddToWallet(MakeTransactionRef(tx), state, [&](CWalletTx& wtx, bool /* new_tx */) { + // Assign wtx.m_state to simplify test and avoid the need to simulate + // reorg events. Without this, AddToWallet asserts false when the same + // transaction is confirmed in different blocks. + wtx.m_state = state; return true; })->nTimeSmart; } @@ -503,7 +500,7 @@ public: ListCoinsTestingSetup() { CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); - wallet = CreateSyncedWallet(*m_node.chain, m_node.chainman->ActiveChain(), coinbaseKey); + wallet = CreateSyncedWallet(*m_node.chain, m_node.chainman->ActiveChain(), m_args, coinbaseKey); } ~ListCoinsTestingSetup() @@ -534,8 +531,7 @@ public: wallet->SetLastBlockProcessed(wallet->GetLastBlockHeight() + 1, m_node.chainman->ActiveChain().Tip()->GetBlockHash()); auto it = wallet->mapWallet.find(tx->GetHash()); BOOST_CHECK(it != wallet->mapWallet.end()); - CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash(), 1); - it->second.m_confirm = confirm; + it->second.m_state = TxStateConfirmed{m_node.chainman->ActiveChain().Tip()->GetBlockHash(), m_node.chainman->ActiveChain().Height(), /*position_in_block=*/1}; return it->second; } @@ -606,7 +602,7 @@ BOOST_FIXTURE_TEST_CASE(ListCoinsTest, ListCoinsTestingSetup) BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) { { - const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); wallet->SetupLegacyScriptPubKeyMan(); wallet->SetMinVersion(FEATURE_LATEST); wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); @@ -616,7 +612,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) BOOST_CHECK(!wallet->GetNewDestination(OutputType::BECH32, "", dest, error)); } { - const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); LOCK(wallet->cs_wallet); wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet->SetMinVersion(FEATURE_LATEST); @@ -686,7 +682,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_descriptor_test, BasicTestingSetup) vw << (int32_t)0; vw << (int32_t)1; - VectorReader vr(0, 0, malformed_record, 0); + SpanReader vr{0, 0, malformed_record}; WalletDescriptor w_desc; BOOST_CHECK_EXCEPTION(vr >> w_desc, std::ios_base::failure, malformed_descriptor); } @@ -783,18 +779,14 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) // deadlock during the sync and simulates a new block notification happening // as soon as possible. addtx_count = 0; - auto handler = HandleLoadWallet(context, [&](std::unique_ptr<interfaces::Wallet> wallet) EXCLUSIVE_LOCKS_REQUIRED(wallet->wallet()->cs_wallet, context.wallets_mutex) { + auto handler = HandleLoadWallet(context, [&](std::unique_ptr<interfaces::Wallet> wallet) { BOOST_CHECK(rescan_completed); m_coinbase_txns.push_back(CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); block_tx = TestSimpleSpend(*m_coinbase_txns[2], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); m_coinbase_txns.push_back(CreateAndProcessBlock({block_tx}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); mempool_tx = TestSimpleSpend(*m_coinbase_txns[3], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); BOOST_CHECK(m_node.chain->broadcastTransaction(MakeTransactionRef(mempool_tx), DEFAULT_TRANSACTION_MAXFEE, false, error)); - LEAVE_CRITICAL_SECTION(context.wallets_mutex); - LEAVE_CRITICAL_SECTION(wallet->wallet()->cs_wallet); SyncWithValidationInterfaceQueue(); - ENTER_CRITICAL_SECTION(wallet->wallet()->cs_wallet); - ENTER_CRITICAL_SECTION(context.wallets_mutex); }); wallet = TestLoadWallet(context); BOOST_CHECK_EQUAL(addtx_count, 4); @@ -824,30 +816,35 @@ BOOST_FIXTURE_TEST_CASE(ZapSelectTx, TestChain100Setup) context.args = &gArgs; context.chain = m_node.chain.get(); auto wallet = TestLoadWallet(context); - CKey key; - key.MakeNewKey(true); - AddKey(*wallet, key); + AddKey(*wallet, coinbaseKey); - std::string error; + // rescan to ensure coinbase transactions from test fixture are picked up by the wallet + { + WalletRescanReserver reserver(*wallet); + reserver.reserve(); + wallet->ScanForWalletTransactions(m_node.chain->getBlockHash(0), 0, /* max height= */ {}, reserver, /* update= */ true); + } + // create one more block to get the first block coinbase to maturity m_coinbase_txns.push_back(CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); - auto block_tx = TestSimpleSpend(*m_coinbase_txns[0], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); - CreateAndProcessBlock({block_tx}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); + // spend first coinbase tx + auto spend_tx = TestSimpleSpend(*m_coinbase_txns[0], 0, coinbaseKey, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); + CreateAndProcessBlock({spend_tx}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); SyncWithValidationInterfaceQueue(); { - auto block_hash = block_tx.GetHash(); + auto spend_tx_hash = spend_tx.GetHash(); auto prev_hash = m_coinbase_txns[0]->GetHash(); LOCK(wallet->cs_wallet); BOOST_CHECK(wallet->HasWalletSpend(prev_hash)); - BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_hash), 1u); + BOOST_CHECK_EQUAL(wallet->mapWallet.count(spend_tx_hash), 1u); - std::vector<uint256> vHashIn{ block_hash }, vHashOut; + std::vector<uint256> vHashIn{spend_tx_hash}, vHashOut; BOOST_CHECK_EQUAL(wallet->ZapSelectTx(vHashIn, vHashOut), DBErrors::LOAD_OK); BOOST_CHECK(!wallet->HasWalletSpend(prev_hash)); - BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_hash), 0u); + BOOST_CHECK_EQUAL(wallet->mapWallet.count(spend_tx_hash), 0u); } TestUnloadWallet(std::move(wallet)); diff --git a/src/wallet/test/wallet_transaction_tests.cpp b/src/wallet/test/wallet_transaction_tests.cpp new file mode 100644 index 0000000000..5ef2904f66 --- /dev/null +++ b/src/wallet/test/wallet_transaction_tests.cpp @@ -0,0 +1,24 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <wallet/transaction.h> + +#include <wallet/test/wallet_test_fixture.h> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(wallet_transaction_tests, WalletTestingSetup) + +BOOST_AUTO_TEST_CASE(roundtrip) +{ + for (uint8_t hash = 0; hash < 5; ++hash) { + for (int index = -2; index < 3; ++index) { + TxState state = TxStateInterpretSerialized(TxStateUnrecognized{uint256{hash}, index}); + BOOST_CHECK_EQUAL(TxStateSerializedBlockHash(state), uint256{hash}); + BOOST_CHECK_EQUAL(TxStateSerializedIndex(state), index); + } + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/transaction.cpp b/src/wallet/transaction.cpp index cf98b516f1..a926c0ecc1 100644 --- a/src/wallet/transaction.cpp +++ b/src/wallet/transaction.cpp @@ -15,7 +15,7 @@ bool CWalletTx::IsEquivalentTo(const CWalletTx& _tx) const bool CWalletTx::InMempool() const { - return fInMempool; + return state<TxStateInMempool>(); } int64_t CWalletTx::GetTxTime() const diff --git a/src/wallet/transaction.h b/src/wallet/transaction.h index 1ccef31056..52d72cccf3 100644 --- a/src/wallet/transaction.h +++ b/src/wallet/transaction.h @@ -11,12 +11,102 @@ #include <wallet/ismine.h> #include <threadsafety.h> #include <tinyformat.h> +#include <util/overloaded.h> #include <util/strencodings.h> #include <util/string.h> #include <list> +#include <variant> #include <vector> +//! State of transaction confirmed in a block. +struct TxStateConfirmed { + uint256 confirmed_block_hash; + int confirmed_block_height; + int position_in_block; + + explicit TxStateConfirmed(const uint256& block_hash, int height, int index) : confirmed_block_hash(block_hash), confirmed_block_height(height), position_in_block(index) {} +}; + +//! State of transaction added to mempool. +struct TxStateInMempool { +}; + +//! State of rejected transaction that conflicts with a confirmed block. +struct TxStateConflicted { + uint256 conflicting_block_hash; + int conflicting_block_height; + + explicit TxStateConflicted(const uint256& block_hash, int height) : conflicting_block_hash(block_hash), conflicting_block_height(height) {} +}; + +//! State of transaction not confirmed or conflicting with a known block and +//! not in the mempool. May conflict with the mempool, or with an unknown block, +//! or be abandoned, never broadcast, or rejected from the mempool for another +//! reason. +struct TxStateInactive { + bool abandoned; + + explicit TxStateInactive(bool abandoned = false) : abandoned(abandoned) {} +}; + +//! State of transaction loaded in an unrecognized state with unexpected hash or +//! index values. Treated as inactive (with serialized hash and index values +//! preserved) by default, but may enter another state if transaction is added +//! to the mempool, or confirmed, or abandoned, or found conflicting. +struct TxStateUnrecognized { + uint256 block_hash; + int index; + + TxStateUnrecognized(const uint256& block_hash, int index) : block_hash(block_hash), index(index) {} +}; + +//! All possible CWalletTx states +using TxState = std::variant<TxStateConfirmed, TxStateInMempool, TxStateConflicted, TxStateInactive, TxStateUnrecognized>; + +//! Subset of states transaction sync logic is implemented to handle. +using SyncTxState = std::variant<TxStateConfirmed, TxStateInMempool, TxStateInactive>; + +//! Try to interpret deserialized TxStateUnrecognized data as a recognized state. +static inline TxState TxStateInterpretSerialized(TxStateUnrecognized data) +{ + if (data.block_hash == uint256::ZERO) { + if (data.index == 0) return TxStateInactive{}; + } else if (data.block_hash == uint256::ONE) { + if (data.index == -1) return TxStateInactive{/*abandoned=*/true}; + } else if (data.index >= 0) { + return TxStateConfirmed{data.block_hash, /*height=*/-1, data.index}; + } else if (data.index == -1) { + return TxStateConflicted{data.block_hash, /*height=*/-1}; + } + return data; +} + +//! Get TxState serialized block hash. Inverse of TxStateInterpretSerialized. +static inline uint256 TxStateSerializedBlockHash(const TxState& state) +{ + return std::visit(util::Overloaded{ + [](const TxStateInactive& inactive) { return inactive.abandoned ? uint256::ONE : uint256::ZERO; }, + [](const TxStateInMempool& in_mempool) { return uint256::ZERO; }, + [](const TxStateConfirmed& confirmed) { return confirmed.confirmed_block_hash; }, + [](const TxStateConflicted& conflicted) { return conflicted.conflicting_block_hash; }, + [](const TxStateUnrecognized& unrecognized) { return unrecognized.block_hash; } + }, state); +} + +//! Get TxState serialized block index. Inverse of TxStateInterpretSerialized. +static inline int TxStateSerializedIndex(const TxState& state) +{ + return std::visit(util::Overloaded{ + [](const TxStateInactive& inactive) { return inactive.abandoned ? -1 : 0; }, + [](const TxStateInMempool& in_mempool) { return 0; }, + [](const TxStateConfirmed& confirmed) { return confirmed.position_in_block; }, + [](const TxStateConflicted& conflicted) { return -1; }, + [](const TxStateUnrecognized& unrecognized) { return unrecognized.index; } + }, state); +} + + typedef std::map<std::string, std::string> mapValue_t; /** Legacy class used for deserializing vtxPrev for backwards compatibility. @@ -45,12 +135,6 @@ public: */ class CWalletTx { -private: - /** Constant used in hashBlock to indicate tx has been abandoned, only used at - * serialization/deserialization to avoid ambiguity with conflicted. - */ - static constexpr const uint256& ABANDON_HASH = uint256::ONE; - public: /** * Key/value map with information about the transaction. @@ -111,11 +195,9 @@ public: */ mutable bool m_is_cache_empty{true}; mutable bool fChangeCached; - mutable bool fInMempool; mutable CAmount nChangeCached; - CWalletTx(CTransactionRef arg) - : tx(std::move(arg)) + CWalletTx(CTransactionRef tx, const TxState& state) : tx(std::move(tx)), m_state(state) { Init(); } @@ -129,44 +211,12 @@ public: nTimeSmart = 0; fFromMe = false; fChangeCached = false; - fInMempool = false; nChangeCached = 0; nOrderPos = -1; - m_confirm = Confirmation{}; } CTransactionRef tx; - - /** New transactions start as UNCONFIRMED. At BlockConnected, - * they will transition to CONFIRMED. In case of reorg, at BlockDisconnected, - * they roll back to UNCONFIRMED. If we detect a conflicting transaction at - * block connection, we update conflicted tx and its dependencies as CONFLICTED. - * If tx isn't confirmed and outside of mempool, the user may switch it to ABANDONED - * by using the abandontransaction call. This last status may be override by a CONFLICTED - * or CONFIRMED transition. - */ - enum Status { - UNCONFIRMED, - CONFIRMED, - CONFLICTED, - ABANDONED - }; - - /** Confirmation includes tx status and a triplet of {block height/block hash/tx index in block} - * at which tx has been confirmed. All three are set to 0 if tx is unconfirmed or abandoned. - * Meaning of these fields changes with CONFLICTED state where they instead point to block hash - * and block height of the deepest conflicting tx. - */ - struct Confirmation { - Status status; - int block_height; - uint256 hashBlock; - int nIndex; - Confirmation(Status status = UNCONFIRMED, int block_height = 0, uint256 block_hash = uint256(), int block_index = 0) - : status{status}, block_height{block_height}, hashBlock{block_hash}, nIndex{block_index} {} - }; - - Confirmation m_confirm; + TxState m_state; template<typename Stream> void Serialize(Stream& s) const @@ -184,8 +234,8 @@ public: std::vector<uint8_t> dummy_vector1; //!< Used to be vMerkleBranch std::vector<uint8_t> dummy_vector2; //!< Used to be vtxPrev bool dummy_bool = false; //!< Used to be fSpent - uint256 serializedHash = isAbandoned() ? ABANDON_HASH : m_confirm.hashBlock; - int serializedIndex = isAbandoned() || isConflicted() ? -1 : m_confirm.nIndex; + uint256 serializedHash = TxStateSerializedBlockHash(m_state); + int serializedIndex = TxStateSerializedIndex(m_state); s << tx << serializedHash << dummy_vector1 << serializedIndex << dummy_vector2 << mapValueCopy << vOrderForm << fTimeReceivedIsTxTime << nTimeReceived << fFromMe << dummy_bool; } @@ -197,24 +247,11 @@ public: std::vector<uint256> dummy_vector1; //!< Used to be vMerkleBranch std::vector<CMerkleTx> dummy_vector2; //!< Used to be vtxPrev bool dummy_bool; //! Used to be fSpent + uint256 serialized_block_hash; int serializedIndex; - s >> tx >> m_confirm.hashBlock >> dummy_vector1 >> serializedIndex >> dummy_vector2 >> mapValue >> vOrderForm >> fTimeReceivedIsTxTime >> nTimeReceived >> fFromMe >> dummy_bool; - - /* At serialization/deserialization, an nIndex == -1 means that hashBlock refers to - * the earliest block in the chain we know this or any in-wallet ancestor conflicts - * with. If nIndex == -1 and hashBlock is ABANDON_HASH, it means transaction is abandoned. - * In same context, an nIndex >= 0 refers to a confirmed transaction (if hashBlock set) or - * unconfirmed one. Older clients interpret nIndex == -1 as unconfirmed for backward - * compatibility (pre-commit 9ac63d6). - */ - if (serializedIndex == -1 && m_confirm.hashBlock == ABANDON_HASH) { - setAbandoned(); - } else if (serializedIndex == -1) { - setConflicted(); - } else if (!m_confirm.hashBlock.IsNull()) { - m_confirm.nIndex = serializedIndex; - setConfirmed(); - } + s >> tx >> serialized_block_hash >> dummy_vector1 >> serializedIndex >> dummy_vector2 >> mapValue >> vOrderForm >> fTimeReceivedIsTxTime >> nTimeReceived >> fFromMe >> dummy_bool; + + m_state = TxStateInterpretSerialized({serialized_block_hash, serializedIndex}); const auto it_op = mapValue.find("n"); nOrderPos = (it_op != mapValue.end()) ? LocaleIndependentAtoi<int64_t>(it_op->second) : -1; @@ -250,20 +287,13 @@ public: int64_t GetTxTime() const; - bool isAbandoned() const { return m_confirm.status == CWalletTx::ABANDONED; } - void setAbandoned() - { - m_confirm.status = CWalletTx::ABANDONED; - m_confirm.hashBlock = uint256(); - m_confirm.block_height = 0; - m_confirm.nIndex = 0; - } - bool isConflicted() const { return m_confirm.status == CWalletTx::CONFLICTED; } - void setConflicted() { m_confirm.status = CWalletTx::CONFLICTED; } - bool isUnconfirmed() const { return m_confirm.status == CWalletTx::UNCONFIRMED; } - void setUnconfirmed() { m_confirm.status = CWalletTx::UNCONFIRMED; } - bool isConfirmed() const { return m_confirm.status == CWalletTx::CONFIRMED; } - void setConfirmed() { m_confirm.status = CWalletTx::CONFIRMED; } + template<typename T> const T* state() const { return std::get_if<T>(&m_state); } + template<typename T> T* state() { return std::get_if<T>(&m_state); } + + bool isAbandoned() const { return state<TxStateInactive>() && state<TxStateInactive>()->abandoned; } + bool isConflicted() const { return state<TxStateConflicted>(); } + bool isUnconfirmed() const { return !isAbandoned() && !isConflicted() && !isConfirmed(); } + bool isConfirmed() const { return state<TxStateConfirmed>(); } const uint256& GetHash() const { return tx->GetHash(); } bool IsCoinBase() const { return tx->IsCoinBase(); } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 4eb9d5560d..cfe550cb3f 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -99,7 +99,11 @@ static void UpdateWalletSetting(interfaces::Chain& chain, */ static void RefreshMempoolStatus(CWalletTx& tx, interfaces::Chain& chain) { - tx.fInMempool = chain.isInMempool(tx.GetHash()); + if (chain.isInMempool(tx.GetHash())) { + tx.m_state = TxStateInMempool(); + } else if (tx.state<TxStateInMempool>()) { + tx.m_state = TxStateInactive(); + } } bool AddWallet(WalletContext& context, const std::shared_ptr<CWallet>& wallet) @@ -885,7 +889,7 @@ bool CWallet::IsSpentKey(const uint256& hash, unsigned int n) const return false; } -CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const CWalletTx::Confirmation& confirm, const UpdateWalletTxFn& update_wtx, bool fFlushOnClose, bool rescanning_old_block) +CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const TxState& state, const UpdateWalletTxFn& update_wtx, bool fFlushOnClose, bool rescanning_old_block) { LOCK(cs_wallet); @@ -906,31 +910,28 @@ CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const CWalletTx::Confirmatio } // Inserts only if not already there, returns tx inserted or tx found - auto ret = mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(hash), std::forward_as_tuple(tx)); + auto ret = mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(hash), std::forward_as_tuple(tx, state)); CWalletTx& wtx = (*ret.first).second; bool fInsertedNew = ret.second; bool fUpdated = update_wtx && update_wtx(wtx, fInsertedNew); if (fInsertedNew) { - wtx.m_confirm = confirm; - wtx.nTimeReceived = chain().getAdjustedTime(); + wtx.nTimeReceived = GetTime(); wtx.nOrderPos = IncOrderPosNext(&batch); wtx.m_it_wtxOrdered = wtxOrdered.insert(std::make_pair(wtx.nOrderPos, &wtx)); wtx.nTimeSmart = ComputeTimeSmart(wtx, rescanning_old_block); - AddToSpends(hash, &batch); + if (IsFromMe(*tx.get())) { + AddToSpends(hash); + } } if (!fInsertedNew) { - if (confirm.status != wtx.m_confirm.status) { - wtx.m_confirm.status = confirm.status; - wtx.m_confirm.nIndex = confirm.nIndex; - wtx.m_confirm.hashBlock = confirm.hashBlock; - wtx.m_confirm.block_height = confirm.block_height; + if (state.index() != wtx.m_state.index()) { + wtx.m_state = state; fUpdated = true; } else { - assert(wtx.m_confirm.nIndex == confirm.nIndex); - assert(wtx.m_confirm.hashBlock == confirm.hashBlock); - assert(wtx.m_confirm.block_height == confirm.block_height); + assert(TxStateSerializedIndex(wtx.m_state) == TxStateSerializedIndex(state)); + assert(TxStateSerializedBlockHash(wtx.m_state) == TxStateSerializedBlockHash(state)); } // If we have a witness-stripped version of this transaction, and we // see a new version with a witness, then we must be upgrading a pre-segwit @@ -959,15 +960,15 @@ CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const CWalletTx::Confirmatio #if HAVE_SYSTEM // notify an external script when a wallet transaction comes in or is updated - std::string strCmd = gArgs.GetArg("-walletnotify", ""); + std::string strCmd = m_args.GetArg("-walletnotify", ""); if (!strCmd.empty()) { boost::replace_all(strCmd, "%s", hash.GetHex()); - if (confirm.status == CWalletTx::Status::CONFIRMED) + if (auto* conf = wtx.state<TxStateConfirmed>()) { - boost::replace_all(strCmd, "%b", confirm.hashBlock.GetHex()); - boost::replace_all(strCmd, "%h", ToString(confirm.block_height)); + boost::replace_all(strCmd, "%b", conf->confirmed_block_hash.GetHex()); + boost::replace_all(strCmd, "%h", ToString(conf->confirmed_block_height)); } else { boost::replace_all(strCmd, "%b", "unconfirmed"); boost::replace_all(strCmd, "%h", "-1"); @@ -990,7 +991,7 @@ CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const CWalletTx::Confirmatio bool CWallet::LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx) { - const auto& ins = mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(hash), std::forward_as_tuple(nullptr)); + const auto& ins = mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(hash), std::forward_as_tuple(nullptr, TxStateInactive{})); CWalletTx& wtx = ins.first->second; if (!fill_wtx(wtx, ins.second)) { return false; @@ -998,22 +999,21 @@ bool CWallet::LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx // If wallet doesn't have a chain (e.g wallet-tool), don't bother to update txn. if (HaveChain()) { bool active; - int height; - if (chain().findBlock(wtx.m_confirm.hashBlock, FoundBlock().inActiveChain(active).height(height)) && active) { - // Update cached block height variable since it not stored in the - // serialized transaction. - wtx.m_confirm.block_height = height; - } else if (wtx.isConflicted() || wtx.isConfirmed()) { + auto lookup_block = [&](const uint256& hash, int& height, TxState& state) { // If tx block (or conflicting block) was reorged out of chain // while the wallet was shutdown, change tx status to UNCONFIRMED // and reset block height, hash, and index. ABANDONED tx don't have // associated blocks and don't need to be updated. The case where a // transaction was reorged out while online and then reconfirmed // while offline is covered by the rescan logic. - wtx.setUnconfirmed(); - wtx.m_confirm.hashBlock = uint256(); - wtx.m_confirm.block_height = 0; - wtx.m_confirm.nIndex = 0; + if (!chain().findBlock(hash, FoundBlock().inActiveChain(active).height(height)) || !active) { + state = TxStateInactive{}; + } + }; + if (auto* conf = wtx.state<TxStateConfirmed>()) { + lookup_block(conf->confirmed_block_hash, conf->confirmed_block_height, wtx.m_state); + } else if (auto* conf = wtx.state<TxStateConflicted>()) { + lookup_block(conf->conflicting_block_hash, conf->conflicting_block_height, wtx.m_state); } } if (/* insertion took place */ ins.second) { @@ -1024,27 +1024,27 @@ bool CWallet::LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx auto it = mapWallet.find(txin.prevout.hash); if (it != mapWallet.end()) { CWalletTx& prevtx = it->second; - if (prevtx.isConflicted()) { - MarkConflicted(prevtx.m_confirm.hashBlock, prevtx.m_confirm.block_height, wtx.GetHash()); + if (auto* prev = prevtx.state<TxStateConflicted>()) { + MarkConflicted(prev->conflicting_block_hash, prev->conflicting_block_height, wtx.GetHash()); } } } return true; } -bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::Confirmation confirm, bool fUpdate, bool rescanning_old_block) +bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, const SyncTxState& state, bool fUpdate, bool rescanning_old_block) { const CTransaction& tx = *ptx; { AssertLockHeld(cs_wallet); - if (!confirm.hashBlock.IsNull()) { + if (auto* conf = std::get_if<TxStateConfirmed>(&state)) { for (const CTxIn& txin : tx.vin) { std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(txin.prevout); while (range.first != range.second) { if (range.first->second != tx.GetHash()) { - WalletLogPrintf("Transaction %s (in block %s) conflicts with wallet transaction %s (both spend %s:%i)\n", tx.GetHash().ToString(), confirm.hashBlock.ToString(), range.first->second.ToString(), range.first->first.hash.ToString(), range.first->first.n); - MarkConflicted(confirm.hashBlock, confirm.block_height, range.first->second); + WalletLogPrintf("Transaction %s (in block %s) conflicts with wallet transaction %s (both spend %s:%i)\n", tx.GetHash().ToString(), conf->confirmed_block_hash.ToString(), range.first->second.ToString(), range.first->first.hash.ToString(), range.first->first.n); + MarkConflicted(conf->confirmed_block_hash, conf->confirmed_block_height, range.first->second); } range.first++; } @@ -1063,14 +1063,30 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, CWalletTx::Co // loop though all outputs for (const CTxOut& txout: tx.vout) { - for (const auto& spk_man_pair : m_spk_managers) { - spk_man_pair.second->MarkUnusedAddresses(txout.scriptPubKey); + for (const auto& spk_man : GetScriptPubKeyMans(txout.scriptPubKey)) { + for (auto &dest : spk_man->MarkUnusedAddresses(txout.scriptPubKey)) { + // If internal flag is not defined try to infer it from the ScriptPubKeyMan + if (!dest.internal.has_value()) { + dest.internal = IsInternalScriptPubKeyMan(spk_man); + } + + // skip if can't determine whether it's a receiving address or not + if (!dest.internal.has_value()) continue; + + // If this is a receiving address and it's not in the address book yet + // (e.g. it wasn't generated on this node or we're restoring from backup) + // add it to the address book for proper transaction accounting + if (!*dest.internal && !FindAddressBookEntry(dest.dest, /* allow_change= */ false)) { + SetAddressBook(dest.dest, "", "receive"); + } + } } } // Block disconnection override an abandoned tx as unconfirmed // which means user may have to call abandontransaction again - return AddToWallet(MakeTransactionRef(tx), confirm, /* update_wtx= */ nullptr, /* fFlushOnClose= */ false, rescanning_old_block); + TxState tx_state = std::visit([](auto&& s) -> TxState { return s; }, state); + return AddToWallet(MakeTransactionRef(tx), tx_state, /*update_wtx=*/nullptr, /*fFlushOnClose=*/false, rescanning_old_block); } } return false; @@ -1126,7 +1142,7 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) if (currentconfirm == 0 && !wtx.isAbandoned()) { // If the orig tx was not in block/mempool, none of its spends can be in mempool assert(!wtx.InMempool()); - wtx.setAbandoned(); + wtx.m_state = TxStateInactive{/*abandoned=*/true}; wtx.MarkDirty(); batch.WriteTx(wtx); NotifyTransactionChanged(wtx.GetHash(), CT_UPDATED); @@ -1178,10 +1194,7 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c if (conflictconfirms < currentconfirm) { // Block is 'more conflicted' than current confirm; update. // Mark transaction as conflicted with this block. - wtx.m_confirm.nIndex = 0; - wtx.m_confirm.hashBlock = hashBlock; - wtx.m_confirm.block_height = conflicting_height; - wtx.setConflicted(); + wtx.m_state = TxStateConflicted{hashBlock, conflicting_height}; wtx.MarkDirty(); batch.WriteTx(wtx); // Iterate over all its outputs, and mark transactions in the wallet that spend them conflicted too @@ -1199,9 +1212,9 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c } } -void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Confirmation confirm, bool update_tx, bool rescanning_old_block) +void CWallet::SyncTransaction(const CTransactionRef& ptx, const SyncTxState& state, bool update_tx, bool rescanning_old_block) { - if (!AddToWalletIfInvolvingMe(ptx, confirm, update_tx, rescanning_old_block)) + if (!AddToWalletIfInvolvingMe(ptx, state, update_tx, rescanning_old_block)) return; // Not one of ours // If a transaction changes 'conflicted' state, that changes the balance @@ -1212,7 +1225,7 @@ void CWallet::SyncTransaction(const CTransactionRef& ptx, CWalletTx::Confirmatio void CWallet::transactionAddedToMempool(const CTransactionRef& tx, uint64_t mempool_sequence) { LOCK(cs_wallet); - SyncTransaction(tx, {CWalletTx::Status::UNCONFIRMED, /*block_height=*/0, /*block_hash=*/{}, /*block_index=*/0}); + SyncTransaction(tx, TxStateInMempool{}); auto it = mapWallet.find(tx->GetHash()); if (it != mapWallet.end()) { @@ -1253,7 +1266,7 @@ void CWallet::transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRe // distinguishing between conflicted and unconfirmed transactions are // imperfect, and could be improved in general, see // https://github.com/bitcoin-core/bitcoin-devwiki/wiki/Wallet-Transaction-Conflict-Tracking - SyncTransaction(tx, {CWalletTx::Status::UNCONFIRMED, /*block_height=*/0, /*block_hash=*/{}, /*block_index=*/0}); + SyncTransaction(tx, TxStateInactive{}); } } @@ -1265,7 +1278,7 @@ void CWallet::blockConnected(const CBlock& block, int height) m_last_block_processed_height = height; m_last_block_processed = block_hash; for (size_t index = 0; index < block.vtx.size(); index++) { - SyncTransaction(block.vtx[index], {CWalletTx::Status::CONFIRMED, height, block_hash, (int)index}); + SyncTransaction(block.vtx[index], TxStateConfirmed{block_hash, height, static_cast<int>(index)}); transactionRemovedFromMempool(block.vtx[index], MemPoolRemovalReason::BLOCK, 0 /* mempool_sequence */); } } @@ -1281,7 +1294,7 @@ void CWallet::blockDisconnected(const CBlock& block, int height) m_last_block_processed_height = height - 1; m_last_block_processed = block.hashPrevBlock; for (const CTransactionRef& ptx : block.vtx) { - SyncTransaction(ptx, {CWalletTx::Status::UNCONFIRMED, /*block_height=*/0, /*block_hash=*/{}, /*block_index=*/0}); + SyncTransaction(ptx, TxStateInactive{}); } } @@ -1290,7 +1303,6 @@ void CWallet::updatedBlockTip() m_best_block_time = GetTime(); } - void CWallet::BlockUntilSyncedToCurrentChain() const { AssertLockNotHeld(cs_wallet); // Skip the queue-draining stuff if we know we're caught up with @@ -1645,7 +1657,7 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc break; } for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { - SyncTransaction(block.vtx[posInBlock], {CWalletTx::Status::CONFIRMED, block_height, block_hash, (int)posInBlock}, fUpdate, /* rescanning_old_block */ true); + SyncTransaction(block.vtx[posInBlock], TxStateConfirmed{block_hash, block_height, static_cast<int>(posInBlock)}, fUpdate, /*rescanning_old_block=*/true); } // scan succeeded, record block as most recent successfully scanned result.last_scanned_block = block_hash; @@ -1720,7 +1732,7 @@ void CWallet::ReacceptWalletTransactions() } } -bool CWallet::SubmitTxMemoryPoolAndRelay(const CWalletTx& wtx, std::string& err_string, bool relay) const +bool CWallet::SubmitTxMemoryPoolAndRelay(CWalletTx& wtx, std::string& err_string, bool relay) const { // Can't relay if wallet is not broadcasting if (!GetBroadcastTransactions()) return false; @@ -1734,17 +1746,17 @@ bool CWallet::SubmitTxMemoryPoolAndRelay(const CWalletTx& wtx, std::string& err_ // Submit transaction to mempool for relay WalletLogPrintf("Submitting wtx %s to mempool for relay\n", wtx.GetHash().ToString()); - // We must set fInMempool here - while it will be re-set to true by the + // We must set TxStateInMempool here. Even though it will also be set later by the // entered-mempool callback, if we did not there would be a race where a // user could call sendmoney in a loop and hit spurious out of funds errors // because we think that this newly generated transaction's change is // unavailable as we're not yet aware that it is in the mempool. // - // Irrespective of the failure reason, un-marking fInMempool - // out-of-order is incorrect - it should be unmarked when + // If broadcast fails for any reason, trying to set wtx.m_state here would be incorrect. + // If transaction was previously in the mempool, it should be updated when // TransactionRemovedFromMempool fires. bool ret = chain().broadcastTransaction(wtx.tx, m_default_max_tx_fee, relay, err_string); - wtx.fInMempool |= ret; + if (ret) wtx.m_state = TxStateInMempool{}; return ret; } @@ -1831,7 +1843,8 @@ bool CWallet::SignTransaction(CMutableTransaction& tx) const return false; } const CWalletTx& wtx = mi->second; - coins[input.prevout] = Coin(wtx.tx->vout[input.prevout.n], wtx.m_confirm.block_height, wtx.IsCoinBase()); + int prev_height = wtx.state<TxStateConfirmed>() ? wtx.state<TxStateConfirmed>()->confirmed_block_height : 0; + coins[input.prevout] = Coin(wtx.tx->vout[input.prevout.n], prev_height, wtx.IsCoinBase()); } std::map<int, bilingual_str> input_errors; return SignTransaction(tx, coins, SIGHASH_DEFAULT, input_errors); @@ -1852,7 +1865,7 @@ bool CWallet::SignTransaction(CMutableTransaction& tx, const std::map<COutPoint, return false; } -TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& complete, int sighash_type, bool sign, bool bip32derivs, size_t * n_signed) const +TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& complete, int sighash_type, bool sign, bool bip32derivs, size_t * n_signed, bool finalize) const { if (n_signed) { *n_signed = 0; @@ -1884,7 +1897,7 @@ TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& comp // Fill in information from ScriptPubKeyMans for (ScriptPubKeyMan* spk_man : GetAllScriptPubKeyMans()) { int n_signed_this_spkm = 0; - TransactionError res = spk_man->FillPSBT(psbtx, txdata, sighash_type, sign, bip32derivs, &n_signed_this_spkm); + TransactionError res = spk_man->FillPSBT(psbtx, txdata, sighash_type, sign, bip32derivs, &n_signed_this_spkm, finalize); if (res != TransactionError::OK) { return res; } @@ -1956,7 +1969,7 @@ void CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::ve // Add tx to wallet, because if it has change it's also ours, // otherwise just for transaction history. - AddToWallet(tx, {}, [&](CWalletTx& wtx, bool new_tx) { + AddToWallet(tx, TxStateInactive{}, [&](CWalletTx& wtx, bool new_tx) { CHECK_NONFATAL(wtx.mapValue.empty()); CHECK_NONFATAL(wtx.vOrderForm.empty()); wtx.mapValue = std::move(mapValue); @@ -1974,7 +1987,7 @@ void CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::ve } // Get the inserted-CWalletTx from mapWallet so that the - // fInMempool flag is cached properly + // wtx cached mempool state is updated correctly CWalletTx& wtx = mapWallet.at(tx->GetHash()); if (!fBroadcastTransactions) { @@ -2169,14 +2182,18 @@ bool CWallet::GetNewChangeDestination(const OutputType type, CTxDestination& des return true; } -int64_t CWallet::GetOldestKeyPoolTime() const +std::optional<int64_t> CWallet::GetOldestKeyPoolTime() const { LOCK(cs_wallet); - int64_t oldestKey = std::numeric_limits<int64_t>::max(); + if (m_spk_managers.empty()) { + return std::nullopt; + } + + std::optional<int64_t> oldest_key{std::numeric_limits<int64_t>::max()}; for (const auto& spk_man_pair : m_spk_managers) { - oldestKey = std::min(oldestKey, spk_man_pair.second->GetOldestKeyPoolTime()); + oldest_key = std::min(oldest_key, spk_man_pair.second->GetOldestKeyPoolTime()); } - return oldestKey; + return oldest_key; } void CWallet::MarkDestinationsDirty(const std::set<CTxDestination>& destinations) { @@ -2252,16 +2269,15 @@ void ReserveDestination::ReturnDestination() bool CWallet::DisplayAddress(const CTxDestination& dest) { CScript scriptPubKey = GetScriptForDestination(dest); - const auto spk_man = GetScriptPubKeyMan(scriptPubKey); - if (spk_man == nullptr) { - return false; - } - auto signer_spk_man = dynamic_cast<ExternalSignerScriptPubKeyMan*>(spk_man); - if (signer_spk_man == nullptr) { - return false; + for (const auto& spk_man : GetScriptPubKeyMans(scriptPubKey)) { + auto signer_spk_man = dynamic_cast<ExternalSignerScriptPubKeyMan *>(spk_man); + if (signer_spk_man == nullptr) { + continue; + } + ExternalSigner signer = ExternalSignerScriptPubKeyMan::GetExternalSigner(); + return signer_spk_man->DisplayAddress(scriptPubKey, signer); } - ExternalSigner signer = ExternalSignerScriptPubKeyMan::GetExternalSigner(); - return signer_spk_man->DisplayAddress(scriptPubKey, signer); + return false; } bool CWallet::LockCoin(const COutPoint& output, WalletBatch* batch) @@ -2321,10 +2337,10 @@ void CWallet::GetKeyBirthTimes(std::map<CKeyID, int64_t>& mapKeyBirth) const { mapKeyBirth.clear(); // map in which we'll infer heights of other keys - std::map<CKeyID, const CWalletTx::Confirmation*> mapKeyFirstBlock; - CWalletTx::Confirmation max_confirm; - max_confirm.block_height = GetLastBlockHeight() > 144 ? GetLastBlockHeight() - 144 : 0; // the tip can be reorganized; use a 144-block safety margin - CHECK_NONFATAL(chain().findAncestorByHeight(GetLastBlockHash(), max_confirm.block_height, FoundBlock().hash(max_confirm.hashBlock))); + std::map<CKeyID, const TxStateConfirmed*> mapKeyFirstBlock; + TxStateConfirmed max_confirm{uint256{}, /*height=*/-1, /*index=*/-1}; + max_confirm.confirmed_block_height = GetLastBlockHeight() > 144 ? GetLastBlockHeight() - 144 : 0; // the tip can be reorganized; use a 144-block safety margin + CHECK_NONFATAL(chain().findAncestorByHeight(GetLastBlockHash(), max_confirm.confirmed_block_height, FoundBlock().hash(max_confirm.confirmed_block_hash))); { LegacyScriptPubKeyMan* spk_man = GetLegacyScriptPubKeyMan(); @@ -2352,15 +2368,15 @@ void CWallet::GetKeyBirthTimes(std::map<CKeyID, int64_t>& mapKeyBirth) const { for (const auto& entry : mapWallet) { // iterate over all wallet transactions... const CWalletTx &wtx = entry.second; - if (wtx.m_confirm.status == CWalletTx::CONFIRMED) { + if (auto* conf = wtx.state<TxStateConfirmed>()) { // ... which are already in a block for (const CTxOut &txout : wtx.tx->vout) { // iterate over all their outputs for (const auto &keyid : GetAffectedKeys(txout.scriptPubKey, *spk_man)) { // ... and all their affected keys auto rit = mapKeyFirstBlock.find(keyid); - if (rit != mapKeyFirstBlock.end() && wtx.m_confirm.block_height < rit->second->block_height) { - rit->second = &wtx.m_confirm; + if (rit != mapKeyFirstBlock.end() && conf->confirmed_block_height < rit->second->confirmed_block_height) { + rit->second = conf; } } } @@ -2371,7 +2387,7 @@ void CWallet::GetKeyBirthTimes(std::map<CKeyID, int64_t>& mapKeyBirth) const { // Extract block timestamps for those keys for (const auto& entry : mapKeyFirstBlock) { int64_t block_time; - CHECK_NONFATAL(chain().findBlock(entry.second->hashBlock, FoundBlock().time(block_time))); + CHECK_NONFATAL(chain().findBlock(entry.second->confirmed_block_hash, FoundBlock().time(block_time))); mapKeyBirth[entry.first] = block_time - TIMESTAMP_WINDOW; // block times can be 2h off } } @@ -2401,11 +2417,18 @@ void CWallet::GetKeyBirthTimes(std::map<CKeyID, int64_t>& mapKeyBirth) const { */ unsigned int CWallet::ComputeTimeSmart(const CWalletTx& wtx, bool rescanning_old_block) const { + std::optional<uint256> block_hash; + if (auto* conf = wtx.state<TxStateConfirmed>()) { + block_hash = conf->confirmed_block_hash; + } else if (auto* conf = wtx.state<TxStateConflicted>()) { + block_hash = conf->conflicting_block_hash; + } + unsigned int nTimeSmart = wtx.nTimeReceived; - if (!wtx.isUnconfirmed() && !wtx.isAbandoned()) { + if (block_hash) { int64_t blocktime; int64_t block_max_time; - if (chain().findBlock(wtx.m_confirm.hashBlock, FoundBlock().time(blocktime).maxTime(block_max_time))) { + if (chain().findBlock(*block_hash, FoundBlock().time(blocktime).maxTime(block_max_time))) { if (rescanning_old_block) { nTimeSmart = block_max_time; } else { @@ -2437,7 +2460,7 @@ unsigned int CWallet::ComputeTimeSmart(const CWalletTx& wtx, bool rescanning_old nTimeSmart = std::max(latestEntry, std::min(blocktime, latestNow)); } } else { - WalletLogPrintf("%s: found %s in block %s not in index\n", __func__, wtx.GetHash().ToString(), wtx.m_confirm.hashBlock.ToString()); + WalletLogPrintf("%s: found %s in block %s not in index\n", __func__, wtx.GetHash().ToString(), block_hash->ToString()); } } return nTimeSmart; @@ -2540,7 +2563,7 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri int64_t nStart = GetTimeMillis(); // TODO: Can't use std::make_shared because we need a custom deleter but // should be possible to use std::allocate_shared. - const std::shared_ptr<CWallet> walletInstance(new CWallet(chain, name, std::move(database)), ReleaseWallet); + const std::shared_ptr<CWallet> walletInstance(new CWallet(chain, name, args, std::move(database)), ReleaseWallet); bool rescan_required = false; DBErrors nLoadWalletRet = walletInstance->LoadWallet(); if (nLoadWalletRet != DBErrors::LOAD_OK) { @@ -2732,11 +2755,11 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri walletInstance->m_default_max_tx_fee = max_fee.value(); } - if (gArgs.IsArgSet("-consolidatefeerate")) { - if (std::optional<CAmount> consolidate_feerate = ParseMoney(gArgs.GetArg("-consolidatefeerate", ""))) { + if (args.IsArgSet("-consolidatefeerate")) { + if (std::optional<CAmount> consolidate_feerate = ParseMoney(args.GetArg("-consolidatefeerate", ""))) { walletInstance->m_consolidate_feerate = CFeeRate(*consolidate_feerate); } else { - error = AmountErrMsg("consolidatefeerate", gArgs.GetArg("-consolidatefeerate", "")); + error = AmountErrMsg("consolidatefeerate", args.GetArg("-consolidatefeerate", "")); return nullptr; } } @@ -2755,8 +2778,6 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri // Try to top up keypool. No-op if the wallet is locked. walletInstance->TopUpKeyPool(); - LOCK(walletInstance->cs_wallet); - if (chain && !AttachChain(walletInstance, *chain, rescan_required, error, warnings)) { return nullptr; } @@ -2768,9 +2789,9 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri } } - walletInstance->SetBroadcastTransactions(args.GetBoolArg("-walletbroadcast", DEFAULT_WALLETBROADCAST)); - { + LOCK(walletInstance->cs_wallet); + walletInstance->SetBroadcastTransactions(args.GetBoolArg("-walletbroadcast", DEFAULT_WALLETBROADCAST)); walletInstance->WalletLogPrintf("setKeyPool.size() = %u\n", walletInstance->GetKeyPoolSize()); walletInstance->WalletLogPrintf("mapWallet.size() = %u\n", walletInstance->mapWallet.size()); walletInstance->WalletLogPrintf("m_address_book.size() = %u\n", walletInstance->m_address_book.size()); @@ -2944,9 +2965,13 @@ CKeyPool::CKeyPool(const CPubKey& vchPubKeyIn, bool internalIn) int CWallet::GetTxDepthInMainChain(const CWalletTx& wtx) const { AssertLockHeld(cs_wallet); - if (wtx.isUnconfirmed() || wtx.isAbandoned()) return 0; - - return (GetLastBlockHeight() - wtx.m_confirm.block_height + 1) * (wtx.isConflicted() ? -1 : 1); + if (auto* conf = wtx.state<TxStateConfirmed>()) { + return GetLastBlockHeight() - conf->confirmed_block_height + 1; + } else if (auto* conf = wtx.state<TxStateConflicted>()) { + return -1 * (GetLastBlockHeight() - conf->conflicting_block_height + 1); + } else { + return 0; + } } int CWallet::GetTxBlocksToMaturity(const CWalletTx& wtx) const @@ -3040,9 +3065,10 @@ ScriptPubKeyMan* CWallet::GetScriptPubKeyMan(const OutputType& type, bool intern return it->second; } -std::set<ScriptPubKeyMan*> CWallet::GetScriptPubKeyMans(const CScript& script, SignatureData& sigdata) const +std::set<ScriptPubKeyMan*> CWallet::GetScriptPubKeyMans(const CScript& script) const { std::set<ScriptPubKeyMan*> spk_mans; + SignatureData sigdata; for (const auto& spk_man_pair : m_spk_managers) { if (spk_man_pair.second->CanProvide(script, sigdata)) { spk_mans.insert(spk_man_pair.second.get()); @@ -3051,17 +3077,6 @@ std::set<ScriptPubKeyMan*> CWallet::GetScriptPubKeyMans(const CScript& script, S return spk_mans; } -ScriptPubKeyMan* CWallet::GetScriptPubKeyMan(const CScript& script) const -{ - SignatureData sigdata; - for (const auto& spk_man_pair : m_spk_managers) { - if (spk_man_pair.second->CanProvide(script, sigdata)) { - return spk_man_pair.second.get(); - } - } - return nullptr; -} - ScriptPubKeyMan* CWallet::GetScriptPubKeyMan(const uint256& id) const { if (m_spk_managers.count(id) > 0) { @@ -3160,15 +3175,10 @@ void CWallet::SetupDescriptorScriptPubKeyMans() // Get the extended key CExtKey master_key; - master_key.SetSeed(seed_key.begin(), seed_key.size()); + master_key.SetSeed(seed_key); for (bool internal : {false, true}) { for (OutputType t : OUTPUT_TYPES) { - if (t == OutputType::BECH32M) { - // Skip taproot (bech32m) for now - // TODO: Setup taproot (bech32m) descriptors by default - continue; - } auto spk_manager = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this)); if (IsCrypted()) { if (IsLocked()) { @@ -3282,6 +3292,30 @@ DescriptorScriptPubKeyMan* CWallet::GetDescriptorScriptPubKeyMan(const WalletDes return nullptr; } +std::optional<bool> CWallet::IsInternalScriptPubKeyMan(ScriptPubKeyMan* spk_man) const +{ + // Legacy script pubkey man can't be either external or internal + if (IsLegacy()) { + return std::nullopt; + } + + // only active ScriptPubKeyMan can be internal + if (!GetActiveScriptPubKeyMans().count(spk_man)) { + return std::nullopt; + } + + const auto desc_spk_man = dynamic_cast<DescriptorScriptPubKeyMan*>(spk_man); + if (!desc_spk_man) { + throw std::runtime_error(std::string(__func__) + ": unexpected ScriptPubKeyMan type."); + } + + LOCK(desc_spk_man->cs_desc_man); + const auto& type = desc_spk_man->GetWalletDescriptor().descriptor->GetOutputType(); + assert(type.has_value()); + + return GetScriptPubKeyMan(*type, /* internal= */ true) == desc_spk_man; +} + ScriptPubKeyMan* CWallet::AddWalletDescriptor(WalletDescriptor& desc, const FlatSigningProvider& signing_provider, const std::string& label, bool internal) { AssertLockHeld(cs_wallet); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index c911eb461c..dbf0f6375d 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -275,7 +275,7 @@ private: * Should be called with rescanning_old_block set to true, if the transaction is * not discovered in real time, but during a rescan of old blocks. */ - bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool fUpdate, bool rescanning_old_block) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, const SyncTxState& state, bool fUpdate, bool rescanning_old_block) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** Mark a transaction (and its in-wallet descendants) as conflicting with a particular block. */ void MarkConflicted(const uint256& hashBlock, int conflicting_height, const uint256& hashTx); @@ -285,7 +285,7 @@ private: void SyncMetaData(std::pair<TxSpends::iterator, TxSpends::iterator>) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - void SyncTransaction(const CTransactionRef& tx, CWalletTx::Confirmation confirm, bool update_tx = true, bool rescanning_old_block = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + void SyncTransaction(const CTransactionRef& tx, const SyncTxState& state, bool update_tx = true, bool rescanning_old_block = false) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** WalletFlags set on this wallet. */ std::atomic<uint64_t> m_wallet_flags{0}; @@ -298,6 +298,9 @@ private: //! Unset the blank wallet flag and saves it to disk void UnsetBlankWalletFlag(WalletBatch& batch) override; + /** Provider of aplication-wide arguments. */ + const ArgsManager& m_args; + /** Interface for accessing chain state. */ interfaces::Chain* m_chain; @@ -359,8 +362,9 @@ public: unsigned int nMasterKeyMaxID = 0; /** Construct wallet with specified name and database implementation. */ - CWallet(interfaces::Chain* chain, const std::string& name, std::unique_ptr<WalletDatabase> database) - : m_chain(chain), + CWallet(interfaces::Chain* chain, const std::string& name, const ArgsManager& args, std::unique_ptr<WalletDatabase> database) + : m_args(args), + m_chain(chain), m_name(name), m_database(std::move(database)) { @@ -504,7 +508,7 @@ public: //! @return true if wtx is changed and needs to be saved to disk, otherwise false using UpdateWalletTxFn = std::function<bool(CWalletTx& wtx, bool new_tx)>; - CWalletTx* AddToWallet(CTransactionRef tx, const CWalletTx::Confirmation& confirm, const UpdateWalletTxFn& update_wtx=nullptr, bool fFlushOnClose=true, bool rescanning_old_block = false); + CWalletTx* AddToWallet(CTransactionRef tx, const TxState& state, const UpdateWalletTxFn& update_wtx=nullptr, bool fFlushOnClose=true, bool rescanning_old_block = false); bool LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void transactionAddedToMempool(const CTransactionRef& tx, uint64_t mempool_sequence) override; void blockConnected(const CBlock& block, int height) override; @@ -551,6 +555,8 @@ public: * @param[in] sighash_type the sighash type to use when signing (if PSBT does not specify) * @param[in] sign whether to sign or not * @param[in] bip32derivs whether to fill in bip32 derivation information if available + * @param[out] n_signed the number of inputs signed by this wallet + * @param[in] finalize whether to create the final scriptSig or scriptWitness if possible * return error */ TransactionError FillPSBT(PartiallySignedTransaction& psbtx, @@ -558,7 +564,8 @@ public: int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = true, - size_t* n_signed = nullptr) const; + size_t* n_signed = nullptr, + bool finalize = true) const; /** * Submit the transaction to the node's mempool and then relay to peers. @@ -572,7 +579,7 @@ public: void CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm); /** Pass this transaction to node for mempool insertion and relay to peers if flag set to true */ - bool SubmitTxMemoryPoolAndRelay(const CWalletTx& wtx, std::string& err_string, bool relay) const; + bool SubmitTxMemoryPoolAndRelay(CWalletTx& wtx, std::string& err_string, bool relay) const; bool DummySignTx(CMutableTransaction &txNew, const std::set<CTxOut> &txouts, const CCoinControl* coin_control = nullptr) const { @@ -628,7 +635,7 @@ public: size_t KeypoolCountExternalKeys() const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool TopUpKeyPool(unsigned int kpSize = 0); - int64_t GetOldestKeyPoolTime() const; + std::optional<int64_t> GetOldestKeyPoolTime() const; std::set<CTxDestination> GetLabelAddresses(const std::string& label) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -802,14 +809,11 @@ public: //! Get the ScriptPubKeyMan for the given OutputType and internal/external chain. ScriptPubKeyMan* GetScriptPubKeyMan(const OutputType& type, bool internal) const; - //! Get the ScriptPubKeyMan for a script - ScriptPubKeyMan* GetScriptPubKeyMan(const CScript& script) const; + //! Get all the ScriptPubKeyMans for a script + std::set<ScriptPubKeyMan*> GetScriptPubKeyMans(const CScript& script) const; //! Get the ScriptPubKeyMan by id ScriptPubKeyMan* GetScriptPubKeyMan(const uint256& id) const; - //! Get all of the ScriptPubKeyMans for a script given additional information in sigdata (populated by e.g. a psbt) - std::set<ScriptPubKeyMan*> GetScriptPubKeyMans(const CScript& script, SignatureData& sigdata) const; - //! Get the SigningProvider for a script std::unique_ptr<SigningProvider> GetSolvingProvider(const CScript& script) const; std::unique_ptr<SigningProvider> GetSolvingProvider(const CScript& script, SignatureData& sigdata) const; @@ -875,6 +879,11 @@ public: //! Return the DescriptorScriptPubKeyMan for a WalletDescriptor if it is already in the wallet DescriptorScriptPubKeyMan* GetDescriptorScriptPubKeyMan(const WalletDescriptor& desc) const; + //! Returns whether the provided ScriptPubKeyMan is internal + //! @param[in] spk_man The ScriptPubKeyMan to test + //! @return contains value only for active DescriptorScriptPubKeyMan, otherwise undefined + std::optional<bool> IsInternalScriptPubKeyMan(ScriptPubKeyMan* spk_man) const; + //! Add a descriptor to the wallet, return a ScriptPubKeyMan & associated output type ScriptPubKeyMan* AddWalletDescriptor(WalletDescriptor& desc, const FlatSigningProvider& signing_provider, const std::string& label, bool internal) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); }; diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index c920d4af51..f392649bd9 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -987,7 +987,7 @@ DBErrors WalletBatch::FindWalletTx(std::vector<uint256>& vTxHash, std::list<CWal uint256 hash; ssKey >> hash; vTxHash.push_back(hash); - vWtx.emplace_back(nullptr /* tx */); + vWtx.emplace_back(/*tx=*/nullptr, TxStateInactive{}); ssValue >> vWtx.back(); } } diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 086415f152..d6717ebbca 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -26,7 +26,7 @@ static void WalletCreate(CWallet* wallet_instance, uint64_t wallet_creation_flag { LOCK(wallet_instance->cs_wallet); - wallet_instance->SetMinVersion(FEATURE_HD_SPLIT); + wallet_instance->SetMinVersion(FEATURE_LATEST); wallet_instance->AddWalletFlags(wallet_creation_flags); if (!wallet_instance->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { @@ -40,7 +40,7 @@ static void WalletCreate(CWallet* wallet_instance, uint64_t wallet_creation_flag wallet_instance->TopUpKeyPool(); } -static const std::shared_ptr<CWallet> MakeWallet(const std::string& name, const fs::path& path, DatabaseOptions options) +static const std::shared_ptr<CWallet> MakeWallet(const std::string& name, const fs::path& path, const ArgsManager& args, DatabaseOptions options) { DatabaseStatus status; bilingual_str error; @@ -51,7 +51,7 @@ static const std::shared_ptr<CWallet> MakeWallet(const std::string& name, const } // dummy chain interface - std::shared_ptr<CWallet> wallet_instance{new CWallet(nullptr /* chain */, name, std::move(database)), WalletToolReleaseWallet}; + std::shared_ptr<CWallet> wallet_instance{new CWallet(nullptr /* chain */, name, args, std::move(database)), WalletToolReleaseWallet}; DBErrors load_wallet_ret; try { load_wallet_ret = wallet_instance->LoadWallet(); @@ -151,7 +151,7 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) options.require_format = DatabaseFormat::SQLITE; } - const std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, options); + const std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, args, options); if (wallet_instance) { WalletShowInfo(wallet_instance.get()); wallet_instance->Close(); @@ -159,7 +159,7 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) } else if (command == "info") { DatabaseOptions options; options.require_existing = true; - const std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, options); + const std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, args, options); if (!wallet_instance) return false; WalletShowInfo(wallet_instance.get()); wallet_instance->Close(); @@ -184,7 +184,7 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) } else if (command == "dump") { DatabaseOptions options; options.require_existing = true; - const std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, options); + const std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, args, options); if (!wallet_instance) return false; bilingual_str error; bool ret = DumpWallet(*wallet_instance, error); diff --git a/src/warnings.h b/src/warnings.h index c38edb4570..7ab0a93e3f 100644 --- a/src/warnings.h +++ b/src/warnings.h @@ -20,4 +20,4 @@ void SetfLargeWorkInvalidChainFound(bool flag); */ bilingual_str GetWarnings(bool verbose); -#endif // BITCOIN_WARNINGS_H +#endif // BITCOIN_WARNINGS_H diff --git a/src/zmq/zmqrpc.h b/src/zmq/zmqrpc.h index 5a810a16fb..8538adf9d3 100644 --- a/src/zmq/zmqrpc.h +++ b/src/zmq/zmqrpc.h @@ -9,4 +9,4 @@ class CRPCTable; void RegisterZMQRPCCommands(CRPCTable& t); -#endif // BITCOIN_ZMQ_ZMRRPC_H +#endif // BITCOIN_ZMQ_ZMQRPC_H diff --git a/test/functional/combine_logs.py b/test/functional/combine_logs.py index 71dfb4c01a..33c81bde13 100755 --- a/test/functional/combine_logs.py +++ b/test/functional/combine_logs.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Combine logs from multiple bitcoin nodes as well as the test_framework log. diff --git a/test/functional/data/invalid_txs.py b/test/functional/data/invalid_txs.py index cde0399d8b..33d6282961 100644 --- a/test/functional/data/invalid_txs.py +++ b/test/functional/data/invalid_txs.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ diff --git a/test/functional/data/rpc_decodescript.json b/test/functional/data/rpc_decodescript.json new file mode 100644 index 0000000000..d1aa9ab00d --- /dev/null +++ b/test/functional/data/rpc_decodescript.json @@ -0,0 +1,85 @@ +[ + [ + "5120eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + { + "asm": "1 eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "address": "bcrt1pamhwamhwamhwamhwamhwamhwamhwamhwamhwamhwamhwamhwamhqz6nvlh", + "type": "witness_v1_taproot" + } + ], + [ + "5102eeee", + { + "asm": "1 -28398", + "address": "bcrt1pamhqk96edn", + "type": "witness_unknown" + } + ], + [ + "0020eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + { + "asm": "0 eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "address": "bcrt1qamhwamhwamhwamhwamhwamhwamhwamhwamhwamhwamhwamhwamhqgdn98t", + "type": "witness_v0_scripthash", + "p2sh": "2MwGk8mw1GBP6U9D5X8gTvgvXpuknmAK3fo" + } + ], + [ + "a914eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee87", + { + "asm": "OP_HASH160 eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee OP_EQUAL", + "address": "2NF2b3KS8xXb9XHvbRMXdZh8s5g92rUZHtp", + "type": "scripthash" + } + ], + [ + "6a00", + { + "asm": "OP_RETURN 0", + "type": "nulldata" + } + ], + [ + "6aee", + { + "asm": "OP_RETURN OP_UNKNOWN", + "type": "nonstandard" + } + ], + [ + "6a02ee", + { + "asm": "OP_RETURN [error]", + "type": "nonstandard" + } + ], + [ + "02eeee", + { + "asm": "-28398", + "type": "nonstandard", + "p2sh": "2N34iiGoUUkVSPiaaTFpJjB1FR9TXQu3PGM", + "segwit": { + "asm": "0 96c2368fc30514a438a8bd909f93c49a1549d77198ccbdb792043b666cb24f42", + "hex": "002096c2368fc30514a438a8bd909f93c49a1549d77198ccbdb792043b666cb24f42", + "address": "bcrt1qjmprdr7rq522gw9ghkgfly7yng25n4m3nrxtmdujqsakvm9jfapqk795l5", + "type": "witness_v0_scripthash", + "p2sh-segwit": "2MtoejEictTQ6XtmHYzoYttt35Ec6krqFKN" + } + } + ], + [ + "ba", + { + "asm": "OP_CHECKSIGADD", + "type": "nonstandard" + } + ], + [ + "50", + { + "asm": "OP_RESERVED", + "type": "nonstandard" + } + ] +] diff --git a/test/functional/example_test.py b/test/functional/example_test.py index d6fc2d580f..2ad96da854 100755 --- a/test/functional/example_test.py +++ b/test/functional/example_test.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """An example functional test @@ -141,8 +141,7 @@ class ExampleTest(BitcoinTestFramework): peer_messaging = self.nodes[0].add_p2p_connection(BaseNode()) # Generating a block on one of the nodes will get us out of IBD - blocks = [int(self.generate(self.nodes[0], nblocks=1)[0], 16)] - self.sync_all(self.nodes[0:2]) + blocks = [int(self.generate(self.nodes[0], sync_fun=lambda: self.sync_all(self.nodes[0:2]), nblocks=1)[0], 16)] # Notice above how we called an RPC by calling a method with the same # name on the node object. Notice also how we used a keyword argument diff --git a/test/functional/feature_abortnode.py b/test/functional/feature_abortnode.py index e3cb7725bd..32cf4a47f4 100755 --- a/test/functional/feature_abortnode.py +++ b/test/functional/feature_abortnode.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoind aborts if can't disconnect a block. @@ -26,7 +26,7 @@ class AbortNodeTest(BitcoinTestFramework): # We'll connect the nodes later def run_test(self): - self.generate(self.nodes[0], 3) + self.generate(self.nodes[0], 3, sync_fun=self.no_op) datadir = get_datadir_path(self.options.tmpdir, 0) # Deleting the undo file will result in reorg failure @@ -34,10 +34,10 @@ class AbortNodeTest(BitcoinTestFramework): # Connecting to a node with a more work chain will trigger a reorg # attempt. - self.generate(self.nodes[1], 3) + self.generate(self.nodes[1], 3, sync_fun=self.no_op) with self.nodes[0].assert_debug_log(["Failed to disconnect block"]): self.connect_nodes(0, 1) - self.generate(self.nodes[1], 1) + self.generate(self.nodes[1], 1, sync_fun=self.no_op) # Check that node0 aborted self.log.info("Waiting for crash") diff --git a/test/functional/feature_anchors.py b/test/functional/feature_anchors.py index 7be393a4ea..713c0826d3 100755 --- a/test/functional/feature_anchors.py +++ b/test/functional/feature_anchors.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test block-relay-only anchors functionality""" @@ -8,18 +8,12 @@ import os from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal +from test_framework.util import check_node_connections INBOUND_CONNECTIONS = 5 BLOCK_RELAY_CONNECTIONS = 2 -def check_node_connections(*, node, num_in, num_out): - info = node.getnetworkinfo() - assert_equal(info["connections_in"], num_in) - assert_equal(info["connections_out"], num_out) - - class AnchorsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 diff --git a/test/functional/feature_asmap.py b/test/functional/feature_asmap.py index debd87962f..9440ba11f5 100755 --- a/test/functional/feature_asmap.py +++ b/test/functional/feature_asmap.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test asmap config argument for ASN-based IP bucketing. @@ -89,8 +89,8 @@ class AsmapTest(BitcoinTestFramework): self.restart_node(0, ["-asmap", "-checkaddrman=1"]) with self.node.assert_debug_log( expected_msgs=[ - "Addrman checks started: new 1, tried 1, total 2", - "Addrman checks completed successfully", + "CheckAddrman: new 1, tried 1, total 2 started", + "CheckAddrman: completed", ] ): self.node.getnodeaddresses() # getnodeaddresses re-runs the addrman checks diff --git a/test/functional/feature_assumevalid.py b/test/functional/feature_assumevalid.py index a4480307a7..67cacaa9ce 100755 --- a/test/functional/feature_assumevalid.py +++ b/test/functional/feature_assumevalid.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test logic for skipping signature validation on old blocks. @@ -122,11 +122,8 @@ class AssumeValidTest(BitcoinTestFramework): tx.vout.append(CTxOut(49 * 100000000, CScript([OP_TRUE]))) tx.calc_sha256() - block102 = create_block(self.tip, create_coinbase(height), self.block_time) + block102 = create_block(self.tip, create_coinbase(height), self.block_time, txlist=[tx]) self.block_time += 1 - block102.vtx.extend([tx]) - block102.hashMerkleRoot = block102.calc_merkle_root() - block102.rehash() block102.solve() self.blocks.append(block102) self.tip = block102.sha256 @@ -136,7 +133,6 @@ class AssumeValidTest(BitcoinTestFramework): # Bury the assumed valid block 2100 deep for _ in range(2100): block = create_block(self.tip, create_coinbase(height), self.block_time) - block.nVersion = 4 block.solve() self.blocks.append(block) self.tip = block.sha256 diff --git a/test/functional/feature_backwards_compatibility.py b/test/functional/feature_backwards_compatibility.py index e65525a023..476a6a0c14 100755 --- a/test/functional/feature_backwards_compatibility.py +++ b/test/functional/feature_backwards_compatibility.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Backwards compatibility functional test @@ -66,8 +66,6 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): def run_test(self): self.generatetoaddress(self.nodes[0], COINBASE_MATURITY + 1, self.nodes[0].getnewaddress()) - self.sync_blocks() - # Sanity check the test framework: res = self.nodes[self.num_nodes - 1].getblockchaininfo() assert_equal(res['blocks'], COINBASE_MATURITY + 1) @@ -93,7 +91,6 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): self.nodes[0].sendtoaddress(address, 10) self.sync_mempools() self.generate(self.nodes[0], 1) - self.sync_blocks() # Create a conflicting transaction using RBF return_address = self.nodes[0].getnewaddress() tx1_id = self.nodes[1].sendtoaddress(return_address, 1) @@ -101,7 +98,6 @@ class BackwardsCompatibilityTest(BitcoinTestFramework): # Confirm the transaction self.sync_mempools() self.generate(self.nodes[0], 1) - self.sync_blocks() # Create another conflicting transaction using RBF tx3_id = self.nodes[1].sendtoaddress(return_address, 1) tx4_id = self.nodes[1].bumpfee(tx3_id)["txid"] diff --git a/test/functional/feature_bind_extra.py b/test/functional/feature_bind_extra.py index 6802da8d48..af26f94033 100755 --- a/test/functional/feature_bind_extra.py +++ b/test/functional/feature_bind_extra.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2021 The Bitcoin Core developers +# Copyright (c) 2014-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index 99ac1b5884..05d274a9fe 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP68 implementation.""" @@ -275,7 +275,7 @@ class BIP68Test(BitcoinTestFramework): cur_time = int(time.time()) for _ in range(10): self.nodes[0].setmocktime(cur_time + 600) - self.generate(self.nodes[0], 1) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) cur_time += 600 assert tx2.hash in self.nodes[0].getrawmempool() @@ -335,7 +335,6 @@ class BIP68Test(BitcoinTestFramework): # tx3 to be removed. for i in range(2): block = create_block(tmpl=tmpl, ntime=cur_time) - block.rehash() block.solve() tip = block.sha256 assert_equal(None if i == 1 else 'inconclusive', self.nodes[0].submitblock(block.serialize().hex())) @@ -351,7 +350,7 @@ class BIP68Test(BitcoinTestFramework): # Reset the chain and get rid of the mocktimed-blocks self.nodes[0].setmocktime(0) self.nodes[0].invalidateblock(self.nodes[0].getblockhash(cur_height+1)) - self.generate(self.nodes[0], 10) + self.generate(self.nodes[0], 10, sync_fun=self.no_op) # Make sure that BIP68 isn't being used to validate blocks prior to # activation height. If more blocks are mined prior to this test @@ -389,10 +388,7 @@ class BIP68Test(BitcoinTestFramework): assert_raises_rpc_error(-26, NOT_FINAL_ERROR, self.nodes[0].sendrawtransaction, tx3.serialize().hex()) # make a block that violates bip68; ensure that the tip updates - block = create_block(tmpl=self.nodes[0].getblocktemplate(NORMAL_GBT_REQUEST_PARAMS)) - block.vtx.extend([tx1, tx2, tx3]) - block.hashMerkleRoot = block.calc_merkle_root() - block.rehash() + block = create_block(tmpl=self.nodes[0].getblocktemplate(NORMAL_GBT_REQUEST_PARAMS), txlist=[tx1, tx2, tx3]) add_witness_commitment(block) block.solve() @@ -405,9 +401,9 @@ class BIP68Test(BitcoinTestFramework): min_activation_height = 432 height = self.nodes[0].getblockcount() assert_greater_than(min_activation_height - height, 2) - self.generate(self.nodes[0], min_activation_height - height - 2) + self.generate(self.nodes[0], min_activation_height - height - 2, sync_fun=self.no_op) assert not softfork_active(self.nodes[0], 'csv') - self.generate(self.nodes[0], 1) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) assert softfork_active(self.nodes[0], 'csv') self.sync_blocks() diff --git a/test/functional/feature_block.py b/test/functional/feature_block.py index b06ea8542b..a3253763bd 100755 --- a/test/functional/feature_block.py +++ b/test/functional/feature_block.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test block processing.""" @@ -612,7 +612,6 @@ class FullBlockTest(BitcoinTestFramework): b45.nBits = 0x207fffff b45.vtx.append(non_coinbase) b45.hashMerkleRoot = b45.calc_merkle_root() - b45.calc_sha256() b45.solve() self.block_heights[b45.sha256] = self.block_heights[self.tip.sha256] + 1 self.tip = b45 @@ -1362,11 +1361,10 @@ class FullBlockTest(BitcoinTestFramework): else: coinbase.vout[0].nValue += spend.vout[0].nValue - 1 # all but one satoshi to fees coinbase.rehash() - block = create_block(base_block_hash, coinbase, block_time, version=version) tx = self.create_tx(spend, 0, 1, script) # spend 1 satoshi self.sign_tx(tx, spend) - self.add_transactions_to_block(block, [tx]) - block.hashMerkleRoot = block.calc_merkle_root() + tx.rehash() + block = create_block(base_block_hash, coinbase, block_time, version=version, txlist=[tx]) # Block is created. Find a valid nonce. block.solve() self.tip = block diff --git a/test/functional/feature_blockfilterindex_prune.py b/test/functional/feature_blockfilterindex_prune.py index b740f2cc27..39eb700b4f 100755 --- a/test/functional/feature_blockfilterindex_prune.py +++ b/test/functional/feature_blockfilterindex_prune.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test blockfilterindex in conjunction with prune.""" @@ -26,9 +26,7 @@ class FeatureBlockfilterindexPruneTest(BitcoinTestFramework): assert_greater_than(len(self.nodes[0].getblockfilter(self.nodes[0].getbestblockhash())['filter']), 0) # Mine two batches of blocks to avoid hitting NODE_NETWORK_LIMITED_MIN_BLOCKS disconnection self.generate(self.nodes[0], 250) - self.sync_all() self.generate(self.nodes[0], 250) - self.sync_all() self.sync_index(height=700) self.log.info("prune some blocks") diff --git a/test/functional/feature_blocksdir.py b/test/functional/feature_blocksdir.py index 28e6d6cdf9..e8d2ec3676 100755 --- a/test/functional/feature_blocksdir.py +++ b/test/functional/feature_blocksdir.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the blocksdir option. diff --git a/test/functional/feature_cltv.py b/test/functional/feature_cltv.py index 3dc858f5d2..eb90b2c598 100755 --- a/test/functional/feature_cltv.py +++ b/test/functional/feature_cltv.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP65 (CHECKLOCKTIMEVERIFY). @@ -120,10 +120,7 @@ class BIP65Test(BitcoinTestFramework): tip = self.nodes[0].getbestblockhash() block_time = self.nodes[0].getblockheader(tip)['mediantime'] + 1 - block = create_block(int(tip, 16), create_coinbase(CLTV_HEIGHT - 1), block_time) - block.nVersion = 3 - block.vtx.extend(invalid_cltv_txs) - block.hashMerkleRoot = block.calc_merkle_root() + block = create_block(int(tip, 16), create_coinbase(CLTV_HEIGHT - 1), block_time, version=3, txlist=invalid_cltv_txs) block.solve() self.test_cltv_info(is_active=False) # Not active as of current tip and next block does not need to obey rules @@ -134,8 +131,7 @@ class BIP65Test(BitcoinTestFramework): self.log.info("Test that blocks must now be at least version 4") tip = block.sha256 block_time += 1 - block = create_block(tip, create_coinbase(CLTV_HEIGHT), block_time) - block.nVersion = 3 + block = create_block(tip, create_coinbase(CLTV_HEIGHT), block_time, version=3) block.solve() with self.nodes[0].assert_debug_log(expected_msgs=[f'{block.hash}, bad-version(0x00000003)']): diff --git a/test/functional/feature_coinstatsindex.py b/test/functional/feature_coinstatsindex.py index c592d7bd69..8afd0ff74a 100755 --- a/test/functional/feature_coinstatsindex.py +++ b/test/functional/feature_coinstatsindex.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test coinstatsindex across nodes. @@ -72,8 +72,6 @@ class CoinStatsIndexTest(BitcoinTestFramework): node.sendtoaddress(address=address, amount=10, subtractfeefromamount=True) self.generate(node, 1) - self.sync_blocks(timeout=120) - self.log.info("Test that gettxoutsetinfo() output is consistent with or without coinstatsindex option") res0 = node.gettxoutsetinfo('none') @@ -170,7 +168,6 @@ class CoinStatsIndexTest(BitcoinTestFramework): # Include both txs in a block self.generate(self.nodes[0], 1) - self.sync_all() for hash_option in index_hash_options: # Check all amounts were registered correctly @@ -228,7 +225,7 @@ class CoinStatsIndexTest(BitcoinTestFramework): res9 = index_node.gettxoutsetinfo('muhash') assert_equal(res8, res9) - self.generate(index_node, 1) + self.generate(index_node, 1, sync_fun=self.no_op) res10 = index_node.gettxoutsetinfo('muhash') assert(res8['txouts'] < res10['txouts']) @@ -254,7 +251,7 @@ class CoinStatsIndexTest(BitcoinTestFramework): assert_equal(index_node.gettxoutsetinfo('muhash')['height'], 110) # Add two new blocks - block = self.generate(index_node, 2)[1] + block = self.generate(index_node, 2, sync_fun=self.no_op)[1] res = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=None, use_index=False) # Test that the result of the reorged block is not returned for its old block height @@ -271,7 +268,6 @@ class CoinStatsIndexTest(BitcoinTestFramework): # Add another block, so we don't depend on reconsiderblock remembering which # blocks were touched by invalidateblock self.generate(index_node, 1) - self.sync_all() # Ensure that removing and re-adding blocks yields consistent results block = index_node.getblockhash(99) @@ -280,14 +276,6 @@ class CoinStatsIndexTest(BitcoinTestFramework): res3 = index_node.gettxoutsetinfo(hash_type='muhash', hash_or_height=112) assert_equal(res2, res3) - self.log.info("Test that a node aware of stale blocks syncs them as well") - node = self.nodes[0] - # Ensure the node is aware of a stale block prior to restart - node.getblock(reorg_block) - - self.restart_node(0, ["-coinstatsindex"]) - assert_raises_rpc_error(-32603, "Unable to get data because coinstatsindex is still syncing.", node.gettxoutsetinfo, 'muhash', reorg_block) - def _test_index_rejects_hash_serialized(self): self.log.info("Test that the rpc raises if the legacy hash is passed with the index") diff --git a/test/functional/feature_config_args.py b/test/functional/feature_config_args.py index 3d9d8b7441..eea5fa24ee 100755 --- a/test/functional/feature_config_args.py +++ b/test/functional/feature_config_args.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test various command line arguments and configuration file parameters.""" diff --git a/test/functional/feature_csv_activation.py b/test/functional/feature_csv_activation.py index 5255b13bd1..c200445e81 100755 --- a/test/functional/feature_csv_activation.py +++ b/test/functional/feature_csv_activation.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test CSV soft fork activation. @@ -173,11 +173,7 @@ class BIP68_112_113Test(BitcoinTestFramework): return test_blocks def create_test_block(self, txs): - block = create_block(self.tip, create_coinbase(self.tipheight + 1), self.last_block_time + 600) - block.nVersion = 4 - block.vtx.extend(txs) - block.hashMerkleRoot = block.calc_merkle_root() - block.rehash() + block = create_block(self.tip, create_coinbase(self.tipheight + 1), self.last_block_time + 600, txlist=txs) block.solve() return block diff --git a/test/functional/feature_dbcrash.py b/test/functional/feature_dbcrash.py index f0766ca7c2..3e60efbb3c 100755 --- a/test/functional/feature_dbcrash.py +++ b/test/functional/feature_dbcrash.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test recovery from a crash during chainstate writing. @@ -217,7 +217,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): # Start by creating a lot of utxos on node3 initial_height = self.nodes[3].getblockcount() - utxo_list = create_confirmed_utxos(self, self.nodes[3].getnetworkinfo()['relayfee'], self.nodes[3], 5000) + utxo_list = create_confirmed_utxos(self, self.nodes[3].getnetworkinfo()['relayfee'], self.nodes[3], 5000, sync_fun=self.no_op) self.log.info(f"Prepped {len(utxo_list)} utxo entries") # Sync these blocks with the other nodes @@ -258,6 +258,7 @@ class ChainstateWriteCrashTest(BitcoinTestFramework): nblocks=min(10, current_height + 1 - self.nodes[3].getblockcount()), # new address to avoid mining a block that has just been invalidated address=self.nodes[3].getnewaddress(), + sync_fun=self.no_op, )) self.log.debug(f"Syncing {len(block_hashes)} new blocks...") self.sync_node3blocks(block_hashes) diff --git a/test/functional/feature_dersig.py b/test/functional/feature_dersig.py index 28aff1f2f9..b7cb32c842 100755 --- a/test/functional/feature_dersig.py +++ b/test/functional/feature_dersig.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test BIP66 (DER SIG). @@ -85,10 +85,7 @@ class BIP66Test(BitcoinTestFramework): tip = self.nodes[0].getbestblockhash() block_time = self.nodes[0].getblockheader(tip)['mediantime'] + 1 - block = create_block(int(tip, 16), create_coinbase(DERSIG_HEIGHT - 1), block_time) - block.vtx.append(spendtx) - block.hashMerkleRoot = block.calc_merkle_root() - block.rehash() + block = create_block(int(tip, 16), create_coinbase(DERSIG_HEIGHT - 1), block_time, txlist=[spendtx]) block.solve() assert_equal(self.nodes[0].getblockcount(), DERSIG_HEIGHT - 2) @@ -101,9 +98,7 @@ class BIP66Test(BitcoinTestFramework): self.log.info("Test that blocks must now be at least version 3") tip = block.sha256 block_time += 1 - block = create_block(tip, create_coinbase(DERSIG_HEIGHT), block_time) - block.nVersion = 2 - block.rehash() + block = create_block(tip, create_coinbase(DERSIG_HEIGHT), block_time, version=2) block.solve() with self.nodes[0].assert_debug_log(expected_msgs=[f'{block.hash}, bad-version(0x00000002)']): @@ -133,7 +128,6 @@ class BIP66Test(BitcoinTestFramework): # Now we verify that a block with this transaction is also invalid. block.vtx.append(spendtx) block.hashMerkleRoot = block.calc_merkle_root() - block.rehash() block.solve() with self.nodes[0].assert_debug_log(expected_msgs=[f'CheckInputScripts on {block.vtx[-1].hash} failed with non-mandatory-script-verify-flag (Non-canonical DER signature)']): @@ -144,7 +138,6 @@ class BIP66Test(BitcoinTestFramework): self.log.info("Test that a block with a DERSIG-compliant transaction is accepted") block.vtx[1] = self.create_tx(self.coinbase_txids[1]) block.hashMerkleRoot = block.calc_merkle_root() - block.rehash() block.solve() self.test_dersig_info(is_active=True) # Not active as of current tip, but next block must obey rules diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index ac00db8ff0..46d5bcf1a6 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test fee estimation code.""" @@ -218,7 +218,6 @@ class EstimateFeeTest(BitcoinTestFramework): self.fees_per_kb.append(float(fee) / tx_kbytes) self.sync_mempools(wait=.1) mined = mining_node.getblock(self.generate(mining_node, 1)[0], True)["tx"] - self.sync_blocks(wait=.1) # update which txouts are confirmed newmem = [] for utx in self.memutxo: @@ -237,7 +236,7 @@ class EstimateFeeTest(BitcoinTestFramework): # Mine while len(node.getrawmempool()) > 0: - self.generate(node, 1) + self.generate(node, 1, sync_fun=self.no_op) # Repeatedly split those 2 outputs, doubling twice for each rep # Use txouts to monitor the available utxo, since these won't be tracked in wallet @@ -247,12 +246,12 @@ class EstimateFeeTest(BitcoinTestFramework): while len(self.txouts) > 0: split_inputs(node, self.txouts, self.txouts2) while len(node.getrawmempool()) > 0: - self.generate(node, 1) + self.generate(node, 1, sync_fun=self.no_op) # Double txouts2 to txouts while len(self.txouts2) > 0: split_inputs(node, self.txouts2, self.txouts) while len(node.getrawmempool()) > 0: - self.generate(node, 1) + self.generate(node, 1, sync_fun=self.no_op) reps += 1 def sanity_check_estimates_range(self): @@ -278,8 +277,6 @@ class EstimateFeeTest(BitcoinTestFramework): # Finish by mining a normal-sized block: while len(self.nodes[1].getrawmempool()) > 0: self.generate(self.nodes[1], 1) - - self.sync_blocks(self.nodes[0:3], wait=.1) self.log.info("Final estimates after emptying mempools") check_estimates(self.nodes[1], self.fees_per_kb) @@ -322,7 +319,6 @@ class EstimateFeeTest(BitcoinTestFramework): for txid in txids_to_replace: miner.prioritisetransaction(txid=txid, fee_delta=-COIN) self.generate(miner, 1) - self.sync_blocks(wait=.1, nodes=[node, miner]) # RBF the low-fee transactions while True: try: @@ -334,7 +330,6 @@ class EstimateFeeTest(BitcoinTestFramework): # Mine the last replacement txs self.sync_mempools(wait=.1, nodes=[node, miner]) self.generate(miner, 1) - self.sync_blocks(wait=.1, nodes=[node, miner]) # Only 10% of the transactions were really confirmed with a low feerate, # the rest needed to be RBF'd. We must return the 90% conf rate feerate. diff --git a/test/functional/feature_filelock.py b/test/functional/feature_filelock.py index 0fc654e10a..945ece6a33 100755 --- a/test/functional/feature_filelock.py +++ b/test/functional/feature_filelock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Check that it's not possible to start a second bitcoind instance using the same datadir or wallet.""" diff --git a/test/functional/feature_help.py b/test/functional/feature_help.py index 837e95c128..4b66030b47 100755 --- a/test/functional/feature_help.py +++ b/test/functional/feature_help.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Verify that starting bitcoin with -h works as expected.""" diff --git a/test/functional/feature_includeconf.py b/test/functional/feature_includeconf.py index 448182eded..818e4c923b 100755 --- a/test/functional/feature_includeconf.py +++ b/test/functional/feature_includeconf.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests the includeconf argument diff --git a/test/functional/feature_init.py b/test/functional/feature_init.py new file mode 100755 index 0000000000..cffbd40271 --- /dev/null +++ b/test/functional/feature_init.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Stress tests related to node initialization.""" +import random +import time +import os +from pathlib import Path + +from test_framework.test_framework import BitcoinTestFramework, SkipTest +from test_framework.test_node import ErrorMatch +from test_framework.util import assert_equal + + +class InitStressTest(BitcoinTestFramework): + """ + Ensure that initialization can be interrupted at a number of points and not impair + subsequent starts. + """ + + def set_test_params(self): + self.setup_clean_chain = False + self.num_nodes = 1 + + def run_test(self): + """ + - test terminating initialization after seeing a certain log line. + - test terminating init after seeing a random number of log lines. + - test removing certain essential files to test startup error paths. + """ + # TODO: skip Windows for now since it isn't clear how to SIGTERM. + # + # Windows doesn't support `process.terminate()`. + # and other approaches (like below) don't work: + # + # os.kill(node.process.pid, signal.CTRL_C_EVENT) + if os.name == 'nt': + raise SkipTest("can't SIGTERM on Windows") + + self.stop_node(0) + node = self.nodes[0] + + def sigterm_node(): + node.process.terminate() + node.process.wait() + node.debug_log_path.unlink() + node.debug_log_path.touch() + + def check_clean_start(): + """Ensure that node restarts successfully after various interrupts.""" + # TODO: add -txindex=1 to fully test index initiatlization. + # See https://github.com/bitcoin/bitcoin/pull/23289#discussion_r735159180 for + # a discussion of the related bug. + node.start() + node.wait_for_rpc_connection() + assert_equal(200, node.getblockcount()) + + lines_to_terminate_after = [ + 'scheduler thread start', + 'Loading P2P addresses', + 'Loading banlist', + 'Loading block index', + 'Switching active chainstate', + 'Loaded best chain:', + 'init message: Verifying blocks', + 'loadblk thread start', + # TODO: reenable - see above TODO + # 'txindex thread start', + 'net thread start', + 'addcon thread start', + 'msghand thread start', + ] + if self.is_wallet_compiled(): + lines_to_terminate_after.append('Verifying wallet') + + for terminate_line in lines_to_terminate_after: + self.log.info(f"Starting node and will exit after line '{terminate_line}'") + node.start( + # TODO: add -txindex=1 to fully test index initiatlization. + # extra_args=['-txindex=1'], + ) + logfile = open(node.debug_log_path, 'r', encoding='utf8') + + MAX_SECS_TO_WAIT = 30 + start = time.time() + num_lines = 0 + + while True: + line = logfile.readline() + if line: + num_lines += 1 + + if line and terminate_line.lower() in line.lower(): + self.log.debug(f"Terminating node after {num_lines} log lines seen") + sigterm_node() + break + + if (time.time() - start) > MAX_SECS_TO_WAIT: + raise AssertionError( + f"missed line {terminate_line}; terminating now after {num_lines} lines") + + if node.process.poll() is not None: + raise AssertionError(f"node failed to start (line: '{terminate_line}')") + + check_clean_start() + num_total_logs = len(node.debug_log_path.read_text().splitlines()) + self.stop_node(0) + + self.log.info( + f"Terminate at some random point in the init process (max logs: {num_total_logs})") + + for _ in range(40): + terminate_after = random.randint(1, num_total_logs) + self.log.debug(f"Starting node and will exit after {terminate_after} lines") + node.start( + # TODO: add -txindex=1 to fully test index initiatlization. + # extra_args=['-txindex=1'], + ) + logfile = open(node.debug_log_path, 'r', encoding='utf8') + + MAX_SECS_TO_WAIT = 10 + start = time.time() + num_lines = 0 + + while True: + line = logfile.readline() + if line: + num_lines += 1 + + if num_lines >= terminate_after or (time.time() - start) > MAX_SECS_TO_WAIT: + self.log.debug(f"Terminating node after {num_lines} log lines seen") + sigterm_node() + break + + if node.process.poll() is not None: + raise AssertionError("node failed to start") + + check_clean_start() + self.stop_node(0) + + self.log.info("Test startup errors after removing certain essential files") + + files_to_disturb = { + 'blocks/index/*.ldb': 'Error opening block database.', + 'chainstate/*.ldb': 'Error opening block database.', + 'blocks/blk*.dat': 'Error loading block database.', + } + + for file_patt, err_fragment in files_to_disturb.items(): + target_file = list(node.chain_path.glob(file_patt))[0] + + self.log.info(f"Tweaking file to ensure failure {target_file}") + bak_path = str(target_file) + ".bak" + target_file.rename(bak_path) + + # TODO: at some point, we should test perturbing the files instead of removing + # them, e.g. + # + # contents = target_file.read_bytes() + # tweaked_contents = bytearray(contents) + # tweaked_contents[50:250] = b'1' * 200 + # target_file.write_bytes(bytes(tweaked_contents)) + # + # At the moment I can't get this to work (bitcoind loads successfully?) so + # investigate doing this later. + + node.assert_start_raises_init_error( + # TODO: add -txindex=1 to fully test index initiatlization. + # extra_args=['-txindex=1'], + expected_msg=err_fragment, + match=ErrorMatch.PARTIAL_REGEX, + ) + + self.log.info(f"Restoring file from {bak_path} and restarting") + Path(bak_path).rename(target_file) + check_clean_start() + self.stop_node(0) + + +if __name__ == '__main__': + InitStressTest().main() diff --git a/test/functional/feature_loadblock.py b/test/functional/feature_loadblock.py index 13e6a8d6d7..7f030c6773 100755 --- a/test/functional/feature_loadblock.py +++ b/test/functional/feature_loadblock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test loadblock option @@ -29,7 +29,7 @@ class LoadblockTest(BitcoinTestFramework): def run_test(self): self.nodes[1].setnetworkactive(state=False) - self.generate(self.nodes[0], COINBASE_MATURITY) + self.generate(self.nodes[0], COINBASE_MATURITY, sync_fun=self.no_op) # Parsing the url of our node to get settings for config file data_dir = self.nodes[0].datadir diff --git a/test/functional/feature_logging.py b/test/functional/feature_logging.py index 722219518a..fe4f02dfe6 100755 --- a/test/functional/feature_logging.py +++ b/test/functional/feature_logging.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test debug logging.""" diff --git a/test/functional/feature_maxuploadtarget.py b/test/functional/feature_maxuploadtarget.py index ac4d40638e..b4e0df8a11 100755 --- a/test/functional/feature_maxuploadtarget.py +++ b/test/functional/feature_maxuploadtarget.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test behavior of -maxuploadtarget. diff --git a/test/functional/feature_minchainwork.py b/test/functional/feature_minchainwork.py index 11cb4aa3cb..489a729cfc 100755 --- a/test/functional/feature_minchainwork.py +++ b/test/functional/feature_minchainwork.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test logic for setting nMinimumChainWork on command line. @@ -51,8 +51,7 @@ class MinimumChainWorkTest(BitcoinTestFramework): num_blocks_to_generate = int((self.node_min_work[1] - starting_chain_work) / REGTEST_WORK_PER_BLOCK) self.log.info(f"Generating {num_blocks_to_generate} blocks on node0") - hashes = self.generatetoaddress(self.nodes[0], num_blocks_to_generate, - self.nodes[0].get_deterministic_priv_key().address) + hashes = self.generate(self.nodes[0], num_blocks_to_generate, sync_fun=self.no_op) self.log.info(f"Node0 current chain work: {self.nodes[0].getblockheader(hashes[-1])['chainwork']}") @@ -73,7 +72,7 @@ class MinimumChainWorkTest(BitcoinTestFramework): assert_equal(self.nodes[2].getblockcount(), starting_blockcount) self.log.info("Generating one more block") - self.generatetoaddress(self.nodes[0], 1, self.nodes[0].get_deterministic_priv_key().address) + self.generate(self.nodes[0], 1) self.log.info("Verifying nodes are all synced") diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index 2a507c75c4..e038afa1ad 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the -alertnotify, -blocknotify and -walletnotify options.""" @@ -112,7 +112,6 @@ class NotificationsTest(BitcoinTestFramework): self.log.info("test -walletnotify with conflicting transactions") self.nodes[0].rescanblockchain() self.generatetoaddress(self.nodes[0], 100, ADDRESS_BCRT1_UNSPENDABLE) - self.sync_blocks() # Generate transaction on node 0, sync mempools, and check for # notification on node 1. @@ -149,7 +148,7 @@ class NotificationsTest(BitcoinTestFramework): # about newly confirmed bump2 and newly conflicted tx2. self.disconnect_nodes(0, 1) bump2 = self.nodes[0].bumpfee(tx2)["txid"] - blockhash2 = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE)[0] + blockhash2 = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE, sync_fun=self.no_op)[0] blockheight2 = self.nodes[0].getblockcount() assert_equal(self.nodes[0].gettransaction(bump2)["confirmations"], 1) assert_equal(tx2 in self.nodes[1].getrawmempool(), True) diff --git a/test/functional/feature_nulldummy.py b/test/functional/feature_nulldummy.py index 217a38050d..7a84098a83 100755 --- a/test/functional/feature_nulldummy.py +++ b/test/functional/feature_nulldummy.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2020 The Bitcoin Core developers +# Copyright (c) 2016-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test NULLDUMMY softfork. @@ -22,7 +22,10 @@ from test_framework.blocktools import ( create_transaction, ) from test_framework.messages import CTransaction -from test_framework.script import CScript +from test_framework.script import ( + OP_0, + OP_TRUE, +) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -32,16 +35,11 @@ from test_framework.util import ( NULLDUMMY_ERROR = "non-mandatory-script-verify-flag (Dummy CHECKMULTISIG argument must be zero)" -def trueDummy(tx): - scriptSig = CScript(tx.vin[0].scriptSig) - newscript = [] - for i in scriptSig: - if len(newscript) == 0: - assert len(i) == 0 - newscript.append(b'\x51') - else: - newscript.append(i) - tx.vin[0].scriptSig = CScript(newscript) +def invalidate_nulldummy_tx(tx): + """Transform a NULLDUMMY compliant tx (i.e. scriptSig starts with OP_0) + to be non-NULLDUMMY compliant by replacing the dummy with OP_TRUE""" + assert_equal(tx.vin[0].scriptSig[0], OP_0) + tx.vin[0].scriptSig = bytes([OP_TRUE]) + tx.vin[0].scriptSig[1:] tx.rehash() @@ -94,7 +92,7 @@ class NULLDUMMYTest(BitcoinTestFramework): self.log.info("Test 2: Non-NULLDUMMY base multisig transaction should not be accepted to mempool before activation") test2tx = create_transaction(self.nodes[0], txid2, self.ms_address, amount=47) - trueDummy(test2tx) + invalidate_nulldummy_tx(test2tx) assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test2tx.serialize_with_witness().hex(), 0) self.log.info(f"Test 3: Non-NULLDUMMY base transactions should be accepted in a block before activation [{COINBASE_MATURITY + 4}]") @@ -103,7 +101,7 @@ class NULLDUMMYTest(BitcoinTestFramework): self.log.info("Test 4: Non-NULLDUMMY base multisig transaction is invalid after activation") test4tx = create_transaction(self.nodes[0], test2tx.hash, self.address, amount=46) test6txs = [CTransaction(test4tx)] - trueDummy(test4tx) + invalidate_nulldummy_tx(test4tx) assert_raises_rpc_error(-26, NULLDUMMY_ERROR, self.nodes[0].sendrawtransaction, test4tx.serialize_with_witness().hex(), 0) self.block_submit(self.nodes[0], [test4tx], accept=False) @@ -123,14 +121,9 @@ class NULLDUMMYTest(BitcoinTestFramework): tmpl = node.getblocktemplate(NORMAL_GBT_REQUEST_PARAMS) assert_equal(tmpl['previousblockhash'], self.lastblockhash) assert_equal(tmpl['height'], self.lastblockheight + 1) - block = create_block(tmpl=tmpl, ntime=self.lastblocktime + 1) - for tx in txs: - tx.rehash() - block.vtx.append(tx) - block.hashMerkleRoot = block.calc_merkle_root() + block = create_block(tmpl=tmpl, ntime=self.lastblocktime + 1, txlist=txs) if with_witness: add_witness_commitment(block) - block.rehash() block.solve() assert_equal(None if accept else NULLDUMMY_ERROR, node.submitblock(block.serialize().hex())) if accept: diff --git a/test/functional/feature_presegwit_node_upgrade.py b/test/functional/feature_presegwit_node_upgrade.py index aac42d4dbf..3d762c8197 100755 --- a/test/functional/feature_presegwit_node_upgrade.py +++ b/test/functional/feature_presegwit_node_upgrade.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test a pre-segwit node upgrading to segwit consensus""" diff --git a/test/functional/feature_proxy.py b/test/functional/feature_proxy.py index 2fb5e328f5..7d9e5b70fc 100755 --- a/test/functional/feature_proxy.py +++ b/test/functional/feature_proxy.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoind with different proxy configuration. @@ -12,6 +12,7 @@ Test plan: - `-proxy` (proxy everything) - `-onion` (proxy just onions) - `-proxyrandomize` Circuit randomization + - `-cjdnsreachable` - Proxy configurations to test on proxy side, - support no authentication (other proxy) - support no authentication + user/pass authentication (Tor) @@ -26,6 +27,7 @@ addnode connect to IPv4 addnode connect to IPv6 addnode connect to onion addnode connect to generic DNS name +addnode connect to a CJDNS address - Test getnetworkinfo for each node """ @@ -50,14 +52,15 @@ NET_IPV4 = "ipv4" NET_IPV6 = "ipv6" NET_ONION = "onion" NET_I2P = "i2p" +NET_CJDNS = "cjdns" # Networks returned by RPC getnetworkinfo, defined in src/rpc/net.cpp::GetNetworksInfo() -NETWORKS = frozenset({NET_IPV4, NET_IPV6, NET_ONION, NET_I2P}) +NETWORKS = frozenset({NET_IPV4, NET_IPV6, NET_ONION, NET_I2P, NET_CJDNS}) class ProxyTest(BitcoinTestFramework): def set_test_params(self): - self.num_nodes = 4 + self.num_nodes = 5 self.setup_clean_chain = True def setup_nodes(self): @@ -101,7 +104,9 @@ class ProxyTest(BitcoinTestFramework): ['-listen', f'-proxy={self.conf1.addr[0]}:{self.conf1.addr[1]}',f'-onion={self.conf2.addr[0]}:{self.conf2.addr[1]}', f'-i2psam={self.i2p_sam[0]}:{self.i2p_sam[1]}', '-i2pacceptincoming=0', '-proxyrandomize=0'], ['-listen', f'-proxy={self.conf2.addr[0]}:{self.conf2.addr[1]}','-proxyrandomize=1'], - [] + [], + ['-listen', f'-proxy={self.conf1.addr[0]}:{self.conf1.addr[1]}','-proxyrandomize=1', + '-cjdnsreachable'] ] if self.have_ipv6: args[3] = ['-listen', f'-proxy=[{self.conf3.addr[0]}]:{self.conf3.addr[1]}','-proxyrandomize=0', '-noonion'] @@ -113,7 +118,7 @@ class ProxyTest(BitcoinTestFramework): if peer["addr"] == addr: assert_equal(peer["network"], network) - def node_test(self, node, proxies, auth, test_onion=True): + def node_test(self, node, *, proxies, auth, test_onion, test_cjdns): rv = [] addr = "15.61.23.23:1234" self.log.debug(f"Test: outgoing IPv4 connection through node for address {addr}") @@ -161,6 +166,21 @@ class ProxyTest(BitcoinTestFramework): rv.append(cmd) self.network_test(node, addr, network=NET_ONION) + if test_cjdns: + addr = "[fc00:1:2:3:4:5:6:7]:8888" + self.log.debug(f"Test: outgoing CJDNS connection through node for address {addr}") + node.addnode(addr, "onetry") + cmd = proxies[1].queue.get() + assert isinstance(cmd, Socks5Command) + assert_equal(cmd.atyp, AddressType.DOMAINNAME) + assert_equal(cmd.addr, b"fc00:1:2:3:4:5:6:7") + assert_equal(cmd.port, 8888) + if not auth: + assert_equal(cmd.username, None) + assert_equal(cmd.password, None) + rv.append(cmd) + self.network_test(node, addr, network=NET_CJDNS) + addr = "node.noumenon:8333" self.log.debug(f"Test: outgoing DNS name connection through node for address {addr}") node.addnode(addr, "onetry") @@ -179,20 +199,33 @@ class ProxyTest(BitcoinTestFramework): def run_test(self): # basic -proxy - self.node_test(self.nodes[0], [self.serv1, self.serv1, self.serv1, self.serv1], False) + self.node_test(self.nodes[0], + proxies=[self.serv1, self.serv1, self.serv1, self.serv1], + auth=False, test_onion=True, test_cjdns=False) # -proxy plus -onion - self.node_test(self.nodes[1], [self.serv1, self.serv1, self.serv2, self.serv1], False) + self.node_test(self.nodes[1], + proxies=[self.serv1, self.serv1, self.serv2, self.serv1], + auth=False, test_onion=True, test_cjdns=False) # -proxy plus -onion, -proxyrandomize - rv = self.node_test(self.nodes[2], [self.serv2, self.serv2, self.serv2, self.serv2], True) + rv = self.node_test(self.nodes[2], + proxies=[self.serv2, self.serv2, self.serv2, self.serv2], + auth=True, test_onion=True, test_cjdns=False) # Check that credentials as used for -proxyrandomize connections are unique credentials = set((x.username,x.password) for x in rv) assert_equal(len(credentials), len(rv)) if self.have_ipv6: # proxy on IPv6 localhost - self.node_test(self.nodes[3], [self.serv3, self.serv3, self.serv3, self.serv3], False, False) + self.node_test(self.nodes[3], + proxies=[self.serv3, self.serv3, self.serv3, self.serv3], + auth=False, test_onion=False, test_cjdns=False) + + # -proxy=unauth -proxyrandomize=1 -cjdnsreachable + self.node_test(self.nodes[4], + proxies=[self.serv1, self.serv1, self.serv1, self.serv1], + auth=False, test_onion=True, test_cjdns=True) def networks_dict(d): r = {} @@ -214,6 +247,7 @@ class ProxyTest(BitcoinTestFramework): assert_equal(n0[net]['proxy_randomize_credentials'], expected_randomize) assert_equal(n0['onion']['reachable'], True) assert_equal(n0['i2p']['reachable'], False) + assert_equal(n0['cjdns']['reachable'], False) n1 = networks_dict(self.nodes[1].getnetworkinfo()) assert_equal(NETWORKS, n1.keys()) @@ -240,6 +274,7 @@ class ProxyTest(BitcoinTestFramework): assert_equal(n2[net]['proxy_randomize_credentials'], expected_randomize) assert_equal(n2['onion']['reachable'], True) assert_equal(n2['i2p']['reachable'], False) + assert_equal(n2['cjdns']['reachable'], False) if self.have_ipv6: n3 = networks_dict(self.nodes[3].getnetworkinfo()) @@ -253,6 +288,22 @@ class ProxyTest(BitcoinTestFramework): assert_equal(n3[net]['proxy_randomize_credentials'], False) assert_equal(n3['onion']['reachable'], False) assert_equal(n3['i2p']['reachable'], False) + assert_equal(n3['cjdns']['reachable'], False) + + n4 = networks_dict(self.nodes[4].getnetworkinfo()) + assert_equal(NETWORKS, n4.keys()) + for net in NETWORKS: + if net == NET_I2P: + expected_proxy = '' + expected_randomize = False + else: + expected_proxy = '%s:%i' % (self.conf1.addr) + expected_randomize = True + assert_equal(n4[net]['proxy'], expected_proxy) + assert_equal(n4[net]['proxy_randomize_credentials'], expected_randomize) + assert_equal(n4['onion']['reachable'], True) + assert_equal(n4['i2p']['reachable'], False) + assert_equal(n4['cjdns']['reachable'], True) if __name__ == '__main__': diff --git a/test/functional/feature_pruning.py b/test/functional/feature_pruning.py index c2463d0bcc..0edf1d66c8 100755 --- a/test/functional/feature_pruning.py +++ b/test/functional/feature_pruning.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the pruning code. @@ -118,9 +118,8 @@ class PruneTest(BitcoinTestFramework): def create_big_chain(self): # Start by creating some coinbases we can spend later - self.generate(self.nodes[1], 200) - self.sync_blocks(self.nodes[0:2]) - self.generate(self.nodes[0], 150) + self.generate(self.nodes[1], 200, sync_fun=lambda: self.sync_blocks(self.nodes[0:2])) + self.generate(self.nodes[0], 150, sync_fun=self.no_op) # Then mine enough full blocks to create more than 550MiB of data mine_large_blocks(self.nodes[0], 645) @@ -211,7 +210,7 @@ class PruneTest(BitcoinTestFramework): self.disconnect_nodes(1, 2) self.log.info("Generating new longer chain of 300 more blocks") - self.generate(self.nodes[1], 300) + self.generate(self.nodes[1], 300, sync_fun=self.no_op) self.log.info("Reconnect nodes") self.connect_nodes(0, 1) @@ -263,7 +262,7 @@ class PruneTest(BitcoinTestFramework): self.nodes[0].invalidateblock(curchainhash) assert_equal(self.nodes[0].getblockcount(), self.mainchainheight) assert_equal(self.nodes[0].getbestblockhash(), self.mainchainhash2) - goalbesthash = self.generate(self.nodes[0], blocks_to_mine)[-1] + goalbesthash = self.generate(self.nodes[0], blocks_to_mine, sync_fun=self.no_op)[-1] goalbestheight = first_reorg_height + 1 self.log.info("Verify node 2 reorged back to the main chain, some blocks of which it had to redownload") @@ -306,7 +305,7 @@ class PruneTest(BitcoinTestFramework): assert_equal(block1_details["nTx"], len(block1_details["tx"])) # mine 6 blocks so we are at height 1001 (i.e., above PruneAfterHeight) - self.generate(node, 6) + self.generate(node, 6, sync_fun=self.no_op) assert_equal(node.getblockchaininfo()["blocks"], 1001) # Pruned block should still know the number of transactions @@ -337,7 +336,7 @@ class PruneTest(BitcoinTestFramework): assert has_block(2), "blk00002.dat is still there, should be pruned by now" # advance the tip so blk00002.dat and blk00003.dat can be pruned (the last 288 blocks should now be in blk00004.dat) - self.generate(node, 288) + self.generate(node, 288, sync_fun=self.no_op) prune(1000) assert not has_block(2), "blk00002.dat is still there, should be pruned by now" assert not has_block(3), "blk00003.dat is still there, should be pruned by now" diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index 420147542e..e540cc1574 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RBF code.""" @@ -47,8 +47,8 @@ class ReplaceByFeeTest(BitcoinTestFramework): def run_test(self): self.wallet = MiniWallet(self.nodes[0]) # the pre-mined test framework chain contains coinbase outputs to the - # MiniWallet's default address ADDRESS_BCRT1_P2WSH_OP_TRUE in blocks - # 76-100 (see method BitcoinTestFramework._initialize_chain()) + # MiniWallet's default address in blocks 76-100 (see method + # BitcoinTestFramework._initialize_chain()) self.wallet.rescan_utxos() self.log.info("Running test simple doublespend...") diff --git a/test/functional/feature_reindex.py b/test/functional/feature_reindex.py index f0435b21b2..44040f426f 100755 --- a/test/functional/feature_reindex.py +++ b/test/functional/feature_reindex.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test running bitcoind with -reindex and -reindex-chainstate options. diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index acb7469c6a..6d7f1def88 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2020 The Bitcoin Core developers +# Copyright (c) 2016-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the SegWit changeover logic.""" @@ -191,7 +191,6 @@ class SegWitTest(BitcoinTestFramework): p2sh_ids[n][v].append(send_to_witness(v, self.nodes[0], find_spendable_utxo(self.nodes[0], 50), self.pubkey[n], True, Decimal("49.999"))) self.generate(self.nodes[0], 1) # block 163 - self.sync_blocks() # Make sure all nodes recognize the transactions as theirs assert_equal(self.nodes[0].getbalance(), balance_presetup - 60 * 50 + 20 * Decimal("49.999") + 50) @@ -199,7 +198,6 @@ class SegWitTest(BitcoinTestFramework): assert_equal(self.nodes[2].getbalance(), 20 * Decimal("49.999")) self.generate(self.nodes[0], 260) # block 423 - self.sync_blocks() self.log.info("Verify witness txs are skipped for mining before the fork") self.skip_mine(self.nodes[2], wit_ids[NODE_2][P2WPKH][0], True) # block 424 @@ -216,7 +214,6 @@ class SegWitTest(BitcoinTestFramework): self.log.info("Verify previous witness txs skipped for mining can now be mined") assert_equal(len(self.nodes[2].getrawmempool()), 4) blockhash = self.generate(self.nodes[2], 1)[0] # block 432 (first block with new rules; 432 = 144 * 3) - self.sync_blocks() assert_equal(len(self.nodes[2].getrawmempool()), 0) segwit_tx_list = self.nodes[2].getblock(blockhash)["tx"] assert_equal(len(segwit_tx_list), 5) @@ -630,7 +627,6 @@ class SegWitTest(BitcoinTestFramework): signresults = self.nodes[0].signrawtransactionwithwallet(tx.serialize_without_witness().hex())['hex'] txid = self.nodes[0].sendrawtransaction(hexstring=signresults, maxfeerate=0) txs_mined[txid] = self.generate(self.nodes[0], 1)[0] - self.sync_blocks() watchcount = 0 spendcount = 0 for i in self.nodes[0].listunspent(): @@ -680,7 +676,6 @@ class SegWitTest(BitcoinTestFramework): signresults = self.nodes[0].signrawtransactionwithwallet(tx.serialize_without_witness().hex())['hex'] self.nodes[0].sendrawtransaction(hexstring=signresults, maxfeerate=0) self.generate(self.nodes[0], 1) - self.sync_blocks() if __name__ == '__main__': diff --git a/test/functional/feature_settings.py b/test/functional/feature_settings.py index 26048d37f6..20018f010f 100755 --- a/test/functional/feature_settings.py +++ b/test/functional/feature_settings.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test various command line arguments and configuration file parameters.""" diff --git a/test/functional/feature_signet.py b/test/functional/feature_signet.py index 94138b0e6d..6578caee3f 100755 --- a/test/functional/feature_signet.py +++ b/test/functional/feature_signet.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test basic signet functionality""" @@ -51,7 +51,7 @@ class SignetBasicTest(BitcoinTestFramework): assert_equal(mining_info['networkhashps'], Decimal('0')) assert_equal(mining_info['pooledtx'], 0) - self.generate(self.nodes[0], 1) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) self.log.info("pregenerated signet blocks check") diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index 50a25ee1ef..e9da6edaf6 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # Test Taproot softfork (BIPs 340-342) @@ -22,11 +22,17 @@ from test_framework.messages import ( ) from test_framework.script import ( ANNEX_TAG, + BIP341_sha_amounts, + BIP341_sha_outputs, + BIP341_sha_prevouts, + BIP341_sha_scriptpubkeys, + BIP341_sha_sequences, CScript, CScriptNum, CScriptOp, + hash256, LEAF_VERSION_TAPSCRIPT, - LegacySignatureHash, + LegacySignatureMsg, LOCKTIME_THRESHOLD, MAX_SCRIPT_ELEMENT_SIZE, OP_0, @@ -70,13 +76,15 @@ from test_framework.script import ( SIGHASH_NONE, SIGHASH_SINGLE, SIGHASH_ANYONECANPAY, - SegwitV0SignatureHash, - TaprootSignatureHash, + SegwitV0SignatureMsg, + TaggedHash, + TaprootSignatureMsg, is_op_success, taproot_construct, ) from test_framework.script_util import ( key_to_p2pk_script, + key_to_p2pkh_script, key_to_p2wpkh_script, keyhash_to_p2pkh_script, script_to_p2sh_script, @@ -87,14 +95,19 @@ from test_framework.util import assert_raises_rpc_error, assert_equal from test_framework.key import generate_privkey, compute_xonly_pubkey, sign_schnorr, tweak_add_privkey, ECKey from test_framework.address import ( hash160, + program_to_witness ) from collections import OrderedDict, namedtuple +from enum import Enum from io import BytesIO import json import hashlib import os import random +# Whether or not to output generated test vectors, in JSON format. +GEN_TEST_VECTORS = False + # === Framework for building spending transactions. === # # The computation is represented as a "context" dict, whose entries store potentially-unevaluated expressions that @@ -194,8 +207,8 @@ def default_controlblock(ctx): """Default expression for "controlblock": combine leafversion, negflag, pubkey_internal, merklebranch.""" return bytes([get(ctx, "leafversion") + get(ctx, "negflag")]) + get(ctx, "pubkey_internal") + get(ctx, "merklebranch") -def default_sighash(ctx): - """Default expression for "sighash": depending on mode, compute BIP341, BIP143, or legacy sighash.""" +def default_sigmsg(ctx): + """Default expression for "sigmsg": depending on mode, compute BIP341, BIP143, or legacy sigmsg.""" tx = get(ctx, "tx") idx = get(ctx, "idx") hashtype = get(ctx, "hashtype_actual") @@ -208,18 +221,30 @@ def default_sighash(ctx): codeseppos = get(ctx, "codeseppos") leaf_ver = get(ctx, "leafversion") script = get(ctx, "script_taproot") - return TaprootSignatureHash(tx, utxos, hashtype, idx, scriptpath=True, script=script, leaf_ver=leaf_ver, codeseparator_pos=codeseppos, annex=annex) + return TaprootSignatureMsg(tx, utxos, hashtype, idx, scriptpath=True, script=script, leaf_ver=leaf_ver, codeseparator_pos=codeseppos, annex=annex) else: - return TaprootSignatureHash(tx, utxos, hashtype, idx, scriptpath=False, annex=annex) + return TaprootSignatureMsg(tx, utxos, hashtype, idx, scriptpath=False, annex=annex) elif mode == "witv0": # BIP143 signature hash scriptcode = get(ctx, "scriptcode") utxos = get(ctx, "utxos") - return SegwitV0SignatureHash(scriptcode, tx, idx, hashtype, utxos[idx].nValue) + return SegwitV0SignatureMsg(scriptcode, tx, idx, hashtype, utxos[idx].nValue) else: # Pre-segwit signature hash scriptcode = get(ctx, "scriptcode") - return LegacySignatureHash(scriptcode, tx, idx, hashtype)[0] + return LegacySignatureMsg(scriptcode, tx, idx, hashtype)[0] + +def default_sighash(ctx): + """Default expression for "sighash": depending on mode, compute tagged hash or dsha256 of sigmsg.""" + msg = get(ctx, "sigmsg") + mode = get(ctx, "mode") + if mode == "taproot": + return TaggedHash("TapSighash", msg) + else: + if msg is None: + return (1).to_bytes(32, 'little') + else: + return hash256(msg) def default_tweak(ctx): """Default expression for "tweak": None if a leaf is specified, tap[0] otherwise.""" @@ -239,14 +264,18 @@ def default_key_tweaked(ctx): def default_signature(ctx): """Default expression for "signature": BIP340 signature or ECDSA signature depending on mode.""" sighash = get(ctx, "sighash") + deterministic = get(ctx, "deterministic") if get(ctx, "mode") == "taproot": key = get(ctx, "key_tweaked") flip_r = get(ctx, "flag_flip_r") flip_p = get(ctx, "flag_flip_p") - return sign_schnorr(key, sighash, flip_r=flip_r, flip_p=flip_p) + aux = bytes([0] * 32) + if not deterministic: + aux = random.getrandbits(256).to_bytes(32, 'big') + return sign_schnorr(key, sighash, flip_r=flip_r, flip_p=flip_p, aux=aux) else: key = get(ctx, "key") - return key.sign_ecdsa(sighash) + return key.sign_ecdsa(sighash, rfc6979=deterministic) def default_hashtype_actual(ctx): """Default expression for "hashtype_actual": hashtype, unless mismatching SIGHASH_SINGLE in taproot.""" @@ -340,6 +369,8 @@ DEFAULT_CONTEXT = { "key_tweaked": default_key_tweaked, # The tweak to use (None for script path spends, the actual tweak for key path spends). "tweak": default_tweak, + # The sigmsg value (preimage of sighash) + "sigmsg": default_sigmsg, # The sighash value (32 bytes) "sighash": default_sighash, # The information about the chosen script path spend (TaprootLeafInfo object). @@ -376,6 +407,8 @@ DEFAULT_CONTEXT = { "leaf": None, # The input arguments to provide to the executed script "inputs": [], + # Use deterministic signing nonces + "deterministic": False, # == Parameters to be set before evaluation: == # - mode: what spending style to use ("taproot", "witv0", or "legacy"). @@ -396,6 +429,7 @@ def flatten(lst): ret.append(elem) return ret + def spend(tx, idx, utxos, **kwargs): """Sign transaction input idx of tx, provided utxos is the list of outputs being spent. @@ -423,7 +457,7 @@ def spend(tx, idx, utxos, **kwargs): # Each spender is a tuple of: # - A scriptPubKey which is to be spent from (CScript) # - A comment describing the test (string) -# - Whether the spending (on itself) is expected to be standard (bool) +# - Whether the spending (on itself) is expected to be standard (Enum.Standard) # - A tx-signing lambda returning (scriptsig, witness_stack), taking as inputs: # - A transaction to sign (CTransaction) # - An input position (int) @@ -435,8 +469,14 @@ def spend(tx, idx, utxos, **kwargs): # - Whether this test demands being placed in a txin with no corresponding txout (for testing SIGHASH_SINGLE behavior) Spender = namedtuple("Spender", "script,comment,is_standard,sat_function,err_msg,sigops_weight,no_fail,need_vin_vout_mismatch") +# The full node versions that treat the tx standard. +# ALL means any version +# V23 means the major version 23.0 and any later version +# NONE means no version +Standard = Enum('Standard', 'ALL V23 NONE') + -def make_spender(comment, *, tap=None, witv0=False, script=None, pkh=None, p2sh=False, spk_mutate_pre_p2sh=None, failure=None, standard=True, err_msg=None, sigops_weight=0, need_vin_vout_mismatch=False, **kwargs): +def make_spender(comment, *, tap=None, witv0=False, script=None, pkh=None, p2sh=False, spk_mutate_pre_p2sh=None, failure=None, standard=Standard.ALL, err_msg=None, sigops_weight=0, need_vin_vout_mismatch=False, **kwargs): """Helper for constructing Spender objects using the context signing framework. * tap: a TaprootInfo object (see taproot_construct), for Taproot spends (cannot be combined with pkh, witv0, or script) @@ -446,13 +486,18 @@ def make_spender(comment, *, tap=None, witv0=False, script=None, pkh=None, p2sh= * p2sh: whether the output is P2SH wrapper (this is supported even for Taproot, where it makes the output unencumbered) * spk_mutate_pre_psh: a callable to be applied to the script (before potentially P2SH-wrapping it) * failure: a dict of entries to override in the context when intentionally failing to spend (if None, no_fail will be set) - * standard: whether the (valid version of) spending is expected to be standard + * standard: whether the (valid version of) spending is expected to be standard (True is mapped to Standard.ALL, False is mapped to Standard.NONE) * err_msg: a string with an expected error message for failure (or None, if not cared about) * sigops_weight: the pre-taproot sigops weight consumed by a successful spend * need_vin_vout_mismatch: whether this test requires being tested in a transaction input that has no corresponding transaction output. """ + if standard == True: + standard = Standard.ALL + elif standard == False: + standard = Standard.NONE + conf = dict() # Compute scriptPubKey and set useful defaults based on the inputs. @@ -1137,12 +1182,12 @@ def spenders_taproot_inactive(): tap = taproot_construct(pub, scripts) # Test that keypath spending is valid & non-standard, regardless of validity. - add_spender(spenders, "inactive/keypath_valid", key=sec, tap=tap, standard=False) + add_spender(spenders, "inactive/keypath_valid", key=sec, tap=tap, standard=Standard.V23) add_spender(spenders, "inactive/keypath_invalidsig", key=sec, tap=tap, standard=False, sighash=bitflipper(default_sighash)) add_spender(spenders, "inactive/keypath_empty", key=sec, tap=tap, standard=False, witness=[]) # Same for scriptpath spending (and features like annex, leaf versions, or OP_SUCCESS don't change this) - add_spender(spenders, "inactive/scriptpath_valid", key=sec, tap=tap, leaf="pk", standard=False, inputs=[getter("sign")]) + add_spender(spenders, "inactive/scriptpath_valid", key=sec, tap=tap, leaf="pk", standard=Standard.V23, inputs=[getter("sign")]) add_spender(spenders, "inactive/scriptpath_invalidsig", key=sec, tap=tap, leaf="pk", standard=False, inputs=[getter("sign")], sighash=bitflipper(default_sighash)) add_spender(spenders, "inactive/scriptpath_invalidcb", key=sec, tap=tap, leaf="pk", standard=False, inputs=[getter("sign")], controlblock=bitflipper(default_controlblock)) add_spender(spenders, "inactive/scriptpath_valid_unkleaf", key=sec, tap=tap, leaf="future_leaf", standard=False, inputs=[getter("sign")]) @@ -1172,7 +1217,7 @@ def dump_json_test(tx, input_utxos, idx, success, failure): # The "final" field indicates that a spend should be always valid, even with more validation flags enabled # than the listed ones. Use standardness as a proxy for this (which gives a conservative underestimate). - if spender.is_standard: + if spender.is_standard == Standard.ALL: fields.append(("final", True)) def dump_witness(wit): @@ -1233,14 +1278,9 @@ class TaprootTest(BitcoinTestFramework): # transactions. extra_output_script = CScript([OP_CHECKSIG]*((MAX_BLOCK_SIGOPS_WEIGHT - sigops_weight) // WITNESS_SCALE_FACTOR)) - block = create_block(self.tip, create_coinbase(self.lastblockheight + 1, pubkey=cb_pubkey, extra_output_script=extra_output_script, fees=fees), self.lastblocktime + 1) - block.nVersion = 4 - for tx in txs: - tx.rehash() - block.vtx.append(tx) - block.hashMerkleRoot = block.calc_merkle_root() + coinbase_tx = create_coinbase(self.lastblockheight + 1, pubkey=cb_pubkey, extra_output_script=extra_output_script, fees=fees) + block = create_block(self.tip, coinbase_tx, self.lastblocktime + 1, txlist=txs) witness and add_witness_commitment(block) - block.rehash() block.solve() block_response = node.submitblock(block.serialize().hex()) if err_msg is not None: @@ -1254,6 +1294,14 @@ class TaprootTest(BitcoinTestFramework): else: assert node.getbestblockhash() == self.lastblockhash, "Failed to reject: " + msg + def init_blockinfo(self, node): + # Initialize variables used by block_submit(). + self.lastblockhash = node.getbestblockhash() + self.tip = int(self.lastblockhash, 16) + block = node.getblock(self.lastblockhash) + self.lastblockheight = block['height'] + self.lastblocktime = block['time'] + def test_spenders(self, node, spenders, input_counts): """Run randomized tests with a number of "spenders". @@ -1280,12 +1328,7 @@ class TaprootTest(BitcoinTestFramework): host_spks.append(spk) host_pubkeys.append(bytes.fromhex(info['pubkey'])) - # Initialize variables used by block_submit(). - self.lastblockhash = node.getbestblockhash() - self.tip = int(self.lastblockhash, 16) - block = node.getblock(self.lastblockhash) - self.lastblockheight = block['height'] - self.lastblocktime = block['time'] + self.init_blockinfo(node) # Create transactions spending up to 50 of the wallet's inputs, with one output for each spender, and # one change output at the end. The transaction is constructed on the Python side to enable @@ -1439,8 +1482,13 @@ class TaprootTest(BitcoinTestFramework): for i in range(len(input_utxos)): tx.vin[i].scriptSig = input_data[i][i != fail_input][0] tx.wit.vtxinwit[i].scriptWitness.stack = input_data[i][i != fail_input][1] + taproot_spend_policy = Standard.V23 if node.version is None else Standard.ALL # Submit to mempool to check standardness - is_standard_tx = fail_input is None and all(utxo.spender.is_standard for utxo in input_utxos) and tx.nVersion >= 1 and tx.nVersion <= 2 + is_standard_tx = ( + fail_input is None # Must be valid to be standard + and (all(utxo.spender.is_standard == Standard.ALL or utxo.spender.is_standard == taproot_spend_policy for utxo in input_utxos)) # All inputs must be standard + and tx.nVersion >= 1 # The tx version must be standard + and tx.nVersion <= 2) tx.rehash() msg = ','.join(utxo.spender.comment + ("*" if n == fail_input else "") for n, utxo in enumerate(input_utxos)) if is_standard_tx: @@ -1459,10 +1507,239 @@ class TaprootTest(BitcoinTestFramework): assert len(mismatching_utxos) == 0 self.log.info(" - Done") + def gen_test_vectors(self): + """Run a scenario that corresponds (and optionally produces) to BIP341 test vectors.""" + + self.log.info("Unit test scenario...") + + # Deterministically mine coins to OP_TRUE in block 1 + assert self.nodes[1].getblockcount() == 0 + coinbase = CTransaction() + coinbase.nVersion = 1 + coinbase.vin = [CTxIn(COutPoint(0, 0xffffffff), CScript([OP_1, OP_1]), 0xffffffff)] + coinbase.vout = [CTxOut(5000000000, CScript([OP_1]))] + coinbase.nLockTime = 0 + coinbase.rehash() + assert coinbase.hash == "f60c73405d499a956d3162e3483c395526ef78286458a4cb17b125aa92e49b20" + # Mine it + block = create_block(hashprev=int(self.nodes[1].getbestblockhash(), 16), coinbase=coinbase) + block.rehash() + block.solve() + self.nodes[1].submitblock(block.serialize().hex()) + assert self.nodes[1].getblockcount() == 1 + self.generate(self.nodes[1], COINBASE_MATURITY) + + SEED = 317 + VALID_LEAF_VERS = list(range(0xc0, 0x100, 2)) + [0x66, 0x7e, 0x80, 0x84, 0x96, 0x98, 0xba, 0xbc, 0xbe] + # Generate private keys + prvs = [hashlib.sha256(SEED.to_bytes(2, 'big') + bytes([i])).digest() for i in range(100)] + # Generate corresponding public x-only pubkeys + pubs = [compute_xonly_pubkey(prv)[0] for prv in prvs] + # Generate taproot objects + inner_keys = [pubs[i] for i in range(7)] + + script_lists = [ + None, + [("0", CScript([pubs[50], OP_CHECKSIG]), 0xc0)], + [("0", CScript([pubs[51], OP_CHECKSIG]), 0xc0)], + [("0", CScript([pubs[52], OP_CHECKSIG]), 0xc0), ("1", CScript([b"BIP341"]), VALID_LEAF_VERS[pubs[99][0] % 41])], + [("0", CScript([pubs[53], OP_CHECKSIG]), 0xc0), ("1", CScript([b"Taproot"]), VALID_LEAF_VERS[pubs[99][1] % 41])], + [("0", CScript([pubs[54], OP_CHECKSIG]), 0xc0), [("1", CScript([pubs[55], OP_CHECKSIG]), 0xc0), ("2", CScript([pubs[56], OP_CHECKSIG]), 0xc0)]], + [("0", CScript([pubs[57], OP_CHECKSIG]), 0xc0), [("1", CScript([pubs[58], OP_CHECKSIG]), 0xc0), ("2", CScript([pubs[59], OP_CHECKSIG]), 0xc0)]], + ] + taps = [taproot_construct(inner_keys[i], script_lists[i]) for i in range(len(inner_keys))] + + # Require negated taps[0] + assert taps[0].negflag + # Require one negated and one non-negated in taps 1 and 2. + assert taps[1].negflag != taps[2].negflag + # Require one negated and one non-negated in taps 3 and 4. + assert taps[3].negflag != taps[4].negflag + # Require one negated and one non-negated in taps 5 and 6. + assert taps[5].negflag != taps[6].negflag + + cblks = [{leaf: get({**DEFAULT_CONTEXT, 'tap': taps[i], 'leaf': leaf}, 'controlblock') for leaf in taps[i].leaves} for i in range(7)] + # Require one swapped and one unswapped in taps 3 and 4. + assert (cblks[3]['0'][33:65] < cblks[3]['1'][33:65]) != (cblks[4]['0'][33:65] < cblks[4]['1'][33:65]) + # Require one swapped and one unswapped in taps 5 and 6, both at the top and child level. + assert (cblks[5]['0'][33:65] < cblks[5]['1'][65:]) != (cblks[6]['0'][33:65] < cblks[6]['1'][65:]) + assert (cblks[5]['1'][33:65] < cblks[5]['2'][33:65]) != (cblks[6]['1'][33:65] < cblks[6]['2'][33:65]) + # Require within taps 5 (and thus also 6) that one level is swapped and the other is not. + assert (cblks[5]['0'][33:65] < cblks[5]['1'][65:]) != (cblks[5]['1'][33:65] < cblks[5]['2'][33:65]) + + # Compute a deterministic set of scriptPubKeys + tap_spks = [] + old_spks = [] + spend_info = {} + # First, taproot scriptPubKeys, for the tap objects constructed above + for i, tap in enumerate(taps): + tap_spks.append(tap.scriptPubKey) + d = {'key': prvs[i], 'tap': tap, 'mode': 'taproot'} + spend_info[tap.scriptPubKey] = d + # Then, a number of deterministically generated (keys 0x1,0x2,0x3) with 2x P2PKH, 1x P2WPKH spks. + for i in range(1, 4): + prv = ECKey() + prv.set(i.to_bytes(32, 'big'), True) + pub = prv.get_pubkey().get_bytes() + d = {"key": prv} + d["scriptcode"] = key_to_p2pkh_script(pub) + d["inputs"] = [getter("sign"), pub] + if i < 3: + # P2PKH + d['spk'] = key_to_p2pkh_script(pub) + d['mode'] = 'legacy' + else: + # P2WPKH + d['spk'] = key_to_p2wpkh_script(pub) + d['mode'] = 'witv0' + old_spks.append(d['spk']) + spend_info[d['spk']] = d + + # Construct a deterministic chain of transactions creating UTXOs to the test's spk's (so that they + # come from distinct txids). + txn = [] + lasttxid = coinbase.sha256 + amount = 5000000000 + for i, spk in enumerate(old_spks + tap_spks): + val = 42000000 * (i + 7) + tx = CTransaction() + tx.nVersion = 1 + tx.vin = [CTxIn(COutPoint(lasttxid, i & 1), CScript([]), 0xffffffff)] + tx.vout = [CTxOut(val, spk), CTxOut(amount - val, CScript([OP_1]))] + if i & 1: + tx.vout = list(reversed(tx.vout)) + tx.nLockTime = 0 + tx.rehash() + amount -= val + lasttxid = tx.sha256 + txn.append(tx) + spend_info[spk]['prevout'] = COutPoint(tx.sha256, i & 1) + spend_info[spk]['utxo'] = CTxOut(val, spk) + # Mine those transactions + self.init_blockinfo(self.nodes[1]) + self.block_submit(self.nodes[1], txn, "Crediting txn", None, sigops_weight=10, accept=True) + + # scriptPubKey computation + tests = {"version": 1} + spk_tests = tests.setdefault("scriptPubKey", []) + for i, tap in enumerate(taps): + test_case = {} + given = test_case.setdefault("given", {}) + given['internalPubkey'] = tap.internal_pubkey.hex() + + def pr(node): + if node is None: + return None + elif isinstance(node, tuple): + return {"id": int(node[0]), "script": node[1].hex(), "leafVersion": node[2]} + elif len(node) == 1: + return pr(node[0]) + elif len(node) == 2: + return [pr(node[0]), pr(node[1])] + else: + assert False + + given['scriptTree'] = pr(script_lists[i]) + intermediary = test_case.setdefault("intermediary", {}) + if len(tap.leaves): + leafhashes = intermediary.setdefault('leafHashes', [None] * len(tap.leaves)) + for leaf in tap.leaves: + leafhashes[int(leaf)] = tap.leaves[leaf].leaf_hash.hex() + intermediary['merkleRoot'] = tap.merkle_root.hex() if tap.merkle_root else None + intermediary['tweak'] = tap.tweak.hex() + intermediary['tweakedPubkey'] = tap.output_pubkey.hex() + expected = test_case.setdefault("expected", {}) + expected['scriptPubKey'] = tap.scriptPubKey.hex() + expected['bip350Address'] = program_to_witness(1, bytes(tap.output_pubkey), True) + if len(tap.leaves): + control_blocks = expected.setdefault("scriptPathControlBlocks", [None] * len(tap.leaves)) + for leaf in tap.leaves: + ctx = {**DEFAULT_CONTEXT, 'tap': tap, 'leaf': leaf} + control_blocks[int(leaf)] = get(ctx, "controlblock").hex() + spk_tests.append(test_case) + + # Construct a deterministic transaction spending all outputs created above. + tx = CTransaction() + tx.nVersion = 2 + tx.vin = [] + inputs = [] + input_spks = [tap_spks[0], tap_spks[1], old_spks[0], tap_spks[2], tap_spks[5], old_spks[2], tap_spks[6], tap_spks[3], tap_spks[4]] + sequences = [0, 0xffffffff, 0xffffffff, 0xfffffffe, 0xfffffffe, 0, 0, 0xffffffff, 0xffffffff] + hashtypes = [SIGHASH_SINGLE, SIGHASH_SINGLE|SIGHASH_ANYONECANPAY, SIGHASH_ALL, SIGHASH_ALL, SIGHASH_DEFAULT, SIGHASH_ALL, SIGHASH_NONE, SIGHASH_NONE|SIGHASH_ANYONECANPAY, SIGHASH_ALL|SIGHASH_ANYONECANPAY] + for i, spk in enumerate(input_spks): + tx.vin.append(CTxIn(spend_info[spk]['prevout'], CScript(), sequences[i])) + inputs.append(spend_info[spk]['utxo']) + tx.vout.append(CTxOut(1000000000, old_spks[1])) + tx.vout.append(CTxOut(3410000000, pubs[98])) + tx.nLockTime = 500000000 + precomputed = { + "hashAmounts": BIP341_sha_amounts(inputs), + "hashPrevouts": BIP341_sha_prevouts(tx), + "hashScriptPubkeys": BIP341_sha_scriptpubkeys(inputs), + "hashSequences": BIP341_sha_sequences(tx), + "hashOutputs": BIP341_sha_outputs(tx) + } + keypath_tests = tests.setdefault("keyPathSpending", []) + tx_test = {} + global_given = tx_test.setdefault("given", {}) + global_given['rawUnsignedTx'] = tx.serialize().hex() + utxos_spent = global_given.setdefault("utxosSpent", []) + for i in range(len(input_spks)): + utxos_spent.append({"scriptPubKey": inputs[i].scriptPubKey.hex(), "amountSats": inputs[i].nValue}) + global_intermediary = tx_test.setdefault("intermediary", {}) + for key in sorted(precomputed.keys()): + global_intermediary[key] = precomputed[key].hex() + test_list = tx_test.setdefault('inputSpending', []) + for i in range(len(input_spks)): + ctx = { + **DEFAULT_CONTEXT, + **spend_info[input_spks[i]], + 'tx': tx, + 'utxos': inputs, + 'idx': i, + 'hashtype': hashtypes[i], + 'deterministic': True + } + if ctx['mode'] == 'taproot': + test_case = {} + given = test_case.setdefault("given", {}) + given['txinIndex'] = i + given['internalPrivkey'] = get(ctx, 'key').hex() + if get(ctx, "tap").merkle_root != bytes(): + given['merkleRoot'] = get(ctx, "tap").merkle_root.hex() + else: + given['merkleRoot'] = None + given['hashType'] = get(ctx, "hashtype") + intermediary = test_case.setdefault("intermediary", {}) + intermediary['internalPubkey'] = get(ctx, "tap").internal_pubkey.hex() + intermediary['tweak'] = get(ctx, "tap").tweak.hex() + intermediary['tweakedPrivkey'] = get(ctx, "key_tweaked").hex() + sigmsg = get(ctx, "sigmsg") + intermediary['sigMsg'] = sigmsg.hex() + intermediary['precomputedUsed'] = [key for key in sorted(precomputed.keys()) if sigmsg.count(precomputed[key])] + intermediary['sigHash'] = get(ctx, "sighash").hex() + expected = test_case.setdefault("expected", {}) + expected['witness'] = [get(ctx, "sign").hex()] + test_list.append(test_case) + tx.wit.vtxinwit.append(CTxInWitness()) + tx.vin[i].scriptSig = CScript(flatten(get(ctx, "scriptsig"))) + tx.wit.vtxinwit[i].scriptWitness.stack = flatten(get(ctx, "witness")) + aux = tx_test.setdefault("auxiliary", {}) + aux['fullySignedTx'] = tx.serialize().hex() + keypath_tests.append(tx_test) + assert_equal(hashlib.sha256(tx.serialize()).hexdigest(), "24bab662cb55a7f3bae29b559f651674c62bcc1cd442d44715c0133939107b38") + # Mine the spending transaction + self.block_submit(self.nodes[1], [tx], "Spending txn", None, sigops_weight=10000, accept=True, witness=True) + + if GEN_TEST_VECTORS: + print(json.dumps(tests, indent=4, sort_keys=False)) + + def run_test(self): + self.gen_test_vectors() + # Post-taproot activation tests go first (pre-taproot tests' blocks are invalid post-taproot). self.log.info("Post-activation tests...") - self.generate(self.nodes[1], COINBASE_MATURITY + 1) self.test_spenders(self.nodes[1], spenders_taproot_active(), input_counts=[1, 2, 2, 2, 2, 3]) # Re-connect nodes in case they have been disconnected @@ -1488,7 +1765,6 @@ class TaprootTest(BitcoinTestFramework): # Mine a block with the transaction block = create_block(tmpl=self.nodes[1].getblocktemplate(NORMAL_GBT_REQUEST_PARAMS), txlist=[rawtx]) add_witness_commitment(block) - block.rehash() block.solve() assert_equal(None, self.nodes[1].submitblock(block.serialize().hex())) self.sync_blocks() diff --git a/test/functional/feature_txindex_compatibility.py b/test/functional/feature_txindex_compatibility.py new file mode 100755 index 0000000000..bbe1d1b537 --- /dev/null +++ b/test/functional/feature_txindex_compatibility.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test that legacy txindex will be disabled on upgrade. + +Previous releases are required by this test, see test/README.md. +""" + +import os +import shutil + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.wallet import MiniWallet + + +class MempoolCompatibilityTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 3 + self.extra_args = [ + ["-reindex", "-txindex"], + [], + [], + ] + + def skip_test_if_missing_module(self): + self.skip_if_no_previous_releases() + + def setup_network(self): + self.add_nodes( + self.num_nodes, + self.extra_args, + versions=[ + 160300, # Last release with legacy txindex + None, # For MiniWallet, without migration code + 200100, # Any release with migration code (0.17.x - 22.x) + ], + ) + self.start_nodes() + self.connect_nodes(0, 1) + self.connect_nodes(1, 2) + + def run_test(self): + mini_wallet = MiniWallet(self.nodes[1]) + mini_wallet.rescan_utxos() + spend_utxo = mini_wallet.get_utxo() + mini_wallet.send_self_transfer(from_node=self.nodes[1], utxo_to_spend=spend_utxo) + self.generate(self.nodes[1], 1) + + self.log.info("Check legacy txindex") + self.nodes[0].getrawtransaction(txid=spend_utxo["txid"]) # Requires -txindex + + self.stop_nodes() + legacy_chain_dir = os.path.join(self.nodes[0].datadir, self.chain) + + self.log.info("Migrate legacy txindex") + migrate_chain_dir = os.path.join(self.nodes[2].datadir, self.chain) + shutil.rmtree(migrate_chain_dir) + shutil.copytree(legacy_chain_dir, migrate_chain_dir) + with self.nodes[2].assert_debug_log([ + "Upgrading txindex database...", + "txindex is enabled at height 200", + ]): + self.start_node(2, extra_args=["-txindex"]) + self.nodes[2].getrawtransaction(txid=spend_utxo["txid"]) # Requires -txindex + + self.log.info("Drop legacy txindex") + drop_index_chain_dir = os.path.join(self.nodes[1].datadir, self.chain) + shutil.rmtree(drop_index_chain_dir) + shutil.copytree(legacy_chain_dir, drop_index_chain_dir) + self.nodes[1].assert_start_raises_init_error( + extra_args=["-txindex"], + expected_msg="Error: The block index db contains a legacy 'txindex'. To clear the occupied disk space, run a full -reindex, otherwise ignore this error. This error message will not be displayed again.", + ) + # Build txindex from scratch and check there is no error this time + self.start_node(1, extra_args=["-txindex"]) + self.nodes[2].getrawtransaction(txid=spend_utxo["txid"]) # Requires -txindex + + self.stop_nodes() + + self.log.info("Check migrated txindex can not be read by legacy node") + err_msg = f": You need to rebuild the database using -reindex to change -txindex.{os.linesep}Please restart with -reindex or -reindex-chainstate to recover." + shutil.rmtree(legacy_chain_dir) + shutil.copytree(migrate_chain_dir, legacy_chain_dir) + self.nodes[0].assert_start_raises_init_error(extra_args=["-txindex"], expected_msg=err_msg) + shutil.rmtree(legacy_chain_dir) + shutil.copytree(drop_index_chain_dir, legacy_chain_dir) + self.nodes[0].assert_start_raises_init_error(extra_args=["-txindex"], expected_msg=err_msg) + + +if __name__ == "__main__": + MempoolCompatibilityTest().main() diff --git a/test/functional/feature_utxo_set_hash.py b/test/functional/feature_utxo_set_hash.py index b1b4703d37..33b7615aea 100755 --- a/test/functional/feature_utxo_set_hash.py +++ b/test/functional/feature_utxo_set_hash.py @@ -69,8 +69,8 @@ class UTXOSetHashTest(BitcoinTestFramework): assert_equal(finalized[::-1].hex(), node_muhash) self.log.info("Test deterministic UTXO set hash results") - assert_equal(node.gettxoutsetinfo()['hash_serialized_2'], "5b1b44097406226c0eb8e1362cd17a1f346522cf9390a8175a57a5262cb1963f") - assert_equal(node.gettxoutsetinfo("muhash")['muhash'], "4b8803075d7151d06fad3e88b68ba726886794873fbfa841d12aefb2cc2b881b") + assert_equal(node.gettxoutsetinfo()['hash_serialized_2'], "221f245cf4c9010eeb7f5183d342c002ae6c1c27e98aa357dccb788c21d98049") + assert_equal(node.gettxoutsetinfo("muhash")['muhash'], "7c0890c68501f7630d36aeb3999dc924e63af084ae1bbfba11dd462144637635") def run_test(self): self.test_muhash_implementation() diff --git a/test/functional/feature_versionbits_warning.py b/test/functional/feature_versionbits_warning.py index d74ef5e088..e83dd7f446 100755 --- a/test/functional/feature_versionbits_warning.py +++ b/test/functional/feature_versionbits_warning.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2020 The Bitcoin Core developers +# Copyright (c) 2016-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test version bits warning system. @@ -48,8 +48,7 @@ class VersionBitsWarningTest(BitcoinTestFramework): tip = int(tip, 16) for _ in range(numblocks): - block = create_block(tip, create_coinbase(height + 1), block_time) - block.nVersion = version + block = create_block(tip, create_coinbase(height + 1), block_time, version=version) block.solve() peer.send_message(msg_block(block)) block_time += 1 diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index c28186cde7..8a1edb7eb2 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoin-cli""" @@ -136,10 +136,13 @@ class TestBitcoinCli(BitcoinTestFramework): network_info = self.nodes[0].getnetworkinfo() cli_get_info_string = self.nodes[0].cli('-getinfo').send_cli() cli_get_info = cli_get_info_string_to_dict(cli_get_info_string) - assert_equal(cli_get_info["Proxies"], "127.0.0.1:9050 (ipv4, ipv6, onion), 127.0.0.1:7656 (i2p)") + assert_equal(cli_get_info["Proxies"], "127.0.0.1:9050 (ipv4, ipv6, onion, cjdns), 127.0.0.1:7656 (i2p)") if self.is_wallet_compiled(): self.log.info("Test -getinfo and bitcoin-cli getwalletinfo return expected wallet info") + # Explicitely set the output type in order to have constintent tx vsize / fees + # for both legacy and descriptor wallets (disables the change address type detection algorithm) + self.restart_node(0, extra_args=["-addresstype=bech32", "-changetype=bech32"]) assert_equal(Decimal(cli_get_info['Balance']), BALANCE) assert 'Balances' not in cli_get_info_string wallet_info = self.nodes[0].getwalletinfo() diff --git a/test/functional/interface_http.py b/test/functional/interface_http.py index 075224c011..6e32009e05 100755 --- a/test/functional/interface_http.py +++ b/test/functional/interface_http.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RPC HTTP basics.""" diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index 868bb42604..a2f84573da 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the REST API.""" @@ -81,9 +81,7 @@ class RESTTest (BitcoinTestFramework): not_related_address = "2MxqoHEdNQTyYeX1mHcbrrpzgojbosTpCvJ" self.generate(self.nodes[0], 1) - self.sync_all() self.generatetoaddress(self.nodes[1], 100, not_related_address) - self.sync_all() assert_equal(self.nodes[0].getbalance(), 50) @@ -108,7 +106,6 @@ class RESTTest (BitcoinTestFramework): self.log.info("Query an unspent TXO using the /getutxos URI") self.generatetoaddress(self.nodes[1], 1, not_related_address) - self.sync_all() bb_hash = self.nodes[0].getbestblockhash() assert_equal(self.nodes[1].getbalance(), Decimal("0.1")) @@ -183,7 +180,6 @@ class RESTTest (BitcoinTestFramework): assert_equal(len(json_obj['utxos']), 0) self.generate(self.nodes[0], 1) - self.sync_all() json_obj = self.test_rest_request(f"/getutxos/{spending[0]}-{spending[1]}") assert_equal(len(json_obj['utxos']), 1) @@ -204,7 +200,6 @@ class RESTTest (BitcoinTestFramework): self.test_rest_request(f"/getutxos/checkmempool/{long_uri}", http_method='POST', status=200) self.generate(self.nodes[0], 1) # generate block to not affect upcoming tests - self.sync_all() self.log.info("Test the /block, /blockhashbyheight and /headers URIs") bb_hash = self.nodes[0].getbestblockhash() @@ -275,7 +270,6 @@ class RESTTest (BitcoinTestFramework): # See if we can get 5 headers in one response self.generate(self.nodes[1], 5) - self.sync_all() json_obj = self.test_rest_request(f"/headers/5/{bb_hash}") assert_equal(len(json_obj), 5) # now we should have 5 header objects @@ -310,7 +304,6 @@ class RESTTest (BitcoinTestFramework): # Now mine the transactions newblockhash = self.generate(self.nodes[1], 1) - self.sync_all() # Check if the 3 tx show up in the new block json_obj = self.test_rest_request(f"/block/{newblockhash[0]}") diff --git a/test/functional/interface_rpc.py b/test/functional/interface_rpc.py index 89a7d29b24..48082f3a17 100755 --- a/test/functional/interface_rpc.py +++ b/test/functional/interface_rpc.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests some generic aspects of the RPC interface.""" diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 5a11a62ec4..1ee12c0040 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the ZMQ notification interface.""" @@ -18,7 +18,6 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.messages import ( CTransaction, hash256, - tx_from_hex, ) from test_framework.util import ( assert_equal, @@ -82,9 +81,8 @@ class ZMQTestSetupBlock: the generated block's hash, it's (coinbase) transaction id, the raw block or raw transaction data. """ - def __init__(self, test_framework, node): - self.block_hash = test_framework.generate(node, 1)[0] + self.block_hash = test_framework.generate(node, 1, sync_fun=test_framework.no_op)[0] coinbase = node.getblock(self.block_hash, 2)['tx'][0] self.tx_hash = coinbase['txid'] self.raw_tx = coinbase['hex'] @@ -191,8 +189,6 @@ class ZMQTest (BitcoinTestFramework): self.log.info(f"Generate {num_blocks} blocks (and {num_blocks} coinbase txes)") genhashes = self.generatetoaddress(self.nodes[0], num_blocks, ADDRESS_BCRT1_UNSPENDABLE) - self.sync_all() - for x in range(num_blocks): # Should receive the coinbase txid. txid = hashtx.receive() @@ -261,14 +257,14 @@ class ZMQTest (BitcoinTestFramework): # Generate 1 block in nodes[0] with 1 mempool tx and receive all notifications payment_txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1.0) - disconnect_block = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE)[0] + disconnect_block = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE, sync_fun=self.no_op)[0] disconnect_cb = self.nodes[0].getblock(disconnect_block)["tx"][0] assert_equal(self.nodes[0].getbestblockhash(), hashblock.receive().hex()) assert_equal(hashtx.receive().hex(), payment_txid) assert_equal(hashtx.receive().hex(), disconnect_cb) # Generate 2 blocks in nodes[1] to a different address to ensure split - connect_blocks = self.generatetoaddress(self.nodes[1], 2, ADDRESS_BCRT1_P2WSH_OP_TRUE) + connect_blocks = self.generatetoaddress(self.nodes[1], 2, ADDRESS_BCRT1_P2WSH_OP_TRUE, sync_fun=self.no_op) # nodes[0] will reorg chain after connecting back nodes[1] self.connect_nodes(0, 1) @@ -312,13 +308,13 @@ class ZMQTest (BitcoinTestFramework): seq_num = 1 # Generate 1 block in nodes[0] and receive all notifications - dc_block = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE)[0] + dc_block = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE, sync_fun=self.no_op)[0] # Note: We are not notified of any block transactions, coinbase or mined assert_equal((self.nodes[0].getbestblockhash(), "C", None), seq.receive_sequence()) # Generate 2 blocks in nodes[1] to a different address to ensure a chain split - self.generatetoaddress(self.nodes[1], 2, ADDRESS_BCRT1_P2WSH_OP_TRUE) + self.generatetoaddress(self.nodes[1], 2, ADDRESS_BCRT1_P2WSH_OP_TRUE, sync_fun=self.no_op) # nodes[0] will reorg chain after connecting back nodes[1] self.connect_nodes(0, 1) @@ -354,7 +350,6 @@ class ZMQTest (BitcoinTestFramework): # removed from the mempool by the block mining it. mempool_size = len(self.nodes[0].getrawmempool()) c_block = self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE)[0] - self.sync_all() # Make sure the number of mined transactions matches the number of txs out of mempool mempool_size_delta = mempool_size - len(self.nodes[0].getrawmempool()) assert_equal(len(self.nodes[0].getblock(c_block)["tx"])-1, mempool_size_delta) @@ -394,7 +389,6 @@ class ZMQTest (BitcoinTestFramework): # Other things may happen but aren't wallet-deterministic so we don't test for them currently self.nodes[0].reconsiderblock(best_hash) self.generatetoaddress(self.nodes[1], 1, ADDRESS_BCRT1_UNSPENDABLE) - self.sync_all() self.log.info("Evict mempool transaction by block conflict") orig_txid = self.nodes[0].sendtoaddress(address=self.nodes[0].getnewaddress(), amount=1.0, replaceable=True) @@ -407,12 +401,8 @@ class ZMQTest (BitcoinTestFramework): raw_tx = self.nodes[0].getrawtransaction(orig_txid) bump_info = self.nodes[0].bumpfee(orig_txid) # Mine the pre-bump tx - block = create_block(int(self.nodes[0].getbestblockhash(), 16), create_coinbase(self.nodes[0].getblockcount()+1)) - tx = tx_from_hex(raw_tx) - block.vtx.append(tx) - for txid in more_tx: - tx = tx_from_hex(self.nodes[0].getrawtransaction(txid)) - block.vtx.append(tx) + txs_to_add = [raw_tx] + [self.nodes[0].getrawtransaction(txid) for txid in more_tx] + block = create_block(int(self.nodes[0].getbestblockhash(), 16), create_coinbase(self.nodes[0].getblockcount()+1), txlist=txs_to_add) add_witness_commitment(block) block.solve() assert_equal(self.nodes[0].submitblock(block.serialize().hex()), None) @@ -474,7 +464,7 @@ class ZMQTest (BitcoinTestFramework): # 1) Consume backlog until we get a mempool sequence number (hash_str, label, zmq_mem_seq) = seq.receive_sequence() while zmq_mem_seq is None: - (hash_str, label, zmq_mem_seq) = seq.receive_sequence() + (hash_str, label, zmq_mem_seq) = seq.receive_sequence() assert label == "A" or label == "R" assert hash_str is not None @@ -566,7 +556,7 @@ class ZMQTest (BitcoinTestFramework): ], sync_blocks=False) # Generate 1 block in nodes[0] and receive all notifications - self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) + self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE, sync_fun=self.no_op) # Should receive the same block hash on both subscribers assert_equal(self.nodes[0].getbestblockhash(), subscribers[0].receive().hex()) diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index 71be2b4a82..44db8bb00a 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool acceptance of raw transactions.""" diff --git a/test/functional/mempool_compatibility.py b/test/functional/mempool_compatibility.py index a9f09b1cf8..c545a7f68d 100755 --- a/test/functional/mempool_compatibility.py +++ b/test/functional/mempool_compatibility.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that mempool.dat is both backward and forward compatible between versions @@ -7,14 +7,17 @@ NOTE: The test is designed to prevent cases when compatibility is broken accidentally. In case we need to break mempool compatibility we can continue to use the test by just bumping the version number. -The previous release v0.15.2 is required by this test, see test/README.md. +The previous release v0.19.1 is required by this test, see test/README.md. """ import os from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework -from test_framework.wallet import MiniWallet +from test_framework.wallet import ( + MiniWallet, + MiniWalletMode, +) class MempoolCompatibilityTest(BitcoinTestFramework): @@ -37,9 +40,9 @@ class MempoolCompatibilityTest(BitcoinTestFramework): self.log.info("Test that mempool.dat is compatible between versions") old_node, new_node = self.nodes - new_wallet = MiniWallet(new_node) - self.generate(new_wallet, 1) - self.generate(new_node, COINBASE_MATURITY) + new_wallet = MiniWallet(new_node, mode=MiniWalletMode.RAW_P2PK) + self.generate(new_wallet, 1, sync_fun=self.no_op) + self.generate(new_node, COINBASE_MATURITY, sync_fun=self.no_op) # Sync the nodes to ensure old_node has the block that contains the coinbase that new_wallet will spend. # Otherwise, because coinbases are only valid in a block and not as loose txns, if the nodes aren't synced # unbroadcasted_tx won't pass old_node's `MemPoolAccept::PreChecks`. diff --git a/test/functional/mempool_expiry.py b/test/functional/mempool_expiry.py index 942f79e8b0..f301b29c25 100755 --- a/test/functional/mempool_expiry.py +++ b/test/functional/mempool_expiry.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests that a mempool transaction expires after a given timeout and that its diff --git a/test/functional/mempool_limit.py b/test/functional/mempool_limit.py index 79f6f9dc70..3619e05761 100755 --- a/test/functional/mempool_limit.py +++ b/test/functional/mempool_limit.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool limiting together/eviction with the wallet.""" diff --git a/test/functional/mempool_package_onemore.py b/test/functional/mempool_package_onemore.py index 69c21f32bc..a6fb1dcf35 100755 --- a/test/functional/mempool_package_onemore.py +++ b/test/functional/mempool_package_onemore.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test descendant package tracking carve-out allowing one final transaction in diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index 872c178103..068fdc0b65 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test descendant package tracking code.""" @@ -187,7 +187,6 @@ class MempoolPackagesTest(BitcoinTestFramework): # Check that prioritising a tx before it's added to the mempool works # First clear the mempool by mining a block. self.generate(self.nodes[0], 1) - self.sync_blocks() assert_equal(len(self.nodes[0].getrawmempool()), 0) # Prioritise a transaction that has been mined, then add it back to the # mempool by using invalidateblock. @@ -276,7 +275,6 @@ class MempoolPackagesTest(BitcoinTestFramework): # Test reorg handling # First, the basics: self.generate(self.nodes[0], 1) - self.sync_blocks() self.nodes[1].invalidateblock(self.nodes[0].getbestblockhash()) self.nodes[1].reconsiderblock(self.nodes[0].getbestblockhash()) @@ -323,7 +321,6 @@ class MempoolPackagesTest(BitcoinTestFramework): # Mine these in a block self.generate(self.nodes[0], 1) - self.sync_all() # Now generate tx8, with a big fee inputs = [ {'txid' : tx1_id, 'vout': 0}, {'txid' : txid, 'vout': 0} ] diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index 71a132dca3..8c9379b90b 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool persistence. @@ -149,8 +149,9 @@ class MempoolPersistTest(BitcoinTestFramework): mempooldat1 = os.path.join(self.nodes[1].datadir, self.chain, 'mempool.dat') self.log.debug("Remove the mempool.dat file. Verify that savemempool to disk via RPC re-creates it") os.remove(mempooldat0) - self.nodes[0].savemempool() + result0 = self.nodes[0].savemempool() assert os.path.isfile(mempooldat0) + assert_equal(result0['filename'], mempooldat0) self.log.debug("Stop nodes, make node1 use mempool.dat from node0. Verify it has 6 transactions") os.rename(mempooldat0, mempooldat1) @@ -174,7 +175,7 @@ class MempoolPersistTest(BitcoinTestFramework): self.start_node(0) # clear out mempool - self.generate(node0, 1) + self.generate(node0, 1, sync_fun=self.no_op) # ensure node0 doesn't have any connections # make a transaction that will remain in the unbroadcast set diff --git a/test/functional/mempool_reorg.py b/test/functional/mempool_reorg.py index 260b41ef12..7e940fa3ca 100755 --- a/test/functional/mempool_reorg.py +++ b/test/functional/mempool_reorg.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool re-org scenarios. @@ -79,9 +79,8 @@ class MempoolCoinbaseTest(BitcoinTestFramework): spend_3_1_id = self.nodes[0].sendrawtransaction(spend_3_1['hex']) self.log.info("Generate a block") last_block = self.generate(self.nodes[0], 1) - # Sync blocks, so that peer 1 gets the block before timelock_tx + # generate() implicitly syncs blocks, so that peer 1 gets the block before timelock_tx # Otherwise, peer 1 would put the timelock_tx in m_recent_rejects - self.sync_all() self.log.info("The time-locked transaction can now be spent") timelock_tx_id = self.nodes[0].sendrawtransaction(timelock_tx) diff --git a/test/functional/mempool_resurrect.py b/test/functional/mempool_resurrect.py index 4fce07dad3..3e610d02ac 100755 --- a/test/functional/mempool_resurrect.py +++ b/test/functional/mempool_resurrect.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test resurrection of mined transactions when the blockchain is re-organized.""" diff --git a/test/functional/mempool_spend_coinbase.py b/test/functional/mempool_spend_coinbase.py index 4e1dd80ba7..5afa6be925 100755 --- a/test/functional/mempool_spend_coinbase.py +++ b/test/functional/mempool_spend_coinbase.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test spending coinbase transactions. diff --git a/test/functional/mempool_unbroadcast.py b/test/functional/mempool_unbroadcast.py index 4d6379fe86..adf7326dac 100755 --- a/test/functional/mempool_unbroadcast.py +++ b/test/functional/mempool_unbroadcast.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that the mempool ensures transaction delivery by periodically sending @@ -109,7 +109,8 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): # a block removal_reason = "Removed {} from set of unbroadcast txns before confirmation that txn was sent out".format(txhsh) with node.assert_debug_log([removal_reason]): - self.generate(node, 1) + self.generate(node, 1, sync_fun=self.no_op) + if __name__ == "__main__": MempoolUnbroadcastTest().main() diff --git a/test/functional/mempool_updatefromblock.py b/test/functional/mempool_updatefromblock.py index 22f136d1a5..16c15e3f74 100755 --- a/test/functional/mempool_updatefromblock.py +++ b/test/functional/mempool_updatefromblock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mempool descendants/ancestors information update. diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index f141d201eb..f8d002e664 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test mining RPCs @@ -58,7 +58,7 @@ class MiningTest(BitcoinTestFramework): self.log.info('Create some old blocks') for t in range(TIME_GENESIS_BLOCK, TIME_GENESIS_BLOCK + 200 * 600, 600): self.nodes[0].setmocktime(t) - self.generate(self.nodes[0], 1) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) mining_info = self.nodes[0].getmininginfo() assert_equal(mining_info['blocks'], 200) assert_equal(mining_info['currentblocktx'], 0) diff --git a/test/functional/mining_getblocktemplate_longpoll.py b/test/functional/mining_getblocktemplate_longpoll.py index 0879fb9f2d..e928ee4936 100755 --- a/test/functional/mining_getblocktemplate_longpoll.py +++ b/test/functional/mining_getblocktemplate_longpoll.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test longpolling with getblocktemplate.""" @@ -64,7 +64,6 @@ class GetBlockTemplateLPTest(BitcoinTestFramework): # Add enough mature utxos to the wallets, so that all txs spend confirmed coins self.generate(self.nodes[0], COINBASE_MATURITY) - self.sync_blocks() self.log.info("Test that introducing a new transaction into the mempool will terminate the longpoll") thr = LongpollThread(self.nodes[0]) diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index 35274d3500..6f2ac805a0 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the prioritisetransaction mining RPC.""" @@ -105,7 +105,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): # the other high fee transactions. Keep mining until our mempool has # decreased by all the high fee size that we calculated above. while (self.nodes[0].getmempoolinfo()['bytes'] > sizes[0] + sizes[1]): - self.generate(self.nodes[0], 1) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) # High fee transaction should not have been mined, but other high fee rate # transactions should have been. diff --git a/test/functional/mocks/signer.py b/test/functional/mocks/signer.py index 676d0a0a4d..b732b26a53 100755 --- a/test/functional/mocks/signer.py +++ b/test/functional/mocks/signer.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,7 +18,7 @@ def perform_pre_checks(): sys.exit(int(mock_result[0])) def enumerate(args): - sys.stdout.write(json.dumps([{"fingerprint": "00000001", "type": "trezor", "model": "trezor_t"}, {"fingerprint": "00000002"}])) + sys.stdout.write(json.dumps([{"fingerprint": "00000001", "type": "trezor", "model": "trezor_t"}, {"fingerprint": "00000002"}])) def getdescriptors(args): xpub = "tpubD6NzVbkrYhZ4WaWSyoBvQwbpLkojyoTZPRsgXELWz3Popb3qkjcJyJUGLnL4qHHoQvao8ESaAstxYSnhyswJ76uZPStJRJCTKvosUCJZL5B" @@ -93,7 +93,7 @@ parser_signtx.set_defaults(func=signtx) if not sys.stdin.isatty(): buffer = sys.stdin.read() if buffer and buffer.rstrip() != "": - sys.argv.extend(buffer.rstrip().split(" ")) + sys.argv.extend(buffer.rstrip().split(" ")) args = parser.parse_args() diff --git a/test/functional/p2p_add_connections.py b/test/functional/p2p_add_connections.py index a04ba5db2d..b86502dc85 100755 --- a/test/functional/p2p_add_connections.py +++ b/test/functional/p2p_add_connections.py @@ -6,13 +6,7 @@ from test_framework.p2p import P2PInterface from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal - - -def check_node_connections(*, node, num_in, num_out): - info = node.getnetworkinfo() - assert_equal(info["connections_in"], num_in) - assert_equal(info["connections_out"], num_out) +from test_framework.util import check_node_connections class P2PAddConnections(BitcoinTestFramework): diff --git a/test/functional/p2p_addr_relay.py b/test/functional/p2p_addr_relay.py index 9df74ad3a0..5532056dbe 100755 --- a/test/functional/p2p_addr_relay.py +++ b/test/functional/p2p_addr_relay.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ diff --git a/test/functional/p2p_addrv2_relay.py b/test/functional/p2p_addrv2_relay.py index f4be893d2c..9ab190871f 100755 --- a/test/functional/p2p_addrv2_relay.py +++ b/test/functional/p2p_addrv2_relay.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ diff --git a/test/functional/p2p_blockfilters.py b/test/functional/p2p_blockfilters.py index 3a4fcc4549..e73fad439f 100755 --- a/test/functional/p2p_blockfilters.py +++ b/test/functional/p2p_blockfilters.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests NODE_COMPACT_FILTERS (BIP 157/158). @@ -57,16 +57,15 @@ class CompactFiltersTest(BitcoinTestFramework): # Nodes 0 & 1 share the same first 999 blocks in the chain. self.generate(self.nodes[0], 999) - self.sync_blocks(timeout=600) # Stale blocks by disconnecting nodes 0 & 1, mining, then reconnecting self.disconnect_nodes(0, 1) - stale_block_hash = self.generate(self.nodes[0], 1)[0] + stale_block_hash = self.generate(self.nodes[0], 1, sync_fun=self.no_op)[0] self.nodes[0].syncwithvalidationinterfacequeue() assert_equal(self.nodes[0].getblockcount(), 1000) - self.generate(self.nodes[1], 1001) + self.generate(self.nodes[1], 1001, sync_fun=self.no_op) assert_equal(self.nodes[1].getblockcount(), 2000) # Check that nodes have signalled NODE_COMPACT_FILTERS correctly. diff --git a/test/functional/p2p_blocksonly.py b/test/functional/p2p_blocksonly.py index 94ae758d46..6e48341259 100755 --- a/test/functional/p2p_blocksonly.py +++ b/test/functional/p2p_blocksonly.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test p2p blocksonly mode & block-relay-only connections.""" diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index 3f01d552b2..a314e8edfd 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2020 The Bitcoin Core developers +# Copyright (c) 2016-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test compact blocks (BIP 152). diff --git a/test/functional/p2p_compactblocks_blocksonly.py b/test/functional/p2p_compactblocks_blocksonly.py index 5f01fa4dfe..6367eb26a3 100755 --- a/test/functional/p2p_compactblocks_blocksonly.py +++ b/test/functional/p2p_compactblocks_blocksonly.py @@ -33,7 +33,7 @@ class P2PCompactBlocksBlocksOnly(BitcoinTestFramework): self.sync_all() def build_block_on_tip(self): - blockhash = self.generate(self.nodes[2], 1)[0] + blockhash = self.generate(self.nodes[2], 1, sync_fun=self.no_op)[0] block_hex = self.nodes[2].getblock(blockhash=blockhash, verbosity=0) block = from_hex(CBlock(), block_hex) block.rehash() diff --git a/test/functional/p2p_compactblocks_hb.py b/test/functional/p2p_compactblocks_hb.py index 72b3897b4f..c985a1f98d 100755 --- a/test/functional/p2p_compactblocks_hb.py +++ b/test/functional/p2p_compactblocks_hb.py @@ -31,7 +31,6 @@ class CompactBlocksConnectionTest(BitcoinTestFramework): """Relay a new block through peer peer, and return HB status between 1 and [2,3,4,5].""" self.connect_nodes(peer, 0) self.generate(self.nodes[0], 1) - self.sync_blocks() self.disconnect_nodes(peer, 0) status_to = [self.peer_info(1, i)['bip152_hb_to'] for i in range(2, 6)] status_from = [self.peer_info(i, 1)['bip152_hb_from'] for i in range(2, 6)] @@ -45,7 +44,6 @@ class CompactBlocksConnectionTest(BitcoinTestFramework): for i in range(1, 6): self.connect_nodes(i, 0) self.generate(self.nodes[0], 2) - self.sync_blocks() for i in range(1, 6): self.disconnect_nodes(i, 0) diff --git a/test/functional/p2p_dos_header_tree.py b/test/functional/p2p_dos_header_tree.py index 52a47c9bc2..fde1e4bfa2 100755 --- a/test/functional/p2p_dos_header_tree.py +++ b/test/functional/p2p_dos_header_tree.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that we reject low difficulty headers to prevent our block tree from filling up with useless bloat""" diff --git a/test/functional/p2p_eviction.py b/test/functional/p2p_eviction.py index 4ccc942164..1f4797a89d 100755 --- a/test/functional/p2p_eviction.py +++ b/test/functional/p2p_eviction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/functional/p2p_feefilter.py b/test/functional/p2p_feefilter.py index 60adc2c7fa..b65e927d5b 100755 --- a/test/functional/p2p_feefilter.py +++ b/test/functional/p2p_feefilter.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2020 The Bitcoin Core developers +# Copyright (c) 2016-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of feefilter messages.""" diff --git a/test/functional/p2p_filter.py b/test/functional/p2p_filter.py index 0d8c298bea..2192363a89 100755 --- a/test/functional/p2p_filter.py +++ b/test/functional/p2p_filter.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ diff --git a/test/functional/p2p_fingerprint.py b/test/functional/p2p_fingerprint.py index 2962dc8085..e8bba8555f 100755 --- a/test/functional/p2p_fingerprint.py +++ b/test/functional/p2p_fingerprint.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test various fingerprinting protections. diff --git a/test/functional/p2p_ibd_txrelay.py b/test/functional/p2p_ibd_txrelay.py index c35053d9d4..65a94ad31c 100755 --- a/test/functional/p2p_ibd_txrelay.py +++ b/test/functional/p2p_ibd_txrelay.py @@ -1,12 +1,31 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test fee filters during and after IBD.""" +"""Test transaction relay behavior during IBD: +- Set fee filters to MAX_MONEY +- Don't request transactions +- Ignore all transaction messages +""" from decimal import Decimal +import time -from test_framework.messages import COIN +from test_framework.messages import ( + CInv, + COIN, + CTransaction, + from_hex, + msg_inv, + msg_tx, + MSG_WTX, +) +from test_framework.p2p import ( + NONPREF_PEER_TX_DELAY, + P2PDataStore, + P2PInterface, + p2p_lock +) from test_framework.test_framework import BitcoinTestFramework MAX_FEE_FILTER = Decimal(9170997) / COIN @@ -28,15 +47,43 @@ class P2PIBDTxRelayTest(BitcoinTestFramework): assert node.getblockchaininfo()['initialblockdownload'] self.wait_until(lambda: all(peer['minfeefilter'] == MAX_FEE_FILTER for peer in node.getpeerinfo())) + self.log.info("Check that nodes don't send getdatas for transactions while still in IBD") + peer_inver = self.nodes[0].add_p2p_connection(P2PDataStore()) + txid = 0xdeadbeef + peer_inver.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=txid)])) + # The node should not send a getdata, but if it did, it would first delay 2 seconds + self.nodes[0].setmocktime(int(time.time() + NONPREF_PEER_TX_DELAY)) + peer_inver.sync_send_with_ping() + with p2p_lock: + assert txid not in peer_inver.getdata_requests + self.nodes[0].disconnect_p2ps() + + self.log.info("Check that nodes don't process unsolicited transactions while still in IBD") + # A transaction hex pulled from tx_valid.json. There are no valid transactions since no UTXOs + # exist yet, but it should be a well-formed transaction. + rawhex = "0100000001b14bdcbc3e01bdaad36cc08e81e69c82e1060bc14e518db2b49aa43ad90ba260000000004a01ff473" + \ + "04402203f16c6f40162ab686621ef3000b04e75418a0c0cb2d8aebeac894ae360ac1e780220ddc15ecdfc3507ac48e168" + \ + "1a33eb60996631bf6bf5bc0a0682c4db743ce7ca2b01ffffffff0140420f00000000001976a914660d4ef3a743e3e696a" + \ + "d990364e555c271ad504b88ac00000000" + assert self.nodes[1].decoderawtransaction(rawhex) # returns a dict, should not throw + tx = from_hex(CTransaction(), rawhex) + peer_txer = self.nodes[0].add_p2p_connection(P2PInterface()) + with self.nodes[0].assert_debug_log(expected_msgs=["received: tx"], unexpected_msgs=["was not accepted"]): + peer_txer.send_and_ping(msg_tx(tx)) + self.nodes[0].disconnect_p2ps() + # Come out of IBD by generating a block self.generate(self.nodes[0], 1) - self.sync_all() self.log.info("Check that nodes reset minfilter after coming out of IBD") for node in self.nodes: assert not node.getblockchaininfo()['initialblockdownload'] self.wait_until(lambda: all(peer['minfeefilter'] == NORMAL_FEE_FILTER for peer in node.getpeerinfo())) + self.log.info("Check that nodes process the same transaction, even when unsolicited, when no longer in IBD") + peer_txer = self.nodes[0].add_p2p_connection(P2PInterface()) + with self.nodes[0].assert_debug_log(expected_msgs=["was not accepted"]): + peer_txer.send_and_ping(msg_tx(tx)) if __name__ == '__main__': P2PIBDTxRelayTest().main() diff --git a/test/functional/p2p_invalid_block.py b/test/functional/p2p_invalid_block.py index 875ab52db4..710f3d93e1 100755 --- a/test/functional/p2p_invalid_block.py +++ b/test/functional/p2p_invalid_block.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid blocks. @@ -15,14 +15,18 @@ becomes valid. import copy import time -from test_framework.blocktools import create_block, create_coinbase, create_tx_with_script +from test_framework.blocktools import ( + MAX_FUTURE_BLOCK_TIME, + create_block, + create_coinbase, + create_tx_with_script, +) from test_framework.messages import COIN from test_framework.p2p import P2PDataStore +from test_framework.script import OP_TRUE from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal -MAX_FUTURE_BLOCK_TIME = 2 * 60 * 60 - class InvalidBlockRequestTest(BitcoinTestFramework): def set_test_params(self): @@ -66,16 +70,10 @@ class InvalidBlockRequestTest(BitcoinTestFramework): # For more information on merkle-root malleability see src/consensus/merkle.cpp. self.log.info("Test merkle root malleability.") - block2 = create_block(tip, create_coinbase(height), block_time) + tx1 = create_tx_with_script(block1.vtx[0], 0, script_sig=bytes([OP_TRUE]), amount=50 * COIN) + tx2 = create_tx_with_script(tx1, 0, script_sig=bytes([OP_TRUE]), amount=50 * COIN) + block2 = create_block(tip, create_coinbase(height), block_time, txlist=[tx1, tx2]) block_time += 1 - - # b'0x51' is OP_TRUE - tx1 = create_tx_with_script(block1.vtx[0], 0, script_sig=b'\x51', amount=50 * COIN) - tx2 = create_tx_with_script(tx1, 0, script_sig=b'\x51', amount=50 * COIN) - - block2.vtx.extend([tx1, tx2]) - block2.hashMerkleRoot = block2.calc_merkle_root() - block2.rehash() block2.solve() orig_hash = block2.sha256 block2_orig = copy.deepcopy(block2) @@ -95,19 +93,13 @@ class InvalidBlockRequestTest(BitcoinTestFramework): block2_dup.vtx[2].vin.append(block2_dup.vtx[2].vin[0]) block2_dup.vtx[2].rehash() block2_dup.hashMerkleRoot = block2_dup.calc_merkle_root() - block2_dup.rehash() block2_dup.solve() peer.send_blocks_and_test([block2_dup], node, success=False, reject_reason='bad-txns-inputs-duplicate') self.log.info("Test very broken block.") - block3 = create_block(tip, create_coinbase(height), block_time) + block3 = create_block(tip, create_coinbase(height, nValue=100), block_time) block_time += 1 - block3.vtx[0].vout[0].nValue = 100 * COIN # Too high! - block3.vtx[0].sha256 = None - block3.vtx[0].calc_sha256() - block3.hashMerkleRoot = block3.calc_merkle_root() - block3.rehash() block3.solve() peer.send_blocks_and_test([block3], node, success=False, reject_reason='bad-cb-amount') @@ -126,15 +118,10 @@ class InvalidBlockRequestTest(BitcoinTestFramework): # Complete testing of CVE-2018-17144, by checking for the inflation bug. # Create a block that spends the output of a tx in a previous block. - block4 = create_block(tip, create_coinbase(height), block_time) - tx3 = create_tx_with_script(tx2, 0, script_sig=b'\x51', amount=50 * COIN) - - # Duplicates input - tx3.vin.append(tx3.vin[0]) + tx3 = create_tx_with_script(tx2, 0, script_sig=bytes([OP_TRUE]), amount=50 * COIN) + tx3.vin.append(tx3.vin[0]) # Duplicates input tx3.rehash() - block4.vtx.append(tx3) - block4.hashMerkleRoot = block4.calc_merkle_root() - block4.rehash() + block4 = create_block(tip, create_coinbase(height), block_time, txlist=[tx3]) block4.solve() self.log.info("Test inflation by duplicating input") peer.send_blocks_and_test([block4], node, success=False, reject_reason='bad-txns-inputs-duplicate') @@ -144,7 +131,6 @@ class InvalidBlockRequestTest(BitcoinTestFramework): node.setmocktime(t) # Set block time +1 second past max future validity block = create_block(tip, create_coinbase(height), t + MAX_FUTURE_BLOCK_TIME + 1) - block.hashMerkleRoot = block.calc_merkle_root() block.solve() # Need force_send because the block will get rejected without a getdata otherwise peer.send_blocks_and_test([block], node, force_send=True, success=False, reject_reason='time-too-new') diff --git a/test/functional/p2p_invalid_locator.py b/test/functional/p2p_invalid_locator.py index a586b48d4c..626422370a 100755 --- a/test/functional/p2p_invalid_locator.py +++ b/test/functional/p2p_invalid_locator.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid locators. diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index 82c7e94c59..3109ad2b56 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid network messages.""" diff --git a/test/functional/p2p_invalid_tx.py b/test/functional/p2p_invalid_tx.py index 0a3ae23f58..139f4d64e7 100755 --- a/test/functional/p2p_invalid_tx.py +++ b/test/functional/p2p_invalid_tx.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test node responses to invalid transactions. diff --git a/test/functional/p2p_leak.py b/test/functional/p2p_leak.py index de58e07aad..af8e45d578 100755 --- a/test/functional/p2p_leak.py +++ b/test/functional/p2p_leak.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test message sending before handshake completion. diff --git a/test/functional/p2p_leak_tx.py b/test/functional/p2p_leak_tx.py index 9b80e1b877..4c064b359e 100755 --- a/test/functional/p2p_leak_tx.py +++ b/test/functional/p2p_leak_tx.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that we don't leak txs to inbound peers that we haven't yet announced to""" diff --git a/test/functional/p2p_message_capture.py b/test/functional/p2p_message_capture.py index 080b2d93ad..edde9a6ecf 100755 --- a/test/functional/p2p_message_capture.py +++ b/test/functional/p2p_message_capture.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test per-peer message capture capability. diff --git a/test/functional/p2p_node_network_limited.py b/test/functional/p2p_node_network_limited.py index e491fe7e07..5a0003d3ef 100755 --- a/test/functional/p2p_node_network_limited.py +++ b/test/functional/p2p_node_network_limited.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Tests NODE_NETWORK_LIMITED. @@ -59,8 +59,7 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): self.log.info("Mine enough blocks to reach the NODE_NETWORK_LIMITED range.") self.connect_nodes(0, 1) - blocks = self.generatetoaddress(self.nodes[1], 292, self.nodes[1].get_deterministic_priv_key().address) - self.sync_blocks([self.nodes[0], self.nodes[1]]) + blocks = self.generate(self.nodes[1], 292, sync_fun=lambda: self.sync_blocks([self.nodes[0], self.nodes[1]])) self.log.info("Make sure we can max retrieve block at tip-288.") node.send_getdata_for_block(blocks[1]) # last block in valid range @@ -101,7 +100,7 @@ class NodeNetworkLimitedTest(BitcoinTestFramework): self.disconnect_all() # mine 10 blocks on node 0 (pruned node) - self.generatetoaddress(self.nodes[0], 10, self.nodes[0].get_deterministic_priv_key().address) + self.generate(self.nodes[0], 10, sync_fun=self.no_op) # connect node1 (non pruned) with node0 (pruned) and check if the can sync self.connect_nodes(0, 1) diff --git a/test/functional/p2p_permissions.py b/test/functional/p2p_permissions.py index 32f2ea14e1..1dc3a5b9a0 100755 --- a/test/functional/p2p_permissions.py +++ b/test/functional/p2p_permissions.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test p2p permission message. @@ -94,7 +94,6 @@ class P2PPermissionsTests(BitcoinTestFramework): def check_tx_relay(self): block_op_true = self.nodes[0].getblock(self.generatetoaddress(self.nodes[0], 100, ADDRESS_BCRT1_P2WSH_OP_TRUE)[0]) - self.sync_all() self.log.debug("Create a connection from a forcerelay peer that rebroadcasts raw txs") # A test framework p2p connection is needed to send the raw transaction directly. If a full node was used, it could only diff --git a/test/functional/p2p_ping.py b/test/functional/p2p_ping.py index d67e97acf7..52dae90d19 100755 --- a/test/functional/p2p_ping.py +++ b/test/functional/p2p_ping.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test ping message diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 5195c9fd04..f377fbaaa6 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2020 The Bitcoin Core developers +# Copyright (c) 2016-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test segwit transactions and blocks on P2P network.""" @@ -101,6 +101,23 @@ class UTXO(): self.n = n self.nValue = value + +def subtest(func): + """Wraps the subtests for logging and state assertions.""" + def func_wrapper(self, *args, **kwargs): + self.log.info("Subtest: {} (Segwit active = {})".format(func.__name__, self.segwit_active)) + # Assert segwit status is as expected + assert_equal(softfork_active(self.nodes[0], 'segwit'), self.segwit_active) + func(self, *args, **kwargs) + # Each subtest should leave some utxos for the next subtest + assert self.utxo + self.sync_blocks() + # Assert segwit status is as expected at end of subtest + assert_equal(softfork_active(self.nodes[0], 'segwit'), self.segwit_active) + + return func_wrapper + + def sign_p2pk_witness_input(script, tx_to, in_idx, hashtype, value, key): """Add signature for a P2PK witness script.""" tx_hash = SegwitV0SignatureHash(script, tx_to, in_idx, hashtype, value) @@ -215,7 +232,6 @@ class SegWitTest(BitcoinTestFramework): height = self.nodes[0].getblockcount() + 1 block_time = self.nodes[0].getblockheader(tip)["mediantime"] + 1 block = create_block(int(tip, 16), create_coinbase(height), block_time) - block.nVersion = 4 block.rehash() return block @@ -280,22 +296,7 @@ class SegWitTest(BitcoinTestFramework): # Individual tests - def subtest(func): # noqa: N805 - """Wraps the subtests for logging and state assertions.""" - def func_wrapper(self, *args, **kwargs): - self.log.info("Subtest: {} (Segwit active = {})".format(func.__name__, self.segwit_active)) - # Assert segwit status is as expected - assert_equal(softfork_active(self.nodes[0], 'segwit'), self.segwit_active) - func(self, *args, **kwargs) - # Each subtest should leave some utxos for the next subtest - assert self.utxo - self.sync_blocks() - # Assert segwit status is as expected at end of subtest - assert_equal(softfork_active(self.nodes[0], 'segwit'), self.segwit_active) - - return func_wrapper - - @subtest # type: ignore + @subtest def test_non_witness_transaction(self): """See if sending a regular transaction works, and create a utxo to use in later tests.""" # Mine a block with an anyone-can-spend coinbase, @@ -324,7 +325,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.append(UTXO(tx.sha256, 0, 49 * 100000000)) self.generate(self.nodes[0], 1) - @subtest # type: ignore + @subtest def test_unnecessary_witness_before_segwit_activation(self): """Verify that blocks with witnesses are rejected before activation.""" @@ -355,7 +356,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(tx.sha256, 0, tx.vout[0].nValue)) - @subtest # type: ignore + @subtest def test_block_relay(self): """Test that block requests to NODE_WITNESS peer are with MSG_WITNESS_FLAG. @@ -444,7 +445,7 @@ class SegWitTest(BitcoinTestFramework): self.old_node.announce_tx_and_wait_for_getdata(block4.vtx[0]) assert block4.sha256 not in self.old_node.getdataset - @subtest # type: ignore + @subtest def test_v0_outputs_arent_spendable(self): """Test that v0 outputs aren't spendable before segwit activation. @@ -516,7 +517,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(txid, 2, value)) - @subtest # type: ignore + @subtest def test_witness_tx_relay_before_segwit_activation(self): # Generate a transaction that doesn't require a witness, but send it @@ -558,7 +559,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(tx_hash, 0, tx_value)) - @subtest # type: ignore + @subtest def test_standardness_v0(self): """Test V0 txout standardness. @@ -578,7 +579,6 @@ class SegWitTest(BitcoinTestFramework): # Mine it on test_node to create the confirmed output. test_transaction_acceptance(self.nodes[0], self.test_node, p2sh_tx, with_witness=True, accepted=True) self.generate(self.nodes[0], 1) - self.sync_blocks() # Now test standardness of v0 P2WSH outputs. # Start by creating a transaction with two outputs. @@ -651,12 +651,11 @@ class SegWitTest(BitcoinTestFramework): test_transaction_acceptance(self.nodes[0], self.test_node, tx3, with_witness=True, accepted=True) self.generate(self.nodes[0], 1) - self.sync_blocks() self.utxo.pop(0) self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) assert_equal(len(self.nodes[1].getrawmempool()), 0) - @subtest # type: ignore + @subtest def advance_to_segwit_active(self): """Mine enough blocks to activate segwit.""" assert not softfork_active(self.nodes[0], 'segwit') @@ -667,7 +666,7 @@ class SegWitTest(BitcoinTestFramework): assert softfork_active(self.nodes[0], 'segwit') self.segwit_active = True - @subtest # type: ignore + @subtest def test_p2sh_witness(self): """Test P2SH wrapped witness programs.""" @@ -734,7 +733,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(spend_tx.sha256, 0, spend_tx.vout[0].nValue)) - @subtest # type: ignore + @subtest def test_witness_commitments(self): """Test witness commitments. @@ -788,7 +787,6 @@ class SegWitTest(BitcoinTestFramework): block_3.vtx[0].vout.append(CTxOut(0, CScript([OP_RETURN, WITNESS_COMMITMENT_HEADER + ser_uint256(2), 10]))) block_3.vtx[0].rehash() block_3.hashMerkleRoot = block_3.calc_merkle_root() - block_3.rehash() block_3.solve() test_witness_block(self.nodes[0], self.test_node, block_3, accepted=False, reason='bad-witness-merkle-match') @@ -802,7 +800,6 @@ class SegWitTest(BitcoinTestFramework): block_3.vtx[0].vout[-1].nValue += 1 block_3.vtx[0].rehash() block_3.hashMerkleRoot = block_3.calc_merkle_root() - block_3.rehash() assert len(block_3.vtx[0].vout) == 4 # 3 OP_returns block_3.solve() test_witness_block(self.nodes[0], self.test_node, block_3, accepted=True) @@ -823,7 +820,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) - @subtest # type: ignore + @subtest def test_block_malleability(self): # Make sure that a block that has too big a virtual size @@ -863,7 +860,7 @@ class SegWitTest(BitcoinTestFramework): block.vtx[0].wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(0)] test_witness_block(self.nodes[0], self.test_node, block, accepted=True) - @subtest # type: ignore + @subtest def test_witness_block_size(self): # TODO: Test that non-witness carrying blocks can't exceed 1MB # Skipping this test for now; this is covered in feature_block.py @@ -938,7 +935,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(block.vtx[-1].sha256, 0, block.vtx[-1].vout[0].nValue)) - @subtest # type: ignore + @subtest def test_submit_block(self): """Test that submitblock adds the nonce automatically when possible.""" block = self.build_next_block() @@ -974,7 +971,7 @@ class SegWitTest(BitcoinTestFramework): # Tip should not advance! assert self.nodes[0].getbestblockhash() != block_2.hash - @subtest # type: ignore + @subtest def test_extra_witness_data(self): """Test extra witness data in a transaction.""" @@ -1049,7 +1046,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) - @subtest # type: ignore + @subtest def test_max_witness_push_length(self): """Test that witness stack can only allow up to 520 byte pushes.""" @@ -1086,7 +1083,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop() self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) - @subtest # type: ignore + @subtest def test_max_witness_script_length(self): """Test that witness outputs greater than 10kB can't be spent.""" @@ -1133,7 +1130,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop() self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) - @subtest # type: ignore + @subtest def test_witness_input_length(self): """Test that vin length must match vtxinwit length.""" @@ -1217,7 +1214,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop() self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) - @subtest # type: ignore + @subtest def test_tx_relay_after_segwit_activation(self): """Test transaction relay after segwit activation. @@ -1309,7 +1306,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) - @subtest # type: ignore + @subtest def test_segwit_versions(self): """Test validity of future segwit version transactions. @@ -1354,7 +1351,6 @@ class SegWitTest(BitcoinTestFramework): temp_utxo.append(UTXO(tx.sha256, 0, tx.vout[0].nValue)) self.generate(self.nodes[0], 1) # Mine all the transactions - self.sync_blocks() assert len(self.nodes[0].getrawmempool()) == 0 # Finally, verify that version 0 -> version 2 transactions @@ -1403,7 +1399,7 @@ class SegWitTest(BitcoinTestFramework): # Add utxo to our list self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) - @subtest # type: ignore + @subtest def test_premature_coinbase_witness_spend(self): block = self.build_next_block() @@ -1425,7 +1421,6 @@ class SegWitTest(BitcoinTestFramework): # Now test a premature spend. self.generate(self.nodes[0], 98) - self.sync_blocks() block2 = self.build_next_block() self.update_witness_block_with_transactions(block2, [spend_tx]) test_witness_block(self.nodes[0], self.test_node, block2, accepted=False, reason='bad-txns-premature-spend-of-coinbase') @@ -1437,7 +1432,7 @@ class SegWitTest(BitcoinTestFramework): test_witness_block(self.nodes[0], self.test_node, block2, accepted=True) self.sync_blocks() - @subtest # type: ignore + @subtest def test_uncompressed_pubkey(self): """Test uncompressed pubkey validity in segwit transactions. @@ -1540,7 +1535,7 @@ class SegWitTest(BitcoinTestFramework): test_witness_block(self.nodes[0], self.test_node, block, accepted=True) self.utxo.append(UTXO(tx5.sha256, 0, tx5.vout[0].nValue)) - @subtest # type: ignore + @subtest def test_signature_version_1(self): key = ECKey() @@ -1726,7 +1721,7 @@ class SegWitTest(BitcoinTestFramework): for i in range(len(tx.vout)): self.utxo.append(UTXO(tx.sha256, i, tx.vout[i].nValue)) - @subtest # type: ignore + @subtest def test_non_standard_witness_blinding(self): """Test behavior of unnecessary witnesses in transactions does not blind the node for the transaction""" @@ -1744,7 +1739,6 @@ class SegWitTest(BitcoinTestFramework): tx.rehash() test_transaction_acceptance(self.nodes[0], self.test_node, tx, False, True) self.generate(self.nodes[0], 1) - self.sync_blocks() # We'll add an unnecessary witness to this transaction that would cause # it to be non-standard, to test that violating policy with a witness @@ -1773,13 +1767,12 @@ class SegWitTest(BitcoinTestFramework): test_transaction_acceptance(self.nodes[0], self.test_node, tx3, False, True) self.generate(self.nodes[0], 1) - self.sync_blocks() # Update our utxo list; we spent the first entry. self.utxo.pop(0) self.utxo.append(UTXO(tx3.sha256, 0, tx3.vout[0].nValue)) - @subtest # type: ignore + @subtest def test_non_standard_witness(self): """Test detection of non-standard P2WSH witness""" pad = chr(1).encode('latin-1') @@ -1808,7 +1801,6 @@ class SegWitTest(BitcoinTestFramework): test_transaction_acceptance(self.nodes[0], self.test_node, tx, with_witness=False, accepted=True) self.generate(self.nodes[0], 1) - self.sync_blocks() # Creating transactions for tests p2wsh_txs = [] @@ -1878,7 +1870,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) - @subtest # type: ignore + @subtest def test_witness_sigops(self): """Test sigop counting is correct inside witnesses.""" @@ -1980,7 +1972,7 @@ class SegWitTest(BitcoinTestFramework): self.utxo.pop(0) self.utxo.append(UTXO(tx2.sha256, 0, tx2.vout[0].nValue)) - @subtest # type: ignore + @subtest def test_superfluous_witness(self): # Serialization of tx that puts witness flag to 3 always def serialize_with_bogus_witness(tx): @@ -2024,7 +2016,7 @@ class SegWitTest(BitcoinTestFramework): with self.nodes[0].assert_debug_log(['Unknown transaction optional data']): self.test_node.send_and_ping(msg_bogus_tx(tx)) - @subtest # type: ignore + @subtest def test_wtxid_relay(self): # Use brand new nodes to avoid contamination from earlier tests self.wtx_node = self.nodes[0].add_p2p_connection(TestP2PConn(wtxidrelay=True), services=P2P_SERVICES) diff --git a/test/functional/p2p_sendheaders.py b/test/functional/p2p_sendheaders.py index 7bf1803780..1ccc447b89 100755 --- a/test/functional/p2p_sendheaders.py +++ b/test/functional/p2p_sendheaders.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test behavior of headers messages to announce blocks. @@ -217,7 +217,6 @@ class SendHeadersTest(BitcoinTestFramework): # make sure all invalidated blocks are node0's self.generatetoaddress(self.nodes[0], length, self.nodes[0].get_deterministic_priv_key().address) - self.sync_blocks(self.nodes, wait=0.1) for x in self.nodes[0].p2ps: x.wait_for_block_announcement(int(self.nodes[0].getbestblockhash(), 16)) x.clear_block_announcements() @@ -226,7 +225,6 @@ class SendHeadersTest(BitcoinTestFramework): hash_to_invalidate = self.nodes[1].getblockhash(tip_height - (length - 1)) self.nodes[1].invalidateblock(hash_to_invalidate) all_hashes = self.generatetoaddress(self.nodes[1], length + 1, self.nodes[1].get_deterministic_priv_key().address) # Must be longer than the orig chain - self.sync_blocks(self.nodes, wait=0.1) return [int(x, 16) for x in all_hashes] def run_test(self): diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index 3e962b4450..7356b8bbb3 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ diff --git a/test/functional/p2p_unrequested_blocks.py b/test/functional/p2p_unrequested_blocks.py index a9d5ed970a..9c4e1dd1b1 100755 --- a/test/functional/p2p_unrequested_blocks.py +++ b/test/functional/p2p_unrequested_blocks.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test processing of unrequested blocks. @@ -77,7 +77,7 @@ class AcceptBlockTest(BitcoinTestFramework): min_work_node = self.nodes[1].add_p2p_connection(P2PInterface()) # 1. Have nodes mine a block (leave IBD) - [self.generatetoaddress(n, 1, n.get_deterministic_priv_key().address) for n in self.nodes] + [self.generate(n, 1, sync_fun=self.no_op) for n in self.nodes] tips = [int("0x" + n.getbestblockhash(), 0) for n in self.nodes] # 2. Send one block that builds on each tip. @@ -225,10 +225,9 @@ class AcceptBlockTest(BitcoinTestFramework): block_289f.solve() block_290f = create_block(block_289f.sha256, create_coinbase(290), block_289f.nTime+1) block_290f.solve() - block_291 = create_block(block_290f.sha256, create_coinbase(291), block_290f.nTime+1) # block_291 spends a coinbase below maturity! - block_291.vtx.append(create_tx_with_script(block_290f.vtx[0], 0, script_sig=b"42", amount=1)) - block_291.hashMerkleRoot = block_291.calc_merkle_root() + tx_to_add = create_tx_with_script(block_290f.vtx[0], 0, script_sig=b"42", amount=1) + block_291 = create_block(block_290f.sha256, create_coinbase(291), block_290f.nTime+1, txlist=[tx_to_add]) block_291.solve() block_292 = create_block(block_291.sha256, create_coinbase(292), block_291.nTime+1) block_292.solve() diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index eea9ee26cb..8f7d5114fa 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPCs related to blockchainstate. @@ -25,11 +25,11 @@ import http.client import os import subprocess -from test_framework.address import ADDRESS_BCRT1_P2WSH_OP_TRUE from test_framework.blocktools import ( + MAX_FUTURE_BLOCK_TIME, + TIME_GENESIS_BLOCK, create_block, create_coinbase, - TIME_GENESIS_BLOCK, ) from test_framework.messages import ( CBlockHeader, @@ -54,6 +54,7 @@ from test_framework.wallet import MiniWallet HEIGHT = 200 # blocks mined TIME_RANGE_STEP = 600 # ten-minute steps TIME_RANGE_MTP = TIME_GENESIS_BLOCK + (HEIGHT - 6) * TIME_RANGE_STEP +TIME_RANGE_TIP = TIME_GENESIS_BLOCK + (HEIGHT - 1) * TIME_RANGE_STEP TIME_RANGE_END = TIME_GENESIS_BLOCK + HEIGHT * TIME_RANGE_STEP @@ -64,7 +65,9 @@ class BlockchainTest(BitcoinTestFramework): self.supports_cli = False def run_test(self): + self.wallet = MiniWallet(self.nodes[0]) self.mine_chain() + self._test_max_future_block_time() self.restart_node(0, extra_args=['-stopatheight=207', '-prune=1']) # Set extra args with pruning after rescan is complete self._test_getblockchaininfo() @@ -82,9 +85,22 @@ class BlockchainTest(BitcoinTestFramework): self.log.info(f"Generate {HEIGHT} blocks after the genesis block in ten-minute steps") for t in range(TIME_GENESIS_BLOCK, TIME_RANGE_END, TIME_RANGE_STEP): self.nodes[0].setmocktime(t) - self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_P2WSH_OP_TRUE) + self.generate(self.wallet, 1) assert_equal(self.nodes[0].getblockchaininfo()['blocks'], HEIGHT) + def _test_max_future_block_time(self): + self.stop_node(0) + self.log.info("A block tip of more than MAX_FUTURE_BLOCK_TIME in the future raises an error") + self.nodes[0].assert_start_raises_init_error( + extra_args=[f"-mocktime={TIME_RANGE_TIP - MAX_FUTURE_BLOCK_TIME - 1}"], + expected_msg=": The block database contains a block which appears to be from the future." + " This may be due to your computer's date and time being set incorrectly." + f" Only rebuild the block database if you are sure that your computer's date and time are correct.{os.linesep}" + "Please restart with -reindex or -reindex-chainstate to recover.", + ) + self.log.info("A block tip of MAX_FUTURE_BLOCK_TIME in the future is fine") + self.start_node(0, extra_args=[f"-mocktime={TIME_RANGE_TIP - MAX_FUTURE_BLOCK_TIME}"]) + def _test_getblockchaininfo(self): self.log.info("Test getblockchaininfo") @@ -371,12 +387,12 @@ class BlockchainTest(BitcoinTestFramework): def _test_stopatheight(self): self.log.info("Test stopping at height") assert_equal(self.nodes[0].getblockcount(), HEIGHT) - self.generatetoaddress(self.nodes[0], 6, ADDRESS_BCRT1_P2WSH_OP_TRUE) + self.generate(self.wallet, 6) assert_equal(self.nodes[0].getblockcount(), HEIGHT + 6) self.log.debug('Node should not stop at this height') assert_raises(subprocess.TimeoutExpired, lambda: self.nodes[0].process.wait(timeout=3)) try: - self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_P2WSH_OP_TRUE) + self.generatetoaddress(self.nodes[0], 1, self.wallet.get_address(), sync_fun=self.no_op) except (ConnectionError, http.client.BadStatusLine): pass # The node already shut down before response self.log.debug('Node should stop at this height...') @@ -424,14 +440,10 @@ class BlockchainTest(BitcoinTestFramework): def _test_getblock(self): node = self.nodes[0] - - miniwallet = MiniWallet(node) - miniwallet.rescan_utxos() - fee_per_byte = Decimal('0.00000010') fee_per_kb = 1000 * fee_per_byte - miniwallet.send_self_transfer(fee_rate=fee_per_kb, from_node=node) + self.wallet.send_self_transfer(fee_rate=fee_per_kb, from_node=node) blockhash = self.generate(node, 1)[0] def assert_fee_not_in_block(verbosity): diff --git a/test/functional/rpc_createmultisig.py b/test/functional/rpc_createmultisig.py index 696438ccfe..f56e71ae7a 100755 --- a/test/functional/rpc_createmultisig.py +++ b/test/functional/rpc_createmultisig.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test multisig RPCs""" @@ -46,7 +46,6 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): self.log.info('Generating blocks ...') self.generate(node0, 149) - self.sync_all() self.moved = 0 for self.nkeys in [3, 5]: @@ -117,7 +116,6 @@ class RpcCreateMultiSigTest(BitcoinTestFramework): def checkbalances(self): node0, node1, node2 = self.nodes self.generate(node0, COINBASE_MATURITY) - self.sync_all() bal0 = node0.getbalance() bal1 = node1.getbalance() diff --git a/test/functional/rpc_decodescript.py b/test/functional/rpc_decodescript.py index 5b1514af6f..56f596d419 100755 --- a/test/functional/rpc_decodescript.py +++ b/test/functional/rpc_decodescript.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2019 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test decoding scripts via decodescript RPC command.""" +import json +import os + from test_framework.messages import ( sha256, tx_from_hex, @@ -27,29 +30,29 @@ class DecodeScriptTest(BitcoinTestFramework): # below are test cases for all of the standard transaction types - # 1) P2PK scriptSig + self.log.info("- P2PK") # the scriptSig of a public key scriptPubKey simply pushes a signature onto the stack rpc_result = self.nodes[0].decodescript(push_signature) assert_equal(signature, rpc_result['asm']) - # 2) P2PKH scriptSig + self.log.info("- P2PKH") rpc_result = self.nodes[0].decodescript(push_signature + push_public_key) assert_equal(signature + ' ' + public_key, rpc_result['asm']) - # 3) multisig scriptSig + self.log.info("- multisig") # this also tests the leading portion of a P2SH multisig scriptSig # OP_0 <A sig> <B sig> rpc_result = self.nodes[0].decodescript('00' + push_signature + push_signature) assert_equal('0 ' + signature + ' ' + signature, rpc_result['asm']) - # 4) P2SH scriptSig + self.log.info("- P2SH") # an empty P2SH redeemScript is valid and makes for a very simple test case. # thus, such a spending scriptSig would just need to pass the outer redeemScript # hash test and leave true on the top of the stack. rpc_result = self.nodes[0].decodescript('5100') assert_equal('1 0', rpc_result['asm']) - # 5) null data scriptSig - no such thing because null data scripts can not be spent. + # null data scriptSig - no such thing because null data scripts can not be spent. # thus, no test case for that standard transaction type is here. def decodescript_script_pub_key(self): @@ -63,50 +66,58 @@ class DecodeScriptTest(BitcoinTestFramework): # below are test cases for all of the standard transaction types - # 1) P2PK scriptPubKey + self.log.info("- P2PK") # <pubkey> OP_CHECKSIG rpc_result = self.nodes[0].decodescript(push_public_key + 'ac') assert_equal(public_key + ' OP_CHECKSIG', rpc_result['asm']) + assert_equal('pubkey', rpc_result['type']) # P2PK is translated to P2WPKH assert_equal('0 ' + public_key_hash, rpc_result['segwit']['asm']) - # 2) P2PKH scriptPubKey + self.log.info("- P2PKH") # OP_DUP OP_HASH160 <PubKeyHash> OP_EQUALVERIFY OP_CHECKSIG rpc_result = self.nodes[0].decodescript('76a9' + push_public_key_hash + '88ac') + assert_equal('pubkeyhash', rpc_result['type']) assert_equal('OP_DUP OP_HASH160 ' + public_key_hash + ' OP_EQUALVERIFY OP_CHECKSIG', rpc_result['asm']) # P2PKH is translated to P2WPKH + assert_equal('witness_v0_keyhash', rpc_result['segwit']['type']) assert_equal('0 ' + public_key_hash, rpc_result['segwit']['asm']) - # 3) multisig scriptPubKey + self.log.info("- multisig") # <m> <A pubkey> <B pubkey> <C pubkey> <n> OP_CHECKMULTISIG # just imagine that the pub keys used below are different. # for our purposes here it does not matter that they are the same even though it is unrealistic. multisig_script = '52' + push_public_key + push_public_key + push_public_key + '53ae' rpc_result = self.nodes[0].decodescript(multisig_script) + assert_equal('multisig', rpc_result['type']) assert_equal('2 ' + public_key + ' ' + public_key + ' ' + public_key + ' 3 OP_CHECKMULTISIG', rpc_result['asm']) # multisig in P2WSH multisig_script_hash = sha256(bytes.fromhex(multisig_script)).hex() + assert_equal('witness_v0_scripthash', rpc_result['segwit']['type']) assert_equal('0 ' + multisig_script_hash, rpc_result['segwit']['asm']) - # 4) P2SH scriptPubKey + self.log.info ("- P2SH") # OP_HASH160 <Hash160(redeemScript)> OP_EQUAL. # push_public_key_hash here should actually be the hash of a redeem script. # but this works the same for purposes of this test. rpc_result = self.nodes[0].decodescript('a9' + push_public_key_hash + '87') + assert_equal('scripthash', rpc_result['type']) assert_equal('OP_HASH160 ' + public_key_hash + ' OP_EQUAL', rpc_result['asm']) # P2SH does not work in segwit secripts. decodescript should not return a result for it. assert 'segwit' not in rpc_result - # 5) null data scriptPubKey + self.log.info("- null data") # use a signature look-alike here to make sure that we do not decode random data as a signature. # this matters if/when signature sighash decoding comes along. # would want to make sure that no such decoding takes place in this case. signature_imposter = '48304502207fa7a6d1e0ee81132a269ad84e68d695483745cde8b541e3bf630749894e342a022100c1f7ab20e13e22fb95281a870f3dcf38d782e53023ee313d741ad0cfbc0c509001' # OP_RETURN <data> rpc_result = self.nodes[0].decodescript('6a' + signature_imposter) + assert_equal('nulldata', rpc_result['type']) assert_equal('OP_RETURN ' + signature_imposter[2:], rpc_result['asm']) - # 6) a CLTV redeem script. redeem scripts are in-effect scriptPubKey scripts, so adding a test here. + self.log.info("- CLTV redeem script") + # redeem scripts are in-effect scriptPubKey scripts, so adding a test here. # OP_NOP2 is also known as OP_CHECKLOCKTIMEVERIFY. # just imagine that the pub keys used below are different. # for our purposes here it does not matter that they are the same even though it is unrealistic. @@ -121,55 +132,69 @@ class DecodeScriptTest(BitcoinTestFramework): # lock until block 500,000 cltv_script = '63' + push_public_key + 'ad670320a107b17568' + push_public_key + 'ac' rpc_result = self.nodes[0].decodescript(cltv_script) + assert_equal('nonstandard', rpc_result['type']) assert_equal('OP_IF ' + public_key + ' OP_CHECKSIGVERIFY OP_ELSE 500000 OP_CHECKLOCKTIMEVERIFY OP_DROP OP_ENDIF ' + public_key + ' OP_CHECKSIG', rpc_result['asm']) # CLTV script in P2WSH cltv_script_hash = sha256(bytes.fromhex(cltv_script)).hex() assert_equal('0 ' + cltv_script_hash, rpc_result['segwit']['asm']) - # 7) P2PK scriptPubKey + self.log.info("- P2PK with uncompressed pubkey") # <pubkey> OP_CHECKSIG rpc_result = self.nodes[0].decodescript(push_uncompressed_public_key + 'ac') + assert_equal('pubkey', rpc_result['type']) assert_equal(uncompressed_public_key + ' OP_CHECKSIG', rpc_result['asm']) # uncompressed pubkeys are invalid for checksigs in segwit scripts. # decodescript should not return a P2WPKH equivalent. assert 'segwit' not in rpc_result - # 8) multisig scriptPubKey with an uncompressed pubkey + self.log.info("- multisig with uncompressed pubkey") # <m> <A pubkey> <B pubkey> <n> OP_CHECKMULTISIG # just imagine that the pub keys used below are different. # the purpose of this test is to check that a segwit script is not returned for bare multisig scripts # with an uncompressed pubkey in them. rpc_result = self.nodes[0].decodescript('52' + push_public_key + push_uncompressed_public_key +'52ae') + assert_equal('multisig', rpc_result['type']) assert_equal('2 ' + public_key + ' ' + uncompressed_public_key + ' 2 OP_CHECKMULTISIG', rpc_result['asm']) # uncompressed pubkeys are invalid for checksigs in segwit scripts. # decodescript should not return a P2WPKH equivalent. assert 'segwit' not in rpc_result - # 9) P2WPKH scriptpubkey + self.log.info("- P2WPKH") # 0 <PubKeyHash> rpc_result = self.nodes[0].decodescript('00' + push_public_key_hash) + assert_equal('witness_v0_keyhash', rpc_result['type']) assert_equal('0 ' + public_key_hash, rpc_result['asm']) # segwit scripts do not work nested into each other. # a nested segwit script should not be returned in the results. assert 'segwit' not in rpc_result - # 10) P2WSH scriptpubkey + self.log.info("- P2WSH") # 0 <ScriptHash> # even though this hash is of a P2PK script which is better used as bare P2WPKH, it should not matter # for the purpose of this test. rpc_result = self.nodes[0].decodescript('0020' + p2wsh_p2pk_script_hash) + assert_equal('witness_v0_scripthash', rpc_result['type']) assert_equal('0 ' + p2wsh_p2pk_script_hash, rpc_result['asm']) # segwit scripts do not work nested into each other. # a nested segwit script should not be returned in the results. assert 'segwit' not in rpc_result + self.log.info("- P2TR") + # 1 <x-only pubkey> + xonly_public_key = '01'*32 # first ever P2TR output on mainnet + rpc_result = self.nodes[0].decodescript('5120' + xonly_public_key) + assert_equal('witness_v1_taproot', rpc_result['type']) + assert_equal('1 ' + xonly_public_key, rpc_result['asm']) + assert 'segwit' not in rpc_result + def decoderawtransaction_asm_sighashtype(self): """Test decoding scripts via RPC command "decoderawtransaction". This test is in with the "decodescript" tests because they are testing the same "asm" script decodes. """ - # this test case uses a random plain vanilla mainnet transaction with a single P2PKH input and output + self.log.info("- various mainnet txs") + # this test case uses a mainnet transaction that has a P2SH input and both P2PKH and P2SH outputs. tx = '0100000001696a20784a2c70143f634e95227dbdfdf0ecd51647052e70854512235f5986ca010000008a47304402207174775824bec6c2700023309a168231ec80b82c6069282f5133e6f11cbb04460220570edc55c7c5da2ca687ebd0372d3546ebc3f810516a002350cac72dfe192dfb014104d3f898e6487787910a690410b7a917ef198905c27fb9d3b0a42da12aceae0544fc7088d239d9a48f2828a15a09e84043001f27cc80d162cb95404e1210161536ffffffff0100e1f505000000001976a914eb6c6e0cdb2d256a32d97b8df1fc75d1920d9bca88ac00000000' rpc_result = self.nodes[0].decoderawtransaction(tx) assert_equal('304402207174775824bec6c2700023309a168231ec80b82c6069282f5133e6f11cbb04460220570edc55c7c5da2ca687ebd0372d3546ebc3f810516a002350cac72dfe192dfb[ALL] 04d3f898e6487787910a690410b7a917ef198905c27fb9d3b0a42da12aceae0544fc7088d239d9a48f2828a15a09e84043001f27cc80d162cb95404e1210161536', rpc_result['vin'][0]['scriptSig']['asm']) @@ -185,11 +210,13 @@ class DecodeScriptTest(BitcoinTestFramework): assert_equal('OP_HASH160 2a5edea39971049a540474c6a99edf0aa4074c58 OP_EQUAL', rpc_result['vout'][1]['scriptPubKey']['asm']) txSave = tx_from_hex(tx) + self.log.info("- tx not passing DER signature checks") # make sure that a specifically crafted op_return value will not pass all the IsDERSignature checks and then get decoded as a sighash type tx = '01000000015ded05872fdbda629c7d3d02b194763ce3b9b1535ea884e3c8e765d42e316724020000006b48304502204c10d4064885c42638cbff3585915b322de33762598321145ba033fc796971e2022100bb153ad3baa8b757e30a2175bd32852d2e1cb9080f84d7e32fcdfd667934ef1b012103163c0ff73511ea1743fb5b98384a2ff09dd06949488028fd819f4d83f56264efffffffff0200000000000000000b6a0930060201000201000180380100000000001976a9141cabd296e753837c086da7a45a6c2fe0d49d7b7b88ac00000000' rpc_result = self.nodes[0].decoderawtransaction(tx) assert_equal('OP_RETURN 300602010002010001', rpc_result['vout'][0]['scriptPubKey']['asm']) + self.log.info("- tx passing DER signature checks") # verify that we have not altered scriptPubKey processing even of a specially crafted P2PKH pubkeyhash and P2SH redeem script hash that is made to pass the der signature checks tx = '01000000018d1f5635abd06e2c7e2ddf58dc85b3de111e4ad6e0ab51bb0dcf5e84126d927300000000fdfe0000483045022100ae3b4e589dfc9d48cb82d41008dc5fa6a86f94d5c54f9935531924602730ab8002202f88cf464414c4ed9fa11b773c5ee944f66e9b05cc1e51d97abc22ce098937ea01483045022100b44883be035600e9328a01b66c7d8439b74db64187e76b99a68f7893b701d5380220225bf286493e4c4adcf928c40f785422572eb232f84a0b83b0dea823c3a19c75014c695221020743d44be989540d27b1b4bbbcfd17721c337cb6bc9af20eb8a32520b393532f2102c0120a1dda9e51a938d39ddd9fe0ebc45ea97e1d27a7cbd671d5431416d3dd87210213820eb3d5f509d7438c9eeecb4157b2f595105e7cd564b3cdbb9ead3da41eed53aeffffffff02611e0000000000001976a914301102070101010101010102060101010101010188acee2a02000000000017a91430110207010101010101010206010101010101018700000000' rpc_result = self.nodes[0].decoderawtransaction(tx) @@ -207,7 +234,7 @@ class DecodeScriptTest(BitcoinTestFramework): push_signature_2 = '48' + signature_2 signature_2_sighash_decoded = der_signature + '[NONE|ANYONECANPAY]' - # 1) P2PK scriptSig + self.log.info("- P2PK scriptSig") txSave.vin[0].scriptSig = bytes.fromhex(push_signature) rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal(signature_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) @@ -217,21 +244,34 @@ class DecodeScriptTest(BitcoinTestFramework): rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal(signature_2_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) - # 2) multisig scriptSig + self.log.info("- multisig scriptSig") txSave.vin[0].scriptSig = bytes.fromhex('00' + push_signature + push_signature_2) rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal('0 ' + signature_sighash_decoded + ' ' + signature_2_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) - # 3) test a scriptSig that contains more than push operations. + self.log.info("- scriptSig that contains more than push operations") # in fact, it contains an OP_RETURN with data specially crafted to cause improper decode if the code does not catch it. txSave.vin[0].scriptSig = bytes.fromhex('6a143011020701010101010101020601010101010101') rpc_result = self.nodes[0].decoderawtransaction(txSave.serialize().hex()) assert_equal('OP_RETURN 3011020701010101010101020601010101010101', rpc_result['vin'][0]['scriptSig']['asm']) + def decodescript_datadriven_tests(self): + with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/rpc_decodescript.json'), encoding='utf-8') as f: + dd_tests = json.load(f) + + for script, result in dd_tests: + rpc_result = self.nodes[0].decodescript(script) + assert_equal(result, rpc_result) + def run_test(self): + self.log.info("Test decoding of standard input scripts [scriptSig]") self.decodescript_script_sig() + self.log.info("Test decoding of standard output scripts [scriptPubKey]") self.decodescript_script_pub_key() + self.log.info("Test 'asm' script decoding of transactions") self.decoderawtransaction_asm_sighashtype() + self.log.info("Data-driven tests") + self.decodescript_datadriven_tests() if __name__ == '__main__': DecodeScriptTest().main() diff --git a/test/functional/rpc_deprecated.py b/test/functional/rpc_deprecated.py index fdaed918a1..15c77ed856 100755 --- a/test/functional/rpc_deprecated.py +++ b/test/functional/rpc_deprecated.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test deprecation of RPC calls.""" diff --git a/test/functional/rpc_dumptxoutset.py b/test/functional/rpc_dumptxoutset.py index 89388df555..1721b6ffe8 100755 --- a/test/functional/rpc_dumptxoutset.py +++ b/test/functional/rpc_dumptxoutset.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the generation of UTXO snapshots using `dumptxoutset`. @@ -45,6 +45,10 @@ class DumptxoutsetTest(BitcoinTestFramework): assert_equal( digest, '7ae82c986fa5445678d2a21453bb1c86d39e47af13da137640c2b1cf8093691c') + assert_equal( + out['txoutset_hash'], 'd4b614f476b99a6e569973bf1c0120d88b1a168076f8ce25691fb41dd1cef149') + assert_equal(out['nchaintx'], 101) + # Specifying a path to an existing file will fail. assert_raises_rpc_error( -8, '{} already exists'.format(FILENAME), node.dumptxoutset, FILENAME) diff --git a/test/functional/rpc_fundrawtransaction.py b/test/functional/rpc_fundrawtransaction.py index 98f91221ea..17d11a3ceb 100755 --- a/test/functional/rpc_fundrawtransaction.py +++ b/test/functional/rpc_fundrawtransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the fundrawtransaction RPC.""" @@ -99,9 +99,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.fee_tolerance = 2 * self.min_relay_tx_fee / 1000 self.generate(self.nodes[2], 1) - self.sync_all() self.generate(self.nodes[0], 121) - self.sync_all() self.test_change_position() self.test_simple() @@ -136,6 +134,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.test_include_unsafe() self.test_external_inputs() self.test_22670() + self.test_feerate_rounding() def test_change_position(self): """Ensure setting changePosition in fundraw with an exact match is handled properly.""" @@ -163,7 +162,6 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 5.0) self.generate(self.nodes[0], 1) - self.sync_all() wwatch.unloadwallet() @@ -549,7 +547,6 @@ class RawTransactionsTest(BitcoinTestFramework): # Send 1.2 BTC to msig addr. self.nodes[0].sendtoaddress(mSigObj, 1.2) self.generate(self.nodes[0], 1) - self.sync_all() oldBalance = self.nodes[1].getbalance() inputs = [] @@ -560,7 +557,6 @@ class RawTransactionsTest(BitcoinTestFramework): final_psbt = w2.finalizepsbt(signed_psbt['psbt']) self.nodes[2].sendrawtransaction(final_psbt['hex']) self.generate(self.nodes[2], 1) - self.sync_all() # Make sure funds are received at node1. assert_equal(oldBalance+Decimal('1.10000000'), self.nodes[1].getbalance()) @@ -575,12 +571,12 @@ class RawTransactionsTest(BitcoinTestFramework): if self.options.descriptors: self.nodes[1].walletpassphrase('test', 10) self.nodes[1].importdescriptors([{ - 'desc': descsum_create('wpkh(tprv8ZgxMBicQKsPdYeeZbPSKd2KYLmeVKtcFA7kqCxDvDR13MQ6us8HopUR2wLcS2ZKPhLyKsqpDL2FtL73LMHcgoCL7DXsciA8eX8nbjCR2eG/0h/*h)'), + 'desc': descsum_create('tr(tprv8ZgxMBicQKsPdYeeZbPSKd2KYLmeVKtcFA7kqCxDvDR13MQ6us8HopUR2wLcS2ZKPhLyKsqpDL2FtL73LMHcgoCL7DXsciA8eX8nbjCR2eG/0h/*h)'), 'timestamp': 'now', 'active': True }, { - 'desc': descsum_create('wpkh(tprv8ZgxMBicQKsPdYeeZbPSKd2KYLmeVKtcFA7kqCxDvDR13MQ6us8HopUR2wLcS2ZKPhLyKsqpDL2FtL73LMHcgoCL7DXsciA8eX8nbjCR2eG/1h/*h)'), + 'desc': descsum_create('tr(tprv8ZgxMBicQKsPdYeeZbPSKd2KYLmeVKtcFA7kqCxDvDR13MQ6us8HopUR2wLcS2ZKPhLyKsqpDL2FtL73LMHcgoCL7DXsciA8eX8nbjCR2eG/1h/*h)'), 'timestamp': 'now', 'active': True, 'internal': True @@ -624,7 +620,6 @@ class RawTransactionsTest(BitcoinTestFramework): signedTx = self.nodes[1].signrawtransactionwithwallet(fundedTx['hex']) self.nodes[1].sendrawtransaction(signedTx['hex']) self.generate(self.nodes[1], 1) - self.sync_all() # Make sure funds are received at node1. assert_equal(oldBalance+Decimal('51.10000000'), self.nodes[0].getbalance()) @@ -636,12 +631,10 @@ class RawTransactionsTest(BitcoinTestFramework): # Empty node1, send some small coins from node0 to node1. self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), self.nodes[1].getbalance(), "", "", True) self.generate(self.nodes[1], 1) - self.sync_all() for _ in range(20): self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.01) self.generate(self.nodes[0], 1) - self.sync_all() # Fund a tx with ~20 small inputs. inputs = [] @@ -664,12 +657,10 @@ class RawTransactionsTest(BitcoinTestFramework): # Again, empty node1, send some small coins from node0 to node1. self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), self.nodes[1].getbalance(), "", "", True) self.generate(self.nodes[1], 1) - self.sync_all() for _ in range(20): self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.01) self.generate(self.nodes[0], 1) - self.sync_all() # Fund a tx with ~20 small inputs. oldBalance = self.nodes[0].getbalance() @@ -681,7 +672,6 @@ class RawTransactionsTest(BitcoinTestFramework): fundedAndSignedTx = self.nodes[1].signrawtransactionwithwallet(fundedTx['hex']) self.nodes[1].sendrawtransaction(fundedAndSignedTx['hex']) self.generate(self.nodes[1], 1) - self.sync_all() assert_equal(oldBalance+Decimal('50.19000000'), self.nodes[0].getbalance()) #0.19+block reward def test_op_return(self): @@ -759,7 +749,6 @@ class RawTransactionsTest(BitcoinTestFramework): assert signedtx["complete"] self.nodes[0].sendrawtransaction(signedtx["hex"]) self.generate(self.nodes[0], 1) - self.sync_all() wwatch.unloadwallet() @@ -789,11 +778,18 @@ class RawTransactionsTest(BitcoinTestFramework): for param, zero_value in product(["fee_rate", "feeRate"], [0, 0.000, 0.00000000, "0", "0.000", "0.00000000"]): assert_equal(self.nodes[3].fundrawtransaction(rawtx, {param: zero_value})["fee"], 0) - # With no arguments passed, expect fee of 141 satoshis. - assert_approx(node.fundrawtransaction(rawtx)["fee"], vexp=0.00000141, vspan=0.00000001) - # Expect fee to be 10,000x higher when an explicit fee rate 10,000x greater is specified. - result = node.fundrawtransaction(rawtx, {"fee_rate": 10000}) - assert_approx(result["fee"], vexp=0.0141, vspan=0.0001) + if self.options.descriptors: + # With no arguments passed, expect fee of 153 satoshis as descriptor wallets now have a taproot output. + assert_approx(node.fundrawtransaction(rawtx)["fee"], vexp=0.00000153, vspan=0.00000001) + # Expect fee to be 10,000x higher when an explicit fee rate 10,000x greater is specified. + result = node.fundrawtransaction(rawtx, {"fee_rate": 10000}) + assert_approx(result["fee"], vexp=0.0153, vspan=0.0001) + else: + # With no arguments passed, expect fee of 141 satoshis as legacy wallets only support up to segwit v0. + assert_approx(node.fundrawtransaction(rawtx)["fee"], vexp=0.00000141, vspan=0.00000001) + # Expect fee to be 10,000x higher when an explicit fee rate 10,000x greater is specified. + result = node.fundrawtransaction(rawtx, {"fee_rate": 10000}) + assert_approx(result["fee"], vexp=0.0141, vspan=0.0001) self.log.info("Test fundrawtxn with invalid estimate_mode settings") for k, v in {"number": 42, "object": {"foo": "bar"}}.items(): @@ -1011,7 +1007,6 @@ class RawTransactionsTest(BitcoinTestFramework): self.nodes[0].sendtoaddress(addr, 10) self.nodes[0].sendtoaddress(wallet.getnewaddress(), 10) self.generate(self.nodes[0], 6) - self.sync_all() ext_utxo = self.nodes[0].listunspent(addresses=[addr])[0] # An external input without solving data should result in an error @@ -1085,7 +1080,7 @@ class RawTransactionsTest(BitcoinTestFramework): # Make sure the default wallet will not be loaded when restarted with a high minrelaytxfee self.nodes[0].unloadwallet(self.default_wallet_name, False) feerate = Decimal("0.1") - self.restart_node(0, [f"-minrelaytxfee={feerate}", "-discardfee=0"]) # Set high minrelayfee, set discardfee to 0 for easier calculation + self.restart_node(0, [f"-minrelaytxfee={feerate}", "-discardfee=0", "-changetype=bech32", "-addresstype=bech32"]) # Set high minrelayfee, set discardfee to 0 for easier calculation self.nodes[0].loadwallet(self.default_wallet_name, True) funds = self.nodes[0].get_wallet_rpc(self.default_wallet_name) @@ -1096,7 +1091,7 @@ class RawTransactionsTest(BitcoinTestFramework): # than any single input available, and require more than 1 input. So we make 3 outputs for i in range(0, 3): funds.sendtoaddress(tester.getnewaddress(address_type="bech32"), 1) - self.generate(self.nodes[0], 1) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) # Create transactions in order to calculate fees for the target bounds that can trigger this bug change_tx = tester.fundrawtransaction(tester.createrawtransaction([], [{funds.getnewaddress(): 1.5}])) @@ -1129,6 +1124,32 @@ class RawTransactionsTest(BitcoinTestFramework): do_fund_send(upper_bound) self.restart_node(0) + self.connect_nodes(0, 1) + self.connect_nodes(0, 2) + self.connect_nodes(0, 3) + + def test_feerate_rounding(self): + self.log.info("Test that rounding of GetFee does not result in an assertion") + + self.nodes[1].createwallet("roundtest") + w = self.nodes[1].get_wallet_rpc("roundtest") + + addr = w.getnewaddress(address_type="bech32") + self.nodes[0].sendtoaddress(addr, 1) + self.generate(self.nodes[0], 1) + + # A P2WPKH input costs 68 vbytes; With a single P2WPKH output, the rest of the tx is 42 vbytes for a total of 110 vbytes. + # At a feerate of 1.85 sat/vb, the input will need a fee of 125.8 sats and the rest 77.7 sats + # The entire tx fee should be 203.5 sats. + # Coin selection rounds the fee individually instead of at the end (due to how CFeeRate::GetFee works). + # If rounding down (which is the incorrect behavior), then the calculated fee will be 125 + 77 = 202. + # If rounding up, then the calculated fee will be 126 + 78 = 204. + # In the former case, the calculated needed fee is higher than the actual fee being paid, so an assertion is reached + # To test this does not happen, we subtract 202 sats from the input value. If working correctly, this should + # fail with insufficient funds rather than bitcoind asserting. + rawtx = w.createrawtransaction(inputs=[], outputs=[{self.nodes[0].getnewaddress(address_type="bech32"): 1 - 0.00000202}]) + assert_raises_rpc_error(-4, "Insufficient funds", w.fundrawtransaction, rawtx, {"fee_rate": 1.85}) + if __name__ == '__main__': RawTransactionsTest().main() diff --git a/test/functional/rpc_generateblock.py b/test/functional/rpc_generateblock.py index 3c6b3fb125..7aede0e947 100755 --- a/test/functional/rpc_generateblock.py +++ b/test/functional/rpc_generateblock.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. '''Test generateblock rpc. ''' from test_framework.test_framework import BitcoinTestFramework +from test_framework.wallet import MiniWallet from test_framework.util import ( assert_equal, assert_raises_rpc_error, @@ -16,14 +17,13 @@ class GenerateBlockTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 1 - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def run_test(self): node = self.nodes[0] + miniwallet = MiniWallet(node) + miniwallet.rescan_utxos() self.log.info('Generate an empty block to address') - address = node.getnewaddress() + address = miniwallet.get_address() hash = self.generateblock(node, output=address, transactions=[])['hash'] block = node.getblock(blockhash=hash, verbose=2) assert_equal(len(block['tx']), 1) @@ -51,37 +51,31 @@ class GenerateBlockTest(BitcoinTestFramework): assert_equal(len(block['tx']), 1) assert_equal(block['tx'][0]['vout'][0]['scriptPubKey']['address'], combo_address) - # Generate 110 blocks to spend - self.generatetoaddress(node, 110, address) - # Generate some extra mempool transactions to verify they don't get mined for _ in range(10): - node.sendtoaddress(address, 0.001) + miniwallet.send_self_transfer(from_node=node) self.log.info('Generate block with txid') - txid = node.sendtoaddress(address, 1) + txid = miniwallet.send_self_transfer(from_node=node)['txid'] hash = self.generateblock(node, address, [txid])['hash'] block = node.getblock(hash, 1) assert_equal(len(block['tx']), 2) assert_equal(block['tx'][1], txid) self.log.info('Generate block with raw tx') - utxos = node.listunspent(addresses=[address]) - raw = node.createrawtransaction([{'txid':utxos[0]['txid'], 'vout':utxos[0]['vout']}],[{address:1}]) - signed_raw = node.signrawtransactionwithwallet(raw)['hex'] - hash = self.generateblock(node, address, [signed_raw])['hash'] + rawtx = miniwallet.create_self_transfer(from_node=node)['hex'] + hash = self.generateblock(node, address, [rawtx])['hash'] + block = node.getblock(hash, 1) assert_equal(len(block['tx']), 2) txid = block['tx'][1] - assert_equal(node.gettransaction(txid)['hex'], signed_raw) + assert_equal(node.getrawtransaction(txid=txid, verbose=False, blockhash=hash), rawtx) self.log.info('Fail to generate block with out of order txs') - raw1 = node.createrawtransaction([{'txid':txid, 'vout':0}],[{address:0.9999}]) - signed_raw1 = node.signrawtransactionwithwallet(raw1)['hex'] - txid1 = node.sendrawtransaction(signed_raw1) - raw2 = node.createrawtransaction([{'txid':txid1, 'vout':0}],[{address:0.999}]) - signed_raw2 = node.signrawtransactionwithwallet(raw2)['hex'] - assert_raises_rpc_error(-25, 'TestBlockValidity failed: bad-txns-inputs-missingorspent', self.generateblock, node, address, [signed_raw2, txid1]) + txid1 = miniwallet.send_self_transfer(from_node=node)['txid'] + utxo1 = miniwallet.get_utxo(txid=txid1) + rawtx2 = miniwallet.create_self_transfer(from_node=node, utxo_to_spend=utxo1)['hex'] + assert_raises_rpc_error(-25, 'TestBlockValidity failed: bad-txns-inputs-missingorspent', self.generateblock, node, address, [rawtx2, txid1]) self.log.info('Fail to generate block with txid not in mempool') missing_txid = '0000000000000000000000000000000000000000000000000000000000000000' diff --git a/test/functional/rpc_getblockfilter.py b/test/functional/rpc_getblockfilter.py index 4d860d0f36..b09af9e078 100755 --- a/test/functional/rpc_getblockfilter.py +++ b/test/functional/rpc_getblockfilter.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the getblockfilter RPC.""" @@ -21,8 +21,8 @@ class GetBlockFilterTest(BitcoinTestFramework): # Create two chains by disconnecting nodes 0 & 1, mining, then reconnecting self.disconnect_nodes(0, 1) - self.generate(self.nodes[0], 3) - self.generate(self.nodes[1], 4) + self.generate(self.nodes[0], 3, sync_fun=self.no_op) + self.generate(self.nodes[1], 4, sync_fun=self.no_op) assert_equal(self.nodes[0].getblockcount(), 3) chain0_hashes = [self.nodes[0].getblockhash(block_height) for block_height in range(4)] diff --git a/test/functional/rpc_getblockstats.py b/test/functional/rpc_getblockstats.py index 456e2cb0ad..8c08d2ced5 100755 --- a/test/functional/rpc_getblockstats.py +++ b/test/functional/rpc_getblockstats.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -48,7 +48,6 @@ class GetblockstatsTest(BitcoinTestFramework): address = self.nodes[0].get_deterministic_priv_key().address self.nodes[0].sendtoaddress(address=address, amount=10, subtractfeefromamount=True) self.generate(self.nodes[0], 1) - self.sync_all() self.nodes[0].sendtoaddress(address=address, amount=10, subtractfeefromamount=True) self.nodes[0].sendtoaddress(address=address, amount=10, subtractfeefromamount=False) diff --git a/test/functional/rpc_getchaintips.py b/test/functional/rpc_getchaintips.py index ab0ee9142d..7efa306c8c 100755 --- a/test/functional/rpc_getchaintips.py +++ b/test/functional/rpc_getchaintips.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the getchaintips RPC. @@ -26,10 +26,8 @@ class GetChainTipsTest (BitcoinTestFramework): # Split the network and build two chains of different lengths. self.split_network() - self.generatetoaddress(self.nodes[0], 10, self.nodes[0].get_deterministic_priv_key().address) - self.generatetoaddress(self.nodes[2], 20, self.nodes[2].get_deterministic_priv_key().address) - self.sync_all(self.nodes[:2]) - self.sync_all(self.nodes[2:]) + self.generate(self.nodes[0], 10, sync_fun=lambda: self.sync_all(self.nodes[:2])) + self.generate(self.nodes[2], 20, sync_fun=lambda: self.sync_all(self.nodes[2:])) tips = self.nodes[1].getchaintips () assert_equal (len (tips), 1) diff --git a/test/functional/rpc_help.py b/test/functional/rpc_help.py index de21f43747..ccb380e25b 100755 --- a/test/functional/rpc_help.py +++ b/test/functional/rpc_help.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPC help output.""" diff --git a/test/functional/rpc_invalid_address_message.py b/test/functional/rpc_invalid_address_message.py index 085f6582b5..fcc49d0a75 100755 --- a/test/functional/rpc_invalid_address_message.py +++ b/test/functional/rpc_invalid_address_message.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test error messages for 'getaddressinfo' and 'validateaddress' RPC commands.""" @@ -12,79 +12,96 @@ from test_framework.util import ( ) BECH32_VALID = 'bcrt1qtmp74ayg7p24uslctssvjm06q5phz4yrxucgnv' +BECH32_VALID_CAPITALS = 'BCRT1QPLMTZKC2XHARPPZDLNPAQL78RSHJ68U33RAH7R' +BECH32_VALID_MULTISIG = 'bcrt1qdg3myrgvzw7ml9q0ejxhlkyxm7vl9r56yzkfgvzclrf4hkpx9yfqhpsuks' + BECH32_INVALID_BECH32 = 'bcrt1p0xlxvlhemja6c4dqv22uapctqupfhlxm9h8z3k2e72q4k9hcz7vqdmchcc' BECH32_INVALID_BECH32M = 'bcrt1qw508d6qejxtdg4y5r3zarvary0c5xw7k35mrzd' BECH32_INVALID_VERSION = 'bcrt130xlxvlhemja6c4dqv22uapctqupfhlxm9h8z3k2e72q4k9hcz7vqynjegk' BECH32_INVALID_SIZE = 'bcrt1s0xlxvlhemja6c4dqv22uapctqupfhlxm9h8z3k2e72q4k9hcz7v8n0nx0muaewav25430mtr' BECH32_INVALID_V0_SIZE = 'bcrt1qw508d6qejxtdg4y5r3zarvary0c5xw7kqqq5k3my' BECH32_INVALID_PREFIX = 'bc1pw508d6qejxtdg4y5r3zarvary0c5xw7kw508d6qejxtdg4y5r3zarvary0c5xw7k7grplx' +BECH32_TOO_LONG = 'bcrt1q049edschfnwystcqnsvyfpj23mpsg3jcedq9xv049edschfnwystcqnsvyfpj23mpsg3jcedq9xv049edschfnwystcqnsvyfpj23m' +BECH32_ONE_ERROR = 'bcrt1q049edschfnwystcqnsvyfpj23mpsg3jcedq9xv' +BECH32_ONE_ERROR_CAPITALS = 'BCRT1QPLMTZKC2XHARPPZDLNPAQL78RSHJ68U32RAH7R' +BECH32_TWO_ERRORS = 'bcrt1qax9suht3qv95sw33xavx8crpxduefdrsvgsklu' # should be bcrt1qax9suht3qv95sw33wavx8crpxduefdrsvgsklx +BECH32_NO_SEPARATOR = 'bcrtq049ldschfnwystcqnsvyfpj23mpsg3jcedq9xv' +BECH32_INVALID_CHAR = 'bcrt1q04oldschfnwystcqnsvyfpj23mpsg3jcedq9xv' +BECH32_MULTISIG_TWO_ERRORS = 'bcrt1qdg3myrgvzw7ml8q0ejxhlkyxn7vl9r56yzkfgvzclrf4hkpx9yfqhpsuks' +BECH32_WRONG_VERSION = 'bcrt1ptmp74ayg7p24uslctssvjm06q5phz4yrxucgnv' BASE58_VALID = 'mipcBbFg9gMiCh81Kj8tqqdgoZub1ZJRfn' BASE58_INVALID_PREFIX = '17VZNX1SN5NtKa8UQFxwQbFeFc3iqRYhem' +BASE58_INVALID_CHECKSUM = 'mipcBbFg9gMiCh81Kj8tqqdgoZub1ZJJfn' +BASE58_INVALID_LENGTH = '2VKf7XKMrp4bVNVmuRbyCewkP8FhGLP2E54LHDPakr9Sq5mtU2' INVALID_ADDRESS = 'asfah14i8fajz0123f' +INVALID_ADDRESS_2 = '1q049ldschfnwystcqnsvyfpj23mpsg3jcedq9xv' class InvalidAddressErrorMessageTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 - def test_validateaddress(self): - node = self.nodes[0] - - # Bech32 - info = node.validateaddress(BECH32_INVALID_SIZE) - assert not info['isvalid'] - assert_equal(info['error'], 'Invalid Bech32 address data size') - - info = node.validateaddress(BECH32_INVALID_PREFIX) - assert not info['isvalid'] - assert_equal(info['error'], 'Invalid prefix for Bech32 address') - - info = node.validateaddress(BECH32_INVALID_BECH32) - assert not info['isvalid'] - assert_equal(info['error'], 'Version 1+ witness address must use Bech32m checksum') - - info = node.validateaddress(BECH32_INVALID_BECH32M) - assert not info['isvalid'] - assert_equal(info['error'], 'Version 0 witness address must use Bech32 checksum') - - info = node.validateaddress(BECH32_INVALID_V0_SIZE) - assert not info['isvalid'] - assert_equal(info['error'], 'Invalid Bech32 v0 address data size') - - info = node.validateaddress(BECH32_VALID) + def check_valid(self, addr): + info = self.nodes[0].validateaddress(addr) assert info['isvalid'] assert 'error' not in info + assert 'error_locations' not in info - info = node.validateaddress(BECH32_INVALID_VERSION) - assert not info['isvalid'] - assert_equal(info['error'], 'Invalid Bech32 address witness version') - - # Base58 - info = node.validateaddress(BASE58_INVALID_PREFIX) - assert not info['isvalid'] - assert_equal(info['error'], 'Invalid prefix for Base58-encoded address') + def check_invalid(self, addr, error_str, error_locations=None): + res = self.nodes[0].validateaddress(addr) + assert not res['isvalid'] + assert_equal(res['error'], error_str) + if error_locations: + assert_equal(res['error_locations'], error_locations) + else: + assert_equal(res['error_locations'], []) - info = node.validateaddress(BASE58_VALID) - assert info['isvalid'] - assert 'error' not in info + def test_validateaddress(self): + # Invalid Bech32 + self.check_invalid(BECH32_INVALID_SIZE, 'Invalid Bech32 address data size') + self.check_invalid(BECH32_INVALID_PREFIX, 'Not a valid Bech32 or Base58 encoding') + self.check_invalid(BECH32_INVALID_BECH32, 'Version 1+ witness address must use Bech32m checksum') + self.check_invalid(BECH32_INVALID_BECH32M, 'Version 0 witness address must use Bech32 checksum') + self.check_invalid(BECH32_INVALID_VERSION, 'Invalid Bech32 address witness version') + self.check_invalid(BECH32_INVALID_V0_SIZE, 'Invalid Bech32 v0 address data size') + self.check_invalid(BECH32_TOO_LONG, 'Bech32 string too long', list(range(90, 108))) + self.check_invalid(BECH32_ONE_ERROR, 'Invalid Bech32 checksum', [9]) + self.check_invalid(BECH32_TWO_ERRORS, 'Invalid Bech32 checksum', [22, 43]) + self.check_invalid(BECH32_ONE_ERROR_CAPITALS, 'Invalid Bech32 checksum', [38]) + self.check_invalid(BECH32_NO_SEPARATOR, 'Missing separator') + self.check_invalid(BECH32_INVALID_CHAR, 'Invalid Base 32 character', [8]) + self.check_invalid(BECH32_MULTISIG_TWO_ERRORS, 'Invalid Bech32 checksum', [19, 30]) + self.check_invalid(BECH32_WRONG_VERSION, 'Invalid Bech32 checksum', [5]) + + # Valid Bech32 + self.check_valid(BECH32_VALID) + self.check_valid(BECH32_VALID_CAPITALS) + self.check_valid(BECH32_VALID_MULTISIG) + + # Invalid Base58 + self.check_invalid(BASE58_INVALID_PREFIX, 'Invalid prefix for Base58-encoded address') + self.check_invalid(BASE58_INVALID_CHECKSUM, 'Invalid checksum or length of Base58 address') + self.check_invalid(BASE58_INVALID_LENGTH, 'Invalid checksum or length of Base58 address') + + # Valid Base58 + self.check_valid(BASE58_VALID) # Invalid address format - info = node.validateaddress(INVALID_ADDRESS) - assert not info['isvalid'] - assert_equal(info['error'], 'Invalid address format') + self.check_invalid(INVALID_ADDRESS, 'Not a valid Bech32 or Base58 encoding') + self.check_invalid(INVALID_ADDRESS_2, 'Not a valid Bech32 or Base58 encoding') def test_getaddressinfo(self): node = self.nodes[0] assert_raises_rpc_error(-5, "Invalid Bech32 address data size", node.getaddressinfo, BECH32_INVALID_SIZE) - assert_raises_rpc_error(-5, "Invalid prefix for Bech32 address", node.getaddressinfo, BECH32_INVALID_PREFIX) + assert_raises_rpc_error(-5, "Not a valid Bech32 or Base58 encoding", node.getaddressinfo, BECH32_INVALID_PREFIX) assert_raises_rpc_error(-5, "Invalid prefix for Base58-encoded address", node.getaddressinfo, BASE58_INVALID_PREFIX) - assert_raises_rpc_error(-5, "Invalid address format", node.getaddressinfo, INVALID_ADDRESS) + assert_raises_rpc_error(-5, "Not a valid Bech32 or Base58 encoding", node.getaddressinfo, INVALID_ADDRESS) def run_test(self): self.test_validateaddress() diff --git a/test/functional/rpc_invalidateblock.py b/test/functional/rpc_invalidateblock.py index a91ce85855..f1c2537ef9 100755 --- a/test/functional/rpc_invalidateblock.py +++ b/test/functional/rpc_invalidateblock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the invalidateblock RPC.""" @@ -22,12 +22,12 @@ class InvalidateTest(BitcoinTestFramework): def run_test(self): self.log.info("Make sure we repopulate setBlockIndexCandidates after InvalidateBlock:") self.log.info("Mine 4 blocks on Node 0") - self.generatetoaddress(self.nodes[0], 4, self.nodes[0].get_deterministic_priv_key().address) + self.generate(self.nodes[0], 4, sync_fun=self.no_op) assert_equal(self.nodes[0].getblockcount(), 4) besthash_n0 = self.nodes[0].getbestblockhash() self.log.info("Mine competing 6 blocks on Node 1") - self.generatetoaddress(self.nodes[1], 6, self.nodes[1].get_deterministic_priv_key().address) + self.generate(self.nodes[1], 6, sync_fun=self.no_op) assert_equal(self.nodes[1].getblockcount(), 6) self.log.info("Connect nodes to force a reorg") @@ -53,14 +53,14 @@ class InvalidateTest(BitcoinTestFramework): self.nodes[2].invalidateblock(self.nodes[2].getblockhash(3)) assert_equal(self.nodes[2].getblockcount(), 2) self.log.info("..and then mine a block") - self.generatetoaddress(self.nodes[2], 1, self.nodes[2].get_deterministic_priv_key().address) + self.generate(self.nodes[2], 1, sync_fun=self.no_op) self.log.info("Verify all nodes are at the right height") self.wait_until(lambda: self.nodes[2].getblockcount() == 3, timeout=5) self.wait_until(lambda: self.nodes[0].getblockcount() == 4, timeout=5) self.wait_until(lambda: self.nodes[1].getblockcount() == 4, timeout=5) self.log.info("Verify that we reconsider all ancestors as well") - blocks = self.generatetodescriptor(self.nodes[1], 10, ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR) + blocks = self.generatetodescriptor(self.nodes[1], 10, ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR, sync_fun=self.no_op) assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) # Invalidate the two blocks at the tip self.nodes[1].invalidateblock(blocks[-1]) @@ -72,7 +72,7 @@ class InvalidateTest(BitcoinTestFramework): assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) self.log.info("Verify that we reconsider all descendants") - blocks = self.generatetodescriptor(self.nodes[1], 10, ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR) + blocks = self.generatetodescriptor(self.nodes[1], 10, ADDRESS_BCRT1_UNSPENDABLE_DESCRIPTOR, sync_fun=self.no_op) assert_equal(self.nodes[1].getbestblockhash(), blocks[-1]) # Invalidate the two blocks at the tip self.nodes[1].invalidateblock(blocks[-2]) diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py index e32e562bce..2f1796d7cc 100755 --- a/test/functional/rpc_misc.py +++ b/test/functional/rpc_misc.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPC misc output.""" @@ -50,7 +50,7 @@ class RpcMiscTest(BitcoinTestFramework): assert_equal(tree.tag, 'malloc') except JSONRPCException: self.log.info('getmemoryinfo(mode="mallocinfo") not available') - assert_raises_rpc_error(-8, 'mallocinfo is only available when compiled with glibc 2.10+', node.getmemoryinfo, mode="mallocinfo") + assert_raises_rpc_error(-8, 'mallocinfo mode not available', node.getmemoryinfo, mode="mallocinfo") assert_raises_rpc_error(-8, "unknown mode foobar", node.getmemoryinfo, mode="foobar") diff --git a/test/functional/rpc_net.py b/test/functional/rpc_net.py index 0f3bbce54c..81a3cfee97 100755 --- a/test/functional/rpc_net.py +++ b/test/functional/rpc_net.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPC calls related to net. @@ -81,7 +81,6 @@ class NetTest(BitcoinTestFramework): # Create a few getpeerinfo last_block/last_transaction values. self.wallet.send_self_transfer(from_node=self.nodes[0]) # Make a transaction so we can see it in the getpeerinfo results self.generate(self.nodes[1], 1) - self.sync_all() time_now = int(time.time()) peer_info = [x.getpeerinfo() for x in self.nodes] # Verify last_block and last_transaction keys/values. @@ -106,7 +105,7 @@ class NetTest(BitcoinTestFramework): assert_equal(peer_info[1][1]['connection_type'], 'inbound') # Check dynamically generated networks list in getpeerinfo help output. - assert "(ipv4, ipv6, onion, i2p, not_publicly_routable)" in self.nodes[0].help("getpeerinfo") + assert "(ipv4, ipv6, onion, i2p, cjdns, not_publicly_routable)" in self.nodes[0].help("getpeerinfo") def test_getnettotals(self): self.log.info("Test getnettotals") @@ -157,7 +156,7 @@ class NetTest(BitcoinTestFramework): assert_net_servicesnames(int(info["localservices"], 0x10), info["localservicesnames"]) # Check dynamically generated networks list in getnetworkinfo help output. - assert "(ipv4, ipv6, onion, i2p)" in self.nodes[0].help("getnetworkinfo") + assert "(ipv4, ipv6, onion, i2p, cjdns)" in self.nodes[0].help("getnetworkinfo") def test_getaddednodeinfo(self): self.log.info("Test getaddednodeinfo") @@ -228,8 +227,8 @@ class NetTest(BitcoinTestFramework): assert_equal(res[0]["port"], 8333) assert_equal(res[0]["services"], P2P_SERVICES) - # Test for the absence of onion and I2P addresses. - for network in ["onion", "i2p"]: + # Test for the absence of onion, I2P and CJDNS addresses. + for network in ["onion", "i2p", "cjdns"]: assert_equal(self.nodes[0].getnodeaddresses(0, network), []) # Test invalid arguments. @@ -260,7 +259,7 @@ class NetTest(BitcoinTestFramework): self.log.debug("Test that adding a valid address to the tried table succeeds") assert_equal(node.addpeeraddress(address="1.2.3.4", tried=True, port=8333), {"success": True}) - with node.assert_debug_log(expected_msgs=["Addrman checks started: new 0, tried 1, total 1"]): + with node.assert_debug_log(expected_msgs=["CheckAddrman: new 0, tried 1, total 1 started"]): addrs = node.getnodeaddresses(count=0) # getnodeaddresses re-runs the addrman checks assert_equal(len(addrs), 1) assert_equal(addrs[0]["address"], "1.2.3.4") @@ -273,7 +272,7 @@ class NetTest(BitcoinTestFramework): self.log.debug("Test that adding a second address, this time to the new table, succeeds") assert_equal(node.addpeeraddress(address="2.0.0.0", port=8333), {"success": True}) - with node.assert_debug_log(expected_msgs=["Addrman checks started: new 1, tried 1, total 2"]): + with node.assert_debug_log(expected_msgs=["CheckAddrman: new 1, tried 1, total 2 started"]): addrs = node.getnodeaddresses(count=0) # getnodeaddresses re-runs the addrman checks assert_equal(len(addrs), 2) diff --git a/test/functional/rpc_preciousblock.py b/test/functional/rpc_preciousblock.py index 3a00992ddc..91298937fd 100755 --- a/test/functional/rpc_preciousblock.py +++ b/test/functional/rpc_preciousblock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the preciousblock RPC.""" @@ -42,19 +42,18 @@ class PreciousTest(BitcoinTestFramework): def run_test(self): self.log.info("Ensure submitblock can in principle reorg to a competing chain") - gen_address = lambda i: self.nodes[i].get_deterministic_priv_key().address # A non-wallet address to mine to - self.generatetoaddress(self.nodes[0], 1, gen_address(0)) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) assert_equal(self.nodes[0].getblockcount(), 1) - hashZ = self.generatetoaddress(self.nodes[1], 2, gen_address(1))[-1] + hashZ = self.generate(self.nodes[1], 2, sync_fun=self.no_op)[-1] assert_equal(self.nodes[1].getblockcount(), 2) node_sync_via_rpc(self.nodes[0:3]) assert_equal(self.nodes[0].getbestblockhash(), hashZ) self.log.info("Mine blocks A-B-C on Node 0") - hashC = self.generatetoaddress(self.nodes[0], 3, gen_address(0))[-1] + hashC = self.generate(self.nodes[0], 3, sync_fun=self.no_op)[-1] assert_equal(self.nodes[0].getblockcount(), 5) self.log.info("Mine competing blocks E-F-G on Node 1") - hashG = self.generatetoaddress(self.nodes[1], 3, gen_address(1))[-1] + hashG = self.generate(self.nodes[1], 3, sync_fun=self.no_op)[-1] assert_equal(self.nodes[1].getblockcount(), 5) assert hashC != hashG self.log.info("Connect nodes and check no reorg occurs") @@ -83,7 +82,7 @@ class PreciousTest(BitcoinTestFramework): self.nodes[1].preciousblock(hashC) assert_equal(self.nodes[1].getbestblockhash(), hashC) self.log.info("Mine another block (E-F-G-)H on Node 0 and reorg Node 1") - self.generatetoaddress(self.nodes[0], 1, gen_address(0)) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) assert_equal(self.nodes[0].getblockcount(), 6) self.sync_blocks(self.nodes[0:2]) hashH = self.nodes[0].getbestblockhash() @@ -92,7 +91,7 @@ class PreciousTest(BitcoinTestFramework): self.nodes[1].preciousblock(hashC) assert_equal(self.nodes[1].getbestblockhash(), hashH) self.log.info("Mine competing blocks I-J-K-L on Node 2") - self.generatetoaddress(self.nodes[2], 4, gen_address(2)) + self.generate(self.nodes[2], 4, sync_fun=self.no_op) assert_equal(self.nodes[2].getblockcount(), 6) hashL = self.nodes[2].getbestblockhash() self.log.info("Connect nodes and check no reorg occurs") diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index b132ac3d31..a8034849cc 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the Partially Signed Transaction RPCs. @@ -31,7 +31,7 @@ class PSBTTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 3 self.extra_args = [ - ["-walletrbf=1"], + ["-walletrbf=1", "-addresstype=bech32", "-changetype=bech32"], #TODO: Remove address type restrictions once taproot has psbt extensions ["-walletrbf=0", "-changetype=legacy"], [] ] @@ -61,7 +61,6 @@ class PSBTTest(BitcoinTestFramework): wonline.importaddress(offline_addr, "", False) mining_node.sendtoaddress(address=offline_addr, amount=1.0) self.generate(mining_node, nblocks=1) - self.sync_blocks([mining_node, online_node]) # Construct an unsigned PSBT on the online node (who doesn't know the output is Segwit, so will include a non-witness UTXO) utxos = wonline.listunspent(addresses=[offline_addr]) @@ -76,7 +75,6 @@ class PSBTTest(BitcoinTestFramework): # Make sure we can mine the resulting transaction txid = mining_node.sendrawtransaction(mining_node.finalizepsbt(signed_psbt)["hex"]) self.generate(mining_node, 1) - self.sync_blocks([mining_node, online_node]) assert_equal(online_node.gettxout(txid,0)["confirmations"], 1) wonline.unloadwallet() @@ -122,7 +120,9 @@ class PSBTTest(BitcoinTestFramework): self.nodes[0].walletpassphrase(passphrase="password", timeout=1000000) # Sign the transaction and send - signed_tx = self.nodes[0].walletprocesspsbt(psbtx)['psbt'] + signed_tx = self.nodes[0].walletprocesspsbt(psbt=psbtx, finalize=False)['psbt'] + finalized_tx = self.nodes[0].walletprocesspsbt(psbt=psbtx, finalize=True)['psbt'] + assert signed_tx != finalized_tx final_tx = self.nodes[0].finalizepsbt(signed_tx)['hex'] self.nodes[0].sendrawtransaction(final_tx) @@ -162,7 +162,6 @@ class PSBTTest(BitcoinTestFramework): signed_tx = self.nodes[0].signrawtransactionwithwallet(rawtx['hex'])['hex'] txid = self.nodes[0].sendrawtransaction(signed_tx) self.generate(self.nodes[0], 6) - self.sync_all() # Find the output pos p2sh_pos = -1 @@ -321,7 +320,6 @@ class PSBTTest(BitcoinTestFramework): txid1 = self.nodes[0].sendtoaddress(node1_addr, 13) txid2 = self.nodes[0].sendtoaddress(node2_addr, 13) blockhash = self.generate(self.nodes[0], 6)[0] - self.sync_all() vout1 = find_output(self.nodes[1], txid1, 13, blockhash=blockhash) vout2 = find_output(self.nodes[2], txid2, 13, blockhash=blockhash) @@ -349,7 +347,6 @@ class PSBTTest(BitcoinTestFramework): finalized = self.nodes[0].finalizepsbt(combined)['hex'] self.nodes[0].sendrawtransaction(finalized) self.generate(self.nodes[0], 6) - self.sync_all() # Test additional args in walletcreatepsbt # Make sure both pre-included and funded inputs @@ -544,7 +541,6 @@ class PSBTTest(BitcoinTestFramework): txid4 = self.nodes[0].sendtoaddress(addr4, 5) vout4 = find_output(self.nodes[0], txid4, 5) self.generate(self.nodes[0], 6) - self.sync_all() psbt2 = self.nodes[1].createpsbt([{"txid":txid4, "vout":vout4}], {self.nodes[0].getnewaddress():Decimal('4.999')}) psbt2 = self.nodes[1].walletprocesspsbt(psbt2)['psbt'] psbt2_decoded = self.nodes[0].decodepsbt(psbt2) @@ -568,7 +564,6 @@ class PSBTTest(BitcoinTestFramework): txid = self.nodes[0].sendtoaddress(addr, 7) addrinfo = self.nodes[1].getaddressinfo(addr) blockhash = self.generate(self.nodes[0], 6)[0] - self.sync_all() vout = find_output(self.nodes[0], txid, 7, blockhash=blockhash) psbt = self.nodes[1].createpsbt([{"txid":txid, "vout":vout}], {self.nodes[0].getnewaddress("", "p2sh-segwit"):Decimal('6.999')}) analyzed = self.nodes[0].analyzepsbt(psbt) @@ -628,7 +623,6 @@ class PSBTTest(BitcoinTestFramework): self.nodes[0].sendtoaddress(addr, 10) self.generate(self.nodes[0], 6) - self.sync_all() ext_utxo = self.nodes[0].listunspent(addresses=[addr])[0] # An external input without solving data should result in an error diff --git a/test/functional/rpc_rawtransaction.py b/test/functional/rpc_rawtransaction.py index fc812340fa..96691b2686 100755 --- a/test/functional/rpc_rawtransaction.py +++ b/test/functional/rpc_rawtransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the rawtransaction RPCs. @@ -75,14 +75,11 @@ class RawTransactionsTest(BitcoinTestFramework): def run_test(self): self.log.info("Prepare some coins for multiple *rawtransaction commands") self.generate(self.nodes[2], 1) - self.sync_all() self.generate(self.nodes[0], COINBASE_MATURITY + 1) - self.sync_all() for amount in [1.5, 1.0, 5.0]: self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), amount) self.sync_all() self.generate(self.nodes[0], 5) - self.sync_all() self.getrawtransaction_tests() self.createrawtransaction_tests() @@ -98,13 +95,11 @@ class RawTransactionsTest(BitcoinTestFramework): addr = self.nodes[1].getnewaddress() txid = self.nodes[0].sendtoaddress(addr, 10) self.generate(self.nodes[0], 1) - self.sync_all() vout = find_vout_for_address(self.nodes[1], txid, addr) rawTx = self.nodes[1].createrawtransaction([{'txid': txid, 'vout': vout}], {self.nodes[1].getnewaddress(): 9.999}) rawTxSigned = self.nodes[1].signrawtransactionwithwallet(rawTx) txId = self.nodes[1].sendrawtransaction(rawTxSigned['hex']) self.generate(self.nodes[0], 1) - self.sync_all() for n in [0, 3]: self.log.info(f"Test getrawtransaction {'with' if n == 0 else 'without'} -txindex") @@ -137,7 +132,6 @@ class RawTransactionsTest(BitcoinTestFramework): # Make a tx by sending, then generate 2 blocks; block1 has the tx in it tx = self.nodes[2].sendtoaddress(self.nodes[1].getnewaddress(), 1) block1, block2 = self.generate(self.nodes[2], 2) - self.sync_all() for n in [0, 3]: self.log.info(f"Test getrawtransaction {'with' if n == 0 else 'without'} -txindex, with blockhash") # We should be able to get the raw transaction by providing the correct block @@ -369,7 +363,6 @@ class RawTransactionsTest(BitcoinTestFramework): self.log.info("Test sendrawtransaction/testmempoolaccept with tx already in the chain") self.generate(self.nodes[2], 1) - self.sync_blocks() for node in self.nodes: testres = node.testmempoolaccept([rawTxSigned['hex']])[0] assert_equal(testres['allowed'], False) @@ -443,7 +436,6 @@ class RawTransactionsTest(BitcoinTestFramework): txId = self.nodes[0].sendtoaddress(mSigObj, 1.2) self.sync_all() self.generate(self.nodes[0], 1) - self.sync_all() # node2 has both keys of the 2of2 ms addr, tx should affect the balance assert_equal(self.nodes[2].getbalance(), bal + Decimal('1.20000000')) @@ -465,7 +457,6 @@ class RawTransactionsTest(BitcoinTestFramework): rawTx = self.nodes[0].decoderawtransaction(decTx['hex']) self.sync_all() self.generate(self.nodes[0], 1) - self.sync_all() # THIS IS AN INCOMPLETE FEATURE # NODE2 HAS TWO OF THREE KEYS AND THE FUNDS SHOULD BE SPENDABLE AND COUNT AT BALANCE CALCULATION @@ -488,7 +479,6 @@ class RawTransactionsTest(BitcoinTestFramework): rawTx = self.nodes[0].decoderawtransaction(rawTxSigned['hex']) self.sync_all() self.generate(self.nodes[0], 1) - self.sync_all() assert_equal(self.nodes[0].getbalance(), bal + Decimal('50.00000000') + Decimal('2.19000000')) # block reward + tx # 2of2 test for combining transactions @@ -508,7 +498,6 @@ class RawTransactionsTest(BitcoinTestFramework): rawTx2 = self.nodes[0].decoderawtransaction(decTx['hex']) self.sync_all() self.generate(self.nodes[0], 1) - self.sync_all() assert_equal(self.nodes[2].getbalance(), bal) # the funds of a 2of2 multisig tx should not be marked as spendable @@ -533,7 +522,6 @@ class RawTransactionsTest(BitcoinTestFramework): rawTx2 = self.nodes[0].decoderawtransaction(rawTxComb) self.sync_all() self.generate(self.nodes[0], 1) - self.sync_all() assert_equal(self.nodes[0].getbalance(), bal + Decimal('50.00000000') + Decimal('2.19000000')) # block reward + tx diff --git a/test/functional/rpc_scantxoutset.py b/test/functional/rpc_scantxoutset.py index ec8205acd5..8703bfab8e 100755 --- a/test/functional/rpc_scantxoutset.py +++ b/test/functional/rpc_scantxoutset.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the scantxoutset rpc call.""" diff --git a/test/functional/rpc_setban.py b/test/functional/rpc_setban.py index 36873f964b..97354f480c 100755 --- a/test/functional/rpc_setban.py +++ b/test/functional/rpc_setban.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the setban rpc call.""" diff --git a/test/functional/rpc_signer.py b/test/functional/rpc_signer.py index 5c3722ef8f..f1107197c5 100755 --- a/test/functional/rpc_signer.py +++ b/test/functional/rpc_signer.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test external signer. diff --git a/test/functional/rpc_signmessagewithprivkey.py b/test/functional/rpc_signmessagewithprivkey.py index 95beba8730..80555eab75 100755 --- a/test/functional/rpc_signmessagewithprivkey.py +++ b/test/functional/rpc_signmessagewithprivkey.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test RPC commands for signing messages with private key.""" diff --git a/test/functional/rpc_signrawtransaction.py b/test/functional/rpc_signrawtransaction.py index c519d0c7d1..e648040278 100755 --- a/test/functional/rpc_signrawtransaction.py +++ b/test/functional/rpc_signrawtransaction.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test transaction signing using the signrawtransaction* RPCs.""" @@ -203,7 +203,6 @@ class SignRawTransactionsTest(BitcoinTestFramework): self.generate(self.nodes[0], COINBASE_MATURITY + 1) self.nodes[0].sendtoaddress(p2sh_p2wsh_address["address"], 49.999) self.generate(self.nodes[0], 1) - self.sync_all() # Get the UTXO info from scantxoutset unspent_output = self.nodes[1].scantxoutset('start', [p2sh_p2wsh_address['descriptor']])['unspents'][0] spk = script_to_p2sh_p2wsh_script(p2sh_p2wsh_address['redeemScript']).hex() diff --git a/test/functional/rpc_txoutproof.py b/test/functional/rpc_txoutproof.py index a45694328a..d04d05962f 100755 --- a/test/functional/rpc_txoutproof.py +++ b/test/functional/rpc_txoutproof.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test gettxoutproof and verifytxoutproof RPCs.""" @@ -31,7 +31,6 @@ class MerkleBlockTest(BitcoinTestFramework): # Add enough mature utxos to the wallet, so that all txs spend confirmed coins self.generate(miniwallet, 5) self.generate(self.nodes[0], COINBASE_MATURITY) - self.sync_all() chain_height = self.nodes[1].getblockcount() assert_equal(chain_height, 105) @@ -43,7 +42,6 @@ class MerkleBlockTest(BitcoinTestFramework): self.generate(self.nodes[0], 1) blockhash = self.nodes[0].getblockhash(chain_height + 1) - self.sync_all() txlist = [] blocktxn = self.nodes[0].getblock(blockhash, True)["tx"] @@ -54,11 +52,10 @@ class MerkleBlockTest(BitcoinTestFramework): assert_equal(self.nodes[0].verifytxoutproof(self.nodes[0].gettxoutproof([txid1, txid2])), txlist) assert_equal(self.nodes[0].verifytxoutproof(self.nodes[0].gettxoutproof([txid1, txid2], blockhash)), txlist) - txin_spent = miniwallet.get_utxo() # Get the change from txid2 + txin_spent = miniwallet.get_utxo(txid=txid2) # Get the change from txid2 tx3 = miniwallet.send_self_transfer(from_node=self.nodes[0], utxo_to_spend=txin_spent) txid3 = tx3['txid'] self.generate(self.nodes[0], 1) - self.sync_all() txid_spent = txin_spent["txid"] txid_unspent = txid1 # Input was change from txid2, so txid1 should be unspent diff --git a/test/functional/rpc_uptime.py b/test/functional/rpc_uptime.py index 6177970872..1a82d1fa41 100755 --- a/test/functional/rpc_uptime.py +++ b/test/functional/rpc_uptime.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2019 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the RPC call related to the uptime command. diff --git a/test/functional/test_framework/address.py b/test/functional/test_framework/address.py index fe733e9368..013522a5e1 100644 --- a/test/functional/test_framework/address.py +++ b/test/functional/test_framework/address.py @@ -1,16 +1,25 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2020 The Bitcoin Core developers +# Copyright (c) 2016-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Encode and decode Bitcoin addresses. - base58 P2PKH and P2SH addresses. -- bech32 segwit v0 P2WPKH and P2WSH addresses.""" +- bech32 segwit v0 P2WPKH and P2WSH addresses. +- bech32m segwit v1 P2TR addresses.""" import enum import unittest -from .script import hash256, hash160, sha256, CScript, OP_0 +from .script import ( + CScript, + OP_0, + OP_TRUE, + hash160, + hash256, + sha256, + taproot_construct, +) from .segwit_addr import encode_segwit_address from .util import assert_equal @@ -29,6 +38,21 @@ class AddressType(enum.Enum): chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' +def create_deterministic_address_bcrt1_p2tr_op_true(): + """ + Generates a deterministic bech32m address (segwit v1 output) that + can be spent with a witness stack of OP_TRUE and the control block + with internal public key (script-path spending). + + Returns a tuple with the generated address and the internal key. + """ + internal_key = (1).to_bytes(32, 'big') + scriptPubKey = taproot_construct(internal_key, [(None, CScript([OP_TRUE]))]).scriptPubKey + address = encode_segwit_address("bcrt", 1, scriptPubKey[2:]) + assert_equal(address, 'bcrt1p9yfmy5h72durp7zrhlw9lf7jpwjgvwdg0jr0lqmmjtgg83266lqsekaqka') + return (address, internal_key) + + def byte_to_base58(b, version): result = '' str = b.hex() diff --git a/test/functional/test_framework/bdb.py b/test/functional/test_framework/bdb.py index d623bcdf6e..41886c09fd 100644 --- a/test/functional/test_framework/bdb.py +++ b/test/functional/test_framework/bdb.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """ diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 5d0113465f..caa9b86969 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Utilities for manipulating blocks and transactions.""" @@ -50,6 +50,8 @@ MAX_BLOCK_SIGOPS_WEIGHT = MAX_BLOCK_SIGOPS * WITNESS_SCALE_FACTOR # Genesis block time (regtest) TIME_GENESIS_BLOCK = 1296688602 +MAX_FUTURE_BLOCK_TIME = 2 * 60 * 60 + # Coinbase transaction outputs can only be spent after this number of new blocks (network rule) COINBASE_MATURITY = 100 diff --git a/test/functional/test_framework/coverage.py b/test/functional/test_framework/coverage.py index ad8cfe5c9a..4fb4f8bb82 100644 --- a/test/functional/test_framework/coverage.py +++ b/test/functional/test_framework/coverage.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Utilities for doing coverage analysis on the RPC interface. diff --git a/test/functional/test_framework/key.py b/test/functional/test_framework/key.py index 26526e35fa..e5dea66963 100644 --- a/test/functional/test_framework/key.py +++ b/test/functional/test_framework/key.py @@ -8,6 +8,7 @@ keys, and is trivially vulnerable to side channel attacks. Do not use for anything but tests.""" import csv import hashlib +import hmac import os import random import unittest @@ -326,6 +327,16 @@ def generate_privkey(): """Generate a valid random 32-byte private key.""" return random.randrange(1, SECP256K1_ORDER).to_bytes(32, 'big') +def rfc6979_nonce(key): + """Compute signing nonce using RFC6979.""" + v = bytes([1] * 32) + k = bytes([0] * 32) + k = hmac.new(k, v + b"\x00" + key, 'sha256').digest() + v = hmac.new(k, v, 'sha256').digest() + k = hmac.new(k, v + b"\x01" + key, 'sha256').digest() + v = hmac.new(k, v, 'sha256').digest() + return hmac.new(k, v, 'sha256').digest() + class ECKey(): """A secp256k1 private key""" @@ -368,15 +379,18 @@ class ECKey(): ret.compressed = self.compressed return ret - def sign_ecdsa(self, msg, low_s=True): + def sign_ecdsa(self, msg, low_s=True, rfc6979=False): """Construct a DER-encoded ECDSA signature with this key. See https://en.wikipedia.org/wiki/Elliptic_Curve_Digital_Signature_Algorithm for the ECDSA signer algorithm.""" assert(self.valid) z = int.from_bytes(msg, 'big') - # Note: no RFC6979, but a simple random nonce (some tests rely on distinct transactions for the same operation) - k = random.randrange(1, SECP256K1_ORDER) + # Note: no RFC6979 by default, but a simple random nonce (some tests rely on distinct transactions for the same operation) + if rfc6979: + k = int.from_bytes(rfc6979_nonce(self.secret.to_bytes(32, 'big') + msg), 'big') + else: + k = random.randrange(1, SECP256K1_ORDER) R = SECP256K1.affine(SECP256K1.mul([(SECP256K1_G, k)])) r = R[0] % SECP256K1_ORDER s = (modinv(k, SECP256K1_ORDER) * (z + self.secret * r)) % SECP256K1_ORDER diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 65d90f8448..71ac5c5bfd 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright (c) 2010 ArtForz -- public domain half-a-node # Copyright (c) 2012 Jeff Garzik -# Copyright (c) 2010-2020 The Bitcoin Core developers +# Copyright (c) 2010-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Bitcoin test framework primitive and message structures diff --git a/test/functional/test_framework/netutil.py b/test/functional/test_framework/netutil.py index b5f78e0cf3..174dc44a2a 100644 --- a/test/functional/test_framework/netutil.py +++ b/test/functional/test_framework/netutil.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Linux network utilities. diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index 78c63b57a1..251d3d5eae 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright (c) 2010 ArtForz -- public domain half-a-node # Copyright (c) 2012 Jeff Garzik -# Copyright (c) 2010-2020 The Bitcoin Core developers +# Copyright (c) 2010-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test objects for interacting with a bitcoind node over the p2p protocol. @@ -89,6 +89,8 @@ P2P_SERVICES = NODE_NETWORK | NODE_WITNESS P2P_SUBVERSION = "/python-p2p-tester:0.0.3/" # Value for relay that this test framework sends in its `version` message P2P_VERSION_RELAY = 1 +# Delay after receiving a tx inv before requesting transactions from non-preferred peers, in seconds +NONPREF_PEER_TX_DELAY = 2 MESSAGEMAP = { b"addr": msg_addr, diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 3c9b8a6e69..947a1f9808 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Functionality to build scripts, as well as signature hash functions. @@ -619,16 +619,15 @@ def FindAndDelete(script, sig): r += script[last_sop_idx:] return CScript(r) -def LegacySignatureHash(script, txTo, inIdx, hashtype): - """Consensus-correct SignatureHash +def LegacySignatureMsg(script, txTo, inIdx, hashtype): + """Preimage of the signature hash, if it exists. - Returns (hash, err) to precisely match the consensus-critical behavior of - the SIGHASH_SINGLE bug. (inIdx is *not* checked for validity) + Returns either (None, err) to indicate error (which translates to sighash 1), + or (msg, None). """ - HASH_ONE = b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' if inIdx >= len(txTo.vin): - return (HASH_ONE, "inIdx %d out of range (%d)" % (inIdx, len(txTo.vin))) + return (None, "inIdx %d out of range (%d)" % (inIdx, len(txTo.vin))) txtmp = CTransaction(txTo) for txin in txtmp.vin: @@ -645,7 +644,7 @@ def LegacySignatureHash(script, txTo, inIdx, hashtype): elif (hashtype & 0x1f) == SIGHASH_SINGLE: outIdx = inIdx if outIdx >= len(txtmp.vout): - return (HASH_ONE, "outIdx %d out of range (%d)" % (outIdx, len(txtmp.vout))) + return (None, "outIdx %d out of range (%d)" % (outIdx, len(txtmp.vout))) tmp = txtmp.vout[outIdx] txtmp.vout = [] @@ -665,15 +664,27 @@ def LegacySignatureHash(script, txTo, inIdx, hashtype): s = txtmp.serialize_without_witness() s += struct.pack(b"<I", hashtype) - hash = hash256(s) + return (s, None) + +def LegacySignatureHash(*args, **kwargs): + """Consensus-correct SignatureHash + + Returns (hash, err) to precisely match the consensus-critical behavior of + the SIGHASH_SINGLE bug. (inIdx is *not* checked for validity) + """ - return (hash, None) + HASH_ONE = b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + msg, err = LegacySignatureMsg(*args, **kwargs) + if msg is None: + return (HASH_ONE, err) + else: + return (hash256(msg), err) # TODO: Allow cached hashPrevouts/hashSequence/hashOutputs to be provided. # Performance optimization probably not necessary for python tests, however. # Note that this corresponds to sigversion == 1 in EvalScript, which is used # for version 0 witnesses. -def SegwitV0SignatureHash(script, txTo, inIdx, hashtype, amount): +def SegwitV0SignatureMsg(script, txTo, inIdx, hashtype, amount): hashPrevouts = 0 hashSequence = 0 @@ -711,8 +722,10 @@ def SegwitV0SignatureHash(script, txTo, inIdx, hashtype, amount): ss += ser_uint256(hashOutputs) ss += struct.pack("<i", txTo.nLockTime) ss += struct.pack("<I", hashtype) + return ss - return hash256(ss) +def SegwitV0SignatureHash(*args, **kwargs): + return hash256(SegwitV0SignatureMsg(*args, **kwargs)) class TestFrameworkScript(unittest.TestCase): def test_bn2vch(self): @@ -742,7 +755,22 @@ class TestFrameworkScript(unittest.TestCase): for value in values: self.assertEqual(CScriptNum.decode(CScriptNum.encode(CScriptNum(value))), value) -def TaprootSignatureHash(txTo, spent_utxos, hash_type, input_index = 0, scriptpath = False, script = CScript(), codeseparator_pos = -1, annex = None, leaf_ver = LEAF_VERSION_TAPSCRIPT): +def BIP341_sha_prevouts(txTo): + return sha256(b"".join(i.prevout.serialize() for i in txTo.vin)) + +def BIP341_sha_amounts(spent_utxos): + return sha256(b"".join(struct.pack("<q", u.nValue) for u in spent_utxos)) + +def BIP341_sha_scriptpubkeys(spent_utxos): + return sha256(b"".join(ser_string(u.scriptPubKey) for u in spent_utxos)) + +def BIP341_sha_sequences(txTo): + return sha256(b"".join(struct.pack("<I", i.nSequence) for i in txTo.vin)) + +def BIP341_sha_outputs(txTo): + return sha256(b"".join(o.serialize() for o in txTo.vout)) + +def TaprootSignatureMsg(txTo, spent_utxos, hash_type, input_index = 0, scriptpath = False, script = CScript(), codeseparator_pos = -1, annex = None, leaf_ver = LEAF_VERSION_TAPSCRIPT): assert (len(txTo.vin) == len(spent_utxos)) assert (input_index < len(txTo.vin)) out_type = SIGHASH_ALL if hash_type == 0 else hash_type & 3 @@ -752,12 +780,12 @@ def TaprootSignatureHash(txTo, spent_utxos, hash_type, input_index = 0, scriptpa ss += struct.pack("<i", txTo.nVersion) ss += struct.pack("<I", txTo.nLockTime) if in_type != SIGHASH_ANYONECANPAY: - ss += sha256(b"".join(i.prevout.serialize() for i in txTo.vin)) - ss += sha256(b"".join(struct.pack("<q", u.nValue) for u in spent_utxos)) - ss += sha256(b"".join(ser_string(u.scriptPubKey) for u in spent_utxos)) - ss += sha256(b"".join(struct.pack("<I", i.nSequence) for i in txTo.vin)) + ss += BIP341_sha_prevouts(txTo) + ss += BIP341_sha_amounts(spent_utxos) + ss += BIP341_sha_scriptpubkeys(spent_utxos) + ss += BIP341_sha_sequences(txTo) if out_type == SIGHASH_ALL: - ss += sha256(b"".join(o.serialize() for o in txTo.vout)) + ss += BIP341_sha_outputs(txTo) spend_type = 0 if annex is not None: spend_type |= 1 @@ -783,7 +811,10 @@ def TaprootSignatureHash(txTo, spent_utxos, hash_type, input_index = 0, scriptpa ss += bytes([0]) ss += struct.pack("<i", codeseparator_pos) assert len(ss) == 175 - (in_type == SIGHASH_ANYONECANPAY) * 49 - (out_type != SIGHASH_ALL and out_type != SIGHASH_SINGLE) * 32 + (annex is not None) * 32 + scriptpath * 37 - return TaggedHash("TapSighash", ss) + return ss + +def TaprootSignatureHash(*args, **kwargs): + return TaggedHash("TapSighash", TaprootSignatureMsg(*args, **kwargs)) def taproot_tree_helper(scripts): if len(scripts) == 0: @@ -805,20 +836,20 @@ def taproot_tree_helper(scripts): h = TaggedHash("TapLeaf", bytes([version]) + ser_string(code)) if name is None: return ([], h) - return ([(name, version, code, bytes())], h) + return ([(name, version, code, bytes(), h)], h) elif len(scripts) == 2 and callable(scripts[1]): # Two entries, and the right one is a function left, left_h = taproot_tree_helper(scripts[0:1]) right_h = scripts[1](left_h) - left = [(name, version, script, control + right_h) for name, version, script, control in left] + left = [(name, version, script, control + right_h, leaf) for name, version, script, control, leaf in left] right = [] else: # Two or more entries: descend into each side split_pos = len(scripts) // 2 left, left_h = taproot_tree_helper(scripts[0:split_pos]) right, right_h = taproot_tree_helper(scripts[split_pos:]) - left = [(name, version, script, control + right_h) for name, version, script, control in left] - right = [(name, version, script, control + left_h) for name, version, script, control in right] + left = [(name, version, script, control + right_h, leaf) for name, version, script, control, leaf in left] + right = [(name, version, script, control + left_h, leaf) for name, version, script, control, leaf in right] if right_h < left_h: right_h, left_h = left_h, right_h h = TaggedHash("TapBranch", left_h + right_h) @@ -830,13 +861,14 @@ def taproot_tree_helper(scripts): # - negflag: whether the pubkey in the scriptPubKey was negated from internal_pubkey+tweak*G (bool). # - tweak: the tweak (32 bytes) # - leaves: a dict of name -> TaprootLeafInfo objects for all known leaves -TaprootInfo = namedtuple("TaprootInfo", "scriptPubKey,internal_pubkey,negflag,tweak,leaves") +# - merkle_root: the script tree's Merkle root, or bytes() if no leaves are present +TaprootInfo = namedtuple("TaprootInfo", "scriptPubKey,internal_pubkey,negflag,tweak,leaves,merkle_root,output_pubkey") # A TaprootLeafInfo object has the following fields: # - script: the leaf script (CScript or bytes) # - version: the leaf version (0xc0 for BIP342 tapscript) # - merklebranch: the merkle branch to use for this leaf (32*N bytes) -TaprootLeafInfo = namedtuple("TaprootLeafInfo", "script,version,merklebranch") +TaprootLeafInfo = namedtuple("TaprootLeafInfo", "script,version,merklebranch,leaf_hash") def taproot_construct(pubkey, scripts=None): """Construct a tree of Taproot spending conditions @@ -858,8 +890,8 @@ def taproot_construct(pubkey, scripts=None): ret, h = taproot_tree_helper(scripts) tweak = TaggedHash("TapTweak", pubkey + h) tweaked, negated = tweak_add_pubkey(pubkey, tweak) - leaves = dict((name, TaprootLeafInfo(script, version, merklebranch)) for name, version, script, merklebranch in ret) - return TaprootInfo(CScript([OP_1, tweaked]), pubkey, negated + 0, tweak, leaves) + leaves = dict((name, TaprootLeafInfo(script, version, merklebranch, leaf)) for name, version, script, merklebranch, leaf in ret) + return TaprootInfo(CScript([OP_1, tweaked]), pubkey, negated + 0, tweak, leaves, h, tweaked) def is_op_success(o): return o == 0x50 or o == 0x62 or o == 0x89 or o == 0x8a or o == 0x8d or o == 0x8e or (o >= 0x7e and o <= 0x81) or (o >= 0x83 and o <= 0x86) or (o >= 0x95 and o <= 0x99) or (o >= 0xbb and o <= 0xfe) diff --git a/test/functional/test_framework/script_util.py b/test/functional/test_framework/script_util.py index cbc4a560db..f7d8422eee 100755 --- a/test/functional/test_framework/script_util.py +++ b/test/functional/test_framework/script_util.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Useful Script constants and utils.""" diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index ec3561b1f2..6746fbf9f9 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Base class for RPC testing.""" @@ -19,7 +19,7 @@ import tempfile import time from typing import List -from .address import ADDRESS_BCRT1_P2WSH_OP_TRUE +from .address import create_deterministic_address_bcrt1_p2tr_op_true from .authproxy import JSONRPCException from . import coverage from .p2p import NetworkThread @@ -101,7 +101,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.supports_cli = True self.bind_to_localhost_only = True self.parse_args() - self.disable_syscall_sandbox = self.options.nosandbox + self.disable_syscall_sandbox = self.options.nosandbox or self.options.valgrind self.default_wallet_name = "default_wallet" if self.options.descriptors else "" self.wallet_data_filename = "wallet.dat" # Optional list of wallet names that can be set in set_test_params to @@ -188,7 +188,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): parser.add_argument("--perf", dest="perf", default=False, action="store_true", help="profile running nodes with perf for the duration of the test") parser.add_argument("--valgrind", dest="valgrind", default=False, action="store_true", - help="run nodes under the valgrind memory error detector: expect at least a ~10x slowdown, valgrind 3.14 or later required") + help="run nodes under the valgrind memory error detector: expect at least a ~10x slowdown. valgrind 3.14 or later required. Forces --nosandbox.") parser.add_argument("--randomseed", type=int, help="set a random seed for deterministically reproducing a previous test run") parser.add_argument('--timeout-factor', dest="timeout_factor", type=float, default=1.0, help='adjust test timeouts by a factor. Setting it to 0 disables all timeouts') @@ -413,7 +413,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # To ensure that all nodes are out of IBD, the most recent block # must have a timestamp not too old (see IsInitialBlockDownload()). self.log.debug('Generate a block with current time') - block_hash = self.generate(self.nodes[0], 1)[0] + block_hash = self.generate(self.nodes[0], 1, sync_fun=self.no_op)[0] block = self.nodes[0].getblock(blockhash=block_hash, verbosity=0) for n in self.nodes: n.submitblock(block) @@ -431,7 +431,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): n = self.nodes[node] if wallet_name is not None: n.createwallet(wallet_name=wallet_name, descriptors=self.options.descriptors, load_on_startup=True) - n.importprivkey(privkey=n.get_deterministic_priv_key().key, label='coinbase') + n.importprivkey(privkey=n.get_deterministic_priv_key().key, label='coinbase', rescan=True) def run_test(self): """Tests must override this method to define test logic""" @@ -627,20 +627,27 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.connect_nodes(1, 2) self.sync_all() - def generate(self, generator, *args, **kwargs): + def no_op(self): + pass + + def generate(self, generator, *args, sync_fun=None, **kwargs): blocks = generator.generate(*args, invalid_call=False, **kwargs) + sync_fun() if sync_fun else self.sync_all() return blocks - def generateblock(self, generator, *args, **kwargs): + def generateblock(self, generator, *args, sync_fun=None, **kwargs): blocks = generator.generateblock(*args, invalid_call=False, **kwargs) + sync_fun() if sync_fun else self.sync_all() return blocks - def generatetoaddress(self, generator, *args, **kwargs): + def generatetoaddress(self, generator, *args, sync_fun=None, **kwargs): blocks = generator.generatetoaddress(*args, invalid_call=False, **kwargs) + sync_fun() if sync_fun else self.sync_all() return blocks - def generatetodescriptor(self, generator, *args, **kwargs): + def generatetodescriptor(self, generator, *args, sync_fun=None, **kwargs): blocks = generator.generatetodescriptor(*args, invalid_call=False, **kwargs) + sync_fun() if sync_fun else self.sync_all() return blocks def sync_blocks(self, nodes=None, wait=1, timeout=60): @@ -770,7 +777,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): # block in the cache does not age too much (have an old tip age). # This is needed so that we are out of IBD when the test starts, # see the tip age check in IsInitialBlockDownload(). - gen_addresses = [k.address for k in TestNode.PRIV_KEYS][:3] + [ADDRESS_BCRT1_P2WSH_OP_TRUE] + gen_addresses = [k.address for k in TestNode.PRIV_KEYS][:3] + [create_deterministic_address_bcrt1_p2tr_op_true()[0]] assert_equal(len(gen_addresses), 4) for i in range(8): self.generatetoaddress( diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index e8ff41a46d..b3279666b2 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Class for bitcoind node under test""" @@ -20,6 +20,7 @@ import urllib.parse import collections import shlex import sys +from pathlib import Path from .authproxy import JSONRPCException from .descriptors import descsum_create @@ -380,13 +381,20 @@ class TestNode(): def wait_until_stopped(self, timeout=BITCOIND_PROC_WAIT_TIMEOUT): wait_until_helper(self.is_node_stopped, timeout=timeout, timeout_factor=self.timeout_factor) + @property + def chain_path(self) -> Path: + return Path(self.datadir) / self.chain + + @property + def debug_log_path(self) -> Path: + return self.chain_path / 'debug.log' + @contextlib.contextmanager def assert_debug_log(self, expected_msgs, unexpected_msgs=None, timeout=2): if unexpected_msgs is None: unexpected_msgs = [] time_end = time.time() + timeout * self.timeout_factor - debug_log = os.path.join(self.datadir, self.chain, 'debug.log') - with open(debug_log, encoding='utf-8') as dl: + with open(self.debug_log_path, encoding='utf-8') as dl: dl.seek(0, 2) prev_size = dl.tell() @@ -394,7 +402,7 @@ class TestNode(): while True: found = True - with open(debug_log, encoding='utf-8') as dl: + with open(self.debug_log_path, encoding='utf-8') as dl: dl.seek(prev_size) log = dl.read() print_log = " - " + "\n - ".join(log.splitlines()) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 9f5bca6884..c3a9029d55 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Helpful routines for regression testing.""" @@ -36,12 +36,12 @@ def assert_approx(v, vexp, vspan=0.00001): def assert_fee_amount(fee, tx_size, feerate_BTC_kvB): """Assert the fee is in range.""" - feerate_BTC_vB = feerate_BTC_kvB / 1000 - target_fee = satoshi_round(tx_size * feerate_BTC_vB) + target_fee = get_fee(tx_size, feerate_BTC_kvB) if fee < target_fee: raise AssertionError("Fee of %s BTC too low! (Should be %s BTC)" % (str(fee), str(target_fee))) # allow the wallet's estimation to be at most 2 bytes off - if fee > (tx_size + 2) * feerate_BTC_vB: + high_fee = get_fee(tx_size + 2, feerate_BTC_kvB) + if fee > high_fee: raise AssertionError("Fee of %s BTC too high! (Should be %s BTC)" % (str(fee), str(target_fee))) @@ -218,6 +218,18 @@ def str_to_b64str(string): return b64encode(string.encode('utf-8')).decode('ascii') +def ceildiv(a, b): + """Divide 2 ints and round up to next int rather than round down""" + return -(-a // b) + + +def get_fee(tx_size, feerate_btc_kvb): + """Calculate the fee in BTC given a feerate is BTC/kvB. Reflects CFeeRate::GetFee""" + feerate_sat_kvb = int(feerate_btc_kvb * Decimal(1e8)) # Fee in sat/kvb as an int to avoid float precision errors + target_fee_sat = ceildiv(feerate_sat_kvb * tx_size, 1000) # Round calculated fee up to nearest sat + return satoshi_round(target_fee_sat / Decimal(1e8)) # Truncate BTC result to nearest sat + + def satoshi_round(amount): return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN) @@ -257,6 +269,7 @@ def wait_until_helper(predicate, *, attempts=float('inf'), timeout=float('inf'), raise AssertionError("Predicate {} not true after {} seconds".format(predicate_source, timeout)) raise RuntimeError('Unreachable') + def sha256sum_file(filename): h = hashlib.sha256() with open(filename, 'rb') as f: @@ -433,6 +446,12 @@ def set_node_times(nodes, t): node.setmocktime(t) +def check_node_connections(*, node, num_in, num_out): + info = node.getnetworkinfo() + assert_equal(info["connections_in"], num_in) + assert_equal(info["connections_out"], num_out) + + # Transaction/Block functions ############################# @@ -451,10 +470,10 @@ def find_output(node, txid, amount, *, blockhash=None): # Helper to create at least "count" utxos # Pass in a fee that is sufficient for relay and mining new transactions. -def create_confirmed_utxos(test_framework, fee, node, count): +def create_confirmed_utxos(test_framework, fee, node, count, **kwargs): to_generate = int(0.5 * count) + 101 while to_generate > 0: - test_framework.generate(node, min(25, to_generate)) + test_framework.generate(node, min(25, to_generate), **kwargs) to_generate -= 25 utxos = node.listunspent() iterations = count - len(utxos) @@ -475,7 +494,7 @@ def create_confirmed_utxos(test_framework, fee, node, count): node.sendrawtransaction(signed_tx) while (node.getmempoolinfo()['size'] > 0): - test_framework.generate(node, 1) + test_framework.generate(node, 1, **kwargs) utxos = node.listunspent() assert len(utxos) >= count diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index d2c21f3525..f724cb2af3 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """A limited-functionality wallet, which may replace a real wallet in tests""" @@ -9,7 +9,7 @@ from decimal import Decimal from enum import Enum from random import choice from typing import Optional -from test_framework.address import ADDRESS_BCRT1_P2WSH_OP_TRUE +from test_framework.address import create_deterministic_address_bcrt1_p2tr_op_true from test_framework.descriptors import descsum_create from test_framework.key import ECKey from test_framework.messages import ( @@ -24,8 +24,9 @@ from test_framework.messages import ( from test_framework.script import ( CScript, LegacySignatureHash, - OP_TRUE, + LEAF_VERSION_TAPSCRIPT, OP_NOP, + OP_TRUE, SIGHASH_ALL, ) from test_framework.script_util import ( @@ -43,7 +44,7 @@ class MiniWalletMode(Enum): """Determines the transaction type the MiniWallet is creating and spending. For most purposes, the default mode ADDRESS_OP_TRUE should be sufficient; - it simply uses a fixed bech32 P2WSH address whose coins are spent with a + it simply uses a fixed bech32m P2TR address whose coins are spent with a witness stack of OP_TRUE, i.e. following an anyone-can-spend policy. However, if the transactions need to be modified by the user (e.g. prepending scriptSig for testing opcodes that are activated by a soft-fork), or the txs @@ -53,7 +54,7 @@ class MiniWalletMode(Enum): | output | | tx is | can modify | needs mode | description | address | standard | scriptSig | signing ----------------+-------------------+-----------+----------+------------+---------- - ADDRESS_OP_TRUE | anyone-can-spend | bech32 | yes | no | no + ADDRESS_OP_TRUE | anyone-can-spend | bech32m | yes | no | no RAW_OP_TRUE | anyone-can-spend | - (raw) | no | yes | no RAW_P2PK | pay-to-public-key | - (raw) | yes | yes | yes """ @@ -79,7 +80,7 @@ class MiniWallet: pub_key = self._priv_key.get_pubkey() self._scriptPubKey = key_to_p2pk_script(pub_key.get_bytes()) elif mode == MiniWalletMode.ADDRESS_OP_TRUE: - self._address = ADDRESS_BCRT1_P2WSH_OP_TRUE + self._address, self._internal_key = create_deterministic_address_bcrt1_p2tr_op_true() self._scriptPubKey = bytes.fromhex(self._test_node.validateaddress(self._address)['scriptPubKey']) def rescan_utxos(self): @@ -132,10 +133,9 @@ class MiniWallet: Args: txid: get the first utxo we find from a specific transaction - - Note: Can be used to get the change output immediately after a send_self_transfer """ index = -1 # by default the last utxo + self._utxos = sorted(self._utxos, key=lambda k: (k['value'], -k['height'])) # Put the largest utxo last if txid: utxo = next(filter(lambda utxo: txid == utxo['txid'], self._utxos)) index = self._utxos.index(utxo) @@ -171,10 +171,9 @@ class MiniWallet: def create_self_transfer(self, *, fee_rate=Decimal("0.003"), from_node, utxo_to_spend=None, mempool_valid=True, locktime=0, sequence=0): """Create and return a tx with the specified fee_rate. Fee may be exact or at most one satoshi higher than needed.""" - self._utxos = sorted(self._utxos, key=lambda k: (k['value'], -k['height'])) - utxo_to_spend = utxo_to_spend or self._utxos.pop() # Pick the largest utxo (if none provided) and hope it covers the fee + utxo_to_spend = utxo_to_spend or self.get_utxo() if self._priv_key is None: - vsize = Decimal(96) # anyone-can-spend + vsize = Decimal(104) # anyone-can-spend else: vsize = Decimal(168) # P2PK (73 bytes scriptSig + 35 bytes scriptPubKey + 60 bytes other) send_value = int(COIN * (utxo_to_spend['value'] - fee_rate * (vsize / 1000))) @@ -191,10 +190,10 @@ class MiniWallet: self.sign_tx(tx) else: # anyone-can-spend - tx.vin[0].scriptSig = CScript([OP_NOP] * 35) # pad to identical size + tx.vin[0].scriptSig = CScript([OP_NOP] * 43) # pad to identical size else: tx.wit.vtxinwit = [CTxInWitness()] - tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE])] + tx.wit.vtxinwit[0].scriptWitness.stack = [CScript([OP_TRUE]), bytes([LEAF_VERSION_TAPSCRIPT]) + self._internal_key] tx_hex = tx.serialize().hex() tx_info = from_node.testmempoolaccept([tx_hex])[0] diff --git a/test/functional/test_framework/wallet_util.py b/test/functional/test_framework/wallet_util.py index c307ded542..410d85cd8c 100755 --- a/test/functional/test_framework/wallet_util.py +++ b/test/functional/test_framework/wallet_util.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Useful util functions for testing the wallet""" diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 2fc5d2c56e..8e7e791538 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Run regression test suite. @@ -138,7 +138,8 @@ BASE_SCRIPTS = [ 'feature_fee_estimation.py', 'interface_zmq.py', 'rpc_invalid_address_message.py', - 'interface_bitcoin_cli.py', + 'interface_bitcoin_cli.py --legacy-wallet', + 'interface_bitcoin_cli.py --descriptors', 'feature_bind_extra.py', 'mempool_resurrect.py', 'wallet_txn_doublespend.py --mineblock', @@ -178,7 +179,8 @@ BASE_SCRIPTS = [ 'rpc_rawtransaction.py --legacy-wallet', 'rpc_rawtransaction.py --descriptors', 'wallet_groups.py --legacy-wallet', - 'wallet_transactiontime_rescan.py', + 'wallet_transactiontime_rescan.py --descriptors', + 'wallet_transactiontime_rescan.py --legacy-wallet', 'p2p_addrv2_relay.py', 'wallet_groups.py --descriptors', 'p2p_compactblocks_hb.py', @@ -217,7 +219,8 @@ BASE_SCRIPTS = [ 'rpc_getblockfilter.py', 'rpc_invalidateblock.py', 'feature_utxo_set_hash.py', - 'feature_rbf.py', + 'feature_rbf.py --legacy-wallet', + 'feature_rbf.py --descriptors', 'mempool_packages.py', 'mempool_package_onemore.py', 'rpc_createmultisig.py --legacy-wallet', @@ -277,6 +280,7 @@ BASE_SCRIPTS = [ 'wallet_taproot.py', 'p2p_fingerprint.py', 'feature_uacomment.py', + 'feature_init.py', 'wallet_coinbase_category.py --legacy-wallet', 'wallet_coinbase_category.py --descriptors', 'feature_filelock.py', @@ -296,6 +300,7 @@ BASE_SCRIPTS = [ 'rpc_deriveaddresses.py --usecli', 'p2p_ping.py', 'rpc_scantxoutset.py', + 'feature_txindex_compatibility.py', 'feature_logging.py', 'feature_anchors.py', 'feature_coinstatsindex.py', @@ -348,7 +353,7 @@ def main(): parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.') parser.add_argument('--quiet', '-q', action='store_true', help='only print dots, results summary and failure logs') parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs") - parser.add_argument('--failfast', action='store_true', help='stop execution after the first test failure') + parser.add_argument('--failfast', '-F', action='store_true', help='stop execution after the first test failure') parser.add_argument('--filter', help='filter scripts to run by regular expression') args, unknown_args = parser.parse_known_args() diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index cebaa02524..afe4dba7b4 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoin-wallet.""" @@ -70,8 +70,8 @@ class ToolWalletTest(BitcoinTestFramework): def get_expected_info_output(self, name="", transactions=0, keypool=2, address=0): wallet_name = self.default_wallet_name if name == "" else name - output_types = 3 # p2pkh, p2sh, segwit if self.options.descriptors: + output_types = 4 # p2pkh, p2sh, segwit, bech32m return textwrap.dedent('''\ Wallet info =========== @@ -85,6 +85,7 @@ class ToolWalletTest(BitcoinTestFramework): Address Book: %d ''' % (wallet_name, keypool * output_types, transactions, address)) else: + output_types = 3 # p2pkh, p2sh, segwit. Legacy wallets do not support bech32m. return textwrap.dedent('''\ Wallet info =========== @@ -298,8 +299,8 @@ class ToolWalletTest(BitcoinTestFramework): assert_equal(1000, out['keypoolsize_hd_internal']) assert_equal(True, 'hdseedid' in out) else: - assert_equal(3000, out['keypoolsize']) - assert_equal(3000, out['keypoolsize_hd_internal']) + assert_equal(4000, out['keypoolsize']) + assert_equal(4000, out['keypoolsize_hd_internal']) self.log_wallet_timestamp_comparison(timestamp_before, timestamp_after) assert_equal(timestamp_before, timestamp_after) diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index 8f54e50598..27d9d8da88 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the abandontransaction RPC. @@ -30,7 +30,6 @@ class AbandonConflictTest(BitcoinTestFramework): def run_test(self): self.generate(self.nodes[1], COINBASE_MATURITY) - self.sync_blocks() balance = self.nodes[0].getbalance() txA = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) txB = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) @@ -43,7 +42,6 @@ class AbandonConflictTest(BitcoinTestFramework): # Can not abandon confirmed transaction assert_raises_rpc_error(-5, 'Transaction not eligible for abandonment', lambda: self.nodes[0].abandontransaction(txid=txA)) - self.sync_blocks() newbalance = self.nodes[0].getbalance() assert balance - newbalance < Decimal("0.001") #no more than fees lost balance = newbalance @@ -167,7 +165,7 @@ class AbandonConflictTest(BitcoinTestFramework): tx = self.nodes[0].createrawtransaction(inputs, outputs) signed = self.nodes[0].signrawtransactionwithwallet(tx) self.nodes[1].sendrawtransaction(signed["hex"]) - self.generate(self.nodes[1], 1) + self.generate(self.nodes[1], 1, sync_fun=self.no_op) self.connect_nodes(0, 1) self.sync_blocks() diff --git a/test/functional/wallet_address_types.py b/test/functional/wallet_address_types.py index 7a448e8590..eb6e497951 100755 --- a/test/functional/wallet_address_types.py +++ b/test/functional/wallet_address_types.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that the wallet can send and receive using all combinations of address types. @@ -121,6 +121,12 @@ class AddressTypeTest(BitcoinTestFramework): assert_equal(info['witness_version'], 0) assert_equal(len(info['witness_program']), 40) assert 'pubkey' in info + elif not multisig and typ == "bech32m": + # P2TR single sig + assert info["isscript"] + assert info["iswitness"] + assert_equal(info["witness_version"], 1) + assert_equal(len(info["witness_program"]), 64) elif typ == 'legacy': # P2SH-multisig assert info['isscript'] @@ -221,7 +227,6 @@ class AddressTypeTest(BitcoinTestFramework): # Mine 101 blocks on node5 to bring nodes out of IBD and make sure that # no coinbases are maturing for the nodes-under-test during the test self.generate(self.nodes[5], COINBASE_MATURITY + 1) - self.sync_blocks() uncompressed_1 = "0496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858ee" uncompressed_2 = "047211a824f55b505228e4c3d5194c1fcfaa15a456abdf37f9b9d97a4040afc073dee6c89064984f03385237d92167c13e236446b417ab79a0fcae412ae3316b77" @@ -306,7 +311,6 @@ class AddressTypeTest(BitcoinTestFramework): # node5 collects fee and block subsidy to keep accounting simple self.generate(self.nodes[5], 1) - self.sync_blocks() # Verify that the receiving wallet contains a UTXO with the expected address, and expected descriptor for n, to_node in enumerate(range(from_node, from_node + 4)): @@ -336,25 +340,36 @@ class AddressTypeTest(BitcoinTestFramework): # Fund node 4: self.nodes[5].sendtoaddress(self.nodes[4].getnewaddress(), Decimal("1")) self.generate(self.nodes[5], 1) - self.sync_blocks() assert_equal(self.nodes[4].getbalance(), 1) self.log.info("Nodes with addresstype=legacy never use a P2WPKH change output (unless changetype is set otherwise):") self.test_change_output_type(0, [to_address_bech32_1], 'legacy') - self.log.info("Nodes with addresstype=p2sh-segwit only use a P2WPKH change output if any destination address is bech32:") - self.test_change_output_type(1, [to_address_p2sh], 'p2sh-segwit') - self.test_change_output_type(1, [to_address_bech32_1], 'bech32') - self.test_change_output_type(1, [to_address_p2sh, to_address_bech32_1], 'bech32') - self.test_change_output_type(1, [to_address_bech32_1, to_address_bech32_2], 'bech32') + if self.options.descriptors: + self.log.info("Nodes with addresstype=p2sh-segwit only use a bech32m change output if any destination address is bech32:") + self.test_change_output_type(1, [to_address_p2sh], 'p2sh-segwit') + self.test_change_output_type(1, [to_address_bech32_1], 'bech32m') + self.test_change_output_type(1, [to_address_p2sh, to_address_bech32_1], 'bech32m') + self.test_change_output_type(1, [to_address_bech32_1, to_address_bech32_2], 'bech32m') + else: + self.log.info("Nodes with addresstype=p2sh-segwit only use a P2WPKH change output if any destination address is bech32:") + self.test_change_output_type(1, [to_address_p2sh], 'p2sh-segwit') + self.test_change_output_type(1, [to_address_bech32_1], 'bech32') + self.test_change_output_type(1, [to_address_p2sh, to_address_bech32_1], 'bech32') + self.test_change_output_type(1, [to_address_bech32_1, to_address_bech32_2], 'bech32') self.log.info("Nodes with change_type=bech32 always use a P2WPKH change output:") self.test_change_output_type(2, [to_address_bech32_1], 'bech32') self.test_change_output_type(2, [to_address_p2sh], 'bech32') - self.log.info("Nodes with addresstype=bech32 always use a P2WPKH change output (unless changetype is set otherwise):") - self.test_change_output_type(3, [to_address_bech32_1], 'bech32') - self.test_change_output_type(3, [to_address_p2sh], 'bech32') + if self.options.descriptors: + self.log.info("Nodes with addresstype=bech32 always use either a bech32 or bech32m change output (unless changetype is set otherwise):") + self.test_change_output_type(3, [to_address_bech32_1], 'bech32m') + self.test_change_output_type(3, [to_address_p2sh], 'bech32') + else: + self.log.info("Nodes with addresstype=bech32 always use a P2WPKH change output (unless changetype is set otherwise):") + self.test_change_output_type(3, [to_address_bech32_1], 'bech32') + self.test_change_output_type(3, [to_address_p2sh], 'bech32') self.log.info('getrawchangeaddress defaults to addresstype if -changetype is not set and argument is absent') self.test_address(3, self.nodes[3].getrawchangeaddress(), multisig=False, typ='bech32') @@ -373,10 +388,9 @@ class AddressTypeTest(BitcoinTestFramework): self.test_address(4, self.nodes[4].getrawchangeaddress('bech32'), multisig=False, typ='bech32') if self.options.descriptors: - self.log.info("Descriptor wallets do not have bech32m addresses by default yet") - # TODO: Remove this when they do - assert_raises_rpc_error(-12, "Error: No bech32m addresses available", self.nodes[0].getnewaddress, "", "bech32m") - assert_raises_rpc_error(-12, "Error: No bech32m addresses available", self.nodes[0].getrawchangeaddress, "bech32m") + self.log.info("Descriptor wallets have bech32m addresses") + self.test_address(4, self.nodes[4].getnewaddress("", "bech32m"), multisig=False, typ="bech32m") + self.test_address(4, self.nodes[4].getrawchangeaddress("bech32m"), multisig=False, typ="bech32m") else: self.log.info("Legacy wallets cannot make bech32m addresses") assert_raises_rpc_error(-8, "Legacy wallets cannot provide bech32m addresses", self.nodes[0].getnewaddress, "", "bech32m") diff --git a/test/functional/wallet_avoidreuse.py b/test/functional/wallet_avoidreuse.py index 12357e2d63..dc823c2c60 100755 --- a/test/functional/wallet_avoidreuse.py +++ b/test/functional/wallet_avoidreuse.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the avoid_reuse and setwalletflag features.""" @@ -80,7 +80,6 @@ class AvoidReuseTest(BitcoinTestFramework): self.test_immutable() self.generate(self.nodes[0], 110) - self.sync_all() self.test_change_remains_change(self.nodes[1]) reset_balance(self.nodes[1], self.nodes[0].getnewaddress()) self.test_sending_from_reused_address_without_avoid_reuse() @@ -175,7 +174,6 @@ class AvoidReuseTest(BitcoinTestFramework): self.nodes[0].sendtoaddress(fundaddr, 10) self.generate(self.nodes[0], 1) - self.sync_all() # listunspent should show 1 single, unused 10 btc output assert_unspent(self.nodes[1], total_count=1, total_sum=10, reused_supported=True, reused_count=0) @@ -186,7 +184,6 @@ class AvoidReuseTest(BitcoinTestFramework): self.nodes[1].sendtoaddress(retaddr, 5) self.generate(self.nodes[0], 1) - self.sync_all() # listunspent should show 1 single, unused 5 btc output assert_unspent(self.nodes[1], total_count=1, total_sum=5, reused_supported=True, reused_count=0) @@ -195,7 +192,6 @@ class AvoidReuseTest(BitcoinTestFramework): self.nodes[0].sendtoaddress(fundaddr, 10) self.generate(self.nodes[0], 1) - self.sync_all() # listunspent should show 2 total outputs (5, 10 btc), one unused (5), one reused (10) assert_unspent(self.nodes[1], total_count=2, total_sum=15, reused_count=1, reused_sum=10) @@ -229,7 +225,6 @@ class AvoidReuseTest(BitcoinTestFramework): self.nodes[0].sendtoaddress(fundaddr, 10) self.generate(self.nodes[0], 1) - self.sync_all() # listunspent should show 1 single, unused 10 btc output assert_unspent(self.nodes[1], total_count=1, total_sum=10, reused_supported=True, reused_count=0) @@ -238,7 +233,6 @@ class AvoidReuseTest(BitcoinTestFramework): self.nodes[1].sendtoaddress(retaddr, 5) self.generate(self.nodes[0], 1) - self.sync_all() # listunspent should show 1 single, unused 5 btc output assert_unspent(self.nodes[1], total_count=1, total_sum=5, reused_supported=True, reused_count=0) @@ -260,7 +254,6 @@ class AvoidReuseTest(BitcoinTestFramework): self.nodes[0].sendtoaddress(new_fundaddr, 10) self.generate(self.nodes[0], 1) - self.sync_all() # listunspent should show 2 total outputs (5, 10 btc), one unused (5), one reused (10) assert_unspent(self.nodes[1], total_count=2, total_sum=15, reused_count=1, reused_sum=10) @@ -303,7 +296,6 @@ class AvoidReuseTest(BitcoinTestFramework): self.nodes[0].sendtoaddress(new_addr, 1) self.generate(self.nodes[0], 1) - self.sync_all() # send transaction that should not use all the available outputs # per the current coin selection algorithm @@ -335,7 +327,6 @@ class AvoidReuseTest(BitcoinTestFramework): self.nodes[0].sendtoaddress(new_addr, 1) self.generate(self.nodes[0], 1) - self.sync_all() # Sending a transaction that is smaller than each one of the # available outputs @@ -364,7 +355,6 @@ class AvoidReuseTest(BitcoinTestFramework): self.nodes[0].sendtoaddress(new_addr, 1) self.generate(self.nodes[0], 1) - self.sync_all() # Sending a transaction that needs to use the full groups # of 100 inputs but also the incomplete group of 2 inputs. diff --git a/test/functional/wallet_backup.py b/test/functional/wallet_backup.py index a07c28c8a4..932df4fbff 100755 --- a/test/functional/wallet_backup.py +++ b/test/functional/wallet_backup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet backup features. @@ -89,7 +89,6 @@ class WalletBackupTest(BitcoinTestFramework): # Must sync mempools before mining. self.sync_mempools() self.generate(self.nodes[3], 1) - self.sync_blocks() # As above, this mirrors the original bash test. def start_three(self, args=()): @@ -131,13 +130,9 @@ class WalletBackupTest(BitcoinTestFramework): def run_test(self): self.log.info("Generating initial blockchain") self.generate(self.nodes[0], 1) - self.sync_blocks() self.generate(self.nodes[1], 1) - self.sync_blocks() self.generate(self.nodes[2], 1) - self.sync_blocks() self.generate(self.nodes[3], COINBASE_MATURITY) - self.sync_blocks() assert_equal(self.nodes[0].getbalance(), 50) assert_equal(self.nodes[1].getbalance(), 50) @@ -166,7 +161,6 @@ class WalletBackupTest(BitcoinTestFramework): # Generate 101 more blocks, so any fees paid mature self.generate(self.nodes[3], COINBASE_MATURITY + 1) - self.sync_all() balance0 = self.nodes[0].getbalance() balance1 = self.nodes[1].getbalance() diff --git a/test/functional/wallet_balance.py b/test/functional/wallet_balance.py index 2abac7bb92..0cfbefb719 100755 --- a/test/functional/wallet_balance.py +++ b/test/functional/wallet_balance.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet balance RPC methods.""" @@ -71,10 +71,8 @@ class WalletTest(BitcoinTestFramework): self.log.info("Mining blocks ...") self.generate(self.nodes[0], 1) - self.sync_all() self.generate(self.nodes[1], 1) self.generatetoaddress(self.nodes[1], COINBASE_MATURITY + 1, ADDRESS_WATCHONLY) - self.sync_all() if not self.options.descriptors: # Tests legacy watchonly behavior which is not present (and does not need to be tested) in descriptor wallets @@ -197,7 +195,6 @@ class WalletTest(BitcoinTestFramework): test_balances(fee_node_1=Decimal('0.02')) self.generatetoaddress(self.nodes[1], 1, ADDRESS_WATCHONLY) - self.sync_all() # balances are correct after the transactions are confirmed balance_node0 = Decimal('69.99') # node 1's send plus change from node 0's send @@ -211,7 +208,6 @@ class WalletTest(BitcoinTestFramework): txs = create_transactions(self.nodes[1], self.nodes[0].getnewaddress(), Decimal('29.97'), [Decimal('0.01')]) self.nodes[1].sendrawtransaction(txs[0]['hex']) self.generatetoaddress(self.nodes[1], 2, ADDRESS_WATCHONLY) - self.sync_all() # getbalance with a minconf incorrectly excludes coins that have been spent more recently than the minconf blocks ago # TODO: fix getbalance tracking of coin spentness depth @@ -258,14 +254,13 @@ class WalletTest(BitcoinTestFramework): # Now confirm tx_replace block_reorg = self.generatetoaddress(self.nodes[1], 1, ADDRESS_WATCHONLY)[0] - self.sync_all() assert_equal(self.nodes[0].getbalance(minconf=0), total_amount) self.log.info('Put txs back into mempool of node 1 (not node 0)') self.nodes[0].invalidateblock(block_reorg) self.nodes[1].invalidateblock(block_reorg) assert_equal(self.nodes[0].getbalance(minconf=0), 0) # wallet txs not in the mempool are untrusted - self.generatetoaddress(self.nodes[0], 1, ADDRESS_WATCHONLY) + self.generatetoaddress(self.nodes[0], 1, ADDRESS_WATCHONLY, sync_fun=self.no_op) assert_equal(self.nodes[0].getbalance(minconf=0), 0) # wallet txs not in the mempool are untrusted # Now confirm tx_orig @@ -274,7 +269,6 @@ class WalletTest(BitcoinTestFramework): self.sync_blocks() self.nodes[1].sendrawtransaction(tx_orig) self.generatetoaddress(self.nodes[1], 1, ADDRESS_WATCHONLY) - self.sync_all() assert_equal(self.nodes[0].getbalance(minconf=0), total_amount + 1) # The reorg recovered our fee of 1 coin diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 92da54d97c..69f9df57d8 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet.""" @@ -60,15 +60,14 @@ class WalletTest(BitcoinTestFramework): self.log.info("Mining blocks...") - self.generate(self.nodes[0], 1) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) walletinfo = self.nodes[0].getwalletinfo() assert_equal(walletinfo['immature_balance'], 50) assert_equal(walletinfo['balance'], 0) self.sync_all(self.nodes[0:3]) - self.generate(self.nodes[1], COINBASE_MATURITY + 1) - self.sync_all(self.nodes[0:3]) + self.generate(self.nodes[1], COINBASE_MATURITY + 1, sync_fun=lambda: self.sync_all(self.nodes[0:3])) assert_equal(self.nodes[0].getbalance(), 50) assert_equal(self.nodes[1].getbalance(), 50) @@ -116,8 +115,7 @@ class WalletTest(BitcoinTestFramework): assert_equal(walletinfo['immature_balance'], 0) # Have node0 mine a block, thus it will collect its own fee. - self.generate(self.nodes[0], 1) - self.sync_all(self.nodes[0:3]) + self.generate(self.nodes[0], 1, sync_fun=lambda: self.sync_all(self.nodes[0:3])) # Exercise locking of unspent outputs unspent_0 = self.nodes[2].listunspent()[0] @@ -196,8 +194,7 @@ class WalletTest(BitcoinTestFramework): assert_equal(len(self.nodes[1].listlockunspent()), 0) # Have node1 generate 100 blocks (so node0 can recover the fee) - self.generate(self.nodes[1], COINBASE_MATURITY) - self.sync_all(self.nodes[0:3]) + self.generate(self.nodes[1], COINBASE_MATURITY, sync_fun=lambda: self.sync_all(self.nodes[0:3])) # node0 should end up with 100 btc in block rewards plus fees, but # minus the 21 plus fees sent to node2 @@ -225,8 +222,7 @@ class WalletTest(BitcoinTestFramework): self.nodes[1].sendrawtransaction(hexstring=txns_to_send[1]["hex"], maxfeerate=0) # Have node1 mine a block to confirm transactions: - self.generate(self.nodes[1], 1) - self.sync_all(self.nodes[0:3]) + self.generate(self.nodes[1], 1, sync_fun=lambda: self.sync_all(self.nodes[0:3])) assert_equal(self.nodes[0].getbalance(), 0) assert_equal(self.nodes[2].getbalance(), 94) @@ -240,15 +236,13 @@ class WalletTest(BitcoinTestFramework): fee_per_byte = Decimal('0.001') / 1000 self.nodes[2].settxfee(fee_per_byte * 1000) txid = self.nodes[2].sendtoaddress(address, 10, "", "", False) - self.generate(self.nodes[2], 1) - self.sync_all(self.nodes[0:3]) + self.generate(self.nodes[2], 1, sync_fun=lambda: self.sync_all(self.nodes[0:3])) node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), Decimal('84'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) assert_equal(self.nodes[0].getbalance(), Decimal('10')) # Send 10 BTC with subtract fee from amount txid = self.nodes[2].sendtoaddress(address, 10, "", "", True) - self.generate(self.nodes[2], 1) - self.sync_all(self.nodes[0:3]) + self.generate(self.nodes[2], 1, sync_fun=lambda: self.sync_all(self.nodes[0:3])) node_2_bal -= Decimal('10') assert_equal(self.nodes[2].getbalance(), node_2_bal) node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), Decimal('20'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) @@ -257,16 +251,14 @@ class WalletTest(BitcoinTestFramework): # Sendmany 10 BTC txid = self.nodes[2].sendmany('', {address: 10}, 0, "", []) - self.generate(self.nodes[2], 1) - self.sync_all(self.nodes[0:3]) + self.generate(self.nodes[2], 1, sync_fun=lambda: self.sync_all(self.nodes[0:3])) node_0_bal += Decimal('10') node_2_bal = self.check_fee_amount(self.nodes[2].getbalance(), node_2_bal - Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) assert_equal(self.nodes[0].getbalance(), node_0_bal) # Sendmany 10 BTC with subtract fee from amount txid = self.nodes[2].sendmany('', {address: 10}, 0, "", [address]) - self.generate(self.nodes[2], 1) - self.sync_all(self.nodes[0:3]) + self.generate(self.nodes[2], 1, sync_fun=lambda: self.sync_all(self.nodes[0:3])) node_2_bal -= Decimal('10') assert_equal(self.nodes[2].getbalance(), node_2_bal) node_0_bal = self.check_fee_amount(self.nodes[0].getbalance(), node_0_bal + Decimal('10'), fee_per_byte, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) @@ -278,8 +270,7 @@ class WalletTest(BitcoinTestFramework): # Test passing fee_rate as a string txid = self.nodes[2].sendmany(amounts={address: 10}, fee_rate=str(fee_rate_sat_vb)) - self.generate(self.nodes[2], 1) - self.sync_all(self.nodes[0:3]) + self.generate(self.nodes[2], 1, sync_fun=lambda: self.sync_all(self.nodes[0:3])) balance = self.nodes[2].getbalance() node_2_bal = self.check_fee_amount(balance, node_2_bal - Decimal('10'), explicit_fee_rate_btc_kvb, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) assert_equal(balance, node_2_bal) @@ -289,8 +280,7 @@ class WalletTest(BitcoinTestFramework): # Test passing fee_rate as an integer amount = Decimal("0.0001") txid = self.nodes[2].sendmany(amounts={address: amount}, fee_rate=fee_rate_sat_vb) - self.generate(self.nodes[2], 1) - self.sync_all(self.nodes[0:3]) + self.generate(self.nodes[2], 1, sync_fun=lambda: self.sync_all(self.nodes[0:3])) balance = self.nodes[2].getbalance() node_2_bal = self.check_fee_amount(balance, node_2_bal - amount, explicit_fee_rate_btc_kvb, self.get_vsize(self.nodes[2].gettransaction(txid)['hex'])) assert_equal(balance, node_2_bal) @@ -352,7 +342,6 @@ class WalletTest(BitcoinTestFramework): self.sync_all() self.generate(self.nodes[1], 1) # mine a block - self.sync_all() unspent_txs = self.nodes[0].listunspent() # zero value tx must be in listunspents output found = False @@ -374,14 +363,12 @@ class WalletTest(BitcoinTestFramework): txid_not_broadcast = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), 2) tx_obj_not_broadcast = self.nodes[0].gettransaction(txid_not_broadcast) - self.generate(self.nodes[1], 1) # mine a block, tx should not be in there - self.sync_all(self.nodes[0:3]) + self.generate(self.nodes[1], 1, sync_fun=lambda: self.sync_all(self.nodes[0:3])) # mine a block, tx should not be in there assert_equal(self.nodes[2].getbalance(), node_2_bal) # should not be changed because tx was not broadcasted # now broadcast from another node, mine a block, sync, and check the balance self.nodes[1].sendrawtransaction(tx_obj_not_broadcast['hex']) - self.generate(self.nodes[1], 1) - self.sync_all(self.nodes[0:3]) + self.generate(self.nodes[1], 1, sync_fun=lambda: self.sync_all(self.nodes[0:3])) node_2_bal += 2 tx_obj_not_broadcast = self.nodes[0].gettransaction(txid_not_broadcast) assert_equal(self.nodes[2].getbalance(), node_2_bal) @@ -399,8 +386,7 @@ class WalletTest(BitcoinTestFramework): self.connect_nodes(0, 2) self.sync_blocks(self.nodes[0:3]) - self.generate(self.nodes[0], 1) - self.sync_blocks(self.nodes[0:3]) + self.generate(self.nodes[0], 1, sync_fun=lambda: self.sync_blocks(self.nodes[0:3])) node_2_bal += 2 # tx should be added to balance because after restarting the nodes tx should be broadcast @@ -467,8 +453,7 @@ class WalletTest(BitcoinTestFramework): self.sync_mempools(self.nodes[0:3]) vout = find_vout_for_address(self.nodes[2], txid, address_to_import) self.nodes[2].lockunspent(False, [{"txid": txid, "vout": vout}]) - self.generate(self.nodes[0], 1) - self.sync_all(self.nodes[0:3]) + self.generate(self.nodes[0], 1, sync_fun=lambda: self.sync_all(self.nodes[0:3])) self.log.info("Test sendtoaddress with fee_rate param (explicit fee rate in sat/vB)") prebalance = self.nodes[2].getbalance() @@ -480,8 +465,7 @@ class WalletTest(BitcoinTestFramework): # Test passing fee_rate as an integer txid = self.nodes[2].sendtoaddress(address=address, amount=amount, fee_rate=fee_rate_sat_vb) tx_size = self.get_vsize(self.nodes[2].gettransaction(txid)['hex']) - self.generate(self.nodes[0], 1) - self.sync_all(self.nodes[0:3]) + self.generate(self.nodes[0], 1, sync_fun=lambda: self.sync_all(self.nodes[0:3])) postbalance = self.nodes[2].getbalance() fee = prebalance - postbalance - Decimal(amount) assert_fee_amount(fee, tx_size, Decimal(fee_rate_btc_kvb)) @@ -493,8 +477,7 @@ class WalletTest(BitcoinTestFramework): # Test passing fee_rate as a string txid = self.nodes[2].sendtoaddress(address=address, amount=amount, fee_rate=str(fee_rate_sat_vb)) tx_size = self.get_vsize(self.nodes[2].gettransaction(txid)['hex']) - self.generate(self.nodes[0], 1) - self.sync_all(self.nodes[0:3]) + self.generate(self.nodes[0], 1, sync_fun=lambda: self.sync_all(self.nodes[0:3])) postbalance = self.nodes[2].getbalance() fee = prebalance - postbalance - amount assert_fee_amount(fee, tx_size, Decimal(fee_rate_btc_kvb)) @@ -555,17 +538,15 @@ class WalletTest(BitcoinTestFramework): # Mine a block from node0 to an address from node1 coinbase_addr = self.nodes[1].getnewaddress() - block_hash = self.generatetoaddress(self.nodes[0], 1, coinbase_addr)[0] + block_hash = self.generatetoaddress(self.nodes[0], 1, coinbase_addr, sync_fun=lambda: self.sync_all(self.nodes[0:3]))[0] coinbase_txid = self.nodes[0].getblock(block_hash)['tx'][0] - self.sync_all(self.nodes[0:3]) # Check that the txid and balance is found by node1 self.nodes[1].gettransaction(coinbase_txid) # check if wallet or blockchain maintenance changes the balance self.sync_all(self.nodes[0:3]) - blocks = self.generate(self.nodes[0], 2) - self.sync_all(self.nodes[0:3]) + blocks = self.generate(self.nodes[0], 2, sync_fun=lambda: self.sync_all(self.nodes[0:3])) balance_nodes = [self.nodes[i].getbalance() for i in range(3)] block_count = self.nodes[0].getblockcount() @@ -606,13 +587,13 @@ class WalletTest(BitcoinTestFramework): # Get all non-zero utxos together chain_addrs = [self.nodes[0].getnewaddress(), self.nodes[0].getnewaddress()] singletxid = self.nodes[0].sendtoaddress(chain_addrs[0], self.nodes[0].getbalance(), "", "", True) - self.generate(self.nodes[0], 1) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) node0_balance = self.nodes[0].getbalance() # Split into two chains rawtx = self.nodes[0].createrawtransaction([{"txid": singletxid, "vout": 0}], {chain_addrs[0]: node0_balance / 2 - Decimal('0.01'), chain_addrs[1]: node0_balance / 2 - Decimal('0.01')}) signedtx = self.nodes[0].signrawtransactionwithwallet(rawtx) singletxid = self.nodes[0].sendrawtransaction(hexstring=signedtx["hex"], maxfeerate=0) - self.generate(self.nodes[0], 1) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) # Make a long chain of unconfirmed payments without hitting mempool limit # Each tx we make leaves only one output of change on a chain 1 longer @@ -663,7 +644,7 @@ class WalletTest(BitcoinTestFramework): assert not address_info["ischange"] # Test getaddressinfo 'ischange' field on change address. - self.generate(self.nodes[0], 1) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) destination = self.nodes[1].getnewaddress() txid = self.nodes[0].sendtoaddress(destination, 0.123) tx = self.nodes[0].gettransaction(txid=txid, verbose=True)['decoded'] diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index 46a5df4a8e..f6843d597d 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2020 The Bitcoin Core developers +# Copyright (c) 2016-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the bumpfee RPC. @@ -24,7 +24,6 @@ from test_framework.blocktools import ( ) from test_framework.messages import ( BIP125_SEQUENCE_NUMBER, - tx_from_hex, ) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( @@ -62,7 +61,6 @@ class BumpFeeTest(BitcoinTestFramework): def clear_mempool(self): # Clear mempool between subtests. The subtests may only depend on chainstate (utxos) self.generate(self.nodes[1], 1) - self.sync_all() def run_test(self): # Encrypt wallet for test_locked_wallet_fails test @@ -75,12 +73,10 @@ class BumpFeeTest(BitcoinTestFramework): # fund rbf node with 10 coins of 0.001 btc (100,000 satoshis) self.log.info("Mining blocks...") self.generate(peer_node, 110) - self.sync_all() for _ in range(25): peer_node.sendtoaddress(rbf_node_address, 0.001) self.sync_all() self.generate(peer_node, 1) - self.sync_all() assert_equal(rbf_node.getbalance(), Decimal("0.025")) self.log.info("Running tests") @@ -444,7 +440,6 @@ def test_watchonly_psbt(self, peer_node, rbf_node, dest_address): funding_address2 = watcher.getnewaddress(address_type='bech32') peer_node.sendmany("", {funding_address1: 0.001, funding_address2: 0.001}) self.generate(peer_node, 1) - self.sync_all() # Create single-input PSBT for transaction to be bumped psbt = watcher.walletcreatefundedpsbt([], {dest_address: 0.0005}, 0, {"fee_rate": 1}, True)['psbt'] @@ -529,7 +524,7 @@ def test_unconfirmed_not_spendable(self, rbf_node, rbf_node_address): assert_equal([t for t in rbf_node.listunspent(minconf=0, include_unsafe=False) if t["txid"] == rbfid], []) # check that the main output from the rbf tx is spendable after confirmed - self.generate(rbf_node, 1) + self.generate(rbf_node, 1, sync_fun=self.no_op) assert_equal( sum(1 for t in rbf_node.listunspent(minconf=0, include_unsafe=False) if t["txid"] == rbfid and t["address"] == rbf_node_address and t["spendable"]), 1) @@ -592,14 +587,10 @@ def spend_one_input(node, dest_address, change_size=Decimal("0.00049000")): def submit_block_with_tx(node, tx): - ctx = tx_from_hex(tx) tip = node.getbestblockhash() height = node.getblockcount() + 1 block_time = node.getblockheader(tip)["mediantime"] + 1 - block = create_block(int(tip, 16), create_coinbase(height), block_time) - block.vtx.append(ctx) - block.rehash() - block.hashMerkleRoot = block.calc_merkle_root() + block = create_block(int(tip, 16), create_coinbase(height), block_time, txlist=[tx]) add_witness_commitment(block) block.solve() node.submitblock(block.serialize().hex()) diff --git a/test/functional/wallet_coinbase_category.py b/test/functional/wallet_coinbase_category.py index 3c7abd0800..5a6b6cee59 100755 --- a/test/functional/wallet_coinbase_category.py +++ b/test/functional/wallet_coinbase_category.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2018 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test coinbase transactions return the correct categories. diff --git a/test/functional/wallet_create_tx.py b/test/functional/wallet_create_tx.py index 00ee08002e..a213a261ef 100755 --- a/test/functional/wallet_create_tx.py +++ b/test/functional/wallet_create_tx.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/test/functional/wallet_createwallet.py b/test/functional/wallet_createwallet.py index d806f8f6d2..4416a9655f 100755 --- a/test/functional/wallet_createwallet.py +++ b/test/functional/wallet_createwallet.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test createwallet arguments. @@ -146,7 +146,7 @@ class CreateWalletTest(BitcoinTestFramework): w6.keypoolrefill(1) # There should only be 1 key for legacy, 3 for descriptors walletinfo = w6.getwalletinfo() - keys = 3 if self.options.descriptors else 1 + keys = 4 if self.options.descriptors else 1 assert_equal(walletinfo['keypoolsize'], keys) assert_equal(walletinfo['keypoolsize_hd_internal'], keys) # Allow empty passphrase, but there should be a warning diff --git a/test/functional/wallet_descriptor.py b/test/functional/wallet_descriptor.py index 4ec44a8a6c..e47d021210 100755 --- a/test/functional/wallet_descriptor.py +++ b/test/functional/wallet_descriptor.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test descriptor wallet function.""" @@ -37,12 +37,12 @@ class WalletDescriptorTest(BitcoinTestFramework): self.log.info("Making a descriptor wallet") self.nodes[0].createwallet(wallet_name="desc1", descriptors=True) - # A descriptor wallet should have 100 addresses * 3 types = 300 keys + # A descriptor wallet should have 100 addresses * 4 types = 400 keys self.log.info("Checking wallet info") wallet_info = self.nodes[0].getwalletinfo() assert_equal(wallet_info['format'], 'sqlite') - assert_equal(wallet_info['keypoolsize'], 300) - assert_equal(wallet_info['keypoolsize_hd_internal'], 300) + assert_equal(wallet_info['keypoolsize'], 400) + assert_equal(wallet_info['keypoolsize_hd_internal'], 400) assert 'keypoololdest' not in wallet_info # Check that getnewaddress works diff --git a/test/functional/wallet_disable.py b/test/functional/wallet_disable.py index d0043e9bbb..2c7996ca6b 100755 --- a/test/functional/wallet_disable.py +++ b/test/functional/wallet_disable.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2020 The Bitcoin Core developers +# Copyright (c) 2015-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test a node with the -disablewallet option. diff --git a/test/functional/wallet_dump.py b/test/functional/wallet_dump.py index 06460e17d2..9f0d666270 100755 --- a/test/functional/wallet_dump.py +++ b/test/functional/wallet_dump.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2020 The Bitcoin Core developers +# Copyright (c) 2016-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the dumpwallet RPC.""" diff --git a/test/functional/wallet_fallbackfee.py b/test/functional/wallet_fallbackfee.py index 674c37dc73..acd92097ff 100755 --- a/test/functional/wallet_fallbackfee.py +++ b/test/functional/wallet_fallbackfee.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test wallet replace-by-fee capabilities in conjunction with the fallbackfee.""" diff --git a/test/functional/wallet_groups.py b/test/functional/wallet_groups.py index 802fed6e7d..9052bc7f7f 100755 --- a/test/functional/wallet_groups.py +++ b/test/functional/wallet_groups.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test wallet group functionality.""" @@ -46,7 +46,6 @@ class WalletGroupTest(BitcoinTestFramework): [self.nodes[0].sendtoaddress(addr, 0.5) for addr in addrs] self.generate(self.nodes[0], 1) - self.sync_all() # For each node, send 0.2 coins back to 0; # - node[1] should pick one 0.5 UTXO and leave the rest @@ -109,13 +108,24 @@ class WalletGroupTest(BitcoinTestFramework): assert_equal(input_addrs[0], input_addrs[1]) # Node 2 enforces avoidpartialspends so needs no checking here + if self.options.descriptors: + # Descriptor wallets will use Taproot change by default which has different fees + tx4_ungrouped_fee = 3060 + tx4_grouped_fee = 4400 + tx5_6_ungrouped_fee = 5760 + tx5_6_grouped_fee = 8480 + else: + tx4_ungrouped_fee = 2820 + tx4_grouped_fee = 4160 + tx5_6_ungrouped_fee = 5520 + tx5_6_grouped_fee = 8240 + self.log.info("Test wallet option maxapsfee") addr_aps = self.nodes[3].getnewaddress() self.nodes[0].sendtoaddress(addr_aps, 1.0) self.nodes[0].sendtoaddress(addr_aps, 1.0) self.generate(self.nodes[0], 1) - self.sync_all() - with self.nodes[3].assert_debug_log(['Fee non-grouped = 2820, grouped = 4160, using grouped']): + with self.nodes[3].assert_debug_log([f'Fee non-grouped = {tx4_ungrouped_fee}, grouped = {tx4_grouped_fee}, using grouped']): txid4 = self.nodes[3].sendtoaddress(self.nodes[0].getnewaddress(), 0.1) tx4 = self.nodes[3].getrawtransaction(txid4, True) # tx4 should have 2 inputs and 2 outputs although one output would @@ -126,8 +136,7 @@ class WalletGroupTest(BitcoinTestFramework): addr_aps2 = self.nodes[3].getnewaddress() [self.nodes[0].sendtoaddress(addr_aps2, 1.0) for _ in range(5)] self.generate(self.nodes[0], 1) - self.sync_all() - with self.nodes[3].assert_debug_log(['Fee non-grouped = 5520, grouped = 8240, using non-grouped']): + with self.nodes[3].assert_debug_log([f'Fee non-grouped = {tx5_6_ungrouped_fee}, grouped = {tx5_6_grouped_fee}, using non-grouped']): txid5 = self.nodes[3].sendtoaddress(self.nodes[0].getnewaddress(), 2.95) tx5 = self.nodes[3].getrawtransaction(txid5, True) # tx5 should have 3 inputs (1.0, 1.0, 1.0) and 2 outputs @@ -140,8 +149,7 @@ class WalletGroupTest(BitcoinTestFramework): addr_aps3 = self.nodes[4].getnewaddress() [self.nodes[0].sendtoaddress(addr_aps3, 1.0) for _ in range(5)] self.generate(self.nodes[0], 1) - self.sync_all() - with self.nodes[4].assert_debug_log(['Fee non-grouped = 5520, grouped = 8240, using grouped']): + with self.nodes[4].assert_debug_log([f'Fee non-grouped = {tx5_6_ungrouped_fee}, grouped = {tx5_6_grouped_fee}, using grouped']): txid6 = self.nodes[4].sendtoaddress(self.nodes[0].getnewaddress(), 2.95) tx6 = self.nodes[4].getrawtransaction(txid6, True) # tx6 should have 5 inputs and 2 outputs @@ -163,7 +171,6 @@ class WalletGroupTest(BitcoinTestFramework): signed_tx = self.nodes[0].signrawtransactionwithwallet(funded_tx['hex']) self.nodes[0].sendrawtransaction(signed_tx['hex']) self.generate(self.nodes[0], 1) - self.sync_all() # Check that we can create a transaction that only requires ~100 of our # utxos, without pulling in all outputs and creating a transaction that diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index f54ae89c04..d78afb4212 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2020 The Bitcoin Core developers +# Copyright (c) 2016-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test Hierarchical Deterministic wallet function.""" @@ -136,7 +136,7 @@ class WalletHDTest(BitcoinTestFramework): keypath = self.nodes[1].getaddressinfo(out['scriptPubKey']['address'])['hdkeypath'] if self.options.descriptors: - assert_equal(keypath[0:14], "m/84'/1'/0'/1/") + assert_equal(keypath[0:14], "m/86'/1'/0'/1/") else: assert_equal(keypath[0:7], "m/0'/1'") @@ -229,7 +229,6 @@ class WalletHDTest(BitcoinTestFramework): txid = self.nodes[0].sendtoaddress(addr, 1) origin_rpc.sendrawtransaction(self.nodes[0].gettransaction(txid)['hex']) self.generate(self.nodes[0], 1) - self.sync_blocks() origin_rpc.gettransaction(txid) assert_raises_rpc_error(-5, 'Invalid or non-wallet transaction id', restore_rpc.gettransaction, txid) out_of_kp_txid = txid @@ -240,7 +239,6 @@ class WalletHDTest(BitcoinTestFramework): txid = self.nodes[0].sendtoaddress(last_addr, 1) origin_rpc.sendrawtransaction(self.nodes[0].gettransaction(txid)['hex']) self.generate(self.nodes[0], 1) - self.sync_blocks() origin_rpc.gettransaction(txid) restore_rpc.gettransaction(txid) assert_raises_rpc_error(-5, 'Invalid or non-wallet transaction id', restore_rpc.gettransaction, out_of_kp_txid) diff --git a/test/functional/wallet_import_rescan.py b/test/functional/wallet_import_rescan.py index 27a2a42dac..d9acc8cea5 100755 --- a/test/functional/wallet_import_rescan.py +++ b/test/functional/wallet_import_rescan.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test wallet import RPCs. @@ -181,7 +181,6 @@ class ImportRescanTest(BitcoinTestFramework): self.generate(self.nodes[0], 1) # Generate one block for each send variant.confirmation_height = self.nodes[0].getblockcount() variant.timestamp = self.nodes[0].getblockheader(self.nodes[0].getbestblockhash())["time"] - self.sync_all() # Conclude sync before calling setmocktime to avoid timeouts # Generate a block further in the future (past the rescan window). assert_equal(self.nodes[0].getrawmempool(), []) @@ -190,7 +189,6 @@ class ImportRescanTest(BitcoinTestFramework): self.nodes[0].getblockheader(self.nodes[0].getbestblockhash())["time"] + TIMESTAMP_WINDOW + 1, ) self.generate(self.nodes[0], 1) - self.sync_all() # For each variation of wallet key import, invoke the import RPC and # check the results from getbalance and listtransactions. diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index fc9eac1d74..ac74ff2484 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the importdescriptors RPC. @@ -53,7 +53,7 @@ class ImportDescriptorsTest(BitcoinTestFramework): result = wrpc.importdescriptors([req]) observed_warnings = [] if 'warnings' in result[0]: - observed_warnings = result[0]['warnings'] + observed_warnings = result[0]['warnings'] assert_equal("\n".join(sorted(warnings)), "\n".join(sorted(observed_warnings))) assert_equal(result[0]['success'], success) if error_code is not None: @@ -406,7 +406,6 @@ class ImportDescriptorsTest(BitcoinTestFramework): ismine=True) txid = w0.sendtoaddress(address, 49.99995540) self.generatetoaddress(self.nodes[0], 6, w0.getnewaddress()) - self.sync_blocks() tx = wpriv.createrawtransaction([{"txid": txid, "vout": 0}], {w0.getnewaddress(): 49.999}) signed_tx = wpriv.signrawtransactionwithwallet(tx) w1.sendrawtransaction(signed_tx['hex']) @@ -452,12 +451,10 @@ class ImportDescriptorsTest(BitcoinTestFramework): assert_equal(wmulti_priv.getwalletinfo()['keypoolsize'], 1000) txid = w0.sendtoaddress(addr, 10) self.generate(self.nodes[0], 6) - self.sync_all() send_txid = wmulti_priv.sendtoaddress(w0.getnewaddress(), 8) decoded = wmulti_priv.gettransaction(txid=send_txid, verbose=True)['decoded'] assert_equal(len(decoded['vin'][0]['txinwitness']), 4) self.generate(self.nodes[0], 6) - self.sync_all() self.nodes[1].createwallet(wallet_name="wmulti_pub", disable_private_keys=True, blank=True, descriptors=True) wmulti_pub = self.nodes[1].get_wallet_rpc("wmulti_pub") @@ -495,7 +492,6 @@ class ImportDescriptorsTest(BitcoinTestFramework): vout2 = find_vout_for_address(self.nodes[0], txid2, addr2) self.generate(self.nodes[0], 6) - self.sync_all() assert_equal(wmulti_pub.getbalance(), wmulti_priv.getbalance()) # Make sure that descriptor wallets containing multiple xpubs in a single descriptor load correctly @@ -583,7 +579,6 @@ class ImportDescriptorsTest(BitcoinTestFramework): addr = wmulti_priv_big.getnewaddress() w0.sendtoaddress(addr, 10) self.generate(self.nodes[0], 1) - self.sync_all() # It is standard and would relay. txid = wmulti_priv_big.sendtoaddress(w0.getnewaddress(), 9.999) decoded = wmulti_priv_big.gettransaction(txid=txid, verbose=True)['decoded'] @@ -618,7 +613,6 @@ class ImportDescriptorsTest(BitcoinTestFramework): addr = multi_priv_big.getnewaddress("", "legacy") w0.sendtoaddress(addr, 10) self.generate(self.nodes[0], 6) - self.sync_all() # It is standard and would relay. txid = multi_priv_big.sendtoaddress(w0.getnewaddress(), 10, "", "", True) decoded = multi_priv_big.gettransaction(txid=txid, verbose=True)['decoded'] diff --git a/test/functional/wallet_importmulti.py b/test/functional/wallet_importmulti.py index 4e8907bc3d..3953851491 100755 --- a/test/functional/wallet_importmulti.py +++ b/test/functional/wallet_importmulti.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the importmulti RPC. @@ -62,10 +62,9 @@ class ImportMultiTest(BitcoinTestFramework): def run_test(self): self.log.info("Mining blocks...") - self.generate(self.nodes[0], 1) - self.generate(self.nodes[1], 1) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) + self.generate(self.nodes[1], 1, sync_fun=self.no_op) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] - self.nodes[1].syncwithvalidationinterfacequeue() # Sync the timestamp to the wallet, so that importmulti works node0_address1 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) @@ -256,11 +255,10 @@ class ImportMultiTest(BitcoinTestFramework): # P2SH address multisig = get_multisig(self.nodes[0]) - self.generate(self.nodes[1], COINBASE_MATURITY) + self.generate(self.nodes[1], COINBASE_MATURITY, sync_fun=self.no_op) self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) - self.generate(self.nodes[1], 1) + self.generate(self.nodes[1], 1, sync_fun=self.no_op) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] - self.nodes[1].syncwithvalidationinterfacequeue() self.log.info("Should import a p2sh") self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_addr}, @@ -277,11 +275,10 @@ class ImportMultiTest(BitcoinTestFramework): # P2SH + Redeem script multisig = get_multisig(self.nodes[0]) - self.generate(self.nodes[1], COINBASE_MATURITY) + self.generate(self.nodes[1], COINBASE_MATURITY, sync_fun=self.no_op) self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) - self.generate(self.nodes[1], 1) + self.generate(self.nodes[1], 1, sync_fun=self.no_op) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] - self.nodes[1].syncwithvalidationinterfacequeue() self.log.info("Should import a p2sh with respective redeem script") self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_addr}, @@ -298,11 +295,10 @@ class ImportMultiTest(BitcoinTestFramework): # P2SH + Redeem script + Private Keys + !Watchonly multisig = get_multisig(self.nodes[0]) - self.generate(self.nodes[1], COINBASE_MATURITY) + self.generate(self.nodes[1], COINBASE_MATURITY, sync_fun=self.no_op) self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) - self.generate(self.nodes[1], 1) + self.generate(self.nodes[1], 1, sync_fun=self.no_op) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] - self.nodes[1].syncwithvalidationinterfacequeue() self.log.info("Should import a p2sh with respective redeem script and private keys") self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_addr}, @@ -324,11 +320,10 @@ class ImportMultiTest(BitcoinTestFramework): # P2SH + Redeem script + Private Keys + Watchonly multisig = get_multisig(self.nodes[0]) - self.generate(self.nodes[1], COINBASE_MATURITY) + self.generate(self.nodes[1], COINBASE_MATURITY, sync_fun=self.no_op) self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) - self.generate(self.nodes[1], 1) + self.generate(self.nodes[1], 1, sync_fun=self.no_op) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] - self.nodes[1].syncwithvalidationinterfacequeue() self.log.info("Should import a p2sh with respective redeem script and private keys") self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_addr}, diff --git a/test/functional/wallet_importprunedfunds.py b/test/functional/wallet_importprunedfunds.py index 74c5100f40..cdb5823109 100755 --- a/test/functional/wallet_importprunedfunds.py +++ b/test/functional/wallet_importprunedfunds.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the importprunedfunds and removeprunedfunds RPCs.""" @@ -27,8 +27,6 @@ class ImportPrunedFundsTest(BitcoinTestFramework): self.log.info("Mining blocks...") self.generate(self.nodes[0], COINBASE_MATURITY + 1) - self.sync_all() - # address address1 = self.nodes[0].getnewaddress() # pubkey diff --git a/test/functional/wallet_keypool.py b/test/functional/wallet_keypool.py index 79235646b0..54c47511a9 100755 --- a/test/functional/wallet_keypool.py +++ b/test/functional/wallet_keypool.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet keypool and interaction with wallet encryption/locking.""" @@ -87,8 +87,8 @@ class KeyPoolTest(BitcoinTestFramework): nodes[0].walletlock() wi = nodes[0].getwalletinfo() if self.options.descriptors: - assert_equal(wi['keypoolsize_hd_internal'], 18) - assert_equal(wi['keypoolsize'], 18) + assert_equal(wi['keypoolsize_hd_internal'], 24) + assert_equal(wi['keypoolsize'], 24) else: assert_equal(wi['keypoolsize_hd_internal'], 6) assert_equal(wi['keypoolsize'], 6) @@ -132,8 +132,8 @@ class KeyPoolTest(BitcoinTestFramework): nodes[0].keypoolrefill(100) wi = nodes[0].getwalletinfo() if self.options.descriptors: - assert_equal(wi['keypoolsize_hd_internal'], 300) - assert_equal(wi['keypoolsize'], 300) + assert_equal(wi['keypoolsize_hd_internal'], 400) + assert_equal(wi['keypoolsize'], 400) else: assert_equal(wi['keypoolsize_hd_internal'], 100) assert_equal(wi['keypoolsize'], 100) @@ -193,7 +193,7 @@ class KeyPoolTest(BitcoinTestFramework): assert_equal("psbt" in res, True) # create a transaction without change at the maximum fee rate, such that the output is still spendable: - res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00010000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.0008824}) + res = w2.walletcreatefundedpsbt(inputs=[], outputs=[{destination: 0.00010000}], options={"subtractFeeFromOutputs": [0], "feeRate": 0.0008823}) assert_equal("psbt" in res, True) assert_equal(res["fee"], Decimal("0.00009706")) diff --git a/test/functional/wallet_keypool_topup.py b/test/functional/wallet_keypool_topup.py index f730f82397..4c965b7160 100755 --- a/test/functional/wallet_keypool_topup.py +++ b/test/functional/wallet_keypool_topup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test HD Wallet keypool restore function. @@ -66,7 +66,6 @@ class KeypoolRestoreTest(BitcoinTestFramework): self.generate(self.nodes[0], 1) self.nodes[0].sendtoaddress(addr_extpool, 5) self.generate(self.nodes[0], 1) - self.sync_blocks() self.log.info("Restart node with wallet backup") self.stop_node(idx) diff --git a/test/functional/wallet_labels.py b/test/functional/wallet_labels.py index 150f2b341e..c29b02e661 100755 --- a/test/functional/wallet_labels.py +++ b/test/functional/wallet_labels.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2020 The Bitcoin Core developers +# Copyright (c) 2016-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test label RPCs. diff --git a/test/functional/wallet_listdescriptors.py b/test/functional/wallet_listdescriptors.py index 436bbdcfcc..202ef92887 100755 --- a/test/functional/wallet_listdescriptors.py +++ b/test/functional/wallet_listdescriptors.py @@ -43,9 +43,9 @@ class ListDescriptorsTest(BitcoinTestFramework): node.createwallet(wallet_name='w3', descriptors=True) result = node.get_wallet_rpc('w3').listdescriptors() assert_equal("w3", result['wallet_name']) - assert_equal(6, len(result['descriptors'])) - assert_equal(6, len([d for d in result['descriptors'] if d['active']])) - assert_equal(3, len([d for d in result['descriptors'] if d['internal']])) + assert_equal(8, len(result['descriptors'])) + assert_equal(8, len([d for d in result['descriptors'] if d['active']])) + assert_equal(4, len([d for d in result['descriptors'] if d['internal']])) for item in result['descriptors']: assert item['desc'] != '' assert item['next'] == 0 diff --git a/test/functional/wallet_listreceivedby.py b/test/functional/wallet_listreceivedby.py index 975bf9a84b..42a2685a0f 100755 --- a/test/functional/wallet_listreceivedby.py +++ b/test/functional/wallet_listreceivedby.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the listreceivedbyaddress RPC.""" @@ -25,7 +25,6 @@ class ReceivedByTest(BitcoinTestFramework): def run_test(self): # Generate block to get out of IBD self.generate(self.nodes[0], 1) - self.sync_blocks() # save the number of coinbase reward addresses so far num_cb_reward_addresses = len(self.nodes[1].listreceivedbyaddress(minconf=0, include_empty=True, include_watchonly=True)) @@ -44,7 +43,6 @@ class ReceivedByTest(BitcoinTestFramework): True) # Bury Tx under 10 block so it will be returned by listreceivedbyaddress self.generate(self.nodes[1], 10) - self.sync_all() assert_array_result(self.nodes[1].listreceivedbyaddress(), {"address": addr}, {"address": addr, "label": "", "amount": Decimal("0.1"), "confirmations": 10, "txids": [txid, ]}) @@ -79,7 +77,6 @@ class ReceivedByTest(BitcoinTestFramework): other_addr = self.nodes[1].getnewaddress() txid2 = self.nodes[0].sendtoaddress(other_addr, 0.1) self.generate(self.nodes[0], 1) - self.sync_all() # Same test as above should still pass expected = {"address": addr, "label": "", "amount": Decimal("0.1"), "confirmations": 11, "txids": [txid, ]} res = self.nodes[1].listreceivedbyaddress(0, True, True, addr) @@ -116,7 +113,6 @@ class ReceivedByTest(BitcoinTestFramework): # Bury Tx under 10 block so it will be returned by the default getreceivedbyaddress self.generate(self.nodes[1], 10) - self.sync_all() balance = self.nodes[1].getreceivedbyaddress(addr) assert_equal(balance, Decimal("0.1")) @@ -145,7 +141,6 @@ class ReceivedByTest(BitcoinTestFramework): assert_equal(balance, balance_by_label) self.generate(self.nodes[1], 10) - self.sync_all() # listreceivedbylabel should return updated received list assert_array_result(self.nodes[1].listreceivedbylabel(), {"label": label}, diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index f4a00a8ec8..fc06565983 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the listsinceblock RPC.""" @@ -31,7 +31,6 @@ class ListSinceBlockTest(BitcoinTestFramework): # only one connection. (See fPreferredDownload in net_processing) self.connect_nodes(1, 2) self.generate(self.nodes[2], COINBASE_MATURITY + 1) - self.sync_all() self.test_no_blockhash() self.test_invalid_blockhash() @@ -54,7 +53,6 @@ class ListSinceBlockTest(BitcoinTestFramework): blockhash, = self.generate(self.nodes[2], 1) blockheight = self.nodes[2].getblockheader(blockhash)['height'] - self.sync_all() txs = self.nodes[0].listtransactions() assert_array_result(txs, {"txid": txid}, { @@ -99,7 +97,6 @@ class ListSinceBlockTest(BitcoinTestFramework): self.log.info("Test target_confirmations") blockhash, = self.generate(self.nodes[2], 1) blockheight = self.nodes[2].getblockheader(blockhash)['height'] - self.sync_all() assert_equal( self.nodes[0].getblockhash(0), @@ -147,14 +144,11 @@ class ListSinceBlockTest(BitcoinTestFramework): senttx = self.nodes[2].sendtoaddress(self.nodes[0].getnewaddress(), 1) # generate on both sides - nodes1_last_blockhash = self.generate(self.nodes[1], 6)[-1] - nodes2_first_blockhash = self.generate(self.nodes[2], 7)[0] + nodes1_last_blockhash = self.generate(self.nodes[1], 6, sync_fun=lambda: self.sync_all(self.nodes[:2]))[-1] + nodes2_first_blockhash = self.generate(self.nodes[2], 7, sync_fun=lambda: self.sync_all(self.nodes[2:]))[0] self.log.debug("nodes[1] last blockhash = {}".format(nodes1_last_blockhash)) self.log.debug("nodes[2] first blockhash = {}".format(nodes2_first_blockhash)) - self.sync_all(self.nodes[:2]) - self.sync_all(self.nodes[2:]) - self.join_network() # listsinceblock(nodes1_last_blockhash) should now include tx as seen from nodes[0] @@ -203,7 +197,6 @@ class ListSinceBlockTest(BitcoinTestFramework): address = key_to_p2wpkh(eckey.get_pubkey().get_bytes()) self.nodes[2].sendtoaddress(address, 10) self.generate(self.nodes[2], 6) - self.sync_all() self.nodes[2].importprivkey(privkey) utxos = self.nodes[2].listunspent() utxo = [u for u in utxos if u["address"] == address][0] @@ -236,8 +229,8 @@ class ListSinceBlockTest(BitcoinTestFramework): self.nodes[2].createrawtransaction(utxo_dicts, recipient_dict2))['hex']) # generate on both sides - lastblockhash = self.generate(self.nodes[1], 3)[2] - self.generate(self.nodes[2], 4) + lastblockhash = self.generate(self.nodes[1], 3, sync_fun=self.no_op)[2] + self.generate(self.nodes[2], 4, sync_fun=self.no_op) self.join_network() @@ -308,7 +301,7 @@ class ListSinceBlockTest(BitcoinTestFramework): txid1 = self.nodes[1].sendrawtransaction(signedtx) # generate bb1-bb2 on right side - self.generate(self.nodes[2], 2) + self.generate(self.nodes[2], 2, sync_fun=self.no_op) # send from nodes[2]; this will end up in bb3 txid2 = self.nodes[2].sendrawtransaction(signedtx) @@ -316,8 +309,8 @@ class ListSinceBlockTest(BitcoinTestFramework): assert_equal(txid1, txid2) # generate on both sides - lastblockhash = self.generate(self.nodes[1], 3)[2] - self.generate(self.nodes[2], 2) + lastblockhash = self.generate(self.nodes[1], 3, sync_fun=self.no_op)[2] + self.generate(self.nodes[2], 2, sync_fun=self.no_op) self.join_network() diff --git a/test/functional/wallet_listtransactions.py b/test/functional/wallet_listtransactions.py index ca6a6ab540..1b29d163c2 100755 --- a/test/functional/wallet_listtransactions.py +++ b/test/functional/wallet_listtransactions.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the listtransactions API.""" + from decimal import Decimal +import os +import shutil from test_framework.messages import ( COIN, @@ -15,9 +18,10 @@ from test_framework.util import ( assert_equal, ) + class ListTransactionsTest(BitcoinTestFramework): def set_test_params(self): - self.num_nodes = 2 + self.num_nodes = 3 # This test isn't testing txn relay/timing, so set whitelist on the # peers for instant txn relay. This speeds up the test run time 2-3x. self.extra_args = [["-whitelist=noban@127.0.0.1"]] * self.num_nodes @@ -38,7 +42,6 @@ class ListTransactionsTest(BitcoinTestFramework): self.log.info("Test confirmations change after mining a block") blockhash = self.generate(self.nodes[0], 1)[0] blockheight = self.nodes[0].getblockheader(blockhash)['height'] - self.sync_all() assert_array_result(self.nodes[0].listtransactions(), {"txid": txid}, {"category": "send", "amount": Decimal("-0.1"), "confirmations": 1, "blockhash": blockhash, "blockheight": blockheight}) @@ -95,7 +98,6 @@ class ListTransactionsTest(BitcoinTestFramework): self.nodes[0].importaddress(multisig["redeemScript"], "watchonly", False, True) txid = self.nodes[1].sendtoaddress(multisig["address"], 0.1) self.generate(self.nodes[1], 1) - self.sync_all() assert_equal(len(self.nodes[0].listtransactions(label="watchonly", include_watchonly=True)), 1) assert_equal(len(self.nodes[0].listtransactions(dummy="watchonly", include_watchonly=True)), 1) assert len(self.nodes[0].listtransactions(label="watchonly", count=100, include_watchonly=False)) == 0 @@ -104,7 +106,7 @@ class ListTransactionsTest(BitcoinTestFramework): {"txid": txid, "label": "watchonly"}) self.run_rbf_opt_in_test() - + self.run_externally_generated_address_test() def run_rbf_opt_in_test(self): """Test the opt-in-rbf flag for sent and received transactions.""" @@ -219,5 +221,60 @@ class ListTransactionsTest(BitcoinTestFramework): assert_equal(self.nodes[0].gettransaction(txid_3b)["bip125-replaceable"], "no") assert_equal(self.nodes[0].gettransaction(txid_4)["bip125-replaceable"], "unknown") + def run_externally_generated_address_test(self): + """Test behavior when receiving address is not in the address book.""" + + self.log.info("Setup the same wallet on two nodes") + # refill keypool otherwise the second node wouldn't recognize addresses generated on the first nodes + self.nodes[0].keypoolrefill(1000) + self.stop_nodes() + wallet0 = os.path.join(self.nodes[0].datadir, self.chain, self.default_wallet_name, "wallet.dat") + wallet2 = os.path.join(self.nodes[2].datadir, self.chain, self.default_wallet_name, "wallet.dat") + shutil.copyfile(wallet0, wallet2) + self.start_nodes() + # reconnect nodes + self.connect_nodes(0, 1) + self.connect_nodes(1, 2) + self.connect_nodes(2, 0) + + addr1 = self.nodes[0].getnewaddress("pizza1", 'legacy') + addr2 = self.nodes[0].getnewaddress("pizza2", 'p2sh-segwit') + addr3 = self.nodes[0].getnewaddress("pizza3", 'bech32') + + self.log.info("Send to externally generated addresses") + # send to an address beyond the next to be generated to test the keypool gap + self.nodes[1].sendtoaddress(addr3, "0.001") + self.generate(self.nodes[1], 1) + + # send to an address that is already marked as used due to the keypool gap mechanics + self.nodes[1].sendtoaddress(addr2, "0.001") + self.generate(self.nodes[1], 1) + + # send to self transaction + self.nodes[0].sendtoaddress(addr1, "0.001") + self.generate(self.nodes[0], 1) + + self.log.info("Verify listtransactions is the same regardless of where the address was generated") + transactions0 = self.nodes[0].listtransactions() + transactions2 = self.nodes[2].listtransactions() + + # normalize results: remove fields that normally could differ and sort + def normalize_list(txs): + for tx in txs: + tx.pop('label', None) + tx.pop('time', None) + tx.pop('timereceived', None) + txs.sort(key=lambda x: x['txid']) + + normalize_list(transactions0) + normalize_list(transactions2) + assert_equal(transactions0, transactions2) + + self.log.info("Verify labels are persistent on the node that generated the addresses") + assert_equal(['pizza1'], self.nodes[0].getaddressinfo(addr1)['labels']) + assert_equal(['pizza2'], self.nodes[0].getaddressinfo(addr2)['labels']) + assert_equal(['pizza3'], self.nodes[0].getaddressinfo(addr3)['labels']) + + if __name__ == '__main__': ListTransactionsTest().main() diff --git a/test/functional/wallet_multisig_descriptor_psbt.py b/test/functional/wallet_multisig_descriptor_psbt.py index ed855d2525..2b565db137 100755 --- a/test/functional/wallet_multisig_descriptor_psbt.py +++ b/test/functional/wallet_multisig_descriptor_psbt.py @@ -85,7 +85,7 @@ class WalletMultisigDescriptorPSBTTest(BitcoinTestFramework): # This wallet will be the participant's `signer` for the resulting multisig. Avoid reusing this wallet for any other purpose (for privacy reasons). "signers": [node.get_wallet_rpc(node.createwallet(wallet_name=f"participant_{self.nodes.index(node)}", descriptors=True)["name"]) for node in self.nodes], # After participants generate and exchange their xpubs they will each create their own watch-only multisig. - # Note: these multisigs are all the same, this justs highlights that each participant can independently verify everything on their own node. + # Note: these multisigs are all the same, this just highlights that each participant can independently verify everything on their own node. "multisigs": [] } @@ -111,7 +111,6 @@ class WalletMultisigDescriptorPSBTTest(BitcoinTestFramework): self.log.info("Send funds to the resulting multisig receiving address...") coordinator_wallet.sendtoaddress(multisig_receiving_address, deposit_amount) self.generate(self.nodes[0], 1) - self.sync_all() for participant in participants["multisigs"]: assert_approx(participant.getbalance(), deposit_amount, vspan=0.001) @@ -137,7 +136,6 @@ class WalletMultisigDescriptorPSBTTest(BitcoinTestFramework): self.log.info("Check that balances are correct after the transaction has been included in a block.") self.generate(self.nodes[0], 1) - self.sync_all() assert_approx(participants["multisigs"][0].getbalance(), deposit_amount - value, vspan=0.001) assert_equal(participants["signers"][self.N - 1].getbalance(), value) @@ -154,7 +152,6 @@ class WalletMultisigDescriptorPSBTTest(BitcoinTestFramework): self.log.info("Check that balances are correct after the transaction has been included in a block.") self.generate(self.nodes[0], 1) - self.sync_all() assert_approx(participants["multisigs"][0].getbalance(), deposit_amount - (value * 2), vspan=0.001) assert_equal(participants["signers"][self.N - 1].getbalance(), value * 2) diff --git a/test/functional/wallet_multiwallet.py b/test/functional/wallet_multiwallet.py index 68ca005649..0b868dde6c 100755 --- a/test/functional/wallet_multiwallet.py +++ b/test/functional/wallet_multiwallet.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test multiwallet. @@ -185,7 +185,7 @@ class MultiWalletTest(BitcoinTestFramework): self.nodes[0].createwallet("w5") assert_equal(set(node.listwallets()), {"w4", "w5"}) w5 = wallet("w5") - self.generatetoaddress(node, nblocks=1, address=w5.getnewaddress()) + self.generatetoaddress(node, nblocks=1, address=w5.getnewaddress(), sync_fun=self.no_op) # now if wallets/ exists again, but the rootdir is specified as the walletdir, w4 and w5 should still be loaded os.rename(wallet_dir2, wallet_dir()) @@ -217,7 +217,7 @@ class MultiWalletTest(BitcoinTestFramework): wallet_bad = wallet("bad") # check wallet names and balances - self.generatetoaddress(node, nblocks=1, address=wallets[0].getnewaddress()) + self.generatetoaddress(node, nblocks=1, address=wallets[0].getnewaddress(), sync_fun=self.no_op) for wallet_name, wallet in zip(wallet_names, wallets): info = wallet.getwalletinfo() assert_equal(info['immature_balance'], 50 if wallet is wallets[0] else 0) @@ -230,7 +230,7 @@ class MultiWalletTest(BitcoinTestFramework): assert_raises_rpc_error(-19, "Wallet file not specified", node.getwalletinfo) w1, w2, w3, w4, *_ = wallets - self.generatetoaddress(node, nblocks=COINBASE_MATURITY + 1, address=w1.getnewaddress()) + self.generatetoaddress(node, nblocks=COINBASE_MATURITY + 1, address=w1.getnewaddress(), sync_fun=self.no_op) assert_equal(w1.getbalance(), 100) assert_equal(w2.getbalance(), 0) assert_equal(w3.getbalance(), 0) @@ -239,7 +239,7 @@ class MultiWalletTest(BitcoinTestFramework): w1.sendtoaddress(w2.getnewaddress(), 1) w1.sendtoaddress(w3.getnewaddress(), 2) w1.sendtoaddress(w4.getnewaddress(), 3) - self.generatetoaddress(node, nblocks=1, address=w1.getnewaddress()) + self.generatetoaddress(node, nblocks=1, address=w1.getnewaddress(), sync_fun=self.no_op) assert_equal(w2.getbalance(), 1) assert_equal(w3.getbalance(), 2) assert_equal(w4.getbalance(), 3) diff --git a/test/functional/wallet_orphanedreward.py b/test/functional/wallet_orphanedreward.py index ff1d1bd49b..7295db4653 100755 --- a/test/functional/wallet_orphanedreward.py +++ b/test/functional/wallet_orphanedreward.py @@ -26,12 +26,10 @@ class OrphanedBlockRewardTest(BitcoinTestFramework): # it later. self.sync_blocks() blk = self.generate(self.nodes[1], 1)[0] - self.sync_blocks() # Let the block reward mature and send coins including both # the existing balance and the block reward. self.generate(self.nodes[0], 150) - self.sync_blocks() assert_equal(self.nodes[1].getbalance(), 10 + 25) txid = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 30) @@ -39,7 +37,6 @@ class OrphanedBlockRewardTest(BitcoinTestFramework): # from the wallet can still be spent. self.nodes[0].invalidateblock(blk) self.generate(self.nodes[0], 152) - self.sync_blocks() # Without the following abandontransaction call, the coins are # not considered available yet. assert_equal(self.nodes[1].getbalances()["mine"], { diff --git a/test/functional/wallet_reorgsrestore.py b/test/functional/wallet_reorgsrestore.py index 1f452f8337..f2bdb114b7 100755 --- a/test/functional/wallet_reorgsrestore.py +++ b/test/functional/wallet_reorgsrestore.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -33,7 +33,6 @@ class ReorgsRestoreTest(BitcoinTestFramework): # Send a tx from which to conflict outputs later txid_conflict_from = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) self.generate(self.nodes[0], 1) - self.sync_blocks() # Disconnect node1 from others to reorg its chain later self.disconnect_nodes(0, 1) @@ -43,7 +42,7 @@ class ReorgsRestoreTest(BitcoinTestFramework): # Send a tx to be unconfirmed later txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), Decimal("10")) tx = self.nodes[0].gettransaction(txid) - self.generate(self.nodes[0], 4) + self.generate(self.nodes[0], 4, sync_fun=self.no_op) tx_before_reorg = self.nodes[0].gettransaction(txid) assert_equal(tx_before_reorg["confirmations"], 4) @@ -62,9 +61,9 @@ class ReorgsRestoreTest(BitcoinTestFramework): conflicting = self.nodes[0].signrawtransactionwithwallet(self.nodes[0].createrawtransaction(inputs, outputs_2)) conflicted_txid = self.nodes[0].sendrawtransaction(conflicted["hex"]) - self.generate(self.nodes[0], 1) + self.generate(self.nodes[0], 1, sync_fun=self.no_op) conflicting_txid = self.nodes[2].sendrawtransaction(conflicting["hex"]) - self.generate(self.nodes[2], 9) + self.generate(self.nodes[2], 9, sync_fun=self.no_op) # Reconnect node0 and node2 and check that conflicted_txid is effectively conflicted self.connect_nodes(0, 2) @@ -78,11 +77,11 @@ class ReorgsRestoreTest(BitcoinTestFramework): self.restart_node(0) # The block chain re-orgs and the tx is included in a different block - self.generate(self.nodes[1], 9) + self.generate(self.nodes[1], 9, sync_fun=self.no_op) self.nodes[1].sendrawtransaction(tx["hex"]) - self.generate(self.nodes[1], 1) + self.generate(self.nodes[1], 1, sync_fun=self.no_op) self.nodes[1].sendrawtransaction(conflicted["hex"]) - self.generate(self.nodes[1], 1) + self.generate(self.nodes[1], 1, sync_fun=self.no_op) # Node0 wallet file is loaded on longest sync'ed node1 self.stop_node(1) diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index 37dee219e7..5aae2c813a 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test that the wallet resends transactions periodically.""" @@ -48,7 +48,6 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): block_time = int(time.time()) + 6 * 60 node.setmocktime(block_time) block = create_block(int(node.getbestblockhash(), 16), create_coinbase(node.getblockcount() + 1), block_time) - block.rehash() block.solve() node.submitblock(block.serialize().hex()) diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py index c9daeabeb9..d77d554baa 100755 --- a/test/functional/wallet_send.py +++ b/test/functional/wallet_send.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the send RPC command.""" @@ -246,7 +246,6 @@ class WalletSendTest(BitcoinTestFramework): w0.sendtoaddress(a2_receive, 10) # fund w3 self.generate(self.nodes[0], 1) - self.sync_blocks() if not self.options.descriptors: # w4 has private keys enabled, but only contains watch-only keys (from w2) @@ -265,7 +264,6 @@ class WalletSendTest(BitcoinTestFramework): w0.sendtoaddress(a2_receive, 10) # fund w4 self.generate(self.nodes[0], 1) - self.sync_blocks() self.log.info("Send to address...") self.test_send(from_wallet=w0, to_wallet=w1, amount=1) @@ -448,7 +446,6 @@ class WalletSendTest(BitcoinTestFramework): res = self.nodes[0].sendrawtransaction(hex) self.generate(self.nodes[0], 1) assert_equal(self.nodes[0].gettransaction(txid)["confirmations"], 1) - self.sync_all() self.log.info("Lock unspents...") utxo1 = w0.listunspent()[0] @@ -503,7 +500,6 @@ class WalletSendTest(BitcoinTestFramework): self.nodes[0].sendtoaddress(addr, 10) self.nodes[0].sendtoaddress(ext_wallet.getnewaddress(), 10) self.generate(self.nodes[0], 6) - self.sync_all() ext_utxo = ext_fund.listunspent(addresses=[addr])[0] # An external input without solving data should result in an error diff --git a/test/functional/wallet_signer.py b/test/functional/wallet_signer.py index c6c1cc8784..6dadc57b1a 100755 --- a/test/functional/wallet_signer.py +++ b/test/functional/wallet_signer.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2018 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test external signer. @@ -112,7 +112,6 @@ class WalletSignerTest(BitcoinTestFramework): self.log.info('Prepare mock PSBT') self.nodes[0].sendtoaddress(address1, 1) self.generate(self.nodes[0], 1) - self.sync_all() # Load private key into wallet to generate a signed PSBT for the mock self.nodes[1].createwallet(wallet_name="mock", disable_private_keys=False, blank=True, descriptors=True) diff --git a/test/functional/wallet_signmessagewithaddress.py b/test/functional/wallet_signmessagewithaddress.py index bf6f95e3f1..74a8f2eef2 100755 --- a/test/functional/wallet_signmessagewithaddress.py +++ b/test/functional/wallet_signmessagewithaddress.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2019 The Bitcoin Core developers +# Copyright (c) 2016-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test Wallet commands for signing and verifying messages.""" diff --git a/test/functional/wallet_taproot.py b/test/functional/wallet_taproot.py index 80c125df97..17eab25457 100755 --- a/test/functional/wallet_taproot.py +++ b/test/functional/wallet_taproot.py @@ -10,7 +10,12 @@ from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal from test_framework.descriptors import descsum_create -from test_framework.script import (CScript, OP_CHECKSIG, taproot_construct) +from test_framework.script import ( + CScript, + OP_1, + OP_CHECKSIG, + taproot_construct, +) from test_framework.segwit_addr import encode_segwit_address # xprvs/xpubs, and m/* derived x-only pubkeys (created using independent implementation) @@ -165,7 +170,7 @@ def pk(hex_key): def compute_taproot_address(pubkey, scripts): """Compute the address for a taproot output with given inner key and scripts.""" tap = taproot_construct(pubkey, scripts) - assert tap.scriptPubKey[0] == 0x51 + assert tap.scriptPubKey[0] == OP_1 assert tap.scriptPubKey[1] == 0x20 return encode_segwit_address("bcrt", 1, tap.scriptPubKey[2:]) @@ -237,20 +242,15 @@ class WalletTaprootTest(BitcoinTestFramework): assert_equal(len(rederive), 1) assert_equal(rederive[0], addr_g) - # tr descriptors cannot be imported when Taproot is not active + # tr descriptors can be imported regardless of Taproot status result = self.privs_tr_enabled.importdescriptors([{"desc": desc, "timestamp": "now"}]) assert(result[0]["success"]) result = self.pubs_tr_enabled.importdescriptors([{"desc": desc_pub, "timestamp": "now"}]) assert(result[0]["success"]) - if desc.startswith("tr"): - result = self.privs_tr_disabled.importdescriptors([{"desc": desc, "timestamp": "now"}]) - assert(not result[0]["success"]) - assert_equal(result[0]["error"]["code"], -4) - assert_equal(result[0]["error"]["message"], "Cannot import tr() descriptor when Taproot is not active") - result = self.pubs_tr_disabled.importdescriptors([{"desc": desc_pub, "timestamp": "now"}]) - assert(not result[0]["success"]) - assert_equal(result[0]["error"]["code"], -4) - assert_equal(result[0]["error"]["message"], "Cannot import tr() descriptor when Taproot is not active") + result = self.privs_tr_disabled.importdescriptors([{"desc": desc, "timestamp": "now"}]) + assert result[0]["success"] + result = self.pubs_tr_disabled.importdescriptors([{"desc": desc_pub, "timestamp": "now"}]) + assert result[0]["success"] def do_test_sendtoaddress(self, comment, pattern, privmap, treefn, keys_pay, keys_change): self.log.info("Testing %s through sendtoaddress" % comment) @@ -272,11 +272,11 @@ class WalletTaprootTest(BitcoinTestFramework): boring_balance = int(self.boring.getbalance() * 100000000) to_amnt = random.randrange(1000000, boring_balance) self.boring.sendtoaddress(address=addr_g, amount=Decimal(to_amnt) / 100000000, subtractfeefromamount=True) - self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress()) + self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress(), sync_fun=self.no_op) test_balance = int(self.rpc_online.getbalance() * 100000000) ret_amnt = random.randrange(100000, test_balance) res = self.rpc_online.sendtoaddress(address=self.boring.getnewaddress(), amount=Decimal(ret_amnt) / 100000000, subtractfeefromamount=True) - self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress()) + self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress(), sync_fun=self.no_op) assert(self.rpc_online.gettransaction(res)["confirmations"] > 0) def do_test_psbt(self, comment, pattern, privmap, treefn, keys_pay, keys_change): @@ -303,7 +303,7 @@ class WalletTaprootTest(BitcoinTestFramework): boring_balance = int(self.boring.getbalance() * 100000000) to_amnt = random.randrange(1000000, boring_balance) self.boring.sendtoaddress(address=addr_g, amount=Decimal(to_amnt) / 100000000, subtractfeefromamount=True) - self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress()) + self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress(), sync_fun=self.no_op) test_balance = int(self.psbt_online.getbalance() * 100000000) ret_amnt = random.randrange(100000, test_balance) psbt = self.psbt_online.walletcreatefundedpsbt([], [{self.boring.getnewaddress(): Decimal(ret_amnt) / 100000000}], None, {"subtractFeeFromOutputs":[0]})['psbt'] @@ -311,7 +311,7 @@ class WalletTaprootTest(BitcoinTestFramework): assert(res['complete']) rawtx = self.nodes[0].finalizepsbt(res['psbt'])['hex'] txid = self.nodes[0].sendrawtransaction(rawtx) - self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress()) + self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress(), sync_fun=self.no_op) assert(self.psbt_online.gettransaction(txid)['confirmations'] > 0) def do_test(self, comment, pattern, privmap, treefn, nkeys): @@ -343,7 +343,7 @@ class WalletTaprootTest(BitcoinTestFramework): self.log.info("Mining blocks...") gen_addr = self.boring.getnewaddress() - self.generatetoaddress(self.nodes[0], 101, gen_addr) + self.generatetoaddress(self.nodes[0], 101, gen_addr, sync_fun=self.no_op) self.do_test( "tr(XPRV)", @@ -412,7 +412,7 @@ class WalletTaprootTest(BitcoinTestFramework): self.log.info("Sending everything back...") txid = self.rpc_online.sendtoaddress(address=self.boring.getnewaddress(), amount=self.rpc_online.getbalance(), subtractfeefromamount=True) - self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress()) + self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress(), sync_fun=self.no_op) assert(self.rpc_online.gettransaction(txid)["confirmations"] > 0) psbt = self.psbt_online.walletcreatefundedpsbt([], [{self.boring.getnewaddress(): self.psbt_online.getbalance()}], None, {"subtractFeeFromOutputs": [0]})['psbt'] @@ -420,7 +420,7 @@ class WalletTaprootTest(BitcoinTestFramework): assert(res['complete']) rawtx = self.nodes[0].finalizepsbt(res['psbt'])['hex'] txid = self.nodes[0].sendrawtransaction(rawtx) - self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress()) + self.generatetoaddress(self.nodes[0], 1, self.boring.getnewaddress(), sync_fun=self.no_op) assert(self.psbt_online.gettransaction(txid)['confirmations'] > 0) if __name__ == '__main__': diff --git a/test/functional/wallet_transactiontime_rescan.py b/test/functional/wallet_transactiontime_rescan.py index afa5139da7..21941084a3 100755 --- a/test/functional/wallet_transactiontime_rescan.py +++ b/test/functional/wallet_transactiontime_rescan.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test transaction time during old block rescanning @@ -10,7 +10,8 @@ import time from test_framework.blocktools import COINBASE_MATURITY from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( - assert_equal + assert_equal, + set_node_times, ) @@ -35,9 +36,7 @@ class TransactionTimeRescanTest(BitcoinTestFramework): # synchronize nodes and time self.sync_all() - minernode.setmocktime(cur_time) - usernode.setmocktime(cur_time) - restorenode.setmocktime(cur_time) + set_node_times(self.nodes, cur_time) # prepare miner wallet minernode.createwallet(wallet_name='default') @@ -68,9 +67,7 @@ class TransactionTimeRescanTest(BitcoinTestFramework): # synchronize nodes and time self.sync_all() - minernode.setmocktime(cur_time + ten_days) - usernode.setmocktime(cur_time + ten_days) - restorenode.setmocktime(cur_time + ten_days) + set_node_times(self.nodes, cur_time + ten_days) # send 10 btc to user's first watch-only address self.log.info('Send 10 btc to user') miner_wallet.sendtoaddress(wo1, 10) @@ -81,9 +78,7 @@ class TransactionTimeRescanTest(BitcoinTestFramework): # synchronize nodes and time self.sync_all() - minernode.setmocktime(cur_time + ten_days + ten_days) - usernode.setmocktime(cur_time + ten_days + ten_days) - restorenode.setmocktime(cur_time + ten_days + ten_days) + set_node_times(self.nodes, cur_time + ten_days + ten_days) # send 5 btc to our second watch-only address self.log.info('Send 5 btc to user') miner_wallet.sendtoaddress(wo2, 5) @@ -94,9 +89,7 @@ class TransactionTimeRescanTest(BitcoinTestFramework): # synchronize nodes and time self.sync_all() - minernode.setmocktime(cur_time + ten_days + ten_days + ten_days) - usernode.setmocktime(cur_time + ten_days + ten_days + ten_days) - restorenode.setmocktime(cur_time + ten_days + ten_days + ten_days) + set_node_times(self.nodes, cur_time + ten_days + ten_days + ten_days) # send 1 btc to our third watch-only address self.log.info('Send 1 btc to user') miner_wallet.sendtoaddress(wo3, 1) @@ -126,6 +119,14 @@ class TransactionTimeRescanTest(BitcoinTestFramework): restorenode.createwallet(wallet_name='wo', disable_private_keys=True) restorewo_wallet = restorenode.get_wallet_rpc('wo') + # for descriptor wallets, the test framework maps the importaddress RPC to the + # importdescriptors RPC (with argument 'timestamp'='now'), which always rescans + # blocks of the past 2 hours, based on the current MTP timestamp; in order to avoid + # importing the last address (wo3), we advance the time further and generate 10 blocks + if self.options.descriptors: + set_node_times(self.nodes, cur_time + ten_days + ten_days + ten_days + ten_days) + self.generatetoaddress(minernode, 10, m1) + restorewo_wallet.importaddress(wo1, rescan=False) restorewo_wallet.importaddress(wo2, rescan=False) restorewo_wallet.importaddress(wo3, rescan=False) diff --git a/test/functional/wallet_txn_clone.py b/test/functional/wallet_txn_clone.py index 7f178d7d46..5bdde13aa4 100755 --- a/test/functional/wallet_txn_clone.py +++ b/test/functional/wallet_txn_clone.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet accounts properly when there are cloned transactions with malleated scriptsigs.""" @@ -93,8 +93,7 @@ class TxnMallTest(BitcoinTestFramework): # Have node0 mine a block, if requested: if (self.options.mine_block): - self.generate(self.nodes[0], 1) - self.sync_blocks(self.nodes[0:2]) + self.generate(self.nodes[0], 1, sync_fun=lambda: self.sync_blocks(self.nodes[0:2])) tx1 = self.nodes[0].gettransaction(txid1) tx2 = self.nodes[0].gettransaction(txid2) @@ -123,14 +122,13 @@ class TxnMallTest(BitcoinTestFramework): return # ... mine a block... - self.generate(self.nodes[2], 1) + self.generate(self.nodes[2], 1, sync_fun=self.no_op) # Reconnect the split network, and sync chain: self.connect_nodes(1, 2) self.nodes[2].sendrawtransaction(node0_tx2["hex"]) self.nodes[2].sendrawtransaction(tx2["hex"]) self.generate(self.nodes[2], 1) # Mine another block to make sure we sync - self.sync_blocks() # Re-fetch transaction info: tx1 = self.nodes[0].gettransaction(txid1) diff --git a/test/functional/wallet_txn_doublespend.py b/test/functional/wallet_txn_doublespend.py index 150e4083b9..206187fb61 100755 --- a/test/functional/wallet_txn_doublespend.py +++ b/test/functional/wallet_txn_doublespend.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2020 The Bitcoin Core developers +# Copyright (c) 2014-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the wallet accounts properly when there is a double-spend conflict.""" @@ -91,8 +91,7 @@ class TxnMallTest(BitcoinTestFramework): # Have node0 mine a block: if (self.options.mine_block): - self.generate(self.nodes[0], 1) - self.sync_blocks(self.nodes[0:2]) + self.generate(self.nodes[0], 1, sync_fun=lambda: self.sync_blocks(self.nodes[0:2])) tx1 = self.nodes[0].gettransaction(txid1) tx2 = self.nodes[0].gettransaction(txid2) @@ -120,12 +119,11 @@ class TxnMallTest(BitcoinTestFramework): self.nodes[2].sendrawtransaction(fund_bar_tx["hex"]) doublespend_txid = self.nodes[2].sendrawtransaction(doublespend["hex"]) # ... mine a block... - self.generate(self.nodes[2], 1) + self.generate(self.nodes[2], 1, sync_fun=self.no_op) # Reconnect the split network, and sync chain: self.connect_nodes(1, 2) self.generate(self.nodes[2], 1) # Mine another block to make sure we sync - self.sync_blocks() assert_equal(self.nodes[0].gettransaction(doublespend_txid)["confirmations"], 2) # Re-fetch transaction info: diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index 5800880830..36e72f2dd9 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """upgradewallet RPC functional test @@ -119,8 +119,7 @@ class UpgradeWalletTest(BitcoinTestFramework): assert_equal(wallet.getwalletinfo()["walletversion"], previous_version) def run_test(self): - self.generatetoaddress(self.nodes[0], COINBASE_MATURITY + 1, self.nodes[0].getnewaddress()) - self.dumb_sync_blocks() + self.generatetoaddress(self.nodes[0], COINBASE_MATURITY + 1, self.nodes[0].getnewaddress(), sync_fun=lambda: self.dumb_sync_blocks()) # # Sanity check the test framework: res = self.nodes[0].getblockchaininfo() assert_equal(res['blocks'], COINBASE_MATURITY + 1) @@ -131,8 +130,7 @@ class UpgradeWalletTest(BitcoinTestFramework): # Send coins to old wallets for later conversion checks. v16_3_wallet = v16_3_node.get_wallet_rpc('wallet.dat') v16_3_address = v16_3_wallet.getnewaddress() - self.generatetoaddress(node_master, COINBASE_MATURITY + 1, v16_3_address) - self.dumb_sync_blocks() + self.generatetoaddress(node_master, COINBASE_MATURITY + 1, v16_3_address, sync_fun=lambda: self.dumb_sync_blocks()) v16_3_balance = v16_3_wallet.getbalance() self.log.info("Test upgradewallet RPC...") diff --git a/test/functional/wallet_watchonly.py b/test/functional/wallet_watchonly.py index 3a9800111b..69c32ba54c 100755 --- a/test/functional/wallet_watchonly.py +++ b/test/functional/wallet_watchonly.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test createwallet watchonly arguments. diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index eeff7a4515..e2eab2a0fe 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Run fuzz test targets. diff --git a/test/get_previous_releases.py b/test/get_previous_releases.py index 62fcad04b3..177aa74191 100755 --- a/test/get_previous_releases.py +++ b/test/get_previous_releases.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/README.md b/test/lint/README.md index c4d76eac94..f4165f908e 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -31,6 +31,7 @@ maintained: * for `src/univalue`: https://github.com/bitcoin-core/univalue-subtree.git (branch master) * for `src/crypto/ctaes`: https://github.com/bitcoin-core/ctaes.git (branch master) * for `src/crc32c`: https://github.com/bitcoin-core/crc32c-subtree.git (branch bitcoin-fork) +* for `src/minisketch`: https://github.com/sipa/minisketch.git (branch master) To do so, add the upstream repository as remote: diff --git a/test/lint/commit-script-check.sh b/test/lint/commit-script-check.sh index 9fca1129b6..6a8a15d05c 100755 --- a/test/lint/commit-script-check.sh +++ b/test/lint/commit-script-check.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,7 +12,7 @@ # one. Any remaining diff signals an error. export LC_ALL=C -if test -z $1; then +if test -z "$1"; then echo "Usage: $0 <commit>..." exit 1 fi @@ -20,10 +20,10 @@ fi RET=0 PREV_BRANCH=$(git name-rev --name-only HEAD) PREV_HEAD=$(git rev-parse HEAD) -for commit in $(git rev-list --reverse $1); do - if git rev-list -n 1 --pretty="%s" $commit | grep -q "^scripted-diff:"; then - git checkout --quiet $commit^ || exit - SCRIPT="$(git rev-list --format=%b -n1 $commit | sed '/^-BEGIN VERIFY SCRIPT-$/,/^-END VERIFY SCRIPT-$/{//!b};d')" +for commit in $(git rev-list --reverse "$1"); do + if git rev-list -n 1 --pretty="%s" "$commit" | grep -q "^scripted-diff:"; then + git checkout --quiet "$commit"^ || exit + SCRIPT="$(git rev-list --format=%b -n1 "$commit" | sed '/^-BEGIN VERIFY SCRIPT-$/,/^-END VERIFY SCRIPT-$/{//!b};d')" if test -z "$SCRIPT"; then echo "Error: missing script for: $commit" echo "Failed" @@ -32,16 +32,16 @@ for commit in $(git rev-list --reverse $1); do echo "Running script for: $commit" echo "$SCRIPT" (eval "$SCRIPT") - git --no-pager diff --exit-code $commit && echo "OK" || (echo "Failed"; false) || RET=1 + git --no-pager diff --exit-code "$commit" && echo "OK" || (echo "Failed"; false) || RET=1 fi git reset --quiet --hard HEAD else - if git rev-list "--format=%b" -n1 $commit | grep -q '^-\(BEGIN\|END\)[ a-zA-Z]*-$'; then + if git rev-list "--format=%b" -n1 "$commit" | grep -q '^-\(BEGIN\|END\)[ a-zA-Z]*-$'; then echo "Error: script block marker but no scripted-diff in title of commit $commit" echo "Failed" RET=1 fi fi done -git checkout --quiet $PREV_BRANCH 2>/dev/null || git checkout --quiet $PREV_HEAD +git checkout --quiet "$PREV_BRANCH" 2>/dev/null || git checkout --quiet "$PREV_HEAD" exit $RET diff --git a/test/lint/extended-lint-cppcheck.sh b/test/lint/extended-lint-cppcheck.sh index 0ab6aad50c..2af39ed60a 100755 --- a/test/lint/extended-lint-cppcheck.sh +++ b/test/lint/extended-lint-cppcheck.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2020 The Bitcoin Core developers +# Copyright (c) 2019-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -73,7 +73,7 @@ function join_array { ENABLED_CHECKS_REGEXP=$(join_array "|" "${ENABLED_CHECKS[@]}") IGNORED_WARNINGS_REGEXP=$(join_array "|" "${IGNORED_WARNINGS[@]}") -WARNINGS=$(git ls-files -- "*.cpp" "*.h" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" | \ +WARNINGS=$(git ls-files -- "*.cpp" "*.h" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" | \ xargs cppcheck --enable=all -j "$(getconf _NPROCESSORS_ONLN)" --language=c++ --std=c++17 --template=gcc -D__cplusplus -DCLIENT_VERSION_BUILD -DCLIENT_VERSION_IS_RELEASE -DCLIENT_VERSION_MAJOR -DCLIENT_VERSION_MINOR -DCOPYRIGHT_YEAR -DDEBUG -I src/ -q 2>&1 | sort -u | \ grep -E "${ENABLED_CHECKS_REGEXP}" | \ grep -vE "${IGNORED_WARNINGS_REGEXP}") diff --git a/test/lint/git-subtree-check.sh b/test/lint/git-subtree-check.sh index 3556e49e08..cdaa5752ac 100755 --- a/test/lint/git-subtree-check.sh +++ b/test/lint/git-subtree-check.sh @@ -82,6 +82,7 @@ if [ -z "$latest_squash" ]; then echo "ERROR: $DIR is not a subtree" >&2 exit 2 fi +# shellcheck disable=SC2086 set $latest_squash old=$1 rev=$2 @@ -92,6 +93,7 @@ if [ -z "$tree_actual" ]; then echo "FAIL: subtree directory $DIR not found in $COMMIT" >&2 exit 1 fi +# shellcheck disable=SC2086 set $tree_actual tree_actual_type=$2 tree_actual_tree=$3 @@ -102,23 +104,23 @@ if [ "d$tree_actual_type" != "dtree" ]; then fi # get the tree at the time of the last subtree update -tree_commit=$(git show -s --format="%T" $old) +tree_commit=$(git show -s --format="%T" "$old") echo "$DIR in $COMMIT was last updated in commit $old (tree $tree_commit)" # ... and compare the actual tree with it if [ "$tree_actual_tree" != "$tree_commit" ]; then - git diff $tree_commit $tree_actual_tree >&2 + git diff "$tree_commit" "$tree_actual_tree" >&2 echo "FAIL: subtree directory was touched without subtree merge" >&2 exit 1 fi if [ "$check_remote" != "0" ]; then # get the tree in the subtree commit referred to - if [ "d$(git cat-file -t $rev 2>/dev/null)" != dcommit ]; then + if [ "d$(git cat-file -t "$rev" 2>/dev/null)" != dcommit ]; then echo "subtree commit $rev unavailable: cannot compare. Did you add and fetch the remote?" >&2 exit 1 fi - tree_subtree=$(git show -s --format="%T" $rev) + tree_subtree=$(git show -s --format="%T" "$rev") echo "$DIR in $COMMIT was last updated to upstream commit $rev (tree $tree_subtree)" # ... and compare the actual tree with it diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh index 8e74f41bb6..69185090d1 100755 --- a/test/lint/lint-circular-dependencies.sh +++ b/test/lint/lint-circular-dependencies.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -15,12 +15,10 @@ EXPECTED_CIRCULAR_DEPENDENCIES=( "index/base -> validation -> index/blockfilterindex -> index/base" "index/coinstatsindex -> node/coinstats -> index/coinstatsindex" "policy/fees -> txmempool -> policy/fees" - "policy/rbf -> txmempool -> validation -> policy/rbf" "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel" "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel" "qt/sendcoinsdialog -> qt/walletmodel -> qt/sendcoinsdialog" "qt/transactiontablemodel -> qt/walletmodel -> qt/transactiontablemodel" - "txmempool -> validation -> txmempool" "wallet/fees -> wallet/wallet -> wallet/fees" "wallet/wallet -> wallet/walletdb -> wallet/wallet" "node/coinstats -> validation -> node/coinstats" diff --git a/test/lint/lint-files.py b/test/lint/lint-files.py index 400921e5f3..68b795eef7 100755 --- a/test/lint/lint-files.py +++ b/test/lint/lint-files.py @@ -19,7 +19,7 @@ CMD_SHEBANG_FILES = "git grep --full-name --line-number -I '^#!'" ALLOWED_FILENAME_REGEXP = "^[a-zA-Z0-9/_.@][a-zA-Z0-9/_.@-]*$" ALLOWED_SOURCE_FILENAME_REGEXP = "^[a-z0-9_./-]+$" ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP = ( - "^src/(secp256k1/|univalue/|test/fuzz/FuzzedDataProvider.h)" + "^src/(secp256k1/|minisketch/|univalue/|test/fuzz/FuzzedDataProvider.h)" ) ALLOWED_PERMISSION_NON_EXECUTABLES = 644 ALLOWED_PERMISSION_EXECUTABLES = 755 diff --git a/test/lint/lint-files.sh b/test/lint/lint-files.sh index 1e115778bd..f9ede4bc68 100755 --- a/test/lint/lint-files.sh +++ b/test/lint/lint-files.sh @@ -3,5 +3,5 @@ export LC_ALL=C set -e -cd "$(dirname $0)/../.." +cd "$(dirname "$0")/../.." test/lint/lint-files.py diff --git a/test/lint/lint-format-strings.sh b/test/lint/lint-format-strings.sh index cb3ec09ae6..d98f12b1a1 100755 --- a/test/lint/lint-format-strings.sh +++ b/test/lint/lint-format-strings.sh @@ -34,7 +34,7 @@ if ! python3 -m doctest test/lint/lint-format-strings.py; then fi for S in "${FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS[@]}"; do IFS="," read -r FUNCTION_NAME SKIP_ARGUMENTS <<< "${S}" - for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(leveldb|secp256k1|tinyformat|univalue|test/fuzz/strprintf.cpp)"); do + for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(leveldb|secp256k1|minisketch|tinyformat|univalue|test/fuzz/strprintf.cpp)"); do MATCHING_FILES+=("${MATCHING_FILE}") done if ! test/lint/lint-format-strings.py --skip-arguments "${SKIP_ARGUMENTS}" "${FUNCTION_NAME}" "${MATCHING_FILES[@]}"; then diff --git a/test/lint/lint-git-commit-check.sh b/test/lint/lint-git-commit-check.sh index 2b3a9b87c2..d1ab72658b 100755 --- a/test/lint/lint-git-commit-check.sh +++ b/test/lint/lint-git-commit-check.sh @@ -38,7 +38,7 @@ while IFS= read -r commit_hash || [[ -n "$commit_hash" ]]; do while IFS= read -r line || [[ -n "$line" ]]; do n_line=$((n_line+1)) length=${#line} - if [ $n_line -eq 2 ] && [ $length -ne 0 ]; then + if [ $n_line -eq 2 ] && [ "$length" -ne 0 ]; then echo "The subject line of commit hash ${commit_hash} is followed by a non-empty line. Subject lines should always be followed by a blank line." EXIT_CODE=1 fi diff --git a/test/lint/lint-include-guards.sh b/test/lint/lint-include-guards.sh index c23b903bce..23f53f027e 100755 --- a/test/lint/lint-include-guards.sh +++ b/test/lint/lint-include-guards.sh @@ -10,14 +10,14 @@ export LC_ALL=C HEADER_ID_PREFIX="BITCOIN_" HEADER_ID_SUFFIX="_H" -REGEXP_EXCLUDE_FILES_WITH_PREFIX="src/(crypto/ctaes/|leveldb/|crc32c/|secp256k1/|test/fuzz/FuzzedDataProvider.h|tinyformat.h|bench/nanobench.h|univalue/)" +REGEXP_EXCLUDE_FILES_WITH_PREFIX="src/(crypto/ctaes/|leveldb/|crc32c/|secp256k1/|minisketch/|test/fuzz/FuzzedDataProvider.h|tinyformat.h|bench/nanobench.h|univalue/)" EXIT_CODE=0 for HEADER_FILE in $(git ls-files -- "*.h" | grep -vE "^${REGEXP_EXCLUDE_FILES_WITH_PREFIX}") do HEADER_ID_BASE=$(cut -f2- -d/ <<< "${HEADER_FILE}" | sed "s/\.h$//g" | tr / _ | tr - _ | tr "[:lower:]" "[:upper:]") HEADER_ID="${HEADER_ID_PREFIX}${HEADER_ID_BASE}${HEADER_ID_SUFFIX}" - if [[ $(grep -cE "^#(ifndef|define) ${HEADER_ID}" "${HEADER_FILE}") != 2 ]]; then + if [[ $(grep --count --extended-regexp "^#(ifndef|define|endif //) ${HEADER_ID}" "${HEADER_FILE}") != 3 ]]; then echo "${HEADER_FILE} seems to be missing the expected include guard:" echo " #ifndef ${HEADER_ID}" echo " #define ${HEADER_ID}" diff --git a/test/lint/lint-includes.sh b/test/lint/lint-includes.sh index bf7aeb5b4f..98d5021657 100755 --- a/test/lint/lint-includes.sh +++ b/test/lint/lint-includes.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -9,10 +9,10 @@ # Check includes: Check for duplicate includes. Enforce bracket syntax includes. export LC_ALL=C -IGNORE_REGEXP="/(leveldb|secp256k1|univalue|crc32c)/" +IGNORE_REGEXP="/(leveldb|secp256k1|minisketch|univalue|crc32c)/" # cd to root folder of git repo for git ls-files to work properly -cd "$(dirname $0)/../.." || exit 1 +cd "$(dirname "$0")/../.." || exit 1 filter_suffix() { git ls-files | grep -E "^src/.*\.${1}"'$' | grep -Ev "${IGNORE_REGEXP}" diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index b119cffec8..661bc35175 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -37,18 +37,15 @@ export LC_ALL=C # See https://doc.qt.io/qt-5/qcoreapplication.html#locale-settings and # https://stackoverflow.com/a/34878283 for more details. -# TODO: Reduce KNOWN_VIOLATIONS by replacing uses of locale dependent stoul/strtol with locale -# independent ToIntegral<T>(...) or the ParseInt*() functions. # TODO: Reduce KNOWN_VIOLATIONS by replacing uses of locale dependent snprintf with strprintf. KNOWN_VIOLATIONS=( "src/dbwrapper.cpp:.*vsnprintf" "src/test/dbwrapper_tests.cpp:.*snprintf" "src/test/fuzz/locale.cpp" "src/test/fuzz/string.cpp" - "src/torcontrol.cpp:.*strtol" ) -REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(crypto/ctaes/|leveldb/|secp256k1/|tinyformat.h|univalue/)" +REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(crypto/ctaes/|leveldb/|secp256k1/|minisketch/|tinyformat.h|univalue/)" LOCALE_DEPENDENT_FUNCTIONS=( alphasort # LC_COLLATE (via strcoll) diff --git a/test/lint/lint-logs.sh b/test/lint/lint-logs.sh index d6c53e8ff3..6d5165f649 100755 --- a/test/lint/lint-logs.sh +++ b/test/lint/lint-logs.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # diff --git a/test/lint/lint-python-dead-code.sh b/test/lint/lint-python-dead-code.sh index c3b6ff3c98..247bfb310a 100755 --- a/test/lint/lint-python-dead-code.sh +++ b/test/lint/lint-python-dead-code.sh @@ -15,9 +15,8 @@ fi # --min-confidence 100 will only report code that is guaranteed to be unused within the analyzed files. # Any value below 100 introduces the risk of false positives, which would create an unacceptable maintenance burden. -if ! vulture \ - --min-confidence 100 \ - $(git ls-files -- "*.py"); then +mapfile -t FILES < <(git ls-files -- "*.py") +if ! vulture --min-confidence 100 "${FILES[@]}"; then echo "Python dead code detection found some issues" exit 1 fi diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh index 3d22407fd1..7d7857d325 100755 --- a/test/lint/lint-python.sh +++ b/test/lint/lint-python.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -92,6 +92,7 @@ fi EXIT_CODE=0 +# shellcheck disable=SC2046 if ! PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}") $( if [[ $# == 0 ]]; then git ls-files "*.py" @@ -102,7 +103,8 @@ if ! PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; e EXIT_CODE=1 fi -if ! mypy --show-error-codes $(git ls-files "test/functional/*.py" "contrib/devtools/*.py"); then +mapfile -t FILES < <(git ls-files "test/functional/*.py" "contrib/devtools/*.py") +if ! mypy --show-error-codes "${FILES[@]}"; then EXIT_CODE=1 fi diff --git a/test/lint/lint-shell-locale.sh b/test/lint/lint-shell-locale.sh index 084dc93f76..bd6b6ce05c 100755 --- a/test/lint/lint-shell-locale.sh +++ b/test/lint/lint-shell-locale.sh @@ -12,7 +12,7 @@ export LC_ALL=C EXIT_CODE=0 -for SHELL_SCRIPT in $(git ls-files -- "*.sh" | grep -vE "src/(secp256k1|univalue)/"); do +for SHELL_SCRIPT in $(git ls-files -- "*.sh" | grep -vE "src/(secp256k1|minisketch|univalue)/"); do if grep -q "# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"" "${SHELL_SCRIPT}"; then continue fi diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh index 73ac583d84..5fa104fce6 100755 --- a/test/lint/lint-shell.sh +++ b/test/lint/lint-shell.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2020 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -10,8 +10,6 @@ export LC_ALL=C # Disabled warnings: disabled=( - SC2046 # Quote this to prevent word splitting. - SC2086 # Double quote to prevent globbing and word splitting. SC2162 # read without -r will mangle backslashes. ) @@ -22,10 +20,13 @@ if ! command -v shellcheck > /dev/null; then exit $EXIT_CODE fi -SHELLCHECK_CMD=(shellcheck --external-sources --check-sourced) +SHELLCHECK_CMD=(shellcheck --external-sources --check-sourced --source-path=SCRIPTDIR) EXCLUDE="--exclude=$(IFS=','; echo "${disabled[*]}")" -SOURCED_FILES=$(git ls-files | xargs gawk '/^# shellcheck shell=/ {print FILENAME} {nextfile}') # Check shellcheck directive used for sourced files -if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE" $SOURCED_FILES $(git ls-files -- '*.sh' | grep -vE 'src/(leveldb|secp256k1|univalue)/'); then +# Check shellcheck directive used for sourced files +mapfile -t SOURCED_FILES < <(git ls-files | xargs gawk '/^# shellcheck shell=/ {print FILENAME} {nextfile}') +mapfile -t GUIX_FILES < <(git ls-files contrib/guix contrib/shell | xargs gawk '/^#!\/usr\/bin\/env bash/ {print FILENAME} {nextfile}') +mapfile -t FILES < <(git ls-files -- '*.sh' | grep -vE 'src/(leveldb|secp256k1|minisketch|univalue)/') +if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE" "${SOURCED_FILES[@]}" "${GUIX_FILES[@]}" "${FILES[@]}"; then EXIT_CODE=1 fi diff --git a/test/lint/lint-spelling.sh b/test/lint/lint-spelling.sh index 111091b7f8..b3e558b02a 100755 --- a/test/lint/lint-spelling.sh +++ b/test/lint/lint-spelling.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -15,6 +15,7 @@ if ! command -v codespell > /dev/null; then fi IGNORE_WORDS_FILE=test/lint/lint-spelling.ignore-words.txt -if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} $(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/qt/locale/" ":(exclude)src/qt/*.qrc" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)contrib/builder-keys/keys.txt" ":(exclude)contrib/guix/patches"); then +mapfile -t FILES < <(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/qt/locale/" ":(exclude)src/qt/*.qrc" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)contrib/builder-keys/keys.txt" ":(exclude)contrib/guix/patches") +if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} "${FILES[@]}"; then echo "^ Warning: codespell identified likely spelling errors. Any false positives? Add them to the list of ignored words in ${IGNORE_WORDS_FILE}" fi diff --git a/test/lint/lint-whitespace.sh b/test/lint/lint-whitespace.sh index 37fcf7804a..9d55c71eb5 100755 --- a/test/lint/lint-whitespace.sh +++ b/test/lint/lint-whitespace.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2017-2020 The Bitcoin Core developers +# Copyright (c) 2017-2021 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -33,14 +33,14 @@ if [ -z "${COMMIT_RANGE}" ]; then fi showdiff() { - if ! git diff -U0 "${COMMIT_RANGE}" -- "." ":(exclude)depends/patches/" ":(exclude)contrib/guix/patches/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/" ":(exclude)src/qt/locale/"; then + if ! git diff -U0 "${COMMIT_RANGE}" -- "." ":(exclude)depends/patches/" ":(exclude)contrib/guix/patches/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/" ":(exclude)src/qt/locale/"; then echo "Failed to get a diff" exit 1 fi } showcodediff() { - if ! git diff -U0 "${COMMIT_RANGE}" -- *.cpp *.h *.md *.py *.sh ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/" ":(exclude)src/qt/locale/"; then + if ! git diff -U0 "${COMMIT_RANGE}" -- *.cpp *.h *.md *.py *.sh ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/" ":(exclude)src/qt/locale/"; then echo "Failed to get a diff" exit 1 fi diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan index 7db051ca37..3c5a15a0c7 100644 --- a/test/sanitizer_suppressions/tsan +++ b/test/sanitizer_suppressions/tsan @@ -41,3 +41,6 @@ race:CZMQAbstractPublishNotifier::SendZmqMessage # https://github.com/bitcoin/bitcoin/pull/20218, https://github.com/bitcoin/bitcoin/pull/20745 race:epoll_ctl + +# https://github.com/bitcoin/bitcoin/issues/23366 +race:std::__1::ios_base::width diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index 1d608b9ec1..bb7a1a5f45 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -34,6 +34,7 @@ unsigned-integer-overflow:crypto/ unsigned-integer-overflow:FuzzedDataProvider.h unsigned-integer-overflow:hash.cpp unsigned-integer-overflow:leveldb/ +unsigned-integer-overflow:minisketch/ unsigned-integer-overflow:policy/fees.cpp unsigned-integer-overflow:prevector.h unsigned-integer-overflow:pubkey.h @@ -59,6 +60,7 @@ implicit-integer-sign-change:crypto/ # implicit-integer-sign-change in FuzzedDataProvider's ConsumeIntegralInRange implicit-integer-sign-change:FuzzedDataProvider.h implicit-integer-sign-change:key.cpp +implicit-integer-sign-change:minisketch/ implicit-integer-sign-change:noui.cpp implicit-integer-sign-change:policy/fees.cpp implicit-integer-sign-change:prevector.h @@ -84,12 +86,10 @@ implicit-signed-integer-truncation:addrman.cpp implicit-signed-integer-truncation:addrman.h implicit-signed-integer-truncation:chain.h implicit-signed-integer-truncation:crypto/ -implicit-signed-integer-truncation:cuckoocache.h implicit-signed-integer-truncation:leveldb/ -implicit-signed-integer-truncation:miner.cpp +implicit-signed-integer-truncation:node/miner.cpp implicit-signed-integer-truncation:net.cpp implicit-signed-integer-truncation:net_processing.cpp -implicit-signed-integer-truncation:netaddress.cpp implicit-signed-integer-truncation:streams.h implicit-signed-integer-truncation:test/arith_uint256_tests.cpp implicit-signed-integer-truncation:test/skiplist_tests.cpp @@ -104,6 +104,7 @@ shift-base:arith_uint256.cpp shift-base:crypto/ shift-base:hash.cpp shift-base:leveldb/ +shift-base:minisketch/ shift-base:net_processing.cpp shift-base:streams.h shift-base:util/bip32.cpp |
