diff options
308 files changed, 6454 insertions, 3148 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 48747ac138..f7962e951c 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -79,18 +79,19 @@ task: << : *FILTER_TEMPLATE windows_container: cpu: 4 - memory: 16G + memory: 8G image: cirrusci/windowsservercore:visualstudio2019 timeout_in: 120m env: PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Current\Bin;%PATH%' PYTHONUTF8: 1 - VCPKG_TAG: '75522bb1f2e7d863078bcd06322348f053a9e33f' - VCPKG_FEATURE_FLAGS: 'manifests' + CI_VCPKG_TAG: '2021.05.12' + VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads' + VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.12/5.12.11/single/qt-everywhere-src-5.12.11.zip' QT_LOCAL_PATH: 'C:\qt-everywhere-src-5.12.11.zip' QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.12.11' - QTBASEDIR: 'C:\Qt5.12.11_x64_static_vs2019_160900' + QTBASEDIR: 'C:\Qt_static' x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"' IgnoreWarnIntDirInTempDetected: 'true' merge_script: @@ -121,8 +122,20 @@ task: - ..\configure -release -silent -opensource -confirm-license -opengl desktop -no-shared -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -no-libjpeg -nomake examples -nomake tests -nomake tools -no-dbus -no-libudev -no-icu -no-gtk -no-opengles3 -no-angle -no-sql-sqlite -no-sql-odbc -no-sqlite -no-libudev -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcanvas3d -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquickcontrols -skip qtquickcontrols2 -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-sql -no-feature-sqlmodel -prefix %QTBASEDIR% - jom - jom install - vcpkg_cache: - folder: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' + vcpkg_tools_cache: + folder: '%VCPKG_DOWNLOADS%\tools' + reupload_on_changes: false + fingerprint_script: + - echo %CI_VCPKG_TAG% + - msbuild -version + vcpkg_binary_cache: + folder: '%VCPKG_DEFAULT_BINARY_CACHE%' + reupload_on_changes: true + fingerprint_script: + - echo %CI_VCPKG_TAG% + - msbuild -version + populate_script: + - mkdir %VCPKG_DEFAULT_BINARY_CACHE% install_python_script: - choco install --yes --no-progress python3 --version=3.9.6 - pip install zmq @@ -131,7 +144,7 @@ task: - cd .. - git clone --quiet https://github.com/microsoft/vcpkg.git - cd vcpkg - - git -c advice.detachedHead=false checkout %VCPKG_TAG% + - git -c advice.detachedHead=false checkout %CI_VCPKG_TAG% - .\bootstrap-vcpkg -disableMetrics - echo set(VCPKG_BUILD_TYPE release) >> triplets\x64-windows-static.cmake - .\vcpkg integrate install diff --git a/Makefile.am b/Makefile.am index ce66331910..af63cf0cbb 100644 --- a/Makefile.am +++ b/Makefile.am @@ -58,8 +58,7 @@ DIST_SHARE = \ BIN_CHECKS=$(top_srcdir)/contrib/devtools/symbol-check.py \ $(top_srcdir)/contrib/devtools/security-check.py \ - $(top_srcdir)/contrib/devtools/utils.py \ - $(top_srcdir)/contrib/devtools/pixie.py + $(top_srcdir)/contrib/devtools/utils.py WINDOWS_PACKAGING = $(top_srcdir)/share/pixmaps/bitcoin.ico \ $(top_srcdir)/share/pixmaps/nsis-header.bmp \ @@ -367,14 +366,14 @@ clean-local: clean-docs test-security-check: if TARGET_DARWIN - $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_MACHO - $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_MACHO + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_MACHO + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_MACHO endif if TARGET_WINDOWS - $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_PE - $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_PE + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_PE + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_PE endif if TARGET_LINUX - $(AM_V_at) CC='$(CC)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_ELF - $(AM_V_at) CC='$(CC)' CPPFILT='$(CPPFILT)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_ELF + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-security-check.py TestSecurityChecks.test_ELF + $(AM_V_at) CC='$(CC)' CFLAGS='$(CFLAGS)' CPPFLAGS='$(CPPFLAGS)' LDFLAGS='$(LDFLAGS)' $(PYTHON) $(top_srcdir)/contrib/devtools/test-symbol-check.py TestSymbolChecks.test_ELF endif diff --git a/build-aux/m4/bitcoin_find_bdb48.m4 b/build-aux/m4/bitcoin_find_bdb48.m4 index 5fc5b493d3..3d6c8210ed 100644 --- a/build-aux/m4/bitcoin_find_bdb48.m4 +++ b/build-aux/m4/bitcoin_find_bdb48.m4 @@ -48,15 +48,22 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ if test "x$bdbpath" = "xX"; then use_bdb=no AC_MSG_RESULT([no]) - AC_MSG_ERROR([libdb_cxx headers missing, ]AC_PACKAGE_NAME[ requires this library for BDB wallet support (--without-bdb to disable BDB wallet support)]) + AC_MSG_WARN([libdb_cxx headers missing]) + AC_MSG_WARN(AC_PACKAGE_NAME[ requires this library for BDB (legacy) wallet support]) + AC_MSG_WARN([Passing --without-bdb will suppress this warning]) elif test "x$bdb48path" = "xX"; then BITCOIN_SUBDIR_TO_INCLUDE(BDB_CPPFLAGS,[${bdbpath}],db_cxx) AC_ARG_WITH([incompatible-bdb],[AS_HELP_STRING([--with-incompatible-bdb], [allow using a bdb version other than 4.8])],[ - AC_MSG_WARN([Found Berkeley DB other than 4.8; BDB wallets opened by this build will not be portable!]) + AC_MSG_WARN([Found Berkeley DB other than 4.8]) + AC_MSG_WARN([BDB (legacy) wallets opened by this build will not be portable!]) + use_bdb=yes ],[ - AC_MSG_ERROR([Found Berkeley DB other than 4.8, required for portable BDB wallets (--with-incompatible-bdb to ignore or --without-bdb to disable BDB wallet support)]) + AC_MSG_WARN([Found Berkeley DB other than 4.8]) + AC_MSG_WARN([BDB (legacy) wallets opened by this build would not be portable!]) + AC_MSG_WARN([If this is intended, pass --with-incompatible-bdb]) + AC_MSG_WARN([Passing --without-bdb will suppress this warning]) + use_bdb=no ]) - use_bdb=yes else BITCOIN_SUBDIR_TO_INCLUDE(BDB_CPPFLAGS,[${bdb48path}],db_cxx) bdbpath="${bdb48path}" @@ -78,7 +85,9 @@ AC_DEFUN([BITCOIN_FIND_BDB48],[ ]) done if test "x$BDB_LIBS" = "x"; then - AC_MSG_ERROR([libdb_cxx missing, ]AC_PACKAGE_NAME[ requires this library for BDB wallet support (--without-bdb to disable BDB wallet support)]) + AC_MSG_WARN([libdb_cxx headers missing]) + AC_MSG_WARN(AC_PACKAGE_NAME[ requires this library for BDB (legacy) wallet support]) + AC_MSG_WARN([Passing --without-bdb will suppress this warning]) fi fi if test "x$use_bdb" != "xno"; then diff --git a/build_msvc/README.md b/build_msvc/README.md index c3705f6b03..36fb942c8e 100644 --- a/build_msvc/README.md +++ b/build_msvc/README.md @@ -3,67 +3,65 @@ Building Bitcoin Core with Visual Studio Introduction --------------------- -Solution and project files to build the Bitcoin Core applications `msbuild` or Visual Studio can be found in the `build_msvc` directory. The build has been tested with Visual Studio 2019 (building with earlier versions of Visual Studio should not be expected to work). +Solution and project files to build Bitcoin Core with `msbuild` or Visual Studio can be found in the `build_msvc` directory. The build has been tested with Visual Studio 2019 (building with earlier versions of Visual Studio should not be expected to work). -Building with Visual Studio is an alternative to the Linux based [cross-compiler build](https://github.com/bitcoin/bitcoin/blob/master/doc/build-windows.md). +To build Bitcoin Core from the command-line, it is sufficient to only install the Visual Studio Build Tools component. -Quick Start ---------------------- -The minimal steps required to build Bitcoin Core with the msbuild toolchain are below. More detailed instructions are contained in the following sections. +Building with Visual Studio is an alternative to the Linux based [cross-compiler build](../doc/build-windows.md). -``` -cd build_msvc -py -3 msvc-autogen.py -msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build -``` -Dependencies +Prerequisites --------------------- -A number of [open source libraries](https://github.com/bitcoin/bitcoin/blob/master/doc/dependencies.md) are required in order to be able to build Bitcoin Core. +To build [dependencies](../doc/dependencies.md) (except for [Qt](#qt)), +the default approach is to use the [vcpkg](https://docs.microsoft.com/en-us/cpp/vcpkg) package manager from Microsoft: -Options for installing the dependencies in a Visual Studio compatible manner are: +1. [Install](https://vcpkg.io/en/getting-started.html) vcpkg. -- Use Microsoft's [vcpkg](https://docs.microsoft.com/en-us/cpp/vcpkg) to download the source packages and build locally. This is the recommended approach. -- Download the source code, build each dependency, add the required include paths, link libraries and binary tools to the Visual Studio project files. -- Use [nuget](https://www.nuget.org/) packages with the understanding that any binary files have been compiled by an untrusted third party. +2. By default, vcpkg makes both `release` and `debug` builds for each package. +To save build time and disk space, one could skip `debug` builds (example uses PowerShell): +```powershell -The [external dependencies](https://github.com/bitcoin/bitcoin/blob/master/doc/dependencies.md) required for building are listed in the `build_msvc/vcpkg.json` file. To ensure `msbuild` project files automatically install the `vcpkg` dependencies use: - -``` -vcpkg integrate install +Add-Content -Path "vcpkg\triplets\x64-windows-static.cmake" -Value "set(VCPKG_BUILD_TYPE release)" ``` Qt --------------------- -In order to build Bitcoin Core a static build of Qt is required. The runtime library version (e.g. v142) and platform type (x86 or x64) must also match. +To build Bitcoin Core with the GUI, a static build of Qt is required. + +1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-src-5.12.11.zip`](https://download.qt.io/official_releases/qt/5.12/5.12.11/single/qt-everywhere-src-5.12.11.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`. + +2. Open "x64 Native Tools Command Prompt for VS 2019", and input the following commands: +```cmd +cd C:\dev\qt-source +mkdir build +cd build +..\configure -release -silent -opensource -confirm-license -opengl desktop -no-shared -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -no-libjpeg -nomake examples -nomake tests -nomake tools -no-dbus -no-libudev -no-icu -no-gtk -no-opengles3 -no-angle -no-sql-sqlite -no-sql-odbc -no-sqlite -no-libudev -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcanvas3d -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquickcontrols -skip qtquickcontrols2 -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-sql -no-feature-sqlmodel -prefix C:\Qt_static +nmake +nmake install +``` -Some prebuilt x64 versions of Qt can be downloaded from [here](https://github.com/sipsorcery/qt_win_binary/releases). Please be aware these downloads are NOT officially sanctioned by Bitcoin Core and are provided for developer convenience only. They should NOT be used for builds that will be used in a production environment or with real funds. +One could speed up building with [`jom`](https://wiki.qt.io/Jom), a replacement for `nmake` which makes use of all CPU cores. -To determine which Qt prebuilt version to download open the `.cirrus.yml` file and note the `QT_DOWNLOAD_URL`. When extracting the zip file the destination path must be set to `C:\`. This is due to the way that Qt includes, libraries and tools use internal paths. +To build Bitcoin Core without Qt, unload or disable the `bitcoin-qt`, `libbitcoin_qt` and `test_bitcoin-qt` projects. -To build Bitcoin Core without Qt unload or disable the `bitcoin-qt`, `libbitcoin_qt` and `test_bitcoin-qt` projects. Building --------------------- -The instructions below use `vcpkg` to install the dependencies. - -- Install [`vcpkg`](https://github.com/Microsoft/vcpkg). - -- Use Python to generate `*.vcxproj` from Makefile +1. Use Python to generate `*.vcxproj` from Makefile: ``` PS >py -3 msvc-autogen.py ``` -- An optional step is to adjust the settings in the `build_msvc` directory and the `common.init.vcxproj` file. This project file contains settings that are common to all projects such as the runtime library version and target Windows SDK version. The Qt directories can also be set. +2. An optional step is to adjust the settings in the `build_msvc` directory and the `common.init.vcxproj` file. This project file contains settings that are common to all projects such as the runtime library version and target Windows SDK version. The Qt directories can also be set. To specify a non-default path to a static Qt package directory, use the `QTBASEDIR` environment variable. -- To build from the command line with the Visual Studio 2019 toolchain use: +3. To build from the command-line with the Visual Studio 2019 toolchain use: -``` -msbuild /m bitcoin.sln /p:Platform=x64 /p:Configuration=Release /t:build +```cmd +msbuild -property:Configuration=Release -maxCpuCount -verbosity:minimal bitcoin.sln ``` -- Alternatively, open the `build_msvc/bitcoin.sln` file in Visual Studio 2019. +Alternatively, open the `build_msvc/bitcoin.sln` file in Visual Studio 2019. Security --------------------- diff --git a/build_msvc/common.qt.init.vcxproj b/build_msvc/common.qt.init.vcxproj index df2fd2fb49..cc8063e545 100644 --- a/build_msvc/common.qt.init.vcxproj +++ b/build_msvc/common.qt.init.vcxproj @@ -2,7 +2,7 @@ <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <PropertyGroup Label="QtGlobals"> - <QtBaseDir>C:\Qt5.12.11_x64_static_vs2019_160900</QtBaseDir> + <QtBaseDir Condition="'$(QTBASEDIR)' == ''">C:\Qt_static</QtBaseDir> <QtPluginsLibraryDir>$(QtBaseDir)\plugins</QtPluginsLibraryDir> <QtLibraryDir>$(QtBaseDir)\lib</QtLibraryDir> <QtIncludeDir>$(QtBaseDir)\include</QtIncludeDir> diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 5587618f2d..991234a436 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -13,7 +13,8 @@ update-alternatives --install /usr/bin/clang-format-diff clang-format-diff $(whi ${CI_RETRY_EXE} pip3 install codespell==2.0.0 ${CI_RETRY_EXE} pip3 install flake8==3.8.3 -${CI_RETRY_EXE} pip3 install mypy==0.781 +${CI_RETRY_EXE} pip3 install mypy==0.910 +${CI_RETRY_EXE} pip3 install pyzmq==22.3.0 ${CI_RETRY_EXE} pip3 install vulture==2.3 SHELLCHECK_VERSION=v0.7.2 diff --git a/ci/test/00_setup_env_android.sh b/ci/test/00_setup_env_android.sh index 4ef3ae1ceb..2f9d1f2a9f 100755 --- a/ci/test/00_setup_env_android.sh +++ b/ci/test/00_setup_env_android.sh @@ -22,4 +22,4 @@ export ANDROID_HOME="${DEPENDS_DIR}/SDKs/android" export ANDROID_NDK_HOME="${ANDROID_HOME}/ndk/${ANDROID_NDK_VERSION}" export DEP_OPTS="ANDROID_SDK=${ANDROID_HOME} ANDROID_NDK=${ANDROID_NDK_HOME} ANDROID_API_LEVEL=${ANDROID_API_LEVEL} ANDROID_TOOLCHAIN_BIN=${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/linux-x86_64/bin/" -export BITCOIN_CONFIG="--disable-ccache" +export BITCOIN_CONFIG="--disable-ccache --disable-tests --enable-gui-tests --disable-bench --disable-fuzz-binary --without-utils --without-libs --without-daemon" diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh index f7f65f6e3a..a25c98a004 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -14,4 +14,5 @@ export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' LDFLAGS='--rtlib=compiler-rt -lgcc_s'" export TEST_RUNNER_ENV="BITCOIND=bitcoin-node" +export TEST_RUNNER_EXTRA="--nosandbox" export PIP_PACKAGES="lief" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 78af869e70..0058a042f5 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -11,6 +11,6 @@ export CONTAINER_NAME=ci_native_valgrind export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 -export TEST_RUNNER_EXTRA="--exclude rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 +export TEST_RUNNER_EXTRA="--nosandbox --exclude rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index a42cd6cee1..b1d83883d1 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -10,7 +10,7 @@ if [ -n "$ANDROID_TOOLS_URL" ]; then DOCKER_EXEC make distclean || true DOCKER_EXEC ./autogen.sh DOCKER_EXEC ./configure $BITCOIN_CONFIG --prefix=$DEPENDS_DIR/aarch64-linux-android || ( (DOCKER_EXEC cat config.log) && false) - DOCKER_EXEC "cd src/qt && make $MAKEJOBS && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk" + DOCKER_EXEC "make $MAKEJOBS && cd src/qt && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk" exit 0 fi diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 194b14beab..311a43755a 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -9,14 +9,12 @@ export LC_ALL=C.UTF-8 if [[ $HOST = *-mingw32 ]]; then # Generate all binaries, so that they can be wrapped DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1 - DOCKER_EXEC make $MAKEJOBS -C src/univalue VERBOSE=1 DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" fi if [ -n "$QEMU_USER_CMD" ]; then # Generate all binaries, so that they can be wrapped DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1 - DOCKER_EXEC make $MAKEJOBS -C src/univalue VERBOSE=1 DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" fi diff --git a/configure.ac b/configure.ac index 26092a7b8d..dc3b07e4bb 100644 --- a/configure.ac +++ b/configure.ac @@ -71,6 +71,12 @@ case $host in ;; esac +AC_ARG_WITH([seccomp], + [AS_HELP_STRING([--with-seccomp], + [enable experimental syscall sandbox feature (-sandbox), default is yes if seccomp-bpf is detected under Linux x86_64])], + [seccomp_found=$withval], + [seccomp_found=auto]) + dnl Require C++17 compiler (no GNU extensions) AX_CXX_COMPILE_STDCXX([17], [noext], [mandatory]) @@ -101,13 +107,12 @@ AC_PATH_TOOL(GCOV, gcov) AC_PATH_TOOL(LLVM_COV, llvm-cov) AC_PATH_PROG(LCOV, lcov) dnl Python 3.6 is specified in .python-version and should be used if available, see doc/dependencies.md -AC_PATH_PROGS([PYTHON], [python3.6 python3.7 python3.8 python3.9 python3 python]) +AC_PATH_PROGS([PYTHON], [python3.6 python3.7 python3.8 python3.9 python3.10 python3.11 python3 python]) AC_PATH_PROG(GENHTML, genhtml) AC_PATH_PROG([GIT], [git]) AC_PATH_PROG(CCACHE,ccache) AC_PATH_PROG(XGETTEXT,xgettext) AC_PATH_PROG(HEXDUMP,hexdump) -AC_PATH_TOOL(CPPFILT, c++filt) AC_PATH_TOOL(OBJCOPY, objcopy) AC_PATH_PROG(DOXYGEN, doxygen) AM_CONDITIONAL([HAVE_DOXYGEN], [test -n "$DOXYGEN"]) @@ -256,12 +261,6 @@ if test "x$use_asm" = xyes; then AC_DEFINE(USE_ASM, 1, [Define this symbol to build in assembly routines]) fi -AC_ARG_WITH([system-univalue], - [AS_HELP_STRING([--with-system-univalue], - [Build with system UniValue (default is no)])], - [system_univalue=$withval], - [system_univalue=no] -) AC_ARG_ENABLE([zmq], [AS_HELP_STRING([--disable-zmq], [disable ZMQ notifications])], @@ -315,7 +314,7 @@ AC_ARG_ENABLE([gprof], dnl Turn warnings into errors AC_ARG_ENABLE([werror], [AS_HELP_STRING([--enable-werror], - [Treat certain compiler warnings as errors (default is no)])], + [Treat compiler warnings as errors (default is no)])], [enable_werror=$enableval], [enable_werror=no]) @@ -403,34 +402,13 @@ if test "x$enable_werror" = "xyes"; then if test "x$CXXFLAG_WERROR" = "x"; then AC_MSG_ERROR("enable-werror set but -Werror is not usable") fi - AX_CHECK_COMPILE_FLAG([-Werror=gnu],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=gnu"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=vla],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=vla"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=shadow-field],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=shadow-field"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=switch],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=switch"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=thread-safety],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=thread-safety"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=range-loop-analysis],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=range-loop-analysis"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=unused-variable],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unused-variable"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=date-time],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=date-time"],,[[$CXXFLAG_WERROR]]) + ERROR_CXXFLAGS=$CXXFLAG_WERROR dnl -Wreturn-type is broken in GCC for MinGW-w64. dnl https://sourceforge.net/p/mingw-w64/bugs/306/ - AX_CHECK_COMPILE_FLAG([-Werror=return-type], [ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=return-type"], [], [$CXXFLAG_WERROR], + AX_CHECK_COMPILE_FLAG([-Werror=return-type], [], [ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Wno-error=return-type"], [$CXXFLAG_WERROR], [AC_LANG_SOURCE([[#include <cassert> int f(){ assert(false); }]])]) - - AX_CHECK_COMPILE_FLAG([-Werror=conditional-uninitialized],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=conditional-uninitialized"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=sign-compare],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=sign-compare"],,[[$CXXFLAG_WERROR]]) - dnl -Wsuggest-override is broken with GCC before 9.2 - dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78010 - AX_CHECK_COMPILE_FLAG([-Werror=suggest-override],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=suggest-override"],,[[$CXXFLAG_WERROR]], - [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) - AX_CHECK_COMPILE_FLAG([-Werror=unreachable-code-loop-increment],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=unreachable-code-loop-increment"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Werror=mismatched-tags], [ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=mismatched-tags"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Werror=implicit-fallthrough], [ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=implicit-fallthrough"], [], [$CXXFLAG_WERROR]) - - if test x$suppress_external_warnings != xno ; then - AX_CHECK_COMPILE_FLAG([-Werror=documentation],[ERROR_CXXFLAGS="$ERROR_CXXFLAGS -Werror=documentation"],,[[$CXXFLAG_WERROR]]) - fi fi if test "x$CXXFLAGS_overridden" = "xno"; then @@ -441,19 +419,18 @@ if test "x$CXXFLAGS_overridden" = "xno"; then AX_CHECK_COMPILE_FLAG([-Wformat -Wformat-security],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat -Wformat-security"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wvla],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wvla"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wshadow-field],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wshadow-field"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wswitch],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wswitch"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wthread-safety],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wrange-loop-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-Wloop-analysis],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wloop-analysis"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wredundant-decls],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wunused-variable],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-variable"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wunused-member-function],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-member-function"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wdate-time],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdate-time"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wconditional-uninitialized],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wconditional-uninitialized"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wsign-compare],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsign-compare"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wduplicated-branches],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-branches"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wduplicated-cond],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-cond"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wlogical-op],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wlogical-op"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Woverloaded-virtual],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Woverloaded-virtual"],,[[$CXXFLAG_WERROR]]) + dnl -Wsuggest-override is broken with GCC before 9.2 + dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78010 AX_CHECK_COMPILE_FLAG([-Wsuggest-override],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsuggest-override"],,[[$CXXFLAG_WERROR]], [AC_LANG_SOURCE([[struct A { virtual void f(); }; struct B : A { void f() final; };]])]) AX_CHECK_COMPILE_FLAG([-Wunreachable-code-loop-increment],[WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunreachable-code-loop-increment"],,[[$CXXFLAG_WERROR]]) @@ -468,7 +445,6 @@ if test "x$CXXFLAGS_overridden" = "xno"; then dnl set the -Wno-foo case if it works. AX_CHECK_COMPILE_FLAG([-Wunused-parameter],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-parameter"],,[[$CXXFLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wself-assign],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-self-assign"],,[[$CXXFLAG_WERROR]]) - AX_CHECK_COMPILE_FLAG([-Wunused-local-typedef],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-local-typedef"],,[[$CXXFLAG_WERROR]]) if test x$suppress_external_warnings != xyes ; then AX_CHECK_COMPILE_FLAG([-Wdeprecated-copy],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-deprecated-copy"],,[[$CXXFLAG_WERROR]]) fi @@ -1243,14 +1219,6 @@ if test "x$have_any_system" != "xno"; then AC_DEFINE(HAVE_SYSTEM, 1, Define to 1 if std::system or ::wsystem is available.) fi -LEVELDB_CPPFLAGS= -LIBLEVELDB= -LIBMEMENV= -AM_CONDITIONAL([EMBEDDED_LEVELDB],[true]) -AC_SUBST(LEVELDB_CPPFLAGS) -AC_SUBST(LIBLEVELDB) -AC_SUBST(LIBMEMENV) - dnl SUPPRESSED_CPPFLAGS=SUPPRESS_WARNINGS([$SOME_CPPFLAGS]) dnl Replace -I with -isystem in $SOME_CPPFLAGS to suppress warnings from dnl headers from its include directories and return the result. @@ -1432,6 +1400,36 @@ if test "x$use_external_signer" != xno; then fi AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER], [test "x$use_external_signer" = "xyes"]) +dnl Do not compile with syscall sandbox support when compiling under the sanitizers. +dnl The sanitizers introduce use of syscalls that are not typically used in bitcoind +dnl (such as execve when the sanitizers execute llvm-symbolizer). +if test x$use_sanitizers != x; then + AC_MSG_WARN(Specifying --with-sanitizers forces --without-seccomp since the sanitizers introduce use of syscalls not allowed by the bitcoind syscall sandbox (-sandbox=<mode>).) + seccomp_found=no +fi +if test "x$seccomp_found" != "xno"; then + AC_MSG_CHECKING([for seccomp-bpf (Linux x86-64)]) + AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ + @%:@include <linux/seccomp.h> + ]], [[ + #if !defined(__x86_64__) + # error Syscall sandbox is an experimental feature currently available only under Linux x86-64. + #endif + ]])],[ + AC_MSG_RESULT(yes) + seccomp_found="yes" + AC_DEFINE(USE_SYSCALL_SANDBOX, 1, [Define this symbol to build with syscall sandbox support.]) + ],[ + AC_MSG_RESULT(no) + seccomp_found="no" + ]) +fi +dnl Currently only enable -sandbox=<mode> feature if seccomp is found. +dnl In the future, sandboxing could be also be supported with other +dnl sandboxing mechanisms besides seccomp. +use_syscall_sandbox=$seccomp_found +AM_CONDITIONAL([ENABLE_SYSCALL_SANDBOX], [test "x$use_syscall_sandbox" != "xno"]) + dnl Check for reduced exports if test x$use_reduce_exports = xyes; then AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[CXXFLAGS="$CXXFLAGS -fvisibility=hidden"], @@ -1510,34 +1508,6 @@ if test "x$use_zmq" = xyes; then esac fi -dnl univalue check - -need_bundled_univalue=yes -if test x$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench = xnononononononono; then - need_bundled_univalue=no -else - if test x$system_univalue != xno; then - PKG_CHECK_MODULES([UNIVALUE], [libunivalue >= 1.0.4], [found_univalue=yes], [found_univalue=no]) - if test x$found_univalue = xyes; then - system_univalue=yes - need_bundled_univalue=no - elif test x$system_univalue = xyes; then - AC_MSG_ERROR([univalue not found]) - else - system_univalue=no - fi - fi - - if test x$need_bundled_univalue = xyes; then - UNIVALUE_CFLAGS='-I$(srcdir)/univalue/include' - UNIVALUE_LIBS='univalue/libunivalue.la' - fi -fi - -AM_CONDITIONAL([EMBEDDED_UNIVALUE],[test x$need_bundled_univalue = xyes]) -AC_SUBST(UNIVALUE_CFLAGS) -AC_SUBST(UNIVALUE_LIBS) - dnl libmultiprocess library check libmultiprocess_found=no @@ -1625,8 +1595,8 @@ if test "x$use_ccache" != "xno"; then fi AC_MSG_RESULT($use_ccache) if test "x$use_ccache" = "xyes"; then - AX_CHECK_COMPILE_FLAG([-fdebug-prefix-map=A=B],[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -fdebug-prefix-map=\$(abs_srcdir)=."],,[[$CXXFLAG_WERROR]]) - AX_CHECK_PREPROC_FLAG([-fmacro-prefix-map=A=B],[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -fmacro-prefix-map=\$(abs_srcdir)=."],,[[$CXXFLAG_WERROR]]) + AX_CHECK_COMPILE_FLAG([-fdebug-prefix-map=A=B],[DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -fdebug-prefix-map=\$(abs_top_srcdir)=."],,[[$CXXFLAG_WERROR]]) + AX_CHECK_PREPROC_FLAG([-fmacro-prefix-map=A=B],[DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -fmacro-prefix-map=\$(abs_top_srcdir)=."],,[[$CXXFLAG_WERROR]]) fi fi @@ -1900,11 +1870,7 @@ PKGCONFIG_LIBDIR_TEMP="$PKG_CONFIG_LIBDIR" unset PKG_CONFIG_LIBDIR PKG_CONFIG_LIBDIR="$PKGCONFIG_LIBDIR_TEMP" -if test x$need_bundled_univalue = xyes; then - AC_CONFIG_SUBDIRS([src/univalue]) -fi - -ac_configure_args="${ac_configure_args} --disable-shared --with-pic --enable-benchmark=no --enable-module-recovery --enable-module-schnorrsig --enable-experimental" +ac_configure_args="${ac_configure_args} --disable-shared --with-pic --enable-benchmark=no --enable-module-recovery --enable-module-schnorrsig --enable-experimental --disable-openssl-tests" AC_CONFIG_SUBDIRS([src/secp256k1]) AC_OUTPUT @@ -1921,6 +1887,7 @@ echo echo "Options used to compile and link:" echo " external signer = $use_external_signer" echo " multiprocess = $build_multiprocess" +echo " with experimental syscall sandbox support = $use_syscall_sandbox" echo " with libs = $build_bitcoin_libs" echo " with wallet = $enable_wallet" if test "x$enable_wallet" != "xno"; then diff --git a/contrib/devtools/pixie.py b/contrib/devtools/pixie.py deleted file mode 100644 index 64660968ad..0000000000 --- a/contrib/devtools/pixie.py +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2020 Wladimir J. van der Laan -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' -Compact, self-contained ELF implementation for bitcoin-core security checks. -''' -import struct -import types -from typing import Dict, List, Optional, Union, Tuple - -# you can find all these values in elf.h -EI_NIDENT = 16 - -# Byte indices in e_ident -EI_CLASS = 4 # ELFCLASSxx -EI_DATA = 5 # ELFDATAxxxx - -ELFCLASS32 = 1 # 32-bit -ELFCLASS64 = 2 # 64-bit - -ELFDATA2LSB = 1 # little endian -ELFDATA2MSB = 2 # big endian - -# relevant values for e_machine -EM_386 = 3 -EM_PPC64 = 21 -EM_ARM = 40 -EM_AARCH64 = 183 -EM_X86_64 = 62 -EM_RISCV = 243 - -# relevant values for e_type -ET_DYN = 3 - -# relevant values for sh_type -SHT_PROGBITS = 1 -SHT_STRTAB = 3 -SHT_DYNAMIC = 6 -SHT_DYNSYM = 11 -SHT_GNU_verneed = 0x6ffffffe -SHT_GNU_versym = 0x6fffffff - -# relevant values for p_type -PT_LOAD = 1 -PT_GNU_STACK = 0x6474e551 -PT_GNU_RELRO = 0x6474e552 - -# relevant values for p_flags -PF_X = (1 << 0) -PF_W = (1 << 1) -PF_R = (1 << 2) - -# relevant values for d_tag -DT_NEEDED = 1 -DT_FLAGS = 30 - -# relevant values of `d_un.d_val' in the DT_FLAGS entry -DF_BIND_NOW = 0x00000008 - -# relevant d_tags with string payload -STRING_TAGS = {DT_NEEDED} - -# rrlevant values for ST_BIND subfield of st_info (symbol binding) -STB_LOCAL = 0 - -class ELFRecord(types.SimpleNamespace): - '''Unified parsing for ELF records.''' - def __init__(self, data: bytes, offset: int, eh: 'ELFHeader', total_size: Optional[int]) -> None: - hdr_struct = self.STRUCT[eh.ei_class][0][eh.ei_data] - if total_size is not None and hdr_struct.size > total_size: - raise ValueError(f'{self.__class__.__name__} header size too small ({total_size} < {hdr_struct.size})') - for field, value in zip(self.STRUCT[eh.ei_class][1], hdr_struct.unpack(data[offset:offset + hdr_struct.size])): - setattr(self, field, value) - -def BiStruct(chars: str) -> Dict[int, struct.Struct]: - '''Compile a struct parser for both endians.''' - return { - ELFDATA2LSB: struct.Struct('<' + chars), - ELFDATA2MSB: struct.Struct('>' + chars), - } - -class ELFHeader(ELFRecord): - FIELDS = ['e_type', 'e_machine', 'e_version', 'e_entry', 'e_phoff', 'e_shoff', 'e_flags', 'e_ehsize', 'e_phentsize', 'e_phnum', 'e_shentsize', 'e_shnum', 'e_shstrndx'] - STRUCT = { - ELFCLASS32: (BiStruct('HHIIIIIHHHHHH'), FIELDS), - ELFCLASS64: (BiStruct('HHIQQQIHHHHHH'), FIELDS), - } - - def __init__(self, data: bytes, offset: int) -> None: - self.e_ident = data[offset:offset + EI_NIDENT] - if self.e_ident[0:4] != b'\x7fELF': - raise ValueError('invalid ELF magic') - self.ei_class = self.e_ident[EI_CLASS] - self.ei_data = self.e_ident[EI_DATA] - - super().__init__(data, offset + EI_NIDENT, self, None) - - def __repr__(self) -> str: - return f'Header(e_ident={self.e_ident!r}, e_type={self.e_type}, e_machine={self.e_machine}, e_version={self.e_version}, e_entry={self.e_entry}, e_phoff={self.e_phoff}, e_shoff={self.e_shoff}, e_flags={self.e_flags}, e_ehsize={self.e_ehsize}, e_phentsize={self.e_phentsize}, e_phnum={self.e_phnum}, e_shentsize={self.e_shentsize}, e_shnum={self.e_shnum}, e_shstrndx={self.e_shstrndx})' - -class Section(ELFRecord): - name: Optional[bytes] = None - FIELDS = ['sh_name', 'sh_type', 'sh_flags', 'sh_addr', 'sh_offset', 'sh_size', 'sh_link', 'sh_info', 'sh_addralign', 'sh_entsize'] - STRUCT = { - ELFCLASS32: (BiStruct('IIIIIIIIII'), FIELDS), - ELFCLASS64: (BiStruct('IIQQQQIIQQ'), FIELDS), - } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: - super().__init__(data, offset, eh, eh.e_shentsize) - self._data = data - - def __repr__(self) -> str: - return f'Section(sh_name={self.sh_name}({self.name!r}), sh_type=0x{self.sh_type:x}, sh_flags={self.sh_flags}, sh_addr=0x{self.sh_addr:x}, sh_offset=0x{self.sh_offset:x}, sh_size={self.sh_size}, sh_link={self.sh_link}, sh_info={self.sh_info}, sh_addralign={self.sh_addralign}, sh_entsize={self.sh_entsize})' - - def contents(self) -> bytes: - '''Return section contents.''' - return self._data[self.sh_offset:self.sh_offset + self.sh_size] - -class ProgramHeader(ELFRecord): - STRUCT = { - # different ELF classes have the same fields, but in a different order to optimize space versus alignment - ELFCLASS32: (BiStruct('IIIIIIII'), ['p_type', 'p_offset', 'p_vaddr', 'p_paddr', 'p_filesz', 'p_memsz', 'p_flags', 'p_align']), - ELFCLASS64: (BiStruct('IIQQQQQQ'), ['p_type', 'p_flags', 'p_offset', 'p_vaddr', 'p_paddr', 'p_filesz', 'p_memsz', 'p_align']), - } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: - super().__init__(data, offset, eh, eh.e_phentsize) - - def __repr__(self) -> str: - return f'ProgramHeader(p_type={self.p_type}, p_offset={self.p_offset}, p_vaddr={self.p_vaddr}, p_paddr={self.p_paddr}, p_filesz={self.p_filesz}, p_memsz={self.p_memsz}, p_flags={self.p_flags}, p_align={self.p_align})' - -class Symbol(ELFRecord): - STRUCT = { - # different ELF classes have the same fields, but in a different order to optimize space versus alignment - ELFCLASS32: (BiStruct('IIIBBH'), ['st_name', 'st_value', 'st_size', 'st_info', 'st_other', 'st_shndx']), - ELFCLASS64: (BiStruct('IBBHQQ'), ['st_name', 'st_info', 'st_other', 'st_shndx', 'st_value', 'st_size']), - } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader, symtab: Section, strings: bytes, version: Optional[bytes]) -> None: - super().__init__(data, offset, eh, symtab.sh_entsize) - self.name = _lookup_string(strings, self.st_name) - self.version = version - - def __repr__(self) -> str: - return f'Symbol(st_name={self.st_name}({self.name!r}), st_value={self.st_value}, st_size={self.st_size}, st_info={self.st_info}, st_other={self.st_other}, st_shndx={self.st_shndx}, version={self.version!r})' - - @property - def is_import(self) -> bool: - '''Returns whether the symbol is an imported symbol.''' - return self.st_bind != STB_LOCAL and self.st_shndx == 0 - - @property - def is_export(self) -> bool: - '''Returns whether the symbol is an exported symbol.''' - return self.st_bind != STB_LOCAL and self.st_shndx != 0 - - @property - def st_bind(self) -> int: - '''Returns STB_*.''' - return self.st_info >> 4 - -class Verneed(ELFRecord): - DEF = (BiStruct('HHIII'), ['vn_version', 'vn_cnt', 'vn_file', 'vn_aux', 'vn_next']) - STRUCT = { ELFCLASS32: DEF, ELFCLASS64: DEF } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader) -> None: - super().__init__(data, offset, eh, None) - - def __repr__(self) -> str: - return f'Verneed(vn_version={self.vn_version}, vn_cnt={self.vn_cnt}, vn_file={self.vn_file}, vn_aux={self.vn_aux}, vn_next={self.vn_next})' - -class Vernaux(ELFRecord): - DEF = (BiStruct('IHHII'), ['vna_hash', 'vna_flags', 'vna_other', 'vna_name', 'vna_next']) - STRUCT = { ELFCLASS32: DEF, ELFCLASS64: DEF } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader, strings: bytes) -> None: - super().__init__(data, offset, eh, None) - self.name = _lookup_string(strings, self.vna_name) - - def __repr__(self) -> str: - return f'Veraux(vna_hash={self.vna_hash}, vna_flags={self.vna_flags}, vna_other={self.vna_other}, vna_name={self.vna_name}({self.name!r}), vna_next={self.vna_next})' - -class DynTag(ELFRecord): - STRUCT = { - ELFCLASS32: (BiStruct('II'), ['d_tag', 'd_val']), - ELFCLASS64: (BiStruct('QQ'), ['d_tag', 'd_val']), - } - - def __init__(self, data: bytes, offset: int, eh: ELFHeader, section: Section) -> None: - super().__init__(data, offset, eh, section.sh_entsize) - - def __repr__(self) -> str: - return f'DynTag(d_tag={self.d_tag}, d_val={self.d_val})' - -def _lookup_string(data: bytes, index: int) -> bytes: - '''Look up string by offset in ELF string table.''' - endx = data.find(b'\x00', index) - assert endx != -1 - return data[index:endx] - -VERSYM_S = BiStruct('H') # .gnu_version section has a single 16-bit integer per symbol in the linked section -def _parse_symbol_table(section: Section, strings: bytes, eh: ELFHeader, versym: bytes, verneed: Dict[int, bytes]) -> List[Symbol]: - '''Parse symbol table, return a list of symbols.''' - data = section.contents() - symbols = [] - versym_iter = (verneed.get(v[0]) for v in VERSYM_S[eh.ei_data].iter_unpack(versym)) - for ofs, version in zip(range(0, len(data), section.sh_entsize), versym_iter): - symbols.append(Symbol(data, ofs, eh, section, strings, version)) - return symbols - -def _parse_verneed(section: Section, strings: bytes, eh: ELFHeader) -> Dict[int, bytes]: - '''Parse .gnu.version_r section, return a dictionary of {versym: 'GLIBC_...'}.''' - data = section.contents() - ofs = 0 - result = {} - while True: - verneed = Verneed(data, ofs, eh) - aofs = ofs + verneed.vn_aux - while True: - vernaux = Vernaux(data, aofs, eh, strings) - result[vernaux.vna_other] = vernaux.name - if not vernaux.vna_next: - break - aofs += vernaux.vna_next - - if not verneed.vn_next: - break - ofs += verneed.vn_next - - return result - -def _parse_dyn_tags(section: Section, strings: bytes, eh: ELFHeader) -> List[Tuple[int, Union[bytes, int]]]: - '''Parse dynamic tags. Return array of tuples.''' - data = section.contents() - ofs = 0 - result = [] - for ofs in range(0, len(data), section.sh_entsize): - tag = DynTag(data, ofs, eh, section) - val = _lookup_string(strings, tag.d_val) if tag.d_tag in STRING_TAGS else tag.d_val - result.append((tag.d_tag, val)) - - return result - -class ELFFile: - sections: List[Section] - program_headers: List[ProgramHeader] - dyn_symbols: List[Symbol] - dyn_tags: List[Tuple[int, Union[bytes, int]]] - - def __init__(self, data: bytes) -> None: - self.data = data - self.hdr = ELFHeader(self.data, 0) - self._load_sections() - self._load_program_headers() - self._load_dyn_symbols() - self._load_dyn_tags() - self._section_to_segment_mapping() - - def _load_sections(self) -> None: - self.sections = [] - for idx in range(self.hdr.e_shnum): - offset = self.hdr.e_shoff + idx * self.hdr.e_shentsize - self.sections.append(Section(self.data, offset, self.hdr)) - - shstr = self.sections[self.hdr.e_shstrndx].contents() - for section in self.sections: - section.name = _lookup_string(shstr, section.sh_name) - - def _load_program_headers(self) -> None: - self.program_headers = [] - for idx in range(self.hdr.e_phnum): - offset = self.hdr.e_phoff + idx * self.hdr.e_phentsize - self.program_headers.append(ProgramHeader(self.data, offset, self.hdr)) - - def _load_dyn_symbols(self) -> None: - # first, load 'verneed' section - verneed = None - for section in self.sections: - if section.sh_type == SHT_GNU_verneed: - strtab = self.sections[section.sh_link].contents() # associated string table - assert verneed is None # only one section of this kind please - verneed = _parse_verneed(section, strtab, self.hdr) - assert verneed is not None - - # then, correlate GNU versym sections with dynamic symbol sections - versym = {} - for section in self.sections: - if section.sh_type == SHT_GNU_versym: - versym[section.sh_link] = section - - # finally, load dynsym sections - self.dyn_symbols = [] - for idx, section in enumerate(self.sections): - if section.sh_type == SHT_DYNSYM: # find dynamic symbol tables - strtab_data = self.sections[section.sh_link].contents() # associated string table - versym_data = versym[idx].contents() # associated symbol version table - self.dyn_symbols += _parse_symbol_table(section, strtab_data, self.hdr, versym_data, verneed) - - def _load_dyn_tags(self) -> None: - self.dyn_tags = [] - for idx, section in enumerate(self.sections): - if section.sh_type == SHT_DYNAMIC: # find dynamic tag tables - strtab = self.sections[section.sh_link].contents() # associated string table - self.dyn_tags += _parse_dyn_tags(section, strtab, self.hdr) - - def _section_to_segment_mapping(self) -> None: - for ph in self.program_headers: - ph.sections = [] - for section in self.sections: - if ph.p_vaddr <= section.sh_addr < (ph.p_vaddr + ph.p_memsz): - ph.sections.append(section) - - def query_dyn_tags(self, tag_in: int) -> List[Union[int, bytes]]: - '''Return the values of all dyn tags with the specified tag.''' - return [val for (tag, val) in self.dyn_tags if tag == tag_in] - - -def load(filename: str) -> ELFFile: - with open(filename, 'rb') as f: - data = f.read() - return ELFFile(data) diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index 0b59d8eada..ef421aebb1 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -8,192 +8,155 @@ Exit status will be 0 if successful, and the program will be silent. Otherwise the exit status will be 1 and it will log which executables failed which checks. ''' import sys -from typing import List, Optional +from typing import List -import lief -import pixie +import lief #type:ignore -def check_ELF_PIE(executable) -> bool: - ''' - Check for position independent executable (PIE), allowing for address space randomization. - ''' - elf = pixie.load(executable) - return elf.hdr.e_type == pixie.ET_DYN - -def check_ELF_NX(executable) -> bool: - ''' - Check that no sections are writable and executable (including the stack) - ''' - elf = pixie.load(executable) - have_wx = False - have_gnu_stack = False - for ph in elf.program_headers: - if ph.p_type == pixie.PT_GNU_STACK: - have_gnu_stack = True - if (ph.p_flags & pixie.PF_W) != 0 and (ph.p_flags & pixie.PF_X) != 0: # section is both writable and executable - have_wx = True - return have_gnu_stack and not have_wx - -def check_ELF_RELRO(executable) -> bool: +def check_ELF_RELRO(binary) -> bool: ''' Check for read-only relocations. GNU_RELRO program header must exist Dynamic section must have BIND_NOW flag ''' - elf = pixie.load(executable) have_gnu_relro = False - for ph in elf.program_headers: + for segment in binary.segments: # Note: not checking p_flags == PF_R: here as linkers set the permission differently # This does not affect security: the permission flags of the GNU_RELRO program # header are ignored, the PT_LOAD header determines the effective permissions. # However, the dynamic linker need to write to this area so these are RW. # Glibc itself takes care of mprotecting this area R after relocations are finished. # See also https://marc.info/?l=binutils&m=1498883354122353 - if ph.p_type == pixie.PT_GNU_RELRO: + if segment.type == lief.ELF.SEGMENT_TYPES.GNU_RELRO: have_gnu_relro = True have_bindnow = False - for flags in elf.query_dyn_tags(pixie.DT_FLAGS): - assert isinstance(flags, int) - if flags & pixie.DF_BIND_NOW: + try: + flags = binary.get(lief.ELF.DYNAMIC_TAGS.FLAGS) + if flags.value & lief.ELF.DYNAMIC_FLAGS.BIND_NOW: have_bindnow = True + except: + have_bindnow = False return have_gnu_relro and have_bindnow -def check_ELF_Canary(executable) -> bool: +def check_ELF_Canary(binary) -> bool: ''' Check for use of stack canary ''' - elf = pixie.load(executable) - ok = False - for symbol in elf.dyn_symbols: - if symbol.name == b'__stack_chk_fail': - ok = True - return ok + return binary.has_symbol('__stack_chk_fail') -def check_ELF_separate_code(executable): +def check_ELF_separate_code(binary): ''' Check that sections are appropriately separated in virtual memory, based on their permissions. This checks for missing -Wl,-z,separate-code and potentially other problems. ''' - elf = pixie.load(executable) - R = pixie.PF_R - W = pixie.PF_W - E = pixie.PF_X + R = lief.ELF.SEGMENT_FLAGS.R + W = lief.ELF.SEGMENT_FLAGS.W + E = lief.ELF.SEGMENT_FLAGS.X EXPECTED_FLAGS = { # Read + execute - b'.init': R | E, - b'.plt': R | E, - b'.plt.got': R | E, - b'.plt.sec': R | E, - b'.text': R | E, - b'.fini': R | E, + '.init': R | E, + '.plt': R | E, + '.plt.got': R | E, + '.plt.sec': R | E, + '.text': R | E, + '.fini': R | E, # Read-only data - b'.interp': R, - b'.note.gnu.property': R, - b'.note.gnu.build-id': R, - b'.note.ABI-tag': R, - b'.gnu.hash': R, - b'.dynsym': R, - b'.dynstr': R, - b'.gnu.version': R, - b'.gnu.version_r': R, - b'.rela.dyn': R, - b'.rela.plt': R, - b'.rodata': R, - b'.eh_frame_hdr': R, - b'.eh_frame': R, - b'.qtmetadata': R, - b'.gcc_except_table': R, - b'.stapsdt.base': R, + '.interp': R, + '.note.gnu.property': R, + '.note.gnu.build-id': R, + '.note.ABI-tag': R, + '.gnu.hash': R, + '.dynsym': R, + '.dynstr': R, + '.gnu.version': R, + '.gnu.version_r': R, + '.rela.dyn': R, + '.rela.plt': R, + '.rodata': R, + '.eh_frame_hdr': R, + '.eh_frame': R, + '.qtmetadata': R, + '.gcc_except_table': R, + '.stapsdt.base': R, # Writable data - b'.init_array': R | W, - b'.fini_array': R | W, - b'.dynamic': R | W, - b'.got': R | W, - b'.data': R | W, - b'.bss': R | W, + '.init_array': R | W, + '.fini_array': R | W, + '.dynamic': R | W, + '.got': R | W, + '.data': R | W, + '.bss': R | W, } - if elf.hdr.e_machine == pixie.EM_PPC64: + if binary.header.machine_type == lief.ELF.ARCH.PPC64: # .plt is RW on ppc64 even with separate-code - EXPECTED_FLAGS[b'.plt'] = R | W + EXPECTED_FLAGS['.plt'] = R | W # For all LOAD program headers get mapping to the list of sections, # and for each section, remember the flags of the associated program header. flags_per_section = {} - for ph in elf.program_headers: - if ph.p_type == pixie.PT_LOAD: - for section in ph.sections: + for segment in binary.segments: + if segment.type == lief.ELF.SEGMENT_TYPES.LOAD: + for section in segment.sections: assert(section.name not in flags_per_section) - flags_per_section[section.name] = ph.p_flags + flags_per_section[section.name] = segment.flags # Spot-check ELF LOAD program header flags per section # If these sections exist, check them against the expected R/W/E flags for (section, flags) in flags_per_section.items(): if section in EXPECTED_FLAGS: - if EXPECTED_FLAGS[section] != flags: + if int(EXPECTED_FLAGS[section]) != int(flags): return False return True -def check_PE_DYNAMIC_BASE(executable) -> bool: +def check_PE_DYNAMIC_BASE(binary) -> bool: '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' - binary = lief.parse(executable) return lief.PE.DLL_CHARACTERISTICS.DYNAMIC_BASE in binary.optional_header.dll_characteristics_lists # Must support high-entropy 64-bit address space layout randomization # in addition to DYNAMIC_BASE to have secure ASLR. -def check_PE_HIGH_ENTROPY_VA(executable) -> bool: +def check_PE_HIGH_ENTROPY_VA(binary) -> bool: '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' - binary = lief.parse(executable) return lief.PE.DLL_CHARACTERISTICS.HIGH_ENTROPY_VA in binary.optional_header.dll_characteristics_lists -def check_PE_RELOC_SECTION(executable) -> bool: +def check_PE_RELOC_SECTION(binary) -> bool: '''Check for a reloc section. This is required for functional ASLR.''' - binary = lief.parse(executable) return binary.has_relocations -def check_MACHO_NOUNDEFS(executable) -> bool: +def check_MACHO_NOUNDEFS(binary) -> bool: ''' Check for no undefined references. ''' - binary = lief.parse(executable) return binary.header.has(lief.MachO.HEADER_FLAGS.NOUNDEFS) -def check_MACHO_LAZY_BINDINGS(executable) -> bool: +def check_MACHO_LAZY_BINDINGS(binary) -> bool: ''' Check for no lazy bindings. We don't use or check for MH_BINDATLOAD. See #18295. ''' - binary = lief.parse(executable) return binary.dyld_info.lazy_bind == (0,0) -def check_MACHO_Canary(executable) -> bool: +def check_MACHO_Canary(binary) -> bool: ''' Check for use of stack canary ''' - binary = lief.parse(executable) return binary.has_symbol('___stack_chk_fail') -def check_PIE(executable) -> bool: +def check_PIE(binary) -> bool: ''' Check for position independent executable (PIE), allowing for address space randomization. ''' - binary = lief.parse(executable) return binary.is_pie -def check_NX(executable) -> bool: +def check_NX(binary) -> bool: ''' Check for no stack execution ''' - binary = lief.parse(executable) return binary.has_nx -def check_control_flow(executable) -> bool: +def check_control_flow(binary) -> bool: ''' Check for control flow instrumentation ''' - binary = lief.parse(executable) - content = binary.get_content_from_virtual_address(binary.entrypoint, 4, lief.Binary.VA_TYPES.AUTO) if content == [243, 15, 30, 250]: # endbr64 @@ -203,8 +166,8 @@ def check_control_flow(executable) -> bool: CHECKS = { 'ELF': [ - ('PIE', check_ELF_PIE), - ('NX', check_ELF_NX), + ('PIE', check_PIE), + ('NX', check_NX), ('RELRO', check_ELF_RELRO), ('Canary', check_ELF_Canary), ('separate_code', check_ELF_separate_code), @@ -226,30 +189,20 @@ CHECKS = { ] } -def identify_executable(executable) -> Optional[str]: - with open(filename, 'rb') as f: - magic = f.read(4) - if magic.startswith(b'MZ'): - return 'PE' - elif magic.startswith(b'\x7fELF'): - return 'ELF' - elif magic.startswith(b'\xcf\xfa'): - return 'MACHO' - return None - if __name__ == '__main__': retval: int = 0 for filename in sys.argv[1:]: try: - etype = identify_executable(filename) - if etype is None: - print(f'{filename}: unknown format') + binary = lief.parse(filename) + etype = binary.format.name + if etype == lief.EXE_FORMATS.UNKNOWN: + print(f'{filename}: unknown executable format') retval = 1 continue failed: List[str] = [] for (name, func) in CHECKS[etype]: - if not func(filename): + if not func(binary): failed.append(name) if failed: print(f'{filename}: failed {" ".join(failed)}') diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 98cab1b7fc..136a9b70c1 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -10,14 +10,14 @@ Example usage: find ../path/to/binaries -type f -executable | xargs python3 contrib/devtools/symbol-check.py ''' -import subprocess import sys -from typing import List, Optional +from typing import List, Dict -import lief -import pixie +import lief #type:ignore -from utils import determine_wellknown_cmd +# temporary constant, to be replaced with lief.ELF.ARCH.RISCV +# https://github.com/lief-project/LIEF/pull/562 +LIEF_ELF_ARCH_RISCV = lief.ELF.ARCH(243) # Debian 8 (Jessie) EOL: 2020. https://wiki.debian.org/DebianReleases#Production_Releases # @@ -43,12 +43,12 @@ from utils import determine_wellknown_cmd MAX_VERSIONS = { 'GCC': (4,8,0), 'GLIBC': { - pixie.EM_386: (2,17), - pixie.EM_X86_64: (2,17), - pixie.EM_ARM: (2,17), - pixie.EM_AARCH64:(2,17), - pixie.EM_PPC64: (2,17), - pixie.EM_RISCV: (2,27), + lief.ELF.ARCH.i386: (2,17), + lief.ELF.ARCH.x86_64: (2,17), + lief.ELF.ARCH.ARM: (2,17), + lief.ELF.ARCH.AARCH64:(2,17), + lief.ELF.ARCH.PPC64: (2,17), + LIEF_ELF_ARCH_RISCV: (2,27), }, 'LIBATOMIC': (1,0), 'V': (0,5,0), # xkb (bitcoin-qt only) @@ -58,10 +58,35 @@ MAX_VERSIONS = { # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { -'_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', '__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr', +'_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', +'__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr', 'environ', '_environ', '__environ', } +# Expected linker-loader names can be found here: +# https://sourceware.org/glibc/wiki/ABIList?action=recall&rev=16 +ELF_INTERPRETER_NAMES: Dict[lief.ELF.ARCH, Dict[lief.ENDIANNESS, str]] = { + lief.ELF.ARCH.i386: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux.so.2", + }, + lief.ELF.ARCH.x86_64: { + lief.ENDIANNESS.LITTLE: "/lib64/ld-linux-x86-64.so.2", + }, + lief.ELF.ARCH.ARM: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux-armhf.so.3", + }, + lief.ELF.ARCH.AARCH64: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux-aarch64.so.1", + }, + lief.ELF.ARCH.PPC64: { + lief.ENDIANNESS.BIG: "/lib64/ld64.so.1", + lief.ENDIANNESS.LITTLE: "/lib64/ld64.so.2", + }, + LIEF_ELF_ARCH_RISCV: { + lief.ENDIANNESS.LITTLE: "/lib/ld-linux-riscv64-lp64d.so.1", + }, +} + # Allowed NEEDED libraries ELF_ALLOWED_LIBRARIES = { # bitcoind and bitcoin-qt @@ -133,31 +158,8 @@ PE_ALLOWED_LIBRARIES = { 'WTSAPI32.dll', } -class CPPFilt(object): - ''' - Demangle C++ symbol names. - - Use a pipe to the 'c++filt' command. - ''' - def __init__(self): - self.proc = subprocess.Popen(determine_wellknown_cmd('CPPFILT', 'c++filt'), stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) - - def __call__(self, mangled): - self.proc.stdin.write(mangled + '\n') - self.proc.stdin.flush() - return self.proc.stdout.readline().rstrip() - - def close(self): - self.proc.stdin.close() - self.proc.stdout.close() - self.proc.wait() - def check_version(max_versions, version, arch) -> bool: - if '_' in version: - (lib, _, ver) = version.rpartition('_') - else: - lib = version - ver = '0' + (lib, _, ver) = version.rpartition('_') ver = tuple([int(x) for x in ver.split('.')]) if not lib in max_versions: return False @@ -166,48 +168,45 @@ def check_version(max_versions, version, arch) -> bool: else: return ver <= max_versions[lib][arch] -def check_imported_symbols(filename) -> bool: - elf = pixie.load(filename) - cppfilt = CPPFilt() +def check_imported_symbols(binary) -> bool: ok: bool = True - for symbol in elf.dyn_symbols: - if not symbol.is_import: + for symbol in binary.imported_symbols: + if not symbol.imported: continue - sym = symbol.name.decode() - version = symbol.version.decode() if symbol.version is not None else None - if version and not check_version(MAX_VERSIONS, version, elf.hdr.e_machine): - print('{}: symbol {} from unsupported version {}'.format(filename, cppfilt(sym), version)) - ok = False + + version = symbol.symbol_version if symbol.has_version else None + + if version: + aux_version = version.symbol_version_auxiliary.name if version.has_auxiliary_version else None + if aux_version and not check_version(MAX_VERSIONS, aux_version, binary.header.machine_type): + print(f'{filename}: symbol {symbol.name} from unsupported version {version}') + ok = False return ok -def check_exported_symbols(filename) -> bool: - elf = pixie.load(filename) - cppfilt = CPPFilt() +def check_exported_symbols(binary) -> bool: ok: bool = True - for symbol in elf.dyn_symbols: - if not symbol.is_export: + + for symbol in binary.dynamic_symbols: + if not symbol.exported: continue - sym = symbol.name.decode() - if elf.hdr.e_machine == pixie.EM_RISCV or sym in IGNORE_EXPORTS: + name = symbol.name + if binary.header.machine_type == LIEF_ELF_ARCH_RISCV or name in IGNORE_EXPORTS: continue - print('{}: export of symbol {} not allowed'.format(filename, cppfilt(sym))) + print(f'{binary.name}: export of symbol {name} not allowed!') ok = False return ok -def check_ELF_libraries(filename) -> bool: +def check_ELF_libraries(binary) -> bool: ok: bool = True - elf = pixie.load(filename) - for library_name in elf.query_dyn_tags(pixie.DT_NEEDED): - assert(isinstance(library_name, bytes)) - if library_name.decode() not in ELF_ALLOWED_LIBRARIES: - print('{}: NEEDED library {} is not allowed'.format(filename, library_name.decode())) + for library in binary.libraries: + if library not in ELF_ALLOWED_LIBRARIES: + print(f'{filename}: {library} is not in ALLOWED_LIBRARIES!') ok = False return ok -def check_MACHO_libraries(filename) -> bool: +def check_MACHO_libraries(binary) -> bool: ok: bool = True - binary = lief.parse(filename) for dylib in binary.libraries: split = dylib.name.split('/') if split[-1] not in MACHO_ALLOWED_LIBRARIES: @@ -215,40 +214,42 @@ def check_MACHO_libraries(filename) -> bool: ok = False return ok -def check_MACHO_min_os(filename) -> bool: - binary = lief.parse(filename) +def check_MACHO_min_os(binary) -> bool: if binary.build_version.minos == [10,15,0]: return True return False -def check_MACHO_sdk(filename) -> bool: - binary = lief.parse(filename) +def check_MACHO_sdk(binary) -> bool: if binary.build_version.sdk == [10, 15, 6]: return True return False -def check_PE_libraries(filename) -> bool: +def check_PE_libraries(binary) -> bool: ok: bool = True - binary = lief.parse(filename) for dylib in binary.libraries: if dylib not in PE_ALLOWED_LIBRARIES: print(f'{dylib} is not in ALLOWED_LIBRARIES!') ok = False return ok -def check_PE_subsystem_version(filename) -> bool: - binary = lief.parse(filename) +def check_PE_subsystem_version(binary) -> bool: major: int = binary.optional_header.major_subsystem_version minor: int = binary.optional_header.minor_subsystem_version if major == 6 and minor == 1: return True return False +def check_ELF_interpreter(binary) -> bool: + expected_interpreter = ELF_INTERPRETER_NAMES[binary.header.machine_type][binary.abstract.header.endianness] + + return binary.concrete.interpreter == expected_interpreter + CHECKS = { 'ELF': [ ('IMPORTED_SYMBOLS', check_imported_symbols), ('EXPORTED_SYMBOLS', check_exported_symbols), - ('LIBRARY_DEPENDENCIES', check_ELF_libraries) + ('LIBRARY_DEPENDENCIES', check_ELF_libraries), + ('INTERPRETER_NAME', check_ELF_interpreter), ], 'MACHO': [ ('DYNAMIC_LIBRARIES', check_MACHO_libraries), @@ -261,30 +262,20 @@ CHECKS = { ] } -def identify_executable(executable) -> Optional[str]: - with open(filename, 'rb') as f: - magic = f.read(4) - if magic.startswith(b'MZ'): - return 'PE' - elif magic.startswith(b'\x7fELF'): - return 'ELF' - elif magic.startswith(b'\xcf\xfa'): - return 'MACHO' - return None - if __name__ == '__main__': retval: int = 0 for filename in sys.argv[1:]: try: - etype = identify_executable(filename) - if etype is None: - print(f'{filename}: unknown format') + binary = lief.parse(filename) + etype = binary.format.name + if etype == lief.EXE_FORMATS.UNKNOWN: + print(f'{filename}: unknown executable format') retval = 1 continue failed: List[str] = [] for (name, func) in CHECKS[etype]: - if not func(filename): + if not func(binary): failed.append(name) if failed: print(f'{filename}: failed {" ".join(failed)}') diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 14058e2cc8..0af7cdf5e6 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -7,6 +7,7 @@ Test script for security-check.py ''' import os import subprocess +from typing import List import unittest from utils import determine_wellknown_cmd @@ -27,7 +28,16 @@ def clean_files(source, executable): os.remove(executable) def call_security_check(cc, source, executable, options): - subprocess.run([*cc,source,'-o',executable] + options, check=True) + # This should behave the same as AC_TRY_LINK, so arrange well-known flags + # in the same order as autoconf would. + # + # See the definitions for ac_link in autoconf's lib/autoconf/c.m4 file for + # reference. + env_flags: List[str] = [] + for var in ['CFLAGS', 'CPPFLAGS', 'LDFLAGS']: + env_flags += filter(None, os.environ.get(var, '').split(' ')) + + subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True) p = subprocess.run(['./contrib/devtools/security-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) return (p.returncode, p.stdout.rstrip()) diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py index 651589c11b..5246375fe3 100755 --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -13,7 +13,16 @@ import unittest from utils import determine_wellknown_cmd def call_symbol_check(cc: List[str], source, executable, options): - subprocess.run([*cc,source,'-o',executable] + options, check=True) + # This should behave the same as AC_TRY_LINK, so arrange well-known flags + # in the same order as autoconf would. + # + # See the definitions for ac_link in autoconf's lib/autoconf/c.m4 file for + # reference. + env_flags: List[str] = [] + for var in ['CFLAGS', 'CPPFLAGS', 'LDFLAGS']: + env_flags += filter(None, os.environ.get(var, '').split(' ')) + + subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True) p = subprocess.run(['./contrib/devtools/symbol-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) os.remove(source) os.remove(executable) @@ -51,7 +60,7 @@ class TestSymbolChecks(unittest.TestCase): ''') self.assertEqual(call_symbol_check(cc, source, executable, ['-lm']), - (1, executable + ': symbol nextup from unsupported version GLIBC_2.24\n' + + (1, executable + ': symbol nextup from unsupported version GLIBC_2.24(3)\n' + executable + ': failed IMPORTED_SYMBOLS')) # -lutil is part of the libc6 package so a safe bet that it's installed @@ -70,7 +79,7 @@ class TestSymbolChecks(unittest.TestCase): ''') self.assertEqual(call_symbol_check(cc, source, executable, ['-lutil']), - (1, executable + ': NEEDED library libutil.so.1 is not allowed\n' + + (1, executable + ': libutil.so.1 is not in ALLOWED_LIBRARIES!\n' + executable + ': failed LIBRARY_DEPENDENCIES')) # finally, check a simple conforming binary diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 93526f8c45..e009f97c60 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -169,8 +169,8 @@ case "$HOST" in arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; riscv64-linux-gnu) echo /lib/ld-linux-riscv64-lp64d.so.1 ;; - powerpc64-linux-gnu) echo /lib/ld64.so.1;; - powerpc64le-linux-gnu) echo /lib/ld64.so.2;; + powerpc64-linux-gnu) echo /lib64/ld64.so.1;; + powerpc64le-linux-gnu) echo /lib64/ld64.so.2;; *) exit 1 ;; esac ) @@ -297,7 +297,7 @@ mkdir -p "$DISTSRC" ${HOST_CXXFLAGS:+CXXFLAGS="${HOST_CXXFLAGS}"} \ ${HOST_LDFLAGS:+LDFLAGS="${HOST_LDFLAGS}"} - sed -i.old 's/-lstdc++ //g' config.status libtool src/univalue/config.status src/univalue/libtool + sed -i.old 's/-lstdc++ //g' config.status libtool # Build Bitcoin Core make --jobs="$JOBS" ${V:+V=1} diff --git a/contrib/signet/getcoins.py b/contrib/signet/getcoins.py index dc203f1254..3d0aa5d132 100755 --- a/contrib/signet/getcoins.py +++ b/contrib/signet/getcoins.py @@ -4,18 +4,77 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. import argparse +import io +import requests import subprocess import sys -import requests DEFAULT_GLOBAL_FAUCET = 'https://signetfaucet.com/claim' +DEFAULT_GLOBAL_CAPTCHA = 'https://signetfaucet.com/captcha' GLOBAL_FIRST_BLOCK_HASH = '00000086d6b2636cb2a392d45edc4ec544a10024d30141c9adf4bfd9de533b53' +# braille unicode block +BASE = 0x2800 +BIT_PER_PIXEL = [ + [0x01, 0x08], + [0x02, 0x10], + [0x04, 0x20], + [0x40, 0x80], +] +BW = 2 +BH = 4 + +# imagemagick or compatible fork (used for converting SVG) +CONVERT = 'convert' + +class PPMImage: + ''' + Load a PPM image (Pillow-ish API). + ''' + def __init__(self, f): + if f.readline() != b'P6\n': + raise ValueError('Invalid ppm format: header') + line = f.readline() + (width, height) = (int(x) for x in line.rstrip().split(b' ')) + if f.readline() != b'255\n': + raise ValueError('Invalid ppm format: color depth') + data = f.read(width * height * 3) + stride = width * 3 + self.size = (width, height) + self._grid = [[tuple(data[stride * y + 3 * x:stride * y + 3 * (x + 1)]) for x in range(width)] for y in range(height)] + + def getpixel(self, pos): + return self._grid[pos[1]][pos[0]] + +def print_image(img, threshold=128): + '''Print black-and-white image to terminal in braille unicode characters.''' + x_blocks = (img.size[0] + BW - 1) // BW + y_blocks = (img.size[1] + BH - 1) // BH + + for yb in range(y_blocks): + line = [] + for xb in range(x_blocks): + ch = BASE + for y in range(BH): + for x in range(BW): + try: + val = img.getpixel((xb * BW + x, yb * BH + y)) + except IndexError: + pass + else: + if val[0] < threshold: + ch |= BIT_PER_PIXEL[y][x] + line.append(chr(ch)) + print(''.join(line)) + parser = argparse.ArgumentParser(description='Script to get coins from a faucet.', epilog='You may need to start with double-dash (--) when providing bitcoin-cli arguments.') parser.add_argument('-c', '--cmd', dest='cmd', default='bitcoin-cli', help='bitcoin-cli command to use') parser.add_argument('-f', '--faucet', dest='faucet', default=DEFAULT_GLOBAL_FAUCET, help='URL of the faucet') +parser.add_argument('-g', '--captcha', dest='captcha', default=DEFAULT_GLOBAL_CAPTCHA, help='URL of the faucet captcha, or empty if no captcha is needed') parser.add_argument('-a', '--addr', dest='addr', default='', help='Bitcoin address to which the faucet should send') parser.add_argument('-p', '--password', dest='password', default='', help='Faucet password, if any') +parser.add_argument('-n', '--amount', dest='amount', default='0.001', help='Amount to request (0.001-0.1, default is 0.001)') +parser.add_argument('-i', '--imagemagick', dest='imagemagick', default=CONVERT, help='Path to imagemagick convert utility') parser.add_argument('bitcoin_cli_args', nargs='*', help='Arguments to pass on to bitcoin-cli (default: -signet)') args = parser.parse_args() @@ -43,14 +102,43 @@ if args.faucet.lower() == DEFAULT_GLOBAL_FAUCET: if curr_signet_hash != GLOBAL_FIRST_BLOCK_HASH: print('The global faucet cannot be used with a custom Signet network. Please use the global signet or setup your custom faucet to use this functionality.\n') exit(1) +else: + # For custom faucets, don't request captcha by default. + if args.captcha == DEFAULT_GLOBAL_CAPTCHA: + args.captcha = '' if args.addr == '': # get address for receiving coins args.addr = bitcoin_cli(['getnewaddress', 'faucet', 'bech32']) -data = {'address': args.addr, 'password': args.password} +data = {'address': args.addr, 'password': args.password, 'amount': args.amount} + +# Store cookies +# for debugging: print(session.cookies.get_dict()) +session = requests.Session() + +if args.captcha != '': # Retrieve a captcha + try: + res = session.get(args.captcha) + except: + print('Unexpected error when contacting faucet:', sys.exc_info()[0]) + exit(1) + + # Convert SVG image to PPM, and load it + try: + rv = subprocess.run([args.imagemagick, '-', '-depth', '8', 'ppm:-'], input=res.content, check=True, capture_output=True) + except FileNotFoundError: + print('The binary', args.imagemagick, 'could not be found. Please make sure ImageMagick (or a compatible fork) is installed and that the correct path is specified.') + exit(1) + img = PPMImage(io.BytesIO(rv.stdout)) + + # Terminal interaction + print_image(img) + print('Enter captcha: ', end='') + data['captcha'] = input() + try: - res = requests.post(args.faucet, data=data) + res = session.post(args.faucet, data=data) except: print('Unexpected error when contacting faucet:', sys.exc_info()[0]) exit(1) diff --git a/contrib/tracing/README.md b/contrib/tracing/README.md index 047354cda1..1f93474fa0 100644 --- a/contrib/tracing/README.md +++ b/contrib/tracing/README.md @@ -176,17 +176,12 @@ third acts as a duration threshold in milliseconds. When the `ConnectBlock()` function takes longer than the threshold, information about the block, is printed. For more details, see the header comment in the script. -By default, `bpftrace` limits strings to 64 bytes due to the limited stack size -in the kernel VM. Block hashes as zero-terminated hex strings are 65 bytes which -exceed the string limit. The string size limit can be set to 65 bytes with the -environment variable `BPFTRACE_STRLEN`. - The following command can be used to benchmark, for example, `ConnectBlock()` between height 20000 and 38000 on SigNet while logging all blocks that take longer than 25ms to connect. ``` -$ BPFTRACE_STRLEN=65 bpftrace contrib/tracing/connectblock_benchmark.bt 20000 38000 25 +$ bpftrace contrib/tracing/connectblock_benchmark.bt 20000 38000 25 ``` In a different terminal, starting Bitcoin Core in SigNet mode and with diff --git a/contrib/tracing/connectblock_benchmark.bt b/contrib/tracing/connectblock_benchmark.bt index d268eff7f8..6e7a98ef07 100755 --- a/contrib/tracing/connectblock_benchmark.bt +++ b/contrib/tracing/connectblock_benchmark.bt @@ -4,11 +4,8 @@ USAGE: - BPFTRACE_STRLEN=65 bpftrace contrib/tracing/connectblock_benchmark.bt <start height> <end height> <logging threshold in ms> + bpftrace contrib/tracing/connectblock_benchmark.bt <start height> <end height> <logging threshold in ms> - - The environment variable BPFTRACE_STRLEN needs to be set to 65 chars as - strings are limited to 64 chars by default. Hex strings with Bitcoin block - hashes are 64 hex chars + 1 null-termination char. - <start height> sets the height at which the benchmark should start. Setting the start height to 0 starts the benchmark immediately, even before the first block is connected. @@ -23,7 +20,7 @@ EXAMPLES: - BPFTRACE_STRLEN=65 bpftrace contrib/tracing/connectblock_benchmark.bt 300000 680000 1000 + bpftrace contrib/tracing/connectblock_benchmark.bt 300000 680000 1000 When run together 'bitcoind -reindex', this benchmarks the time it takes to connect the blocks between height 300.000 and 680.000 (inclusive) and prints @@ -31,7 +28,7 @@ histogram with block connection times when the benchmark is finished. - BPFTRACE_STRLEN=65 bpftrace contrib/tracing/connectblock_benchmark.bt 0 0 500 + bpftrace contrib/tracing/connectblock_benchmark.bt 0 0 500 When running together 'bitcoind', all newly connected blocks that take longer than 500ms to connect are logged. A histogram with block @@ -107,14 +104,23 @@ usdt:./src/bitcoind:validation:block_connected /arg1 >= $1 && (arg1 <= $2 || $2 */ usdt:./src/bitcoind:validation:block_connected / (uint64) arg5 / 1000> $3 / { - $hash_str = str(arg0); + $hash = arg0; $height = (int32) arg1; $transactions = (uint64) arg2; $inputs = (int32) arg3; $sigops = (int64) arg4; $duration = (int64) arg5; - printf("Block %d (%s) %4d tx %5d ins %5d sigops took %4d ms\n", $height, $hash_str, $transactions, $inputs, $sigops, (uint64) $duration / 1000); + + printf("Block %d (", $height); + /* Prints each byte of the block hash as hex in big-endian (the block-explorer format) */ + $p = $hash + 31; + unroll(32) { + $b = *(uint8*)$p; + printf("%02x", $b); + $p -= 1; + } + printf(") %4d tx %5d ins %5d sigops took %4d ms\n", $transactions, $inputs, $sigops, (uint64) $duration / 1000); } diff --git a/depends/packages/expat.mk b/depends/packages/expat.mk index 902fe43be2..41c1114be0 100644 --- a/depends/packages/expat.mk +++ b/depends/packages/expat.mk @@ -23,5 +23,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/*.la + rm -rf share lib/*.la endef diff --git a/depends/packages/fontconfig.mk b/depends/packages/fontconfig.mk index 0d5f94f380..22b5022f06 100644 --- a/depends/packages/fontconfig.mk +++ b/depends/packages/fontconfig.mk @@ -29,5 +29,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/*.la + rm -rf var lib/*.la endef diff --git a/depends/packages/freetype.mk b/depends/packages/freetype.mk index a1584608e1..aebc8a5f3b 100644 --- a/depends/packages/freetype.mk +++ b/depends/packages/freetype.mk @@ -23,5 +23,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/*.la + rm -rf share/man lib/*.la endef diff --git a/depends/packages/libXau.mk b/depends/packages/libXau.mk index 4c55c2df04..24e0e9d325 100644 --- a/depends/packages/libXau.mk +++ b/depends/packages/libXau.mk @@ -30,5 +30,5 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/*.la + rm -rf share lib/*.la endef diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 9004b064d6..12e0494ad4 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -248,7 +248,6 @@ endef define $(package)_config_cmds export PKG_CONFIG_SYSROOT_DIR=/ && \ export PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig && \ - export PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig && \ cd qtbase && \ ./configure -top-level $($(package)_config_opts) endef diff --git a/doc/descriptors.md b/doc/descriptors.md index 3bbb626a42..57a0f99d70 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -139,6 +139,47 @@ Key order does not matter for `sortedmulti()`. `sortedmulti()` behaves in the sa as `multi()` does but the keys are reordered in the resulting script such that they are lexicographically ordered as described in BIP67. +#### Basic multisig example + +For a good example of a basic M-of-N multisig between multiple participants using descriptor +wallets and PSBTs, as well as a signing flow, see [this functional test](/test/functional/wallet_multisig_descriptor_psbt.py). + +Disclaimers: It is important to note that this example serves as a quick-start and is kept basic for readability. A downside of the approach +outlined here is that each participant must maintain (and backup) two separate wallets: a signer and the corresponding multisig. +It should also be noted that privacy best-practices are not "by default" here - participants should take care to only use the signer to sign +transactions related to the multisig. Lastly, it is not recommended to use anything other than a Bitcoin Core descriptor wallet to serve as your +signer(s). Other wallets, whether hardware or software, likely impose additional checks and safeguards to prevent users from signing transactions that +could lead to loss of funds, or are deemed security hazards. Conforming to various 3rd-party checks and verifications is not in the scope of this example. + +The basic steps are: + + 1. Every participant generates an xpub. The most straightforward way is to create a new descriptor wallet which we will refer to as + the participant's signer wallet. Avoid reusing this wallet for any purpose other than signing transactions from the + corresponding multisig we are about to create. Hint: extract the wallet's xpubs using `listdescriptors` and pick the one from the + `pkh` descriptor since it's least likely to be accidentally reused (legacy addresses) + 2. Create a watch-only descriptor wallet (blank, private keys disabled). Now the multisig is created by importing the two descriptors: + `wsh(sortedmulti(<M>,XPUB1/0/*,XPUB2/0/*,…,XPUBN/0/*))` and `wsh(sortedmulti(<M>,XPUB1/1/*,XPUB2/1/*,…,XPUBN/1/*))` + (one descriptor w/ `0` for receiving addresses and another w/ `1` for change). Every participant does this + 3. A receiving address is generated for the multisig. As a check to ensure step 2 was done correctly, every participant + should verify they get the same addresses + 4. Funds are sent to the resulting address + 5. A sending transaction from the multisig is created using `walletcreatefundedpsbt` (anyone can initiate this). It is simple to do + this in the GUI by going to the `Send` tab in the multisig wallet and creating an unsigned transaction (PSBT) + 6. At least `M` participants check the PSBT with their multisig using `decodepsbt` to verify the transaction is OK before signing it. + 7. (If OK) the participant signs the PSBT with their signer wallet using `walletprocesspsbt`. It is simple to do this in the GUI by + loading the PSBT from file and signing it + 8. The signed PSBTs are collected with `combinepsbt`, finalized w/ `finalizepsbt`, and then the resulting transaction is broadcasted + to the network. Note that any wallet (eg one of the signers or multisig) is capable of doing this. + 9. Checks that balances are correct after the transaction has been included in a block + +You may prefer a daisy chained signing flow where each participant signs the PSBT one after another until +the PSBT has been signed `M` times and is "complete." For the most part, the steps above remain the same, except (6, 7) +change slightly from signing the original PSBT in parallel to signing it in series. `combinepsbt` is not necessary with +this signing flow and the last (`m`th) signer can just broadcast the PSBT after signing. Note that a parallel signing flow may be +preferable in cases where there are more signers. This signing flow is also included in the test / Python example. +[The test](/test/functional/wallet_multisig_descriptor_psbt.py) is meant to be documentation as much as it is a functional test, so +it is kept as simple and readable as possible. + ### BIP32 derived keys and chains Most modern wallet software and hardware uses keys that are derived using diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 0a5a7066ab..a05ea93a46 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -12,6 +12,7 @@ Developer Notes - [Generating Documentation](#generating-documentation) - [Development tips and tricks](#development-tips-and-tricks) - [Compiling for debugging](#compiling-for-debugging) + - [Show sources in debugging](#show-sources-in-debugging) - [Compiling for gprof profiling](#compiling-for-gprof-profiling) - [`debug.log`](#debuglog) - [Signet, testnet, and regtest modes](#signet-testnet-and-regtest-modes) @@ -253,6 +254,35 @@ Development tips and tricks Run configure with `--enable-debug` to add additional compiler flags that produce better debugging builds. +### Show sources in debugging + +If you have ccache enabled, absolute paths are stripped from debug information +with the -fdebug-prefix-map and -fmacro-prefix-map options (if supported by the +compiler). This might break source file detection in case you move binaries +after compilation, debug from the directory other than the project root or use +an IDE that only supports absolute paths for debugging. + +There are a few possible fixes: + +1. Configure source file mapping. + +For `gdb` create or append to `.gdbinit` file: +``` +set substitute-path ./src /path/to/project/root/src +``` + +For `lldb` create or append to `.lldbinit` file: +``` +settings set target.source-map ./src /path/to/project/root/src +``` + +2. Add a symlink to the `./src` directory: +``` +ln -s /path/to/project/root/src src +``` + +3. Use `debugedit` to modify debug information in the binary. + ### Compiling for gprof profiling Run configure with the `--enable-gprof` option, then make. diff --git a/doc/psbt.md b/doc/psbt.md index c411b31d5d..0f31cb8eba 100644 --- a/doc/psbt.md +++ b/doc/psbt.md @@ -92,6 +92,9 @@ hardware implementations will typically implement multiple roles simultaneously. #### Multisig with multiple Bitcoin Core instances +For a quick start see [Basic M-of-N multisig example using descriptor wallets and PSBTs](./descriptors.md#basic-multisig-example). +If you are using legacy wallets feel free to continue with the example provided here. + Alice, Bob, and Carol want to create a 2-of-3 multisig address. They're all using Bitcoin Core. We assume their wallets only contain the multisig funds. In case they also have a personal wallet, this can be accomplished through the diff --git a/doc/release-notes-12677.md b/doc/release-notes-12677.md deleted file mode 100644 index d6fea9eae7..0000000000 --- a/doc/release-notes-12677.md +++ /dev/null @@ -1,8 +0,0 @@ -Notable changes -=============== - -Updated RPCs ------------- - -- `listunspent` now includes `ancestorcount`, `ancestorsize`, and -`ancestorfees` for each transaction output that is still in the mempool. diff --git a/doc/release-notes-23065.md b/doc/release-notes-23065.md deleted file mode 100644 index 6ec002b2df..0000000000 --- a/doc/release-notes-23065.md +++ /dev/null @@ -1,15 +0,0 @@ -Notable changes -=============== - -Updated RPCs ------------- - -- `lockunspent` now optionally takes a third parameter, `persistent`, which -causes the lock to be written persistently to the wallet database. This -allows UTXOs to remain locked even after node restarts or crashes. - -GUI changes ------------ - -- UTXOs which are locked via the GUI are now stored persistently in the -wallet database, so are not lost on node shutdown or crash.
\ No newline at end of file diff --git a/doc/release-notes-23093.md b/doc/release-notes-23093.md new file mode 100644 index 0000000000..68fbaec53c --- /dev/null +++ b/doc/release-notes-23093.md @@ -0,0 +1,11 @@ +Notable changes +=============== + +Updated RPCs +------------ + +- `upgradewallet` will now automatically flush the keypool if upgrading +from a non-HD wallet to an HD wallet, to immediately start using the +newly-generated HD keys. +- a new RPC `newkeypool` has been added, which will flush (entirely +clear and refill) the keypool. diff --git a/doc/release-notes-remove-rescan.md b/doc/release-notes-remove-rescan.md deleted file mode 100644 index fcbe7e6a82..0000000000 --- a/doc/release-notes-remove-rescan.md +++ /dev/null @@ -1,9 +0,0 @@ -Notable changes -=============== - -Rescan startup parameter removed --------------------------------- - -The `-rescan` startup parameter has been removed. Wallets which require -rescanning due to corruption will still be rescanned on startup. -Otherwise, please use the `rescanblockchain` RPC to trigger a rescan.
\ No newline at end of file diff --git a/doc/release-notes.md b/doc/release-notes.md index a0c1ed3b31..b460cd3eb2 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -61,6 +61,19 @@ P2P and network changes They will become eligible for address gossip after sending an ADDR, ADDRV2, or GETADDR message. (#21528) +Fee estimation changes +---------------------- + +- Fee estimation now takes the feerate of replacement (RBF) transactions into + account. (#22539) + +Rescan startup parameter removed +-------------------------------- + +The `-rescan` startup parameter has been removed. Wallets which require +rescanning due to corruption will still be rescanned on startup. +Otherwise, please use the `rescanblockchain` RPC to trigger a rescan. (#23123) + Updated RPCs ------------ @@ -69,6 +82,22 @@ Updated RPCs `gettransaction verbose=true` and REST endpoints `/rest/tx`, `/rest/getutxos`, `/rest/block` no longer return the `addresses` and `reqSigs` fields, which were previously deprecated in 22.0. (#22650) +- The `getblock` RPC command now supports verbose level 3 containing transaction inputs + `prevout` information. The existing `/rest/block/` REST endpoint is modified to contain + this information too. Every `vin` field will contain an additional `prevout` subfield + describing the spent output. `prevout` contains the following keys: + - `generated` - true if the spent coins was a coinbase. + - `height` + - `value` + - `scriptPubKey` + +- `listunspent` now includes `ancestorcount`, `ancestorsize`, and + `ancestorfees` for each transaction output that is still in the mempool. + (#12677) + +- `lockunspent` now optionally takes a third parameter, `persistent`, which + causes the lock to be written persistently to the wallet database. This + allows UTXOs to remain locked even after node restarts or crashes. (#23065) New RPCs -------- @@ -113,6 +142,9 @@ Wallet GUI changes ----------- +- UTXOs which are locked via the GUI are now stored persistently in the + wallet database, so are not lost on node shutdown or crash. (#23065) + Low-level changes ================= diff --git a/doc/release-process.md b/doc/release-process.md index 6a5202d0f9..14567d4f15 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -60,7 +60,7 @@ Release Process To tag the version (or release candidate) in git, use the `make-tag.py` script from [bitcoin-maintainer-tools](https://github.com/bitcoin-core/bitcoin-maintainer-tools). From the root of the repository run: - ../bitcoin-maintainer-tools/make-tag.py v(new version, e.g. 0.20.0) + ../bitcoin-maintainer-tools/make-tag.py v(new version, e.g. 23.0) This will perform a few last-minute consistency checks in the build system files, and if they pass, create a signed tag. @@ -253,6 +253,10 @@ cat "$VERSION"/*/all.SHA256SUMS.asc > SHA256SUMS.asc - bitcoincore.org maintained versions update: [table](https://github.com/bitcoin-core/bitcoincore.org/commits/master/_includes/posts/maintenance-table.md) + - Delete post-EOL [release branches](https://github.com/bitcoin/bitcoin/branches/all) and create a tag `v${branch_name}-final`. + + - Delete ["Needs backport" labels](https://github.com/bitcoin/bitcoin/labels?q=backport) for non-existing branches. + - bitcoincore.org RPC documentation update - Install [golang](https://golang.org/doc/install) diff --git a/doc/tracing.md b/doc/tracing.md index 87fc9603fe..57104c43a0 100644 --- a/doc/tracing.md +++ b/doc/tracing.md @@ -101,19 +101,12 @@ Is called *after* a block is connected to the chain. Can, for example, be used to benchmark block connections together with `-reindex`. Arguments passed: -1. Block Header Hash as `pointer to C-style String` (64 characters) +1. Block Header Hash as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) 2. Block Height as `int32` 3. Transactions in the Block as `uint64` 4. Inputs spend in the Block as `int32` 5. SigOps in the Block (excluding coinbase SigOps) `uint64` 6. Time it took to connect the Block in microseconds (µs) as `uint64` -7. Block Header Hash as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) - -Note: The 7th argument can't be accessed by bpftrace and is purposefully chosen -to be the block header hash as bytes. See [bpftrace argument limit] for more -details. - -[bpftrace argument limit]: #bpftrace-argument-limit ## Adding tracepoints to Bitcoin Core diff --git a/src/Makefile.am b/src/Makefile.am index 52c8b85357..856e6f1985 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -6,7 +6,7 @@ print-%: FORCE @echo '$*'='$($*)' -DIST_SUBDIRS = secp256k1 univalue +DIST_SUBDIRS = secp256k1 AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) @@ -15,18 +15,7 @@ AM_LIBTOOLFLAGS = --preserve-dup-deps PTHREAD_FLAGS = $(PTHREAD_CFLAGS) $(PTHREAD_LIBS) EXTRA_LIBRARIES = -if EMBEDDED_UNIVALUE -LIBUNIVALUE = univalue/libunivalue.la - -$(LIBUNIVALUE): $(wildcard univalue/lib/*) $(wildcard univalue/include/*) - $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) -else -LIBUNIVALUE = $(UNIVALUE_LIBS) -endif - -BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/secp256k1/include $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) - -BITCOIN_INCLUDES += $(UNIVALUE_CFLAGS) +BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) LIBBITCOIN_SERVER=libbitcoin_server.a LIBBITCOIN_COMMON=libbitcoin_common.a @@ -80,6 +69,7 @@ EXTRA_LIBRARIES += \ $(LIBBITCOIN_ZMQ) lib_LTLIBRARIES = $(LIBBITCOINCONSENSUS) +noinst_LTLIBRARIES = bin_PROGRAMS = noinst_PROGRAMS = @@ -117,13 +107,13 @@ endif BITCOIN_CORE_H = \ addrdb.h \ addrman.h \ + addrman_impl.h \ attributes.h \ banman.h \ base58.h \ bech32.h \ blockencodings.h \ blockfilter.h \ - bloom.h \ chain.h \ chainparams.h \ chainparamsbase.h \ @@ -131,6 +121,7 @@ BITCOIN_CORE_H = \ checkqueue.h \ clientversion.h \ coins.h \ + common/bloom.h \ compat.h \ compat/assumptions.h \ compat/byteswap.h \ @@ -261,6 +252,7 @@ BITCOIN_CORE_H = \ util/sock.h \ util/spanparsing.h \ util/string.h \ + util/syscall_sandbox.h \ util/system.h \ util/thread.h \ util/threadnames.h \ @@ -498,9 +490,9 @@ crypto_libbitcoin_crypto_shani_a_SOURCES = crypto/sha256_shani.cpp libbitcoin_consensus_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_consensus_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_consensus_a_SOURCES = \ - amount.h \ arith_uint256.cpp \ arith_uint256.h \ + consensus/amount.h \ consensus/merkle.cpp \ consensus/merkle.h \ consensus/params.h \ @@ -537,9 +529,9 @@ libbitcoin_common_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_common_a_SOURCES = \ base58.cpp \ bech32.cpp \ - bloom.cpp \ chainparams.cpp \ coins.cpp \ + common/bloom.cpp \ compressor.cpp \ core_read.cpp \ core_write.cpp \ @@ -611,6 +603,7 @@ libbitcoin_util_a_SOURCES = \ util/spanparsing.cpp \ util/strencodings.cpp \ util/string.cpp \ + util/syscall_sandbox.cpp \ util/time.cpp \ util/tokenpipe.cpp \ $(BITCOIN_CORE_H) @@ -794,7 +787,6 @@ $(top_srcdir)/$(subdir)/config/bitcoin-config.h.in: $(am__configure_deps) clean-local: -$(MAKE) -C secp256k1 clean - -$(MAKE) -C univalue clean -rm -f leveldb/*/*.gcda leveldb/*/*.gcno leveldb/helpers/memenv/*.gcda leveldb/helpers/memenv/*.gcno -rm -f config.h -rm -rf test/__pycache__ @@ -805,20 +797,8 @@ clean-local: $(AM_V_GEN) $(WINDRES) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(CPPFLAGS) -DWINDRES_PREPROC -i $< -o $@ check-symbols: $(bin_PROGRAMS) -if TARGET_DARWIN - @echo "Checking macOS dynamic libraries..." + @echo "Running symbol and dynamic library checks..." $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) -endif - -if TARGET_WINDOWS - @echo "Checking Windows dynamic libraries..." - $(AM_V_at) $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) -endif - -if TARGET_LINUX - @echo "Checking glibc back compat..." - $(AM_V_at) CPPFILT='$(CPPFILT)' $(PYTHON) $(top_srcdir)/contrib/devtools/symbol-check.py $(bin_PROGRAMS) -endif check-security: $(bin_PROGRAMS) if HARDEN @@ -861,11 +841,8 @@ nodist_libbitcoin_ipc_a_SOURCES = $(libbitcoin_ipc_mpgen_output) CLEANFILES += $(libbitcoin_ipc_mpgen_output) endif -if EMBEDDED_LEVELDB include Makefile.crc32c.include include Makefile.leveldb.include -endif - include Makefile.test_util.include include Makefile.test_fuzz.include @@ -884,3 +861,5 @@ endif if ENABLE_QT_TESTS include Makefile.qttest.include endif + +include Makefile.univalue.include diff --git a/src/Makefile.leveldb.include b/src/Makefile.leveldb.include index ce1f93f11f..3bec92482a 100644 --- a/src/Makefile.leveldb.include +++ b/src/Makefile.leveldb.include @@ -8,9 +8,10 @@ LIBMEMENV_INT = leveldb/libmemenv.a EXTRA_LIBRARIES += $(LIBLEVELDB_INT) EXTRA_LIBRARIES += $(LIBMEMENV_INT) -LIBLEVELDB += $(LIBLEVELDB_INT) $(LIBCRC32C) -LIBMEMENV += $(LIBMEMENV_INT) +LIBLEVELDB = $(LIBLEVELDB_INT) $(LIBCRC32C) +LIBMEMENV = $(LIBMEMENV_INT) +LEVELDB_CPPFLAGS = LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/include LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/helpers/memenv diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index f4b0b3adbe..1e3d75a8d8 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -168,10 +168,10 @@ BITCOIN_QT_H = \ qt/walletview.h \ qt/winshutdownmonitor.h -RES_FONTS = \ +QT_RES_FONTS = \ qt/res/fonts/RobotoMono-Bold.ttf -RES_ICONS = \ +QT_RES_ICONS = \ qt/res/icons/add.png \ qt/res/icons/address-book.png \ qt/res/icons/bitcoin.ico \ @@ -287,9 +287,9 @@ if ENABLE_WALLET BITCOIN_QT_CPP += $(BITCOIN_QT_WALLET_CPP) endif # ENABLE_WALLET -RES_ANIMATION = $(wildcard $(srcdir)/qt/res/animation/spinner-*.png) +QT_RES_ANIMATION = $(wildcard $(srcdir)/qt/res/animation/spinner-*.png) -BITCOIN_RC = qt/res/bitcoin-qt-res.rc +BITCOIN_QT_RC = qt/res/bitcoin-qt-res.rc BITCOIN_QT_INCLUDES = -DQT_NO_KEYWORDS -DQT_USE_QSTRINGBUILDER @@ -299,7 +299,7 @@ qt_libbitcoinqt_a_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) qt_libbitcoinqt_a_OBJCXXFLAGS = $(AM_OBJCXXFLAGS) $(QT_PIE_FLAGS) qt_libbitcoinqt_a_SOURCES = $(BITCOIN_QT_CPP) $(BITCOIN_QT_H) $(QT_FORMS_UI) \ - $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(RES_FONTS) $(RES_ICONS) $(RES_ANIMATION) + $(QT_QRC) $(QT_QRC_LOCALE) $(QT_TS) $(QT_RES_FONTS) $(QT_RES_ICONS) $(QT_RES_ANIMATION) if TARGET_DARWIN qt_libbitcoinqt_a_SOURCES += $(BITCOIN_MM) endif @@ -321,7 +321,7 @@ bitcoin_qt_cxxflags = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) bitcoin_qt_sources = qt/main.cpp if TARGET_WINDOWS - bitcoin_qt_sources += $(BITCOIN_RC) + bitcoin_qt_sources += $(BITCOIN_QT_RC) endif bitcoin_qt_ldadd = qt/libbitcoinqt.a $(LIBBITCOIN_SERVER) if ENABLE_WALLET @@ -371,7 +371,7 @@ $(QT_QRC_LOCALE_CPP): $(QT_QRC_LOCALE) $(QT_QM) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin_locale --format-version 1 $(@D)/temp_$(<F) > $@ @rm $(@D)/temp_$(<F) -$(QT_QRC_CPP): $(QT_QRC) $(QT_FORMS_H) $(RES_FONTS) $(RES_ICONS) $(RES_ANIMATION) +$(QT_QRC_CPP): $(QT_QRC) $(QT_FORMS_H) $(QT_RES_FONTS) $(QT_RES_ICONS) $(QT_RES_ANIMATION) @test -f $(RCC) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin --format-version 1 $< > $@ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index be63214c23..27f9382631 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -119,8 +119,9 @@ BITCOIN_TESTS =\ test/sanity_tests.cpp \ test/scheduler_tests.cpp \ test/script_p2sh_tests.cpp \ - test/script_tests.cpp \ + test/script_parse_tests.cpp \ test/script_standard_tests.cpp \ + test/script_tests.cpp \ test/scriptnum_tests.cpp \ test/serfloat_tests.cpp \ test/serialize_tests.cpp \ @@ -349,8 +350,26 @@ if ENABLE_BENCH endif endif $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check -if EMBEDDED_UNIVALUE - $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C univalue check + +if !ENABLE_FUZZ +UNIVALUE_TESTS = univalue/test/object univalue/test/unitester univalue/test/no_nul +noinst_PROGRAMS += $(UNIVALUE_TESTS) +TESTS += $(UNIVALUE_TESTS) + +univalue_test_unitester_SOURCES = $(UNIVALUE_TEST_UNITESTER_INT) +univalue_test_unitester_LDADD = $(LIBUNIVALUE) +univalue_test_unitester_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) -DJSON_TEST_SRC=\"$(srcdir)/$(UNIVALUE_TEST_DATA_DIR_INT)\" +univalue_test_unitester_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) + +univalue_test_no_nul_SOURCES = $(UNIVALUE_TEST_NO_NUL_INT) +univalue_test_no_nul_LDADD = $(LIBUNIVALUE) +univalue_test_no_nul_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) +univalue_test_no_nul_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) + +univalue_test_object_SOURCES = $(UNIVALUE_TEST_OBJECT_INT) +univalue_test_object_LDADD = $(LIBUNIVALUE) +univalue_test_object_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) +univalue_test_object_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) endif %.cpp.test: %.cpp diff --git a/src/Makefile.univalue.include b/src/Makefile.univalue.include new file mode 100644 index 0000000000..3644e36368 --- /dev/null +++ b/src/Makefile.univalue.include @@ -0,0 +1,6 @@ +include univalue/sources.mk + +LIBUNIVALUE = libunivalue.la +noinst_LTLIBRARIES += $(LIBUNIVALUE) +libunivalue_la_SOURCES = $(UNIVALUE_LIB_SOURCES_INT) $(UNIVALUE_DIST_HEADERS_INT) $(UNIVALUE_LIB_HEADERS_INT) $(UNIVALUE_TEST_FILES_INT) +libunivalue_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 94c77a6d89..bdb1fc6b2b 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -58,7 +58,7 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data if (fileout.IsNull()) { fileout.fclose(); remove(pathTmp); - return error("%s: Failed to open file %s", __func__, pathTmp.string()); + return error("%s: Failed to open file %s", __func__, fs::PathToString(pathTmp)); } // Serialize @@ -70,7 +70,7 @@ bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data if (!FileCommit(fileout.Get())) { fileout.fclose(); remove(pathTmp); - return error("%s: Failed to flush file %s", __func__, pathTmp.string()); + return error("%s: Failed to flush file %s", __func__, fs::PathToString(pathTmp)); } fileout.fclose(); @@ -122,8 +122,8 @@ void DeserializeFileDB(const fs::path& path, Data& data, int version) } // namespace CBanDB::CBanDB(fs::path ban_list_path) - : m_banlist_dat(ban_list_path.string() + ".dat"), - m_banlist_json(ban_list_path.string() + ".json") + : m_banlist_dat(ban_list_path + ".dat"), + m_banlist_json(ban_list_path + ".json") { } @@ -143,7 +143,7 @@ bool CBanDB::Write(const banmap_t& banSet) bool CBanDB::Read(banmap_t& banSet) { if (fs::exists(m_banlist_dat)) { - LogPrintf("banlist.dat ignored because it can only be read by " PACKAGE_NAME " version 22.x. Remove %s to silence this warning.\n", m_banlist_dat); + LogPrintf("banlist.dat ignored because it can only be read by " PACKAGE_NAME " version 22.x. Remove %s to silence this warning.\n", fs::quoted(fs::PathToString(m_banlist_dat))); } // If the JSON banlist does not exist, then recreate it if (!fs::exists(m_banlist_json)) { @@ -155,7 +155,7 @@ bool CBanDB::Read(banmap_t& banSet) if (!util::ReadSettings(m_banlist_json, settings, errors)) { for (const auto& err : errors) { - LogPrintf("Cannot load banlist %s: %s\n", m_banlist_json.string(), err); + LogPrintf("Cannot load banlist %s: %s\n", fs::PathToString(m_banlist_json), err); } return false; } @@ -163,28 +163,28 @@ bool CBanDB::Read(banmap_t& banSet) try { BanMapFromJson(settings[JSON_KEY], banSet); } catch (const std::runtime_error& e) { - LogPrintf("Cannot parse banlist %s: %s\n", m_banlist_json.string(), e.what()); + LogPrintf("Cannot parse banlist %s: %s\n", fs::PathToString(m_banlist_json), e.what()); return false; } return true; } -bool DumpPeerAddresses(const ArgsManager& args, const CAddrMan& addr) +bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr) { const auto pathAddr = args.GetDataDirNet() / "peers.dat"; return SerializeFileDB("peers", pathAddr, addr, CLIENT_VERSION); } -void ReadFromStream(CAddrMan& addr, CDataStream& ssPeers) +void ReadFromStream(AddrMan& addr, CDataStream& ssPeers) { DeserializeDB(ssPeers, addr, false); } -std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const ArgsManager& args, std::unique_ptr<CAddrMan>& addrman) +std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman) { auto check_addrman = std::clamp<int32_t>(args.GetIntArg("-checkaddrman", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), 0, 1000000); - addrman = std::make_unique<CAddrMan>(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); + addrman = std::make_unique<AddrMan>(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); int64_t nStart = GetTimeMillis(); const auto path_addr{args.GetDataDirNet() / "peers.dat"}; @@ -193,13 +193,13 @@ std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const A LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman->size(), GetTimeMillis() - nStart); } catch (const DbNotFoundError&) { // Addrman can be in an inconsistent state after failure, reset it - addrman = std::make_unique<CAddrMan>(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); - LogPrintf("Creating peers.dat because the file was not found (%s)\n", path_addr); + addrman = std::make_unique<AddrMan>(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); + LogPrintf("Creating peers.dat because the file was not found (%s)\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const std::exception& e) { addrman = nullptr; return strprintf(_("Invalid or corrupt peers.dat (%s). If you believe this is a bug, please report it to %s. As a workaround, you can move the file (%s) out of the way (rename, move, or delete) to have a new one created on the next start."), - e.what(), PACKAGE_BUGREPORT, path_addr); + e.what(), PACKAGE_BUGREPORT, fs::quoted(fs::PathToString(path_addr))); } return std::nullopt; } @@ -215,7 +215,7 @@ std::vector<CAddress> ReadAnchors(const fs::path& anchors_db_path) std::vector<CAddress> anchors; try { DeserializeFileDB(anchors_db_path, anchors, CLIENT_VERSION | ADDRV2_FORMAT); - LogPrintf("Loaded %i addresses from %s\n", anchors.size(), anchors_db_path.filename()); + LogPrintf("Loaded %i addresses from %s\n", anchors.size(), fs::quoted(fs::PathToString(anchors_db_path.filename()))); } catch (const std::exception&) { anchors.clear(); } diff --git a/src/addrdb.h b/src/addrdb.h index 33cc1f9204..19be4b5bb4 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -14,14 +14,14 @@ #include <vector> class ArgsManager; -class CAddrMan; +class AddrMan; class CAddress; class CDataStream; struct bilingual_str; -bool DumpPeerAddresses(const ArgsManager& args, const CAddrMan& addr); +bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr); /** Only used by tests. */ -void ReadFromStream(CAddrMan& addr, CDataStream& ssPeers); +void ReadFromStream(AddrMan& addr, CDataStream& ssPeers); /** Access to the banlist database (banlist.json) */ class CBanDB @@ -48,7 +48,7 @@ public: }; /** Returns an error string on failure */ -std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const ArgsManager& args, std::unique_ptr<CAddrMan>& addrman); +std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman); /** * Dump the anchor IP address database (anchors.dat) diff --git a/src/addrman.cpp b/src/addrman.cpp index 7c6b8fe64d..cf6e847b04 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -4,25 +4,27 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <addrman.h> +#include <addrman_impl.h> -#include <clientversion.h> #include <hash.h> -#include <logging.h> #include <netaddress.h> +#include <protocol.h> +#include <random.h> #include <serialize.h> #include <streams.h> +#include <timedata.h> +#include <tinyformat.h> +#include <uint256.h> #include <util/check.h> #include <cmath> #include <optional> -#include <unordered_map> -#include <unordered_set> /** Over how many buckets entries with tried addresses from a single group (/16 for IPv4) are spread */ static constexpr uint32_t ADDRMAN_TRIED_BUCKETS_PER_GROUP{8}; /** Over how many buckets entries with new addresses originating from a single group are spread */ static constexpr uint32_t ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP{64}; -/** Maximum number of times an address can be added to the new table */ +/** Maximum number of times an address can occur in the new table */ static constexpr int32_t ADDRMAN_NEW_BUCKETS_PER_ADDRESS{8}; /** How old addresses can maximally be */ static constexpr int64_t ADDRMAN_HORIZON_DAYS{30}; @@ -39,34 +41,28 @@ static constexpr size_t ADDRMAN_SET_TRIED_COLLISION_SIZE{10}; /** The maximum time we'll spend trying to resolve a tried table collision, in seconds */ static constexpr int64_t ADDRMAN_TEST_WINDOW{40*60}; // 40 minutes -int CAddrInfo::GetTriedBucket(const uint256& nKey, const std::vector<bool> &asmap) const +int AddrInfo::GetTriedBucket(const uint256& nKey, const std::vector<bool>& asmap) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetCheapHash(); uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); - int tried_bucket = hash2 % ADDRMAN_TRIED_BUCKET_COUNT; - uint32_t mapped_as = GetMappedAS(asmap); - LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to tried bucket %i\n", ToStringIP(), mapped_as, tried_bucket); - return tried_bucket; + return hash2 % ADDRMAN_TRIED_BUCKET_COUNT; } -int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std::vector<bool> &asmap) const +int AddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std::vector<bool>& asmap) const { std::vector<unsigned char> vchSourceGroupKey = src.GetGroup(asmap); uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << vchSourceGroupKey).GetCheapHash(); uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << vchSourceGroupKey << (hash1 % ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP)).GetCheapHash(); - int new_bucket = hash2 % ADDRMAN_NEW_BUCKET_COUNT; - uint32_t mapped_as = GetMappedAS(asmap); - LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to new bucket %i\n", ToStringIP(), mapped_as, new_bucket); - return new_bucket; + return hash2 % ADDRMAN_NEW_BUCKET_COUNT; } -int CAddrInfo::GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const +int AddrInfo::GetBucketPosition(const uint256& nKey, bool fNew, int nBucket) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << (fNew ? uint8_t{'N'} : uint8_t{'K'}) << nBucket << GetKey()).GetCheapHash(); return hash1 % ADDRMAN_BUCKET_SIZE; } -bool CAddrInfo::IsTerrible(int64_t nNow) const +bool AddrInfo::IsTerrible(int64_t nNow) const { if (nLastTry && nLastTry >= nNow - 60) // never remove things tried in the last minute return false; @@ -86,7 +82,7 @@ bool CAddrInfo::IsTerrible(int64_t nNow) const return false; } -double CAddrInfo::GetChance(int64_t nNow) const +double AddrInfo::GetChance(int64_t nNow) const { double fChance = 1.0; int64_t nSinceLastTry = std::max<int64_t>(nNow - nLastTry, 0); @@ -101,7 +97,7 @@ double CAddrInfo::GetChance(int64_t nNow) const return fChance; } -CAddrMan::CAddrMan(std::vector<bool> asmap, bool deterministic, int32_t consistency_check_ratio) +AddrManImpl::AddrManImpl(std::vector<bool>&& asmap, bool deterministic, int32_t consistency_check_ratio) : insecure_rand{deterministic} , nKey{deterministic ? uint256{1} : insecure_rand.rand256()} , m_consistency_check_ratio{consistency_check_ratio} @@ -119,8 +115,13 @@ CAddrMan::CAddrMan(std::vector<bool> asmap, bool deterministic, int32_t consiste } } +AddrManImpl::~AddrManImpl() +{ + nKey.SetNull(); +} + template <typename Stream> -void CAddrMan::Serialize(Stream& s_) const +void AddrManImpl::Serialize(Stream& s_) const { LOCK(cs); @@ -183,7 +184,7 @@ void CAddrMan::Serialize(Stream& s_) const int nIds = 0; for (const auto& entry : mapInfo) { mapUnkIds[entry.first] = nIds; - const CAddrInfo &info = entry.second; + const AddrInfo& info = entry.second; if (info.nRefCount) { assert(nIds != nNew); // this means nNew was wrong, oh ow s << info; @@ -192,7 +193,7 @@ void CAddrMan::Serialize(Stream& s_) const } nIds = 0; for (const auto& entry : mapInfo) { - const CAddrInfo &info = entry.second; + const AddrInfo& info = entry.second; if (info.fInTried) { assert(nIds != nTried); // this means nTried was wrong, oh ow s << info; @@ -223,7 +224,7 @@ void CAddrMan::Serialize(Stream& s_) const } template <typename Stream> -void CAddrMan::Unserialize(Stream& s_) +void AddrManImpl::Unserialize(Stream& s_) { LOCK(cs); @@ -262,21 +263,21 @@ void CAddrMan::Unserialize(Stream& s_) if (nNew > ADDRMAN_NEW_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE || nNew < 0) { throw std::ios_base::failure( - strprintf("Corrupt CAddrMan serialization: nNew=%d, should be in [0, %d]", + strprintf("Corrupt AddrMan serialization: nNew=%d, should be in [0, %d]", nNew, ADDRMAN_NEW_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE)); } if (nTried > ADDRMAN_TRIED_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE || nTried < 0) { throw std::ios_base::failure( - strprintf("Corrupt CAddrMan serialization: nTried=%d, should be in [0, %d]", + strprintf("Corrupt AddrMan serialization: nTried=%d, should be in [0, %d]", nTried, ADDRMAN_TRIED_BUCKET_COUNT * ADDRMAN_BUCKET_SIZE)); } // Deserialize entries from the new table. for (int n = 0; n < nNew; n++) { - CAddrInfo &info = mapInfo[n]; + AddrInfo& info = mapInfo[n]; s >> info; mapAddr[info] = n; info.nRandomPos = vRandom.size(); @@ -287,7 +288,7 @@ void CAddrMan::Unserialize(Stream& s_) // Deserialize entries from the tried table. int nLost = 0; for (int n = 0; n < nTried; n++) { - CAddrInfo info; + AddrInfo info; s >> info; int nKBucket = info.GetTriedBucket(nKey, m_asmap); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); @@ -344,7 +345,7 @@ void CAddrMan::Unserialize(Stream& s_) for (auto bucket_entry : bucket_entries) { int bucket{bucket_entry.first}; const int entry_index{bucket_entry.second}; - CAddrInfo& info = mapInfo[entry_index]; + AddrInfo& info = mapInfo[entry_index]; // Don't store the entry in the new bucket if it's not a valid address for our addrman if (!info.IsValid()) continue; @@ -394,16 +395,7 @@ void CAddrMan::Unserialize(Stream& s_) } } -// explicit instantiation -template void CAddrMan::Serialize(CHashWriter& s) const; -template void CAddrMan::Serialize(CAutoFile& s) const; -template void CAddrMan::Serialize(CDataStream& s) const; -template void CAddrMan::Unserialize(CAutoFile& s); -template void CAddrMan::Unserialize(CHashVerifier<CAutoFile>& s); -template void CAddrMan::Unserialize(CDataStream& s); -template void CAddrMan::Unserialize(CHashVerifier<CDataStream>& s); - -CAddrInfo* CAddrMan::Find(const CNetAddr& addr, int* pnId) +AddrInfo* AddrManImpl::Find(const CNetAddr& addr, int* pnId) { AssertLockHeld(cs); @@ -418,12 +410,12 @@ CAddrInfo* CAddrMan::Find(const CNetAddr& addr, int* pnId) return nullptr; } -CAddrInfo* CAddrMan::Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId) +AddrInfo* AddrManImpl::Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId) { AssertLockHeld(cs); int nId = nIdCount++; - mapInfo[nId] = CAddrInfo(addr, addrSource); + mapInfo[nId] = AddrInfo(addr, addrSource); mapAddr[addr] = nId; mapInfo[nId].nRandomPos = vRandom.size(); vRandom.push_back(nId); @@ -432,7 +424,7 @@ CAddrInfo* CAddrMan::Create(const CAddress& addr, const CNetAddr& addrSource, in return &mapInfo[nId]; } -void CAddrMan::SwapRandom(unsigned int nRndPos1, unsigned int nRndPos2) const +void AddrManImpl::SwapRandom(unsigned int nRndPos1, unsigned int nRndPos2) const { AssertLockHeld(cs); @@ -456,12 +448,12 @@ void CAddrMan::SwapRandom(unsigned int nRndPos1, unsigned int nRndPos2) const vRandom[nRndPos2] = nId1; } -void CAddrMan::Delete(int nId) +void AddrManImpl::Delete(int nId) { AssertLockHeld(cs); assert(mapInfo.count(nId) != 0); - CAddrInfo& info = mapInfo[nId]; + AddrInfo& info = mapInfo[nId]; assert(!info.fInTried); assert(info.nRefCount == 0); @@ -472,24 +464,25 @@ void CAddrMan::Delete(int nId) nNew--; } -void CAddrMan::ClearNew(int nUBucket, int nUBucketPos) +void AddrManImpl::ClearNew(int nUBucket, int nUBucketPos) { AssertLockHeld(cs); // if there is an entry in the specified bucket, delete it. if (vvNew[nUBucket][nUBucketPos] != -1) { int nIdDelete = vvNew[nUBucket][nUBucketPos]; - CAddrInfo& infoDelete = mapInfo[nIdDelete]; + AddrInfo& infoDelete = mapInfo[nIdDelete]; assert(infoDelete.nRefCount > 0); infoDelete.nRefCount--; vvNew[nUBucket][nUBucketPos] = -1; + LogPrint(BCLog::ADDRMAN, "Removed %s from new[%i][%i]\n", infoDelete.ToString(), nUBucket, nUBucketPos); if (infoDelete.nRefCount == 0) { Delete(nIdDelete); } } } -void CAddrMan::MakeTried(CAddrInfo& info, int nId) +void AddrManImpl::MakeTried(AddrInfo& info, int nId) { AssertLockHeld(cs); @@ -517,7 +510,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) // find an item to evict int nIdEvict = vvTried[nKBucket][nKBucketPos]; assert(mapInfo.count(nIdEvict) == 1); - CAddrInfo& infoOld = mapInfo[nIdEvict]; + AddrInfo& infoOld = mapInfo[nIdEvict]; // Remove the to-be-evicted item from the tried set. infoOld.fInTried = false; @@ -534,6 +527,8 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) infoOld.nRefCount = 1; vvNew[nUBucket][nUBucketPos] = nIdEvict; nNew++; + LogPrint(BCLog::ADDRMAN, "Moved %s from tried[%i][%i] to new[%i][%i] to make space\n", + infoOld.ToString(), nKBucket, nKBucketPos, nUBucket, nUBucketPos); } assert(vvTried[nKBucket][nKBucketPos] == -1); @@ -542,7 +537,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) info.fInTried = true; } -void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime) +void AddrManImpl::Good_(const CService& addr, bool test_before_evict, int64_t nTime) { AssertLockHeld(cs); @@ -550,13 +545,13 @@ void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime nLastGood = nTime; - CAddrInfo* pinfo = Find(addr, &nId); + AddrInfo* pinfo = Find(addr, &nId); // if not found, bail out if (!pinfo) return; - CAddrInfo& info = *pinfo; + AddrInfo& info = *pinfo; // check whether we are talking about the exact same CService (including same port) if (info != addr) @@ -584,21 +579,24 @@ void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime // Will moving this address into tried evict another entry? if (test_before_evict && (vvTried[tried_bucket][tried_bucket_pos] != -1)) { - // Output the entry we'd be colliding with, for debugging purposes - auto colliding_entry = mapInfo.find(vvTried[tried_bucket][tried_bucket_pos]); - LogPrint(BCLog::ADDRMAN, "Collision inserting element into tried table (%s), moving %s to m_tried_collisions=%d\n", colliding_entry != mapInfo.end() ? colliding_entry->second.ToString() : "", addr.ToString(), m_tried_collisions.size()); if (m_tried_collisions.size() < ADDRMAN_SET_TRIED_COLLISION_SIZE) { m_tried_collisions.insert(nId); } + // Output the entry we'd be colliding with, for debugging purposes + auto colliding_entry = mapInfo.find(vvTried[tried_bucket][tried_bucket_pos]); + LogPrint(BCLog::ADDRMAN, "Collision with %s while attempting to move %s to tried table. Collisions=%d\n", + colliding_entry != mapInfo.end() ? colliding_entry->second.ToString() : "", + addr.ToString(), + m_tried_collisions.size()); } else { - LogPrint(BCLog::ADDRMAN, "Moving %s to tried\n", addr.ToString()); - // move nId to the tried tables MakeTried(info, nId); + LogPrint(BCLog::ADDRMAN, "Moved %s mapped to AS%i to tried[%i][%i]\n", + addr.ToString(), addr.GetMappedAS(m_asmap), tried_bucket, tried_bucket_pos); } } -bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimePenalty) +bool AddrManImpl::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimePenalty) { AssertLockHeld(cs); @@ -607,7 +605,7 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP bool fNew = false; int nId; - CAddrInfo* pinfo = Find(addr, &nId); + AddrInfo* pinfo = Find(addr, &nId); // Do not set a penalty for a source's self-announcement if (addr == source) { @@ -654,7 +652,7 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP if (vvNew[nUBucket][nUBucketPos] != nId) { bool fInsert = vvNew[nUBucket][nUBucketPos] == -1; if (!fInsert) { - CAddrInfo& infoExisting = mapInfo[vvNew[nUBucket][nUBucketPos]]; + AddrInfo& infoExisting = mapInfo[vvNew[nUBucket][nUBucketPos]]; if (infoExisting.IsTerrible() || (infoExisting.nRefCount > 1 && pinfo->nRefCount == 0)) { // Overwrite the existing new table entry. fInsert = true; @@ -664,6 +662,8 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP ClearNew(nUBucket, nUBucketPos); pinfo->nRefCount++; vvNew[nUBucket][nUBucketPos] = nId; + LogPrint(BCLog::ADDRMAN, "Added %s mapped to AS%i to new[%i][%i]\n", + addr.ToString(), addr.GetMappedAS(m_asmap), nUBucket, nUBucketPos); } else { if (pinfo->nRefCount == 0) { Delete(nId); @@ -673,17 +673,17 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP return fNew; } -void CAddrMan::Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) +void AddrManImpl::Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) { AssertLockHeld(cs); - CAddrInfo* pinfo = Find(addr); + AddrInfo* pinfo = Find(addr); // if not found, bail out if (!pinfo) return; - CAddrInfo& info = *pinfo; + AddrInfo& info = *pinfo; // check whether we are talking about the exact same CService (including same port) if (info != addr) @@ -697,15 +697,13 @@ void CAddrMan::Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) } } -CAddrInfo CAddrMan::Select_(bool newOnly) const +std::pair<CAddress, int64_t> AddrManImpl::Select_(bool newOnly) const { AssertLockHeld(cs); - if (vRandom.empty()) - return CAddrInfo(); + if (vRandom.empty()) return {}; - if (newOnly && nNew == 0) - return CAddrInfo(); + if (newOnly && nNew == 0) return {}; // Use a 50% chance for choosing between tried and new table entries. if (!newOnly && @@ -713,145 +711,62 @@ CAddrInfo CAddrMan::Select_(bool newOnly) const // use a tried node double fChanceFactor = 1.0; while (1) { + // Pick a tried bucket, and an initial position in that bucket. int nKBucket = insecure_rand.randrange(ADDRMAN_TRIED_BUCKET_COUNT); int nKBucketPos = insecure_rand.randrange(ADDRMAN_BUCKET_SIZE); - while (vvTried[nKBucket][nKBucketPos] == -1) { - nKBucket = (nKBucket + insecure_rand.randbits(ADDRMAN_TRIED_BUCKET_COUNT_LOG2)) % ADDRMAN_TRIED_BUCKET_COUNT; - nKBucketPos = (nKBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; + // Iterate over the positions of that bucket, starting at the initial one, + // and looping around. + int i; + for (i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) { + if (vvTried[nKBucket][(nKBucketPos + i) % ADDRMAN_BUCKET_SIZE] != -1) break; } - int nId = vvTried[nKBucket][nKBucketPos]; + // If the bucket is entirely empty, start over with a (likely) different one. + if (i == ADDRMAN_BUCKET_SIZE) continue; + // Find the entry to return. + int nId = vvTried[nKBucket][(nKBucketPos + i) % ADDRMAN_BUCKET_SIZE]; const auto it_found{mapInfo.find(nId)}; assert(it_found != mapInfo.end()); - const CAddrInfo& info{it_found->second}; - if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) - return info; + const AddrInfo& info{it_found->second}; + // With probability GetChance() * fChanceFactor, return the entry. + if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) { + LogPrint(BCLog::ADDRMAN, "Selected %s from tried\n", info.ToString()); + return {info, info.nLastTry}; + } + // Otherwise start over with a (likely) different bucket, and increased chance factor. fChanceFactor *= 1.2; } } else { // use a new node double fChanceFactor = 1.0; while (1) { + // Pick a new bucket, and an initial position in that bucket. int nUBucket = insecure_rand.randrange(ADDRMAN_NEW_BUCKET_COUNT); int nUBucketPos = insecure_rand.randrange(ADDRMAN_BUCKET_SIZE); - while (vvNew[nUBucket][nUBucketPos] == -1) { - nUBucket = (nUBucket + insecure_rand.randbits(ADDRMAN_NEW_BUCKET_COUNT_LOG2)) % ADDRMAN_NEW_BUCKET_COUNT; - nUBucketPos = (nUBucketPos + insecure_rand.randbits(ADDRMAN_BUCKET_SIZE_LOG2)) % ADDRMAN_BUCKET_SIZE; + // Iterate over the positions of that bucket, starting at the initial one, + // and looping around. + int i; + for (i = 0; i < ADDRMAN_BUCKET_SIZE; ++i) { + if (vvNew[nUBucket][(nUBucketPos + i) % ADDRMAN_BUCKET_SIZE] != -1) break; } - int nId = vvNew[nUBucket][nUBucketPos]; + // If the bucket is entirely empty, start over with a (likely) different one. + if (i == ADDRMAN_BUCKET_SIZE) continue; + // Find the entry to return. + int nId = vvNew[nUBucket][(nUBucketPos + i) % ADDRMAN_BUCKET_SIZE]; const auto it_found{mapInfo.find(nId)}; assert(it_found != mapInfo.end()); - const CAddrInfo& info{it_found->second}; - if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) - return info; - fChanceFactor *= 1.2; - } - } -} - -void CAddrMan::Check() const -{ - AssertLockHeld(cs); - - // Run consistency checks 1 in m_consistency_check_ratio times if enabled - if (m_consistency_check_ratio == 0) return; - if (insecure_rand.randrange(m_consistency_check_ratio) >= 1) return; - - const int err{ForceCheckAddrman()}; - if (err) { - LogPrintf("ADDRMAN CONSISTENCY CHECK FAILED!!! err=%i\n", err); - assert(false); - } -} - -int CAddrMan::ForceCheckAddrman() const -{ - AssertLockHeld(cs); - - LogPrint(BCLog::ADDRMAN, "Addrman checks started: new %i, tried %i, total %u\n", nNew, nTried, vRandom.size()); - - std::unordered_set<int> setTried; - std::unordered_map<int, int> mapNew; - - if (vRandom.size() != (size_t)(nTried + nNew)) - return -7; - - for (const auto& entry : mapInfo) { - int n = entry.first; - const CAddrInfo& info = entry.second; - if (info.fInTried) { - if (!info.nLastSuccess) - return -1; - if (info.nRefCount) - return -2; - setTried.insert(n); - } else { - if (info.nRefCount < 0 || info.nRefCount > ADDRMAN_NEW_BUCKETS_PER_ADDRESS) - return -3; - if (!info.nRefCount) - return -4; - mapNew[n] = info.nRefCount; - } - const auto it{mapAddr.find(info)}; - if (it == mapAddr.end() || it->second != n) { - return -5; - } - if (info.nRandomPos < 0 || (size_t)info.nRandomPos >= vRandom.size() || vRandom[info.nRandomPos] != n) - return -14; - if (info.nLastTry < 0) - return -6; - if (info.nLastSuccess < 0) - return -8; - } - - if (setTried.size() != (size_t)nTried) - return -9; - if (mapNew.size() != (size_t)nNew) - return -10; - - for (int n = 0; n < ADDRMAN_TRIED_BUCKET_COUNT; n++) { - for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { - if (vvTried[n][i] != -1) { - if (!setTried.count(vvTried[n][i])) - return -11; - const auto it{mapInfo.find(vvTried[n][i])}; - if (it == mapInfo.end() || it->second.GetTriedBucket(nKey, m_asmap) != n) { - return -17; - } - if (it->second.GetBucketPosition(nKey, false, n) != i) { - return -18; - } - setTried.erase(vvTried[n][i]); - } - } - } - - for (int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; n++) { - for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { - if (vvNew[n][i] != -1) { - if (!mapNew.count(vvNew[n][i])) - return -12; - const auto it{mapInfo.find(vvNew[n][i])}; - if (it == mapInfo.end() || it->second.GetBucketPosition(nKey, true, n) != i) { - return -19; - } - if (--mapNew[vvNew[n][i]] == 0) - mapNew.erase(vvNew[n][i]); + const AddrInfo& info{it_found->second}; + // With probability GetChance() * fChanceFactor, return the entry. + if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) { + LogPrint(BCLog::ADDRMAN, "Selected %s from new\n", info.ToString()); + return {info, info.nLastTry}; } + // Otherwise start over with a (likely) different bucket, and increased chance factor. + fChanceFactor *= 1.2; } } - - if (setTried.size()) - return -13; - if (mapNew.size()) - return -15; - if (nKey.IsNull()) - return -16; - - LogPrint(BCLog::ADDRMAN, "Addrman checks completed successfully\n"); - return 0; } -void CAddrMan::GetAddr_(std::vector<CAddress>& vAddr, size_t max_addresses, size_t max_pct, std::optional<Network> network) const +std::vector<CAddress> AddrManImpl::GetAddr_(size_t max_addresses, size_t max_pct, std::optional<Network> network) const { AssertLockHeld(cs); @@ -865,8 +780,9 @@ void CAddrMan::GetAddr_(std::vector<CAddress>& vAddr, size_t max_addresses, size // gather a list of random nodes, skipping those of low quality const int64_t now{GetAdjustedTime()}; + std::vector<CAddress> addresses; for (unsigned int n = 0; n < vRandom.size(); n++) { - if (vAddr.size() >= nNodes) + if (addresses.size() >= nNodes) break; int nRndPos = insecure_rand.randrange(vRandom.size() - n) + n; @@ -874,7 +790,7 @@ void CAddrMan::GetAddr_(std::vector<CAddress>& vAddr, size_t max_addresses, size const auto it{mapInfo.find(vRandom[n])}; assert(it != mapInfo.end()); - const CAddrInfo& ai{it->second}; + const AddrInfo& ai{it->second}; // Filter by network (optional) if (network != std::nullopt && ai.GetNetClass() != network) continue; @@ -882,21 +798,23 @@ void CAddrMan::GetAddr_(std::vector<CAddress>& vAddr, size_t max_addresses, size // Filter for quality if (ai.IsTerrible(now)) continue; - vAddr.push_back(ai); + addresses.push_back(ai); } + LogPrint(BCLog::ADDRMAN, "GetAddr returned %d random addresses\n", addresses.size()); + return addresses; } -void CAddrMan::Connected_(const CService& addr, int64_t nTime) +void AddrManImpl::Connected_(const CService& addr, int64_t nTime) { AssertLockHeld(cs); - CAddrInfo* pinfo = Find(addr); + AddrInfo* pinfo = Find(addr); // if not found, bail out if (!pinfo) return; - CAddrInfo& info = *pinfo; + AddrInfo& info = *pinfo; // check whether we are talking about the exact same CService (including same port) if (info != addr) @@ -908,17 +826,17 @@ void CAddrMan::Connected_(const CService& addr, int64_t nTime) info.nTime = nTime; } -void CAddrMan::SetServices_(const CService& addr, ServiceFlags nServices) +void AddrManImpl::SetServices_(const CService& addr, ServiceFlags nServices) { AssertLockHeld(cs); - CAddrInfo* pinfo = Find(addr); + AddrInfo* pinfo = Find(addr); // if not found, bail out if (!pinfo) return; - CAddrInfo& info = *pinfo; + AddrInfo& info = *pinfo; // check whether we are talking about the exact same CService (including same port) if (info != addr) @@ -928,7 +846,7 @@ void CAddrMan::SetServices_(const CService& addr, ServiceFlags nServices) info.nServices = nServices; } -void CAddrMan::ResolveCollisions_() +void AddrManImpl::ResolveCollisions_() { AssertLockHeld(cs); @@ -941,7 +859,7 @@ void CAddrMan::ResolveCollisions_() if (mapInfo.count(id_new) != 1) { erase_collision = true; } else { - CAddrInfo& info_new = mapInfo[id_new]; + AddrInfo& info_new = mapInfo[id_new]; // Which tried bucket to move the entry to. int tried_bucket = info_new.GetTriedBucket(nKey, m_asmap); @@ -952,7 +870,7 @@ void CAddrMan::ResolveCollisions_() // Get the to-be-evicted address that is being tested int id_old = vvTried[tried_bucket][tried_bucket_pos]; - CAddrInfo& info_old = mapInfo[id_old]; + AddrInfo& info_old = mapInfo[id_old]; // Has successfully connected in last X hours if (GetAdjustedTime() - info_old.nLastSuccess < ADDRMAN_REPLACEMENT_HOURS*(60*60)) { @@ -989,11 +907,11 @@ void CAddrMan::ResolveCollisions_() } } -CAddrInfo CAddrMan::SelectTriedCollision_() +std::pair<CAddress, int64_t> AddrManImpl::SelectTriedCollision_() { AssertLockHeld(cs); - if (m_tried_collisions.size() == 0) return CAddrInfo(); + if (m_tried_collisions.size() == 0) return {}; std::set<int>::iterator it = m_tried_collisions.begin(); @@ -1004,16 +922,291 @@ CAddrInfo CAddrMan::SelectTriedCollision_() // If id_new not found in mapInfo remove it from m_tried_collisions if (mapInfo.count(id_new) != 1) { m_tried_collisions.erase(it); - return CAddrInfo(); + return {}; } - const CAddrInfo& newInfo = mapInfo[id_new]; + const AddrInfo& newInfo = mapInfo[id_new]; // which tried bucket to move the entry to int tried_bucket = newInfo.GetTriedBucket(nKey, m_asmap); int tried_bucket_pos = newInfo.GetBucketPosition(nKey, false, tried_bucket); - int id_old = vvTried[tried_bucket][tried_bucket_pos]; + const AddrInfo& info_old = mapInfo[vvTried[tried_bucket][tried_bucket_pos]]; + return {info_old, info_old.nLastTry}; +} + +void AddrManImpl::Check() const +{ + AssertLockHeld(cs); + + // Run consistency checks 1 in m_consistency_check_ratio times if enabled + if (m_consistency_check_ratio == 0) return; + if (insecure_rand.randrange(m_consistency_check_ratio) >= 1) return; + + const int err{ForceCheckAddrman()}; + if (err) { + LogPrintf("ADDRMAN CONSISTENCY CHECK FAILED!!! err=%i\n", err); + assert(false); + } +} + +int AddrManImpl::ForceCheckAddrman() const +{ + AssertLockHeld(cs); + + LogPrint(BCLog::ADDRMAN, "Addrman checks started: new %i, tried %i, total %u\n", nNew, nTried, vRandom.size()); + + std::unordered_set<int> setTried; + std::unordered_map<int, int> mapNew; + + if (vRandom.size() != (size_t)(nTried + nNew)) + return -7; + + for (const auto& entry : mapInfo) { + int n = entry.first; + const AddrInfo& info = entry.second; + if (info.fInTried) { + if (!info.nLastSuccess) + return -1; + if (info.nRefCount) + return -2; + setTried.insert(n); + } else { + if (info.nRefCount < 0 || info.nRefCount > ADDRMAN_NEW_BUCKETS_PER_ADDRESS) + return -3; + if (!info.nRefCount) + return -4; + mapNew[n] = info.nRefCount; + } + const auto it{mapAddr.find(info)}; + if (it == mapAddr.end() || it->second != n) { + return -5; + } + if (info.nRandomPos < 0 || (size_t)info.nRandomPos >= vRandom.size() || vRandom[info.nRandomPos] != n) + return -14; + if (info.nLastTry < 0) + return -6; + if (info.nLastSuccess < 0) + return -8; + } + + if (setTried.size() != (size_t)nTried) + return -9; + if (mapNew.size() != (size_t)nNew) + return -10; + + for (int n = 0; n < ADDRMAN_TRIED_BUCKET_COUNT; n++) { + for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { + if (vvTried[n][i] != -1) { + if (!setTried.count(vvTried[n][i])) + return -11; + const auto it{mapInfo.find(vvTried[n][i])}; + if (it == mapInfo.end() || it->second.GetTriedBucket(nKey, m_asmap) != n) { + return -17; + } + if (it->second.GetBucketPosition(nKey, false, n) != i) { + return -18; + } + setTried.erase(vvTried[n][i]); + } + } + } - return mapInfo[id_old]; + for (int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; n++) { + for (int i = 0; i < ADDRMAN_BUCKET_SIZE; i++) { + if (vvNew[n][i] != -1) { + if (!mapNew.count(vvNew[n][i])) + return -12; + const auto it{mapInfo.find(vvNew[n][i])}; + if (it == mapInfo.end() || it->second.GetBucketPosition(nKey, true, n) != i) { + return -19; + } + if (--mapNew[vvNew[n][i]] == 0) + mapNew.erase(vvNew[n][i]); + } + } + } + + if (setTried.size()) + return -13; + if (mapNew.size()) + return -15; + if (nKey.IsNull()) + return -16; + + LogPrint(BCLog::ADDRMAN, "Addrman checks completed successfully\n"); + return 0; +} + +size_t AddrManImpl::size() const +{ + LOCK(cs); // TODO: Cache this in an atomic to avoid this overhead + return vRandom.size(); +} + +bool AddrManImpl::Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, int64_t nTimePenalty) +{ + LOCK(cs); + int nAdd = 0; + Check(); + for (std::vector<CAddress>::const_iterator it = vAddr.begin(); it != vAddr.end(); it++) + nAdd += Add_(*it, source, nTimePenalty) ? 1 : 0; + Check(); + if (nAdd) { + LogPrint(BCLog::ADDRMAN, "Added %i addresses from %s: %i tried, %i new\n", nAdd, source.ToString(), nTried, nNew); + } + return nAdd > 0; +} + +void AddrManImpl::Good(const CService& addr, int64_t nTime) +{ + LOCK(cs); + Check(); + Good_(addr, /* test_before_evict */ true, nTime); + Check(); +} + +void AddrManImpl::Attempt(const CService& addr, bool fCountFailure, int64_t nTime) +{ + LOCK(cs); + Check(); + Attempt_(addr, fCountFailure, nTime); + Check(); +} + +void AddrManImpl::ResolveCollisions() +{ + LOCK(cs); + Check(); + ResolveCollisions_(); + Check(); +} + +std::pair<CAddress, int64_t> AddrManImpl::SelectTriedCollision() +{ + LOCK(cs); + Check(); + const auto ret = SelectTriedCollision_(); + Check(); + return ret; +} + +std::pair<CAddress, int64_t> AddrManImpl::Select(bool newOnly) const +{ + LOCK(cs); + Check(); + const auto addrRet = Select_(newOnly); + Check(); + return addrRet; +} + +std::vector<CAddress> AddrManImpl::GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const +{ + LOCK(cs); + Check(); + const auto addresses = GetAddr_(max_addresses, max_pct, network); + Check(); + return addresses; +} + +void AddrManImpl::Connected(const CService& addr, int64_t nTime) +{ + LOCK(cs); + Check(); + Connected_(addr, nTime); + Check(); +} + +void AddrManImpl::SetServices(const CService& addr, ServiceFlags nServices) +{ + LOCK(cs); + Check(); + SetServices_(addr, nServices); + Check(); +} + +const std::vector<bool>& AddrManImpl::GetAsmap() const +{ + return m_asmap; +} + +AddrMan::AddrMan(std::vector<bool> asmap, bool deterministic, int32_t consistency_check_ratio) + : m_impl(std::make_unique<AddrManImpl>(std::move(asmap), deterministic, consistency_check_ratio)) {} + +AddrMan::~AddrMan() = default; + +template <typename Stream> +void AddrMan::Serialize(Stream& s_) const +{ + m_impl->Serialize<Stream>(s_); +} + +template <typename Stream> +void AddrMan::Unserialize(Stream& s_) +{ + m_impl->Unserialize<Stream>(s_); +} + +// explicit instantiation +template void AddrMan::Serialize(CHashWriter& s) const; +template void AddrMan::Serialize(CAutoFile& s) const; +template void AddrMan::Serialize(CDataStream& s) const; +template void AddrMan::Unserialize(CAutoFile& s); +template void AddrMan::Unserialize(CHashVerifier<CAutoFile>& s); +template void AddrMan::Unserialize(CDataStream& s); +template void AddrMan::Unserialize(CHashVerifier<CDataStream>& s); + +size_t AddrMan::size() const +{ + return m_impl->size(); +} + +bool AddrMan::Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, int64_t nTimePenalty) +{ + return m_impl->Add(vAddr, source, nTimePenalty); +} + +void AddrMan::Good(const CService& addr, int64_t nTime) +{ + m_impl->Good(addr, nTime); +} + +void AddrMan::Attempt(const CService& addr, bool fCountFailure, int64_t nTime) +{ + m_impl->Attempt(addr, fCountFailure, nTime); +} + +void AddrMan::ResolveCollisions() +{ + m_impl->ResolveCollisions(); +} + +std::pair<CAddress, int64_t> AddrMan::SelectTriedCollision() +{ + return m_impl->SelectTriedCollision(); +} + +std::pair<CAddress, int64_t> AddrMan::Select(bool newOnly) const +{ + return m_impl->Select(newOnly); +} + +std::vector<CAddress> AddrMan::GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const +{ + return m_impl->GetAddr(max_addresses, max_pct, network); +} + +void AddrMan::Connected(const CService& addr, int64_t nTime) +{ + m_impl->Connected(addr, nTime); +} + +void AddrMan::SetServices(const CService& addr, ServiceFlags nServices) +{ + m_impl->SetServices(addr, nServices); +} + +const std::vector<bool>& AddrMan::GetAsmap() const +{ + return m_impl->GetAsmap(); } diff --git a/src/addrman.h b/src/addrman.h index 7dd8528bef..174ab4f811 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -6,94 +6,22 @@ #ifndef BITCOIN_ADDRMAN_H #define BITCOIN_ADDRMAN_H -#include <fs.h> -#include <logging.h> #include <netaddress.h> #include <protocol.h> -#include <sync.h> +#include <streams.h> #include <timedata.h> #include <cstdint> +#include <memory> #include <optional> -#include <set> -#include <unordered_map> +#include <utility> #include <vector> +class AddrManImpl; + /** Default for -checkaddrman */ static constexpr int32_t DEFAULT_ADDRMAN_CONSISTENCY_CHECKS{0}; -/** - * Extended statistics about a CAddress - */ -class CAddrInfo : public CAddress -{ -public: - //! last try whatsoever by us (memory only) - int64_t nLastTry{0}; - - //! last counted attempt (memory only) - int64_t nLastCountAttempt{0}; - -private: - //! where knowledge about this address first came from - CNetAddr source; - - //! last successful connection by us - int64_t nLastSuccess{0}; - - //! connection attempts since last successful attempt - int nAttempts{0}; - - //! reference count in new sets (memory only) - int nRefCount{0}; - - //! in tried set? (memory only) - bool fInTried{false}; - - //! position in vRandom - mutable int nRandomPos{-1}; - - friend class CAddrMan; - friend class CAddrManDeterministic; - -public: - - SERIALIZE_METHODS(CAddrInfo, obj) - { - READWRITEAS(CAddress, obj); - READWRITE(obj.source, obj.nLastSuccess, obj.nAttempts); - } - - CAddrInfo(const CAddress &addrIn, const CNetAddr &addrSource) : CAddress(addrIn), source(addrSource) - { - } - - CAddrInfo() : CAddress(), source() - { - } - - //! Calculate in which "tried" bucket this entry belongs - int GetTriedBucket(const uint256 &nKey, const std::vector<bool> &asmap) const; - - //! Calculate in which "new" bucket this entry belongs, given a certain source - int GetNewBucket(const uint256 &nKey, const CNetAddr& src, const std::vector<bool> &asmap) const; - - //! Calculate in which "new" bucket this entry belongs, using its default source - int GetNewBucket(const uint256 &nKey, const std::vector<bool> &asmap) const - { - return GetNewBucket(nKey, source, asmap); - } - - //! Calculate in which position of a bucket to store this entry. - int GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const; - - //! Determine whether the statistics about this entry are bad enough so that it can just be deleted - bool IsTerrible(int64_t nNow = GetAdjustedTime()) const; - - //! Calculate the relative chance this entry should be given when selecting nodes to connect to - double GetChance(int64_t nNow = GetAdjustedTime()) const; -}; - /** Stochastic address manager * * Design goals: @@ -123,290 +51,64 @@ public: * * Several indexes are kept for high performance. Setting m_consistency_check_ratio with the -checkaddrman * configuration option will introduce (expensive) consistency checks for the entire data structure. */ +class AddrMan +{ + const std::unique_ptr<AddrManImpl> m_impl; -/** Total number of buckets for tried addresses */ -static constexpr int32_t ADDRMAN_TRIED_BUCKET_COUNT_LOG2{8}; -static constexpr int ADDRMAN_TRIED_BUCKET_COUNT{1 << ADDRMAN_TRIED_BUCKET_COUNT_LOG2}; - -/** Total number of buckets for new addresses */ -static constexpr int32_t ADDRMAN_NEW_BUCKET_COUNT_LOG2{10}; -static constexpr int ADDRMAN_NEW_BUCKET_COUNT{1 << ADDRMAN_NEW_BUCKET_COUNT_LOG2}; +public: + explicit AddrMan(std::vector<bool> asmap, bool deterministic, int32_t consistency_check_ratio); -/** Maximum allowed number of entries in buckets for new and tried addresses */ -static constexpr int32_t ADDRMAN_BUCKET_SIZE_LOG2{6}; -static constexpr int ADDRMAN_BUCKET_SIZE{1 << ADDRMAN_BUCKET_SIZE_LOG2}; + ~AddrMan(); -/** - * Stochastical (IP) address manager - */ -class CAddrMan -{ -public: template <typename Stream> - void Serialize(Stream& s_) const EXCLUSIVE_LOCKS_REQUIRED(!cs); + void Serialize(Stream& s_) const; template <typename Stream> - void Unserialize(Stream& s_) EXCLUSIVE_LOCKS_REQUIRED(!cs); - - explicit CAddrMan(std::vector<bool> asmap, bool deterministic, int32_t consistency_check_ratio); - - ~CAddrMan() - { - nKey.SetNull(); - } + void Unserialize(Stream& s_); //! Return the number of (unique) addresses in all tables. - size_t size() const - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); // TODO: Cache this in an atomic to avoid this overhead - return vRandom.size(); - } + size_t size() const; //! Add addresses to addrman's new table. - bool Add(const std::vector<CAddress> &vAddr, const CNetAddr& source, int64_t nTimePenalty = 0) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - int nAdd = 0; - Check(); - for (std::vector<CAddress>::const_iterator it = vAddr.begin(); it != vAddr.end(); it++) - nAdd += Add_(*it, source, nTimePenalty) ? 1 : 0; - Check(); - if (nAdd) { - LogPrint(BCLog::ADDRMAN, "Added %i addresses from %s: %i tried, %i new\n", nAdd, source.ToString(), nTried, nNew); - } - return nAdd > 0; - } + bool Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, int64_t nTimePenalty = 0); - //! Mark an entry as accessible. - void Good(const CService &addr, int64_t nTime = GetAdjustedTime()) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - Good_(addr, /* test_before_evict */ true, nTime); - Check(); - } + //! Mark an entry as accessible, possibly moving it from "new" to "tried". + void Good(const CService& addr, int64_t nTime = GetAdjustedTime()); //! Mark an entry as connection attempted to. - void Attempt(const CService &addr, bool fCountFailure, int64_t nTime = GetAdjustedTime()) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - Attempt_(addr, fCountFailure, nTime); - Check(); - } + void Attempt(const CService& addr, bool fCountFailure, int64_t nTime = GetAdjustedTime()); //! See if any to-be-evicted tried table entries have been tested and if so resolve the collisions. - void ResolveCollisions() - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - ResolveCollisions_(); - Check(); - } - - //! Randomly select an address in tried that another address is attempting to evict. - CAddrInfo SelectTriedCollision() - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - const CAddrInfo ret = SelectTriedCollision_(); - Check(); - return ret; - } + void ResolveCollisions(); /** - * Choose an address to connect to. + * Randomly select an address in the tried table that another address is + * attempting to evict. + * + * @return CAddress The record for the selected tried peer. + * int64_t The last time we attempted to connect to that peer. */ - CAddrInfo Select(bool newOnly = false) const - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - const CAddrInfo addrRet = Select_(newOnly); - Check(); - return addrRet; - } + std::pair<CAddress, int64_t> SelectTriedCollision(); /** - * Return all or many randomly selected addresses, optionally by network. + * Choose an address to connect to. * - * @param[in] max_addresses Maximum number of addresses to return (0 = all). - * @param[in] max_pct Maximum percentage of addresses to return (0 = all). - * @param[in] network Select only addresses of this network (nullopt = all). + * @param[in] newOnly Whether to only select addresses from the new table. + * @return CAddress The record for the selected peer. + * int64_t The last time we attempted to connect to that peer. */ - std::vector<CAddress> GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - std::vector<CAddress> vAddr; - GetAddr_(vAddr, max_addresses, max_pct, network); - Check(); - return vAddr; - } - - //! Outer function for Connected_() - void Connected(const CService &addr, int64_t nTime = GetAdjustedTime()) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - Connected_(addr, nTime); - Check(); - } - - void SetServices(const CService &addr, ServiceFlags nServices) - EXCLUSIVE_LOCKS_REQUIRED(!cs) - { - LOCK(cs); - Check(); - SetServices_(addr, nServices); - Check(); - } - - const std::vector<bool>& GetAsmap() const { return m_asmap; } - -private: - //! A mutex to protect the inner data structures. - mutable Mutex cs; - - //! Source of random numbers for randomization in inner loops - mutable FastRandomContext insecure_rand GUARDED_BY(cs); - - //! secret key to randomize bucket select with - uint256 nKey; - - //! Serialization versions. - enum Format : uint8_t { - V0_HISTORICAL = 0, //!< historic format, before commit e6b343d88 - V1_DETERMINISTIC = 1, //!< for pre-asmap files - V2_ASMAP = 2, //!< for files including asmap version - V3_BIP155 = 3, //!< same as V2_ASMAP plus addresses are in BIP155 format - }; - - //! The maximum format this software knows it can unserialize. Also, we always serialize - //! in this format. - //! The format (first byte in the serialized stream) can be higher than this and - //! still this software may be able to unserialize the file - if the second byte - //! (see `lowest_compatible` in `Unserialize()`) is less or equal to this. - static constexpr Format FILE_FORMAT = Format::V3_BIP155; - - //! The initial value of a field that is incremented every time an incompatible format - //! change is made (such that old software versions would not be able to parse and - //! understand the new file format). This is 32 because we overtook the "key size" - //! field which was 32 historically. - //! @note Don't increment this. Increment `lowest_compatible` in `Serialize()` instead. - static constexpr uint8_t INCOMPATIBILITY_BASE = 32; - - //! last used nId - int nIdCount GUARDED_BY(cs){0}; - - //! table with information about all nIds - std::unordered_map<int, CAddrInfo> mapInfo GUARDED_BY(cs); - - //! find an nId based on its network address - std::unordered_map<CNetAddr, int, CNetAddrHash> mapAddr GUARDED_BY(cs); - - //! randomly-ordered vector of all nIds - //! This is mutable because it is unobservable outside the class, so any - //! changes to it (even in const methods) are also unobservable. - mutable std::vector<int> vRandom GUARDED_BY(cs); - - // number of "tried" entries - int nTried GUARDED_BY(cs){0}; - - //! list of "tried" buckets - int vvTried[ADDRMAN_TRIED_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); - - //! number of (unique) "new" entries - int nNew GUARDED_BY(cs){0}; - - //! list of "new" buckets - int vvNew[ADDRMAN_NEW_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); - - //! last time Good was called (memory only). Initially set to 1 so that "never" is strictly worse. - int64_t nLastGood GUARDED_BY(cs){1}; - - //! Holds addrs inserted into tried table that collide with existing entries. Test-before-evict discipline used to resolve these collisions. - std::set<int> m_tried_collisions; - - /** Perform consistency checks every m_consistency_check_ratio operations (if non-zero). */ - const int32_t m_consistency_check_ratio; - - // Compressed IP->ASN mapping, loaded from a file when a node starts. - // Should be always empty if no file was provided. - // This mapping is then used for bucketing nodes in Addrman. - // - // If asmap is provided, nodes will be bucketed by - // AS they belong to, in order to make impossible for a node - // to connect to several nodes hosted in a single AS. - // This is done in response to Erebus attack, but also to generally - // diversify the connections every node creates, - // especially useful when a large fraction of nodes - // operate under a couple of cloud providers. - // - // If a new asmap was provided, the existing records - // would be re-bucketed accordingly. - const std::vector<bool> m_asmap; - - //! Find an entry. - CAddrInfo* Find(const CNetAddr& addr, int *pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Create a new entry and add it to the internal data structures mapInfo, mapAddr and vRandom. - CAddrInfo* Create(const CAddress &addr, const CNetAddr &addrSource, int *pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Swap two elements in vRandom. - void SwapRandom(unsigned int nRandomPos1, unsigned int nRandomPos2) const EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Move an entry from the "new" table(s) to the "tried" table - void MakeTried(CAddrInfo& info, int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Delete an entry. It must not be in tried, and have refcount 0. - void Delete(int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Clear a position in a "new" table. This is the only place where entries are actually deleted. - void ClearNew(int nUBucket, int nUBucketPos) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Mark an entry "good", possibly moving it from "new" to "tried". - void Good_(const CService &addr, bool test_before_evict, int64_t time) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Add an entry to the "new" table. - bool Add_(const CAddress &addr, const CNetAddr& source, int64_t nTimePenalty) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Mark an entry as attempted to connect. - void Attempt_(const CService &addr, bool fCountFailure, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Select an address to connect to, if newOnly is set to true, only the new table is selected from. - CAddrInfo Select_(bool newOnly) const EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! See if any to-be-evicted tried table entries have been tested and if so resolve the collisions. - void ResolveCollisions_() EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Return a random to-be-evicted tried table address. - CAddrInfo SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Consistency check, taking into account m_consistency_check_ratio. Will std::abort if an inconsistency is detected. - void Check() const EXCLUSIVE_LOCKS_REQUIRED(cs); - - //! Perform consistency check, regardless of m_consistency_check_ratio. - //! @returns an error code or zero. - int ForceCheckAddrman() const EXCLUSIVE_LOCKS_REQUIRED(cs); + std::pair<CAddress, int64_t> Select(bool newOnly = false) const; /** * Return all or many randomly selected addresses, optionally by network. * - * @param[out] vAddr Vector of randomly selected addresses from vRandom. * @param[in] max_addresses Maximum number of addresses to return (0 = all). * @param[in] max_pct Maximum percentage of addresses to return (0 = all). * @param[in] network Select only addresses of this network (nullopt = all). + * + * @return A vector of randomly selected addresses from vRandom. */ - void GetAddr_(std::vector<CAddress>& vAddr, size_t max_addresses, size_t max_pct, std::optional<Network> network) const EXCLUSIVE_LOCKS_REQUIRED(cs); + std::vector<CAddress> GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const; /** We have successfully connected to this peer. Calling this function * updates the CAddress's nTime, which is used in our IsTerrible() @@ -419,13 +121,15 @@ private: * @param[in] addr The address of the peer we were connected to * @param[in] nTime The time that we were last connected to this peer */ - void Connected_(const CService& addr, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); + void Connected(const CService& addr, int64_t nTime = GetAdjustedTime()); //! Update an entry's service bits. - void SetServices_(const CService &addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); + void SetServices(const CService& addr, ServiceFlags nServices); + + const std::vector<bool>& GetAsmap() const; - friend class CAddrManTest; - friend class CAddrManDeterministic; + friend class AddrManTest; + friend class AddrManDeterministic; }; #endif // BITCOIN_ADDRMAN_H diff --git a/src/addrman_impl.h b/src/addrman_impl.h new file mode 100644 index 0000000000..1dc7f25f9c --- /dev/null +++ b/src/addrman_impl.h @@ -0,0 +1,271 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_ADDRMAN_IMPL_H +#define BITCOIN_ADDRMAN_IMPL_H + +#include <logging.h> +#include <netaddress.h> +#include <protocol.h> +#include <serialize.h> +#include <sync.h> +#include <uint256.h> + +#include <cstdint> +#include <optional> +#include <set> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +/** Total number of buckets for tried addresses */ +static constexpr int32_t ADDRMAN_TRIED_BUCKET_COUNT_LOG2{8}; +static constexpr int ADDRMAN_TRIED_BUCKET_COUNT{1 << ADDRMAN_TRIED_BUCKET_COUNT_LOG2}; +/** Total number of buckets for new addresses */ +static constexpr int32_t ADDRMAN_NEW_BUCKET_COUNT_LOG2{10}; +static constexpr int ADDRMAN_NEW_BUCKET_COUNT{1 << ADDRMAN_NEW_BUCKET_COUNT_LOG2}; +/** Maximum allowed number of entries in buckets for new and tried addresses */ +static constexpr int32_t ADDRMAN_BUCKET_SIZE_LOG2{6}; +static constexpr int ADDRMAN_BUCKET_SIZE{1 << ADDRMAN_BUCKET_SIZE_LOG2}; + +/** + * Extended statistics about a CAddress + */ +class AddrInfo : public CAddress +{ +public: + //! last try whatsoever by us (memory only) + int64_t nLastTry{0}; + + //! last counted attempt (memory only) + int64_t nLastCountAttempt{0}; + + //! where knowledge about this address first came from + CNetAddr source; + + //! last successful connection by us + int64_t nLastSuccess{0}; + + //! connection attempts since last successful attempt + int nAttempts{0}; + + //! reference count in new sets (memory only) + int nRefCount{0}; + + //! in tried set? (memory only) + bool fInTried{false}; + + //! position in vRandom + mutable int nRandomPos{-1}; + + SERIALIZE_METHODS(AddrInfo, obj) + { + READWRITEAS(CAddress, obj); + READWRITE(obj.source, obj.nLastSuccess, obj.nAttempts); + } + + AddrInfo(const CAddress &addrIn, const CNetAddr &addrSource) : CAddress(addrIn), source(addrSource) + { + } + + AddrInfo() : CAddress(), source() + { + } + + //! Calculate in which "tried" bucket this entry belongs + int GetTriedBucket(const uint256 &nKey, const std::vector<bool> &asmap) const; + + //! Calculate in which "new" bucket this entry belongs, given a certain source + int GetNewBucket(const uint256 &nKey, const CNetAddr& src, const std::vector<bool> &asmap) const; + + //! Calculate in which "new" bucket this entry belongs, using its default source + int GetNewBucket(const uint256 &nKey, const std::vector<bool> &asmap) const + { + return GetNewBucket(nKey, source, asmap); + } + + //! Calculate in which position of a bucket to store this entry. + int GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const; + + //! Determine whether the statistics about this entry are bad enough so that it can just be deleted + bool IsTerrible(int64_t nNow = GetAdjustedTime()) const; + + //! Calculate the relative chance this entry should be given when selecting nodes to connect to + double GetChance(int64_t nNow = GetAdjustedTime()) const; +}; + +class AddrManImpl +{ +public: + AddrManImpl(std::vector<bool>&& asmap, bool deterministic, int32_t consistency_check_ratio); + + ~AddrManImpl(); + + template <typename Stream> + void Serialize(Stream& s_) const EXCLUSIVE_LOCKS_REQUIRED(!cs); + + template <typename Stream> + void Unserialize(Stream& s_) EXCLUSIVE_LOCKS_REQUIRED(!cs); + + size_t size() const EXCLUSIVE_LOCKS_REQUIRED(!cs); + + bool Add(const std::vector<CAddress>& vAddr, const CNetAddr& source, int64_t nTimePenalty) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void Good(const CService& addr, int64_t nTime) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void Attempt(const CService& addr, bool fCountFailure, int64_t nTime) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void ResolveCollisions() EXCLUSIVE_LOCKS_REQUIRED(!cs); + + std::pair<CAddress, int64_t> SelectTriedCollision() EXCLUSIVE_LOCKS_REQUIRED(!cs); + + std::pair<CAddress, int64_t> Select(bool newOnly) const + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + std::vector<CAddress> GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void Connected(const CService& addr, int64_t nTime) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + void SetServices(const CService& addr, ServiceFlags nServices) + EXCLUSIVE_LOCKS_REQUIRED(!cs); + + const std::vector<bool>& GetAsmap() const; + + friend class AddrManTest; + friend class AddrManDeterministic; + +private: + //! A mutex to protect the inner data structures. + mutable Mutex cs; + + //! Source of random numbers for randomization in inner loops + mutable FastRandomContext insecure_rand GUARDED_BY(cs); + + //! secret key to randomize bucket select with + uint256 nKey; + + //! Serialization versions. + enum Format : uint8_t { + V0_HISTORICAL = 0, //!< historic format, before commit e6b343d88 + V1_DETERMINISTIC = 1, //!< for pre-asmap files + V2_ASMAP = 2, //!< for files including asmap version + V3_BIP155 = 3, //!< same as V2_ASMAP plus addresses are in BIP155 format + }; + + //! The maximum format this software knows it can unserialize. Also, we always serialize + //! in this format. + //! The format (first byte in the serialized stream) can be higher than this and + //! still this software may be able to unserialize the file - if the second byte + //! (see `lowest_compatible` in `Unserialize()`) is less or equal to this. + static constexpr Format FILE_FORMAT = Format::V3_BIP155; + + //! The initial value of a field that is incremented every time an incompatible format + //! change is made (such that old software versions would not be able to parse and + //! understand the new file format). This is 32 because we overtook the "key size" + //! field which was 32 historically. + //! @note Don't increment this. Increment `lowest_compatible` in `Serialize()` instead. + static constexpr uint8_t INCOMPATIBILITY_BASE = 32; + + //! last used nId + int nIdCount GUARDED_BY(cs){0}; + + //! table with information about all nIds + std::unordered_map<int, AddrInfo> mapInfo GUARDED_BY(cs); + + //! find an nId based on its network address + std::unordered_map<CNetAddr, int, CNetAddrHash> mapAddr GUARDED_BY(cs); + + //! randomly-ordered vector of all nIds + //! This is mutable because it is unobservable outside the class, so any + //! changes to it (even in const methods) are also unobservable. + mutable std::vector<int> vRandom GUARDED_BY(cs); + + // number of "tried" entries + int nTried GUARDED_BY(cs){0}; + + //! list of "tried" buckets + int vvTried[ADDRMAN_TRIED_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); + + //! number of (unique) "new" entries + int nNew GUARDED_BY(cs){0}; + + //! list of "new" buckets + int vvNew[ADDRMAN_NEW_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); + + //! last time Good was called (memory only). Initially set to 1 so that "never" is strictly worse. + int64_t nLastGood GUARDED_BY(cs){1}; + + //! Holds addrs inserted into tried table that collide with existing entries. Test-before-evict discipline used to resolve these collisions. + std::set<int> m_tried_collisions; + + /** Perform consistency checks every m_consistency_check_ratio operations (if non-zero). */ + const int32_t m_consistency_check_ratio; + + // Compressed IP->ASN mapping, loaded from a file when a node starts. + // Should be always empty if no file was provided. + // This mapping is then used for bucketing nodes in Addrman. + // + // If asmap is provided, nodes will be bucketed by + // AS they belong to, in order to make impossible for a node + // to connect to several nodes hosted in a single AS. + // This is done in response to Erebus attack, but also to generally + // diversify the connections every node creates, + // especially useful when a large fraction of nodes + // operate under a couple of cloud providers. + // + // If a new asmap was provided, the existing records + // would be re-bucketed accordingly. + const std::vector<bool> m_asmap; + + //! Find an entry. + AddrInfo* Find(const CNetAddr& addr, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Create a new entry and add it to the internal data structures mapInfo, mapAddr and vRandom. + AddrInfo* Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Swap two elements in vRandom. + void SwapRandom(unsigned int nRandomPos1, unsigned int nRandomPos2) const EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Delete an entry. It must not be in tried, and have refcount 0. + void Delete(int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Clear a position in a "new" table. This is the only place where entries are actually deleted. + void ClearNew(int nUBucket, int nUBucketPos) EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Move an entry from the "new" table(s) to the "tried" table + void MakeTried(AddrInfo& info, int nId) EXCLUSIVE_LOCKS_REQUIRED(cs); + + void Good_(const CService& addr, bool test_before_evict, int64_t time) EXCLUSIVE_LOCKS_REQUIRED(cs); + + bool Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimePenalty) EXCLUSIVE_LOCKS_REQUIRED(cs); + + void Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); + + std::pair<CAddress, int64_t> Select_(bool newOnly) const EXCLUSIVE_LOCKS_REQUIRED(cs); + + std::vector<CAddress> GetAddr_(size_t max_addresses, size_t max_pct, std::optional<Network> network) const EXCLUSIVE_LOCKS_REQUIRED(cs); + + void Connected_(const CService& addr, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); + + void SetServices_(const CService& addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); + + void ResolveCollisions_() EXCLUSIVE_LOCKS_REQUIRED(cs); + + std::pair<CAddress, int64_t> SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Consistency check, taking into account m_consistency_check_ratio. Will std::abort if an inconsistency is detected. + void Check() const EXCLUSIVE_LOCKS_REQUIRED(cs); + + //! Perform consistency check, regardless of m_consistency_check_ratio. + //! @returns an error code or zero. + int ForceCheckAddrman() const EXCLUSIVE_LOCKS_REQUIRED(cs); +}; + +#endif // BITCOIN_ADDRMAN_IMPL_H diff --git a/src/banman.h b/src/banman.h index 8a03a9e3fc..f495dab49d 100644 --- a/src/banman.h +++ b/src/banman.h @@ -6,7 +6,7 @@ #define BITCOIN_BANMAN_H #include <addrdb.h> -#include <bloom.h> +#include <common/bloom.h> #include <fs.h> #include <net_types.h> // For banmap_t #include <sync.h> diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index bebf86a09d..d6834a239b 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -53,14 +53,14 @@ static void CreateAddresses() } } -static void AddAddressesToAddrMan(CAddrMan& addrman) +static void AddAddressesToAddrMan(AddrMan& addrman) { for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { addrman.Add(g_addresses[source_i], g_sources[source_i]); } } -static void FillAddrMan(CAddrMan& addrman) +static void FillAddrMan(AddrMan& addrman) { CreateAddresses(); @@ -74,26 +74,26 @@ static void AddrManAdd(benchmark::Bench& bench) CreateAddresses(); bench.run([&] { - CAddrMan addrman{/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0}; + AddrMan addrman{/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0}; AddAddressesToAddrMan(addrman); }); } static void AddrManSelect(benchmark::Bench& bench) { - CAddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + AddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); FillAddrMan(addrman); bench.run([&] { const auto& address = addrman.Select(); - assert(address.GetPort() > 0); + assert(address.first.GetPort() > 0); }); } static void AddrManGetAddr(benchmark::Bench& bench) { - CAddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + AddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); FillAddrMan(addrman); @@ -105,7 +105,7 @@ static void AddrManGetAddr(benchmark::Bench& bench) static void AddrManAddThenGood(benchmark::Bench& bench) { - auto markSomeAsGood = [](CAddrMan& addrman) { + auto markSomeAsGood = [](AddrMan& addrman) { for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { for (size_t addr_i = 0; addr_i < NUM_ADDRESSES_PER_SOURCE; ++addr_i) { addrman.Good(g_addresses[source_i][addr_i]); @@ -117,12 +117,12 @@ static void AddrManAddThenGood(benchmark::Bench& bench) bench.run([&] { // To make the benchmark independent of the number of evaluations, we always prepare a new addrman. - // This is necessary because CAddrMan::Good() method modifies the object, affecting the timing of subsequent calls + // This is necessary because AddrMan::Good() method modifies the object, affecting the timing of subsequent calls // to the same method and we want to do the same amount of work in every loop iteration. // // This has some overhead (exactly the result of AddrManAdd benchmark), but that overhead is constant so improvements in - // CAddrMan::Good() will still be noticeable. - CAddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + // AddrMan::Good() will still be noticeable. + AddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); AddAddressesToAddrMan(addrman); markSomeAsGood(addrman); diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 934b574f8b..fd5145950b 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -33,7 +33,6 @@ static void CoinSelection(benchmark::Bench& bench) NodeContext node; auto chain = interfaces::MakeChain(node); CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); - wallet.SetupLegacyScriptPubKeyMan(); std::vector<std::unique_ptr<CWalletTx>> wtxs; LOCK(wallet.cs_wallet); @@ -65,10 +64,6 @@ static void CoinSelection(benchmark::Bench& bench) } typedef std::set<CInputCoin> CoinSet; -static NodeContext testNode; -static auto testChain = interfaces::MakeChain(testNode); -static CWallet testWallet(testChain.get(), "", CreateDummyWalletDatabase()); -std::vector<std::unique_ptr<CWalletTx>> wtxn; // Copied from src/wallet/test/coinselector_tests.cpp static void add_coin(const CAmount& nValue, int nInput, std::vector<OutputGroup>& set) @@ -76,10 +71,9 @@ static void add_coin(const CAmount& nValue, int nInput, std::vector<OutputGroup> CMutableTransaction tx; tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; - std::unique_ptr<CWalletTx> wtx = std::make_unique<CWalletTx>(MakeTransactionRef(std::move(tx))); + CInputCoin coin(MakeTransactionRef(tx), nInput); set.emplace_back(); - set.back().Insert(COutput(testWallet, *wtx, nInput, 0, true, true, true).GetInputCoin(), 0, true, 0, 0, false); - wtxn.emplace_back(std::move(wtx)); + set.back().Insert(coin, 0, true, 0, 0, false); } // Copied from src/wallet/test/coinselector_tests.cpp static CAmount make_hard_case(int utxos, std::vector<OutputGroup>& utxo_pool) @@ -97,7 +91,6 @@ static CAmount make_hard_case(int utxos, std::vector<OutputGroup>& utxo_pool) static void BnBExhaustion(benchmark::Bench& bench) { // Setup - testWallet.SetupLegacyScriptPubKeyMan(); std::vector<OutputGroup> utxo_pool; CoinSet selection; CAmount value_ret = 0; diff --git a/src/bench/rollingbloom.cpp b/src/bench/rollingbloom.cpp index 28167767db..30bc1d5fdf 100644 --- a/src/bench/rollingbloom.cpp +++ b/src/bench/rollingbloom.cpp @@ -4,7 +4,7 @@ #include <bench/bench.h> -#include <bloom.h> +#include <common/bloom.h> static void RollingBloom(benchmark::Bench& bench) { diff --git a/src/bench/rpc_blockchain.cpp b/src/bench/rpc_blockchain.cpp index c8886a4c23..3bef64f720 100644 --- a/src/bench/rpc_blockchain.cpp +++ b/src/bench/rpc_blockchain.cpp @@ -40,7 +40,7 @@ static void BlockToJsonVerbose(benchmark::Bench& bench) { TestBlockAndIndex data; bench.run([&] { - auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, /*verbose*/ true); + auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, TxVerbosity::SHOW_DETAILS_AND_PREVOUT); ankerl::nanobench::doNotOptimizeAway(univalue); }); } @@ -50,7 +50,7 @@ BENCHMARK(BlockToJsonVerbose); static void BlockToJsonVerboseWrite(benchmark::Bench& bench) { TestBlockAndIndex data; - auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, /*verbose*/ true); + auto univalue = blockToJSON(data.block, &data.blockindex, &data.blockindex, TxVerbosity::SHOW_DETAILS_AND_PREVOUT); bench.run([&] { auto str = univalue.write(); ankerl::nanobench::doNotOptimizeAway(str); diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index a205d8b6e7..166ed16042 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -14,7 +14,7 @@ #include <optional> -static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const bool add_watchonly, const bool add_mine) +static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const bool add_mine) { const auto test_setup = MakeNoLogFileContext<const TestingSetup>(); @@ -22,13 +22,14 @@ static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const b CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockWalletDatabase()}; { - wallet.SetupLegacyScriptPubKeyMan(); + LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet.SetupDescriptorScriptPubKeyMans(); if (wallet.LoadWallet() != DBErrors::LOAD_OK) assert(false); } auto handler = test_setup->m_node.chain->handleNotifications({&wallet, [](CWallet*) {}}); const std::optional<std::string> address_mine{add_mine ? std::optional<std::string>{getnewaddress(wallet)} : std::nullopt}; - if (add_watchonly) importaddress(wallet, ADDRESS_WATCHONLY); for (int i = 0; i < 100; ++i) { generatetoaddress(test_setup->m_node, address_mine.value_or(ADDRESS_WATCHONLY)); @@ -42,14 +43,13 @@ static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const b if (set_dirty) wallet.MarkDirty(); bal = GetBalance(wallet); if (add_mine) assert(bal.m_mine_trusted > 0); - if (add_watchonly) assert(bal.m_watchonly_trusted > 0); }); } -static void WalletBalanceDirty(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ true, /* add_watchonly */ true, /* add_mine */ true); } -static void WalletBalanceClean(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_watchonly */ true, /* add_mine */ true); } -static void WalletBalanceMine(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_watchonly */ false, /* add_mine */ true); } -static void WalletBalanceWatch(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_watchonly */ true, /* add_mine */ false); } +static void WalletBalanceDirty(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ true, /* add_mine */ true); } +static void WalletBalanceClean(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_mine */ true); } +static void WalletBalanceMine(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_mine */ true); } +static void WalletBalanceWatch(benchmark::Bench& bench) { WalletBalance(bench, /* set_dirty */ false, /* add_mine */ false); } BENCHMARK(WalletBalanceDirty); BENCHMARK(WalletBalanceClean); diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 3c22ee0f67..b6344ec413 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -551,15 +551,26 @@ public: } // Report peer connection totals by type. - result += " ipv4 ipv6 onion"; - const bool any_i2p_peers = m_counts.at(2).at(3); // false if total i2p peers count is 0, otherwise true - if (any_i2p_peers) result += " i2p"; + result += " "; + std::vector<int8_t> reachable_networks; + for (const UniValue& network : networkinfo["networks"].getValues()) { + if (network["reachable"].get_bool()) { + const std::string& network_name{network["name"].get_str()}; + const int8_t network_id{NetworkStringToId(network_name)}; + if (network_id == UNKNOWN_NETWORK) continue; + result += strprintf("%8s", network_name); // column header + reachable_networks.push_back(network_id); + } + }; result += " total block"; if (m_manual_peers_count) result += " manual"; + const std::array rows{"in", "out", "total"}; - for (uint8_t i = 0; i < 3; ++i) { - result += strprintf("\n%-5s %5i %5i %5i", rows.at(i), m_counts.at(i).at(0), m_counts.at(i).at(1), m_counts.at(i).at(2)); // ipv4/ipv6/onion peers counts - if (any_i2p_peers) result += strprintf(" %5i", m_counts.at(i).at(3)); // i2p peers count + for (size_t i = 0; i < rows.size(); ++i) { + result += strprintf("\n%-5s", rows[i]); // row header + for (int8_t n : reachable_networks) { + result += strprintf("%8i", m_counts.at(i).at(n)); // network peers count + } result += strprintf(" %5i", m_counts.at(i).at(m_networks.size())); // total peers count if (i == 1) { // the outbound row has two extra columns for block relay and manual peer counts result += strprintf(" %5i", m_block_relay_peers_count); @@ -786,7 +797,7 @@ static UniValue CallRPC(BaseRequestHandler* rh, const std::string& strMethod, co if (failedToGetAuthCookie) { throw std::runtime_error(strprintf( "Could not locate RPC credentials. No authentication cookie could be found, and RPC password is not set. See -rpcpassword and -stdinrpcpass. Configuration file: (%s)", - GetConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME)).string())); + fs::PathToString(GetConfigFile(gArgs.GetArg("-conf", BITCOIN_CONF_FILENAME))))); } else { throw std::runtime_error("Authorization failed: Incorrect rpcuser or rpcpassword"); } diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 98916460aa..eb97cfc6f6 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -8,6 +8,7 @@ #include <clientversion.h> #include <coins.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <core_io.h> #include <key_io.h> @@ -234,6 +235,16 @@ static void MutateTxRBFOptIn(CMutableTransaction& tx, const std::string& strInId } } +template <typename T> +static T TrimAndParse(const std::string& int_str, const std::string& err) +{ + const auto parsed{ToIntegral<T>(TrimString(int_str))}; + if (!parsed.has_value()) { + throw std::runtime_error(err + " '" + int_str + "'"); + } + return parsed.value(); +} + static void MutateTxAddInput(CMutableTransaction& tx, const std::string& strInput) { std::vector<std::string> vStrInputParts; @@ -260,8 +271,9 @@ static void MutateTxAddInput(CMutableTransaction& tx, const std::string& strInpu // extract the optional sequence number uint32_t nSequenceIn = CTxIn::SEQUENCE_FINAL; - if (vStrInputParts.size() > 2) - nSequenceIn = std::stoul(vStrInputParts[2]); + if (vStrInputParts.size() > 2) { + nSequenceIn = TrimAndParse<uint32_t>(vStrInputParts.at(2), "invalid TX sequence id"); + } // append to transaction input list CTxIn txin(txid, vout, CScript(), nSequenceIn); @@ -351,10 +363,10 @@ static void MutateTxAddOutMultiSig(CMutableTransaction& tx, const std::string& s CAmount value = ExtractAndValidateValue(vStrInputParts[0]); // Extract REQUIRED - uint32_t required = stoul(vStrInputParts[1]); + const uint32_t required{TrimAndParse<uint32_t>(vStrInputParts.at(1), "invalid multisig required number")}; // Extract NUMKEYS - uint32_t numkeys = stoul(vStrInputParts[2]); + const uint32_t numkeys{TrimAndParse<uint32_t>(vStrInputParts.at(2), "invalid multisig total number")}; // Validate there are the correct number of pubkeys if (vStrInputParts.size() < numkeys + 3) diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 765954c92e..21d4df5b01 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -30,6 +30,7 @@ static void SetupWalletToolArgs(ArgsManager& argsman) argsman.AddArg("-dumpfile=<file name>", "When used with 'dump', writes out the records to this file. When used with 'createfromdump', loads the records into a new wallet.", ArgsManager::ALLOW_STRING, OptionsCategory::OPTIONS); argsman.AddArg("-debug=<category>", "Output debugging information (default: 0).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-descriptors", "Create descriptors wallet. Only for 'create'", ArgsManager::ALLOW_BOOL, OptionsCategory::OPTIONS); + argsman.AddArg("-legacy", "Create legacy wallet. Only for 'create'", ArgsManager::ALLOW_BOOL, OptionsCategory::OPTIONS); argsman.AddArg("-format=<format>", "The format of the wallet file to create. Either \"bdb\" or \"sqlite\". Only used with 'createfromdump'", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-printtoconsole", "Send trace/debug info to console (default: 1 when no -debug is true, 0 otherwise).", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 654679af27..25ec2809e9 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -19,6 +19,7 @@ #include <shutdown.h> #include <util/check.h> #include <util/strencodings.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <util/threadnames.h> #include <util/tokenpipe.h> @@ -238,6 +239,7 @@ static bool AppInit(NodeContext& node, int argc, char* argv[]) daemon_ep.Close(); } #endif + SetSyscallSandboxPolicy(SyscallSandboxPolicy::SHUTOFF); if (fRet) { WaitForShutdown(); } diff --git a/src/chain.h b/src/chain.h index 365a7f79b6..15ca8f8750 100644 --- a/src/chain.h +++ b/src/chain.h @@ -178,7 +178,7 @@ public: //! (memory only) Number of transactions in the chain up to and including this block. //! This value will be non-zero only if and only if transactions for this block and all its parents are available. - //! Change to 64-bit type when necessary; won't happen before 2030 + //! Change to 64-bit type before 2024 (assuming worst case of 60 byte transactions). //! //! Note: this value is faked during use of a UTXO snapshot because we don't //! have the underlying block data available during snapshot load. diff --git a/src/chainparams.cpp b/src/chainparams.cpp index b155745794..2e823c1211 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -118,15 +118,15 @@ public: // This is fine at runtime as we'll fall back to using them as an addrfetch if they don't support the // service bits we want, but we should get them updated to support all service bits wanted by any // release ASAP to avoid it where possible. - vSeeds.emplace_back("seed.bitcoin.sipa.be"); // Pieter Wuille, only supports x1, x5, x9, and xd - vSeeds.emplace_back("dnsseed.bluematt.me"); // Matt Corallo, only supports x9 - vSeeds.emplace_back("dnsseed.bitcoin.dashjr.org"); // Luke Dashjr - vSeeds.emplace_back("seed.bitcoinstats.com"); // Christian Decker, supports x1 - xf - vSeeds.emplace_back("seed.bitcoin.jonasschnelli.ch"); // Jonas Schnelli, only supports x1, x5, x9, and xd - vSeeds.emplace_back("seed.btc.petertodd.org"); // Peter Todd, only supports x1, x5, x9, and xd - vSeeds.emplace_back("seed.bitcoin.sprovoost.nl"); // Sjors Provoost - vSeeds.emplace_back("dnsseed.emzy.de"); // Stephan Oeste - vSeeds.emplace_back("seed.bitcoin.wiz.biz"); // Jason Maurice + vSeeds.emplace_back("seed.bitcoin.sipa.be."); // Pieter Wuille, only supports x1, x5, x9, and xd + vSeeds.emplace_back("dnsseed.bluematt.me."); // Matt Corallo, only supports x9 + vSeeds.emplace_back("dnsseed.bitcoin.dashjr.org."); // Luke Dashjr + vSeeds.emplace_back("seed.bitcoinstats.com."); // Christian Decker, supports x1 - xf + vSeeds.emplace_back("seed.bitcoin.jonasschnelli.ch."); // Jonas Schnelli, only supports x1, x5, x9, and xd + vSeeds.emplace_back("seed.btc.petertodd.org."); // Peter Todd, only supports x1, x5, x9, and xd + vSeeds.emplace_back("seed.bitcoin.sprovoost.nl."); // Sjors Provoost + vSeeds.emplace_back("dnsseed.emzy.de."); // Stephan Oeste + vSeeds.emplace_back("seed.bitcoin.wiz.biz."); // Jason Maurice base58Prefixes[PUBKEY_ADDRESS] = std::vector<unsigned char>(1,0); base58Prefixes[SCRIPT_ADDRESS] = std::vector<unsigned char>(1,5); @@ -230,10 +230,10 @@ public: vFixedSeeds.clear(); vSeeds.clear(); // nodes with support for servicebits filtering should be at the top - vSeeds.emplace_back("testnet-seed.bitcoin.jonasschnelli.ch"); - vSeeds.emplace_back("seed.tbtc.petertodd.org"); - vSeeds.emplace_back("seed.testnet.bitcoin.sprovoost.nl"); - vSeeds.emplace_back("testnet-seed.bluematt.me"); // Just a static list of stable node(s), only supports x9 + vSeeds.emplace_back("testnet-seed.bitcoin.jonasschnelli.ch."); + vSeeds.emplace_back("seed.tbtc.petertodd.org."); + vSeeds.emplace_back("seed.testnet.bitcoin.sprovoost.nl."); + vSeeds.emplace_back("testnet-seed.bluematt.me."); // Just a static list of stable node(s), only supports x9 base58Prefixes[PUBKEY_ADDRESS] = std::vector<unsigned char>(1,111); base58Prefixes[SCRIPT_ADDRESS] = std::vector<unsigned char>(1,196); @@ -280,7 +280,7 @@ public: if (!args.IsArgSet("-signetchallenge")) { bin = ParseHex("512103ad5e0edad18cb1f0fc0d28a3d4f1f3e445640337489abb10404f2d1e086be430210359ef5021964fe22d6f8e05b2463c9540ce96883fe3b278760f048f5189f2e6c452ae"); - vSeeds.emplace_back("seed.signet.bitcoin.sprovoost.nl"); + vSeeds.emplace_back("seed.signet.bitcoin.sprovoost.nl."); // Hardcoded nodes can be removed once there are more DNS seeds vSeeds.emplace_back("178.128.221.177"); diff --git a/src/checkqueue.h b/src/checkqueue.h index 4ceeb3600a..7c20e2013c 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -7,6 +7,7 @@ #include <sync.h> #include <tinyformat.h> +#include <util/syscall_sandbox.h> #include <util/threadnames.h> #include <algorithm> @@ -151,6 +152,7 @@ public: for (int n = 0; n < threads_num; ++n) { m_worker_threads.emplace_back([this, n]() { util::ThreadRename(strprintf("scriptch.%i", n)); + SetSyscallSandboxPolicy(SyscallSandboxPolicy::VALIDATION_SCRIPT_CHECK); Loop(false /* worker thread */); }); } diff --git a/src/bloom.cpp b/src/common/bloom.cpp index 15e06389de..26b70b4d14 100644 --- a/src/bloom.cpp +++ b/src/common/bloom.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <bloom.h> +#include <common/bloom.h> #include <hash.h> #include <primitives/transaction.h> @@ -82,7 +82,7 @@ bool CBloomFilter::contains(const COutPoint& outpoint) const { CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); stream << outpoint; - return contains(MakeUCharSpan(stream)); + return contains(stream); } bool CBloomFilter::IsWithinSizeConstraints() const diff --git a/src/bloom.h b/src/common/bloom.h index 422646d8b9..25c16fbfe2 100644 --- a/src/bloom.h +++ b/src/common/bloom.h @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_BLOOM_H -#define BITCOIN_BLOOM_H +#ifndef BITCOIN_COMMON_BLOOM_H +#define BITCOIN_COMMON_BLOOM_H #include <serialize.h> #include <span.h> @@ -124,4 +124,4 @@ private: int nHashFuncs; }; -#endif // BITCOIN_BLOOM_H +#endif // BITCOIN_COMMON_BLOOM_H diff --git a/src/amount.h b/src/consensus/amount.h index 47968e80b1..96566ea13f 100644 --- a/src/amount.h +++ b/src/consensus/amount.h @@ -3,15 +3,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_AMOUNT_H -#define BITCOIN_AMOUNT_H +#ifndef BITCOIN_CONSENSUS_AMOUNT_H +#define BITCOIN_CONSENSUS_AMOUNT_H -#include <stdint.h> +#include <cstdint> /** Amount in satoshis (Can be negative) */ typedef int64_t CAmount; -static const CAmount COIN = 100000000; +/** The amount of satoshis in one BTC. */ +static constexpr CAmount COIN = 100000000; /** No amount larger than this (in satoshi) is valid. * @@ -22,7 +23,7 @@ static const CAmount COIN = 100000000; * critical; in unusual circumstances like a(nother) overflow bug that allowed * for the creation of coins out of thin air modification could lead to a fork. * */ -static const CAmount MAX_MONEY = 21000000 * COIN; +static constexpr CAmount MAX_MONEY = 21000000 * COIN; inline bool MoneyRange(const CAmount& nValue) { return (nValue >= 0 && nValue <= MAX_MONEY); } -#endif // BITCOIN_AMOUNT_H +#endif // BITCOIN_CONSENSUS_AMOUNT_H diff --git a/src/consensus/tx_check.cpp b/src/consensus/tx_check.cpp index bb8cd10c63..de4824fadc 100644 --- a/src/consensus/tx_check.cpp +++ b/src/consensus/tx_check.cpp @@ -4,6 +4,7 @@ #include <consensus/tx_check.h> +#include <consensus/amount.h> #include <primitives/transaction.h> #include <consensus/validation.h> diff --git a/src/consensus/tx_verify.cpp b/src/consensus/tx_verify.cpp index 0ab790ccdc..a07adae536 100644 --- a/src/consensus/tx_verify.cpp +++ b/src/consensus/tx_verify.cpp @@ -4,6 +4,7 @@ #include <consensus/tx_verify.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <primitives/transaction.h> #include <script/interpreter.h> diff --git a/src/consensus/tx_verify.h b/src/consensus/tx_verify.h index 264433c33d..777556808a 100644 --- a/src/consensus/tx_verify.h +++ b/src/consensus/tx_verify.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_CONSENSUS_TX_VERIFY_H #define BITCOIN_CONSENSUS_TX_VERIFY_H -#include <amount.h> +#include <consensus/amount.h> #include <stdint.h> #include <vector> diff --git a/src/core_io.h b/src/core_io.h index f00f155249..4d7199ab12 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_CORE_IO_H #define BITCOIN_CORE_IO_H -#include <amount.h> +#include <consensus/amount.h> #include <attributes.h> #include <string> @@ -20,6 +20,15 @@ class uint256; class UniValue; class CTxUndo; +/** + * Verbose level for block's transaction + */ +enum class TxVerbosity { + SHOW_TXID, //!< Only TXID for each block's transaction + SHOW_DETAILS, //!< Include TXID, inputs, outputs, and other common block's transaction information + SHOW_DETAILS_AND_PREVOUT //!< The same as previous option with information about prevouts if available +}; + // core_read.cpp CScript ParseScript(const std::string& s); std::string ScriptToAsmStr(const CScript& script, const bool fAttemptSighashDecode = false); @@ -46,6 +55,6 @@ std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags = 0); std::string SighashToStr(unsigned char sighash_type); void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool include_hex, bool include_address = true); void ScriptToUniv(const CScript& script, UniValue& out); -void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex = true, int serialize_flags = 0, const CTxUndo* txundo = nullptr); +void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex = true, int serialize_flags = 0, const CTxUndo* txundo = nullptr, TxVerbosity verbosity = TxVerbosity::SHOW_DETAILS); #endif // BITCOIN_CORE_IO_H diff --git a/src/core_read.cpp b/src/core_read.cpp index 6108961010..2149b428d2 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -26,20 +26,20 @@ opcodetype ParseOpCode(const std::string& s) { static std::map<std::string, opcodetype> mapOpNames; - if (mapOpNames.empty()) - { - for (unsigned int op = 0; op <= MAX_OPCODE; op++) - { + if (mapOpNames.empty()) { + for (unsigned int op = 0; op <= MAX_OPCODE; op++) { // Allow OP_RESERVED to get into mapOpNames - if (op < OP_NOP && op != OP_RESERVED) + if (op < OP_NOP && op != OP_RESERVED) { continue; + } std::string strName = GetOpName(static_cast<opcodetype>(op)); - if (strName == "OP_UNKNOWN") + if (strName == "OP_UNKNOWN") { continue; + } mapOpNames[strName] = static_cast<opcodetype>(op); // Convenience: OP_ADD and just ADD are both recognized: - if (strName.compare(0, 3, "OP_") == 0) { // strName starts with "OP_" + if (strName.compare(0, 3, "OP_") == 0) { // strName starts with "OP_" mapOpNames[strName.substr(3)] = static_cast<opcodetype>(op); } } @@ -59,44 +59,35 @@ CScript ParseScript(const std::string& s) std::vector<std::string> words; boost::algorithm::split(words, s, boost::algorithm::is_any_of(" \t\n"), boost::algorithm::token_compress_on); - for (std::vector<std::string>::const_iterator w = words.begin(); w != words.end(); ++w) - { - if (w->empty()) - { + for (const std::string& w : words) { + if (w.empty()) { // Empty string, ignore. (boost::split given '' will return one word) - } - else if (std::all_of(w->begin(), w->end(), ::IsDigit) || - (w->front() == '-' && w->size() > 1 && std::all_of(w->begin()+1, w->end(), ::IsDigit))) + } else if (std::all_of(w.begin(), w.end(), ::IsDigit) || + (w.front() == '-' && w.size() > 1 && std::all_of(w.begin() + 1, w.end(), ::IsDigit))) { // Number - int64_t n = atoi64(*w); + const auto num{ToIntegral<int64_t>(w)}; - //limit the range of numbers ParseScript accepts in decimal - //since numbers outside -0xFFFFFFFF...0xFFFFFFFF are illegal in scripts - if (n > int64_t{0xffffffff} || n < -1 * int64_t{0xffffffff}) { + // limit the range of numbers ParseScript accepts in decimal + // since numbers outside -0xFFFFFFFF...0xFFFFFFFF are illegal in scripts + if (!num.has_value() || num > int64_t{0xffffffff} || num < -1 * int64_t{0xffffffff}) { throw std::runtime_error("script parse error: decimal numeric value only allowed in the " "range -0xFFFFFFFF...0xFFFFFFFF"); } - result << n; - } - else if (w->substr(0,2) == "0x" && w->size() > 2 && IsHex(std::string(w->begin()+2, w->end()))) - { + result << num.value(); + } else if (w.substr(0, 2) == "0x" && w.size() > 2 && IsHex(std::string(w.begin() + 2, w.end()))) { // Raw hex data, inserted NOT pushed onto stack: - std::vector<unsigned char> raw = ParseHex(std::string(w->begin()+2, w->end())); + std::vector<unsigned char> raw = ParseHex(std::string(w.begin() + 2, w.end())); result.insert(result.end(), raw.begin(), raw.end()); - } - else if (w->size() >= 2 && w->front() == '\'' && w->back() == '\'') - { + } else if (w.size() >= 2 && w.front() == '\'' && w.back() == '\'') { // Single-quoted string, pushed as data. NOTE: this is poor-man's // parsing, spaces/tabs/newlines in single-quoted strings won't work. - std::vector<unsigned char> value(w->begin()+1, w->end()-1); + std::vector<unsigned char> value(w.begin() + 1, w.end() - 1); result << value; - } - else - { + } else { // opcode, e.g. OP_ADD or ADD: - result << ParseOpCode(*w); + result << ParseOpCode(w); } } diff --git a/src/core_write.cpp b/src/core_write.cpp index d92c970cb6..468694b011 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -4,6 +4,7 @@ #include <core_io.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/validation.h> #include <key_io.h> @@ -162,7 +163,7 @@ void ScriptPubKeyToUniv(const CScript& scriptPubKey, UniValue& out, bool include out.pushKV("type", GetTxnOutputType(type)); } -void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex, int serialize_flags, const CTxUndo* txundo) +void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, bool include_hex, int serialize_flags, const CTxUndo* txundo, TxVerbosity verbosity) { entry.pushKV("txid", tx.GetHash().GetHex()); entry.pushKV("hash", tx.GetWitnessHash().GetHex()); @@ -178,7 +179,7 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, // If available, use Undo data to calculate the fee. Note that txundo == nullptr // for coinbase transactions and for transactions where undo data is unavailable. - const bool calculate_fee = txundo != nullptr; + const bool have_undo = txundo != nullptr; CAmount amt_total_in = 0; CAmount amt_total_out = 0; @@ -202,9 +203,28 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, } in.pushKV("txinwitness", txinwitness); } - if (calculate_fee) { - const CTxOut& prev_txout = txundo->vprevout[i].out; + if (have_undo) { + const Coin& prev_coin = txundo->vprevout[i]; + const CTxOut& prev_txout = prev_coin.out; + amt_total_in += prev_txout.nValue; + switch (verbosity) { + case TxVerbosity::SHOW_TXID: + case TxVerbosity::SHOW_DETAILS: + break; + + case TxVerbosity::SHOW_DETAILS_AND_PREVOUT: + UniValue o_script_pub_key(UniValue::VOBJ); + ScriptPubKeyToUniv(prev_txout.scriptPubKey, o_script_pub_key, /* includeHex */ true); + + UniValue p(UniValue::VOBJ); + p.pushKV("generated", bool(prev_coin.fCoinBase)); + p.pushKV("height", uint64_t(prev_coin.nHeight)); + p.pushKV("value", ValueFromAmount(prev_txout.nValue)); + p.pushKV("scriptPubKey", o_script_pub_key); + in.pushKV("prevout", p); + break; + } } in.pushKV("sequence", (int64_t)txin.nSequence); vin.push_back(in); @@ -225,13 +245,13 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, out.pushKV("scriptPubKey", o); vout.push_back(out); - if (calculate_fee) { + if (have_undo) { amt_total_out += txout.nValue; } } entry.pushKV("vout", vout); - if (calculate_fee) { + if (have_undo) { const CAmount fee = amt_total_in - amt_total_out; CHECK_NONFATAL(MoneyRange(fee)); entry.pushKV("fee", ValueFromAmount(fee)); diff --git a/src/crypto/chacha_poly_aead.h b/src/crypto/chacha_poly_aead.h index 0afe8fcc14..6a7998335d 100644 --- a/src/crypto/chacha_poly_aead.h +++ b/src/crypto/chacha_poly_aead.h @@ -117,8 +117,8 @@ static constexpr int AAD_PACKAGES_PER_ROUND = 21; /* 64 / 3 round down*/ class ChaCha20Poly1305AEAD { private: - ChaCha20 m_chacha_main; // payload and poly1305 key-derivation cipher instance - ChaCha20 m_chacha_header; // AAD cipher instance (encrypted length) + ChaCha20 m_chacha_header; // AAD cipher instance (encrypted length) and poly1305 key-derivation cipher instance + ChaCha20 m_chacha_main; // payload unsigned char m_aad_keystream_buffer[CHACHA20_ROUND_OUTPUT]; // aad keystream cache uint64_t m_cached_aad_seqnr; // aad keystream cache hint diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 3a1086bf4c..2fdc54464a 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -115,7 +115,7 @@ static leveldb::Options GetOptions(size_t nCacheSize) } CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate) - : m_name{path.stem().string()} + : m_name{fs::PathToString(path.stem())} { penv = nullptr; readoptions.verify_checksums = true; @@ -129,21 +129,21 @@ CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bo options.env = penv; } else { if (fWipe) { - LogPrintf("Wiping LevelDB in %s\n", path.string()); - leveldb::Status result = leveldb::DestroyDB(path.string(), options); + LogPrintf("Wiping LevelDB in %s\n", fs::PathToString(path)); + leveldb::Status result = leveldb::DestroyDB(fs::PathToString(path), options); dbwrapper_private::HandleError(result); } TryCreateDirectories(path); - LogPrintf("Opening LevelDB in %s\n", path.string()); + LogPrintf("Opening LevelDB in %s\n", fs::PathToString(path)); } - leveldb::Status status = leveldb::DB::Open(options, path.string(), &pdb); + leveldb::Status status = leveldb::DB::Open(options, fs::PathToString(path), &pdb); dbwrapper_private::HandleError(status); LogPrintf("Opened LevelDB successfully\n"); if (gArgs.GetBoolArg("-forcecompactdb", false)) { - LogPrintf("Starting database compaction of %s\n", path.string()); + LogPrintf("Starting database compaction of %s\n", fs::PathToString(path)); pdb->CompactRange(nullptr, nullptr); - LogPrintf("Finished database compaction of %s\n", path.string()); + LogPrintf("Finished database compaction of %s\n", fs::PathToString(path)); } // The base-case obfuscation key, which is a noop. @@ -160,10 +160,10 @@ CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bo Write(OBFUSCATE_KEY_KEY, new_key); obfuscate_key = new_key; - LogPrintf("Wrote new obfuscate key for %s: %s\n", path.string(), HexStr(obfuscate_key)); + LogPrintf("Wrote new obfuscate key for %s: %s\n", fs::PathToString(path), HexStr(obfuscate_key)); } - LogPrintf("Using obfuscation key for %s: %s\n", path.string(), HexStr(obfuscate_key)); + LogPrintf("Using obfuscation key for %s: %s\n", fs::PathToString(path), HexStr(obfuscate_key)); } CDBWrapper::~CDBWrapper() @@ -197,13 +197,15 @@ bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync) return true; } -size_t CDBWrapper::DynamicMemoryUsage() const { +size_t CDBWrapper::DynamicMemoryUsage() const +{ std::string memory; - if (!pdb->GetProperty("leveldb.approximate-memory-usage", &memory)) { + std::optional<size_t> parsed; + if (!pdb->GetProperty("leveldb.approximate-memory-usage", &memory) || !(parsed = ToIntegral<size_t>(memory))) { LogPrint(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n"); return 0; } - return stoul(memory); + return parsed.value(); } // Prefixed with null character to avoid collisions with other keys diff --git a/src/external_signer.cpp b/src/external_signer.cpp index d6388b759a..75070899c6 100644 --- a/src/external_signer.cpp +++ b/src/external_signer.cpp @@ -9,6 +9,7 @@ #include <util/system.h> #include <external_signer.h> +#include <algorithm> #include <stdexcept> #include <string> #include <vector> @@ -75,15 +76,14 @@ bool ExternalSigner::SignTransaction(PartiallySignedTransaction& psbtx, std::str ssTx << psbtx; // Check if signer fingerprint matches any input master key fingerprint - bool match = false; - for (unsigned int i = 0; i < psbtx.inputs.size(); ++i) { - const PSBTInput& input = psbtx.inputs[i]; + auto matches_signer_fingerprint = [&](const PSBTInput& input) { for (const auto& entry : input.hd_keypaths) { - if (m_fingerprint == strprintf("%08x", ReadBE32(entry.second.fingerprint))) match = true; + if (m_fingerprint == strprintf("%08x", ReadBE32(entry.second.fingerprint))) return true; } - } + return false; + }; - if (!match) { + if (!std::any_of(psbtx.inputs.begin(), psbtx.inputs.end(), matches_signer_fingerprint)) { error = "Signer fingerprint " + m_fingerprint + " does not match any of the inputs:\n" + EncodeBase64(ssTx.str()); return false; } diff --git a/src/flatfile.cpp b/src/flatfile.cpp index 151f1a38f1..929808c7fa 100644 --- a/src/flatfile.cpp +++ b/src/flatfile.cpp @@ -41,11 +41,11 @@ FILE* FlatFileSeq::Open(const FlatFilePos& pos, bool read_only) if (!file && !read_only) file = fsbridge::fopen(path, "wb+"); if (!file) { - LogPrintf("Unable to open file %s\n", path.string()); + LogPrintf("Unable to open file %s\n", fs::PathToString(path)); return nullptr; } if (pos.nPos && fseek(file, pos.nPos, SEEK_SET)) { - LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string()); + LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, fs::PathToString(path)); fclose(file); return nullptr; } diff --git a/src/fs.cpp b/src/fs.cpp index b9b3c46d8d..7a99444eef 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -16,6 +16,7 @@ #define NOMINMAX #endif #include <codecvt> +#include <limits> #include <windows.h> #endif @@ -24,7 +25,7 @@ namespace fsbridge { FILE *fopen(const fs::path& p, const char *mode) { #ifndef WIN32 - return ::fopen(p.string().c_str(), mode); + return ::fopen(p.c_str(), mode); #else std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>,wchar_t> utf8_cvt; return ::_wfopen(p.wstring().c_str(), utf8_cvt.from_bytes(mode).c_str()); @@ -46,7 +47,7 @@ static std::string GetErrorReason() FileLock::FileLock(const fs::path& file) { - fd = open(file.string().c_str(), O_RDWR); + fd = open(file.c_str(), O_RDWR); if (fd == -1) { reason = GetErrorReason(); } @@ -249,9 +250,9 @@ void ofstream::close() #else // __GLIBCXX__ #if BOOST_VERSION >= 107700 -static_assert(sizeof(*BOOST_FILESYSTEM_C_STR(fs::path())) == sizeof(wchar_t), +static_assert(sizeof(*BOOST_FILESYSTEM_C_STR(boost::filesystem::path())) == sizeof(wchar_t), #else -static_assert(sizeof(*fs::path().BOOST_FILESYSTEM_C_STR) == sizeof(wchar_t), +static_assert(sizeof(*boost::filesystem::path().BOOST_FILESYSTEM_C_STR) == sizeof(wchar_t), #endif // BOOST_VERSION >= 107700 "Warning: This build is using boost::filesystem ofstream and ifstream " "implementations which will fail to open paths containing multibyte " @@ -13,9 +13,132 @@ #include <boost/filesystem.hpp> #include <boost/filesystem/fstream.hpp> +#include <tinyformat.h> /** Filesystem operations and types */ -namespace fs = boost::filesystem; +namespace fs { + +using namespace boost::filesystem; + +/** + * Path class wrapper to prepare application code for transition from + * boost::filesystem library to std::filesystem implementation. The main + * purpose of the class is to define fs::path::u8string() and fs::u8path() + * functions not present in boost. It also blocks calls to the + * fs::path(std::string) implicit constructor and the fs::path::string() + * method, which worked well in the boost::filesystem implementation, but have + * unsafe and unpredictable behavior on Windows in the std::filesystem + * implementation (see implementation note in \ref PathToString for details). + */ +class path : public boost::filesystem::path +{ +public: + using boost::filesystem::path::path; + + // Allow path objects arguments for compatibility. + path(boost::filesystem::path path) : boost::filesystem::path::path(std::move(path)) {} + path& operator=(boost::filesystem::path path) { boost::filesystem::path::operator=(std::move(path)); return *this; } + path& operator/=(boost::filesystem::path path) { boost::filesystem::path::operator/=(std::move(path)); return *this; } + + // Allow literal string arguments, which are safe as long as the literals are ASCII. + path(const char* c) : boost::filesystem::path(c) {} + path& operator=(const char* c) { boost::filesystem::path::operator=(c); return *this; } + path& operator/=(const char* c) { boost::filesystem::path::operator/=(c); return *this; } + path& append(const char* c) { boost::filesystem::path::append(c); return *this; } + + // Disallow std::string arguments to avoid locale-dependent decoding on windows. + path(std::string) = delete; + path& operator=(std::string) = delete; + path& operator/=(std::string) = delete; + path& append(std::string) = delete; + + // Disallow std::string conversion method to avoid locale-dependent encoding on windows. + std::string string() const = delete; + + // Define UTF-8 string conversion method not present in boost::filesystem but present in std::filesystem. + std::string u8string() const { return boost::filesystem::path::string(); } +}; + +// Define UTF-8 string conversion function not present in boost::filesystem but present in std::filesystem. +static inline path u8path(const std::string& string) +{ + return boost::filesystem::path(string); +} + +// Disallow implicit std::string conversion for system_complete to avoid +// locale-dependent encoding on windows. +static inline path system_complete(const path& p) +{ + return boost::filesystem::system_complete(p); +} + +// Disallow implicit std::string conversion for exists to avoid +// locale-dependent encoding on windows. +static inline bool exists(const path& p) +{ + return boost::filesystem::exists(p); +} + +// Allow explicit quoted stream I/O. +static inline auto quoted(const std::string& s) +{ + return boost::io::quoted(s, '&'); +} + +// Allow safe path append operations. +static inline path operator+(path p1, path p2) +{ + p1 += std::move(p2); + return p1; +} + +/** + * Convert path object to byte string. On POSIX, paths natively are byte + * strings so this is trivial. On Windows, paths natively are Unicode, so an + * encoding step is necessary. + * + * The inverse of \ref PathToString is \ref PathFromString. The strings + * returned and parsed by these functions can be used to call POSIX APIs, and + * for roundtrip conversion, logging, and debugging. But they are not + * guaranteed to be valid UTF-8, and are generally meant to be used internally, + * not externally. When communicating with external programs and libraries that + * require UTF-8, fs::path::u8string() and fs::u8path() methods can be used. + * For other applications, if support for non UTF-8 paths is required, or if + * higher-level JSON or XML or URI or C-style escapes are preferred, it may be + * also be appropriate to use different path encoding functions. + * + * Implementation note: On Windows, the std::filesystem::path(string) + * constructor and std::filesystem::path::string() method are not safe to use + * here, because these methods encode the path using C++'s narrow multibyte + * encoding, which on Windows corresponds to the current "code page", which is + * unpredictable and typically not able to represent all valid paths. So + * std::filesystem::path::u8string() and std::filesystem::u8path() functions + * are used instead on Windows. On POSIX, u8string/u8path functions are not + * safe to use because paths are not always valid UTF-8, so plain string + * methods which do not transform the path there are used. + */ +static inline std::string PathToString(const path& path) +{ +#ifdef WIN32 + return path.u8string(); +#else + static_assert(std::is_same<path::string_type, std::string>::value, "PathToString not implemented on this platform"); + return path.boost::filesystem::path::string(); +#endif +} + +/** + * Convert byte string to path object. Inverse of \ref PathToString. + */ +static inline path PathFromString(const std::string& string) +{ +#ifdef WIN32 + return u8path(string); +#else + return boost::filesystem::path(string); +#endif +} +} // namespace fs /** Bridge operations to C stdio */ namespace fsbridge { @@ -103,4 +226,11 @@ namespace fsbridge { #endif // WIN32 && __GLIBCXX__ }; +// Disallow path operator<< formatting in tinyformat to avoid locale-dependent +// encoding on windows. +namespace tinyformat { +template<> inline void formatValue(std::ostream&, const char*, const char*, int, const boost::filesystem::path&) = delete; +template<> inline void formatValue(std::ostream&, const char*, const char*, int, const fs::path&) = delete; +} // namespace tinyformat + #endif // BITCOIN_FS_H diff --git a/src/hash.cpp b/src/hash.cpp index 92c923fbd2..0e5bd975e4 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -75,10 +75,7 @@ unsigned int MurmurHash3(unsigned int nHashSeed, Span<const unsigned char> vData void BIP32Hash(const ChainCode &chainCode, unsigned int nChild, unsigned char header, const unsigned char data[32], unsigned char output[64]) { unsigned char num[4]; - num[0] = (nChild >> 24) & 0xFF; - num[1] = (nChild >> 16) & 0xFF; - num[2] = (nChild >> 8) & 0xFF; - num[3] = (nChild >> 0) & 0xFF; + WriteBE32(num, nChild); CHMAC_SHA512(chainCode.begin(), chainCode.size()).Write(&header, 1).Write(data, 32).Write(num, 4).Finalize(output); } diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 36bbece0ea..6e75e28596 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -12,6 +12,7 @@ #include <shutdown.h> #include <sync.h> #include <util/strencodings.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <util/threadnames.h> #include <util/translation.h> @@ -279,6 +280,7 @@ static void http_reject_request_cb(struct evhttp_request* req, void*) static bool ThreadHTTP(struct event_base* base) { util::ThreadRename("http"); + SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET_HTTP_SERVER); LogPrint(BCLog::HTTP, "Entering http event loop\n"); event_base_dispatch(base); // Event loop will be interrupted by InterruptHTTPServer() @@ -332,6 +334,7 @@ static bool HTTPBindAddresses(struct evhttp* http) static void HTTPWorkQueueRun(WorkQueue<HTTPClosure>* queue, int worker_num) { util::ThreadRename(strprintf("httpworker.%i", worker_num)); + SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET_HTTP_SERVER_WORKER); queue->Run(); } diff --git a/src/i2p.cpp b/src/i2p.cpp index 5e7e42fb77..35ac8731f2 100644 --- a/src/i2p.cpp +++ b/src/i2p.cpp @@ -328,7 +328,7 @@ void Session::GenerateAndSavePrivateKey(const Sock& sock) if (!WriteBinaryFile(m_private_key_file, std::string(m_private_key.begin(), m_private_key.end()))) { throw std::runtime_error( - strprintf("Cannot save I2P private key to %s", m_private_key_file)); + strprintf("Cannot save I2P private key to %s", fs::quoted(fs::PathToString(m_private_key_file)))); } } diff --git a/src/index/base.cpp b/src/index/base.cpp index 6fd2701e2e..fc6dd77a72 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -8,6 +8,7 @@ #include <node/ui_interface.h> #include <shutdown.h> #include <tinyformat.h> +#include <util/syscall_sandbox.h> #include <util/thread.h> #include <util/translation.h> #include <validation.h> // For g_chainman @@ -123,6 +124,7 @@ static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev, CChain& void BaseIndex::ThreadSync() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::TX_INDEX); const CBlockIndex* pindex = m_best_block_index.load(); if (!m_synced) { auto& consensus_params = Params().GetConsensus(); @@ -319,7 +321,7 @@ bool BaseIndex::BlockUntilSyncedToCurrentChain() const { // Skip the queue-draining stuff if we know we're caught up with - // ::ChainActive().Tip(). + // m_chain.Tip(). LOCK(cs_main); const CBlockIndex* chain_tip = m_chainstate->m_chain.Tip(); const CBlockIndex* best_block_index = m_best_block_index.load(); diff --git a/src/init.cpp b/src/init.cpp index 65e3c275e9..164b7bb55d 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -10,12 +10,12 @@ #include <init.h> #include <addrman.h> -#include <amount.h> #include <banman.h> #include <blockfilter.h> #include <chain.h> #include <chainparams.h> #include <compat/sanity.h> +#include <consensus/amount.h> #include <deploymentstatus.h> #include <fs.h> #include <hash.h> @@ -60,6 +60,7 @@ #include <util/check.h> #include <util/moneystr.h> #include <util/string.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <util/thread.h> #include <util/threadnames.h> @@ -112,7 +113,7 @@ static const char* BITCOIN_PID_FILENAME = "bitcoind.pid"; static fs::path GetPidFile(const ArgsManager& args) { - return AbsPathForConfigVal(fs::path(args.GetArg("-pid", BITCOIN_PID_FILENAME))); + return AbsPathForConfigVal(fs::PathFromString(args.GetArg("-pid", BITCOIN_PID_FILENAME))); } [[nodiscard]] static bool CreatePidFile(const ArgsManager& args) @@ -126,7 +127,7 @@ static fs::path GetPidFile(const ArgsManager& args) #endif return true; } else { - return InitError(strprintf(_("Unable to create the PID file '%s': %s"), GetPidFile(args).string(), std::strerror(errno))); + return InitError(strprintf(_("Unable to create the PID file '%s': %s"), fs::PathToString(GetPidFile(args)), std::strerror(errno))); } } @@ -562,6 +563,10 @@ void SetupServerArgs(ArgsManager& argsman) hidden_args.emplace_back("-daemonwait"); #endif +#if defined(USE_SYSCALL_SANDBOX) + argsman.AddArg("-sandbox=<mode>", "Use the experimental syscall sandbox in the specified mode (-sandbox=log-and-abort or -sandbox=abort). Allow only expected syscalls to be used by bitcoind. Note that this is an experimental new feature that may cause bitcoind to exit or crash unexpectedly: use with caution. In the \"log-and-abort\" mode the invocation of an unexpected syscall results in a debug handler being invoked which will log the incident and terminate the program (without executing the unexpected syscall). In the \"abort\" mode the invocation of an unexpected syscall results in the entire process being killed immediately by the kernel without executing the unexpected syscall.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); +#endif // USE_SYSCALL_SANDBOX + // Add the hidden options argsman.AddHiddenArgs(hidden_args); } @@ -1018,6 +1023,37 @@ bool AppInitParameterInteraction(const ArgsManager& args) return InitError(_("No proxy server specified. Use -proxy=<ip> or -proxy=<ip:port>.")); } +#if defined(USE_SYSCALL_SANDBOX) + if (args.IsArgSet("-sandbox") && !args.IsArgNegated("-sandbox")) { + const std::string sandbox_arg{args.GetArg("-sandbox", "")}; + bool log_syscall_violation_before_terminating{false}; + if (sandbox_arg == "log-and-abort") { + log_syscall_violation_before_terminating = true; + } else if (sandbox_arg == "abort") { + // log_syscall_violation_before_terminating is false by default. + } else { + return InitError(Untranslated("Unknown syscall sandbox mode (-sandbox=<mode>). Available modes are \"log-and-abort\" and \"abort\".")); + } + // execve(...) is not allowed by the syscall sandbox. + const std::vector<std::string> features_using_execve{ + "-alertnotify", + "-blocknotify", + "-signer", + "-startupnotify", + "-walletnotify", + }; + for (const std::string& feature_using_execve : features_using_execve) { + if (!args.GetArg(feature_using_execve, "").empty()) { + return InitError(Untranslated(strprintf("The experimental syscall sandbox feature (-sandbox=<mode>) is incompatible with %s (which uses execve).", feature_using_execve))); + } + } + if (!SetupSyscallSandbox(log_syscall_violation_before_terminating)) { + return InitError(Untranslated("Installation of the syscall sandbox failed.")); + } + LogPrintf("Experimental syscall sandbox enabled (-sandbox=%s): bitcoind will terminate if an unexpected (not allowlisted) syscall is invoked.\n", sandbox_arg); + } +#endif // USE_SYSCALL_SANDBOX + return true; } @@ -1026,10 +1062,10 @@ static bool LockDataDirectory(bool probeOnly) // Make sure only a single Bitcoin process is using the data directory. fs::path datadir = gArgs.GetDataDirNet(); if (!DirIsWritable(datadir)) { - return InitError(strprintf(_("Cannot write to data directory '%s'; check permissions."), datadir.string())); + return InitError(strprintf(_("Cannot write to data directory '%s'; check permissions."), fs::PathToString(datadir))); } if (!LockDirectory(datadir, ".lock", probeOnly)) { - return InitError(strprintf(_("Cannot obtain a lock on data directory %s. %s is probably already running."), datadir.string(), PACKAGE_NAME)); + return InitError(strprintf(_("Cannot obtain a lock on data directory %s. %s is probably already running."), fs::PathToString(datadir), PACKAGE_NAME)); } return true; } @@ -1090,12 +1126,12 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) LogPrintf("Using at most %i automatic connections (%i file descriptors available)\n", nMaxConnections, nFD); // Warn about relative -datadir path. - if (args.IsArgSet("-datadir") && !fs::path(args.GetArg("-datadir", "")).is_absolute()) { + if (args.IsArgSet("-datadir") && !fs::PathFromString(args.GetArg("-datadir", "")).is_absolute()) { LogPrintf("Warning: relative datadir option '%s' specified, which will be interpreted relative to the " /* Continued */ "current working directory '%s'. This is fragile, because if bitcoin is started in the future " "from a different location, it will be unable to locate the current data files. There could " "also be data loss if bitcoin is started while in a temporary directory.\n", - args.GetArg("-datadir", ""), fs::current_path().string()); + args.GetArg("-datadir", ""), fs::PathToString(fs::current_path())); } InitSignatureCache(); @@ -1179,20 +1215,20 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // Read asmap file if configured std::vector<bool> asmap; if (args.IsArgSet("-asmap")) { - fs::path asmap_path = fs::path(args.GetArg("-asmap", "")); + fs::path asmap_path = fs::PathFromString(args.GetArg("-asmap", "")); if (asmap_path.empty()) { - asmap_path = DEFAULT_ASMAP_FILENAME; + asmap_path = fs::PathFromString(DEFAULT_ASMAP_FILENAME); } if (!asmap_path.is_absolute()) { asmap_path = gArgs.GetDataDirNet() / asmap_path; } if (!fs::exists(asmap_path)) { - InitError(strprintf(_("Could not find asmap file %s"), asmap_path)); + InitError(strprintf(_("Could not find asmap file %s"), fs::quoted(fs::PathToString(asmap_path)))); return false; } asmap = DecodeAsmap(asmap_path); if (asmap.size() == 0) { - InitError(strprintf(_("Could not parse asmap file %s"), asmap_path)); + InitError(strprintf(_("Could not parse asmap file %s"), fs::quoted(fs::PathToString(asmap_path)))); return false; } const uint256 asmap_version = SerializeHash(asmap); @@ -1617,11 +1653,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // ********************************************************* Step 11: import blocks if (!CheckDiskSpace(gArgs.GetDataDirNet())) { - InitError(strprintf(_("Error: Disk space is low for %s"), gArgs.GetDataDirNet())); + InitError(strprintf(_("Error: Disk space is low for %s"), fs::quoted(fs::PathToString(gArgs.GetDataDirNet())))); return false; } if (!CheckDiskSpace(gArgs.GetBlocksDirPath())) { - InitError(strprintf(_("Error: Disk space is low for %s"), gArgs.GetBlocksDirPath())); + InitError(strprintf(_("Error: Disk space is low for %s"), fs::quoted(fs::PathToString(gArgs.GetBlocksDirPath())))); return false; } @@ -1649,7 +1685,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) std::vector<fs::path> vImportFiles; for (const std::string& strFile : args.GetArgs("-loadblock")) { - vImportFiles.push_back(strFile); + vImportFiles.push_back(fs::PathFromString(strFile)); } chainman.m_load_block = std::thread(&util::TraceThread, "loadblk", [=, &chainman, &args] { diff --git a/src/init/common.cpp b/src/init/common.cpp index 5c1f469081..8f9e0ebc87 100644 --- a/src/init/common.cpp +++ b/src/init/common.cpp @@ -81,7 +81,7 @@ void AddLoggingArgs(ArgsManager& argsman) void SetLoggingOptions(const ArgsManager& args) { LogInstance().m_print_to_file = !args.IsArgNegated("-debuglogfile"); - LogInstance().m_file_path = AbsPathForConfigVal(args.GetArg("-debuglogfile", DEFAULT_DEBUGLOGFILE)); + LogInstance().m_file_path = AbsPathForConfigVal(fs::PathFromString(args.GetArg("-debuglogfile", DEFAULT_DEBUGLOGFILE))); LogInstance().m_print_to_console = args.GetBoolArg("-printtoconsole", !args.GetBoolArg("-daemon", false)); LogInstance().m_log_timestamps = args.GetBoolArg("-logtimestamps", DEFAULT_LOGTIMESTAMPS); LogInstance().m_log_time_micros = args.GetBoolArg("-logtimemicros", DEFAULT_LOGTIMEMICROS); @@ -128,24 +128,24 @@ bool StartLogging(const ArgsManager& args) } if (!LogInstance().StartLogging()) { return InitError(strprintf(Untranslated("Could not open debug log file %s"), - LogInstance().m_file_path.string())); + fs::PathToString(LogInstance().m_file_path))); } if (!LogInstance().m_log_timestamps) LogPrintf("Startup time: %s\n", FormatISO8601DateTime(GetTime())); - LogPrintf("Default data directory %s\n", GetDefaultDataDir().string()); - LogPrintf("Using data directory %s\n", gArgs.GetDataDirNet().string()); + LogPrintf("Default data directory %s\n", fs::PathToString(GetDefaultDataDir())); + LogPrintf("Using data directory %s\n", fs::PathToString(gArgs.GetDataDirNet())); // Only log conf file usage message if conf file actually exists. fs::path config_file_path = GetConfigFile(args.GetArg("-conf", BITCOIN_CONF_FILENAME)); if (fs::exists(config_file_path)) { - LogPrintf("Config file: %s\n", config_file_path.string()); + LogPrintf("Config file: %s\n", fs::PathToString(config_file_path)); } else if (args.IsArgSet("-conf")) { // Warn if no conf file exists at path provided by user - InitWarning(strprintf(_("The specified config file %s does not exist"), config_file_path.string())); + InitWarning(strprintf(_("The specified config file %s does not exist"), fs::PathToString(config_file_path))); } else { // Not categorizing as "Warning" because it's the default behavior - LogPrintf("Config file: %s (not found, skipping)\n", config_file_path.string()); + LogPrintf("Config file: %s (not found, skipping)\n", fs::PathToString(config_file_path)); } // Log the config arguments to debug.log diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 9a97cad1f8..d4ceb517dd 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -289,7 +289,7 @@ public: virtual void requestMempoolTransactions(Notifications& notifications) = 0; //! Check if Taproot has activated - virtual bool isTaprootActive() const = 0; + virtual bool isTaprootActive() = 0; }; //! Interface to let node manage chain clients (wallets, or maybe tools for diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 770b1b8753..34fdde3774 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_INTERFACES_NODE_H #define BITCOIN_INTERFACES_NODE_H -#include <amount.h> // For CAmount +#include <consensus/amount.h> #include <external_signer.h> #include <net.h> // For NodeId #include <net_types.h> // For banmap_t diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index 6766e0510f..490563426c 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_INTERFACES_WALLET_H #define BITCOIN_INTERFACES_WALLET_H -#include <amount.h> // For CAmount +#include <consensus/amount.h> #include <interfaces/chain.h> // For ChainClient #include <pubkey.h> // For CKeyID and CScriptID (definitions needed in CTxDestination instantiation) #include <script/standard.h> // For CTxDestination diff --git a/src/ipc/process.cpp b/src/ipc/process.cpp index 43ed1f1bae..9036b80c45 100644 --- a/src/ipc/process.cpp +++ b/src/ipc/process.cpp @@ -30,8 +30,8 @@ public: return mp::SpawnProcess(pid, [&](int fd) { fs::path path = argv0_path; path.remove_filename(); - path.append(new_exe_name); - return std::vector<std::string>{path.string(), "-ipcfd", strprintf("%i", fd)}; + path /= fs::PathFromString(new_exe_name); + return std::vector<std::string>{fs::PathToString(path), "-ipcfd", strprintf("%i", fd)}; }); } int waitSpawned(int pid) override { return mp::WaitProcess(pid); } diff --git a/src/key.cpp b/src/key.cpp index 40df248e02..39155e4311 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -343,8 +343,7 @@ CExtPubKey CExtKey::Neuter() const { void CExtKey::Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const { code[0] = nDepth; memcpy(code+1, vchFingerprint, 4); - code[5] = (nChild >> 24) & 0xFF; code[6] = (nChild >> 16) & 0xFF; - code[7] = (nChild >> 8) & 0xFF; code[8] = (nChild >> 0) & 0xFF; + WriteBE32(code+5, nChild); memcpy(code+9, chaincode.begin(), 32); code[41] = 0; assert(key.size() == 32); @@ -354,7 +353,7 @@ void CExtKey::Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const { void CExtKey::Decode(const unsigned char code[BIP32_EXTKEY_SIZE]) { nDepth = code[0]; memcpy(vchFingerprint, code+1, 4); - nChild = (code[5] << 24) | (code[6] << 16) | (code[7] << 8) | code[8]; + nChild = ReadBE32(code+5); memcpy(chaincode.begin(), code+9, 32); key.Set(code+42, code+BIP32_EXTKEY_SIZE, true); if ((nDepth == 0 && (nChild != 0 || ReadLE32(vchFingerprint) != 0)) || code[41] != 0) key = CKey(); diff --git a/src/logging.cpp b/src/logging.cpp index eb2c750296..1efce21bdb 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -160,6 +160,8 @@ const CLogCategoryDesc LogCategories[] = {BCLog::I2P, "i2p"}, {BCLog::IPC, "ipc"}, {BCLog::LOCK, "lock"}, + {BCLog::UTIL, "util"}, + {BCLog::BLOCKSTORE, "blockstorage"}, {BCLog::ALL, "1"}, {BCLog::ALL, "all"}, }; diff --git a/src/logging.h b/src/logging.h index 53a89d28bd..f46104364c 100644 --- a/src/logging.h +++ b/src/logging.h @@ -60,6 +60,8 @@ namespace BCLog { I2P = (1 << 22), IPC = (1 << 23), LOCK = (1 << 24), + UTIL = (1 << 25), + BLOCKSTORE = (1 << 26), ALL = ~(uint32_t)0, }; diff --git a/src/mapport.cpp b/src/mapport.cpp index 135efb561e..a2d06c68b4 100644 --- a/src/mapport.cpp +++ b/src/mapport.cpp @@ -14,6 +14,7 @@ #include <netaddress.h> #include <netbase.h> #include <threadinterrupt.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <util/thread.h> @@ -222,6 +223,7 @@ static bool ProcessUpnp() static void ThreadMapPort() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION_MAP_PORT); bool ok; do { ok = false; diff --git a/src/merkleblock.h b/src/merkleblock.h index 0e4ed72130..70749b6378 100644 --- a/src/merkleblock.h +++ b/src/merkleblock.h @@ -6,10 +6,10 @@ #ifndef BITCOIN_MERKLEBLOCK_H #define BITCOIN_MERKLEBLOCK_H +#include <common/bloom.h> +#include <primitives/block.h> #include <serialize.h> #include <uint256.h> -#include <primitives/block.h> -#include <bloom.h> #include <vector> diff --git a/src/miner.cpp b/src/miner.cpp index 33d115f279..1ef246cd14 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -5,10 +5,10 @@ #include <miner.h> -#include <amount.h> #include <chain.h> #include <chainparams.h> #include <coins.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/merkle.h> #include <consensus/tx_verify.h> diff --git a/src/net.cpp b/src/net.cpp index b8ff0b13ea..ad558dd598 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -25,6 +25,7 @@ #include <scheduler.h> #include <util/sock.h> #include <util/strencodings.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <util/thread.h> #include <util/trace.h> @@ -1294,9 +1295,8 @@ void CConnman::NotifyNumConnectionsChanged() } } -bool CConnman::ShouldRunInactivityChecks(const CNode& node, std::optional<int64_t> now_in) const +bool CConnman::ShouldRunInactivityChecks(const CNode& node, int64_t now) const { - const int64_t now = now_in ? now_in.value() : GetTimeSeconds(); return node.nTimeConnected + m_peer_connect_timeout < now; } @@ -1615,6 +1615,7 @@ void CConnman::SocketHandler() void CConnman::ThreadSocketHandler() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET); while (!interruptNet) { DisconnectNodes(); @@ -1634,6 +1635,7 @@ void CConnman::WakeMessageHandler() void CConnman::ThreadDNSAddressSeed() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION_DNS_SEED); FastRandomContext rng; std::vector<std::string> seeds = Params().DNSSeeds(); Shuffle(seeds.begin(), seeds.end(), rng); @@ -1816,6 +1818,7 @@ int CConnman::GetExtraBlockRelayCount() const void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET_OPEN_CONNECTION); // Connect to specific addresses if (!connect.empty()) { @@ -2006,17 +2009,18 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) if (nTries > 100) break; - CAddrInfo addr; + CAddress addr; + int64_t addr_last_try{0}; if (fFeeler) { // First, try to get a tried table collision address. This returns // an empty (invalid) address if there are no collisions to try. - addr = addrman.SelectTriedCollision(); + std::tie(addr, addr_last_try) = addrman.SelectTriedCollision(); if (!addr.IsValid()) { // No tried table collisions. Select a new table address // for our feeler. - addr = addrman.Select(true); + std::tie(addr, addr_last_try) = addrman.Select(true); } else if (AlreadyConnectedToAddress(addr)) { // If test-before-evict logic would have us connect to a // peer that we're already connected to, just mark that @@ -2025,11 +2029,11 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // a currently-connected peer. addrman.Good(addr); // Select a new table address for our feeler instead. - addr = addrman.Select(true); + std::tie(addr, addr_last_try) = addrman.Select(true); } } else { // Not a feeler - addr = addrman.Select(); + std::tie(addr, addr_last_try) = addrman.Select(); } // Require outbound connections, other than feelers, to be to distinct network groups @@ -2046,7 +2050,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) continue; // only consider very recently tried nodes after 30 failed attempts - if (nANow - addr.nLastTry < 600 && nTries < 30) + if (nANow - addr_last_try < 600 && nTries < 30) continue; // for non-feelers, require all the services we'll want, @@ -2155,6 +2159,7 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const void CConnman::ThreadOpenAddedConnections() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::NET_ADD_CONNECTION); while (true) { CSemaphoreGrant grant(*semAddnode); @@ -2218,6 +2223,7 @@ void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai void CConnman::ThreadMessageHandler() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::MESSAGE_HANDLER); FastRandomContext rng; while (!flagInterruptMsgProc) { @@ -2437,7 +2443,7 @@ void CConnman::SetNetworkActive(bool active) } } -CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In, CAddrMan& addrman_in, bool network_active) +CConnman::CConnman(uint64_t nSeed0In, uint64_t nSeed1In, AddrMan& addrman_in, bool network_active) : addrman(addrman_in), nSeed0(nSeed0In), nSeed1(nSeed1In) { SetTryNewOutboundPeer(false); @@ -7,10 +7,10 @@ #define BITCOIN_NET_H #include <addrman.h> -#include <amount.h> -#include <bloom.h> #include <chainparams.h> +#include <common/bloom.h> #include <compat.h> +#include <consensus/amount.h> #include <crypto/siphash.h> #include <hash.h> #include <i2p.h> @@ -797,7 +797,7 @@ public: m_onion_binds = connOptions.onion_binds; } - CConnman(uint64_t seed0, uint64_t seed1, CAddrMan& addrman, bool network_active = true); + CConnman(uint64_t seed0, uint64_t seed1, AddrMan& addrman, bool network_active = true); ~CConnman(); bool Start(CScheduler& scheduler, const Options& options); @@ -942,7 +942,7 @@ public: std::chrono::microseconds PoissonNextSendInbound(std::chrono::microseconds now, std::chrono::seconds average_interval); /** Return true if we should disconnect the peer for failing an inactivity check. */ - bool ShouldRunInactivityChecks(const CNode& node, std::optional<int64_t> now=std::nullopt) const; + bool ShouldRunInactivityChecks(const CNode& node, int64_t secs_now) const; private: struct ListenSocket { @@ -1049,7 +1049,7 @@ private: std::vector<ListenSocket> vhListenSocket; std::atomic<bool> fNetworkActive{true}; bool fAddressesInitialized{false}; - CAddrMan& addrman; + AddrMan& addrman; std::deque<std::string> m_addr_fetches GUARDED_BY(m_addr_fetches_mutex); RecursiveMutex m_addr_fetches_mutex; std::vector<std::string> vAddedNodes GUARDED_BY(cs_vAddedNodes); diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 12c4eece56..9f3aa5b4a3 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -10,6 +10,7 @@ #include <blockencodings.h> #include <blockfilter.h> #include <chainparams.h> +#include <consensus/amount.h> #include <consensus/validation.h> #include <deploymentstatus.h> #include <hash.h> @@ -291,7 +292,7 @@ using PeerRef = std::shared_ptr<Peer>; class PeerManagerImpl final : public PeerManager { public: - PeerManagerImpl(const CChainParams& chainparams, CConnman& connman, CAddrMan& addrman, + PeerManagerImpl(const CChainParams& chainparams, CConnman& connman, AddrMan& addrman, BanMan* banman, ChainstateManager& chainman, CTxMemPool& pool, bool ignore_incoming_txs); @@ -409,7 +410,7 @@ private: const CChainParams& m_chainparams; CConnman& m_connman; - CAddrMan& m_addrman; + AddrMan& m_addrman; /** Pointer to this node's banman. May be nullptr - check existence before dereferencing. */ BanMan* const m_banman; ChainstateManager& m_chainman; @@ -1425,14 +1426,14 @@ bool PeerManagerImpl::BlockRequestAllowed(const CBlockIndex* pindex) (GetBlockProofEquivalentTime(*pindexBestHeader, *pindex, *pindexBestHeader, m_chainparams.GetConsensus()) < STALE_RELAY_AGE_LIMIT); } -std::unique_ptr<PeerManager> PeerManager::make(const CChainParams& chainparams, CConnman& connman, CAddrMan& addrman, +std::unique_ptr<PeerManager> PeerManager::make(const CChainParams& chainparams, CConnman& connman, AddrMan& addrman, BanMan* banman, ChainstateManager& chainman, CTxMemPool& pool, bool ignore_incoming_txs) { return std::make_unique<PeerManagerImpl>(chainparams, connman, addrman, banman, chainman, pool, ignore_incoming_txs); } -PeerManagerImpl::PeerManagerImpl(const CChainParams& chainparams, CConnman& connman, CAddrMan& addrman, +PeerManagerImpl::PeerManagerImpl(const CChainParams& chainparams, CConnman& connman, AddrMan& addrman, BanMan* banman, ChainstateManager& chainman, CTxMemPool& pool, bool ignore_incoming_txs) : m_chainparams(chainparams), @@ -2664,7 +2665,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // table is also potentially detrimental because new-table entries // are subject to eviction in the event of addrman collisions. We // mitigate the information-leak by never calling - // CAddrMan::Connected() on block-relay-only peers; see + // AddrMan::Connected() on block-relay-only peers; see // FinalizeNode(). // // This moves an address from New to Tried table in Addrman, @@ -2960,16 +2961,17 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, best_block = &inv.hash; } } else if (inv.IsGenTxMsg()) { + if (reject_tx_invs) { + LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol, disconnecting peer=%d\n", inv.hash.ToString(), pfrom.GetId()); + pfrom.fDisconnect = true; + return; + } const GenTxid gtxid = ToGenTxid(inv); const bool fAlreadyHave = AlreadyHaveTx(gtxid); LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom.GetId()); pfrom.AddKnownTx(inv.hash); - if (reject_tx_invs) { - LogPrint(BCLog::NET, "transaction (%s) inv sent in violation of protocol, disconnecting peer=%d\n", inv.hash.ToString(), pfrom.GetId()); - pfrom.fDisconnect = true; - return; - } else if (!fAlreadyHave && !m_chainman.ActiveChainstate().IsInitialBlockDownload()) { + if (!fAlreadyHave && !m_chainman.ActiveChainstate().IsInitialBlockDownload()) { AddTxAnnouncement(pfrom, gtxid, current_time); } } else { @@ -3242,12 +3244,12 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // already; and an adversary can already relay us old transactions // (older than our recency filter) if trying to DoS us, without any need // for witness malleation. - if (AlreadyHaveTx(GenTxid(/* is_wtxid=*/true, wtxid))) { + if (AlreadyHaveTx(GenTxid::Wtxid(wtxid))) { if (pfrom.HasPermission(NetPermissionFlags::ForceRelay)) { // Always relay transactions received from peers with forcerelay // permission, even if they were already in the mempool, allowing // the node to function as a gateway for nodes hidden behind it. - if (!m_mempool.exists(tx.GetHash())) { + if (!m_mempool.exists(GenTxid::Txid(tx.GetHash()))) { LogPrintf("Not relaying non-mempool transaction %s from forcerelay peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); } else { LogPrintf("Force relaying tx %s from peer=%d\n", tx.GetHash().ToString(), pfrom.GetId()); @@ -3313,7 +3315,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // wtxidrelay peers. // Eventually we should replace this with an improved // protocol for getting all unconfirmed parents. - const GenTxid gtxid{/* is_wtxid=*/false, parent_txid}; + const auto gtxid{GenTxid::Txid(parent_txid)}; pfrom.AddKnownTx(parent_txid); if (!AlreadyHaveTx(gtxid)) AddTxAnnouncement(pfrom, gtxid, current_time); } @@ -4313,8 +4315,11 @@ void PeerManagerImpl::CheckForStaleTipAndEvictPeers() void PeerManagerImpl::MaybeSendPing(CNode& node_to, Peer& peer, std::chrono::microseconds now) { - if (m_connman.ShouldRunInactivityChecks(node_to) && peer.m_ping_nonce_sent && + if (m_connman.ShouldRunInactivityChecks(node_to, std::chrono::duration_cast<std::chrono::seconds>(now).count()) && + peer.m_ping_nonce_sent && now > peer.m_ping_start.load() + std::chrono::seconds{TIMEOUT_INTERVAL}) { + // The ping timeout is using mocktime. To disable the check during + // testing, increase -peertimeout. LogPrint(BCLog::NET, "ping timeout: %fs peer=%d\n", 0.000001 * count_microseconds(now - peer.m_ping_start.load()), peer.m_id); node_to.fDisconnect = true; return; diff --git a/src/net_processing.h b/src/net_processing.h index 9d8d788583..27bc40687a 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -9,7 +9,7 @@ #include <net.h> #include <validationinterface.h> -class CAddrMan; +class AddrMan; class CChainParams; class CTxMemPool; class ChainstateManager; @@ -37,7 +37,7 @@ struct CNodeStateStats { class PeerManager : public CValidationInterface, public NetEventsInterface { public: - static std::unique_ptr<PeerManager> make(const CChainParams& chainparams, CConnman& connman, CAddrMan& addrman, + static std::unique_ptr<PeerManager> make(const CChainParams& chainparams, CConnman& connman, AddrMan& addrman, BanMan* banman, ChainstateManager& chainman, CTxMemPool& pool, bool ignore_incoming_txs); virtual ~PeerManager() { } diff --git a/src/netaddress.cpp b/src/netaddress.cpp index b2f4945e3b..f9fff5a6d5 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -165,7 +165,7 @@ void CNetAddr::SetLegacyIPv6(Span<const uint8_t> ipv6) } /** - * Create an "internal" address that represents a name or FQDN. CAddrMan uses + * Create an "internal" address that represents a name or FQDN. AddrMan uses * these fake addresses to keep track of which DNS seeds were used. * @returns Whether or not the operation was successful. * @see NET_INTERNAL, INTERNAL_IN_IPV6_PREFIX, CNetAddr::IsInternal(), CNetAddr::IsRFC4193() diff --git a/src/netaddress.h b/src/netaddress.h index cfb2edcd34..66c8c48f08 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -62,7 +62,7 @@ enum Network { NET_CJDNS, /// A set of addresses that represent the hash of a string or FQDN. We use - /// them in CAddrMan to keep track of which DNS seeds were used. + /// them in AddrMan to keep track of which DNS seeds were used. NET_INTERNAL, /// Dummy value to indicate the number of NET_* constants. diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 5ddcf95c84..53bc2b5069 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -16,6 +16,7 @@ #include <signet.h> #include <streams.h> #include <undo.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <validation.h> @@ -67,13 +68,14 @@ void CleanupBlockRevFiles() LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n"); fs::path blocksdir = gArgs.GetBlocksDirPath(); for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) { + const std::string path = fs::PathToString(it->path().filename()); if (fs::is_regular_file(*it) && - it->path().filename().string().length() == 12 && - it->path().filename().string().substr(8,4) == ".dat") + path.length() == 12 && + path.substr(8,4) == ".dat") { - if (it->path().filename().string().substr(0, 3) == "blk") { - mapBlockFiles[it->path().filename().string().substr(3, 5)] = it->path(); - } else if (it->path().filename().string().substr(0, 3) == "rev") { + if (path.substr(0, 3) == "blk") { + mapBlockFiles[path.substr(3, 5)] = it->path(); + } else if (path.substr(0, 3) == "rev") { remove(it->path()); } } @@ -85,7 +87,7 @@ void CleanupBlockRevFiles() // start removing block files. int nContigCounter = 0; for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) { - if (atoi(item.first) == nContigCounter) { + if (LocaleIndependentAtoi<int>(item.first) == nContigCounter) { nContigCounter++; continue; } @@ -203,7 +205,7 @@ void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) FlatFilePos pos(*it, 0); fs::remove(BlockFileSeq().FileName(pos)); fs::remove(UndoFileSeq().FileName(pos)); - LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); + LogPrint(BCLog::BLOCKSTORE, "Prune: %s deleted blk/rev (%05u)\n", __func__, *it); } } @@ -260,7 +262,7 @@ bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, if ((int)nFile != nLastBlockFile) { if (!fKnown) { - LogPrint(BCLog::VALIDATION, "Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); + LogPrint(BCLog::BLOCKSTORE, "Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); } FlushBlockFile(!fKnown, finalize_undo); nLastBlockFile = nFile; @@ -489,6 +491,7 @@ struct CImportingNow { void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args) { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION_LOAD_BLOCKS); ScheduleBatchPriority(); { @@ -525,14 +528,14 @@ void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFile for (const fs::path& path : vImportFiles) { FILE* file = fsbridge::fopen(path, "rb"); if (file) { - LogPrintf("Importing blocks file %s...\n", path.string()); + LogPrintf("Importing blocks file %s...\n", fs::PathToString(path)); chainman.ActiveChainstate().LoadExternalBlockFile(file); if (ShutdownRequested()) { LogPrintf("Shutdown requested. Exit %s\n", __func__); return; } } else { - LogPrintf("Warning: Could not open blocks file %s\n", path.string()); + LogPrintf("Warning: Could not open blocks file %s\n", fs::PathToString(path)); } } diff --git a/src/node/coinstats.h b/src/node/coinstats.h index 69e856dd15..9e9503ff5d 100644 --- a/src/node/coinstats.h +++ b/src/node/coinstats.h @@ -6,9 +6,9 @@ #ifndef BITCOIN_NODE_COINSTATS_H #define BITCOIN_NODE_COINSTATS_H -#include <amount.h> #include <chain.h> #include <coins.h> +#include <consensus/amount.h> #include <streams.h> #include <uint256.h> diff --git a/src/node/context.h b/src/node/context.h index 135f9ea1c6..26873345b4 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -12,7 +12,7 @@ class ArgsManager; class BanMan; -class CAddrMan; +class AddrMan; class CBlockPolicyEstimator; class CConnman; class CScheduler; @@ -39,7 +39,7 @@ class WalletClient; struct NodeContext { //! Init interface for initializing current process and connecting to other processes. interfaces::Init* init{nullptr}; - std::unique_ptr<CAddrMan> addrman; + std::unique_ptr<AddrMan> addrman; std::unique_ptr<CConnman> connman; std::unique_ptr<CTxMemPool> mempool; std::unique_ptr<CBlockPolicyEstimator> fee_estimator; diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 5b6d8416a7..192caf7994 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -555,7 +555,7 @@ public: { if (!m_node.mempool) return false; LOCK(m_node.mempool->cs); - return m_node.mempool->exists(txid); + return m_node.mempool->exists(GenTxid::Txid(txid)); } bool hasDescendantsInMempool(const uint256& txid) override { @@ -698,7 +698,7 @@ public: notifications.transactionAddedToMempool(entry.GetSharedTx(), 0 /* mempool_sequence */); } } - bool isTaprootActive() const override + bool isTaprootActive() override { LOCK(::cs_main); const CBlockIndex* tip = Assert(m_node.chainman)->ActiveChain().Tip(); diff --git a/src/node/psbt.cpp b/src/node/psbt.cpp index b013b6d579..9ad65d15d2 100644 --- a/src/node/psbt.cpp +++ b/src/node/psbt.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> #include <coins.h> +#include <consensus/amount.h> #include <consensus/tx_verify.h> #include <node/psbt.h> #include <policy/policy.h> diff --git a/src/policy/feerate.h b/src/policy/feerate.h index d296d32774..b16f3f8251 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -6,7 +6,7 @@ #ifndef BITCOIN_POLICY_FEERATE_H #define BITCOIN_POLICY_FEERATE_H -#include <amount.h> +#include <consensus/amount.h> #include <serialize.h> #include <string> diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 2ae5798ebe..d8c21bd833 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -527,7 +527,7 @@ CBlockPolicyEstimator::CBlockPolicyEstimator() fs::path est_filepath = gArgs.GetDataDirNet() / FEE_ESTIMATES_FILENAME; CAutoFile est_file(fsbridge::fopen(est_filepath, "rb"), SER_DISK, CLIENT_VERSION); if (est_file.IsNull() || !Read(est_file)) { - LogPrintf("Failed to read fee estimates from %s. Continue anyway.\n", est_filepath.string()); + LogPrintf("Failed to read fee estimates from %s. Continue anyway.\n", fs::PathToString(est_filepath)); } } @@ -549,7 +549,7 @@ void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, boo if (txHeight != nBestSeenHeight) { // Ignore side chains and re-orgs; assuming they are random they don't // affect the estimate. We'll potentially double count transactions in 1-block reorgs. - // Ignore txs if BlockPolicyEstimator is not in sync with ::ChainActive().Tip(). + // Ignore txs if BlockPolicyEstimator is not in sync with ActiveChain().Tip(). // It will be synced next time a block is processed. return; } @@ -887,7 +887,7 @@ void CBlockPolicyEstimator::Flush() { fs::path est_filepath = gArgs.GetDataDirNet() / FEE_ESTIMATES_FILENAME; CAutoFile est_file(fsbridge::fopen(est_filepath, "wb"), SER_DISK, CLIENT_VERSION); if (est_file.IsNull() || !Write(est_file)) { - LogPrintf("Failed to write fee estimates to %s. Continue anyway.\n", est_filepath.string()); + LogPrintf("Failed to write fee estimates to %s. Continue anyway.\n", fs::PathToString(est_filepath)); } } diff --git a/src/policy/fees.h b/src/policy/fees.h index c444d71a31..27f9120c64 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_POLICY_FEES_H #define BITCOIN_POLICY_FEES_H -#include <amount.h> +#include <consensus/amount.h> #include <policy/feerate.h> #include <uint256.h> #include <random.h> diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index 9e433584e7..fced397e51 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -22,7 +22,7 @@ CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) // so dust is a spendable txout less than // 182*dustRelayFee/1000 (in satoshis). // 546 satoshis at the default rate of 3000 sat/kvB. - // A typical spendable segwit txout is 31 bytes big, and will + // A typical spendable segwit P2WPKH txout is 31 bytes big, and will // need a CTxIn of at least 67 bytes to spend: // so dust is a spendable txout less than // 98*dustRelayFee/1000 (in satoshis). @@ -34,6 +34,11 @@ CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) int witnessversion = 0; std::vector<unsigned char> witnessprogram; + // Note this computation is for spending a Segwit v0 P2WPKH output (a 33 bytes + // public key + an ECDSA signature). For Segwit v1 Taproot outputs the minimum + // satisfaction is lower (a single BIP340 signature) but this computation was + // kept to not further reduce the dust level. + // See discussion in https://github.com/bitcoin/bitcoin/pull/22779 for details. if (txout.scriptPubKey.IsWitnessProgram(witnessversion, witnessprogram)) { // sum the sizes of the parts of a transaction input // with 75% segwit discount applied to the script size. diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index 7ac2e22006..7e6b0cf245 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -22,7 +22,7 @@ RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) // If this transaction is not in our mempool, then we can't be sure // we will know about all its inputs. - if (!pool.exists(tx.GetHash())) { + if (!pool.exists(GenTxid::Txid(tx.GetHash()))) { return RBFTransactionState::UNKNOWN; } @@ -98,7 +98,7 @@ std::optional<std::string> HasNoNewUnconfirmed(const CTransaction& tx, if (!parents_of_conflicts.count(tx.vin[j].prevout.hash)) { // Rather than check the UTXO set - potentially expensive - it's cheaper to just check // if the new input refers to a tx that's in the mempool. - if (pool.exists(tx.vin[j].prevout.hash)) { + if (pool.exists(GenTxid::Txid(tx.vin[j].prevout.hash))) { return strprintf("replacement %s adds unconfirmed input, idx %d", tx.GetHash().ToString(), j); } diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index 245206b906..a871912225 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -5,6 +5,7 @@ #include <primitives/transaction.h> +#include <consensus/amount.h> #include <hash.h> #include <tinyformat.h> #include <util/strencodings.h> diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 6bf36ee854..947c1c60bb 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -7,7 +7,7 @@ #define BITCOIN_PRIMITIVES_TRANSACTION_H #include <stdint.h> -#include <amount.h> +#include <consensus/amount.h> #include <script/script.h> #include <serialize.h> #include <uint256.h> @@ -391,8 +391,11 @@ class GenTxid { bool m_is_wtxid; uint256 m_hash; -public: GenTxid(bool is_wtxid, const uint256& hash) : m_is_wtxid(is_wtxid), m_hash(hash) {} + +public: + static GenTxid Txid(const uint256& hash) { return GenTxid{false, hash}; } + static GenTxid Wtxid(const uint256& hash) { return GenTxid{true, hash}; } bool IsWtxid() const { return m_is_wtxid; } const uint256& GetHash() const { return m_hash; } friend bool operator==(const GenTxid& a, const GenTxid& b) { return a.m_is_wtxid == b.m_is_wtxid && a.m_hash == b.m_hash; } diff --git a/src/protocol.cpp b/src/protocol.cpp index 2e70b41e4c..7506c81815 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -223,5 +223,5 @@ std::vector<std::string> serviceFlagsToStr(uint64_t flags) GenTxid ToGenTxid(const CInv& inv) { assert(inv.IsGenTxMsg()); - return {inv.IsMsgWtx(), inv.hash}; + return inv.IsMsgWtx() ? GenTxid::Wtxid(inv.hash) : GenTxid::Txid(inv.hash); } diff --git a/src/pubkey.cpp b/src/pubkey.cpp index d14a20b870..956ff2b34a 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -337,8 +337,7 @@ bool CPubKey::Derive(CPubKey& pubkeyChild, ChainCode &ccChild, unsigned int nChi void CExtPubKey::Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const { code[0] = nDepth; memcpy(code+1, vchFingerprint, 4); - code[5] = (nChild >> 24) & 0xFF; code[6] = (nChild >> 16) & 0xFF; - code[7] = (nChild >> 8) & 0xFF; code[8] = (nChild >> 0) & 0xFF; + WriteBE32(code+5, nChild); memcpy(code+9, chaincode.begin(), 32); assert(pubkey.size() == CPubKey::COMPRESSED_SIZE); memcpy(code+41, pubkey.begin(), CPubKey::COMPRESSED_SIZE); @@ -347,7 +346,7 @@ void CExtPubKey::Encode(unsigned char code[BIP32_EXTKEY_SIZE]) const { void CExtPubKey::Decode(const unsigned char code[BIP32_EXTKEY_SIZE]) { nDepth = code[0]; memcpy(vchFingerprint, code+1, 4); - nChild = (code[5] << 24) | (code[6] << 16) | (code[7] << 8) | code[8]; + nChild = ReadBE32(code+5); memcpy(chaincode.begin(), code+9, 32); pubkey.Set(code+41, code+BIP32_EXTKEY_SIZE); if ((nDepth == 0 && (nChild != 0 || ReadLE32(vchFingerprint) != 0)) || !pubkey.IsFullyValid()) pubkey = CPubKey(); diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 00c9fd3059..5b586b9d89 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -46,7 +46,6 @@ #include <QApplication> #include <QDebug> -#include <QFontDatabase> #include <QLatin1String> #include <QLibraryInfo> #include <QLocale> @@ -155,10 +154,11 @@ static bool InitSettings() std::vector<std::string> errors; if (!gArgs.ReadSettingsFile(&errors)) { - bilingual_str error = _("Settings file could not be read"); - InitError(Untranslated(strprintf("%s:\n%s\n", error.original, MakeUnorderedList(errors)))); + std::string error = QT_TRANSLATE_NOOP("bitcoin-core", "Settings file could not be read"); + std::string error_translated = QCoreApplication::translate("bitcoin-core", error.c_str()).toStdString(); + InitError(Untranslated(strprintf("%s:\n%s\n", error, MakeUnorderedList(errors)))); - QMessageBox messagebox(QMessageBox::Critical, PACKAGE_NAME, QString::fromStdString(strprintf("%s.", error.translated)), QMessageBox::Reset | QMessageBox::Abort); + QMessageBox messagebox(QMessageBox::Critical, PACKAGE_NAME, QString::fromStdString(strprintf("%s.", error_translated)), QMessageBox::Reset | QMessageBox::Abort); /*: Explanatory text shown on startup when the settings file cannot be read. Prompts user to make a choice between resetting or aborting. */ messagebox.setInformativeText(QObject::tr("Do you want to reset settings to default values, or to abort without making changes?")); @@ -177,10 +177,11 @@ static bool InitSettings() errors.clear(); if (!gArgs.WriteSettingsFile(&errors)) { - bilingual_str error = _("Settings file could not be written"); - InitError(Untranslated(strprintf("%s:\n%s\n", error.original, MakeUnorderedList(errors)))); + std::string error = QT_TRANSLATE_NOOP("bitcoin-core", "Settings file could not be written"); + std::string error_translated = QCoreApplication::translate("bitcoin-core", error.c_str()).toStdString(); + InitError(Untranslated(strprintf("%s:\n%s\n", error, MakeUnorderedList(errors)))); - QMessageBox messagebox(QMessageBox::Critical, PACKAGE_NAME, QString::fromStdString(strprintf("%s.", error.translated)), QMessageBox::Ok); + QMessageBox messagebox(QMessageBox::Critical, PACKAGE_NAME, QString::fromStdString(strprintf("%s.", error_translated)), QMessageBox::Ok); /*: Explanatory text shown on startup when the settings file could not be written. Prompts user to check that we have the ability to write to the file. Explains that the user has the option of running without a settings file.*/ @@ -492,7 +493,7 @@ int GuiMain(int argc, char* argv[]) #endif BitcoinApplication app; - QFontDatabase::addApplicationFont(":/fonts/monospace"); + GUIUtil::LoadFont(QStringLiteral(":/fonts/monospace")); /// 2. Parse command-line options. We do this after qt in order to show an error if there are problems parsing these // Command-line options take precedence: diff --git a/src/qt/bitcoinamountfield.h b/src/qt/bitcoinamountfield.h index c60d9a2c90..4855ada513 100644 --- a/src/qt/bitcoinamountfield.h +++ b/src/qt/bitcoinamountfield.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_BITCOINAMOUNTFIELD_H #define BITCOIN_QT_BITCOINAMOUNTFIELD_H -#include <amount.h> +#include <consensus/amount.h> #include <QWidget> diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 610637360b..b68ce39b53 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -1304,8 +1304,6 @@ void BitcoinGUI::setHDStatus(bool privkeyDisabled, int hdEnabled) labelWalletHDStatusIcon->setThemedPixmap(privkeyDisabled ? QStringLiteral(":/icons/eye") : hdEnabled ? QStringLiteral(":/icons/hd_enabled") : QStringLiteral(":/icons/hd_disabled"), STATUSBAR_ICONSIZE, STATUSBAR_ICONSIZE); labelWalletHDStatusIcon->setToolTip(privkeyDisabled ? tr("Private key <b>disabled</b>") : hdEnabled ? tr("HD key generation is <b>enabled</b>") : tr("HD key generation is <b>disabled</b>")); labelWalletHDStatusIcon->show(); - // eventually disable the QLabel to set its opacity to 50% - labelWalletHDStatusIcon->setEnabled(hdEnabled); } void BitcoinGUI::setEncryptionStatus(int status) diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 27045f5cc3..fa7ae4b87d 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -12,7 +12,7 @@ #include <qt/guiutil.h> #include <qt/optionsdialog.h> -#include <amount.h> +#include <consensus/amount.h> #include <QLabel> #include <QMainWindow> diff --git a/src/qt/bitcoinunits.cpp b/src/qt/bitcoinunits.cpp index 9660ba99f7..66d5eea7ac 100644 --- a/src/qt/bitcoinunits.cpp +++ b/src/qt/bitcoinunits.cpp @@ -4,6 +4,8 @@ #include <qt/bitcoinunits.h> +#include <consensus/amount.h> + #include <QStringList> #include <cassert> diff --git a/src/qt/bitcoinunits.h b/src/qt/bitcoinunits.h index e22ba0a938..e78a347bb1 100644 --- a/src/qt/bitcoinunits.h +++ b/src/qt/bitcoinunits.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_BITCOINUNITS_H #define BITCOIN_QT_BITCOINUNITS_H -#include <amount.h> +#include <consensus/amount.h> #include <QAbstractListModel> #include <QString> diff --git a/src/qt/coincontroldialog.h b/src/qt/coincontroldialog.h index bcaf45df42..ec2619d115 100644 --- a/src/qt/coincontroldialog.h +++ b/src/qt/coincontroldialog.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_COINCONTROLDIALOG_H #define BITCOIN_QT_COINCONTROLDIALOG_H -#include <amount.h> +#include <consensus/amount.h> #include <QAbstractButton> #include <QAction> diff --git a/src/qt/forms/createwalletdialog.ui b/src/qt/forms/createwalletdialog.ui index b11fb026b0..56adbe17a5 100644 --- a/src/qt/forms/createwalletdialog.ui +++ b/src/qt/forms/createwalletdialog.ui @@ -107,6 +107,9 @@ <property name="text"> <string>Descriptor Wallet</string> </property> + <property name="checked"> + <bool>true</bool> + </property> </widget> </item> <item> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 7b1384b485..4262866f32 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -272,6 +272,12 @@ bool hasEntryData(const QAbstractItemView *view, int column, int role) return !selection.at(0).data(role).toString().isEmpty(); } +void LoadFont(const QString& file_name) +{ + const int id = QFontDatabase::addApplicationFont(file_name); + assert(id != -1); +} + QString getDefaultDataDirectory() { return boostPathToQString(GetDefaultDataDir()); @@ -647,12 +653,12 @@ void setClipboard(const QString& str) fs::path qstringToBoostPath(const QString &path) { - return fs::path(path.toStdString()); + return fs::u8path(path.toStdString()); } QString boostPathToQString(const fs::path &path) { - return QString::fromStdString(path.string()); + return QString::fromStdString(path.u8string()); } QString NetworkToQString(Network net) diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index 274f0bdcbf..211f3f506d 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_GUIUTIL_H #define BITCOIN_QT_GUIUTIL_H -#include <amount.h> +#include <consensus/amount.h> #include <fs.h> #include <net.h> #include <netaddress.h> @@ -114,6 +114,11 @@ namespace GUIUtil void setClipboard(const QString& str); /** + * Loads the font from the file specified by file_name, aborts if it fails. + */ + void LoadFont(const QString& file_name); + + /** * Determine default data directory for operating system. */ QString getDefaultDataDirectory(); diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index 4c78fba752..2ca4b6a21e 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -263,7 +263,7 @@ bool Intro::showIfNeeded(bool& did_show_intro, int64_t& prune_MiB) * (to be consistent with bitcoind behavior) */ if(dataDir != GUIUtil::getDefaultDataDirectory()) { - gArgs.SoftSetArg("-datadir", GUIUtil::qstringToBoostPath(dataDir).string()); // use OS locale for path setting + gArgs.SoftSetArg("-datadir", fs::PathToString(GUIUtil::qstringToBoostPath(dataDir))); // use OS locale for path setting } return true; } diff --git a/src/qt/optionsmodel.h b/src/qt/optionsmodel.h index 65544acfbd..8f1513e48d 100644 --- a/src/qt/optionsmodel.h +++ b/src/qt/optionsmodel.h @@ -5,7 +5,6 @@ #ifndef BITCOIN_QT_OPTIONSMODEL_H #define BITCOIN_QT_OPTIONSMODEL_H -#include <amount.h> #include <cstdint> #include <qt/guiconstants.h> diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 1c8ed22ada..0c3332ab76 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -247,10 +247,11 @@ bool RPCConsole::RPCParseCommandLine(interfaces::Node* node, std::string &strRes UniValue subelement; if (lastResult.isArray()) { - for(char argch: curarg) - if (!IsDigit(argch)) - throw std::runtime_error("Invalid result query"); - subelement = lastResult[atoi(curarg.c_str())]; + const auto parsed{ToIntegral<size_t>(curarg)}; + if (!parsed) { + throw std::runtime_error("Invalid result query"); + } + subelement = lastResult[parsed.value()]; } else if (lastResult.isObject()) subelement = find_value(lastResult, curarg); diff --git a/src/qt/sendcoinsrecipient.h b/src/qt/sendcoinsrecipient.h index 01135cdfef..c23afcab24 100644 --- a/src/qt/sendcoinsrecipient.h +++ b/src/qt/sendcoinsrecipient.h @@ -9,7 +9,7 @@ #include <config/bitcoin-config.h> #endif -#include <amount.h> +#include <consensus/amount.h> #include <serialize.h> #include <string> diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index 2292c01d6a..61b52fd08a 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -184,8 +184,8 @@ static void InitMessage(SplashScreen *splash, const std::string &message) static void ShowProgress(SplashScreen *splash, const std::string &title, int nProgress, bool resume_possible) { InitMessage(splash, title + std::string("\n") + - (resume_possible ? _("(press q to shutdown and continue later)").translated - : _("press q to shutdown").translated) + + (resume_possible ? SplashScreen::tr("(press q to shutdown and continue later)").toStdString() + : SplashScreen::tr("press q to shutdown").toStdString()) + strprintf("\n%d", nProgress) + "%"); } diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 0de781661a..729957699a 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -64,8 +64,12 @@ void TestAddAddressesToSendBook(interfaces::Node& node) test.m_node.wallet_client = wallet_client.get(); node.setContext(&test.m_node); std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockWalletDatabase()); - wallet->SetupLegacyScriptPubKeyMan(); wallet->LoadWallet(); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + { + LOCK(wallet->cs_wallet); + wallet->SetupDescriptorScriptPubKeyMans(); + } auto build_address = [&wallet]() { CKey key; diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 62b135d3f1..c74c8f25b3 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -143,11 +143,20 @@ void TestGUI(interfaces::Node& node) node.setContext(&test.m_node); std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockWalletDatabase()); wallet->LoadWallet(); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); { - auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); - LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); - wallet->SetAddressBook(GetDestinationForKey(test.coinbaseKey.GetPubKey(), wallet->m_default_address_type), "", "receive"); - spk_man->AddKeyPubKey(test.coinbaseKey, test.coinbaseKey.GetPubKey()); + LOCK(wallet->cs_wallet); + wallet->SetupDescriptorScriptPubKeyMans(); + + // Add the coinbase key + FlatSigningProvider provider; + std::string error; + std::unique_ptr<Descriptor> desc = Parse("combo(" + EncodeSecret(test.coinbaseKey) + ")", provider, error, /* require_checksum=*/ false); + assert(desc); + WalletDescriptor w_desc(std::move(desc), 0, 0, 1, 1); + if (!wallet->AddWalletDescriptor(w_desc, provider, "", false)) assert(false); + CTxDestination dest = GetDestinationForKey(test.coinbaseKey.GetPubKey(), wallet->m_default_address_type); + wallet->SetAddressBook(dest, "", "receive"); wallet->SetLastBlockProcessed(105, node.context()->chainman->ActiveChain().Tip()->GetBlockHash()); } { diff --git a/src/qt/transactionfilterproxy.h b/src/qt/transactionfilterproxy.h index 09bc9e75db..270b8ef78c 100644 --- a/src/qt/transactionfilterproxy.h +++ b/src/qt/transactionfilterproxy.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_TRANSACTIONFILTERPROXY_H #define BITCOIN_QT_TRANSACTIONFILTERPROXY_H -#include <amount.h> +#include <consensus/amount.h> #include <QDateTime> #include <QSortFilterProxyModel> diff --git a/src/qt/transactionrecord.h b/src/qt/transactionrecord.h index e10243a28a..fb88ca424f 100644 --- a/src/qt/transactionrecord.h +++ b/src/qt/transactionrecord.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_TRANSACTIONRECORD_H #define BITCOIN_QT_TRANSACTIONRECORD_H -#include <amount.h> +#include <consensus/amount.h> #include <uint256.h> #include <QList> diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index 4c74bcd480..a0ad59f12a 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -184,12 +184,13 @@ WalletControllerActivity::WalletControllerActivity(WalletController* wallet_cont connect(this, &WalletControllerActivity::finished, this, &QObject::deleteLater); } -void WalletControllerActivity::showProgressDialog(const QString& label_text) +void WalletControllerActivity::showProgressDialog(const QString& title_text, const QString& label_text) { auto progress_dialog = new QProgressDialog(m_parent_widget); progress_dialog->setAttribute(Qt::WA_DeleteOnClose); connect(this, &WalletControllerActivity::finished, progress_dialog, &QWidget::close); + progress_dialog->setWindowTitle(title_text); progress_dialog->setLabelText(label_text); progress_dialog->setRange(0, 0); progress_dialog->setCancelButton(nullptr); @@ -231,7 +232,12 @@ void CreateWalletActivity::askPassphrase() void CreateWalletActivity::createWallet() { - showProgressDialog(tr("Creating Wallet <b>%1</b>…").arg(m_create_wallet_dialog->walletName().toHtmlEscaped())); + showProgressDialog( + //: Title of window indicating the progress of creation of a new wallet. + tr("Create Wallet"), + /*: Descriptive text of the create wallet progress window which indicates + to the user which wallet is currently being created. */ + tr("Creating Wallet <b>%1</b>…").arg(m_create_wallet_dialog->walletName().toHtmlEscaped())); std::string name = m_create_wallet_dialog->walletName().toStdString(); uint64_t flags = 0; @@ -322,7 +328,12 @@ void OpenWalletActivity::open(const std::string& path) { QString name = path.empty() ? QString("["+tr("default wallet")+"]") : QString::fromStdString(path); - showProgressDialog(tr("Opening Wallet <b>%1</b>…").arg(name.toHtmlEscaped())); + showProgressDialog( + //: Title of window indicating the progress of opening of a wallet. + tr("Open Wallet"), + /*: Descriptive text of the open wallet progress window which indicates + to the user which wallet is currently being opened. */ + tr("Opening Wallet <b>%1</b>…").arg(name.toHtmlEscaped())); QTimer::singleShot(0, worker(), [this, path] { std::unique_ptr<interfaces::Wallet> wallet = node().walletClient().loadWallet(path, m_error_message, m_warning_message); @@ -340,7 +351,12 @@ LoadWalletsActivity::LoadWalletsActivity(WalletController* wallet_controller, QW void LoadWalletsActivity::load() { - showProgressDialog(tr("Loading wallets…")); + showProgressDialog( + //: Title of progress window which is displayed when wallets are being loaded. + tr("Load Wallets"), + /*: Descriptive text of the load wallets progress window which indicates to + the user that wallets are currently being loaded.*/ + tr("Loading wallets…")); QTimer::singleShot(0, worker(), [this] { for (auto& wallet : node().walletClient().getWallets()) { diff --git a/src/qt/walletcontroller.h b/src/qt/walletcontroller.h index f97a7a1e84..bbd990228f 100644 --- a/src/qt/walletcontroller.h +++ b/src/qt/walletcontroller.h @@ -96,7 +96,7 @@ protected: interfaces::Node& node() const { return m_wallet_controller->m_node; } QObject* worker() const { return m_wallet_controller->m_activity_worker; } - void showProgressDialog(const QString& label_text); + void showProgressDialog(const QString& title_text, const QString& label_text); WalletController* const m_wallet_controller; QWidget* const m_parent_widget; diff --git a/src/qt/walletmodeltransaction.h b/src/qt/walletmodeltransaction.h index 120d240d91..0bae4bade3 100644 --- a/src/qt/walletmodeltransaction.h +++ b/src/qt/walletmodeltransaction.h @@ -8,7 +8,7 @@ #include <primitives/transaction.h> #include <qt/sendcoinsrecipient.h> -#include <amount.h> +#include <consensus/amount.h> #include <QObject> diff --git a/src/qt/walletview.h b/src/qt/walletview.h index eebc163624..86a835c484 100644 --- a/src/qt/walletview.h +++ b/src/qt/walletview.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_QT_WALLETVIEW_H #define BITCOIN_QT_WALLETVIEW_H -#include <amount.h> +#include <consensus/amount.h> #include <QStackedWidget> diff --git a/src/rest.cpp b/src/rest.cpp index e50ab33e54..3746fd752a 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -189,9 +189,10 @@ static bool rest_headers(const std::any& context, if (path.size() != 2) return RESTERR(req, HTTP_BAD_REQUEST, "No header count specified. Use /rest/headers/<count>/<hash>.<ext>."); - long count = strtol(path[0].c_str(), nullptr, 10); - if (count < 1 || count > 2000) + const auto parsed_count{ToIntegral<size_t>(path[0])}; + if (!parsed_count.has_value() || *parsed_count < 1 || *parsed_count > 2000) { return RESTERR(req, HTTP_BAD_REQUEST, "Header count out of range: " + path[0]); + } std::string hashStr = path[1]; uint256 hash; @@ -199,8 +200,8 @@ static bool rest_headers(const std::any& context, return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); const CBlockIndex* tip = nullptr; - std::vector<const CBlockIndex *> headers; - headers.reserve(count); + std::vector<const CBlockIndex*> headers; + headers.reserve(*parsed_count); { ChainstateManager* maybe_chainman = GetChainman(context, req); if (!maybe_chainman) return false; @@ -211,8 +212,9 @@ static bool rest_headers(const std::any& context, const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(hash); while (pindex != nullptr && active_chain.Contains(pindex)) { headers.push_back(pindex); - if (headers.size() == (unsigned long)count) + if (headers.size() == *parsed_count) { break; + } pindex = active_chain.Next(pindex); } } @@ -260,7 +262,7 @@ static bool rest_headers(const std::any& context, static bool rest_block(const std::any& context, HTTPRequest* req, const std::string& strURIPart, - bool showTxDetails) + TxVerbosity tx_verbosity) { if (!CheckWarmup(req)) return false; @@ -312,7 +314,7 @@ static bool rest_block(const std::any& context, } case RetFormat::JSON: { - UniValue objBlock = blockToJSON(block, tip, pblockindex, showTxDetails); + UniValue objBlock = blockToJSON(block, tip, pblockindex, tx_verbosity); std::string strJSON = objBlock.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); @@ -327,12 +329,12 @@ static bool rest_block(const std::any& context, static bool rest_block_extended(const std::any& context, HTTPRequest* req, const std::string& strURIPart) { - return rest_block(context, req, strURIPart, true); + return rest_block(context, req, strURIPart, TxVerbosity::SHOW_DETAILS_AND_PREVOUT); } static bool rest_block_notxdetails(const std::any& context, HTTPRequest* req, const std::string& strURIPart) { - return rest_block(context, req, strURIPart, false); + return rest_block(context, req, strURIPart, TxVerbosity::SHOW_TXID); } // A bit of a hack - dependency on a function defined in rpc/blockchain.cpp diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 3370afc75f..aa7a55e7a9 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -5,11 +5,11 @@ #include <rpc/blockchain.h> -#include <amount.h> #include <blockfilter.h> #include <chain.h> #include <chainparams.h> #include <coins.h> +#include <consensus/amount.h> #include <consensus/params.h> #include <consensus/validation.h> #include <core_io.h> @@ -200,7 +200,7 @@ UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex return result; } -UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, bool txDetails) +UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, TxVerbosity verbosity) { UniValue result = blockheaderToJSON(tip, blockindex); @@ -208,22 +208,29 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIn result.pushKV("size", (int)::GetSerializeSize(block, PROTOCOL_VERSION)); result.pushKV("weight", (int)::GetBlockWeight(block)); UniValue txs(UniValue::VARR); - if (txDetails) { - CBlockUndo blockUndo; - const bool have_undo = !IsBlockPruned(blockindex) && UndoReadFromDisk(blockUndo, blockindex); - for (size_t i = 0; i < block.vtx.size(); ++i) { - const CTransactionRef& tx = block.vtx.at(i); - // coinbase transaction (i == 0) doesn't have undo data - const CTxUndo* txundo = (have_undo && i) ? &blockUndo.vtxundo.at(i - 1) : nullptr; - UniValue objTx(UniValue::VOBJ); - TxToUniv(*tx, uint256(), objTx, true, RPCSerializationFlags(), txundo); - txs.push_back(objTx); - } - } else { - for (const CTransactionRef& tx : block.vtx) { - txs.push_back(tx->GetHash().GetHex()); - } + + switch (verbosity) { + case TxVerbosity::SHOW_TXID: + for (const CTransactionRef& tx : block.vtx) { + txs.push_back(tx->GetHash().GetHex()); + } + break; + + case TxVerbosity::SHOW_DETAILS: + case TxVerbosity::SHOW_DETAILS_AND_PREVOUT: + CBlockUndo blockUndo; + const bool have_undo = !IsBlockPruned(blockindex) && UndoReadFromDisk(blockUndo, blockindex); + + for (size_t i = 0; i < block.vtx.size(); ++i) { + const CTransactionRef& tx = block.vtx.at(i); + // coinbase transaction (i.e. i == 0) doesn't have undo data + const CTxUndo* txundo = (have_undo && i > 0) ? &blockUndo.vtxundo.at(i - 1) : nullptr; + UniValue objTx(UniValue::VOBJ); + TxToUniv(*tx, uint256(), objTx, true, RPCSerializationFlags(), txundo, verbosity); + txs.push_back(objTx); + } } + result.pushKV("tx", txs); return result; @@ -509,7 +516,7 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool std::set<std::string> setDepends; for (const CTxIn& txin : tx.vin) { - if (pool.exists(txin.prevout.hash)) + if (pool.exists(GenTxid::Txid(txin.prevout.hash))) setDepends.insert(txin.prevout.hash.ToString()); } @@ -931,7 +938,8 @@ static RPCHelpMan getblock() return RPCHelpMan{"getblock", "\nIf verbosity is 0, returns a string that is serialized, hex-encoded data for block 'hash'.\n" "If verbosity is 1, returns an Object with information about block <hash>.\n" - "If verbosity is 2, returns an Object with information about block <hash> and information about each transaction. \n", + "If verbosity is 2, returns an Object with information about block <hash> and information about each transaction.\n" + "If verbosity is 3, returns an Object with information about block <hash> and information about each transaction, including prevout information for inputs (only for unpruned blocks in the current best chain).\n", { {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The block hash"}, {"verbosity|verbose", RPCArg::Type::NUM, RPCArg::Default{1}, "0 for hex-encoded data, 1 for a json object, and 2 for json object with transaction data"}, @@ -1018,7 +1026,16 @@ static RPCHelpMan getblock() return strHex; } - return blockToJSON(block, tip, pblockindex, verbosity >= 2); + TxVerbosity tx_verbosity; + if (verbosity == 1) { + tx_verbosity = TxVerbosity::SHOW_TXID; + } else if (verbosity == 2) { + tx_verbosity = TxVerbosity::SHOW_DETAILS; + } else { + tx_verbosity = TxVerbosity::SHOW_DETAILS_AND_PREVOUT; + } + + return blockToJSON(block, tip, pblockindex, tx_verbosity); }, }; } @@ -2537,15 +2554,15 @@ static RPCHelpMan dumptxoutset() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - const fs::path path = fsbridge::AbsPathJoin(gArgs.GetDataDirNet(), request.params[0].get_str()); + const fs::path path = fsbridge::AbsPathJoin(gArgs.GetDataDirNet(), fs::u8path(request.params[0].get_str())); // Write to a temporary path and then move into `path` on completion // to avoid confusion due to an interruption. - const fs::path temppath = fsbridge::AbsPathJoin(gArgs.GetDataDirNet(), request.params[0].get_str() + ".incomplete"); + const fs::path temppath = fsbridge::AbsPathJoin(gArgs.GetDataDirNet(), fs::u8path(request.params[0].get_str() + ".incomplete")); if (fs::exists(path)) { throw JSONRPCError( RPC_INVALID_PARAMETER, - path.string() + " already exists. If you are sure this is what you want, " + path.u8string() + " already exists. If you are sure this is what you want, " "move it out of the way first"); } @@ -2555,7 +2572,7 @@ static RPCHelpMan dumptxoutset() UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), afile); fs::rename(temppath, path); - result.pushKV("path", path.string()); + result.pushKV("path", path.u8string()); return result; }, }; diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index 4b0d855685..d9c6761f47 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -5,7 +5,8 @@ #ifndef BITCOIN_RPC_BLOCKCHAIN_H #define BITCOIN_RPC_BLOCKCHAIN_H -#include <amount.h> +#include <consensus/amount.h> +#include <core_io.h> #include <streams.h> #include <sync.h> @@ -38,7 +39,7 @@ double GetDifficulty(const CBlockIndex* blockindex); void RPCNotifyBlockChange(const CBlockIndex*); /** Block description to JSON */ -UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, bool txDetails = false) LOCKS_EXCLUDED(cs_main); +UniValue blockToJSON(const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, TxVerbosity verbosity) LOCKS_EXCLUDED(cs_main); /** Mempool information to JSON */ UniValue MempoolInfoToJSON(const CTxMemPool& pool); diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 066a60b71b..518c41d12a 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -3,9 +3,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> #include <chain.h> #include <chainparams.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/params.h> #include <consensus/validation.h> @@ -702,7 +702,7 @@ static RPCHelpMan getblocktemplate() std::string lpstr = lpval.get_str(); hashWatchedChain = ParseHashV(lpstr.substr(0, 64), "longpollid"); - nTransactionsUpdatedLastLP = atoi64(lpstr.substr(64)); + nTransactionsUpdatedLastLP = LocaleIndependentAtoi<int64_t>(lpstr.substr(64)); } else { diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index dff2129980..39bd9c6091 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -22,6 +22,7 @@ #include <util/check.h> #include <util/message.h> // For MessageSign(), MessageVerify() #include <util/strencodings.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <optional> @@ -417,6 +418,27 @@ static RPCHelpMan setmocktime() }; } +#if defined(USE_SYSCALL_SANDBOX) +static RPCHelpMan invokedisallowedsyscall() +{ + return RPCHelpMan{ + "invokedisallowedsyscall", + "\nInvoke a disallowed syscall to trigger a syscall sandbox violation. Used for testing purposes.\n", + {}, + RPCResult{RPCResult::Type::NONE, "", ""}, + RPCExamples{ + HelpExampleCli("invokedisallowedsyscall", "") + HelpExampleRpc("invokedisallowedsyscall", "")}, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + if (!Params().IsTestChain()) { + throw std::runtime_error("invokedisallowedsyscall is used for testing only."); + } + TestDisallowedSandboxCall(); + return NullUniValue; + }, + }; +} +#endif // USE_SYSCALL_SANDBOX + static RPCHelpMan mockscheduler() { return RPCHelpMan{"mockscheduler", @@ -777,6 +799,9 @@ static const CRPCCommand commands[] = { "hidden", &echo, }, { "hidden", &echojson, }, { "hidden", &echoipc, }, +#if defined(USE_SYSCALL_SANDBOX) + { "hidden", &invokedisallowedsyscall, }, +#endif // USE_SYSCALL_SANDBOX }; // clang-format on for (const auto& c : commands) { diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index b32bc670b6..483717aa7a 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -5,6 +5,7 @@ #include <chain.h> #include <coins.h> +#include <consensus/amount.h> #include <consensus/validation.h> #include <core_io.h> #include <index/txindex.h> diff --git a/src/rpc/rawtransaction_util.cpp b/src/rpc/rawtransaction_util.cpp index f21eddf56c..d550160260 100644 --- a/src/rpc/rawtransaction_util.cpp +++ b/src/rpc/rawtransaction_util.cpp @@ -6,6 +6,7 @@ #include <rpc/rawtransaction_util.h> #include <coins.h> +#include <consensus/amount.h> #include <core_io.h> #include <key_io.h> #include <policy/policy.h> diff --git a/src/rpc/request.cpp b/src/rpc/request.cpp index a7866474e1..3245e04cdf 100644 --- a/src/rpc/request.cpp +++ b/src/rpc/request.cpp @@ -70,7 +70,7 @@ static fs::path GetAuthCookieFile(bool temp=false) if (temp) { arg += ".tmp"; } - return AbsPathForConfigVal(fs::path(arg)); + return AbsPathForConfigVal(fs::PathFromString(arg)); } bool GenerateAuthCookie(std::string *cookie_out) @@ -87,7 +87,7 @@ bool GenerateAuthCookie(std::string *cookie_out) fs::path filepath_tmp = GetAuthCookieFile(true); file.open(filepath_tmp); if (!file.is_open()) { - LogPrintf("Unable to open cookie authentication file %s for writing\n", filepath_tmp.string()); + LogPrintf("Unable to open cookie authentication file %s for writing\n", fs::PathToString(filepath_tmp)); return false; } file << cookie; @@ -95,10 +95,10 @@ bool GenerateAuthCookie(std::string *cookie_out) fs::path filepath = GetAuthCookieFile(false); if (!RenameOver(filepath_tmp, filepath)) { - LogPrintf("Unable to rename cookie authentication file %s to %s\n", filepath_tmp.string(), filepath.string()); + LogPrintf("Unable to rename cookie authentication file %s to %s\n", fs::PathToString(filepath_tmp), fs::PathToString(filepath)); return false; } - LogPrintf("Generated RPC authentication cookie %s\n", filepath.string()); + LogPrintf("Generated RPC authentication cookie %s\n", fs::PathToString(filepath)); if (cookie_out) *cookie_out = cookie; diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 0d02ec5c47..9bcfba3507 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -239,7 +239,7 @@ static RPCHelpMan getrpcinfo() UniValue result(UniValue::VOBJ); result.pushKV("active_commands", active_commands); - const std::string path = LogInstance().m_file_path.string(); + const std::string path = LogInstance().m_file_path.u8string(); UniValue log_path(UniValue::VSTR, path); result.pushKV("logpath", log_path); diff --git a/src/rpc/server.h b/src/rpc/server.h index 03967020c2..e6bb35fc33 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -6,7 +6,6 @@ #ifndef BITCOIN_RPC_SERVER_H #define BITCOIN_RPC_SERVER_H -#include <amount.h> #include <rpc/request.h> #include <rpc/util.h> diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 2059628b54..2d7f5f2894 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <consensus/amount.h> #include <key_io.h> #include <outputtype.h> #include <rpc/util.h> diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 02ada969a4..162cced6c7 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -5,6 +5,7 @@ #include <scheduler.h> #include <random.h> +#include <util/syscall_sandbox.h> #include <util/time.h> #include <assert.h> @@ -24,6 +25,7 @@ CScheduler::~CScheduler() void CScheduler::serviceQueue() { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::SCHEDULER); WAIT_LOCK(newTaskMutex, lock); ++nThreadsServicingQueue; diff --git a/src/script/sign.cpp b/src/script/sign.cpp index b912b00365..4cb2125747 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -5,6 +5,7 @@ #include <script/sign.h> +#include <consensus/amount.h> #include <key.h> #include <policy/policy.h> #include <primitives/transaction.h> diff --git a/src/support/allocators/secure.h b/src/support/allocators/secure.h index 0e31ad3ce3..c4923dc56f 100644 --- a/src/support/allocators/secure.h +++ b/src/support/allocators/secure.h @@ -9,6 +9,7 @@ #include <support/lockedpool.h> #include <support/cleanse.h> +#include <memory> #include <string> // @@ -17,15 +18,13 @@ // template <typename T> struct secure_allocator : public std::allocator<T> { - // MSVC8 default copy constructor is broken - typedef std::allocator<T> base; - typedef typename base::size_type size_type; - typedef typename base::difference_type difference_type; - typedef typename base::pointer pointer; - typedef typename base::const_pointer const_pointer; - typedef typename base::reference reference; - typedef typename base::const_reference const_reference; - typedef typename base::value_type value_type; + using base = std::allocator<T>; + using traits = std::allocator_traits<base>; + using size_type = typename traits::size_type; + using difference_type = typename traits::difference_type; + using pointer = typename traits::pointer; + using const_pointer = typename traits::const_pointer; + using value_type = typename traits::value_type; secure_allocator() noexcept {} secure_allocator(const secure_allocator& a) noexcept : base(a) {} template <typename U> diff --git a/src/support/allocators/zeroafterfree.h b/src/support/allocators/zeroafterfree.h index 418f0ee656..77de4b1e69 100644 --- a/src/support/allocators/zeroafterfree.h +++ b/src/support/allocators/zeroafterfree.h @@ -13,15 +13,13 @@ template <typename T> struct zero_after_free_allocator : public std::allocator<T> { - // MSVC8 default copy constructor is broken - typedef std::allocator<T> base; - typedef typename base::size_type size_type; - typedef typename base::difference_type difference_type; - typedef typename base::pointer pointer; - typedef typename base::const_pointer const_pointer; - typedef typename base::reference reference; - typedef typename base::const_reference const_reference; - typedef typename base::value_type value_type; + using base = std::allocator<T>; + using traits = std::allocator_traits<base>; + using size_type = typename traits::size_type; + using difference_type = typename traits::difference_type; + using pointer = typename traits::pointer; + using const_pointer = typename traits::const_pointer; + using value_type = typename traits::value_type; zero_after_free_allocator() noexcept {} zero_after_free_allocator(const zero_after_free_allocator& a) noexcept : base(a) {} template <typename U> diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 01a492a20b..bd6f470219 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -4,6 +4,7 @@ #include <addrdb.h> #include <addrman.h> +#include <addrman_impl.h> #include <chainparams.h> #include <clientversion.h> #include <hash.h> @@ -21,26 +22,26 @@ using namespace std::literals; -class CAddrManSerializationMock : public CAddrMan +class AddrManSerializationMock : public AddrMan { public: virtual void Serialize(CDataStream& s) const = 0; - CAddrManSerializationMock() - : CAddrMan(/* asmap */ std::vector<bool>(), /* deterministic */ true, /* consistency_check_ratio */ 100) + AddrManSerializationMock() + : AddrMan(/* asmap */ std::vector<bool>(), /* deterministic */ true, /* consistency_check_ratio */ 100) {} }; -class CAddrManUncorrupted : public CAddrManSerializationMock +class AddrManUncorrupted : public AddrManSerializationMock { public: void Serialize(CDataStream& s) const override { - CAddrMan::Serialize(s); + AddrMan::Serialize(s); } }; -class CAddrManCorrupted : public CAddrManSerializationMock +class AddrManCorrupted : public AddrManSerializationMock { public: void Serialize(CDataStream& s) const override @@ -61,12 +62,12 @@ public: CAddress addr = CAddress(serv, NODE_NONE); CNetAddr resolved; BOOST_CHECK(LookupHost("252.2.2.2", resolved, false)); - CAddrInfo info = CAddrInfo(addr, resolved); + AddrInfo info = AddrInfo(addr, resolved); s << info; } }; -static CDataStream AddrmanToStream(const CAddrManSerializationMock& _addrman) +static CDataStream AddrmanToStream(const AddrManSerializationMock& _addrman) { CDataStream ssPeersIn(SER_DISK, CLIENT_VERSION); ssPeersIn << Params().MessageStart(); @@ -76,44 +77,44 @@ static CDataStream AddrmanToStream(const CAddrManSerializationMock& _addrman) return CDataStream(vchData, SER_DISK, CLIENT_VERSION); } -class CAddrManTest : public CAddrMan +class AddrManTest : public AddrMan { private: bool deterministic; public: - explicit CAddrManTest(bool makeDeterministic = true, - std::vector<bool> asmap = std::vector<bool>()) - : CAddrMan(asmap, makeDeterministic, /* consistency_check_ratio */ 100) + explicit AddrManTest(bool makeDeterministic = true, + std::vector<bool> asmap = std::vector<bool>()) + : AddrMan(asmap, makeDeterministic, /* consistency_check_ratio */ 100) { deterministic = makeDeterministic; } - CAddrInfo* Find(const CNetAddr& addr, int* pnId = nullptr) + AddrInfo* Find(const CNetAddr& addr, int* pnId = nullptr) { - LOCK(cs); - return CAddrMan::Find(addr, pnId); + LOCK(m_impl->cs); + return m_impl->Find(addr, pnId); } - CAddrInfo* Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId = nullptr) + AddrInfo* Create(const CAddress& addr, const CNetAddr& addrSource, int* pnId = nullptr) { - LOCK(cs); - return CAddrMan::Create(addr, addrSource, pnId); + LOCK(m_impl->cs); + return m_impl->Create(addr, addrSource, pnId); } void Delete(int nId) { - LOCK(cs); - CAddrMan::Delete(nId); + LOCK(m_impl->cs); + m_impl->Delete(nId); } // Used to test deserialization std::pair<int, int> GetBucketAndEntry(const CAddress& addr) { - LOCK(cs); - int nId = mapAddr[addr]; + LOCK(m_impl->cs); + int nId = m_impl->mapAddr[addr]; for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; ++bucket) { for (int entry = 0; entry < ADDRMAN_BUCKET_SIZE; ++entry) { - if (nId == vvNew[bucket][entry]) { + if (nId == m_impl->vvNew[bucket][entry]) { return std::pair<int, int>(bucket, entry); } } @@ -165,20 +166,20 @@ BOOST_FIXTURE_TEST_SUITE(addrman_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(addrman_simple) { - auto addrman = std::make_unique<CAddrManTest>(); + auto addrman = std::make_unique<AddrManTest>(); CNetAddr source = ResolveIP("252.2.2.2"); // Test: Does Addrman respond correctly when empty. BOOST_CHECK_EQUAL(addrman->size(), 0U); - CAddrInfo addr_null = addrman->Select(); + auto addr_null = addrman->Select().first; BOOST_CHECK_EQUAL(addr_null.ToString(), "[::]:0"); // Test: Does Addrman::Add work as expected. CService addr1 = ResolveService("250.1.1.1", 8333); BOOST_CHECK(addrman->Add({CAddress(addr1, NODE_NONE)}, source)); BOOST_CHECK_EQUAL(addrman->size(), 1U); - CAddrInfo addr_ret1 = addrman->Select(); + auto addr_ret1 = addrman->Select().first; BOOST_CHECK_EQUAL(addr_ret1.ToString(), "250.1.1.1:8333"); // Test: Does IP address deduplication work correctly. @@ -199,7 +200,7 @@ BOOST_AUTO_TEST_CASE(addrman_simple) BOOST_CHECK(addrman->size() >= 1); // Test: reset addrman and test AddrMan::Add multiple addresses works as expected - addrman = std::make_unique<CAddrManTest>(); + addrman = std::make_unique<AddrManTest>(); std::vector<CAddress> vAddr; vAddr.push_back(CAddress(ResolveService("250.1.1.3", 8333), NODE_NONE)); vAddr.push_back(CAddress(ResolveService("250.1.1.4", 8333), NODE_NONE)); @@ -209,7 +210,7 @@ BOOST_AUTO_TEST_CASE(addrman_simple) BOOST_AUTO_TEST_CASE(addrman_ports) { - CAddrManTest addrman; + AddrManTest addrman; CNetAddr source = ResolveIP("252.2.2.2"); @@ -223,7 +224,7 @@ BOOST_AUTO_TEST_CASE(addrman_ports) CService addr1_port = ResolveService("250.1.1.1", 8334); BOOST_CHECK(!addrman.Add({CAddress(addr1_port, NODE_NONE)}, source)); BOOST_CHECK_EQUAL(addrman.size(), 1U); - CAddrInfo addr_ret2 = addrman.Select(); + auto addr_ret2 = addrman.Select().first; BOOST_CHECK_EQUAL(addr_ret2.ToString(), "250.1.1.1:8333"); // Test: Add same IP but diff port to tried table, it doesn't get added. @@ -231,14 +232,14 @@ BOOST_AUTO_TEST_CASE(addrman_ports) addrman.Good(CAddress(addr1_port, NODE_NONE)); BOOST_CHECK_EQUAL(addrman.size(), 1U); bool newOnly = true; - CAddrInfo addr_ret3 = addrman.Select(newOnly); + auto addr_ret3 = addrman.Select(newOnly).first; BOOST_CHECK_EQUAL(addr_ret3.ToString(), "250.1.1.1:8333"); } BOOST_AUTO_TEST_CASE(addrman_select) { - CAddrManTest addrman; + AddrManTest addrman; CNetAddr source = ResolveIP("252.2.2.2"); @@ -248,16 +249,16 @@ BOOST_AUTO_TEST_CASE(addrman_select) BOOST_CHECK_EQUAL(addrman.size(), 1U); bool newOnly = true; - CAddrInfo addr_ret1 = addrman.Select(newOnly); + auto addr_ret1 = addrman.Select(newOnly).first; BOOST_CHECK_EQUAL(addr_ret1.ToString(), "250.1.1.1:8333"); // Test: move addr to tried, select from new expected nothing returned. addrman.Good(CAddress(addr1, NODE_NONE)); BOOST_CHECK_EQUAL(addrman.size(), 1U); - CAddrInfo addr_ret2 = addrman.Select(newOnly); + auto addr_ret2 = addrman.Select(newOnly).first; BOOST_CHECK_EQUAL(addr_ret2.ToString(), "[::]:0"); - CAddrInfo addr_ret3 = addrman.Select(); + auto addr_ret3 = addrman.Select().first; BOOST_CHECK_EQUAL(addr_ret3.ToString(), "250.1.1.1:8333"); BOOST_CHECK_EQUAL(addrman.size(), 1U); @@ -290,14 +291,14 @@ BOOST_AUTO_TEST_CASE(addrman_select) // Test: Select pulls from new and tried regardless of port number. std::set<uint16_t> ports; for (int i = 0; i < 20; ++i) { - ports.insert(addrman.Select().GetPort()); + ports.insert(addrman.Select().first.GetPort()); } BOOST_CHECK_EQUAL(ports.size(), 3U); } BOOST_AUTO_TEST_CASE(addrman_new_collisions) { - CAddrManTest addrman; + AddrManTest addrman; CNetAddr source = ResolveIP("252.2.2.2"); @@ -326,7 +327,7 @@ BOOST_AUTO_TEST_CASE(addrman_new_collisions) BOOST_AUTO_TEST_CASE(addrman_tried_collisions) { - CAddrManTest addrman; + AddrManTest addrman; CNetAddr source = ResolveIP("252.2.2.2"); @@ -356,7 +357,7 @@ BOOST_AUTO_TEST_CASE(addrman_tried_collisions) BOOST_AUTO_TEST_CASE(addrman_find) { - CAddrManTest addrman; + AddrManTest addrman; BOOST_CHECK_EQUAL(addrman.size(), 0U); @@ -372,24 +373,24 @@ BOOST_AUTO_TEST_CASE(addrman_find) BOOST_CHECK(addrman.Add({addr3}, source1)); // Test: ensure Find returns an IP matching what we searched on. - CAddrInfo* info1 = addrman.Find(addr1); + AddrInfo* info1 = addrman.Find(addr1); BOOST_REQUIRE(info1); BOOST_CHECK_EQUAL(info1->ToString(), "250.1.2.1:8333"); // Test 18; Find does not discriminate by port number. - CAddrInfo* info2 = addrman.Find(addr2); + AddrInfo* info2 = addrman.Find(addr2); BOOST_REQUIRE(info2); BOOST_CHECK_EQUAL(info2->ToString(), info1->ToString()); // Test: Find returns another IP matching what we searched on. - CAddrInfo* info3 = addrman.Find(addr3); + AddrInfo* info3 = addrman.Find(addr3); BOOST_REQUIRE(info3); BOOST_CHECK_EQUAL(info3->ToString(), "251.255.2.1:8333"); } BOOST_AUTO_TEST_CASE(addrman_create) { - CAddrManTest addrman; + AddrManTest addrman; BOOST_CHECK_EQUAL(addrman.size(), 0U); @@ -397,19 +398,19 @@ BOOST_AUTO_TEST_CASE(addrman_create) CNetAddr source1 = ResolveIP("250.1.2.1"); int nId; - CAddrInfo* pinfo = addrman.Create(addr1, source1, &nId); + AddrInfo* pinfo = addrman.Create(addr1, source1, &nId); // Test: The result should be the same as the input addr. BOOST_CHECK_EQUAL(pinfo->ToString(), "250.1.2.1:8333"); - CAddrInfo* info2 = addrman.Find(addr1); + AddrInfo* info2 = addrman.Find(addr1); BOOST_CHECK_EQUAL(info2->ToString(), "250.1.2.1:8333"); } BOOST_AUTO_TEST_CASE(addrman_delete) { - CAddrManTest addrman; + AddrManTest addrman; BOOST_CHECK_EQUAL(addrman.size(), 0U); @@ -423,13 +424,13 @@ BOOST_AUTO_TEST_CASE(addrman_delete) BOOST_CHECK_EQUAL(addrman.size(), 1U); addrman.Delete(nId); BOOST_CHECK_EQUAL(addrman.size(), 0U); - CAddrInfo* info2 = addrman.Find(addr1); + AddrInfo* info2 = addrman.Find(addr1); BOOST_CHECK(info2 == nullptr); } BOOST_AUTO_TEST_CASE(addrman_getaddr) { - CAddrManTest addrman; + AddrManTest addrman; // Test: Sanity check, GetAddr should never return anything if addrman // is empty. @@ -489,7 +490,7 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) { - CAddrManTest addrman; + AddrManTest addrman; CAddress addr1 = CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.1.1", 9999), NODE_NONE); @@ -497,7 +498,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) CNetAddr source1 = ResolveIP("250.1.1.1"); - CAddrInfo info1 = CAddrInfo(addr1, source1); + AddrInfo info1 = AddrInfo(addr1, source1); uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); @@ -512,14 +513,14 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) // Test: Two addresses with same IP but different ports can map to // different buckets because they have different keys. - CAddrInfo info2 = CAddrInfo(addr2, source1); + AddrInfo info2 = AddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); std::set<int> buckets; for (int i = 0; i < 255; i++) { - CAddrInfo infoi = CAddrInfo( + AddrInfo infoi = AddrInfo( CAddress(ResolveService("250.1.1." + ToString(i)), NODE_NONE), ResolveIP("250.1.1." + ToString(i))); int bucket = infoi.GetTriedBucket(nKey1, asmap); @@ -531,7 +532,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) buckets.clear(); for (int j = 0; j < 255; j++) { - CAddrInfo infoj = CAddrInfo( + AddrInfo infoj = AddrInfo( CAddress(ResolveService("250." + ToString(j) + ".1.1"), NODE_NONE), ResolveIP("250." + ToString(j) + ".1.1")); int bucket = infoj.GetTriedBucket(nKey1, asmap); @@ -544,14 +545,14 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) { - CAddrManTest addrman; + AddrManTest addrman; CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); CNetAddr source1 = ResolveIP("250.1.2.1"); - CAddrInfo info1 = CAddrInfo(addr1, source1); + AddrInfo info1 = AddrInfo(addr1, source1); uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); @@ -567,13 +568,13 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); // Test: Ports should not affect bucket placement in the addr - CAddrInfo info2 = CAddrInfo(addr2, source1); + AddrInfo info2 = AddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); std::set<int> buckets; for (int i = 0; i < 255; i++) { - CAddrInfo infoi = CAddrInfo( + AddrInfo infoi = AddrInfo( CAddress(ResolveService("250.1.1." + ToString(i)), NODE_NONE), ResolveIP("250.1.1." + ToString(i))); int bucket = infoi.GetNewBucket(nKey1, asmap); @@ -585,7 +586,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) buckets.clear(); for (int j = 0; j < 4 * 255; j++) { - CAddrInfo infoj = CAddrInfo(CAddress( + AddrInfo infoj = AddrInfo(CAddress( ResolveService( ToString(250 + (j / 255)) + "." + ToString(j % 256) + ".1.1"), NODE_NONE), ResolveIP("251.4.1.1")); @@ -598,7 +599,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) buckets.clear(); for (int p = 0; p < 255; p++) { - CAddrInfo infoj = CAddrInfo( + AddrInfo infoj = AddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), ResolveIP("250." + ToString(p) + ".1.1")); int bucket = infoj.GetNewBucket(nKey1, asmap); @@ -622,7 +623,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) // 101.8.0.0/16 AS8 BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) { - CAddrManTest addrman; + AddrManTest addrman; CAddress addr1 = CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.1.1", 9999), NODE_NONE); @@ -630,7 +631,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) CNetAddr source1 = ResolveIP("250.1.1.1"); - CAddrInfo info1 = CAddrInfo(addr1, source1); + AddrInfo info1 = AddrInfo(addr1, source1); uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); @@ -645,14 +646,14 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) // Test: Two addresses with same IP but different ports can map to // different buckets because they have different keys. - CAddrInfo info2 = CAddrInfo(addr2, source1); + AddrInfo info2 = AddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); std::set<int> buckets; for (int j = 0; j < 255; j++) { - CAddrInfo infoj = CAddrInfo( + AddrInfo infoj = AddrInfo( CAddress(ResolveService("101." + ToString(j) + ".1.1"), NODE_NONE), ResolveIP("101." + ToString(j) + ".1.1")); int bucket = infoj.GetTriedBucket(nKey1, asmap); @@ -664,7 +665,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) buckets.clear(); for (int j = 0; j < 255; j++) { - CAddrInfo infoj = CAddrInfo( + AddrInfo infoj = AddrInfo( CAddress(ResolveService("250." + ToString(j) + ".1.1"), NODE_NONE), ResolveIP("250." + ToString(j) + ".1.1")); int bucket = infoj.GetTriedBucket(nKey1, asmap); @@ -677,14 +678,14 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) { - CAddrManTest addrman; + AddrManTest addrman; CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); CNetAddr source1 = ResolveIP("250.1.2.1"); - CAddrInfo info1 = CAddrInfo(addr1, source1); + AddrInfo info1 = AddrInfo(addr1, source1); uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); @@ -700,13 +701,13 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); // Test: Ports should not affect bucket placement in the addr - CAddrInfo info2 = CAddrInfo(addr2, source1); + AddrInfo info2 = AddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); std::set<int> buckets; for (int i = 0; i < 255; i++) { - CAddrInfo infoi = CAddrInfo( + AddrInfo infoi = AddrInfo( CAddress(ResolveService("250.1.1." + ToString(i)), NODE_NONE), ResolveIP("250.1.1." + ToString(i))); int bucket = infoi.GetNewBucket(nKey1, asmap); @@ -718,7 +719,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) buckets.clear(); for (int j = 0; j < 4 * 255; j++) { - CAddrInfo infoj = CAddrInfo(CAddress( + AddrInfo infoj = AddrInfo(CAddress( ResolveService( ToString(250 + (j / 255)) + "." + ToString(j % 256) + ".1.1"), NODE_NONE), ResolveIP("251.4.1.1")); @@ -731,7 +732,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) buckets.clear(); for (int p = 0; p < 255; p++) { - CAddrInfo infoj = CAddrInfo( + AddrInfo infoj = AddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), ResolveIP("101." + ToString(p) + ".1.1")); int bucket = infoj.GetNewBucket(nKey1, asmap); @@ -743,7 +744,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) buckets.clear(); for (int p = 0; p < 255; p++) { - CAddrInfo infoj = CAddrInfo( + AddrInfo infoj = AddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), ResolveIP("250." + ToString(p) + ".1.1")); int bucket = infoj.GetNewBucket(nKey1, asmap); @@ -759,9 +760,9 @@ BOOST_AUTO_TEST_CASE(addrman_serialization) { std::vector<bool> asmap1 = FromBytes(asmap_raw, sizeof(asmap_raw) * 8); - auto addrman_asmap1 = std::make_unique<CAddrManTest>(true, asmap1); - auto addrman_asmap1_dup = std::make_unique<CAddrManTest>(true, asmap1); - auto addrman_noasmap = std::make_unique<CAddrManTest>(); + auto addrman_asmap1 = std::make_unique<AddrManTest>(true, asmap1); + auto addrman_asmap1_dup = std::make_unique<AddrManTest>(true, asmap1); + auto addrman_noasmap = std::make_unique<AddrManTest>(); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); CAddress addr = CAddress(ResolveService("250.1.1.1"), NODE_NONE); @@ -791,8 +792,8 @@ BOOST_AUTO_TEST_CASE(addrman_serialization) BOOST_CHECK(bucketAndEntry_asmap1.second != bucketAndEntry_noasmap.second); // deserializing non-asmaped peers.dat to asmaped addrman - addrman_asmap1 = std::make_unique<CAddrManTest>(true, asmap1); - addrman_noasmap = std::make_unique<CAddrManTest>(); + addrman_asmap1 = std::make_unique<AddrManTest>(true, asmap1); + addrman_noasmap = std::make_unique<AddrManTest>(); addrman_noasmap->Add({addr}, default_source); stream << *addrman_noasmap; stream >> *addrman_asmap1; @@ -803,8 +804,8 @@ BOOST_AUTO_TEST_CASE(addrman_serialization) BOOST_CHECK(bucketAndEntry_asmap1_deser.second == bucketAndEntry_asmap1_dup.second); // used to map to different buckets, now maps to the same bucket. - addrman_asmap1 = std::make_unique<CAddrManTest>(true, asmap1); - addrman_noasmap = std::make_unique<CAddrManTest>(); + addrman_asmap1 = std::make_unique<AddrManTest>(true, asmap1); + addrman_noasmap = std::make_unique<AddrManTest>(); CAddress addr1 = CAddress(ResolveService("250.1.1.1"), NODE_NONE); CAddress addr2 = CAddress(ResolveService("250.2.1.1"), NODE_NONE); addrman_noasmap->Add({addr, addr2}, default_source); @@ -824,7 +825,7 @@ BOOST_AUTO_TEST_CASE(remove_invalid) { // Confirm that invalid addresses are ignored in unserialization. - auto addrman = std::make_unique<CAddrManTest>(); + auto addrman = std::make_unique<AddrManTest>(); CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); const CAddress new1{ResolveService("5.5.5.5"), NODE_NONE}; @@ -856,19 +857,19 @@ BOOST_AUTO_TEST_CASE(remove_invalid) BOOST_REQUIRE(pos + sizeof(tried2_raw_replacement) <= stream.size()); memcpy(stream.data() + pos, tried2_raw_replacement, sizeof(tried2_raw_replacement)); - addrman = std::make_unique<CAddrManTest>(); + addrman = std::make_unique<AddrManTest>(); stream >> *addrman; BOOST_CHECK_EQUAL(addrman->size(), 2); } BOOST_AUTO_TEST_CASE(addrman_selecttriedcollision) { - CAddrManTest addrman; + AddrManTest addrman; BOOST_CHECK(addrman.size() == 0); // Empty addrman should return blank addrman info. - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); // Add twenty two addresses. CNetAddr source = ResolveIP("252.2.2.2"); @@ -879,7 +880,7 @@ BOOST_AUTO_TEST_CASE(addrman_selecttriedcollision) // No collisions yet. BOOST_CHECK(addrman.size() == i); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } // Ensure Good handles duplicates well. @@ -888,14 +889,14 @@ BOOST_AUTO_TEST_CASE(addrman_selecttriedcollision) addrman.Good(addr); BOOST_CHECK(addrman.size() == 22); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } } BOOST_AUTO_TEST_CASE(addrman_noevict) { - CAddrManTest addrman; + AddrManTest addrman; // Add 35 addresses. CNetAddr source = ResolveIP("252.2.2.2"); @@ -906,7 +907,7 @@ BOOST_AUTO_TEST_CASE(addrman_noevict) // No collision yet. BOOST_CHECK(addrman.size() == i); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } // Collision between 36 and 19. @@ -915,11 +916,11 @@ BOOST_AUTO_TEST_CASE(addrman_noevict) addrman.Good(addr36); BOOST_CHECK(addrman.size() == 36); - BOOST_CHECK_EQUAL(addrman.SelectTriedCollision().ToString(), "250.1.1.19:0"); + BOOST_CHECK_EQUAL(addrman.SelectTriedCollision().first.ToString(), "250.1.1.19:0"); // 36 should be discarded and 19 not evicted. addrman.ResolveCollisions(); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); // Lets create two collisions. for (unsigned int i = 37; i < 59; i++) { @@ -928,7 +929,7 @@ BOOST_AUTO_TEST_CASE(addrman_noevict) addrman.Good(addr); BOOST_CHECK(addrman.size() == i); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } // Cause a collision. @@ -937,26 +938,26 @@ BOOST_AUTO_TEST_CASE(addrman_noevict) addrman.Good(addr59); BOOST_CHECK(addrman.size() == 59); - BOOST_CHECK_EQUAL(addrman.SelectTriedCollision().ToString(), "250.1.1.10:0"); + BOOST_CHECK_EQUAL(addrman.SelectTriedCollision().first.ToString(), "250.1.1.10:0"); // Cause a second collision. BOOST_CHECK(!addrman.Add({CAddress(addr36, NODE_NONE)}, source)); addrman.Good(addr36); BOOST_CHECK(addrman.size() == 59); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() != "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() != "[::]:0"); addrman.ResolveCollisions(); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } BOOST_AUTO_TEST_CASE(addrman_evictionworks) { - CAddrManTest addrman; + AddrManTest addrman; BOOST_CHECK(addrman.size() == 0); // Empty addrman should return blank addrman info. - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); // Add 35 addresses CNetAddr source = ResolveIP("252.2.2.2"); @@ -967,7 +968,7 @@ BOOST_AUTO_TEST_CASE(addrman_evictionworks) // No collision yet. BOOST_CHECK(addrman.size() == i); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } // Collision between 36 and 19. @@ -976,7 +977,7 @@ BOOST_AUTO_TEST_CASE(addrman_evictionworks) addrman.Good(addr); BOOST_CHECK_EQUAL(addrman.size(), 36); - CAddrInfo info = addrman.SelectTriedCollision(); + auto info = addrman.SelectTriedCollision().first; BOOST_CHECK_EQUAL(info.ToString(), "250.1.1.19:0"); // Ensure test of address fails, so that it is evicted. @@ -984,28 +985,28 @@ BOOST_AUTO_TEST_CASE(addrman_evictionworks) // Should swap 36 for 19. addrman.ResolveCollisions(); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); // If 36 was swapped for 19, then this should cause no collisions. BOOST_CHECK(!addrman.Add({CAddress(addr, NODE_NONE)}, source)); addrman.Good(addr); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); // If we insert 19 it should collide with 36 CService addr19 = ResolveService("250.1.1.19"); BOOST_CHECK(!addrman.Add({CAddress(addr19, NODE_NONE)}, source)); addrman.Good(addr19); - BOOST_CHECK_EQUAL(addrman.SelectTriedCollision().ToString(), "250.1.1.36:0"); + BOOST_CHECK_EQUAL(addrman.SelectTriedCollision().first.ToString(), "250.1.1.36:0"); addrman.ResolveCollisions(); - BOOST_CHECK(addrman.SelectTriedCollision().ToString() == "[::]:0"); + BOOST_CHECK(addrman.SelectTriedCollision().first.ToString() == "[::]:0"); } BOOST_AUTO_TEST_CASE(load_addrman) { - CAddrManUncorrupted addrmanUncorrupted; + AddrManUncorrupted addrmanUncorrupted; CService addr1, addr2, addr3; BOOST_CHECK(Lookup("250.7.1.1", addr1, 8333, false)); @@ -1024,7 +1025,7 @@ BOOST_AUTO_TEST_CASE(load_addrman) // Test that the de-serialization does not throw an exception. CDataStream ssPeers1 = AddrmanToStream(addrmanUncorrupted); bool exceptionThrown = false; - CAddrMan addrman1(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); + AddrMan addrman1(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); BOOST_CHECK(addrman1.size() == 0); try { @@ -1041,7 +1042,7 @@ BOOST_AUTO_TEST_CASE(load_addrman) // Test that ReadFromStream creates an addrman with the correct number of addrs. CDataStream ssPeers2 = AddrmanToStream(addrmanUncorrupted); - CAddrMan addrman2(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); + AddrMan addrman2(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); BOOST_CHECK(addrman2.size() == 0); ReadFromStream(addrman2, ssPeers2); BOOST_CHECK(addrman2.size() == 3); @@ -1050,12 +1051,12 @@ BOOST_AUTO_TEST_CASE(load_addrman) BOOST_AUTO_TEST_CASE(load_addrman_corrupted) { - CAddrManCorrupted addrmanCorrupted; + AddrManCorrupted addrmanCorrupted; // Test that the de-serialization of corrupted addrman throws an exception. CDataStream ssPeers1 = AddrmanToStream(addrmanCorrupted); bool exceptionThrown = false; - CAddrMan addrman1(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); + AddrMan addrman1(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); BOOST_CHECK(addrman1.size() == 0); try { unsigned char pchMsgTmp[4]; @@ -1071,7 +1072,7 @@ BOOST_AUTO_TEST_CASE(load_addrman_corrupted) // Test that ReadFromStream fails if peers.dat is corrupt CDataStream ssPeers2 = AddrmanToStream(addrmanCorrupted); - CAddrMan addrman2(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); + AddrMan addrman2(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 100); BOOST_CHECK(addrman2.size() == 0); BOOST_CHECK_THROW(ReadFromStream(addrman2, ssPeers2), std::ios_base::failure); } diff --git a/src/test/amount_tests.cpp b/src/test/amount_tests.cpp index 77b7758a17..114fe3907c 100644 --- a/src/test/amount_tests.cpp +++ b/src/test/amount_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> +#include <consensus/amount.h> #include <policy/feerate.h> #include <limits> diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 23ef2062ef..fe5ed0a3c8 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <bloom.h> +#include <common/bloom.h> #include <clientversion.h> #include <key.h> diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 5b3b39fdb8..1483bd3cb3 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -694,8 +694,8 @@ BOOST_AUTO_TEST_CASE(chacha20_poly1305_aead_testvector) TestChaCha20Poly1305AEAD(true, 0, /* m */ "0000000000000000000000000000000000000000000000000000000000000000", - /* k1 (payload) */ "0000000000000000000000000000000000000000000000000000000000000000", - /* k2 (AAD) */ "0000000000000000000000000000000000000000000000000000000000000000", + /* k1 (AAD) */ "0000000000000000000000000000000000000000000000000000000000000000", + /* k2 (payload) */ "0000000000000000000000000000000000000000000000000000000000000000", /* AAD keystream */ "76b8e0ada0f13d90405d6ae55386bd28bdd219b8a08ded1aa836efcc8b770dc7da41597c5157488d7724e03fb8d84a376a43b8f41518a11cc387b669b2ee6586", /* encrypted message & MAC */ "76b8e09f07e7be5551387a98ba977c732d080dcb0f29a048e3656912c6533e32d2fc11829c1b6c1df1f551cd6131ff08", /* encrypted message & MAC at sequence 999 */ "b0a03d5bd2855d60699e7d3a3133fa47be740fe4e4c1f967555e2d9271f31c3aaa7aa16ec62c5e24f040c08bb20c3598"); diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 0bfe6eecd9..668ff150ee 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -52,6 +52,8 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) { const CChainParams& chainparams = Params(); auto connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman); + // Disable inactivity checks for this test to avoid interference + static_cast<ConnmanTestMsg*>(connman.get())->SetPeerConnectTimeout(99999); auto peerLogic = PeerManager::make(chainparams, *connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, false); diff --git a/src/test/fs_tests.cpp b/src/test/fs_tests.cpp index 526a3c27be..ecb838a7dd 100644 --- a/src/test/fs_tests.cpp +++ b/src/test/fs_tests.cpp @@ -11,6 +11,33 @@ BOOST_FIXTURE_TEST_SUITE(fs_tests, BasicTestingSetup) +BOOST_AUTO_TEST_CASE(fsbridge_pathtostring) +{ + std::string u8_str = "fs_tests_₿_🏃"; + BOOST_CHECK_EQUAL(fs::PathToString(fs::PathFromString(u8_str)), u8_str); + BOOST_CHECK_EQUAL(fs::u8path(u8_str).u8string(), u8_str); + BOOST_CHECK_EQUAL(fs::PathFromString(u8_str).u8string(), u8_str); + BOOST_CHECK_EQUAL(fs::PathToString(fs::u8path(u8_str)), u8_str); +#ifndef WIN32 + // On non-windows systems, verify that arbitrary byte strings containing + // invalid UTF-8 can be round tripped successfully with PathToString and + // PathFromString. On non-windows systems, paths are just byte strings so + // these functions do not do any encoding. On windows, paths are Unicode, + // and these functions do encoding and decoding, so the behavior of this + // test would be undefined. + std::string invalid_u8_str = "\xf0"; + BOOST_CHECK_EQUAL(invalid_u8_str.size(), 1); + BOOST_CHECK_EQUAL(fs::PathToString(fs::PathFromString(invalid_u8_str)), invalid_u8_str); +#endif +} + +BOOST_AUTO_TEST_CASE(fsbridge_stem) +{ + std::string test_filename = "fs_tests_₿_🏃.dat"; + std::string expected_stem = "fs_tests_₿_🏃"; + BOOST_CHECK_EQUAL(fs::PathToString(fs::PathFromString(test_filename).stem()), expected_stem); +} + BOOST_AUTO_TEST_CASE(fsbridge_fstream) { fs::path tmpfolder = m_args.GetDataDirBase(); diff --git a/src/test/fuzz/addrman.cpp b/src/test/fuzz/addrman.cpp index 45ee778b87..8df3707fc9 100644 --- a/src/test/fuzz/addrman.cpp +++ b/src/test/fuzz/addrman.cpp @@ -4,8 +4,10 @@ #include <addrdb.h> #include <addrman.h> +#include <addrman_impl.h> #include <chainparams.h> #include <merkleblock.h> +#include <random.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> @@ -27,99 +29,93 @@ FUZZ_TARGET_INIT(data_stream_addr_man, initialize_addrman) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; CDataStream data_stream = ConsumeDataStream(fuzzed_data_provider); - CAddrMan addr_man(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + AddrMan addr_man(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); try { ReadFromStream(addr_man, data_stream); } catch (const std::exception&) { } } -class CAddrManDeterministic : public CAddrMan +/** + * Generate a random address. Always returns a valid address. + */ +CNetAddr RandAddr(FuzzedDataProvider& fuzzed_data_provider, FastRandomContext& fast_random_context) { -public: - FuzzedDataProvider& m_fuzzed_data_provider; - - explicit CAddrManDeterministic(std::vector<bool> asmap, FuzzedDataProvider& fuzzed_data_provider) - : CAddrMan(std::move(asmap), /* deterministic */ true, /* consistency_check_ratio */ 0) - , m_fuzzed_data_provider(fuzzed_data_provider) - { - WITH_LOCK(cs, insecure_rand = FastRandomContext{ConsumeUInt256(fuzzed_data_provider)}); - } - - /** - * Generate a random address. Always returns a valid address. - */ - CNetAddr RandAddr() EXCLUSIVE_LOCKS_REQUIRED(cs) - { - CNetAddr addr; - if (m_fuzzed_data_provider.remaining_bytes() > 1 && m_fuzzed_data_provider.ConsumeBool()) { - addr = ConsumeNetAddr(m_fuzzed_data_provider); - } else { - // The networks [1..6] correspond to CNetAddr::BIP155Network (private). - static const std::map<uint8_t, uint8_t> net_len_map = {{1, ADDR_IPV4_SIZE}, - {2, ADDR_IPV6_SIZE}, - {4, ADDR_TORV3_SIZE}, - {5, ADDR_I2P_SIZE}, - {6, ADDR_CJDNS_SIZE}}; - uint8_t net = insecure_rand.randrange(5) + 1; // [1..5] - if (net == 3) { - net = 6; - } - - CDataStream s(SER_NETWORK, PROTOCOL_VERSION | ADDRV2_FORMAT); - - s << net; - s << insecure_rand.randbytes(net_len_map.at(net)); - - s >> addr; + CNetAddr addr; + if (fuzzed_data_provider.remaining_bytes() > 1 && fuzzed_data_provider.ConsumeBool()) { + addr = ConsumeNetAddr(fuzzed_data_provider); + } else { + // The networks [1..6] correspond to CNetAddr::BIP155Network (private). + static const std::map<uint8_t, uint8_t> net_len_map = {{1, ADDR_IPV4_SIZE}, + {2, ADDR_IPV6_SIZE}, + {4, ADDR_TORV3_SIZE}, + {5, ADDR_I2P_SIZE}, + {6, ADDR_CJDNS_SIZE}}; + uint8_t net = fast_random_context.randrange(5) + 1; // [1..5] + if (net == 3) { + net = 6; } - // Return a dummy IPv4 5.5.5.5 if we generated an invalid address. - if (!addr.IsValid()) { - in_addr v4_addr = {}; - v4_addr.s_addr = 0x05050505; - addr = CNetAddr{v4_addr}; - } - - return addr; - } - - /** - * Fill this addrman with lots of addresses from lots of sources. - */ - void Fill() - { - LOCK(cs); + CDataStream s(SER_NETWORK, PROTOCOL_VERSION | ADDRV2_FORMAT); - // Add some of the addresses directly to the "tried" table. + s << net; + s << fast_random_context.randbytes(net_len_map.at(net)); - // 0, 1, 2, 3 corresponding to 0%, 100%, 50%, 33% - const size_t n = m_fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 3); + s >> addr; + } - const size_t num_sources = m_fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 50); - CNetAddr prev_source; - // Use insecure_rand inside the loops instead of m_fuzzed_data_provider because when - // the latter is exhausted it just returns 0. - for (size_t i = 0; i < num_sources; ++i) { - const auto source = RandAddr(); - const size_t num_addresses = insecure_rand.randrange(500) + 1; // [1..500] + // Return a dummy IPv4 5.5.5.5 if we generated an invalid address. + if (!addr.IsValid()) { + in_addr v4_addr = {}; + v4_addr.s_addr = 0x05050505; + addr = CNetAddr{v4_addr}; + } - for (size_t j = 0; j < num_addresses; ++j) { - const auto addr = CAddress{CService{RandAddr(), 8333}, NODE_NETWORK}; - const auto time_penalty = insecure_rand.randrange(100000001); - Add_(addr, source, time_penalty); + return addr; +} - if (n > 0 && mapInfo.size() % n == 0) { - Good_(addr, false, GetTime()); - } +/** Fill addrman with lots of addresses from lots of sources. */ +void FillAddrman(AddrMan& addrman, FuzzedDataProvider& fuzzed_data_provider) +{ + // Add a fraction of the addresses to the "tried" table. + // 0, 1, 2, 3 corresponding to 0%, 100%, 50%, 33% + const size_t n = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 3); + + const size_t num_sources = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 50); + CNetAddr prev_source; + // Generate a FastRandomContext seed to use inside the loops instead of + // fuzzed_data_provider. When fuzzed_data_provider is exhausted it + // just returns 0. + FastRandomContext fast_random_context{ConsumeUInt256(fuzzed_data_provider)}; + for (size_t i = 0; i < num_sources; ++i) { + const auto source = RandAddr(fuzzed_data_provider, fast_random_context); + const size_t num_addresses = fast_random_context.randrange(500) + 1; // [1..500] + + for (size_t j = 0; j < num_addresses; ++j) { + const auto addr = CAddress{CService{RandAddr(fuzzed_data_provider, fast_random_context), 8333}, NODE_NETWORK}; + const auto time_penalty = fast_random_context.randrange(100000001); + addrman.Add({addr}, source, time_penalty); + + if (n > 0 && addrman.size() % n == 0) { + addrman.Good(addr, GetTime()); + } - // Add 10% of the addresses from more than one source. - if (insecure_rand.randrange(10) == 0 && prev_source.IsValid()) { - Add_(addr, prev_source, time_penalty); - } + // Add 10% of the addresses from more than one source. + if (fast_random_context.randrange(10) == 0 && prev_source.IsValid()) { + addrman.Add({addr}, prev_source, time_penalty); } - prev_source = source; } + prev_source = source; + } +} + +class AddrManDeterministic : public AddrMan +{ +public: + explicit AddrManDeterministic(std::vector<bool> asmap, FuzzedDataProvider& fuzzed_data_provider) + : AddrMan(std::move(asmap), /* deterministic */ true, /* consistency_check_ratio */ 0) + { + WITH_LOCK(m_impl->cs, m_impl->insecure_rand = FastRandomContext{ConsumeUInt256(fuzzed_data_provider)}); } /** @@ -129,46 +125,46 @@ public: * - vvNew entries refer to the same addresses * - vvTried entries refer to the same addresses */ - bool operator==(const CAddrManDeterministic& other) + bool operator==(const AddrManDeterministic& other) { - LOCK2(cs, other.cs); + LOCK2(m_impl->cs, other.m_impl->cs); - if (mapInfo.size() != other.mapInfo.size() || nNew != other.nNew || - nTried != other.nTried) { + if (m_impl->mapInfo.size() != other.m_impl->mapInfo.size() || m_impl->nNew != other.m_impl->nNew || + m_impl->nTried != other.m_impl->nTried) { return false; } // Check that all values in `mapInfo` are equal to all values in `other.mapInfo`. // Keys may be different. - using CAddrInfoHasher = std::function<size_t(const CAddrInfo&)>; - using CAddrInfoEq = std::function<bool(const CAddrInfo&, const CAddrInfo&)>; + using AddrInfoHasher = std::function<size_t(const AddrInfo&)>; + using AddrInfoEq = std::function<bool(const AddrInfo&, const AddrInfo&)>; CNetAddrHash netaddr_hasher; - CAddrInfoHasher addrinfo_hasher = [&netaddr_hasher](const CAddrInfo& a) { + AddrInfoHasher addrinfo_hasher = [&netaddr_hasher](const AddrInfo& a) { return netaddr_hasher(static_cast<CNetAddr>(a)) ^ netaddr_hasher(a.source) ^ a.nLastSuccess ^ a.nAttempts ^ a.nRefCount ^ a.fInTried; }; - CAddrInfoEq addrinfo_eq = [](const CAddrInfo& lhs, const CAddrInfo& rhs) { + AddrInfoEq addrinfo_eq = [](const AddrInfo& lhs, const AddrInfo& rhs) { return static_cast<CNetAddr>(lhs) == static_cast<CNetAddr>(rhs) && lhs.source == rhs.source && lhs.nLastSuccess == rhs.nLastSuccess && lhs.nAttempts == rhs.nAttempts && lhs.nRefCount == rhs.nRefCount && lhs.fInTried == rhs.fInTried; }; - using Addresses = std::unordered_set<CAddrInfo, CAddrInfoHasher, CAddrInfoEq>; + using Addresses = std::unordered_set<AddrInfo, AddrInfoHasher, AddrInfoEq>; - const size_t num_addresses{mapInfo.size()}; + const size_t num_addresses{m_impl->mapInfo.size()}; Addresses addresses{num_addresses, addrinfo_hasher, addrinfo_eq}; - for (const auto& [id, addr] : mapInfo) { + for (const auto& [id, addr] : m_impl->mapInfo) { addresses.insert(addr); } Addresses other_addresses{num_addresses, addrinfo_hasher, addrinfo_eq}; - for (const auto& [id, addr] : other.mapInfo) { + for (const auto& [id, addr] : other.m_impl->mapInfo) { other_addresses.insert(addr); } @@ -176,14 +172,14 @@ public: return false; } - auto IdsReferToSameAddress = [&](int id, int other_id) EXCLUSIVE_LOCKS_REQUIRED(cs, other.cs) { + auto IdsReferToSameAddress = [&](int id, int other_id) EXCLUSIVE_LOCKS_REQUIRED(m_impl->cs, other.m_impl->cs) { if (id == -1 && other_id == -1) { return true; } if ((id == -1 && other_id != -1) || (id != -1 && other_id == -1)) { return false; } - return mapInfo.at(id) == other.mapInfo.at(other_id); + return m_impl->mapInfo.at(id) == other.m_impl->mapInfo.at(other_id); }; // Check that `vvNew` contains the same addresses as `other.vvNew`. Notice - `vvNew[i][j]` @@ -191,7 +187,7 @@ public: // themselves may differ between `vvNew` and `other.vvNew`. for (size_t i = 0; i < ADDRMAN_NEW_BUCKET_COUNT; ++i) { for (size_t j = 0; j < ADDRMAN_BUCKET_SIZE; ++j) { - if (!IdsReferToSameAddress(vvNew[i][j], other.vvNew[i][j])) { + if (!IdsReferToSameAddress(m_impl->vvNew[i][j], other.m_impl->vvNew[i][j])) { return false; } } @@ -200,7 +196,7 @@ public: // Same for `vvTried`. for (size_t i = 0; i < ADDRMAN_TRIED_BUCKET_COUNT; ++i) { for (size_t j = 0; j < ADDRMAN_BUCKET_SIZE; ++j) { - if (!IdsReferToSameAddress(vvTried[i][j], other.vvTried[i][j])) { + if (!IdsReferToSameAddress(m_impl->vvTried[i][j], other.m_impl->vvTried[i][j])) { return false; } } @@ -222,7 +218,7 @@ FUZZ_TARGET_INIT(addrman, initialize_addrman) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); SetMockTime(ConsumeTime(fuzzed_data_provider)); std::vector<bool> asmap = ConsumeAsmap(fuzzed_data_provider); - auto addr_man_ptr = std::make_unique<CAddrManDeterministic>(asmap, fuzzed_data_provider); + auto addr_man_ptr = std::make_unique<AddrManDeterministic>(asmap, fuzzed_data_provider); if (fuzzed_data_provider.ConsumeBool()) { const std::vector<uint8_t> serialized_data{ConsumeRandomLengthByteVector(fuzzed_data_provider)}; CDataStream ds(serialized_data, SER_DISK, INIT_PROTO_VERSION); @@ -231,10 +227,10 @@ FUZZ_TARGET_INIT(addrman, initialize_addrman) try { ds >> *addr_man_ptr; } catch (const std::ios_base::failure&) { - addr_man_ptr = std::make_unique<CAddrManDeterministic>(asmap, fuzzed_data_provider); + addr_man_ptr = std::make_unique<AddrManDeterministic>(asmap, fuzzed_data_provider); } } - CAddrManDeterministic& addr_man = *addr_man_ptr; + AddrManDeterministic& addr_man = *addr_man_ptr; while (fuzzed_data_provider.ConsumeBool()) { CallOneOf( fuzzed_data_provider, @@ -283,7 +279,7 @@ FUZZ_TARGET_INIT(addrman, initialize_addrman) } }); } - const CAddrMan& const_addr_man{addr_man}; + const AddrMan& const_addr_man{addr_man}; (void)const_addr_man.GetAddr( /* max_addresses */ fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096), /* max_pct */ fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096), @@ -301,12 +297,12 @@ FUZZ_TARGET_INIT(addrman_serdeser, initialize_addrman) SetMockTime(ConsumeTime(fuzzed_data_provider)); std::vector<bool> asmap = ConsumeAsmap(fuzzed_data_provider); - CAddrManDeterministic addr_man1{asmap, fuzzed_data_provider}; - CAddrManDeterministic addr_man2{asmap, fuzzed_data_provider}; + AddrManDeterministic addr_man1{asmap, fuzzed_data_provider}; + AddrManDeterministic addr_man2{asmap, fuzzed_data_provider}; CDataStream data_stream(SER_NETWORK, PROTOCOL_VERSION); - addr_man1.Fill(); + FillAddrman(addr_man1, fuzzed_data_provider); data_stream << addr_man1; data_stream >> addr_man2; assert(addr_man1 == addr_man2); diff --git a/src/test/fuzz/banman.cpp b/src/test/fuzz/banman.cpp index 561cc83c72..fbba25c404 100644 --- a/src/test/fuzz/banman.cpp +++ b/src/test/fuzz/banman.cpp @@ -48,7 +48,7 @@ FUZZ_TARGET_INIT(banman, initialize_banman) const bool start_with_corrupted_banlist{fuzzed_data_provider.ConsumeBool()}; bool force_read_and_write_to_err{false}; if (start_with_corrupted_banlist) { - assert(WriteBinaryFile(banlist_file.string() + ".json", + assert(WriteBinaryFile(banlist_file + ".json", fuzzed_data_provider.ConsumeRandomLengthString())); } else { force_read_and_write_to_err = fuzzed_data_provider.ConsumeBool(); @@ -111,5 +111,5 @@ FUZZ_TARGET_INIT(banman, initialize_banman) assert(banmap == banmap_read); } } - fs::remove(banlist_file.string() + ".json"); + fs::remove(fs::PathToString(banlist_file + ".json")); } diff --git a/src/test/fuzz/bloom_filter.cpp b/src/test/fuzz/bloom_filter.cpp index c5bb8744a4..746591a176 100644 --- a/src/test/fuzz/bloom_filter.cpp +++ b/src/test/fuzz/bloom_filter.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <bloom.h> +#include <common/bloom.h> #include <primitives/transaction.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index bbdb2c6917..87e70861fa 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> #include <chainparams.h> #include <chainparamsbase.h> #include <coins.h> +#include <consensus/amount.h> #include <consensus/tx_check.h> #include <consensus/tx_verify.h> #include <consensus/validation.h> diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index 01741103e4..d381345a0d 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -25,7 +25,7 @@ FUZZ_TARGET_INIT(connman, initialize_connman) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; SetMockTime(ConsumeTime(fuzzed_data_provider)); - CAddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + AddrMan addrman(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); CConnman connman{fuzzed_data_provider.ConsumeIntegral<uint64_t>(), fuzzed_data_provider.ConsumeIntegral<uint64_t>(), addrman, fuzzed_data_provider.ConsumeBool()}; CNetAddr random_netaddr; CNode random_node = ConsumeNode(fuzzed_data_provider); diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index 83ae1680e3..a9325fa738 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -4,6 +4,7 @@ #include <addrdb.h> #include <addrman.h> +#include <addrman_impl.h> #include <blockencodings.h> #include <blockfilter.h> #include <chain.h> @@ -104,7 +105,7 @@ FUZZ_TARGET_DESERIALIZE(block_filter_deserialize, { DeserializeFromFuzzingInput(buffer, block_filter); }) FUZZ_TARGET_DESERIALIZE(addr_info_deserialize, { - CAddrInfo addr_info; + AddrInfo addr_info; DeserializeFromFuzzingInput(buffer, addr_info); }) FUZZ_TARGET_DESERIALIZE(block_file_info_deserialize, { @@ -188,7 +189,7 @@ FUZZ_TARGET_DESERIALIZE(blockmerkleroot, { BlockMerkleRoot(block, &mutated); }) FUZZ_TARGET_DESERIALIZE(addrman_deserialize, { - CAddrMan am(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + AddrMan am(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); DeserializeFromFuzzingInput(buffer, am); }) FUZZ_TARGET_DESERIALIZE(blockheader_deserialize, { diff --git a/src/test/fuzz/fee_rate.cpp b/src/test/fuzz/fee_rate.cpp index dff0e58000..a852f8fb60 100644 --- a/src/test/fuzz/fee_rate.cpp +++ b/src/test/fuzz/fee_rate.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> +#include <consensus/amount.h> #include <policy/feerate.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> diff --git a/src/test/fuzz/fees.cpp b/src/test/fuzz/fees.cpp index 61c7681bf9..b5a07c7ba3 100644 --- a/src/test/fuzz/fees.cpp +++ b/src/test/fuzz/fees.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> +#include <consensus/amount.h> #include <policy/fees.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index 5a732aeeff..b6c40809e3 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> #include <arith_uint256.h> #include <compressor.h> +#include <consensus/amount.h> #include <consensus/merkle.h> #include <core_io.h> #include <crypto/common.h> diff --git a/src/test/fuzz/locale.cpp b/src/test/fuzz/locale.cpp index 5b1acae57b..4ad8123554 100644 --- a/src/test/fuzz/locale.cpp +++ b/src/test/fuzz/locale.cpp @@ -50,8 +50,6 @@ FUZZ_TARGET(locale) const bool parseint32_without_locale = ParseInt32(random_string, &parseint32_out_without_locale); int64_t parseint64_out_without_locale; const bool parseint64_without_locale = ParseInt64(random_string, &parseint64_out_without_locale); - const int64_t atoi64_without_locale = atoi64(random_string); - const int atoi_without_locale = atoi(random_string); const int64_t random_int64 = fuzzed_data_provider.ConsumeIntegral<int64_t>(); const std::string tostring_without_locale = ToString(random_int64); // The variable `random_int32` is no longer used, but the harness still needs to @@ -77,10 +75,6 @@ FUZZ_TARGET(locale) if (parseint64_without_locale) { assert(parseint64_out_without_locale == parseint64_out_with_locale); } - const int64_t atoi64_with_locale = atoi64(random_string); - assert(atoi64_without_locale == atoi64_with_locale); - const int atoi_with_locale = atoi(random_string); - assert(atoi_without_locale == atoi_with_locale); const std::string tostring_with_locale = ToString(random_int64); assert(tostring_without_locale == tostring_with_locale); const std::string strprintf_int_with_locale = strprintf("%d", random_int64); diff --git a/src/test/fuzz/parse_numbers.cpp b/src/test/fuzz/parse_numbers.cpp index 69e58c3f63..85fee062f0 100644 --- a/src/test/fuzz/parse_numbers.cpp +++ b/src/test/fuzz/parse_numbers.cpp @@ -14,9 +14,6 @@ FUZZ_TARGET(parse_numbers) (void)ParseMoney(random_string); - double d; - (void)ParseDouble(random_string, &d); - uint8_t u8; (void)ParseUInt8(random_string, &u8); @@ -25,13 +22,13 @@ FUZZ_TARGET(parse_numbers) int32_t i32; (void)ParseInt32(random_string, &i32); - (void)atoi(random_string); + (void)LocaleIndependentAtoi<int>(random_string); uint32_t u32; (void)ParseUInt32(random_string, &u32); int64_t i64; - (void)atoi64(random_string); + (void)LocaleIndependentAtoi<int64_t>(random_string); (void)ParseFixedPoint(random_string, 3, &i64); (void)ParseInt64(random_string, &i64); diff --git a/src/test/fuzz/rolling_bloom_filter.cpp b/src/test/fuzz/rolling_bloom_filter.cpp index b9ed497e68..9c18ad49cb 100644 --- a/src/test/fuzz/rolling_bloom_filter.cpp +++ b/src/test/fuzz/rolling_bloom_filter.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <bloom.h> +#include <common/bloom.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> diff --git a/src/test/fuzz/script_flags.cpp b/src/test/fuzz/script_flags.cpp index 1278dc87d4..43927772ae 100644 --- a/src/test/fuzz/script_flags.cpp +++ b/src/test/fuzz/script_flags.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <consensus/amount.h> #include <pubkey.h> #include <script/interpreter.h> #include <streams.h> diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index dc2bf7c860..ab646c68fc 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -122,6 +122,12 @@ bool LegacyParseUInt64(const std::string& str, uint64_t* out) return endp && *endp == 0 && !errno && n <= std::numeric_limits<uint64_t>::max(); } + +// For backwards compatibility checking. +int64_t atoi64_legacy(const std::string& str) +{ + return strtoll(str.c_str(), nullptr, 10); +} }; // namespace FUZZ_TARGET(string) @@ -268,4 +274,22 @@ FUZZ_TARGET(string) assert(u8 == u8_legacy); } } + + { + const int atoi_result = atoi(random_string_1.c_str()); + const int locale_independent_atoi_result = LocaleIndependentAtoi<int>(random_string_1); + const int64_t atoi64_result = atoi64_legacy(random_string_1); + const bool out_of_range = atoi64_result < std::numeric_limits<int>::min() || atoi64_result > std::numeric_limits<int>::max(); + if (out_of_range) { + assert(locale_independent_atoi_result == 0); + } else { + assert(atoi_result == locale_independent_atoi_result); + } + } + + { + const int64_t atoi64_result = atoi64_legacy(random_string_1); + const int64_t locale_independent_atoi_result = LocaleIndependentAtoi<int64_t>(random_string_1); + assert(atoi64_result == locale_independent_atoi_result || locale_independent_atoi_result == 0); + } } diff --git a/src/test/fuzz/txrequest.cpp b/src/test/fuzz/txrequest.cpp index 72438ff2d7..a73bbcfc25 100644 --- a/src/test/fuzz/txrequest.cpp +++ b/src/test/fuzz/txrequest.cpp @@ -204,7 +204,7 @@ public: } // Call TxRequestTracker's implementation. - m_tracker.ReceivedInv(peer, GenTxid{is_wtxid, TXHASHES[txhash]}, preferred, reqtime); + m_tracker.ReceivedInv(peer, is_wtxid ? GenTxid::Wtxid(TXHASHES[txhash]) : GenTxid::Txid(TXHASHES[txhash]), preferred, reqtime); } void RequestedTx(int peer, int txhash, std::chrono::microseconds exptime) @@ -252,7 +252,7 @@ public: for (int peer2 = 0; peer2 < MAX_PEERS; ++peer2) { Announcement& ann2 = m_announcements[txhash][peer2]; if (ann2.m_state == State::REQUESTED && ann2.m_time <= m_now) { - expected_expired.emplace_back(peer2, GenTxid{ann2.m_is_wtxid, TXHASHES[txhash]}); + expected_expired.emplace_back(peer2, ann2.m_is_wtxid ? GenTxid::Wtxid(TXHASHES[txhash]) : GenTxid::Txid(TXHASHES[txhash])); ann2.m_state = State::COMPLETED; break; } diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index 0d87f687d3..d83d2924bb 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <consensus/amount.h> #include <pubkey.h> #include <test/fuzz/util.h> #include <test/util/script.h> diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index bb017b3497..1bc6f1db45 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -5,12 +5,12 @@ #ifndef BITCOIN_TEST_FUZZ_UTIL_H #define BITCOIN_TEST_FUZZ_UTIL_H -#include <amount.h> #include <arith_uint256.h> #include <attributes.h> #include <chainparamsbase.h> #include <coins.h> #include <compat.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <merkleblock.h> #include <net.h> diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index bf36f8a6c9..b3497b8ef8 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -444,12 +444,12 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) pool.addUnchecked(entry.Fee(5000LL).FromTx(tx2)); pool.TrimToSize(pool.DynamicMemoryUsage()); // should do nothing - BOOST_CHECK(pool.exists(tx1.GetHash())); - BOOST_CHECK(pool.exists(tx2.GetHash())); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx1.GetHash()))); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx2.GetHash()))); pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); // should remove the lower-feerate transaction - BOOST_CHECK(pool.exists(tx1.GetHash())); - BOOST_CHECK(!pool.exists(tx2.GetHash())); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx1.GetHash()))); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx2.GetHash()))); pool.addUnchecked(entry.FromTx(tx2)); CMutableTransaction tx3 = CMutableTransaction(); @@ -462,14 +462,14 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) pool.addUnchecked(entry.Fee(20000LL).FromTx(tx3)); pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4); // tx3 should pay for tx2 (CPFP) - BOOST_CHECK(!pool.exists(tx1.GetHash())); - BOOST_CHECK(pool.exists(tx2.GetHash())); - BOOST_CHECK(pool.exists(tx3.GetHash())); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx1.GetHash()))); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx2.GetHash()))); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx3.GetHash()))); pool.TrimToSize(GetVirtualTransactionSize(CTransaction(tx1))); // mempool is limited to tx1's size in memory usage, so nothing fits - BOOST_CHECK(!pool.exists(tx1.GetHash())); - BOOST_CHECK(!pool.exists(tx2.GetHash())); - BOOST_CHECK(!pool.exists(tx3.GetHash())); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx1.GetHash()))); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx2.GetHash()))); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx3.GetHash()))); CFeeRate maxFeeRateRemoved(25000, GetVirtualTransactionSize(CTransaction(tx3)) + GetVirtualTransactionSize(CTransaction(tx2))); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), maxFeeRateRemoved.GetFeePerK() + 1000); @@ -529,19 +529,19 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) // we only require this to remove, at max, 2 txn, because it's not clear what we're really optimizing for aside from that pool.TrimToSize(pool.DynamicMemoryUsage() - 1); - BOOST_CHECK(pool.exists(tx4.GetHash())); - BOOST_CHECK(pool.exists(tx6.GetHash())); - BOOST_CHECK(!pool.exists(tx7.GetHash())); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx4.GetHash()))); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx6.GetHash()))); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx7.GetHash()))); - if (!pool.exists(tx5.GetHash())) + if (!pool.exists(GenTxid::Txid(tx5.GetHash()))) pool.addUnchecked(entry.Fee(1000LL).FromTx(tx5)); pool.addUnchecked(entry.Fee(9000LL).FromTx(tx7)); pool.TrimToSize(pool.DynamicMemoryUsage() / 2); // should maximize mempool size by only removing 5/7 - BOOST_CHECK(pool.exists(tx4.GetHash())); - BOOST_CHECK(!pool.exists(tx5.GetHash())); - BOOST_CHECK(pool.exists(tx6.GetHash())); - BOOST_CHECK(!pool.exists(tx7.GetHash())); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx4.GetHash()))); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx5.GetHash()))); + BOOST_CHECK(pool.exists(GenTxid::Txid(tx6.GetHash()))); + BOOST_CHECK(!pool.exists(GenTxid::Txid(tx7.GetHash()))); pool.addUnchecked(entry.Fee(1000LL).FromTx(tx5)); pool.addUnchecked(entry.Fee(9000LL).FromTx(tx7)); diff --git a/src/test/policy_fee_tests.cpp b/src/test/policy_fee_tests.cpp index 4a15be6ca6..f9c7d04d6c 100644 --- a/src/test/policy_fee_tests.cpp +++ b/src/test/policy_fee_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> +#include <consensus/amount.h> #include <policy/fees.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/script_parse_tests.cpp b/src/test/script_parse_tests.cpp new file mode 100644 index 0000000000..004c1a9a84 --- /dev/null +++ b/src/test/script_parse_tests.cpp @@ -0,0 +1,55 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <core_io.h> +#include <script/script.h> +#include <util/strencodings.h> +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +BOOST_AUTO_TEST_SUITE(script_parse_tests) +BOOST_AUTO_TEST_CASE(parse_script) +{ + const std::vector<std::pair<std::string,std::string>> IN_OUT{ + // {IN: script string , OUT: hex string } + {"", ""}, + {"0", "00"}, + {"1", "51"}, + {"2", "52"}, + {"3", "53"}, + {"4", "54"}, + {"5", "55"}, + {"6", "56"}, + {"7", "57"}, + {"8", "58"}, + {"9", "59"}, + {"10", "5a"}, + {"11", "5b"}, + {"12", "5c"}, + {"13", "5d"}, + {"14", "5e"}, + {"15", "5f"}, + {"16", "60"}, + {"17", "0111"}, + {"-9", "0189"}, + {"0x17", "17"}, + {"'17'", "023137"}, + {"ELSE", "67"}, + {"NOP10", "b9"}, + }; + std::string all_in; + std::string all_out; + for (const auto& [in, out] : IN_OUT) { + BOOST_CHECK_EQUAL(HexStr(ParseScript(in)), out); + all_in += " " + in + " "; + all_out += out; + } + BOOST_CHECK_EQUAL(HexStr(ParseScript(all_in)), all_out); + + BOOST_CHECK_EXCEPTION(ParseScript("11111111111111111111"), std::runtime_error, HasReason("script parse error: decimal numeric value only allowed in the range -0xFFFFFFFF...0xFFFFFFFF")); + BOOST_CHECK_EXCEPTION(ParseScript("11111111111"), std::runtime_error, HasReason("script parse error: decimal numeric value only allowed in the range -0xFFFFFFFF...0xFFFFFFFF")); + BOOST_CHECK_EXCEPTION(ParseScript("OP_CHECKSIGADD"), std::runtime_error, HasReason("script parse error: unknown opcode")); +} +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/settings_tests.cpp b/src/test/settings_tests.cpp index 340ce33d91..15cba9e3e5 100644 --- a/src/test/settings_tests.cpp +++ b/src/test/settings_tests.cpp @@ -80,19 +80,19 @@ BOOST_AUTO_TEST_CASE(ReadWrite) "dupe": "dupe" })"); BOOST_CHECK(!util::ReadSettings(path, values, errors)); - std::vector<std::string> dup_keys = {strprintf("Found duplicate key dupe in settings file %s", path.string())}; + std::vector<std::string> dup_keys = {strprintf("Found duplicate key dupe in settings file %s", fs::PathToString(path))}; BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), dup_keys.begin(), dup_keys.end()); // Check non-kv json files not allowed WriteText(path, R"("non-kv")"); BOOST_CHECK(!util::ReadSettings(path, values, errors)); - std::vector<std::string> non_kv = {strprintf("Found non-object value \"non-kv\" in settings file %s", path.string())}; + std::vector<std::string> non_kv = {strprintf("Found non-object value \"non-kv\" in settings file %s", fs::PathToString(path))}; BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), non_kv.begin(), non_kv.end()); // Check invalid json not allowed WriteText(path, R"(invalid json)"); BOOST_CHECK(!util::ReadSettings(path, values, errors)); - std::vector<std::string> fail_parse = {strprintf("Unable to parse settings file %s", path.string())}; + std::vector<std::string> fail_parse = {strprintf("Unable to parse settings file %s", fs::PathToString(path))}; BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), fail_parse.begin(), fail_parse.end()); } diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 24029ea02e..c813fbea32 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -8,6 +8,7 @@ #include <checkqueue.h> #include <clientversion.h> +#include <consensus/amount.h> #include <consensus/tx_check.h> #include <consensus/validation.h> #include <core_io.h> diff --git a/src/test/txrequest_tests.cpp b/src/test/txrequest_tests.cpp index 1d137b03b1..99d41882c9 100644 --- a/src/test/txrequest_tests.cpp +++ b/src/test/txrequest_tests.cpp @@ -221,7 +221,7 @@ public: /** Generate a random GenTxid; the txhash follows NewTxHash; the is_wtxid flag is random. */ GenTxid NewGTxid(const std::vector<std::vector<NodeId>>& orders = {}) { - return {InsecureRandBool(), NewTxHash(orders)}; + return InsecureRandBool() ? GenTxid::Wtxid(NewTxHash(orders)) : GenTxid::Txid(NewTxHash(orders)); } /** Generate a new random NodeId to use as peer. The same NodeId is never returned twice @@ -494,8 +494,8 @@ void BuildWtxidTest(Scenario& scenario, int config) auto peerT = scenario.NewPeer(); auto peerW = scenario.NewPeer(); auto txhash = scenario.NewTxHash(); - GenTxid txid{false, txhash}; - GenTxid wtxid{true, txhash}; + auto txid{GenTxid::Txid(txhash)}; + auto wtxid{GenTxid::Wtxid(txhash)}; auto reqtimeT = InsecureRandBool() ? MIN_TIME : scenario.Now() + RandomTime8s(); auto reqtimeW = InsecureRandBool() ? MIN_TIME : scenario.Now() + RandomTime8s(); diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h index 81ea4c38f5..e95573022c 100644 --- a/src/test/util/chainstate.h +++ b/src/test/util/chainstate.h @@ -36,7 +36,7 @@ CreateAndActivateUTXOSnapshot(NodeContext& node, const fs::path root, F malleati UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), auto_outfile); BOOST_TEST_MESSAGE( - "Wrote UTXO snapshot to " << snapshot_path.make_preferred().string() << ": " << result.write()); + "Wrote UTXO snapshot to " << fs::PathToString(snapshot_path.make_preferred()) << ": " << result.write()); // Read the written snapshot in and then activate it. // diff --git a/src/test/util/net.h b/src/test/util/net.h index 939ec322ed..d89fc34b75 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -17,6 +17,12 @@ struct ConnmanTestMsg : public CConnman { using CConnman::CConnman; + + void SetPeerConnectTimeout(int64_t timeout) + { + m_peer_connect_timeout = timeout; + } + void AddTestNode(CNode& node) { LOCK(cs_vNodes); diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 97e614379c..a3c7564d76 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -91,8 +91,8 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve extra_args); util::ThreadRename("test"); fs::create_directories(m_path_root); - m_args.ForceSetArg("-datadir", m_path_root.string()); - gArgs.ForceSetArg("-datadir", m_path_root.string()); + m_args.ForceSetArg("-datadir", fs::PathToString(m_path_root)); + gArgs.ForceSetArg("-datadir", fs::PathToString(m_path_root)); gArgs.ClearPathCache(); { SetupServerArgs(*m_node.args); @@ -192,7 +192,7 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString())); } - m_node.addrman = std::make_unique<CAddrMan>(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); + m_node.addrman = std::make_unique<AddrMan>(/* asmap */ std::vector<bool>(), /* deterministic */ false, /* consistency_check_ratio */ 0); m_node.banman = std::make_unique<BanMan>(m_args.GetDataDirBase() / "banlist", nullptr, DEFAULT_MISBEHAVING_BANTIME); m_node.connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman); // Deterministic randomness for tests. m_node.peerman = PeerManager::make(chainparams, *m_node.connman, *m_node.addrman, diff --git a/src/test/util/wallet.cpp b/src/test/util/wallet.cpp index 061659818f..76c1bf93a5 100644 --- a/src/test/util/wallet.cpp +++ b/src/test/util/wallet.cpp @@ -25,16 +25,4 @@ std::string getnewaddress(CWallet& w) return EncodeDestination(dest); } -void importaddress(CWallet& wallet, const std::string& address) -{ - auto spk_man = wallet.GetLegacyScriptPubKeyMan(); - LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); - const auto dest = DecodeDestination(address); - assert(IsValidDestination(dest)); - const auto script = GetScriptForDestination(dest); - wallet.MarkDirty(); - assert(!spk_man->HaveWatchOnly(script)); - if (!spk_man->AddWatchOnly(script, 0 /* nCreateTime */)) assert(false); - wallet.SetAddressBook(dest, /* label */ "", "receive"); -} #endif // ENABLE_WALLET diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index a13700d733..b1300d06ba 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -51,23 +51,23 @@ BOOST_AUTO_TEST_CASE(util_datadir) { // Use local args variable instead of m_args to avoid making assumptions about test setup ArgsManager args; - args.ForceSetArg("-datadir", m_path_root.string()); + args.ForceSetArg("-datadir", fs::PathToString(m_path_root)); const fs::path dd_norm = args.GetDataDirBase(); - args.ForceSetArg("-datadir", dd_norm.string() + "/"); + args.ForceSetArg("-datadir", fs::PathToString(dd_norm) + "/"); args.ClearPathCache(); BOOST_CHECK_EQUAL(dd_norm, args.GetDataDirBase()); - args.ForceSetArg("-datadir", dd_norm.string() + "/."); + args.ForceSetArg("-datadir", fs::PathToString(dd_norm) + "/."); args.ClearPathCache(); BOOST_CHECK_EQUAL(dd_norm, args.GetDataDirBase()); - args.ForceSetArg("-datadir", dd_norm.string() + "/./"); + args.ForceSetArg("-datadir", fs::PathToString(dd_norm) + "/./"); args.ClearPathCache(); BOOST_CHECK_EQUAL(dd_norm, args.GetDataDirBase()); - args.ForceSetArg("-datadir", dd_norm.string() + "/.//"); + args.ForceSetArg("-datadir", fs::PathToString(dd_norm) + "/.//"); args.ClearPathCache(); BOOST_CHECK_EQUAL(dd_norm, args.GetDataDirBase()); } @@ -1181,13 +1181,13 @@ BOOST_AUTO_TEST_CASE(util_ReadWriteSettings) { // Test writing setting. TestArgsManager args1; - args1.ForceSetArg("-datadir", m_path_root.string()); + args1.ForceSetArg("-datadir", fs::PathToString(m_path_root)); args1.LockSettings([&](util::Settings& settings) { settings.rw_settings["name"] = "value"; }); args1.WriteSettingsFile(); // Test reading setting. TestArgsManager args2; - args2.ForceSetArg("-datadir", m_path_root.string()); + args2.ForceSetArg("-datadir", fs::PathToString(m_path_root)); args2.ReadSettingsFile(); args2.LockSettings([&](util::Settings& settings) { BOOST_CHECK_EQUAL(settings.rw_settings["name"].get_str(), "value"); }); @@ -1239,6 +1239,11 @@ BOOST_AUTO_TEST_CASE(util_FormatMoney) BOOST_AUTO_TEST_CASE(util_ParseMoney) { BOOST_CHECK_EQUAL(ParseMoney("0.0").value(), 0); + BOOST_CHECK_EQUAL(ParseMoney(".").value(), 0); + BOOST_CHECK_EQUAL(ParseMoney("0.").value(), 0); + BOOST_CHECK_EQUAL(ParseMoney(".0").value(), 0); + BOOST_CHECK_EQUAL(ParseMoney(".6789").value(), 6789'0000); + BOOST_CHECK_EQUAL(ParseMoney("12345.").value(), COIN * 12345); BOOST_CHECK_EQUAL(ParseMoney("12345.6789").value(), (COIN/10000)*123456789); @@ -1276,11 +1281,18 @@ BOOST_AUTO_TEST_CASE(util_ParseMoney) BOOST_CHECK(!ParseMoney(" ")); // Parsing two numbers should fail + BOOST_CHECK(!ParseMoney("..")); + BOOST_CHECK(!ParseMoney("0..0")); BOOST_CHECK(!ParseMoney("1 2")); BOOST_CHECK(!ParseMoney(" 1 2 ")); BOOST_CHECK(!ParseMoney(" 1.2 3 ")); BOOST_CHECK(!ParseMoney(" 1 2.3 ")); + // Embedded whitespace should fail + BOOST_CHECK(!ParseMoney(" -1 .2 ")); + BOOST_CHECK(!ParseMoney(" 1 .2 ")); + BOOST_CHECK(!ParseMoney(" +1 .2 ")); + // Attempted 63 bit overflow should fail BOOST_CHECK(!ParseMoney("92233720368.54775808")); @@ -1474,6 +1486,35 @@ BOOST_AUTO_TEST_CASE(test_ParseInt32) BOOST_CHECK(!ParseInt32("32482348723847471234", nullptr)); } +template <typename T> +static void RunToIntegralTests() +{ + BOOST_CHECK(!ToIntegral<T>(STRING_WITH_EMBEDDED_NULL_CHAR)); + BOOST_CHECK(!ToIntegral<T>(" 1")); + BOOST_CHECK(!ToIntegral<T>("1 ")); + BOOST_CHECK(!ToIntegral<T>("1a")); + BOOST_CHECK(!ToIntegral<T>("1.1")); + BOOST_CHECK(!ToIntegral<T>("1.9")); + BOOST_CHECK(!ToIntegral<T>("+01.9")); + BOOST_CHECK(!ToIntegral<T>("-")); + BOOST_CHECK(!ToIntegral<T>("+")); + BOOST_CHECK(!ToIntegral<T>(" -1")); + BOOST_CHECK(!ToIntegral<T>("-1 ")); + BOOST_CHECK(!ToIntegral<T>(" -1 ")); + BOOST_CHECK(!ToIntegral<T>("+1")); + BOOST_CHECK(!ToIntegral<T>(" +1")); + BOOST_CHECK(!ToIntegral<T>(" +1 ")); + BOOST_CHECK(!ToIntegral<T>("+-1")); + BOOST_CHECK(!ToIntegral<T>("-+1")); + BOOST_CHECK(!ToIntegral<T>("++1")); + BOOST_CHECK(!ToIntegral<T>("--1")); + BOOST_CHECK(!ToIntegral<T>("")); + BOOST_CHECK(!ToIntegral<T>("aap")); + BOOST_CHECK(!ToIntegral<T>("0x1")); + BOOST_CHECK(!ToIntegral<T>("-32482348723847471234")); + BOOST_CHECK(!ToIntegral<T>("32482348723847471234")); +} + BOOST_AUTO_TEST_CASE(test_ToIntegral) { BOOST_CHECK_EQUAL(ToIntegral<int32_t>("1234").value(), 1'234); @@ -1486,27 +1527,14 @@ BOOST_AUTO_TEST_CASE(test_ToIntegral) BOOST_CHECK_EQUAL(ToIntegral<int32_t>("-1234").value(), -1'234); BOOST_CHECK_EQUAL(ToIntegral<int32_t>("-1").value(), -1); - BOOST_CHECK(!ToIntegral<int32_t>(" 1")); - BOOST_CHECK(!ToIntegral<int32_t>("1 ")); - BOOST_CHECK(!ToIntegral<int32_t>("1a")); - BOOST_CHECK(!ToIntegral<int32_t>("1.1")); - BOOST_CHECK(!ToIntegral<int32_t>("1.9")); - BOOST_CHECK(!ToIntegral<int32_t>("+01.9")); - BOOST_CHECK(!ToIntegral<int32_t>(" -1")); - BOOST_CHECK(!ToIntegral<int32_t>("-1 ")); - BOOST_CHECK(!ToIntegral<int32_t>(" -1 ")); - BOOST_CHECK(!ToIntegral<int32_t>("+1")); - BOOST_CHECK(!ToIntegral<int32_t>(" +1")); - BOOST_CHECK(!ToIntegral<int32_t>(" +1 ")); - BOOST_CHECK(!ToIntegral<int32_t>("+-1")); - BOOST_CHECK(!ToIntegral<int32_t>("-+1")); - BOOST_CHECK(!ToIntegral<int32_t>("++1")); - BOOST_CHECK(!ToIntegral<int32_t>("--1")); - BOOST_CHECK(!ToIntegral<int32_t>("")); - BOOST_CHECK(!ToIntegral<int32_t>("aap")); - BOOST_CHECK(!ToIntegral<int32_t>("0x1")); - BOOST_CHECK(!ToIntegral<int32_t>("-32482348723847471234")); - BOOST_CHECK(!ToIntegral<int32_t>("32482348723847471234")); + RunToIntegralTests<uint64_t>(); + RunToIntegralTests<int64_t>(); + RunToIntegralTests<uint32_t>(); + RunToIntegralTests<int32_t>(); + RunToIntegralTests<uint16_t>(); + RunToIntegralTests<int16_t>(); + RunToIntegralTests<uint8_t>(); + RunToIntegralTests<int8_t>(); BOOST_CHECK(!ToIntegral<int64_t>("-9223372036854775809")); BOOST_CHECK_EQUAL(ToIntegral<int64_t>("-9223372036854775808").value(), -9'223'372'036'854'775'807LL - 1LL); @@ -1549,6 +1577,77 @@ BOOST_AUTO_TEST_CASE(test_ToIntegral) BOOST_CHECK(!ToIntegral<uint8_t>("256")); } +BOOST_AUTO_TEST_CASE(test_LocaleIndependentAtoi) +{ + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("1234"), 1'234); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("0"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("01234"), 1'234); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-1234"), -1'234); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>(" 1"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("1 "), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("1a"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("1.1"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("1.9"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("+01.9"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-1"), -1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>(" -1"), -1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-1 "), -1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>(" -1 "), -1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("+1"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>(" +1"), 1); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>(" +1 "), 1); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("+-1"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-+1"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("++1"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("--1"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>(""), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("aap"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("0x1"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-32482348723847471234"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("32482348723847471234"), 0); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int64_t>("-9223372036854775809"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int64_t>("-9223372036854775808"), -9'223'372'036'854'775'807LL - 1LL); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int64_t>("9223372036854775807"), 9'223'372'036'854'775'807); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int64_t>("9223372036854775808"), 0); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint64_t>("-1"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint64_t>("0"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint64_t>("18446744073709551615"), 18'446'744'073'709'551'615ULL); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint64_t>("18446744073709551616"), 0U); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-2147483649"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("-2147483648"), -2'147'483'648LL); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("2147483647"), 2'147'483'647); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int32_t>("2147483648"), 0); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint32_t>("-1"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint32_t>("0"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint32_t>("4294967295"), 4'294'967'295U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint32_t>("4294967296"), 0U); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int16_t>("-32769"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int16_t>("-32768"), -32'768); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int16_t>("32767"), 32'767); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int16_t>("32768"), 0); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint16_t>("-1"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint16_t>("0"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint16_t>("65535"), 65'535U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint16_t>("65536"), 0U); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int8_t>("-129"), 0); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int8_t>("-128"), -128); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int8_t>("127"), 127); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<int8_t>("128"), 0); + + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint8_t>("-1"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint8_t>("0"), 0U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint8_t>("255"), 255U); + BOOST_CHECK_EQUAL(LocaleIndependentAtoi<uint8_t>("256"), 0U); +} + BOOST_AUTO_TEST_CASE(test_ParseInt64) { int64_t n; @@ -1714,32 +1813,6 @@ BOOST_AUTO_TEST_CASE(test_ParseUInt64) BOOST_CHECK(!ParseUInt64("-1234", &n)); } -BOOST_AUTO_TEST_CASE(test_ParseDouble) -{ - double n; - // Valid values - BOOST_CHECK(ParseDouble("1234", nullptr)); - BOOST_CHECK(ParseDouble("0", &n) && n == 0.0); - BOOST_CHECK(ParseDouble("1234", &n) && n == 1234.0); - BOOST_CHECK(ParseDouble("01234", &n) && n == 1234.0); // no octal - BOOST_CHECK(ParseDouble("2147483647", &n) && n == 2147483647.0); - BOOST_CHECK(ParseDouble("-2147483648", &n) && n == -2147483648.0); - BOOST_CHECK(ParseDouble("-1234", &n) && n == -1234.0); - BOOST_CHECK(ParseDouble("1e6", &n) && n == 1e6); - BOOST_CHECK(ParseDouble("-1e6", &n) && n == -1e6); - // Invalid values - BOOST_CHECK(!ParseDouble("", &n)); - BOOST_CHECK(!ParseDouble(" 1", &n)); // no padding inside - BOOST_CHECK(!ParseDouble("1 ", &n)); - BOOST_CHECK(!ParseDouble("1a", &n)); - BOOST_CHECK(!ParseDouble("aap", &n)); - BOOST_CHECK(!ParseDouble("0x1", &n)); // no hex - BOOST_CHECK(!ParseDouble(STRING_WITH_EMBEDDED_NULL_CHAR, &n)); - // Overflow and underflow - BOOST_CHECK(!ParseDouble("-1e10000", nullptr)); - BOOST_CHECK(!ParseDouble("1e10000", nullptr)); -} - BOOST_AUTO_TEST_CASE(test_FormatParagraph) { BOOST_CHECK_EQUAL(FormatParagraph("", 79, 0), ""); diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 726c9ebbb8..9bb08f774f 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -107,20 +107,21 @@ BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) curr_tip = ::g_best_block; - CChainState* background_cs; - BOOST_CHECK_EQUAL(chainman.GetAll().size(), 2); - for (CChainState* cs : chainman.GetAll()) { - if (cs != &chainman.ActiveChainstate()) { - background_cs = cs; + + CChainState& background_cs{*[&] { + for (CChainState* cs : chainman.GetAll()) { + if (cs != &chainman.ActiveChainstate()) { + return cs; + } } - } - BOOST_CHECK(background_cs); + assert(false); + }()}; // Create a block to append to the validation chain. std::vector<CMutableTransaction> noTxns; CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; - CBlock validation_block = this->CreateBlock(noTxns, scriptPubKey, *background_cs); + CBlock validation_block = this->CreateBlock(noTxns, scriptPubKey, background_cs); auto pblock = std::make_shared<const CBlock>(validation_block); BlockValidationState state; CBlockIndex* pindex = nullptr; @@ -133,15 +134,15 @@ BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) LOCK(::cs_main); bool checked = CheckBlock(*pblock, state, chainparams.GetConsensus()); BOOST_CHECK(checked); - bool accepted = background_cs->AcceptBlock( + bool accepted = background_cs.AcceptBlock( pblock, state, &pindex, true, nullptr, &newblock); BOOST_CHECK(accepted); } // UpdateTip is called here - bool block_added = background_cs->ActivateBestChain(state, pblock); + bool block_added = background_cs.ActivateBestChain(state, pblock); // Ensure tip is as expected - BOOST_CHECK_EQUAL(background_cs->m_chain.Tip()->GetBlockHash(), validation_block.GetHash()); + BOOST_CHECK_EQUAL(background_cs.m_chain.Tip()->GetBlockHash(), validation_block.GetHash()); // g_best_block should be unchanged after adding a block to the background // validation chain. diff --git a/src/test/validation_tests.cpp b/src/test/validation_tests.cpp index a0c2e76f00..ca52ecba2f 100644 --- a/src/test/validation_tests.cpp +++ b/src/test/validation_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> +#include <consensus/amount.h> #include <net.h> #include <signet.h> #include <uint256.h> diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index bb296456ba..55618a5c57 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -14,6 +14,7 @@ #include <netbase.h> #include <util/readwritefile.h> #include <util/strencodings.h> +#include <util/syscall_sandbox.h> #include <util/system.h> #include <util/thread.h> #include <util/time.h> @@ -83,7 +84,7 @@ void TorControlConnection::readcb(struct bufferevent *bev, void *ctx) if (s.size() < 4) // Short line continue; // <status>(-|+| )<data><CRLF> - self->message.code = atoi(s.substr(0,3)); + self->message.code = LocaleIndependentAtoi<int>(s.substr(0,3)); self->message.lines.push_back(s.substr(4)); char ch = s[3]; // '-','+' or ' ' if (ch == ' ') { @@ -317,7 +318,7 @@ TorController::TorController(struct event_base* _base, const std::string& tor_co // Read service private key if cached std::pair<bool,std::string> pkf = ReadBinaryFile(GetPrivateKeyFile()); if (pkf.first) { - LogPrint(BCLog::TOR, "tor: Reading cached private key from %s\n", GetPrivateKeyFile().string()); + LogPrint(BCLog::TOR, "tor: Reading cached private key from %s\n", fs::PathToString(GetPrivateKeyFile())); private_key = pkf.second; } } @@ -355,9 +356,9 @@ void TorController::add_onion_cb(TorControlConnection& _conn, const TorControlRe service = LookupNumeric(std::string(service_id+".onion"), Params().GetDefaultPort()); LogPrintf("tor: Got service ID %s, advertising service %s\n", service_id, service.ToString()); if (WriteBinaryFile(GetPrivateKeyFile(), private_key)) { - LogPrint(BCLog::TOR, "tor: Cached service private key to %s\n", GetPrivateKeyFile().string()); + LogPrint(BCLog::TOR, "tor: Cached service private key to %s\n", fs::PathToString(GetPrivateKeyFile())); } else { - LogPrintf("tor: Error writing service private key to %s\n", GetPrivateKeyFile().string()); + LogPrintf("tor: Error writing service private key to %s\n", fs::PathToString(GetPrivateKeyFile())); } AddLocal(service, LOCAL_MANUAL); // ... onion requested - keep connection open @@ -507,7 +508,7 @@ void TorController::protocolinfo_cb(TorControlConnection& _conn, const TorContro } else if (methods.count("SAFECOOKIE")) { // Cookie: hexdump -e '32/1 "%02x""\n"' ~/.tor/control_auth_cookie LogPrint(BCLog::TOR, "tor: Using SAFECOOKIE authentication, reading cookie authentication from %s\n", cookiefile); - std::pair<bool,std::string> status_cookie = ReadBinaryFile(cookiefile, TOR_COOKIE_SIZE); + std::pair<bool,std::string> status_cookie = ReadBinaryFile(fs::PathFromString(cookiefile), TOR_COOKIE_SIZE); if (status_cookie.first && status_cookie.second.size() == TOR_COOKIE_SIZE) { // _conn.Command("AUTHENTICATE " + HexStr(status_cookie.second), std::bind(&TorController::auth_cb, this, std::placeholders::_1, std::placeholders::_2)); cookie = std::vector<uint8_t>(status_cookie.second.begin(), status_cookie.second.end()); @@ -585,6 +586,7 @@ static std::thread torControlThread; static void TorControlThread(CService onion_service_target) { + SetSyscallSandboxPolicy(SyscallSandboxPolicy::TOR_CONTROL); TorController ctrl(gBase, gArgs.GetArg("-torcontrol", DEFAULT_TOR_CONTROL), onion_service_target); event_base_dispatch(gBase); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index a0d9e2a6bf..9bc2377c63 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -874,8 +874,6 @@ TxMempoolInfo CTxMemPool::info(const GenTxid& gtxid) const return GetInfo(i); } -TxMempoolInfo CTxMemPool::info(const uint256& txid) const { return info(GenTxid{false, txid}); } - void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeDelta) { { @@ -948,7 +946,7 @@ CTxMemPool::setEntries CTxMemPool::GetIterSet(const std::set<uint256>& hashes) c bool CTxMemPool::HasNoInputsOf(const CTransaction &tx) const { for (unsigned int i = 0; i < tx.vin.size(); i++) - if (exists(tx.vin[i].prevout.hash)) + if (exists(GenTxid::Txid(tx.vin[i].prevout.hash))) return false; return true; } @@ -1119,7 +1117,7 @@ void CTxMemPool::TrimToSize(size_t sizelimit, std::vector<COutPoint>* pvNoSpends if (pvNoSpendsRemaining) { for (const CTransaction& tx : txn) { for (const CTxIn& txin : tx.vin) { - if (exists(txin.prevout.hash)) continue; + if (exists(GenTxid::Txid(txin.prevout.hash))) continue; pvNoSpendsRemaining->push_back(txin.prevout); } } diff --git a/src/txmempool.h b/src/txmempool.h index a3a11eb72b..90b2aee371 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -14,8 +14,8 @@ #include <utility> #include <vector> -#include <amount.h> #include <coins.h> +#include <consensus/amount.h> #include <indirectmap.h> #include <policy/feerate.h> #include <policy/packages.h> @@ -542,7 +542,7 @@ public: * By design, it is guaranteed that: * * 1. Locking both `cs_main` and `mempool.cs` will give a view of mempool - * that is consistent with current chain tip (`::ChainActive()` and + * that is consistent with current chain tip (`ActiveChain()` and * `CoinsTip()`) and is fully populated. Fully populated means that if the * current active chain is missing transactions that were present in a * previously active chain, all the missing transactions will have been @@ -782,7 +782,6 @@ public: } return (mapTx.count(gtxid.GetHash()) != 0); } - bool exists(const uint256& txid) const { return exists(GenTxid{false, txid}); } CTransactionRef get(const uint256& hash) const; txiter get_iter_from_wtxid(const uint256& wtxid) const EXCLUSIVE_LOCKS_REQUIRED(cs) @@ -790,7 +789,6 @@ public: AssertLockHeld(cs); return mapTx.project<0>(mapTx.get<index_by_wtxid>().find(wtxid)); } - TxMempoolInfo info(const uint256& hash) const; TxMempoolInfo info(const GenTxid& gtxid) const; std::vector<TxMempoolInfo> infoAll() const; @@ -802,7 +800,7 @@ public: LOCK(cs); // Sanity check the transaction is in the mempool & insert into // unbroadcast set. - if (exists(txid)) m_unbroadcast_txids.insert(txid); + if (exists(GenTxid::Txid(txid))) m_unbroadcast_txids.insert(txid); }; /** Removes a transaction from the unbroadcast set */ diff --git a/src/txrequest.cpp b/src/txrequest.cpp index f8d7a1ece8..7d478a5b26 100644 --- a/src/txrequest.cpp +++ b/src/txrequest.cpp @@ -300,7 +300,7 @@ std::map<uint256, TxHashInfo> ComputeTxHashInfo(const Index& index, const Priori GenTxid ToGenTxid(const Announcement& ann) { - return {ann.m_is_wtxid, ann.m_txhash}; + return ann.m_is_wtxid ? GenTxid::Wtxid(ann.m_txhash) : GenTxid::Txid(ann.m_txhash); } } // namespace diff --git a/src/univalue/.cirrus.yml b/src/univalue/.cirrus.yml new file mode 100644 index 0000000000..f140fee12b --- /dev/null +++ b/src/univalue/.cirrus.yml @@ -0,0 +1,44 @@ +env: + MAKEJOBS: "-j4" + RUN_TESTS: "true" + BASE_OUTDIR: "$CIRRUS_WORKING_DIR/out_dir_base" + DEBIAN_FRONTEND: "noninteractive" + +task: + container: + image: ubuntu:focal + cpu: 1 + memory: 1G + greedy: true # https://medium.com/cirruslabs/introducing-greedy-container-instances-29aad06dc2b4 + + matrix: + - name: "gcc" + env: + CC: "gcc" + CXX: "g++" + APT_PKGS: "gcc" + - name: "clang" + env: + CC: "clang" + CXX: "clang++" + APT_PKGS: "clang" + - name: "mingw" + env: + CC: "" + CXX: "" + UNIVALUE_CONFIG: "--host=x86_64-w64-mingw32" + APT_PKGS: "g++-mingw-w64-x86-64 gcc-mingw-w64-x86-64 binutils-mingw-w64-x86-64" + RUN_TESTS: "false" + + install_script: + - apt update + - apt install -y pkg-config build-essential libtool autotools-dev automake bsdmainutils + - apt install -y $APT_PKGS + autogen_script: + - ./autogen.sh + configure_script: + - ./configure --cache-file=config.cache --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib $UNIVALUE_CONFIG + make_script: + - make $MAKEJOBS V=1 + test_script: + - if [ "$RUN_TESTS" = "true" ]; then make $MAKEJOBS distcheck; fi diff --git a/src/univalue/.travis.yml b/src/univalue/.travis.yml deleted file mode 100644 index 43a1ed362e..0000000000 --- a/src/univalue/.travis.yml +++ /dev/null @@ -1,51 +0,0 @@ -language: cpp - -compiler: - - clang - - gcc - -os: - - linux - - osx - -sudo: false - -env: - global: - - MAKEJOBS=-j3 - - RUN_TESTS=true - - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out - -cache: - apt: true - -addons: - apt: - packages: - - pkg-config - -before_script: - - if [ -n "$USE_SHELL" ]; then export CONFIG_SHELL="$USE_SHELL"; fi - - test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh"' || ./autogen.sh - -script: - - if [ -n "$UNIVALUE_CONFIG" ]; then unset CC; unset CXX; fi - - OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST - - UNIVALUE_CONFIG_ALL="--prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib" - - ./configure --cache-file=config.cache $UNIVALUE_CONFIG_ALL $UNIVALUE_CONFIG || ( cat config.log && false) - - make -s $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL ; false ) - - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib - - if [ "$RUN_TESTS" = "true" ]; then make $MAKEJOBS distcheck; fi - -matrix: - fast_finish: true - include: - - os: linux - compiler: gcc - env: UNIVALUE_CONFIG=--host=x86_64-w64-mingw32 RUN_TESTS=false - addons: - apt: - packages: - - g++-mingw-w64-x86-64 - - gcc-mingw-w64-x86-64 - - binutils-mingw-w64-x86-64 diff --git a/src/univalue/Makefile.am b/src/univalue/Makefile.am index 0f5ba59954..476f14b922 100644 --- a/src/univalue/Makefile.am +++ b/src/univalue/Makefile.am @@ -1,20 +1,17 @@ +include sources.mk ACLOCAL_AMFLAGS = -I build-aux/m4 -.PHONY: gen +.PHONY: gen FORCE .INTERMEDIATE: $(GENBIN) -include_HEADERS = include/univalue.h -noinst_HEADERS = lib/univalue_escapes.h lib/univalue_utffilter.h +include_HEADERS = $(UNIVALUE_DIST_HEADERS_INT) +noinst_HEADERS = $(UNIVALUE_LIB_HEADERS_INT) lib_LTLIBRARIES = libunivalue.la pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = pc/libunivalue.pc -libunivalue_la_SOURCES = \ - lib/univalue.cpp \ - lib/univalue_get.cpp \ - lib/univalue_read.cpp \ - lib/univalue_write.cpp +libunivalue_la_SOURCES = $(UNIVALUE_LIB_SOURCES_INT) libunivalue_la_LDFLAGS = \ -version-info $(LIBUNIVALUE_CURRENT):$(LIBUNIVALUE_REVISION):$(LIBUNIVALUE_AGE) \ @@ -30,89 +27,32 @@ $(GENBIN): $(GEN_SRCS) @echo Building $@ $(AM_V_at)c++ -I$(top_srcdir)/include -o $@ $< -gen: lib/univalue_escapes.h $(GENBIN) - @echo Updating $< +gen: $(GENBIN) FORCE + @echo Updating lib/univalue_escapes.h $(AM_V_at)$(GENBIN) > lib/univalue_escapes.h noinst_PROGRAMS = $(TESTS) test/test_json -TEST_DATA_DIR=test - -test_unitester_SOURCES = test/unitester.cpp +test_unitester_SOURCES = $(UNIVALUE_TEST_UNITESTER_INT) test_unitester_LDADD = libunivalue.la -test_unitester_CXXFLAGS = -I$(top_srcdir)/include -DJSON_TEST_SRC=\"$(srcdir)/$(TEST_DATA_DIR)\" +test_unitester_CXXFLAGS = -I$(top_srcdir)/include -DJSON_TEST_SRC=\"$(srcdir)/$(UNIVALUE_TEST_DATA_DIR_INT)\" test_unitester_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) -test_test_json_SOURCES = test/test_json.cpp +test_test_json_SOURCES = $(UNIVALUE_TEST_JSON_INT) test_test_json_LDADD = libunivalue.la test_test_json_CXXFLAGS = -I$(top_srcdir)/include test_test_json_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) -test_no_nul_SOURCES = test/no_nul.cpp +test_no_nul_SOURCES = $(UNIVALUE_TEST_NO_NUL_INT) test_no_nul_LDADD = libunivalue.la test_no_nul_CXXFLAGS = -I$(top_srcdir)/include test_no_nul_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) -test_object_SOURCES = test/object.cpp +test_object_SOURCES = $(UNIVALUE_TEST_OBJECT_INT) test_object_LDADD = libunivalue.la test_object_CXXFLAGS = -I$(top_srcdir)/include test_object_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) -TEST_FILES = \ - $(TEST_DATA_DIR)/fail10.json \ - $(TEST_DATA_DIR)/fail11.json \ - $(TEST_DATA_DIR)/fail12.json \ - $(TEST_DATA_DIR)/fail13.json \ - $(TEST_DATA_DIR)/fail14.json \ - $(TEST_DATA_DIR)/fail15.json \ - $(TEST_DATA_DIR)/fail16.json \ - $(TEST_DATA_DIR)/fail17.json \ - $(TEST_DATA_DIR)/fail18.json \ - $(TEST_DATA_DIR)/fail19.json \ - $(TEST_DATA_DIR)/fail1.json \ - $(TEST_DATA_DIR)/fail20.json \ - $(TEST_DATA_DIR)/fail21.json \ - $(TEST_DATA_DIR)/fail22.json \ - $(TEST_DATA_DIR)/fail23.json \ - $(TEST_DATA_DIR)/fail24.json \ - $(TEST_DATA_DIR)/fail25.json \ - $(TEST_DATA_DIR)/fail26.json \ - $(TEST_DATA_DIR)/fail27.json \ - $(TEST_DATA_DIR)/fail28.json \ - $(TEST_DATA_DIR)/fail29.json \ - $(TEST_DATA_DIR)/fail2.json \ - $(TEST_DATA_DIR)/fail30.json \ - $(TEST_DATA_DIR)/fail31.json \ - $(TEST_DATA_DIR)/fail32.json \ - $(TEST_DATA_DIR)/fail33.json \ - $(TEST_DATA_DIR)/fail34.json \ - $(TEST_DATA_DIR)/fail35.json \ - $(TEST_DATA_DIR)/fail36.json \ - $(TEST_DATA_DIR)/fail37.json \ - $(TEST_DATA_DIR)/fail38.json \ - $(TEST_DATA_DIR)/fail39.json \ - $(TEST_DATA_DIR)/fail40.json \ - $(TEST_DATA_DIR)/fail41.json \ - $(TEST_DATA_DIR)/fail42.json \ - $(TEST_DATA_DIR)/fail44.json \ - $(TEST_DATA_DIR)/fail45.json \ - $(TEST_DATA_DIR)/fail3.json \ - $(TEST_DATA_DIR)/fail4.json \ - $(TEST_DATA_DIR)/fail5.json \ - $(TEST_DATA_DIR)/fail6.json \ - $(TEST_DATA_DIR)/fail7.json \ - $(TEST_DATA_DIR)/fail8.json \ - $(TEST_DATA_DIR)/fail9.json \ - $(TEST_DATA_DIR)/pass1.json \ - $(TEST_DATA_DIR)/pass2.json \ - $(TEST_DATA_DIR)/pass3.json \ - $(TEST_DATA_DIR)/pass4.json \ - $(TEST_DATA_DIR)/round1.json \ - $(TEST_DATA_DIR)/round2.json \ - $(TEST_DATA_DIR)/round3.json \ - $(TEST_DATA_DIR)/round4.json \ - $(TEST_DATA_DIR)/round5.json \ - $(TEST_DATA_DIR)/round6.json \ - $(TEST_DATA_DIR)/round7.json - -EXTRA_DIST=$(TEST_FILES) $(GEN_SRCS) +TEST_FILES = $(UNIVALUE_TEST_FILES_INT) + +EXTRA_DIST=$(UNIVALUE_TEST_FILES_INT) $(GEN_SRCS) diff --git a/src/univalue/build-aux/m4/ax_cxx_compile_stdcxx.m4 b/src/univalue/build-aux/m4/ax_cxx_compile_stdcxx.m4 new file mode 100644 index 0000000000..f7e5137003 --- /dev/null +++ b/src/univalue/build-aux/m4/ax_cxx_compile_stdcxx.m4 @@ -0,0 +1,962 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX(VERSION, [ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the specified +# version of the C++ standard. If necessary, add switches to CXX and +# CXXCPP to enable support. VERSION may be '11' (for the C++11 standard) +# or '14' (for the C++14 standard). +# +# The second argument, if specified, indicates whether you insist on an +# extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g. +# -std=c++11). If neither is specified, you get whatever works, with +# preference for no added switch, and then for an extended mode. +# +# The third argument, if specified 'mandatory' or if left unspecified, +# indicates that baseline support for the specified C++ standard is +# required and that the macro should error out if no mode with that +# support is found. If specified 'optional', then configuration proceeds +# regardless, after defining HAVE_CXX${VERSION} if and only if a +# supporting mode is found. +# +# LICENSE +# +# Copyright (c) 2008 Benjamin Kosnik <bkoz@redhat.com> +# Copyright (c) 2012 Zack Weinberg <zackw@panix.com> +# Copyright (c) 2013 Roy Stogner <roystgnr@ices.utexas.edu> +# Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov <sokolov@google.com> +# Copyright (c) 2015 Paul Norman <penorman@mac.com> +# Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu> +# Copyright (c) 2016, 2018 Krzesimir Nowak <qdlacz@gmail.com> +# Copyright (c) 2019 Enji Cooper <yaneurabeya@gmail.com> +# Copyright (c) 2020 Jason Merrill <jason@redhat.com> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 12 + +dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro +dnl (serial version number 13). + +AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl + m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"], + [$1], [14], [ax_cxx_compile_alternatives="14 1y"], + [$1], [17], [ax_cxx_compile_alternatives="17 1z"], + [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$2], [], [], + [$2], [ext], [], + [$2], [noext], [], + [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$3], [], [ax_cxx_compile_cxx$1_required=true], + [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true], + [$3], [optional], [ax_cxx_compile_cxx$1_required=false], + [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])]) + AC_LANG_PUSH([C++])dnl + ac_success=no + + m4_if([$2], [], [dnl + AC_CACHE_CHECK(whether $CXX supports C++$1 features by default, + ax_cv_cxx_compile_cxx$1, + [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [ax_cv_cxx_compile_cxx$1=yes], + [ax_cv_cxx_compile_cxx$1=no])]) + if test x$ax_cv_cxx_compile_cxx$1 = xyes; then + ac_success=yes + fi]) + + m4_if([$2], [noext], [], [dnl + if test x$ac_success = xno; then + for alternative in ${ax_cxx_compile_alternatives}; do + switch="-std=gnu++${alternative}" + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + fi]) + + m4_if([$2], [ext], [], [dnl + if test x$ac_success = xno; then + dnl HP's aCC needs +std=c++11 according to: + dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf + dnl Cray's crayCC needs "-h std=c++11" + for alternative in ${ax_cxx_compile_alternatives}; do + for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}"; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + if test x$ac_success = xyes; then + break + fi + done + fi]) + AC_LANG_POP([C++]) + if test x$ax_cxx_compile_cxx$1_required = xtrue; then + if test x$ac_success = xno; then + AC_MSG_ERROR([*** A compiler with support for C++$1 language features is required.]) + fi + fi + if test x$ac_success = xno; then + HAVE_CXX$1=0 + AC_MSG_NOTICE([No compiler with C++$1 support was found]) + else + HAVE_CXX$1=1 + AC_DEFINE(HAVE_CXX$1,1, + [define if the compiler supports basic C++$1 syntax]) + fi + AC_SUBST(HAVE_CXX$1) +]) + + +dnl Test body for checking C++11 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 +) + + +dnl Test body for checking C++14 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 +) + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_17 +) + +dnl Tests for new features in C++11 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[ + +// If the compiler admits that it is not ready for C++11, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201103L + +#error "This is not a C++11 compiler" + +#else + +namespace cxx11 +{ + + namespace test_static_assert + { + + template <typename T> + struct check + { + static_assert(sizeof(int) <= sizeof(T), "not big enough"); + }; + + } + + namespace test_final_override + { + + struct Base + { + virtual ~Base() {} + virtual void f() {} + }; + + struct Derived : public Base + { + virtual ~Derived() override {} + virtual void f() override {} + }; + + } + + namespace test_double_right_angle_brackets + { + + template < typename T > + struct check {}; + + typedef check<void> single_type; + typedef check<check<void>> double_type; + typedef check<check<check<void>>> triple_type; + typedef check<check<check<check<void>>>> quadruple_type; + + } + + namespace test_decltype + { + + int + f() + { + int a = 1; + decltype(a) b = 2; + return a + b; + } + + } + + namespace test_type_deduction + { + + template < typename T1, typename T2 > + struct is_same + { + static const bool value = false; + }; + + template < typename T > + struct is_same<T, T> + { + static const bool value = true; + }; + + template < typename T1, typename T2 > + auto + add(T1 a1, T2 a2) -> decltype(a1 + a2) + { + return a1 + a2; + } + + int + test(const int c, volatile int v) + { + static_assert(is_same<int, decltype(0)>::value == true, ""); + static_assert(is_same<int, decltype(c)>::value == false, ""); + static_assert(is_same<int, decltype(v)>::value == false, ""); + auto ac = c; + auto av = v; + auto sumi = ac + av + 'x'; + auto sumf = ac + av + 1.0; + static_assert(is_same<int, decltype(ac)>::value == true, ""); + static_assert(is_same<int, decltype(av)>::value == true, ""); + static_assert(is_same<int, decltype(sumi)>::value == true, ""); + static_assert(is_same<int, decltype(sumf)>::value == false, ""); + static_assert(is_same<int, decltype(add(c, v))>::value == true, ""); + return (sumf > 0.0) ? sumi : add(c, v); + } + + } + + namespace test_noexcept + { + + int f() { return 0; } + int g() noexcept { return 0; } + + static_assert(noexcept(f()) == false, ""); + static_assert(noexcept(g()) == true, ""); + + } + + namespace test_constexpr + { + + template < typename CharT > + unsigned long constexpr + strlen_c_r(const CharT *const s, const unsigned long acc) noexcept + { + return *s ? strlen_c_r(s + 1, acc + 1) : acc; + } + + template < typename CharT > + unsigned long constexpr + strlen_c(const CharT *const s) noexcept + { + return strlen_c_r(s, 0UL); + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("1") == 1UL, ""); + static_assert(strlen_c("example") == 7UL, ""); + static_assert(strlen_c("another\0example") == 7UL, ""); + + } + + namespace test_rvalue_references + { + + template < int N > + struct answer + { + static constexpr int value = N; + }; + + answer<1> f(int&) { return answer<1>(); } + answer<2> f(const int&) { return answer<2>(); } + answer<3> f(int&&) { return answer<3>(); } + + void + test() + { + int i = 0; + const int c = 0; + static_assert(decltype(f(i))::value == 1, ""); + static_assert(decltype(f(c))::value == 2, ""); + static_assert(decltype(f(0))::value == 3, ""); + } + + } + + namespace test_uniform_initialization + { + + struct test + { + static const int zero {}; + static const int one {1}; + }; + + static_assert(test::zero == 0, ""); + static_assert(test::one == 1, ""); + + } + + namespace test_lambdas + { + + void + test1() + { + auto lambda1 = [](){}; + auto lambda2 = lambda1; + lambda1(); + lambda2(); + } + + int + test2() + { + auto a = [](int i, int j){ return i + j; }(1, 2); + auto b = []() -> int { return '0'; }(); + auto c = [=](){ return a + b; }(); + auto d = [&](){ return c; }(); + auto e = [a, &b](int x) mutable { + const auto identity = [](int y){ return y; }; + for (auto i = 0; i < a; ++i) + a += b--; + return x + identity(a + b); + }(0); + return a + b + c + d + e; + } + + int + test3() + { + const auto nullary = [](){ return 0; }; + const auto unary = [](int x){ return x; }; + using nullary_t = decltype(nullary); + using unary_t = decltype(unary); + const auto higher1st = [](nullary_t f){ return f(); }; + const auto higher2nd = [unary](nullary_t f1){ + return [unary, f1](unary_t f2){ return f2(unary(f1())); }; + }; + return higher1st(nullary) + higher2nd(nullary)(unary); + } + + } + + namespace test_variadic_templates + { + + template <int...> + struct sum; + + template <int N0, int... N1toN> + struct sum<N0, N1toN...> + { + static constexpr auto value = N0 + sum<N1toN...>::value; + }; + + template <> + struct sum<> + { + static constexpr auto value = 0; + }; + + static_assert(sum<>::value == 0, ""); + static_assert(sum<1>::value == 1, ""); + static_assert(sum<23>::value == 23, ""); + static_assert(sum<1, 2>::value == 3, ""); + static_assert(sum<5, 5, 11>::value == 21, ""); + static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, ""); + + } + + // http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae + // Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function + // because of this. + namespace test_template_alias_sfinae + { + + struct foo {}; + + template<typename T> + using member = typename T::member_type; + + template<typename T> + void func(...) {} + + template<typename T> + void func(member<T>*) {} + + void test(); + + void test() { func<foo>(0); } + + } + +} // namespace cxx11 + +#endif // __cplusplus >= 201103L + +]]) + + +dnl Tests for new features in C++14 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_14], [[ + +// If the compiler admits that it is not ready for C++14, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201402L + +#error "This is not a C++14 compiler" + +#else + +namespace cxx14 +{ + + namespace test_polymorphic_lambdas + { + + int + test() + { + const auto lambda = [](auto&&... args){ + const auto istiny = [](auto x){ + return (sizeof(x) == 1UL) ? 1 : 0; + }; + const int aretiny[] = { istiny(args)... }; + return aretiny[0]; + }; + return lambda(1, 1L, 1.0f, '1'); + } + + } + + namespace test_binary_literals + { + + constexpr auto ivii = 0b0000000000101010; + static_assert(ivii == 42, "wrong value"); + + } + + namespace test_generalized_constexpr + { + + template < typename CharT > + constexpr unsigned long + strlen_c(const CharT *const s) noexcept + { + auto length = 0UL; + for (auto p = s; *p; ++p) + ++length; + return length; + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("x") == 1UL, ""); + static_assert(strlen_c("test") == 4UL, ""); + static_assert(strlen_c("another\0test") == 7UL, ""); + + } + + namespace test_lambda_init_capture + { + + int + test() + { + auto x = 0; + const auto lambda1 = [a = x](int b){ return a + b; }; + const auto lambda2 = [a = lambda1(x)](){ return a; }; + return lambda2(); + } + + } + + namespace test_digit_separators + { + + constexpr auto ten_million = 100'000'000; + static_assert(ten_million == 100000000, ""); + + } + + namespace test_return_type_deduction + { + + auto f(int& x) { return x; } + decltype(auto) g(int& x) { return x; } + + template < typename T1, typename T2 > + struct is_same + { + static constexpr auto value = false; + }; + + template < typename T > + struct is_same<T, T> + { + static constexpr auto value = true; + }; + + int + test() + { + auto x = 0; + static_assert(is_same<int, decltype(f(x))>::value, ""); + static_assert(is_same<int&, decltype(g(x))>::value, ""); + return x; + } + + } + +} // namespace cxx14 + +#endif // __cplusplus >= 201402L + +]]) + + +dnl Tests for new features in C++17 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_17], [[ + +// If the compiler admits that it is not ready for C++17, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201703L + +#error "This is not a C++17 compiler" + +#else + +#include <initializer_list> +#include <utility> +#include <type_traits> + +namespace cxx17 +{ + + namespace test_constexpr_lambdas + { + + constexpr int foo = [](){return 42;}(); + + } + + namespace test::nested_namespace::definitions + { + + } + + namespace test_fold_expression + { + + template<typename... Args> + int multiply(Args... args) + { + return (args * ... * 1); + } + + template<typename... Args> + bool all(Args... args) + { + return (args && ...); + } + + } + + namespace test_extended_static_assert + { + + static_assert (true); + + } + + namespace test_auto_brace_init_list + { + + auto foo = {5}; + auto bar {5}; + + static_assert(std::is_same<std::initializer_list<int>, decltype(foo)>::value); + static_assert(std::is_same<int, decltype(bar)>::value); + } + + namespace test_typename_in_template_template_parameter + { + + template<template<typename> typename X> struct D; + + } + + namespace test_fallthrough_nodiscard_maybe_unused_attributes + { + + int f1() + { + return 42; + } + + [[nodiscard]] int f2() + { + [[maybe_unused]] auto unused = f1(); + + switch (f1()) + { + case 17: + f1(); + [[fallthrough]]; + case 42: + f1(); + } + return f1(); + } + + } + + namespace test_extended_aggregate_initialization + { + + struct base1 + { + int b1, b2 = 42; + }; + + struct base2 + { + base2() { + b3 = 42; + } + int b3; + }; + + struct derived : base1, base2 + { + int d; + }; + + derived d1 {{1, 2}, {}, 4}; // full initialization + derived d2 {{}, {}, 4}; // value-initialized bases + + } + + namespace test_general_range_based_for_loop + { + + struct iter + { + int i; + + int& operator* () + { + return i; + } + + const int& operator* () const + { + return i; + } + + iter& operator++() + { + ++i; + return *this; + } + }; + + struct sentinel + { + int i; + }; + + bool operator== (const iter& i, const sentinel& s) + { + return i.i == s.i; + } + + bool operator!= (const iter& i, const sentinel& s) + { + return !(i == s); + } + + struct range + { + iter begin() const + { + return {0}; + } + + sentinel end() const + { + return {5}; + } + }; + + void f() + { + range r {}; + + for (auto i : r) + { + [[maybe_unused]] auto v = i; + } + } + + } + + namespace test_lambda_capture_asterisk_this_by_value + { + + struct t + { + int i; + int foo() + { + return [*this]() + { + return i; + }(); + } + }; + + } + + namespace test_enum_class_construction + { + + enum class byte : unsigned char + {}; + + byte foo {42}; + + } + + namespace test_constexpr_if + { + + template <bool cond> + int f () + { + if constexpr(cond) + { + return 13; + } + else + { + return 42; + } + } + + } + + namespace test_selection_statement_with_initializer + { + + int f() + { + return 13; + } + + int f2() + { + if (auto i = f(); i > 0) + { + return 3; + } + + switch (auto i = f(); i + 4) + { + case 17: + return 2; + + default: + return 1; + } + } + + } + + namespace test_template_argument_deduction_for_class_templates + { + + template <typename T1, typename T2> + struct pair + { + pair (T1 p1, T2 p2) + : m1 {p1}, + m2 {p2} + {} + + T1 m1; + T2 m2; + }; + + void f() + { + [[maybe_unused]] auto p = pair{13, 42u}; + } + + } + + namespace test_non_type_auto_template_parameters + { + + template <auto n> + struct B + {}; + + B<5> b1; + B<'a'> b2; + + } + + namespace test_structured_bindings + { + + int arr[2] = { 1, 2 }; + std::pair<int, int> pr = { 1, 2 }; + + auto f1() -> int(&)[2] + { + return arr; + } + + auto f2() -> std::pair<int, int>& + { + return pr; + } + + struct S + { + int x1 : 2; + volatile double y1; + }; + + S f3() + { + return {}; + } + + auto [ x1, y1 ] = f1(); + auto& [ xr1, yr1 ] = f1(); + auto [ x2, y2 ] = f2(); + auto& [ xr2, yr2 ] = f2(); + const auto [ x3, y3 ] = f3(); + + } + + namespace test_exception_spec_type_system + { + + struct Good {}; + struct Bad {}; + + void g1() noexcept; + void g2(); + + template<typename T> + Bad + f(T*, T*); + + template<typename T1, typename T2> + Good + f(T1*, T2*); + + static_assert (std::is_same_v<Good, decltype(f(g1, g2))>); + + } + + namespace test_inline_variables + { + + template<class T> void f(T) + {} + + template<class T> inline T g(T) + { + return T{}; + } + + template<> inline void f<>(int) + {} + + template<> int g<>(int) + { + return 5; + } + + } + +} // namespace cxx17 + +#endif // __cplusplus < 201703L + +]]) diff --git a/src/univalue/configure.ac b/src/univalue/configure.ac index 8298332ac1..495b25a53d 100644 --- a/src/univalue/configure.ac +++ b/src/univalue/configure.ac @@ -1,7 +1,7 @@ m4_define([libunivalue_major_version], [1]) m4_define([libunivalue_minor_version], [1]) -m4_define([libunivalue_micro_version], [3]) -m4_define([libunivalue_interface_age], [3]) +m4_define([libunivalue_micro_version], [4]) +m4_define([libunivalue_interface_age], [4]) # If you need a modifier for the version number. # Normally empty, but can be used to make "fixup" releases. m4_define([libunivalue_extraversion], []) @@ -14,7 +14,7 @@ m4_define([libunivalue_age], [m4_eval(libunivalue_binary_age - libunivalue_inter m4_define([libunivalue_version], [libunivalue_major_version().libunivalue_minor_version().libunivalue_micro_version()libunivalue_extraversion()]) -AC_INIT([univalue], [1.0.3], +AC_INIT([univalue], [1.0.4], [http://github.com/jgarzik/univalue/]) dnl make the compilation flags quiet unless V=1 is used @@ -45,6 +45,9 @@ AC_SUBST(LIBUNIVALUE_AGE) LT_INIT LT_LANG([C++]) +dnl Require C++11 compiler (no GNU extensions) +AX_CXX_COMPILE_STDCXX([11], [noext], [mandatory], [nodefault]) + case $host in *mingw*) LIBTOOL_APP_LDFLAGS="$LIBTOOL_APP_LDFLAGS -all-static" diff --git a/src/univalue/gen/gen.cpp b/src/univalue/gen/gen.cpp index 85fe20924a..b8a6c73f4e 100644 --- a/src/univalue/gen/gen.cpp +++ b/src/univalue/gen/gen.cpp @@ -1,6 +1,6 @@ // Copyright 2014 BitPay Inc. // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. // // To re-create univalue_escapes.h: @@ -45,7 +45,7 @@ static void outputEscape() for (unsigned int i = 0; i < 256; i++) { if (escapes[i].empty()) { - printf("\tNULL,\n"); + printf("\tnullptr,\n"); } else { printf("\t\""); diff --git a/src/univalue/include/univalue.h b/src/univalue/include/univalue.h index 048e162f7d..fc5cf402be 100644 --- a/src/univalue/include/univalue.h +++ b/src/univalue/include/univalue.h @@ -1,7 +1,7 @@ // Copyright 2014 BitPay Inc. // Copyright 2015 Bitcoin Core Developers // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #ifndef __UNIVALUE_H__ #define __UNIVALUE_H__ @@ -14,8 +14,6 @@ #include <map> #include <cassert> -#include <sstream> // .get_int64() - class UniValue { public: enum VType { VNULL, VOBJ, VARR, VSTR, VNUM, VBOOL, }; diff --git a/src/univalue/lib/univalue.cpp b/src/univalue/lib/univalue.cpp index 4c9c15d63e..c4e59fae74 100644 --- a/src/univalue/lib/univalue.cpp +++ b/src/univalue/lib/univalue.cpp @@ -1,7 +1,7 @@ // Copyright 2014 BitPay Inc. // Copyright 2015 Bitcoin Core Developers // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include <stdint.h> #include <iomanip> @@ -178,17 +178,19 @@ bool UniValue::findKey(const std::string& key, size_t& retIdx) const bool UniValue::checkObject(const std::map<std::string,UniValue::VType>& t) const { - if (typ != VOBJ) + if (typ != VOBJ) { return false; + } - for (std::map<std::string,UniValue::VType>::const_iterator it = t.begin(); - it != t.end(); ++it) { + for (const auto& object: t) { size_t idx = 0; - if (!findKey(it->first, idx)) + if (!findKey(object.first, idx)) { return false; + } - if (values.at(idx).getType() != it->second) + if (values.at(idx).getType() != object.second) { return false; + } } return true; @@ -228,7 +230,7 @@ const char *uvTypeName(UniValue::VType t) } // not reached - return NULL; + return nullptr; } const UniValue& find_value(const UniValue& obj, const std::string& name) diff --git a/src/univalue/lib/univalue_escapes.h b/src/univalue/lib/univalue_escapes.h index 74596aab6d..3f714f8e5b 100644 --- a/src/univalue/lib/univalue_escapes.h +++ b/src/univalue/lib/univalue_escapes.h @@ -34,229 +34,229 @@ static const char *escapes[256] = { "\\u001d", "\\u001e", "\\u001f", - NULL, - NULL, + nullptr, + nullptr, "\\\"", - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, "\\\\", - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, "\\u007f", - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, }; #endif // BITCOIN_UNIVALUE_UNIVALUE_ESCAPES_H diff --git a/src/univalue/lib/univalue_get.cpp b/src/univalue/lib/univalue_get.cpp index 0ad6146545..5af89a3561 100644 --- a/src/univalue/lib/univalue_get.cpp +++ b/src/univalue/lib/univalue_get.cpp @@ -1,7 +1,7 @@ // Copyright 2014 BitPay Inc. // Copyright 2015 Bitcoin Core Developers // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include <stdint.h> #include <errno.h> @@ -11,6 +11,7 @@ #include <vector> #include <limits> #include <string> +#include <sstream> #include "univalue.h" @@ -31,7 +32,7 @@ bool ParseInt32(const std::string& str, int32_t *out) { if (!ParsePrechecks(str)) return false; - char *endp = NULL; + char *endp = nullptr; errno = 0; // strtol will not set errno if valid long int n = strtol(str.c_str(), &endp, 10); if(out) *out = (int32_t)n; @@ -47,7 +48,7 @@ bool ParseInt64(const std::string& str, int64_t *out) { if (!ParsePrechecks(str)) return false; - char *endp = NULL; + char *endp = nullptr; errno = 0; // strtoll will not set errno if valid long long int n = strtoll(str.c_str(), &endp, 10); if(out) *out = (int64_t)n; diff --git a/src/univalue/lib/univalue_read.cpp b/src/univalue/lib/univalue_read.cpp index 5c6a1acf75..be39bfe57a 100644 --- a/src/univalue/lib/univalue_read.cpp +++ b/src/univalue/lib/univalue_read.cpp @@ -1,6 +1,6 @@ // Copyright 2014 BitPay Inc. // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include <string.h> #include <vector> @@ -227,7 +227,7 @@ enum jtokentype getJsonToken(std::string& tokenVal, unsigned int& consumed, } else { - writer.push_back(*raw); + writer.push_back(static_cast<unsigned char>(*raw)); raw++; } } @@ -244,7 +244,7 @@ enum jtokentype getJsonToken(std::string& tokenVal, unsigned int& consumed, } } -enum expect_bits { +enum expect_bits : unsigned { EXP_OBJ_NAME = (1U << 0), EXP_COLON = (1U << 1), EXP_ARR_VALUE = (1U << 2), diff --git a/src/univalue/lib/univalue_utffilter.h b/src/univalue/lib/univalue_utffilter.h index 20d4043009..c24ac58eaf 100644 --- a/src/univalue/lib/univalue_utffilter.h +++ b/src/univalue/lib/univalue_utffilter.h @@ -1,6 +1,6 @@ // Copyright 2016 Wladimir J. van der Laan // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #ifndef UNIVALUE_UTFFILTER_H #define UNIVALUE_UTFFILTER_H diff --git a/src/univalue/lib/univalue_write.cpp b/src/univalue/lib/univalue_write.cpp index 827eb9b271..3a2c580c7f 100644 --- a/src/univalue/lib/univalue_write.cpp +++ b/src/univalue/lib/univalue_write.cpp @@ -1,9 +1,8 @@ // Copyright 2014 BitPay Inc. // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include <iomanip> -#include <sstream> #include <stdio.h> #include "univalue.h" #include "univalue_escapes.h" @@ -14,13 +13,13 @@ static std::string json_escape(const std::string& inS) outS.reserve(inS.size() * 2); for (unsigned int i = 0; i < inS.size(); i++) { - unsigned char ch = inS[i]; + unsigned char ch = static_cast<unsigned char>(inS[i]); const char *escStr = escapes[ch]; if (escStr) outS += escStr; else - outS += ch; + outS += static_cast<char>(ch); } return outS; diff --git a/src/univalue/sources.mk b/src/univalue/sources.mk new file mode 100644 index 0000000000..efab6d277f --- /dev/null +++ b/src/univalue/sources.mk @@ -0,0 +1,95 @@ +# - All variables are namespaced with UNIVALUE_ to avoid colliding with +# downstream makefiles. +# - All Variables ending in _HEADERS or _SOURCES confuse automake, so the +# _INT postfix is applied. +# - Convenience variables, for example a UNIVALUE_TEST_DIR should not be used +# as they interfere with automatic dependency generation +# - The %reldir% is the relative path from the Makefile.am. This allows +# downstreams to use these variables without having to manually account for +# the path change. + +UNIVALUE_INCLUDE_DIR_INT = %reldir%/include + +UNIVALUE_DIST_HEADERS_INT = +UNIVALUE_DIST_HEADERS_INT += %reldir%/include/univalue.h + +UNIVALUE_LIB_HEADERS_INT = +UNIVALUE_LIB_HEADERS_INT += %reldir%/lib/univalue_utffilter.h +UNIVALUE_LIB_HEADERS_INT += %reldir%/lib/univalue_escapes.h + +UNIVALUE_LIB_SOURCES_INT = +UNIVALUE_LIB_SOURCES_INT += %reldir%/lib/univalue.cpp +UNIVALUE_LIB_SOURCES_INT += %reldir%/lib/univalue_get.cpp +UNIVALUE_LIB_SOURCES_INT += %reldir%/lib/univalue_read.cpp +UNIVALUE_LIB_SOURCES_INT += %reldir%/lib/univalue_write.cpp + +UNIVALUE_TEST_DATA_DIR_INT = %reldir%/test + +UNIVALUE_TEST_UNITESTER_INT = +UNIVALUE_TEST_UNITESTER_INT += %reldir%/test/unitester.cpp + +UNIVALUE_TEST_JSON_INT = +UNIVALUE_TEST_JSON_INT += %reldir%/test/test_json.cpp + +UNIVALUE_TEST_NO_NUL_INT = +UNIVALUE_TEST_NO_NUL_INT += %reldir%/test/no_nul.cpp + +UNIVALUE_TEST_OBJECT_INT = +UNIVALUE_TEST_OBJECT_INT += %reldir%/test/object.cpp + +UNIVALUE_TEST_FILES_INT = +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail1.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail2.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail3.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail4.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail5.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail6.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail7.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail8.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail9.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail10.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail11.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail12.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail13.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail14.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail15.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail16.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail17.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail18.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail19.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail20.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail21.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail22.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail23.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail24.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail25.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail26.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail27.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail28.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail29.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail30.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail31.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail32.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail33.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail34.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail35.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail36.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail37.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail38.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail39.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail40.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail41.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail42.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail44.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/fail45.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/pass1.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/pass2.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/pass3.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/pass4.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round1.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round2.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round3.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round4.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round5.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round6.json +UNIVALUE_TEST_FILES_INT += %reldir%/test/round7.json diff --git a/src/univalue/test/object.cpp b/src/univalue/test/object.cpp index ccc1344836..c2f52f83ac 100644 --- a/src/univalue/test/object.cpp +++ b/src/univalue/test/object.cpp @@ -1,7 +1,7 @@ // Copyright (c) 2014 BitPay Inc. // Copyright (c) 2014-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include <stdint.h> #include <vector> diff --git a/src/univalue/test/unitester.cpp b/src/univalue/test/unitester.cpp index 2308afbcdf..02e1a83c6d 100644 --- a/src/univalue/test/unitester.cpp +++ b/src/univalue/test/unitester.cpp @@ -1,6 +1,6 @@ // Copyright 2014 BitPay Inc. // Distributed under the MIT/X11 software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// file COPYING or https://opensource.org/licenses/mit-license.php. #include <stdlib.h> #include <stdio.h> @@ -58,7 +58,7 @@ static void runtest_file(const char *filename_) std::string basename(filename_); std::string filename = srcdir + "/" + basename; FILE *f = fopen(filename.c_str(), "r"); - assert(f != NULL); + assert(f != nullptr); std::string jdata; diff --git a/src/util/asmap.cpp b/src/util/asmap.cpp index 5695c62012..b696c65e9d 100644 --- a/src/util/asmap.cpp +++ b/src/util/asmap.cpp @@ -201,7 +201,7 @@ std::vector<bool> DecodeAsmap(fs::path path) } fseek(filestr, 0, SEEK_END); int length = ftell(filestr); - LogPrintf("Opened asmap file %s (%d bytes) from disk\n", path, length); + LogPrintf("Opened asmap file %s (%d bytes) from disk\n", fs::quoted(fs::PathToString(path)), length); fseek(filestr, 0, SEEK_SET); uint8_t cur_byte; for (int i = 0; i < length; ++i) { @@ -211,7 +211,7 @@ std::vector<bool> DecodeAsmap(fs::path path) } } if (!SanityCheckASMap(bits, 128)) { - LogPrintf("Sanity check of asmap file %s failed\n", path); + LogPrintf("Sanity check of asmap file %s failed\n", fs::quoted(fs::PathToString(path))); return {}; } return bits; diff --git a/src/util/moneystr.cpp b/src/util/moneystr.cpp index d3f4029607..1aed7daacf 100644 --- a/src/util/moneystr.cpp +++ b/src/util/moneystr.cpp @@ -5,7 +5,7 @@ #include <util/moneystr.h> -#include <amount.h> +#include <consensus/amount.h> #include <tinyformat.h> #include <util/strencodings.h> #include <util/string.h> @@ -77,8 +77,7 @@ std::optional<CAmount> ParseMoney(const std::string& money_string) return std::nullopt; if (nUnits < 0 || nUnits > COIN) return std::nullopt; - int64_t nWhole = atoi64(strWhole); - + int64_t nWhole = LocaleIndependentAtoi<int64_t>(strWhole); CAmount value = nWhole * COIN + nUnits; if (!MoneyRange(value)) { diff --git a/src/util/moneystr.h b/src/util/moneystr.h index b71dffd0db..f37dc1cffd 100644 --- a/src/util/moneystr.h +++ b/src/util/moneystr.h @@ -9,8 +9,8 @@ #ifndef BITCOIN_UTIL_MONEYSTR_H #define BITCOIN_UTIL_MONEYSTR_H -#include <amount.h> #include <attributes.h> +#include <consensus/amount.h> #include <optional> #include <string> diff --git a/src/util/settings.cpp b/src/util/settings.cpp index 846b34089d..7fb35c073e 100644 --- a/src/util/settings.cpp +++ b/src/util/settings.cpp @@ -66,24 +66,24 @@ bool ReadSettings(const fs::path& path, std::map<std::string, SettingsValue>& va fsbridge::ifstream file; file.open(path); if (!file.is_open()) { - errors.emplace_back(strprintf("%s. Please check permissions.", path.string())); + errors.emplace_back(strprintf("%s. Please check permissions.", fs::PathToString(path))); return false; } SettingsValue in; if (!in.read(std::string{std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>()})) { - errors.emplace_back(strprintf("Unable to parse settings file %s", path.string())); + errors.emplace_back(strprintf("Unable to parse settings file %s", fs::PathToString(path))); return false; } if (file.fail()) { - errors.emplace_back(strprintf("Failed reading settings file %s", path.string())); + errors.emplace_back(strprintf("Failed reading settings file %s", fs::PathToString(path))); return false; } file.close(); // Done with file descriptor. Release while copying data. if (!in.isObject()) { - errors.emplace_back(strprintf("Found non-object value %s in settings file %s", in.write(), path.string())); + errors.emplace_back(strprintf("Found non-object value %s in settings file %s", in.write(), fs::PathToString(path))); return false; } @@ -92,7 +92,7 @@ bool ReadSettings(const fs::path& path, std::map<std::string, SettingsValue>& va for (size_t i = 0; i < in_keys.size(); ++i) { auto inserted = values.emplace(in_keys[i], in_values[i]); if (!inserted.second) { - errors.emplace_back(strprintf("Found duplicate key %s in settings file %s", in_keys[i], path.string())); + errors.emplace_back(strprintf("Found duplicate key %s in settings file %s", in_keys[i], fs::PathToString(path))); } } return errors.empty(); @@ -109,7 +109,7 @@ bool WriteSettings(const fs::path& path, fsbridge::ofstream file; file.open(path); if (file.fail()) { - errors.emplace_back(strprintf("Error: Unable to open settings file %s for writing", path.string())); + errors.emplace_back(strprintf("Error: Unable to open settings file %s for writing", fs::PathToString(path))); return false; } file << out.write(/* prettyIndent= */ 1, /* indentLevel= */ 4) << std::endl; diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index 0aa80ea0ae..15bd07b374 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -281,16 +281,11 @@ std::string DecodeBase32(const std::string& str, bool* pf_invalid) return std::string((const char*)vchRet.data(), vchRet.size()); } -[[nodiscard]] static bool ParsePrechecks(const std::string&); - namespace { template <typename T> bool ParseIntegral(const std::string& str, T* out) { static_assert(std::is_integral<T>::value); - if (!ParsePrechecks(str)) { - return false; - } // Replicate the exact behavior of strtol/strtoll/strtoul/strtoull when // handling leading +/- for backwards compatibility. if (str.length() >= 2 && str[0] == '+' && str[1] == '-') { @@ -307,17 +302,6 @@ bool ParseIntegral(const std::string& str, T* out) } }; // namespace -[[nodiscard]] static bool ParsePrechecks(const std::string& str) -{ - if (str.empty()) // No empty string allowed - return false; - if (str.size() >= 1 && (IsSpace(str[0]) || IsSpace(str[str.size()-1]))) // No padding allowed - return false; - if (!ValidAsCString(str)) // No embedded NUL characters allowed - return false; - return true; -} - bool ParseInt32(const std::string& str, int32_t* out) { return ParseIntegral<int32_t>(str, out); @@ -348,20 +332,6 @@ bool ParseUInt64(const std::string& str, uint64_t* out) return ParseIntegral<uint64_t>(str, out); } -bool ParseDouble(const std::string& str, double *out) -{ - if (!ParsePrechecks(str)) - return false; - if (str.size() >= 2 && str[0] == '0' && str[1] == 'x') // No hexadecimal floats allowed - return false; - std::istringstream text(str); - text.imbue(std::locale::classic()); - double result; - text >> result; - if(out) *out = result; - return text.eof() && !text.fail(); -} - std::string FormatParagraph(const std::string& in, size_t width, size_t indent) { std::stringstream out; @@ -403,20 +373,6 @@ std::string FormatParagraph(const std::string& in, size_t width, size_t indent) return out.str(); } -int64_t atoi64(const std::string& str) -{ -#ifdef _MSC_VER - return _atoi64(str.c_str()); -#else - return strtoll(str.c_str(), nullptr, 10); -#endif -} - -int atoi(const std::string& str) -{ - return atoi(str.c_str()); -} - /** Upper bound for mantissa. * 10^18-1 is the largest arbitrary decimal that will fit in a signed 64-bit integer. * Larger integers cannot consist of arbitrary combinations of 0-9: diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 1217572c45..eedb5ec2f8 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -11,6 +11,7 @@ #include <attributes.h> #include <span.h> +#include <util/string.h> #include <charconv> #include <cstdint> @@ -68,8 +69,33 @@ std::string EncodeBase32(Span<const unsigned char> input, bool pad = true); std::string EncodeBase32(const std::string& str, bool pad = true); void SplitHostPort(std::string in, uint16_t& portOut, std::string& hostOut); -int64_t atoi64(const std::string& str); -int atoi(const std::string& str); + +// LocaleIndependentAtoi is provided for backwards compatibility reasons. +// +// New code should use ToIntegral or the ParseInt* functions +// which provide parse error feedback. +// +// The goal of LocaleIndependentAtoi is to replicate the exact defined behaviour +// of atoi and atoi64 as they behave under the "C" locale. +template <typename T> +T LocaleIndependentAtoi(const std::string& str) +{ + static_assert(std::is_integral<T>::value); + T result; + // Emulate atoi(...) handling of white space and leading +/-. + std::string s = TrimString(str); + if (!s.empty() && s[0] == '+') { + if (s.length() >= 2 && s[1] == '-') { + return 0; + } + s = s.substr(1); + } + auto [_, error_condition] = std::from_chars(s.data(), s.data() + s.size(), result); + if (error_condition != std::errc{}) { + return 0; + } + return result; +} /** * Tests if the given character is a decimal digit. @@ -97,7 +123,9 @@ constexpr inline bool IsSpace(char c) noexcept { } /** - * Convert string to integral type T. + * Convert string to integral type T. Leading whitespace, a leading +, or any + * trailing character fail the parsing. The required format expressed as regex + * is `-?[0-9]+`. The minus sign is only permitted for signed integer types. * * @returns std::nullopt if the entire string could not be parsed, or if the * parsed value is not in the range representable by the type T. @@ -111,7 +139,7 @@ std::optional<T> ToIntegral(const std::string& str) if (first_nonmatching != str.data() + str.size() || error_condition != std::errc{}) { return std::nullopt; } - return {result}; + return result; } /** @@ -157,13 +185,6 @@ std::optional<T> ToIntegral(const std::string& str) [[nodiscard]] bool ParseUInt64(const std::string& str, uint64_t *out); /** - * Convert string to double with strict parse error feedback. - * @returns true if the entire string could be parsed as valid double, - * false if not the entire string could be parsed or when overflow or underflow occurred. - */ -[[nodiscard]] bool ParseDouble(const std::string& str, double *out); - -/** * Convert a span of bytes to a lower-case hexadecimal string. */ std::string HexStr(const Span<const uint8_t> s); diff --git a/src/util/syscall_sandbox.cpp b/src/util/syscall_sandbox.cpp new file mode 100644 index 0000000000..bc69df44f4 --- /dev/null +++ b/src/util/syscall_sandbox.cpp @@ -0,0 +1,919 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif // defined(HAVE_CONFIG_H) + +#include <util/syscall_sandbox.h> + +#if defined(USE_SYSCALL_SANDBOX) +#include <array> +#include <cassert> +#include <cstdint> +#include <exception> +#include <map> +#include <new> +#include <set> +#include <string> +#include <vector> + +#include <logging.h> +#include <tinyformat.h> +#include <util/threadnames.h> + +#include <linux/audit.h> +#include <linux/filter.h> +#include <linux/seccomp.h> +#include <linux/unistd.h> +#include <signal.h> +#include <sys/prctl.h> +#include <sys/types.h> +#include <unistd.h> + +namespace { +bool g_syscall_sandbox_enabled{false}; +bool g_syscall_sandbox_log_violation_before_terminating{false}; + +#if !defined(__x86_64__) +#error Syscall sandbox is an experimental feature currently available only under Linux x86-64. +#endif // defined(__x86_64__) + +#ifndef SECCOMP_RET_KILL_PROCESS +#define SECCOMP_RET_KILL_PROCESS 0x80000000U +#endif + +// Define system call numbers for x86_64 that are referenced in the system call profile +// but not provided by the kernel headers used in the GUIX build. +// Usually, they can be found via "grep name /usr/include/x86_64-linux-gnu/asm/unistd_64.h" + +#ifndef __NR_clone3 +#define __NR_clone3 435 +#endif + +#ifndef __NR_statx +#define __NR_statx 332 +#endif + +#ifndef __NR_getrandom +#define __NR_getrandom 318 +#endif + +#ifndef __NR_membarrier +#define __NR_membarrier 324 +#endif + +#ifndef __NR_copy_file_range +#define __NR_copy_file_range 326 +#endif + +// This list of syscalls in LINUX_SYSCALLS is only used to map syscall numbers to syscall names in +// order to be able to print user friendly error messages which include the syscall name in addition +// to the syscall number. +// +// Example output in case of a syscall violation where the syscall is present in LINUX_SYSCALLS: +// +// ``` +// 2021-06-09T12:34:56Z ERROR: The syscall "execve" (syscall number 59) is not allowed by the syscall sandbox in thread "msghand". Please report. +// ``` +// +// Example output in case of a syscall violation where the syscall is not present in LINUX_SYSCALLS: +// +// ``` +// 2021-06-09T12:34:56Z ERROR: The syscall "*unknown*" (syscall number 314) is not allowed by the syscall sandbox in thread "msghand". Please report. +// `` +// +// LINUX_SYSCALLS contains two types of syscalls: +// 1.) Syscalls that are present under all architectures or relevant Linux kernel versions for which +// we support the syscall sandbox feature (currently only Linux x86-64). Examples include read, +// write, open, close, etc. +// 2.) Syscalls that are present under a subset of architectures or relevant Linux kernel versions +// for which we support the syscall sandbox feature. This type of syscalls should be added to +// LINUX_SYSCALLS conditional on availability like in the following example: +// ... +// #if defined(__NR_arch_dependent_syscall) +// {__NR_arch_dependent_syscall, "arch_dependent_syscall"}, +// #endif // defined(__NR_arch_dependent_syscall) +// ... +const std::map<uint32_t, std::string> LINUX_SYSCALLS{ + {__NR_accept, "accept"}, + {__NR_accept4, "accept4"}, + {__NR_access, "access"}, + {__NR_acct, "acct"}, + {__NR_add_key, "add_key"}, + {__NR_adjtimex, "adjtimex"}, + {__NR_afs_syscall, "afs_syscall"}, + {__NR_alarm, "alarm"}, + {__NR_arch_prctl, "arch_prctl"}, + {__NR_bind, "bind"}, + {__NR_bpf, "bpf"}, + {__NR_brk, "brk"}, + {__NR_capget, "capget"}, + {__NR_capset, "capset"}, + {__NR_chdir, "chdir"}, + {__NR_chmod, "chmod"}, + {__NR_chown, "chown"}, + {__NR_chroot, "chroot"}, + {__NR_clock_adjtime, "clock_adjtime"}, + {__NR_clock_getres, "clock_getres"}, + {__NR_clock_gettime, "clock_gettime"}, + {__NR_clock_nanosleep, "clock_nanosleep"}, + {__NR_clock_settime, "clock_settime"}, + {__NR_clone, "clone"}, + {__NR_clone3, "clone3"}, + {__NR_close, "close"}, + {__NR_connect, "connect"}, + {__NR_copy_file_range, "copy_file_range"}, + {__NR_creat, "creat"}, + {__NR_create_module, "create_module"}, + {__NR_delete_module, "delete_module"}, + {__NR_dup, "dup"}, + {__NR_dup2, "dup2"}, + {__NR_dup3, "dup3"}, + {__NR_epoll_create, "epoll_create"}, + {__NR_epoll_create1, "epoll_create1"}, + {__NR_epoll_ctl, "epoll_ctl"}, + {__NR_epoll_ctl_old, "epoll_ctl_old"}, + {__NR_epoll_pwait, "epoll_pwait"}, + {__NR_epoll_wait, "epoll_wait"}, + {__NR_epoll_wait_old, "epoll_wait_old"}, + {__NR_eventfd, "eventfd"}, + {__NR_eventfd2, "eventfd2"}, + {__NR_execve, "execve"}, + {__NR_execveat, "execveat"}, + {__NR_exit, "exit"}, + {__NR_exit_group, "exit_group"}, + {__NR_faccessat, "faccessat"}, + {__NR_fadvise64, "fadvise64"}, + {__NR_fallocate, "fallocate"}, + {__NR_fanotify_init, "fanotify_init"}, + {__NR_fanotify_mark, "fanotify_mark"}, + {__NR_fchdir, "fchdir"}, + {__NR_fchmod, "fchmod"}, + {__NR_fchmodat, "fchmodat"}, + {__NR_fchown, "fchown"}, + {__NR_fchownat, "fchownat"}, + {__NR_fcntl, "fcntl"}, + {__NR_fdatasync, "fdatasync"}, + {__NR_fgetxattr, "fgetxattr"}, + {__NR_finit_module, "finit_module"}, + {__NR_flistxattr, "flistxattr"}, + {__NR_flock, "flock"}, + {__NR_fork, "fork"}, + {__NR_fremovexattr, "fremovexattr"}, + {__NR_fsetxattr, "fsetxattr"}, + {__NR_fstat, "fstat"}, + {__NR_fstatfs, "fstatfs"}, + {__NR_fsync, "fsync"}, + {__NR_ftruncate, "ftruncate"}, + {__NR_futex, "futex"}, + {__NR_futimesat, "futimesat"}, + {__NR_get_kernel_syms, "get_kernel_syms"}, + {__NR_get_mempolicy, "get_mempolicy"}, + {__NR_get_robust_list, "get_robust_list"}, + {__NR_get_thread_area, "get_thread_area"}, + {__NR_getcpu, "getcpu"}, + {__NR_getcwd, "getcwd"}, + {__NR_getdents, "getdents"}, + {__NR_getdents64, "getdents64"}, + {__NR_getegid, "getegid"}, + {__NR_geteuid, "geteuid"}, + {__NR_getgid, "getgid"}, + {__NR_getgroups, "getgroups"}, + {__NR_getitimer, "getitimer"}, + {__NR_getpeername, "getpeername"}, + {__NR_getpgid, "getpgid"}, + {__NR_getpgrp, "getpgrp"}, + {__NR_getpid, "getpid"}, + {__NR_getpmsg, "getpmsg"}, + {__NR_getppid, "getppid"}, + {__NR_getpriority, "getpriority"}, + {__NR_getrandom, "getrandom"}, + {__NR_getresgid, "getresgid"}, + {__NR_getresuid, "getresuid"}, + {__NR_getrlimit, "getrlimit"}, + {__NR_getrusage, "getrusage"}, + {__NR_getsid, "getsid"}, + {__NR_getsockname, "getsockname"}, + {__NR_getsockopt, "getsockopt"}, + {__NR_gettid, "gettid"}, + {__NR_gettimeofday, "gettimeofday"}, + {__NR_getuid, "getuid"}, + {__NR_getxattr, "getxattr"}, + {__NR_init_module, "init_module"}, + {__NR_inotify_add_watch, "inotify_add_watch"}, + {__NR_inotify_init, "inotify_init"}, + {__NR_inotify_init1, "inotify_init1"}, + {__NR_inotify_rm_watch, "inotify_rm_watch"}, + {__NR_io_cancel, "io_cancel"}, + {__NR_io_destroy, "io_destroy"}, + {__NR_io_getevents, "io_getevents"}, + {__NR_io_setup, "io_setup"}, + {__NR_io_submit, "io_submit"}, + {__NR_ioctl, "ioctl"}, + {__NR_ioperm, "ioperm"}, + {__NR_iopl, "iopl"}, + {__NR_ioprio_get, "ioprio_get"}, + {__NR_ioprio_set, "ioprio_set"}, + {__NR_kcmp, "kcmp"}, + {__NR_kexec_file_load, "kexec_file_load"}, + {__NR_kexec_load, "kexec_load"}, + {__NR_keyctl, "keyctl"}, + {__NR_kill, "kill"}, + {__NR_lchown, "lchown"}, + {__NR_lgetxattr, "lgetxattr"}, + {__NR_link, "link"}, + {__NR_linkat, "linkat"}, + {__NR_listen, "listen"}, + {__NR_listxattr, "listxattr"}, + {__NR_llistxattr, "llistxattr"}, + {__NR_lookup_dcookie, "lookup_dcookie"}, + {__NR_lremovexattr, "lremovexattr"}, + {__NR_lseek, "lseek"}, + {__NR_lsetxattr, "lsetxattr"}, + {__NR_lstat, "lstat"}, + {__NR_madvise, "madvise"}, + {__NR_mbind, "mbind"}, + {__NR_membarrier, "membarrier"}, + {__NR_memfd_create, "memfd_create"}, + {__NR_migrate_pages, "migrate_pages"}, + {__NR_mincore, "mincore"}, + {__NR_mkdir, "mkdir"}, + {__NR_mkdirat, "mkdirat"}, + {__NR_mknod, "mknod"}, + {__NR_mknodat, "mknodat"}, + {__NR_mlock, "mlock"}, + {__NR_mlock2, "mlock2"}, + {__NR_mlockall, "mlockall"}, + {__NR_mmap, "mmap"}, + {__NR_modify_ldt, "modify_ldt"}, + {__NR_mount, "mount"}, + {__NR_move_pages, "move_pages"}, + {__NR_mprotect, "mprotect"}, + {__NR_mq_getsetattr, "mq_getsetattr"}, + {__NR_mq_notify, "mq_notify"}, + {__NR_mq_open, "mq_open"}, + {__NR_mq_timedreceive, "mq_timedreceive"}, + {__NR_mq_timedsend, "mq_timedsend"}, + {__NR_mq_unlink, "mq_unlink"}, + {__NR_mremap, "mremap"}, + {__NR_msgctl, "msgctl"}, + {__NR_msgget, "msgget"}, + {__NR_msgrcv, "msgrcv"}, + {__NR_msgsnd, "msgsnd"}, + {__NR_msync, "msync"}, + {__NR_munlock, "munlock"}, + {__NR_munlockall, "munlockall"}, + {__NR_munmap, "munmap"}, + {__NR_name_to_handle_at, "name_to_handle_at"}, + {__NR_nanosleep, "nanosleep"}, + {__NR_newfstatat, "newfstatat"}, + {__NR_nfsservctl, "nfsservctl"}, + {__NR_open, "open"}, + {__NR_open_by_handle_at, "open_by_handle_at"}, + {__NR_openat, "openat"}, + {__NR_pause, "pause"}, + {__NR_perf_event_open, "perf_event_open"}, + {__NR_personality, "personality"}, + {__NR_pipe, "pipe"}, + {__NR_pipe2, "pipe2"}, + {__NR_pivot_root, "pivot_root"}, +#ifdef __NR_pkey_alloc + {__NR_pkey_alloc, "pkey_alloc"}, +#endif +#ifdef __NR_pkey_free + {__NR_pkey_free, "pkey_free"}, +#endif +#ifdef __NR_pkey_mprotect + {__NR_pkey_mprotect, "pkey_mprotect"}, +#endif + {__NR_poll, "poll"}, + {__NR_ppoll, "ppoll"}, + {__NR_prctl, "prctl"}, + {__NR_pread64, "pread64"}, + {__NR_preadv, "preadv"}, +#ifdef __NR_preadv2 + {__NR_preadv2, "preadv2"}, +#endif + {__NR_prlimit64, "prlimit64"}, + {__NR_process_vm_readv, "process_vm_readv"}, + {__NR_process_vm_writev, "process_vm_writev"}, + {__NR_pselect6, "pselect6"}, + {__NR_ptrace, "ptrace"}, + {__NR_putpmsg, "putpmsg"}, + {__NR_pwrite64, "pwrite64"}, + {__NR_pwritev, "pwritev"}, +#ifdef __NR_pwritev2 + {__NR_pwritev2, "pwritev2"}, +#endif + {__NR__sysctl, "_sysctl"}, + {__NR_query_module, "query_module"}, + {__NR_quotactl, "quotactl"}, + {__NR_read, "read"}, + {__NR_readahead, "readahead"}, + {__NR_readlink, "readlink"}, + {__NR_readlinkat, "readlinkat"}, + {__NR_readv, "readv"}, + {__NR_reboot, "reboot"}, + {__NR_recvfrom, "recvfrom"}, + {__NR_recvmmsg, "recvmmsg"}, + {__NR_recvmsg, "recvmsg"}, + {__NR_remap_file_pages, "remap_file_pages"}, + {__NR_removexattr, "removexattr"}, + {__NR_rename, "rename"}, + {__NR_renameat, "renameat"}, + {__NR_renameat2, "renameat2"}, + {__NR_request_key, "request_key"}, + {__NR_restart_syscall, "restart_syscall"}, + {__NR_rmdir, "rmdir"}, + {__NR_rt_sigaction, "rt_sigaction"}, + {__NR_rt_sigpending, "rt_sigpending"}, + {__NR_rt_sigprocmask, "rt_sigprocmask"}, + {__NR_rt_sigqueueinfo, "rt_sigqueueinfo"}, + {__NR_rt_sigreturn, "rt_sigreturn"}, + {__NR_rt_sigsuspend, "rt_sigsuspend"}, + {__NR_rt_sigtimedwait, "rt_sigtimedwait"}, + {__NR_rt_tgsigqueueinfo, "rt_tgsigqueueinfo"}, + {__NR_sched_get_priority_max, "sched_get_priority_max"}, + {__NR_sched_get_priority_min, "sched_get_priority_min"}, + {__NR_sched_getaffinity, "sched_getaffinity"}, + {__NR_sched_getattr, "sched_getattr"}, + {__NR_sched_getparam, "sched_getparam"}, + {__NR_sched_getscheduler, "sched_getscheduler"}, + {__NR_sched_rr_get_interval, "sched_rr_get_interval"}, + {__NR_sched_setaffinity, "sched_setaffinity"}, + {__NR_sched_setattr, "sched_setattr"}, + {__NR_sched_setparam, "sched_setparam"}, + {__NR_sched_setscheduler, "sched_setscheduler"}, + {__NR_sched_yield, "sched_yield"}, + {__NR_seccomp, "seccomp"}, + {__NR_security, "security"}, + {__NR_select, "select"}, + {__NR_semctl, "semctl"}, + {__NR_semget, "semget"}, + {__NR_semop, "semop"}, + {__NR_semtimedop, "semtimedop"}, + {__NR_sendfile, "sendfile"}, + {__NR_sendmmsg, "sendmmsg"}, + {__NR_sendmsg, "sendmsg"}, + {__NR_sendto, "sendto"}, + {__NR_set_mempolicy, "set_mempolicy"}, + {__NR_set_robust_list, "set_robust_list"}, + {__NR_set_thread_area, "set_thread_area"}, + {__NR_set_tid_address, "set_tid_address"}, + {__NR_setdomainname, "setdomainname"}, + {__NR_setfsgid, "setfsgid"}, + {__NR_setfsuid, "setfsuid"}, + {__NR_setgid, "setgid"}, + {__NR_setgroups, "setgroups"}, + {__NR_sethostname, "sethostname"}, + {__NR_setitimer, "setitimer"}, + {__NR_setns, "setns"}, + {__NR_setpgid, "setpgid"}, + {__NR_setpriority, "setpriority"}, + {__NR_setregid, "setregid"}, + {__NR_setresgid, "setresgid"}, + {__NR_setresuid, "setresuid"}, + {__NR_setreuid, "setreuid"}, + {__NR_setrlimit, "setrlimit"}, + {__NR_setsid, "setsid"}, + {__NR_setsockopt, "setsockopt"}, + {__NR_settimeofday, "settimeofday"}, + {__NR_setuid, "setuid"}, + {__NR_setxattr, "setxattr"}, + {__NR_shmat, "shmat"}, + {__NR_shmctl, "shmctl"}, + {__NR_shmdt, "shmdt"}, + {__NR_shmget, "shmget"}, + {__NR_shutdown, "shutdown"}, + {__NR_sigaltstack, "sigaltstack"}, + {__NR_signalfd, "signalfd"}, + {__NR_signalfd4, "signalfd4"}, + {__NR_socket, "socket"}, + {__NR_socketpair, "socketpair"}, + {__NR_splice, "splice"}, + {__NR_stat, "stat"}, + {__NR_statfs, "statfs"}, + {__NR_statx, "statx"}, + {__NR_swapoff, "swapoff"}, + {__NR_swapon, "swapon"}, + {__NR_symlink, "symlink"}, + {__NR_symlinkat, "symlinkat"}, + {__NR_sync, "sync"}, + {__NR_sync_file_range, "sync_file_range"}, + {__NR_syncfs, "syncfs"}, + {__NR_sysfs, "sysfs"}, + {__NR_sysinfo, "sysinfo"}, + {__NR_syslog, "syslog"}, + {__NR_tee, "tee"}, + {__NR_tgkill, "tgkill"}, + {__NR_time, "time"}, + {__NR_timer_create, "timer_create"}, + {__NR_timer_delete, "timer_delete"}, + {__NR_timer_getoverrun, "timer_getoverrun"}, + {__NR_timer_gettime, "timer_gettime"}, + {__NR_timer_settime, "timer_settime"}, + {__NR_timerfd_create, "timerfd_create"}, + {__NR_timerfd_gettime, "timerfd_gettime"}, + {__NR_timerfd_settime, "timerfd_settime"}, + {__NR_times, "times"}, + {__NR_tkill, "tkill"}, + {__NR_truncate, "truncate"}, + {__NR_tuxcall, "tuxcall"}, + {__NR_umask, "umask"}, + {__NR_umount2, "umount2"}, + {__NR_uname, "uname"}, + {__NR_unlink, "unlink"}, + {__NR_unlinkat, "unlinkat"}, + {__NR_unshare, "unshare"}, + {__NR_uselib, "uselib"}, + {__NR_userfaultfd, "userfaultfd"}, + {__NR_ustat, "ustat"}, + {__NR_utime, "utime"}, + {__NR_utimensat, "utimensat"}, + {__NR_utimes, "utimes"}, + {__NR_vfork, "vfork"}, + {__NR_vhangup, "vhangup"}, + {__NR_vmsplice, "vmsplice"}, + {__NR_vserver, "vserver"}, + {__NR_wait4, "wait4"}, + {__NR_waitid, "waitid"}, + {__NR_write, "write"}, + {__NR_writev, "writev"}, +}; + +std::string GetLinuxSyscallName(uint32_t syscall_number) +{ + const auto element = LINUX_SYSCALLS.find(syscall_number); + if (element != LINUX_SYSCALLS.end()) { + return element->second; + } + return "*unknown*"; +} + +// See Linux kernel developer Kees Cook's seccomp guide at <https://outflux.net/teach-seccomp/> for +// an accessible introduction to using seccomp. +// +// This function largely follows <https://outflux.net/teach-seccomp/step-3/syscall-reporter.c> and +// <https://outflux.net/teach-seccomp/step-3/seccomp-bpf.h>. +// +// Seccomp BPF resources: +// * Seccomp BPF documentation: <https://www.kernel.org/doc/html/latest/userspace-api/seccomp_filter.html> +// * seccomp(2) manual page: <https://www.kernel.org/doc/man-pages/online/pages/man2/seccomp.2.html> +// * Seccomp BPF demo code samples: <https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/samples/seccomp> +void SyscallSandboxDebugSignalHandler(int, siginfo_t* signal_info, void* void_signal_context) +{ + // The si_code field inside the siginfo_t argument that is passed to a SA_SIGINFO signal handler + // is a value indicating why the signal was sent. + // + // The following value can be placed in si_code for a SIGSYS signal: + // * SYS_SECCOMP (since Linux 3.5): Triggered by a seccomp(2) filter rule. + constexpr int32_t SYS_SECCOMP_SI_CODE{1}; + assert(signal_info->si_code == SYS_SECCOMP_SI_CODE); + + // The ucontext_t structure contains signal context information that was saved on the user-space + // stack by the kernel. + const ucontext_t* signal_context = static_cast<ucontext_t*>(void_signal_context); + assert(signal_context != nullptr); + + std::set_new_handler(std::terminate); + // Portability note: REG_RAX is Linux x86_64 specific. + const uint32_t syscall_number = static_cast<uint32_t>(signal_context->uc_mcontext.gregs[REG_RAX]); + const std::string syscall_name = GetLinuxSyscallName(syscall_number); + const std::string thread_name = !util::ThreadGetInternalName().empty() ? util::ThreadGetInternalName() : "*unnamed*"; + const std::string error_message = strprintf("ERROR: The syscall \"%s\" (syscall number %d) is not allowed by the syscall sandbox in thread \"%s\". Please report.", syscall_name, syscall_number, thread_name); + tfm::format(std::cerr, "%s\n", error_message); + LogPrintf("%s\n", error_message); + std::terminate(); +} + +// This function largely follows install_syscall_reporter from Kees Cook's seccomp guide: +// <https://outflux.net/teach-seccomp/step-3/syscall-reporter.c> +bool SetupSyscallSandboxDebugHandler() +{ + struct sigaction action = {}; + sigset_t mask; + sigemptyset(&mask); + sigaddset(&mask, SIGSYS); + action.sa_sigaction = &SyscallSandboxDebugSignalHandler; + action.sa_flags = SA_SIGINFO; + if (sigaction(SIGSYS, &action, nullptr) < 0) { + return false; + } + if (sigprocmask(SIG_UNBLOCK, &mask, nullptr)) { + return false; + } + return true; +} + +enum class SyscallSandboxAction { + KILL_PROCESS, + INVOKE_SIGNAL_HANDLER, +}; + +class SeccompPolicyBuilder +{ + std::set<uint32_t> allowed_syscalls; + +public: + SeccompPolicyBuilder() + { + // Allowed by default. + AllowAddressSpaceAccess(); + AllowEpoll(); + AllowEventFd(); + AllowFutex(); + AllowGeneralIo(); + AllowGetRandom(); + AllowGetSimpleId(); + AllowGetTime(); + AllowGlobalProcessEnvironment(); + AllowGlobalSystemStatus(); + AllowKernelInternalApi(); + AllowNetworkSocketInformation(); + AllowOperationOnExistingFileDescriptor(); + AllowPipe(); + AllowPrctl(); + AllowProcessStartOrDeath(); + AllowScheduling(); + AllowSignalHandling(); + AllowSleep(); + AllowUmask(); + } + + void AllowAddressSpaceAccess() + { + allowed_syscalls.insert(__NR_brk); // change data segment size + allowed_syscalls.insert(__NR_madvise); // give advice about use of memory + allowed_syscalls.insert(__NR_membarrier); // issue memory barriers on a set of threads + allowed_syscalls.insert(__NR_mincore); // check if virtual memory is in RAM + allowed_syscalls.insert(__NR_mlock); // lock memory + allowed_syscalls.insert(__NR_mmap); // map files or devices into memory + allowed_syscalls.insert(__NR_mprotect); // set protection on a region of memory + allowed_syscalls.insert(__NR_mremap); // remap a file in memory + allowed_syscalls.insert(__NR_munlock); // unlock memory + allowed_syscalls.insert(__NR_munmap); // unmap files or devices into memory + } + + void AllowEpoll() + { + allowed_syscalls.insert(__NR_epoll_create1); // open an epoll file descriptor + allowed_syscalls.insert(__NR_epoll_ctl); // control interface for an epoll file descriptor + allowed_syscalls.insert(__NR_epoll_pwait); // wait for an I/O event on an epoll file descriptor + allowed_syscalls.insert(__NR_epoll_wait); // wait for an I/O event on an epoll file descriptor + } + + void AllowEventFd() + { + allowed_syscalls.insert(__NR_eventfd2); // create a file descriptor for event notification + } + + void AllowFileSystem() + { + allowed_syscalls.insert(__NR_access); // check user's permissions for a file + allowed_syscalls.insert(__NR_chdir); // change working directory + allowed_syscalls.insert(__NR_chmod); // change permissions of a file + allowed_syscalls.insert(__NR_copy_file_range); // copy a range of data from one file to another + allowed_syscalls.insert(__NR_fallocate); // manipulate file space + allowed_syscalls.insert(__NR_fchmod); // change permissions of a file + allowed_syscalls.insert(__NR_fchown); // change ownership of a file + allowed_syscalls.insert(__NR_fdatasync); // synchronize a file's in-core state with storage device + allowed_syscalls.insert(__NR_flock); // apply or remove an advisory lock on an open file + allowed_syscalls.insert(__NR_fstat); // get file status + allowed_syscalls.insert(__NR_newfstatat); // get file status + allowed_syscalls.insert(__NR_fsync); // synchronize a file's in-core state with storage device + allowed_syscalls.insert(__NR_ftruncate); // truncate a file to a specified length + allowed_syscalls.insert(__NR_getcwd); // get current working directory + allowed_syscalls.insert(__NR_getdents); // get directory entries + allowed_syscalls.insert(__NR_getdents64); // get directory entries + allowed_syscalls.insert(__NR_lstat); // get file status + allowed_syscalls.insert(__NR_mkdir); // create a directory + allowed_syscalls.insert(__NR_open); // open and possibly create a file + allowed_syscalls.insert(__NR_openat); // open and possibly create a file + allowed_syscalls.insert(__NR_readlink); // read value of a symbolic link + allowed_syscalls.insert(__NR_rename); // change the name or location of a file + allowed_syscalls.insert(__NR_rmdir); // delete a directory + allowed_syscalls.insert(__NR_stat); // get file status + allowed_syscalls.insert(__NR_statfs); // get filesystem statistics + allowed_syscalls.insert(__NR_statx); // get file status (extended) + allowed_syscalls.insert(__NR_unlink); // delete a name and possibly the file it refers to + } + + void AllowFutex() + { + allowed_syscalls.insert(__NR_futex); // fast user-space locking + allowed_syscalls.insert(__NR_set_robust_list); // set list of robust futexes + } + + void AllowGeneralIo() + { + allowed_syscalls.insert(__NR_ioctl); // control device + allowed_syscalls.insert(__NR_lseek); // reposition read/write file offset + allowed_syscalls.insert(__NR_poll); // wait for some event on a file descriptor + allowed_syscalls.insert(__NR_ppoll); // wait for some event on a file descriptor + allowed_syscalls.insert(__NR_pread64); // read from a file descriptor at a given offset + allowed_syscalls.insert(__NR_pwrite64); // write to a file descriptor at a given offset + allowed_syscalls.insert(__NR_read); // read from a file descriptor + allowed_syscalls.insert(__NR_readv); // read data into multiple buffers + allowed_syscalls.insert(__NR_recvfrom); // receive a message from a socket + allowed_syscalls.insert(__NR_recvmsg); // receive a message from a socket + allowed_syscalls.insert(__NR_select); // synchronous I/O multiplexing + allowed_syscalls.insert(__NR_sendmmsg); // send multiple messages on a socket + allowed_syscalls.insert(__NR_sendmsg); // send a message on a socket + allowed_syscalls.insert(__NR_sendto); // send a message on a socket + allowed_syscalls.insert(__NR_write); // write to a file descriptor + allowed_syscalls.insert(__NR_writev); // write data into multiple buffers + } + + void AllowGetRandom() + { + allowed_syscalls.insert(__NR_getrandom); // obtain a series of random bytes + } + + void AllowGetSimpleId() + { + allowed_syscalls.insert(__NR_getegid); // get group identity + allowed_syscalls.insert(__NR_geteuid); // get user identity + allowed_syscalls.insert(__NR_getgid); // get group identity + allowed_syscalls.insert(__NR_getpgid); // get process group + allowed_syscalls.insert(__NR_getpid); // get process identification + allowed_syscalls.insert(__NR_getppid); // get process identification + allowed_syscalls.insert(__NR_getresgid); // get real, effective and saved group IDs + allowed_syscalls.insert(__NR_getresuid); // get real, effective and saved user IDs + allowed_syscalls.insert(__NR_getsid); // get session ID + allowed_syscalls.insert(__NR_gettid); // get thread identification + allowed_syscalls.insert(__NR_getuid); // get user identity + } + + void AllowGetTime() + { + allowed_syscalls.insert(__NR_clock_getres); // find the resolution (precision) of the specified clock + allowed_syscalls.insert(__NR_clock_gettime); // retrieve the time of the specified clock + allowed_syscalls.insert(__NR_gettimeofday); // get timeval + } + + void AllowGlobalProcessEnvironment() + { + allowed_syscalls.insert(__NR_getrlimit); // get resource limits + allowed_syscalls.insert(__NR_getrusage); // get resource usage + allowed_syscalls.insert(__NR_prlimit64); // get/set resource limits + } + + void AllowGlobalSystemStatus() + { + allowed_syscalls.insert(__NR_sysinfo); // return system information + allowed_syscalls.insert(__NR_uname); // get name and information about current kernel + } + + void AllowKernelInternalApi() + { + allowed_syscalls.insert(__NR_restart_syscall); // restart a system call after interruption by a stop signal + } + + void AllowNetwork() + { + allowed_syscalls.insert(__NR_accept); // accept a connection on a socket + allowed_syscalls.insert(__NR_accept4); // accept a connection on a socket + allowed_syscalls.insert(__NR_bind); // bind a name to a socket + allowed_syscalls.insert(__NR_connect); // initiate a connection on a socket + allowed_syscalls.insert(__NR_listen); // listen for connections on a socket + allowed_syscalls.insert(__NR_setsockopt); // set options on sockets + allowed_syscalls.insert(__NR_socket); // create an endpoint for communication + allowed_syscalls.insert(__NR_socketpair); // create a pair of connected sockets + } + + void AllowNetworkSocketInformation() + { + allowed_syscalls.insert(__NR_getpeername); // get name of connected peer socket + allowed_syscalls.insert(__NR_getsockname); // get socket name + allowed_syscalls.insert(__NR_getsockopt); // get options on sockets + } + + void AllowOperationOnExistingFileDescriptor() + { + allowed_syscalls.insert(__NR_close); // close a file descriptor + allowed_syscalls.insert(__NR_dup); // duplicate a file descriptor + allowed_syscalls.insert(__NR_dup2); // duplicate a file descriptor + allowed_syscalls.insert(__NR_fcntl); // manipulate file descriptor + allowed_syscalls.insert(__NR_shutdown); // shut down part of a full-duplex connection + } + + void AllowPipe() + { + allowed_syscalls.insert(__NR_pipe); // create pipe + allowed_syscalls.insert(__NR_pipe2); // create pipe + } + + void AllowPrctl() + { + allowed_syscalls.insert(__NR_arch_prctl); // set architecture-specific thread state + allowed_syscalls.insert(__NR_prctl); // operations on a process + } + + void AllowProcessStartOrDeath() + { + allowed_syscalls.insert(__NR_clone); // create a child process + allowed_syscalls.insert(__NR_clone3); // create a child process + allowed_syscalls.insert(__NR_exit); // terminate the calling process + allowed_syscalls.insert(__NR_exit_group); // exit all threads in a process + allowed_syscalls.insert(__NR_fork); // create a child process + allowed_syscalls.insert(__NR_tgkill); // send a signal to a thread + allowed_syscalls.insert(__NR_wait4); // wait for process to change state, BSD style + } + + void AllowScheduling() + { + allowed_syscalls.insert(__NR_sched_getaffinity); // set a thread's CPU affinity mask + allowed_syscalls.insert(__NR_sched_getparam); // get scheduling parameters + allowed_syscalls.insert(__NR_sched_getscheduler); // get scheduling policy/parameters + allowed_syscalls.insert(__NR_sched_setscheduler); // set scheduling policy/parameters + allowed_syscalls.insert(__NR_sched_yield); // yield the processor + } + + void AllowSignalHandling() + { + allowed_syscalls.insert(__NR_rt_sigaction); // examine and change a signal action + allowed_syscalls.insert(__NR_rt_sigprocmask); // examine and change blocked signals + allowed_syscalls.insert(__NR_rt_sigreturn); // return from signal handler and cleanup stack frame + allowed_syscalls.insert(__NR_sigaltstack); // set and/or get signal stack context + } + + void AllowSleep() + { + allowed_syscalls.insert(__NR_clock_nanosleep); // high-resolution sleep with specifiable clock + allowed_syscalls.insert(__NR_nanosleep); // high-resolution sleep + } + + void AllowUmask() + { + allowed_syscalls.insert(__NR_umask); // set file mode creation mask + } + + // See Linux kernel developer Kees Cook's seccomp guide at <https://outflux.net/teach-seccomp/> + // for an accessible introduction to using seccomp. + // + // This function largely follows <https://outflux.net/teach-seccomp/step-3/seccomp-bpf.h>. + std::vector<sock_filter> BuildFilter(SyscallSandboxAction default_action) + { + std::vector<sock_filter> bpf_policy; + // See VALIDATE_ARCHITECTURE in seccomp-bpf.h referenced above. + bpf_policy.push_back(BPF_STMT(BPF_LD + BPF_W + BPF_ABS, offsetof(struct seccomp_data, arch))); + // Portability note: AUDIT_ARCH_X86_64 is Linux x86_64 specific. + bpf_policy.push_back(BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, AUDIT_ARCH_X86_64, 1, 0)); + bpf_policy.push_back(BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_KILL_PROCESS)); + // See EXAMINE_SYSCALL in seccomp-bpf.h referenced above. + bpf_policy.push_back(BPF_STMT(BPF_LD + BPF_W + BPF_ABS, offsetof(struct seccomp_data, nr))); + for (const uint32_t allowed_syscall : allowed_syscalls) { + // See ALLOW_SYSCALL in seccomp-bpf.h referenced above. + bpf_policy.push_back(BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, allowed_syscall, 0, 1)); + bpf_policy.push_back(BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_ALLOW)); + } + switch (default_action) { + case SyscallSandboxAction::KILL_PROCESS: + // Disallow syscall and kill the process. + // + // See KILL_PROCESS in seccomp-bpf.h referenced above. + // + // Note that we're using SECCOMP_RET_KILL_PROCESS (kill the process) instead + // of SECCOMP_RET_KILL_THREAD (kill the thread). The SECCOMP_RET_KILL_PROCESS + // action was introduced in Linux 4.14. + // + // SECCOMP_RET_KILL_PROCESS: Results in the entire process exiting immediately without + // executing the system call. + // + // SECCOMP_RET_KILL_PROCESS documentation: + // <https://www.kernel.org/doc/html/latest/userspace-api/seccomp_filter.html> + bpf_policy.push_back(BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_KILL_PROCESS)); + break; + case SyscallSandboxAction::INVOKE_SIGNAL_HANDLER: + // Disallow syscall and force a SIGSYS to trigger syscall debug reporter. + // + // SECCOMP_RET_TRAP: Results in the kernel sending a SIGSYS signal to the triggering + // task without executing the system call. + // + // SECCOMP_RET_TRAP documentation: + // <https://www.kernel.org/doc/html/latest/userspace-api/seccomp_filter.html> + bpf_policy.push_back(BPF_STMT(BPF_RET + BPF_K, SECCOMP_RET_TRAP)); + break; + } + return bpf_policy; + } +}; +} // namespace + +bool SetupSyscallSandbox(bool log_syscall_violation_before_terminating) +{ + assert(!g_syscall_sandbox_enabled && "SetupSyscallSandbox(...) should only be called once."); + g_syscall_sandbox_enabled = true; + g_syscall_sandbox_log_violation_before_terminating = log_syscall_violation_before_terminating; + if (log_syscall_violation_before_terminating) { + if (!SetupSyscallSandboxDebugHandler()) { + return false; + } + } + SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION); + return true; +} + +void TestDisallowedSandboxCall() +{ + // The getgroups syscall is assumed NOT to be allowed by the syscall sandbox policy. + std::array<gid_t, 1> groups; + [[maybe_unused]] int32_t ignored = getgroups(groups.size(), groups.data()); +} +#endif // defined(USE_SYSCALL_SANDBOX) + +void SetSyscallSandboxPolicy(SyscallSandboxPolicy syscall_policy) +{ +#if defined(USE_SYSCALL_SANDBOX) + if (!g_syscall_sandbox_enabled) { + return; + } + SeccompPolicyBuilder seccomp_policy_builder; + switch (syscall_policy) { + case SyscallSandboxPolicy::INITIALIZATION: // Thread: main thread (state: init) + // SyscallSandboxPolicy::INITIALIZATION is the first policy loaded. + // + // Subsequently loaded policies can reduce the abilities further, but + // abilities can never be regained. + // + // SyscallSandboxPolicy::INITIALIZATION must thus be a superset of all + // other policies. + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::INITIALIZATION_DNS_SEED: // Thread: dnsseed + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::INITIALIZATION_LOAD_BLOCKS: // Thread: loadblk + seccomp_policy_builder.AllowFileSystem(); + break; + case SyscallSandboxPolicy::INITIALIZATION_MAP_PORT: // Thread: mapport + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::MESSAGE_HANDLER: // Thread: msghand + seccomp_policy_builder.AllowFileSystem(); + break; + case SyscallSandboxPolicy::NET: // Thread: net + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::NET_ADD_CONNECTION: // Thread: addcon + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::NET_HTTP_SERVER: // Thread: http + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::NET_HTTP_SERVER_WORKER: // Thread: httpworker.<N> + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::NET_OPEN_CONNECTION: // Thread: opencon + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::SCHEDULER: // Thread: scheduler + seccomp_policy_builder.AllowFileSystem(); + break; + case SyscallSandboxPolicy::TOR_CONTROL: // Thread: torcontrol + seccomp_policy_builder.AllowFileSystem(); + seccomp_policy_builder.AllowNetwork(); + break; + case SyscallSandboxPolicy::TX_INDEX: // Thread: txindex + seccomp_policy_builder.AllowFileSystem(); + break; + case SyscallSandboxPolicy::VALIDATION_SCRIPT_CHECK: // Thread: scriptch.<N> + break; + case SyscallSandboxPolicy::SHUTOFF: // Thread: main thread (state: shutoff) + seccomp_policy_builder.AllowFileSystem(); + break; + } + + const SyscallSandboxAction default_action = g_syscall_sandbox_log_violation_before_terminating ? SyscallSandboxAction::INVOKE_SIGNAL_HANDLER : SyscallSandboxAction::KILL_PROCESS; + std::vector<sock_filter> filter = seccomp_policy_builder.BuildFilter(default_action); + const sock_fprog prog = { + .len = static_cast<uint16_t>(filter.size()), + .filter = filter.data(), + }; + // Do not allow abilities to be regained after being dropped. + // + // PR_SET_NO_NEW_PRIVS documentation: <https://www.kernel.org/doc/html/latest/userspace-api/no_new_privs.html> + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) { + throw std::runtime_error("Syscall sandbox enforcement failed: prctl(PR_SET_NO_NEW_PRIVS)"); + } + // Install seccomp-bpf syscall filter. + // + // PR_SET_SECCOMP documentation: <https://www.kernel.org/doc/html/latest/userspace-api/seccomp_filter.html> + if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &prog) != 0) { + throw std::runtime_error("Syscall sandbox enforcement failed: prctl(PR_SET_SECCOMP)"); + } + + const std::string thread_name = !util::ThreadGetInternalName().empty() ? util::ThreadGetInternalName() : "*unnamed*"; + LogPrint(BCLog::UTIL, "Syscall filter installed for thread \"%s\"\n", thread_name); +#endif // defined(USE_SYSCALL_SANDBOX) +} diff --git a/src/util/syscall_sandbox.h b/src/util/syscall_sandbox.h new file mode 100644 index 0000000000..0a0c964f94 --- /dev/null +++ b/src/util/syscall_sandbox.h @@ -0,0 +1,57 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_SYSCALL_SANDBOX_H +#define BITCOIN_UTIL_SYSCALL_SANDBOX_H + +enum class SyscallSandboxPolicy { + // 1. Initialization + INITIALIZATION, + INITIALIZATION_DNS_SEED, + INITIALIZATION_LOAD_BLOCKS, + INITIALIZATION_MAP_PORT, + + // 2. Steady state (non-initialization, non-shutdown) + MESSAGE_HANDLER, + NET, + NET_ADD_CONNECTION, + NET_HTTP_SERVER, + NET_HTTP_SERVER_WORKER, + NET_OPEN_CONNECTION, + SCHEDULER, + TOR_CONTROL, + TX_INDEX, + VALIDATION_SCRIPT_CHECK, + + // 3. Shutdown + SHUTOFF, +}; + +//! Force the current thread (and threads created from the current thread) into a restricted-service +//! operating mode where only a subset of all syscalls are available. +//! +//! Subsequent calls to this function can reduce the abilities further, but abilities can never be +//! regained. +//! +//! This function is a no-op unless SetupSyscallSandbox(...) has been called. +//! +//! SetupSyscallSandbox(...) is called during bitcoind initialization if Bitcoin Core was compiled +//! with seccomp-bpf support (--with-seccomp) *and* the parameter -sandbox=<mode> was passed to +//! bitcoind. +//! +//! This experimental feature is available under Linux x86_64 only. +void SetSyscallSandboxPolicy(SyscallSandboxPolicy syscall_policy); + +#if defined(USE_SYSCALL_SANDBOX) +//! Setup and enable the experimental syscall sandbox for the running process. +//! +//! SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION) is called as part of +//! SetupSyscallSandbox(...). +[[nodiscard]] bool SetupSyscallSandbox(bool log_syscall_violation_before_terminating); + +//! Invoke a disallowed syscall. Use for testing purposes. +void TestDisallowedSandboxCall(); +#endif // defined(USE_SYSCALL_SANDBOX) + +#endif // BITCOIN_UTIL_SYSCALL_SANDBOX_H diff --git a/src/util/system.cpp b/src/util/system.cpp index 4defeed4ce..12d7dc49b2 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -98,7 +98,7 @@ bool LockDirectory(const fs::path& directory, const std::string lockfile_name, b fs::path pathLockFile = directory / lockfile_name; // If a lock for this directory already exists in the map, don't try to re-lock it - if (dir_locks.count(pathLockFile.string())) { + if (dir_locks.count(fs::PathToString(pathLockFile))) { return true; } @@ -107,11 +107,11 @@ bool LockDirectory(const fs::path& directory, const std::string lockfile_name, b if (file) fclose(file); auto lock = std::make_unique<fsbridge::FileLock>(pathLockFile); if (!lock->TryLock()) { - return error("Error while attempting to lock directory %s: %s", directory.string(), lock->GetReason()); + return error("Error while attempting to lock directory %s: %s", fs::PathToString(directory), lock->GetReason()); } if (!probe_only) { // Lock successful and we're not just probing, put it into the map - dir_locks.emplace(pathLockFile.string(), std::move(lock)); + dir_locks.emplace(fs::PathToString(pathLockFile), std::move(lock)); } return true; } @@ -119,7 +119,7 @@ bool LockDirectory(const fs::path& directory, const std::string lockfile_name, b void UnlockDirectory(const fs::path& directory, const std::string& lockfile_name) { LOCK(cs_dir_locks); - dir_locks.erase((directory / lockfile_name).string()); + dir_locks.erase(fs::PathToString(directory / lockfile_name)); } void ReleaseDirectoryLocks() @@ -158,16 +158,14 @@ std::streampos GetFileSize(const char* path, std::streamsize max) { /** * Interpret a string argument as a boolean. * - * The definition of atoi() requires that non-numeric string values like "foo", - * return 0. This means that if a user unintentionally supplies a non-integer - * argument here, the return value is always false. This means that -foo=false - * does what the user probably expects, but -foo=true is well defined but does - * not do what they probably expected. + * The definition of LocaleIndependentAtoi<int>() requires that non-numeric string values + * like "foo", return 0. This means that if a user unintentionally supplies a + * non-integer argument here, the return value is always false. This means that + * -foo=false does what the user probably expects, but -foo=true is well defined + * but does not do what they probably expected. * - * The return value of atoi() is undefined when given input not representable as - * an int. On most systems this means string value between "-2147483648" and - * "2147483647" are well defined (this method will return true). Setting - * -txindex=2147483648 on most systems, however, is probably undefined. + * The return value of LocaleIndependentAtoi<int>(...) is zero when given input not + * representable as an int. * * For a more extensive discussion of this topic (and a wide range of opinions * on the Right Way to change this code), see PR12713. @@ -176,7 +174,7 @@ static bool InterpretBool(const std::string& strValue) { if (strValue.empty()) return true; - return (atoi(strValue) != 0); + return (LocaleIndependentAtoi<int>(strValue) != 0); } static std::string SettingName(const std::string& arg) @@ -244,7 +242,7 @@ namespace { fs::path StripRedundantLastElementsOfPath(const fs::path& path) { auto result = path; - while (result.filename().string() == ".") { + while (fs::PathToString(result.filename()) == ".") { result = result.parent_path(); } @@ -404,7 +402,7 @@ const fs::path& ArgsManager::GetBlocksDirPath() const if (!path.empty()) return path; if (IsArgSet("-blocksdir")) { - path = fs::system_complete(GetArg("-blocksdir", "")); + path = fs::system_complete(fs::PathFromString(GetArg("-blocksdir", ""))); if (!fs::is_directory(path)) { path = ""; return path; @@ -413,7 +411,7 @@ const fs::path& ArgsManager::GetBlocksDirPath() const path = GetDataDirBase(); } - path /= BaseParams().DataDir(); + path /= fs::PathFromString(BaseParams().DataDir()); path /= "blocks"; fs::create_directories(path); path = StripRedundantLastElementsOfPath(path); @@ -431,7 +429,7 @@ const fs::path& ArgsManager::GetDataDir(bool net_specific) const std::string datadir = GetArg("-datadir", ""); if (!datadir.empty()) { - path = fs::system_complete(datadir); + path = fs::system_complete(fs::PathFromString(datadir)); if (!fs::is_directory(path)) { path = ""; return path; @@ -440,7 +438,7 @@ const fs::path& ArgsManager::GetDataDir(bool net_specific) const path = GetDefaultDataDir(); } if (net_specific) - path /= BaseParams().DataDir(); + path /= fs::PathFromString(BaseParams().DataDir()); if (fs::create_directories(path)) { // This is the first run, create wallets subdirectory too @@ -519,7 +517,7 @@ bool ArgsManager::GetSettingsPath(fs::path* filepath, bool temp) const } if (filepath) { std::string settings = GetArg("-settings", BITCOIN_SETTINGS_FILENAME); - *filepath = fsbridge::AbsPathJoin(GetDataDirNet(), temp ? settings + ".tmp" : settings); + *filepath = fsbridge::AbsPathJoin(GetDataDirNet(), fs::PathFromString(temp ? settings + ".tmp" : settings)); } return true; } @@ -574,7 +572,7 @@ bool ArgsManager::WriteSettingsFile(std::vector<std::string>* errors) const return false; } if (!RenameOver(path_tmp, path)) { - SaveErrors({strprintf("Failed renaming settings file %s to %s\n", path_tmp.string(), path.string())}, errors); + SaveErrors({strprintf("Failed renaming settings file %s to %s\n", fs::PathToString(path_tmp), fs::PathToString(path))}, errors); return false; } return true; @@ -594,7 +592,7 @@ std::string ArgsManager::GetArg(const std::string& strArg, const std::string& st int64_t ArgsManager::GetIntArg(const std::string& strArg, int64_t nDefault) const { const util::SettingsValue value = GetSetting(strArg); - return value.isNull() ? nDefault : value.isFalse() ? 0 : value.isTrue() ? 1 : value.isNum() ? value.get_int64() : atoi64(value.get_str()); + return value.isNull() ? nDefault : value.isFalse() ? 0 : value.isTrue() ? 1 : value.isNum() ? value.get_int64() : LocaleIndependentAtoi<int64_t>(value.get_str()); } bool ArgsManager::GetBoolArg(const std::string& strArg, bool fDefault) const @@ -811,12 +809,12 @@ fs::path GetDefaultDataDir() bool CheckDataDirOption() { std::string datadir = gArgs.GetArg("-datadir", ""); - return datadir.empty() || fs::is_directory(fs::system_complete(datadir)); + return datadir.empty() || fs::is_directory(fs::system_complete(fs::PathFromString(datadir))); } fs::path GetConfigFile(const std::string& confPath) { - return AbsPathForConfigVal(fs::path(confPath), false); + return AbsPathForConfigVal(fs::PathFromString(confPath), false); } static bool GetConfigOptions(std::istream& stream, const std::string& filepath, std::string& error, std::vector<std::pair<std::string, std::string>>& options, std::list<SectionInfo>& sections) @@ -1067,7 +1065,7 @@ bool RenameOver(fs::path src, fs::path dest) return MoveFileExW(src.wstring().c_str(), dest.wstring().c_str(), MOVEFILE_REPLACE_EXISTING) != 0; #else - int rc = std::rename(src.string().c_str(), dest.string().c_str()); + int rc = std::rename(src.c_str(), dest.c_str()); return (rc == 0); #endif /* WIN32 */ } diff --git a/src/validation.cpp b/src/validation.cpp index 8f0ddd9064..207cdc8233 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -9,6 +9,7 @@ #include <chain.h> #include <chainparams.h> #include <checkqueue.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/merkle.h> #include <consensus/tx_check.h> @@ -218,7 +219,7 @@ bool TestLockPointValidity(CChain& active_chain, const LockPoints* lp) // If there are relative lock times then the maxInputBlock will be set // If there are no relative lock times, the LockPoints don't depend on the chain if (lp->maxInputBlock) { - // Check whether ::ChainActive() is an extension of the block at which the LockPoints + // Check whether active_chain is an extension of the block at which the LockPoints // calculation was valid. If not LockPoints are no longer valid if (!active_chain.Contains(lp->maxInputBlock)) { return false; @@ -354,7 +355,7 @@ void CChainState::MaybeUpdateMempoolForReorg( // If the transaction doesn't make it in to the mempool, remove any // transactions that depend on it (which would now be orphans). m_mempool->removeRecursive(**it, MemPoolRemovalReason::REORG); - } else if (m_mempool->exists((*it)->GetHash())) { + } else if (m_mempool->exists(GenTxid::Txid((*it)->GetHash()))) { vHashUpdate.push_back((*it)->GetHash()); } ++it; @@ -473,7 +474,6 @@ private: std::unique_ptr<CTxMemPoolEntry> m_entry; std::list<CTransactionRef> m_replaced_transactions; - bool m_replacement_transaction; CAmount m_base_fees; CAmount m_modified_fees; /** Total modified fees of all transactions being replaced. */ @@ -555,7 +555,6 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) CTxMemPool::setEntries& allConflicting = ws.m_all_conflicting; CTxMemPool::setEntries& setAncestors = ws.m_ancestors; std::unique_ptr<CTxMemPoolEntry>& entry = ws.m_entry; - bool& fReplacementTransaction = ws.m_replacement_transaction; CAmount& nModifiedFees = ws.m_modified_fees; CAmount& nConflictingFees = ws.m_conflicting_fees; size_t& nConflictingSize = ws.m_conflicting_size; @@ -586,10 +585,10 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) if (!CheckFinalTx(m_active_chainstate.m_chain.Tip(), tx, STANDARD_LOCKTIME_VERIFY_FLAGS)) return state.Invalid(TxValidationResult::TX_PREMATURE_SPEND, "non-final"); - if (m_pool.exists(GenTxid(true, tx.GetWitnessHash()))) { + if (m_pool.exists(GenTxid::Wtxid(tx.GetWitnessHash()))) { // Exact transaction already exists in the mempool. return state.Invalid(TxValidationResult::TX_CONFLICT, "txn-already-in-mempool"); - } else if (m_pool.exists(GenTxid(false, tx.GetHash()))) { + } else if (m_pool.exists(GenTxid::Txid(tx.GetHash()))) { // Transaction with the same non-witness data but different witness (same txid, different // wtxid) already exists in the mempool. return state.Invalid(TxValidationResult::TX_CONFLICT, "txn-same-nonwitness-data-in-mempool"); @@ -778,8 +777,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) } - fReplacementTransaction = setConflicts.size(); - if (fReplacementTransaction) { + if (!setConflicts.empty()) { CFeeRate newFeeRate(nModifiedFees, nSize); // It's possible that the replacement pays more fees than its direct conflicts but not more // than all conflicts (i.e. the direct conflicts have high-fee descendants). However, if the @@ -884,7 +882,6 @@ bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) const CAmount& nModifiedFees = ws.m_modified_fees; const CAmount& nConflictingFees = ws.m_conflicting_fees; const size_t& nConflictingSize = ws.m_conflicting_size; - const bool fReplacementTransaction = ws.m_replacement_transaction; std::unique_ptr<CTxMemPoolEntry>& entry = ws.m_entry; // Remove conflicting transactions from the mempool @@ -900,11 +897,10 @@ bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) m_pool.RemoveStaged(allConflicting, false, MemPoolRemovalReason::REPLACED); // This transaction should only count for fee estimation if: - // - it isn't a BIP 125 replacement transaction (may not be widely supported) // - it's not being re-added during a reorg which bypasses typical mempool fee limits // - the node is not behind // - the transaction is not dependent on any other transactions in the mempool - bool validForFeeEstimation = !fReplacementTransaction && !bypass_limits && IsCurrentForFeeEstimation(m_active_chainstate) && m_pool.HasNoInputsOf(tx); + bool validForFeeEstimation = !bypass_limits && IsCurrentForFeeEstimation(m_active_chainstate) && m_pool.HasNoInputsOf(tx); // Store transaction in memory m_pool.addUnchecked(*entry, setAncestors, validForFeeEstimation); @@ -912,7 +908,7 @@ bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) // trim mempool and check if tx was trimmed if (!bypass_limits) { LimitMempoolSize(m_pool, m_active_chainstate.CoinsTip(), gArgs.GetIntArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, std::chrono::hours{gArgs.GetIntArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY)}); - if (!m_pool.exists(hash)) + if (!m_pool.exists(GenTxid::Txid(hash))) return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool full"); } return true; @@ -1875,14 +1871,13 @@ bool CChainState::ConnectBlock(const CBlock& block, BlockValidationState& state, int64_t nTime5 = GetTimeMicros(); nTimeIndex += nTime5 - nTime4; LogPrint(BCLog::BENCH, " - Index writing: %.2fms [%.2fs (%.2fms/blk)]\n", MILLI * (nTime5 - nTime4), nTimeIndex * MICRO, nTimeIndex * MILLI / nBlocksTotal); - TRACE7(validation, block_connected, - block.GetHash().ToString().c_str(), + TRACE6(validation, block_connected, + block.GetHash().data(), pindex->nHeight, block.vtx.size(), nInputs, nSigOpsCost, - GetTimeMicros() - nTimeStart, // in microseconds (µs) - block.GetHash().data() + GetTimeMicros() - nTimeStart // in microseconds (µs) ); return true; @@ -2027,8 +2022,8 @@ bool CChainState::FlushStateToDisk( } // Flush best chain related state. This can only be done if the blocks / block index write was also done. if (fDoFullFlush && !CoinsTip().GetBestBlock().IsNull()) { - LOG_TIME_SECONDS(strprintf("write coins cache to disk (%d coins, %.2fkB)", - coins_count, coins_mem_usage / 1000)); + LOG_TIME_MILLIS_WITH_CATEGORY(strprintf("write coins cache to disk (%d coins, %.2fkB)", + coins_count, coins_mem_usage / 1000), BCLog::BENCH); // Typical Coin structures on disk are around 48 bytes in size. // Pushing a new one to the database can cause it to be written @@ -3203,7 +3198,7 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS if (ppindex) *ppindex = pindex; if (pindex->nStatus & BLOCK_FAILED_MASK) { - LogPrintf("ERROR: %s: block %s is marked invalid\n", __func__, hash.ToString()); + LogPrint(BCLog::VALIDATION, "%s: block %s is marked invalid\n", __func__, hash.ToString()); return state.Invalid(BlockValidationResult::BLOCK_CACHED_INVALID, "duplicate"); } return true; @@ -3218,16 +3213,18 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS CBlockIndex* pindexPrev = nullptr; BlockMap::iterator mi = m_block_index.find(block.hashPrevBlock); if (mi == m_block_index.end()) { - LogPrintf("ERROR: %s: prev block not found\n", __func__); + LogPrint(BCLog::VALIDATION, "%s: %s prev block not found\n", __func__, hash.ToString()); return state.Invalid(BlockValidationResult::BLOCK_MISSING_PREV, "prev-blk-not-found"); } pindexPrev = (*mi).second; if (pindexPrev->nStatus & BLOCK_FAILED_MASK) { - LogPrintf("ERROR: %s: prev block invalid\n", __func__); + LogPrint(BCLog::VALIDATION, "%s: %s prev block invalid\n", __func__, hash.ToString()); return state.Invalid(BlockValidationResult::BLOCK_INVALID_PREV, "bad-prevblk"); } - if (!ContextualCheckBlockHeader(block, state, *this, chainparams, pindexPrev, GetAdjustedTime())) - return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), state.ToString()); + if (!ContextualCheckBlockHeader(block, state, *this, chainparams, pindexPrev, GetAdjustedTime())) { + LogPrint(BCLog::VALIDATION, "%s: Consensus::ContextualCheckBlockHeader: %s, %s\n", __func__, hash.ToString(), state.ToString()); + return false; + } /* Determine if this block descends from any block which has been found * invalid (m_failed_blocks), then mark pindexPrev and any blocks between @@ -3262,7 +3259,7 @@ bool BlockManager::AcceptBlockHeader(const CBlockHeader& block, BlockValidationS setDirtyBlockIndex.insert(invalid_walk); invalid_walk = invalid_walk->pprev; } - LogPrintf("ERROR: %s: prev block invalid\n", __func__); + LogPrint(BCLog::VALIDATION, "%s: %s prev block invalid\n", __func__, hash.ToString()); return state.Invalid(BlockValidationResult::BLOCK_INVALID_PREV, "bad-prevblk"); } } @@ -4497,7 +4494,7 @@ bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mocka // wallet(s) having loaded it while we were processing // mempool transactions; consider these as valid, instead of // failed, but mark them as 'already there' - if (pool.exists(tx->GetHash())) { + if (pool.exists(GenTxid::Txid(tx->GetHash()))) { ++already_there; } else { ++failed; diff --git a/src/validation.h b/src/validation.h index caa0832dd3..4da8ec8d24 100644 --- a/src/validation.h +++ b/src/validation.h @@ -10,10 +10,10 @@ #include <config/bitcoin-config.h> #endif -#include <amount.h> #include <arith_uint256.h> #include <attributes.h> #include <chain.h> +#include <consensus/amount.h> #include <fs.h> #include <policy/feerate.h> #include <policy/packages.h> @@ -75,7 +75,7 @@ static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; static const bool DEFAULT_PERSIST_MEMPOOL = true; /** Default for -stopatheight */ static const int DEFAULT_STOPATHEIGHT = 0; -/** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ::ChainActive().Tip() will not be pruned. */ +/** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ActiveChain().Tip() will not be pruned. */ static const unsigned int MIN_BLOCKS_TO_KEEP = 288; static const signed int DEFAULT_CHECKBLOCKS = 6; static const unsigned int DEFAULT_CHECKLEVEL = 3; diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index 2eb4d3106c..2290e119fd 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -61,9 +61,9 @@ bool WalletDatabaseFileId::operator==(const WalletDatabaseFileId& rhs) const std::shared_ptr<BerkeleyEnvironment> GetBerkeleyEnv(const fs::path& env_directory) { LOCK(cs_db); - auto inserted = g_dbenvs.emplace(env_directory.string(), std::weak_ptr<BerkeleyEnvironment>()); + auto inserted = g_dbenvs.emplace(fs::PathToString(env_directory), std::weak_ptr<BerkeleyEnvironment>()); if (inserted.second) { - auto env = std::make_shared<BerkeleyEnvironment>(env_directory.string()); + auto env = std::make_shared<BerkeleyEnvironment>(env_directory); inserted.first->second = env; return env; } @@ -101,7 +101,7 @@ void BerkeleyEnvironment::Close() if (error_file) fclose(error_file); - UnlockDirectory(strPath, ".walletlock"); + UnlockDirectory(fs::PathFromString(strPath), ".walletlock"); } void BerkeleyEnvironment::Reset() @@ -111,7 +111,7 @@ void BerkeleyEnvironment::Reset() fMockDb = false; } -BerkeleyEnvironment::BerkeleyEnvironment(const fs::path& dir_path) : strPath(dir_path.string()) +BerkeleyEnvironment::BerkeleyEnvironment(const fs::path& dir_path) : strPath(fs::PathToString(dir_path)) { Reset(); } @@ -129,24 +129,24 @@ bool BerkeleyEnvironment::Open(bilingual_str& err) return true; } - fs::path pathIn = strPath; + fs::path pathIn = fs::PathFromString(strPath); TryCreateDirectories(pathIn); if (!LockDirectory(pathIn, ".walletlock")) { LogPrintf("Cannot obtain a lock on wallet directory %s. Another instance of bitcoin may be using it.\n", strPath); - err = strprintf(_("Error initializing wallet database environment %s!"), Directory()); + err = strprintf(_("Error initializing wallet database environment %s!"), fs::quoted(fs::PathToString(Directory()))); return false; } fs::path pathLogDir = pathIn / "database"; TryCreateDirectories(pathLogDir); fs::path pathErrorFile = pathIn / "db.log"; - LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string()); + LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", fs::PathToString(pathLogDir), fs::PathToString(pathErrorFile)); unsigned int nEnvFlags = 0; if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB)) nEnvFlags |= DB_PRIVATE; - dbenv->set_lg_dir(pathLogDir.string().c_str()); + dbenv->set_lg_dir(fs::PathToString(pathLogDir).c_str()); dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet dbenv->set_lg_bsize(0x10000); dbenv->set_lg_max(1048576); @@ -173,7 +173,7 @@ bool BerkeleyEnvironment::Open(bilingual_str& err) LogPrintf("BerkeleyEnvironment::Open: Error %d closing failed database environment: %s\n", ret2, DbEnv::strerror(ret2)); } Reset(); - err = strprintf(_("Error initializing wallet database environment %s!"), Directory()); + err = strprintf(_("Error initializing wallet database environment %s!"), fs::quoted(fs::PathToString(Directory()))); if (ret == DB_RUNRECOVERY) { err += Untranslated(" ") + _("This error could occur if this wallet was not shutdown cleanly and was last loaded using a build with a newer version of Berkeley DB. If so, please use the software that last loaded this wallet"); } @@ -261,7 +261,7 @@ bool BerkeleyDatabase::Verify(bilingual_str& errorStr) fs::path file_path = walletDir / strFile; LogPrintf("Using BerkeleyDB version %s\n", BerkeleyDatabaseVersion()); - LogPrintf("Using wallet %s\n", file_path.string()); + LogPrintf("Using wallet %s\n", fs::PathToString(file_path)); if (!env->Open(errorStr)) { return false; @@ -274,7 +274,7 @@ bool BerkeleyDatabase::Verify(bilingual_str& errorStr) Db db(env->dbenv.get(), 0); int result = db.verify(strFile.c_str(), nullptr, nullptr, 0); if (result != 0) { - errorStr = strprintf(_("%s corrupt. Try using the wallet tool bitcoin-wallet to salvage or restoring a backup."), file_path); + errorStr = strprintf(_("%s corrupt. Try using the wallet tool bitcoin-wallet to salvage or restoring a backup."), fs::quoted(fs::PathToString(file_path))); return false; } } @@ -566,7 +566,7 @@ void BerkeleyEnvironment::Flush(bool fShutdown) dbenv->log_archive(&listp, DB_ARCH_REMOVE); Close(); if (!fMockDb) { - fs::remove_all(fs::path(strPath) / "database"); + fs::remove_all(fs::PathFromString(strPath) / "database"); } } } @@ -614,21 +614,21 @@ bool BerkeleyDatabase::Backup(const std::string& strDest) const // Copy wallet file fs::path pathSrc = env->Directory() / strFile; - fs::path pathDest(strDest); + fs::path pathDest(fs::PathFromString(strDest)); if (fs::is_directory(pathDest)) - pathDest /= strFile; + pathDest /= fs::PathFromString(strFile); try { if (fs::equivalent(pathSrc, pathDest)) { - LogPrintf("cannot backup to wallet source file %s\n", pathDest.string()); + LogPrintf("cannot backup to wallet source file %s\n", fs::PathToString(pathDest)); return false; } fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists); - LogPrintf("copied %s to %s\n", strFile, pathDest.string()); + LogPrintf("copied %s to %s\n", strFile, fs::PathToString(pathDest)); return true; } catch (const fs::filesystem_error& e) { - LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), fsbridge::get_filesystem_error_message(e)); + LogPrintf("error copying %s to %s - %s\n", strFile, fs::PathToString(pathDest), fsbridge::get_filesystem_error_message(e)); return false; } } @@ -828,10 +828,10 @@ std::unique_ptr<BerkeleyDatabase> MakeBerkeleyDatabase(const fs::path& path, con std::unique_ptr<BerkeleyDatabase> db; { LOCK(cs_db); // Lock env.m_databases until insert in BerkeleyDatabase constructor - std::string data_filename = data_file.filename().string(); + std::string data_filename = fs::PathToString(data_file.filename()); std::shared_ptr<BerkeleyEnvironment> env = GetBerkeleyEnv(data_file.parent_path()); if (env->m_databases.count(data_filename)) { - error = Untranslated(strprintf("Refusing to load database. Data file '%s' is already loaded.", (env->Directory() / data_filename).string())); + error = Untranslated(strprintf("Refusing to load database. Data file '%s' is already loaded.", fs::PathToString(env->Directory() / data_filename))); status = DatabaseStatus::FAILED_ALREADY_LOADED; return nullptr; } diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h index a8209587d7..b666a8e73a 100644 --- a/src/wallet/bdb.h +++ b/src/wallet/bdb.h @@ -63,7 +63,7 @@ public: bool IsMock() const { return fMockDb; } bool IsInitialized() const { return fDbEnvInit; } - fs::path Directory() const { return strPath; } + fs::path Directory() const { return fs::PathFromString(strPath); } bool Open(bilingual_str& error); void Close(); @@ -141,7 +141,7 @@ public: bool Verify(bilingual_str& error); /** Return path to main database filename */ - std::string Filename() override { return (env->Directory() / strFile).string(); } + std::string Filename() override { return fs::PathToString(env->Directory() / strFile); } std::string Format() override { return "bdb"; } /** diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index 85cbec76b7..edd81e590f 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -9,9 +9,14 @@ #include <policy/feerate.h> #include <policy/fees.h> #include <primitives/transaction.h> +#include <script/keyorigin.h> +#include <script/signingprovider.h> #include <script/standard.h> #include <optional> +#include <algorithm> +#include <map> +#include <set> const int DEFAULT_MIN_DEPTH = 0; const int DEFAULT_MAX_DEPTH = 9999999; @@ -53,6 +58,8 @@ public: int m_min_depth = DEFAULT_MIN_DEPTH; //! Maximum chain depth value for coin availability int m_max_depth = DEFAULT_MAX_DEPTH; + //! SigningProvider that has pubkeys and scripts to do spend size estimation for external inputs + FlatSigningProvider m_external_provider; CCoinControl(); @@ -66,11 +73,32 @@ public: return (setSelected.count(output) > 0); } + bool IsExternalSelected(const COutPoint& output) const + { + return (m_external_txouts.count(output) > 0); + } + + bool GetExternalOutput(const COutPoint& outpoint, CTxOut& txout) const + { + const auto ext_it = m_external_txouts.find(outpoint); + if (ext_it == m_external_txouts.end()) { + return false; + } + txout = ext_it->second; + return true; + } + void Select(const COutPoint& output) { setSelected.insert(output); } + void SelectExternal(const COutPoint& outpoint, const CTxOut& txout) + { + setSelected.insert(outpoint); + m_external_txouts.emplace(outpoint, txout); + } + void UnSelect(const COutPoint& output) { setSelected.erase(output); @@ -88,6 +116,7 @@ public: private: std::set<COutPoint> setSelected; + std::map<COutPoint, CTxOut> m_external_txouts; }; #endif // BITCOIN_WALLET_COINCONTROL_H diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 8d08153501..e1ca3fb379 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -4,6 +4,7 @@ #include <wallet/coinselection.h> +#include <consensus/amount.h> #include <policy/feerate.h> #include <util/check.h> #include <util/system.h> diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index a28bee622e..e7d467660f 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_WALLET_COINSELECTION_H #define BITCOIN_WALLET_COINSELECTION_H -#include <amount.h> +#include <consensus/amount.h> #include <policy/feerate.h> #include <primitives/transaction.h> #include <random.h> @@ -37,6 +37,18 @@ public: m_input_bytes = input_bytes; } + CInputCoin(const COutPoint& outpoint_in, const CTxOut& txout_in) + { + outpoint = outpoint_in; + txout = txout_in; + effective_value = txout.nValue; + } + + CInputCoin(const COutPoint& outpoint_in, const CTxOut& txout_in, int input_bytes) : CInputCoin(outpoint_in, txout_in) + { + m_input_bytes = input_bytes; + } + COutPoint outpoint; CTxOut txout; CAmount effective_value; diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 8d5316e0af..c74c69ed09 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -12,7 +12,7 @@ std::vector<fs::path> ListDatabases(const fs::path& wallet_dir) { - const size_t offset = wallet_dir.string().size() + (wallet_dir == wallet_dir.root_name() ? 0 : 1); + const size_t offset = wallet_dir.native().size() + (wallet_dir == wallet_dir.root_name() ? 0 : 1); std::vector<fs::path> paths; boost::system::error_code ec; @@ -20,9 +20,9 @@ std::vector<fs::path> ListDatabases(const fs::path& wallet_dir) if (ec) { if (fs::is_directory(*it)) { it.no_push(); - LogPrintf("%s: %s %s -- skipping.\n", __func__, ec.message(), it->path().string()); + LogPrintf("%s: %s %s -- skipping.\n", __func__, ec.message(), fs::PathToString(it->path())); } else { - LogPrintf("%s: %s %s\n", __func__, ec.message(), it->path().string()); + LogPrintf("%s: %s %s\n", __func__, ec.message(), fs::PathToString(it->path())); } continue; } @@ -30,7 +30,8 @@ std::vector<fs::path> ListDatabases(const fs::path& wallet_dir) try { // Get wallet path relative to walletdir by removing walletdir from the wallet path. // This can be replaced by boost::filesystem::lexically_relative once boost is bumped to 1.60. - const fs::path path = it->path().string().substr(offset); + const auto path_str = it->path().native().substr(offset); + const fs::path path{path_str.begin(), path_str.end()}; if (it->status().type() == fs::directory_file && (IsBDBFile(BDBDataFile(it->path())) || IsSQLiteFile(SQLiteDataFile(it->path())))) { @@ -50,7 +51,7 @@ std::vector<fs::path> ListDatabases(const fs::path& wallet_dir) } } } catch (const std::exception& e) { - LogPrintf("%s: Error scanning %s: %s\n", __func__, it->path().string(), e.what()); + LogPrintf("%s: Error scanning %s: %s\n", __func__, fs::PathToString(it->path()), e.what()); it.no_push(); } } @@ -85,7 +86,7 @@ bool IsBDBFile(const fs::path& path) // This check also prevents opening lock files. boost::system::error_code ec; auto size = fs::file_size(path, ec); - if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), path.string()); + if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), fs::PathToString(path)); if (size < 4096) return false; fsbridge::ifstream file(path, std::ios::binary); @@ -109,7 +110,7 @@ bool IsSQLiteFile(const fs::path& path) // A SQLite Database file is at least 512 bytes. boost::system::error_code ec; auto size = fs::file_size(path, ec); - if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), path.string()); + if (ec) LogPrintf("%s: %s %s\n", __func__, ec.message(), fs::PathToString(path)); if (size < 512) return false; fsbridge::ifstream file(path, std::ios::binary); diff --git a/src/wallet/dump.cpp b/src/wallet/dump.cpp index c39c0c7e73..08d94b76d9 100644 --- a/src/wallet/dump.cpp +++ b/src/wallet/dump.cpp @@ -19,16 +19,16 @@ bool DumpWallet(CWallet& wallet, bilingual_str& error) return false; } - fs::path path = dump_filename; + fs::path path = fs::PathFromString(dump_filename); path = fs::absolute(path); if (fs::exists(path)) { - error = strprintf(_("File %s already exists. If you are sure this is what you want, move it out of the way first."), path.string()); + error = strprintf(_("File %s already exists. If you are sure this is what you want, move it out of the way first."), fs::PathToString(path)); return false; } fsbridge::ofstream dump_file; dump_file.open(path); if (dump_file.fail()) { - error = strprintf(_("Unable to open %s for writing"), path.string()); + error = strprintf(_("Unable to open %s for writing"), fs::PathToString(path)); return false; } @@ -114,10 +114,10 @@ bool CreateFromDump(const std::string& name, const fs::path& wallet_path, biling return false; } - fs::path dump_path = dump_filename; + fs::path dump_path = fs::PathFromString(dump_filename); dump_path = fs::absolute(dump_path); if (!fs::exists(dump_path)) { - error = strprintf(_("Dump file %s does not exist."), dump_path.string()); + error = strprintf(_("Dump file %s does not exist."), fs::PathToString(dump_path)); return false; } fsbridge::ifstream dump_file(dump_path); diff --git a/src/wallet/fees.h b/src/wallet/fees.h index 434f211dc2..d6d625d9c1 100644 --- a/src/wallet/fees.h +++ b/src/wallet/fees.h @@ -6,7 +6,7 @@ #ifndef BITCOIN_WALLET_FEES_H #define BITCOIN_WALLET_FEES_H -#include <amount.h> +#include <consensus/amount.h> class CCoinControl; class CFeeRate; diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index d9fc6de79b..57f1a6a67a 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -4,7 +4,7 @@ #include <interfaces/wallet.h> -#include <amount.h> +#include <consensus/amount.h> #include <interfaces/chain.h> #include <interfaces/handler.h> #include <policy/fees.h> @@ -551,13 +551,13 @@ public: } std::string getWalletDir() override { - return GetWalletDir().string(); + return fs::PathToString(GetWalletDir()); } std::vector<std::string> listWalletDir() override { std::vector<std::string> paths; for (auto& path : ListDatabases(GetWalletDir())) { - paths.push_back(path.string()); + paths.push_back(fs::PathToString(path)); } return paths; } diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 1b841026b8..7ef5a0cf55 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -25,25 +25,25 @@ bool VerifyWallets(WalletContext& context) ArgsManager& args = *Assert(context.args); if (args.IsArgSet("-walletdir")) { - fs::path wallet_dir = args.GetArg("-walletdir", ""); + fs::path wallet_dir = fs::PathFromString(args.GetArg("-walletdir", "")); boost::system::error_code error; // The canonical path cleans the path, preventing >1 Berkeley environment instances for the same directory fs::path canonical_wallet_dir = fs::canonical(wallet_dir, error); if (error || !fs::exists(wallet_dir)) { - chain.initError(strprintf(_("Specified -walletdir \"%s\" does not exist"), wallet_dir.string())); + chain.initError(strprintf(_("Specified -walletdir \"%s\" does not exist"), fs::PathToString(wallet_dir))); return false; } else if (!fs::is_directory(wallet_dir)) { - chain.initError(strprintf(_("Specified -walletdir \"%s\" is not a directory"), wallet_dir.string())); + chain.initError(strprintf(_("Specified -walletdir \"%s\" is not a directory"), fs::PathToString(wallet_dir))); return false; // The canonical path transforms relative paths into absolute ones, so we check the non-canonical version } else if (!wallet_dir.is_absolute()) { - chain.initError(strprintf(_("Specified -walletdir \"%s\" is a relative path"), wallet_dir.string())); + chain.initError(strprintf(_("Specified -walletdir \"%s\" is a relative path"), fs::PathToString(wallet_dir))); return false; } - args.ForceSetArg("-walletdir", canonical_wallet_dir.string()); + args.ForceSetArg("-walletdir", fs::PathToString(canonical_wallet_dir)); } - LogPrintf("Using wallet directory %s\n", GetWalletDir().string()); + LogPrintf("Using wallet directory %s\n", fs::PathToString(GetWalletDir())); chain.initMessage(_("Verifying wallet(s)…").translated); @@ -70,7 +70,7 @@ bool VerifyWallets(WalletContext& context) for (const auto& wallet : chain.getSettingsList("wallet")) { const auto& wallet_file = wallet.get_str(); - const fs::path path = fsbridge::AbsPathJoin(GetWalletDir(), wallet_file); + const fs::path path = fsbridge::AbsPathJoin(GetWalletDir(), fs::PathFromString(wallet_file)); if (!wallet_paths.insert(path).second) { chain.initWarning(strprintf(_("Ignoring duplicate -wallet %s."), wallet_file)); @@ -102,7 +102,7 @@ bool LoadWallets(WalletContext& context) std::set<fs::path> wallet_paths; for (const auto& wallet : chain.getSettingsList("wallet")) { const auto& name = wallet.get_str(); - if (!wallet_paths.insert(name).second) { + if (!wallet_paths.insert(fs::PathFromString(name)).second) { continue; } DatabaseOptions options; diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index 98706dcdf8..2fb274b55f 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <consensus/amount.h> #include <consensus/consensus.h> #include <wallet/receive.h> #include <wallet/transaction.h> diff --git a/src/wallet/receive.h b/src/wallet/receive.h index b4b311636b..f659955fc6 100644 --- a/src/wallet/receive.h +++ b/src/wallet/receive.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_WALLET_RECEIVE_H #define BITCOIN_WALLET_RECEIVE_H -#include <amount.h> +#include <consensus/amount.h> #include <wallet/ismine.h> #include <wallet/transaction.h> #include <wallet/wallet.h> diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 4d7fb2d38c..9b09bc23d6 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -550,7 +550,7 @@ RPCHelpMan importwallet() EnsureWalletIsUnlocked(*pwallet); fsbridge::ifstream file; - file.open(request.params[0].get_str(), std::ios::in | std::ios::ate); + file.open(fs::u8path(request.params[0].get_str()), std::ios::in | std::ios::ate); if (!file.is_open()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot open wallet dump file"); } @@ -745,7 +745,7 @@ RPCHelpMan dumpwallet() EnsureWalletIsUnlocked(wallet); - fs::path filepath = request.params[0].get_str(); + fs::path filepath = fs::u8path(request.params[0].get_str()); filepath = fs::absolute(filepath); /* Prevent arbitrary files from being overwritten. There have been reports @@ -754,7 +754,7 @@ RPCHelpMan dumpwallet() * It may also avoid other security issues. */ if (fs::exists(filepath)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, filepath.string() + " already exists. If you are sure this is what you want, move it out of the way first"); + throw JSONRPCError(RPC_INVALID_PARAMETER, filepath.u8string() + " already exists. If you are sure this is what you want, move it out of the way first"); } fsbridge::ofstream file; @@ -844,7 +844,7 @@ RPCHelpMan dumpwallet() file.close(); UniValue reply(UniValue::VOBJ); - reply.pushKV("filename", filepath.string()); + reply.pushKV("filename", filepath.u8string()); return reply; }, diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index f6cf8868de..f2f28c83ff 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> +#include <consensus/amount.h> #include <core_io.h> #include <interfaces/chain.h> #include <key_io.h> @@ -43,6 +43,7 @@ #include <univalue.h> +#include <map> using interfaces::FoundBlock; @@ -1388,8 +1389,9 @@ static const std::vector<RPCResult> TransactionDescriptionString() { return{{RPCResult::Type::NUM, "confirmations", "The number of confirmations for the transaction. Negative confirmations means the\n" "transaction conflicted that many blocks ago."}, - {RPCResult::Type::BOOL, "generated", /* optional */ true, "Only present if transaction only input is a coinbase one."}, - {RPCResult::Type::BOOL, "trusted", /* optional */ true, "Only present if we consider transaction to be trusted and so safe to spend from."}, + {RPCResult::Type::BOOL, "generated", /* optional */ true, "Only present if the transaction's only input is a coinbase one."}, + {RPCResult::Type::BOOL, "trusted", /* optional */ true, "Whether we consider the transaction to be trusted and safe to spend from.\n" + "Only present when the transaction has 0 confirmations (or negative confirmations, if conflicted)."}, {RPCResult::Type::STR_HEX, "blockhash", /* optional */ true, "The block hash containing the transaction."}, {RPCResult::Type::NUM, "blockheight", /* optional */ true, "The block height containing the transaction."}, {RPCResult::Type::NUM, "blockindex", /* optional */ true, "The index of the transaction in the block that includes it."}, @@ -1407,7 +1409,7 @@ static const std::vector<RPCResult> TransactionDescriptionString() {RPCResult::Type::NUM_TIME, "timereceived", "The time received expressed in " + UNIX_EPOCH_TIME + "."}, {RPCResult::Type::STR, "comment", /* optional */ true, "If a comment is associated with the transaction, only present if not empty."}, {RPCResult::Type::STR, "bip125-replaceable", "(\"yes|no|unknown\") Whether this transaction could be replaced due to BIP125 (replace-by-fee);\n" - "may be unknown for unconfirmed transactions not in the mempool"}}; + "may be unknown for unconfirmed transactions not in the mempool."}}; } static RPCHelpMan listtransactions() @@ -1853,7 +1855,7 @@ static RPCHelpMan keypoolrefill() "\nFills the keypool."+ HELP_REQUIRING_PASSPHRASE, { - {"newsize", RPCArg::Type::NUM, RPCArg::Default{100}, "The new keypool size"}, + {"newsize", RPCArg::Type::NUM, RPCArg::DefaultHint{strprintf("%u, or as set by -keypool", DEFAULT_KEYPOOL_SIZE)}, "The new keypool size"}, }, RPCResult{RPCResult::Type::NONE, "", ""}, RPCExamples{ @@ -1892,6 +1894,33 @@ static RPCHelpMan keypoolrefill() } +static RPCHelpMan newkeypool() +{ + return RPCHelpMan{"newkeypool", + "\nEntirely clears and refills the keypool."+ + HELP_REQUIRING_PASSPHRASE, + {}, + RPCResult{RPCResult::Type::NONE, "", ""}, + RPCExamples{ + HelpExampleCli("newkeypool", "") + + HelpExampleRpc("newkeypool", "") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); + if (!pwallet) return NullUniValue; + + LOCK(pwallet->cs_wallet); + + LegacyScriptPubKeyMan& spk_man = EnsureLegacyScriptPubKeyMan(*pwallet, true); + spk_man.NewKeyPool(); + + return NullUniValue; +}, + }; +} + + static RPCHelpMan walletpassphrase() { return RPCHelpMan{"walletpassphrase", @@ -2554,7 +2583,7 @@ static RPCHelpMan listwalletdir() UniValue wallets(UniValue::VARR); for (const auto& path : ListDatabases(GetWalletDir())) { UniValue wallet(UniValue::VOBJ); - wallet.pushKV("name", path.string()); + wallet.pushKV("name", path.u8string()); wallets.push_back(wallet); } @@ -2742,7 +2771,7 @@ static RPCHelpMan createwallet() {"blank", RPCArg::Type::BOOL, RPCArg::Default{false}, "Create a blank wallet. A blank wallet has no keys or HD seed. One can be set using sethdseed."}, {"passphrase", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "Encrypt the wallet with this passphrase."}, {"avoid_reuse", RPCArg::Type::BOOL, RPCArg::Default{false}, "Keep track of coin reuse, and treat dirty and clean coins differently with privacy considerations in mind."}, - {"descriptors", RPCArg::Type::BOOL, RPCArg::Default{false}, "Create a native descriptor wallet. The wallet will use descriptors internally to handle address creation"}, + {"descriptors", RPCArg::Type::BOOL, RPCArg::Default{true}, "Create a native descriptor wallet. The wallet will use descriptors internally to handle address creation"}, {"load_on_startup", RPCArg::Type::BOOL, RPCArg::Optional::OMITTED_NAMED_ARG, "Save wallet name to persistent settings and load on startup. True to add wallet to startup list, false to remove, null to leave unchanged."}, {"external_signer", RPCArg::Type::BOOL, RPCArg::Default{false}, "Use an external signer such as a hardware wallet. Requires -signer to be configured. Wallet creation will fail if keys cannot be fetched. Requires disable_private_keys and descriptors set to true."}, }, @@ -2784,12 +2813,11 @@ static RPCHelpMan createwallet() if (!request.params[4].isNull() && request.params[4].get_bool()) { flags |= WALLET_FLAG_AVOID_REUSE; } - if (!request.params[5].isNull() && request.params[5].get_bool()) { + if (request.params[5].isNull() || request.params[5].get_bool()) { #ifndef USE_SQLITE throw JSONRPCError(RPC_WALLET_ERROR, "Compiled without sqlite support (required for descriptor wallets)"); #endif flags |= WALLET_FLAG_DESCRIPTORS; - warnings.emplace_back(Untranslated("Wallet is an experimental descriptor wallet")); } if (!request.params[7].isNull() && request.params[7].get_bool()) { #ifdef ENABLE_EXTERNAL_SIGNER @@ -2855,7 +2883,7 @@ static RPCHelpMan restorewallet() WalletContext& context = EnsureWalletContext(request.context); - std::string backup_file = request.params[1].get_str(); + auto backup_file = fs::u8path(request.params[1].get_str()); if (!fs::exists(backup_file)) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Backup file does not exist"); @@ -2863,14 +2891,14 @@ static RPCHelpMan restorewallet() std::string wallet_name = request.params[0].get_str(); - const fs::path wallet_path = fsbridge::AbsPathJoin(GetWalletDir(), wallet_name); + const fs::path wallet_path = fsbridge::AbsPathJoin(GetWalletDir(), fs::u8path(wallet_name)); if (fs::exists(wallet_path)) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Wallet name already exists."); } if (!TryCreateDirectories(wallet_path)) { - throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Failed to create database path '%s'. Database already exists.", wallet_path.string())); + throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Failed to create database path '%s'. Database already exists.", wallet_path.u8string())); } auto wallet_file = wallet_path / "wallet.dat"; @@ -3176,6 +3204,34 @@ static RPCHelpMan listunspent() }; } +// Only includes key documentation where the key is snake_case in all RPC methods. MixedCase keys can be added later. +static std::vector<RPCArg> FundTxDoc() +{ + return { + {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, + {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, std::string() + "The fee estimate mode, must be one of (case insensitive):\n" + " \"" + FeeModes("\"\n\"") + "\""}, + {"replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125 replaceable.\n" + "Allows this transaction to be replaced by a transaction with higher fees"}, + {"solving_data", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "Keys and scripts needed for producing a final transaction with a dummy signature.\n" + "Used for fee estimation during coin selection.", + { + {"pubkeys", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Public keys involved in this transaction.", + { + {"pubkey", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A public key"}, + }}, + {"scripts", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Scripts involved in this transaction.", + { + {"script", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "A script"}, + }}, + {"descriptors", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "Descriptors that provide solving data for this transaction.", + { + {"descriptor", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "A descriptor"}, + }}, + }}, + }; +} + void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, int& change_position, const UniValue& options, CCoinControl& coinControl, bool override_min_fee) { // Make sure the results are valid at least up to the most recent block @@ -3213,6 +3269,7 @@ void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, {"fee_rate", UniValueType()}, // will be checked by AmountFromValue() in SetFeeEstimateMode() {"feeRate", UniValueType()}, // will be checked by AmountFromValue() below {"psbt", UniValueType(UniValue::VBOOL)}, + {"solving_data", UniValueType(UniValue::VOBJ)}, {"subtractFeeFromOutputs", UniValueType(UniValue::VARR)}, {"subtract_fee_from_outputs", UniValueType(UniValue::VARR)}, {"replaceable", UniValueType(UniValue::VBOOL)}, @@ -3289,6 +3346,54 @@ void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, coinControl.fAllowWatchOnly = ParseIncludeWatchonly(NullUniValue, wallet); } + if (options.exists("solving_data")) { + const UniValue solving_data = options["solving_data"].get_obj(); + if (solving_data.exists("pubkeys")) { + for (const UniValue& pk_univ : solving_data["pubkeys"].get_array().getValues()) { + const std::string& pk_str = pk_univ.get_str(); + if (!IsHex(pk_str)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("'%s' is not hex", pk_str)); + } + const std::vector<unsigned char> data(ParseHex(pk_str)); + const CPubKey pubkey(data.begin(), data.end()); + if (!pubkey.IsFullyValid()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("'%s' is not a valid public key", pk_str)); + } + coinControl.m_external_provider.pubkeys.emplace(pubkey.GetID(), pubkey); + // Add witness script for pubkeys + const CScript wit_script = GetScriptForDestination(WitnessV0KeyHash(pubkey)); + coinControl.m_external_provider.scripts.emplace(CScriptID(wit_script), wit_script); + } + } + + if (solving_data.exists("scripts")) { + for (const UniValue& script_univ : solving_data["scripts"].get_array().getValues()) { + const std::string& script_str = script_univ.get_str(); + if (!IsHex(script_str)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("'%s' is not hex", script_str)); + } + std::vector<unsigned char> script_data(ParseHex(script_str)); + const CScript script(script_data.begin(), script_data.end()); + coinControl.m_external_provider.scripts.emplace(CScriptID(script), script); + } + } + + if (solving_data.exists("descriptors")) { + for (const UniValue& desc_univ : solving_data["descriptors"].get_array().getValues()) { + const std::string& desc_str = desc_univ.get_str(); + FlatSigningProvider desc_out; + std::string error; + std::vector<CScript> scripts_temp; + std::unique_ptr<Descriptor> desc = Parse(desc_str, desc_out, error, true); + if (!desc) { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Unable to parse descriptor '%s': %s", desc_str, error)); + } + desc->Expand(0, desc_out, scripts_temp, desc_out); + coinControl.m_external_provider = Merge(coinControl.m_external_provider, desc_out); + } + } + } + if (tx.vout.size() == 0) throw JSONRPCError(RPC_INVALID_PARAMETER, "TX must have at least one output"); @@ -3306,6 +3411,19 @@ void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, setSubtractFeeFromOutputs.insert(pos); } + // Fetch specified UTXOs from the UTXO set to get the scriptPubKeys and values of the outputs being selected + // and to match with the given solving_data. Only used for non-wallet outputs. + std::map<COutPoint, Coin> coins; + for (const CTxIn& txin : tx.vin) { + coins[txin.prevout]; // Create empty map entry keyed by prevout. + } + wallet.chain().findCoins(coins); + for (const auto& coin : coins) { + if (!coin.second.out.IsNull()) { + coinControl.SelectExternal(coin.first, coin.second.out); + } + } + bilingual_str error; if (!FundTransaction(wallet, tx, fee_out, change_position, error, lockUnspents, setSubtractFeeFromOutputs, coinControl)) { @@ -3321,8 +3439,9 @@ static RPCHelpMan fundrawtransaction() "No existing outputs will be modified unless \"subtractFeeFromOutputs\" is specified.\n" "Note that inputs which were signed may need to be resigned after completion since in/outputs have been added.\n" "The inputs added will not be signed, use signrawtransactionwithkey\n" - " or signrawtransactionwithwallet for that.\n" - "Note that all existing inputs must have their previous output transaction be in the wallet.\n" + "or signrawtransactionwithwallet for that.\n" + "All existing inputs must either have their previous output transaction be in the wallet\n" + "or be in the UTXO set. Solving data must be provided for non-wallet inputs.\n" "Note that all inputs selected must be of standard form and P2SH scripts must be\n" "in the wallet using importaddress or addmultisigaddress (to calculate fees).\n" "You can see whether this is the case by checking the \"solvable\" field in the listunspent output.\n" @@ -3330,6 +3449,7 @@ static RPCHelpMan fundrawtransaction() { {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of the raw transaction"}, {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "for backward compatibility: passing in a true instead of an object will result in {\"includeWatching\":true}", + Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::Default{true}, "For a transaction with existing inputs, automatically include more if they are not enough."}, {"include_unsafe", RPCArg::Type::BOOL, RPCArg::Default{false}, "Include inputs that are not safe to spend (unconfirmed transactions from outside keys and unconfirmed replacement transactions).\n" @@ -3352,12 +3472,8 @@ static RPCHelpMan fundrawtransaction() {"vout_index", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "The zero-based output index, before a change output is added."}, }, }, - {"replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125 replaceable.\n" - "Allows this transaction to be replaced by a transaction with higher fees"}, - {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, - {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, std::string() + "The fee estimate mode, must be one of (case insensitive):\n" - " \"" + FeeModes("\"\n\"") + "\""}, }, + FundTxDoc()), "options"}, {"iswitness", RPCArg::Type::BOOL, RPCArg::DefaultHint{"depends on heuristic tests"}, "Whether the transaction hex is a serialized witness transaction.\n" "If iswitness is not present, heuristic tests will be used in decoding.\n" @@ -4166,6 +4282,7 @@ static RPCHelpMan send() " \"" + FeeModes("\"\n\"") + "\""}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", + Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::Default{false}, "If inputs are specified, automatically include more if they are not enough."}, {"include_unsafe", RPCArg::Type::BOOL, RPCArg::Default{false}, "Include inputs that are not safe to spend (unconfirmed transactions from outside keys and unconfirmed replacement transactions).\n" @@ -4175,9 +4292,6 @@ static RPCHelpMan send() {"change_address", RPCArg::Type::STR_HEX, RPCArg::DefaultHint{"pool address"}, "The bitcoin address to receive the change"}, {"change_position", RPCArg::Type::NUM, RPCArg::DefaultHint{"random"}, "The index of the change output"}, {"change_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The output type to use. Only valid if change_address is not specified. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, - {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, - {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, std::string() + "The fee estimate mode, must be one of (case insensitive):\n" - " \"" + FeeModes("\"\n\"") + "\""}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, {"include_watching", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Also select inputs which are watch only.\n" "Only solvable inputs can be used. Watch-only destinations are solvable if the public key and/or output script was imported,\n" @@ -4200,9 +4314,8 @@ static RPCHelpMan send() {"vout_index", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "The zero-based output index, before a change output is added."}, }, }, - {"replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125 replaceable.\n" - "Allows this transaction to be replaced by a transaction with higher fees"}, }, + FundTxDoc()), "options"}, }, RPCResult{ @@ -4489,7 +4602,9 @@ static RPCHelpMan walletcreatefundedpsbt() { return RPCHelpMan{"walletcreatefundedpsbt", "\nCreates and funds a transaction in the Partially Signed Transaction format.\n" - "Implements the Creator and Updater roles.\n", + "Implements the Creator and Updater roles.\n" + "All existing inputs must either have their previous output transaction be in the wallet\n" + "or be in the UTXO set. Solving data must be provided for non-wallet inputs.\n", { {"inputs", RPCArg::Type::ARR, RPCArg::Optional::OMITTED_NAMED_ARG, "Leave empty to add inputs automatically. See add_inputs option.", { @@ -4521,6 +4636,7 @@ static RPCHelpMan walletcreatefundedpsbt() }, {"locktime", RPCArg::Type::NUM, RPCArg::Default{0}, "Raw locktime. Non-0 value also locktime-activates inputs"}, {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", + Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::Default{false}, "If inputs are specified, automatically include more if they are not enough."}, {"include_unsafe", RPCArg::Type::BOOL, RPCArg::Default{false}, "Include inputs that are not safe to spend (unconfirmed transactions from outside keys and unconfirmed replacement transactions).\n" @@ -4541,12 +4657,8 @@ static RPCHelpMan walletcreatefundedpsbt() {"vout_index", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "The zero-based output index, before a change output is added."}, }, }, - {"replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125 replaceable.\n" - "Allows this transaction to be replaced by a transaction with higher fees"}, - {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, - {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, std::string() + "The fee estimate mode, must be one of (case insensitive):\n" - " \"" + FeeModes("\"\n\"") + "\""}, }, + FundTxDoc()), "options"}, {"bip32derivs", RPCArg::Type::BOOL, RPCArg::Default{true}, "Include BIP 32 derivation paths for public keys if we know them"}, }, @@ -4790,6 +4902,7 @@ static const CRPCCommand commands[] = { "wallet", &listwallets, }, { "wallet", &loadwallet, }, { "wallet", &lockunspent, }, + { "wallet", &newkeypool, }, { "wallet", &removeprunedfunds, }, { "wallet", &rescanblockchain, }, { "wallet", &send, }, diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index fdfb36bb0a..619ebc8b4f 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -489,7 +489,7 @@ bool LegacyScriptPubKeyMan::Upgrade(int prev_version, int new_version, bilingual } // Regenerate the keypool if upgraded to HD if (hd_upgrade) { - if (!TopUp()) { + if (!NewKeyPool()) { error = _("Unable to generate keys"); return false; } diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 1724375f4c..5470177440 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -2,9 +2,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <consensus/amount.h> #include <consensus/validation.h> #include <interfaces/chain.h> #include <policy/policy.h> +#include <script/signingprovider.h> #include <util/check.h> #include <util/fees.h> #include <util/moneystr.h> @@ -31,21 +33,27 @@ std::string COutput::ToString() const return strprintf("COutput(%s, %d, %d) [%s]", tx->GetHash().ToString(), i, nDepth, FormatMoney(tx->tx->vout[i].nValue)); } -int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* wallet, bool use_max_sig) +int CalculateMaximumSignedInputSize(const CTxOut& txout, const SigningProvider* provider, bool use_max_sig) { CMutableTransaction txn; txn.vin.push_back(CTxIn(COutPoint())); - if (!wallet->DummySignInput(txn.vin[0], txout, use_max_sig)) { + if (!provider || !DummySignInput(*provider, txn.vin[0], txout, use_max_sig)) { return -1; } return GetVirtualTransactionInputSize(txn.vin[0]); } +int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* wallet, bool use_max_sig) +{ + const std::unique_ptr<SigningProvider> provider = wallet->GetSolvingProvider(txout.scriptPubKey); + return CalculateMaximumSignedInputSize(txout, provider.get(), use_max_sig); +} + // txouts needs to be in the order of tx.vin -TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, const std::vector<CTxOut>& txouts, bool use_max_sig) +TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, const std::vector<CTxOut>& txouts, const CCoinControl* coin_control) { CMutableTransaction txNew(tx); - if (!wallet->DummySignTx(txNew, txouts, use_max_sig)) { + if (!wallet->DummySignTx(txNew, txouts, coin_control)) { return TxSize{-1, -1}; } CTransaction ctx(txNew); @@ -54,19 +62,27 @@ TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *walle return TxSize{vsize, weight}; } -TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, bool use_max_sig) +TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, const CCoinControl* coin_control) { std::vector<CTxOut> txouts; + // Look up the inputs. The inputs are either in the wallet, or in coin_control. for (const CTxIn& input : tx.vin) { const auto mi = wallet->mapWallet.find(input.prevout.hash); // Can not estimate size without knowing the input details - if (mi == wallet->mapWallet.end()) { + if (mi != wallet->mapWallet.end()) { + assert(input.prevout.n < mi->second.tx->vout.size()); + txouts.emplace_back(mi->second.tx->vout.at(input.prevout.n)); + } else if (coin_control) { + CTxOut txout; + if (!coin_control->GetExternalOutput(input.prevout, txout)) { + return TxSize{-1, -1}; + } + txouts.emplace_back(txout); + } else { return TxSize{-1, -1}; } - assert(input.prevout.n < mi->second.tx->vout.size()); - txouts.emplace_back(mi->second.tx->vout[input.prevout.n]); } - return CalculateMaximumSignedTxSize(tx, wallet, txouts, use_max_sig); + return CalculateMaximumSignedTxSize(tx, wallet, txouts, coin_control); } void AvailableCoins(const CWallet& wallet, std::vector<COutput>& vCoins, const CCoinControl* coinControl, const CAmount& nMinimumAmount, const CAmount& nMaximumAmount, const CAmount& nMinimumSumAmount, const uint64_t nMaximumCount) @@ -435,32 +451,40 @@ bool SelectCoins(const CWallet& wallet, const std::vector<COutput>& vAvailableCo std::vector<COutPoint> vPresetInputs; coin_control.ListSelected(vPresetInputs); - for (const COutPoint& outpoint : vPresetInputs) - { + for (const COutPoint& outpoint : vPresetInputs) { + int input_bytes = -1; + CTxOut txout; std::map<uint256, CWalletTx>::const_iterator it = wallet.mapWallet.find(outpoint.hash); - if (it != wallet.mapWallet.end()) - { + if (it != wallet.mapWallet.end()) { const CWalletTx& wtx = it->second; // Clearly invalid input, fail if (wtx.tx->vout.size() <= outpoint.n) { return false; } - // Just to calculate the marginal byte size - CInputCoin coin(wtx.tx, outpoint.n, GetTxSpendSize(wallet, wtx, outpoint.n, false)); - nValueFromPresetInputs += coin.txout.nValue; - if (coin.m_input_bytes <= 0) { - return false; // Not solvable, can't estimate size for fee - } - coin.effective_value = coin.txout.nValue - coin_selection_params.m_effective_feerate.GetFee(coin.m_input_bytes); - if (coin_selection_params.m_subtract_fee_outputs) { - value_to_select -= coin.txout.nValue; - } else { - value_to_select -= coin.effective_value; + input_bytes = GetTxSpendSize(wallet, wtx, outpoint.n, false); + txout = wtx.tx->vout.at(outpoint.n); + } + if (input_bytes == -1) { + // The input is external. We either did not find the tx in mapWallet, or we did but couldn't compute the input size with wallet data + if (!coin_control.GetExternalOutput(outpoint, txout)) { + // Not ours, and we don't have solving data. + return false; } - setPresetCoins.insert(coin); + input_bytes = CalculateMaximumSignedInputSize(txout, &coin_control.m_external_provider, /* use_max_sig */ true); + } + + CInputCoin coin(outpoint, txout, input_bytes); + nValueFromPresetInputs += coin.txout.nValue; + if (coin.m_input_bytes == -1) { + return false; // Not solvable, can't estimate size for fee + } + coin.effective_value = coin.txout.nValue - coin_selection_params.m_effective_feerate.GetFee(coin.m_input_bytes); + if (coin_selection_params.m_subtract_fee_outputs) { + value_to_select -= coin.txout.nValue; } else { - return false; // TODO: Allow non-wallet inputs + value_to_select -= coin.effective_value; } + setPresetCoins.insert(coin); } // remove preset inputs from vCoins so that Coin Selection doesn't pick them. @@ -788,10 +812,10 @@ static bool CreateTransactionInternal( } // Calculate the transaction fee - TxSize tx_sizes = CalculateMaximumSignedTxSize(CTransaction(txNew), &wallet, coin_control.fAllowWatchOnly); + TxSize tx_sizes = CalculateMaximumSignedTxSize(CTransaction(txNew), &wallet, &coin_control); int nBytes = tx_sizes.vsize; - if (nBytes < 0) { - error = _("Signing transaction failed"); + if (nBytes == -1) { + error = _("Missing solving data for estimating transaction size"); return false; } nFeeRet = coin_selection_params.m_effective_feerate.GetFee(nBytes); @@ -813,7 +837,7 @@ static bool CreateTransactionInternal( txNew.vout.erase(change_position); // Because we have dropped this change, the tx size and required fee will be different, so let's recalculate those - tx_sizes = CalculateMaximumSignedTxSize(CTransaction(txNew), &wallet, coin_control.fAllowWatchOnly); + tx_sizes = CalculateMaximumSignedTxSize(CTransaction(txNew), &wallet, &coin_control); nBytes = tx_sizes.vsize; fee_needed = coin_selection_params.m_effective_feerate.GetFee(nBytes); } diff --git a/src/wallet/spend.h b/src/wallet/spend.h index e39f134dc3..7467dd9fa3 100644 --- a/src/wallet/spend.h +++ b/src/wallet/spend.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_WALLET_SPEND_H #define BITCOIN_WALLET_SPEND_H +#include <consensus/amount.h> #include <wallet/coinselection.h> #include <wallet/transaction.h> #include <wallet/wallet.h> @@ -66,6 +67,7 @@ public: //Get the marginal bytes of spending the specified output int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* pwallet, bool use_max_sig = false); +int CalculateMaximumSignedInputSize(const CTxOut& txout, const SigningProvider* pwallet, bool use_max_sig = false); struct TxSize { int64_t vsize{-1}; @@ -76,8 +78,8 @@ struct TxSize { * Use DummySignatureCreator, which inserts 71 byte signatures everywhere. * NOTE: this requires that all inputs must be in mapWallet (eg the tx should * be AllInputsMine). */ -TxSize CalculateMaximumSignedTxSize(const CTransaction& tx, const CWallet* wallet, const std::vector<CTxOut>& txouts, bool use_max_sig = false); -TxSize CalculateMaximumSignedTxSize(const CTransaction& tx, const CWallet* wallet, bool use_max_sig = false) EXCLUSIVE_LOCKS_REQUIRED(wallet->cs_wallet); +TxSize CalculateMaximumSignedTxSize(const CTransaction& tx, const CWallet* wallet, const std::vector<CTxOut>& txouts, const CCoinControl* coin_control = nullptr); +TxSize CalculateMaximumSignedTxSize(const CTransaction& tx, const CWallet* wallet, const CCoinControl* coin_control = nullptr) EXCLUSIVE_LOCKS_REQUIRED(wallet->cs_wallet); /** * populate vCoins with vector of available COutputs. diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index 815d17967c..650e083e8e 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -67,7 +67,7 @@ static void SetPragma(sqlite3* db, const std::string& key, const std::string& va } SQLiteDatabase::SQLiteDatabase(const fs::path& dir_path, const fs::path& file_path, bool mock) - : WalletDatabase(), m_mock(mock), m_dir_path(dir_path.string()), m_file_path(file_path.string()) + : WalletDatabase(), m_mock(mock), m_dir_path(fs::PathToString(dir_path)), m_file_path(fs::PathToString(file_path)) { { LOCK(g_sqlite_mutex); @@ -206,7 +206,7 @@ void SQLiteDatabase::Open() if (m_db == nullptr) { if (!m_mock) { - TryCreateDirectories(m_dir_path); + TryCreateDirectories(fs::PathFromString(m_dir_path)); } int ret = sqlite3_open_v2(m_file_path.c_str(), &m_db, flags, nullptr); if (ret != SQLITE_OK) { diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 40d9e90d56..8606924bb3 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <amount.h> +#include <consensus/amount.h> #include <node/context.h> #include <primitives/transaction.h> #include <random.h> @@ -28,19 +28,9 @@ BOOST_FIXTURE_TEST_SUITE(coinselector_tests, WalletTestingSetup) typedef std::set<CInputCoin> CoinSet; -static std::vector<COutput> vCoins; -static NodeContext testNode; -static auto testChain = interfaces::MakeChain(testNode); -static CWallet testWallet(testChain.get(), "", CreateDummyWalletDatabase()); -static CAmount balance = 0; - -CoinEligibilityFilter filter_standard(1, 6, 0); -CoinEligibilityFilter filter_confirmed(1, 1, 0); -CoinEligibilityFilter filter_standard_extra(6, 6, 0); -CoinSelectionParams coin_selection_params(/* change_output_size= */ 0, - /* change_spend_size= */ 0, /* effective_feerate= */ CFeeRate(0), - /* long_term_feerate= */ CFeeRate(0), /* discard_feerate= */ CFeeRate(0), - /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); +static const CoinEligibilityFilter filter_standard(1, 6, 0); +static const CoinEligibilityFilter filter_confirmed(1, 1, 0); +static const CoinEligibilityFilter filter_standard_extra(6, 6, 0); static void add_coin(const CAmount& nValue, int nInput, std::vector<CInputCoin>& set) { @@ -62,9 +52,8 @@ static void add_coin(const CAmount& nValue, int nInput, CoinSet& set, CAmount fe set.insert(coin); } -static void add_coin(CWallet& wallet, const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = false, int nInput=0, bool spendable = false) +static void add_coin(std::vector<COutput>& coins, CWallet& wallet, const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = false, int nInput=0, bool spendable = false) { - balance += nValue; static int nextLockTime = 0; CMutableTransaction tx; tx.nLockTime = nextLockTime++; // so all transactions get different hashes @@ -82,24 +71,19 @@ static void add_coin(CWallet& wallet, const CAmount& nValue, int nAge = 6*24, bo // so stop vin being empty, and cache a non-zero Debit to fake out IsFromMe() tx.vin.resize(1); } - CWalletTx* wtx = wallet.AddToWallet(MakeTransactionRef(std::move(tx)), /* confirm= */ {}); + uint256 txid = tx.GetHash(); + + LOCK(wallet.cs_wallet); + auto ret = wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(txid), std::forward_as_tuple(MakeTransactionRef(std::move(tx)))); + assert(ret.second); + CWalletTx& wtx = (*ret.first).second; if (fIsFromMe) { - wtx->m_amounts[CWalletTx::DEBIT].Set(ISMINE_SPENDABLE, 1); - wtx->m_is_cache_empty = false; + wtx.m_amounts[CWalletTx::DEBIT].Set(ISMINE_SPENDABLE, 1); + wtx.m_is_cache_empty = false; } - COutput output(wallet, *wtx, nInput, nAge, true /* spendable */, true /* solvable */, true /* safe */); - vCoins.push_back(output); -} -static void add_coin(const CAmount& nValue, int nAge = 6*24, bool fIsFromMe = false, int nInput=0, bool spendable = false) -{ - add_coin(testWallet, nValue, nAge, fIsFromMe, nInput, spendable); -} - -static void empty_wallet(void) -{ - vCoins.clear(); - balance = 0; + COutput output(wallet, wtx, nInput, nAge, true /* spendable */, true /* solvable */, true /* safe */); + coins.push_back(output); } static bool equal_sets(CoinSet a, CoinSet b) @@ -142,20 +126,20 @@ inline std::vector<OutputGroup>& GroupCoins(const std::vector<COutput>& coins) return static_groups; } -inline std::vector<OutputGroup>& KnapsackGroupOutputs(const CoinEligibilityFilter& filter) +inline std::vector<OutputGroup>& KnapsackGroupOutputs(const std::vector<COutput>& coins, CWallet& wallet, const CoinEligibilityFilter& filter) { + CoinSelectionParams coin_selection_params(/* change_output_size= */ 0, + /* change_spend_size= */ 0, /* effective_feerate= */ CFeeRate(0), + /* long_term_feerate= */ CFeeRate(0), /* discard_feerate= */ CFeeRate(0), + /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); static std::vector<OutputGroup> static_groups; - static_groups = GroupOutputs(testWallet, vCoins, coin_selection_params, filter, /* positive_only */false); + static_groups = GroupOutputs(wallet, coins, coin_selection_params, filter, /* positive_only */false); return static_groups; } // Branch and bound coin selection tests BOOST_AUTO_TEST_CASE(bnb_search_test) { - - LOCK(testWallet.cs_wallet); - testWallet.SetupLegacyScriptPubKeyMan(); - // Setup std::vector<CInputCoin> utxo_pool; CoinSet selection; @@ -288,196 +272,213 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) /* change_spend_size= */ 0, /* effective_feerate= */ CFeeRate(3000), /* long_term_feerate= */ CFeeRate(1000), /* discard_feerate= */ CFeeRate(1000), /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); - CoinSet setCoinsRet; - CAmount nValueRet; - empty_wallet(); - add_coin(1); - vCoins.at(0).nInputBytes = 40; // Make sure that it has a negative effective value. The next check should assert if this somehow got through. Otherwise it will fail - BOOST_CHECK(!SelectCoinsBnB(GroupCoins(vCoins), 1 * CENT, coin_selection_params_bnb.m_cost_of_change, setCoinsRet, nValueRet)); - - // Test fees subtracted from output: - empty_wallet(); - add_coin(1 * CENT); - vCoins.at(0).nInputBytes = 40; - coin_selection_params_bnb.m_subtract_fee_outputs = true; - BOOST_CHECK(SelectCoinsBnB(GroupCoins(vCoins), 1 * CENT, coin_selection_params_bnb.m_cost_of_change, setCoinsRet, nValueRet)); - BOOST_CHECK_EQUAL(nValueRet, 1 * CENT); - - // Make sure that can use BnB when there are preset inputs - empty_wallet(); { std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", CreateMockWalletDatabase()); wallet->LoadWallet(); - wallet->SetupLegacyScriptPubKeyMan(); LOCK(wallet->cs_wallet); - add_coin(*wallet, 5 * CENT, 6 * 24, false, 0, true); - add_coin(*wallet, 3 * CENT, 6 * 24, false, 0, true); - add_coin(*wallet, 2 * CENT, 6 * 24, false, 0, true); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetupDescriptorScriptPubKeyMans(); + + std::vector<COutput> coins; + CoinSet setCoinsRet; + CAmount nValueRet; + + add_coin(coins, *wallet, 1); + coins.at(0).nInputBytes = 40; // Make sure that it has a negative effective value. The next check should assert if this somehow got through. Otherwise it will fail + BOOST_CHECK(!SelectCoinsBnB(GroupCoins(coins), 1 * CENT, coin_selection_params_bnb.m_cost_of_change, setCoinsRet, nValueRet)); + + // Test fees subtracted from output: + coins.clear(); + add_coin(coins, *wallet, 1 * CENT); + coins.at(0).nInputBytes = 40; + coin_selection_params_bnb.m_subtract_fee_outputs = true; + BOOST_CHECK(SelectCoinsBnB(GroupCoins(coins), 1 * CENT, coin_selection_params_bnb.m_cost_of_change, setCoinsRet, nValueRet)); + BOOST_CHECK_EQUAL(nValueRet, 1 * CENT); + } + + { + std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", CreateMockWalletDatabase()); + wallet->LoadWallet(); + LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetupDescriptorScriptPubKeyMans(); + + std::vector<COutput> coins; + CoinSet setCoinsRet; + CAmount nValueRet; + + add_coin(coins, *wallet, 5 * CENT, 6 * 24, false, 0, true); + add_coin(coins, *wallet, 3 * CENT, 6 * 24, false, 0, true); + add_coin(coins, *wallet, 2 * CENT, 6 * 24, false, 0, true); CCoinControl coin_control; coin_control.fAllowOtherInputs = true; - coin_control.Select(COutPoint(vCoins.at(0).tx->GetHash(), vCoins.at(0).i)); + coin_control.Select(COutPoint(coins.at(0).tx->GetHash(), coins.at(0).i)); coin_selection_params_bnb.m_effective_feerate = CFeeRate(0); - BOOST_CHECK(SelectCoins(*wallet, vCoins, 10 * CENT, setCoinsRet, nValueRet, coin_control, coin_selection_params_bnb)); + BOOST_CHECK(SelectCoins(*wallet, coins, 10 * CENT, setCoinsRet, nValueRet, coin_control, coin_selection_params_bnb)); } } BOOST_AUTO_TEST_CASE(knapsack_solver_test) { + std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", CreateMockWalletDatabase()); + wallet->LoadWallet(); + LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetupDescriptorScriptPubKeyMans(); + CoinSet setCoinsRet, setCoinsRet2; CAmount nValueRet; - - LOCK(testWallet.cs_wallet); - testWallet.SetupLegacyScriptPubKeyMan(); + std::vector<COutput> coins; // test multiple times to allow for differences in the shuffle order for (int i = 0; i < RUN_TESTS; i++) { - empty_wallet(); + coins.clear(); // with an empty wallet we can't even pay one cent - BOOST_CHECK(!KnapsackSolver(1 * CENT, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); + BOOST_CHECK(!KnapsackSolver(1 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_standard), setCoinsRet, nValueRet)); - add_coin(1*CENT, 4); // add a new 1 cent coin + add_coin(coins, *wallet, 1*CENT, 4); // add a new 1 cent coin // with a new 1 cent coin, we still can't find a mature 1 cent - BOOST_CHECK(!KnapsackSolver(1 * CENT, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); + BOOST_CHECK(!KnapsackSolver(1 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_standard), setCoinsRet, nValueRet)); // but we can find a new 1 cent - BOOST_CHECK(KnapsackSolver(1 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(1 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * CENT); - add_coin(2*CENT); // add a mature 2 cent coin + add_coin(coins, *wallet, 2*CENT); // add a mature 2 cent coin // we can't make 3 cents of mature coins - BOOST_CHECK(!KnapsackSolver(3 * CENT, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); + BOOST_CHECK(!KnapsackSolver(3 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_standard), setCoinsRet, nValueRet)); // we can make 3 cents of new coins - BOOST_CHECK(KnapsackSolver(3 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(3 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 3 * CENT); - add_coin(5*CENT); // add a mature 5 cent coin, - add_coin(10*CENT, 3, true); // a new 10 cent coin sent from one of our own addresses - add_coin(20*CENT); // and a mature 20 cent coin + add_coin(coins, *wallet, 5*CENT); // add a mature 5 cent coin, + add_coin(coins, *wallet, 10*CENT, 3, true); // a new 10 cent coin sent from one of our own addresses + add_coin(coins, *wallet, 20*CENT); // and a mature 20 cent coin // now we have new: 1+10=11 (of which 10 was self-sent), and mature: 2+5+20=27. total = 38 // we can't make 38 cents only if we disallow new coins: - BOOST_CHECK(!KnapsackSolver(38 * CENT, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); + BOOST_CHECK(!KnapsackSolver(38 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_standard), setCoinsRet, nValueRet)); // we can't even make 37 cents if we don't allow new coins even if they're from us - BOOST_CHECK(!KnapsackSolver(38 * CENT, KnapsackGroupOutputs(filter_standard_extra), setCoinsRet, nValueRet)); + BOOST_CHECK(!KnapsackSolver(38 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_standard_extra), setCoinsRet, nValueRet)); // but we can make 37 cents if we accept new coins from ourself - BOOST_CHECK(KnapsackSolver(37 * CENT, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(37 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_standard), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 37 * CENT); // and we can make 38 cents if we accept all new coins - BOOST_CHECK(KnapsackSolver(38 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(38 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 38 * CENT); // try making 34 cents from 1,2,5,10,20 - we can't do it exactly - BOOST_CHECK(KnapsackSolver(34 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(34 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 35 * CENT); // but 35 cents is closest BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // the best should be 20+10+5. it's incredibly unlikely the 1 or 2 got included (but possible) // when we try making 7 cents, the smaller coins (1,2,5) are enough. We should see just 2+5 - BOOST_CHECK(KnapsackSolver(7 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(7 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 7 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); // when we try making 8 cents, the smaller coins (1,2,5) are exactly enough. - BOOST_CHECK(KnapsackSolver(8 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(8 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK(nValueRet == 8 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // when we try making 9 cents, no subset of smaller coins is enough, and we get the next bigger coin (10) - BOOST_CHECK(KnapsackSolver(9 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(9 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 10 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); // now clear out the wallet and start again to test choosing between subsets of smaller coins and the next biggest coin - empty_wallet(); + coins.clear(); - add_coin( 6*CENT); - add_coin( 7*CENT); - add_coin( 8*CENT); - add_coin(20*CENT); - add_coin(30*CENT); // now we have 6+7+8+20+30 = 71 cents total + add_coin(coins, *wallet, 6*CENT); + add_coin(coins, *wallet, 7*CENT); + add_coin(coins, *wallet, 8*CENT); + add_coin(coins, *wallet, 20*CENT); + add_coin(coins, *wallet, 30*CENT); // now we have 6+7+8+20+30 = 71 cents total // check that we have 71 and not 72 - BOOST_CHECK(KnapsackSolver(71 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); - BOOST_CHECK(!KnapsackSolver(72 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(71 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(!KnapsackSolver(72 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); // now try making 16 cents. the best smaller coins can do is 6+7+8 = 21; not as good at the next biggest coin, 20 - BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 20 * CENT); // we should get 20 in one coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); - add_coin( 5*CENT); // now we have 5+6+7+8+20+30 = 75 cents total + add_coin(coins, *wallet, 5*CENT); // now we have 5+6+7+8+20+30 = 75 cents total // now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, better than the next biggest coin, 20 - BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 18 * CENT); // we should get 18 in 3 coins BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); - add_coin( 18*CENT); // now we have 5+6+7+8+18+20+30 + add_coin(coins, *wallet, 18*CENT); // now we have 5+6+7+8+18+20+30 // and now if we try making 16 cents again, the smaller coins can make 5+6+7 = 18 cents, the same as the next biggest coin, 18 - BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(16 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 18 * CENT); // we should get 18 in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); // because in the event of a tie, the biggest coin wins // now try making 11 cents. we should get 5+6 - BOOST_CHECK(KnapsackSolver(11 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(11 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 11 * CENT); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); // check that the smallest bigger coin is used - add_coin( 1*COIN); - add_coin( 2*COIN); - add_coin( 3*COIN); - add_coin( 4*COIN); // now we have 5+6+7+8+18+20+30+100+200+300+400 = 1094 cents - BOOST_CHECK(KnapsackSolver(95 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + add_coin(coins, *wallet, 1*COIN); + add_coin(coins, *wallet, 2*COIN); + add_coin(coins, *wallet, 3*COIN); + add_coin(coins, *wallet, 4*COIN); // now we have 5+6+7+8+18+20+30+100+200+300+400 = 1094 cents + BOOST_CHECK(KnapsackSolver(95 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * COIN); // we should get 1 BTC in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); - BOOST_CHECK(KnapsackSolver(195 * CENT, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(195 * CENT, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 2 * COIN); // we should get 2 BTC in 1 coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); // empty the wallet and start again, now with fractions of a cent, to test small change avoidance - empty_wallet(); - add_coin(MIN_CHANGE * 1 / 10); - add_coin(MIN_CHANGE * 2 / 10); - add_coin(MIN_CHANGE * 3 / 10); - add_coin(MIN_CHANGE * 4 / 10); - add_coin(MIN_CHANGE * 5 / 10); + coins.clear(); + add_coin(coins, *wallet, MIN_CHANGE * 1 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 2 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 3 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 4 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 5 / 10); // try making 1 * MIN_CHANGE from the 1.5 * MIN_CHANGE // we'll get change smaller than MIN_CHANGE whatever happens, so can expect MIN_CHANGE exactly - BOOST_CHECK(KnapsackSolver(MIN_CHANGE, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(MIN_CHANGE, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE); // but if we add a bigger coin, small change is avoided - add_coin(1111*MIN_CHANGE); + add_coin(coins, *wallet, 1111*MIN_CHANGE); // try making 1 from 0.1 + 0.2 + 0.3 + 0.4 + 0.5 + 1111 = 1112.5 - BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * MIN_CHANGE); // we should get the exact amount // if we add more small coins: - add_coin(MIN_CHANGE * 6 / 10); - add_coin(MIN_CHANGE * 7 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 6 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 7 / 10); // and try again to make 1.0 * MIN_CHANGE - BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1 * MIN_CHANGE); // we should get the exact amount // run the 'mtgox' test (see https://blockexplorer.com/tx/29a3efd3ef04f9153d47a990bd7b048a4b2d213daaa5fb8ed670fb85f13bdbcf) // they tried to consolidate 10 50k coins into one 500k coin, and ended up with 50k in change - empty_wallet(); + coins.clear(); for (int j = 0; j < 20; j++) - add_coin(50000 * COIN); + add_coin(coins, *wallet, 50000 * COIN); - BOOST_CHECK(KnapsackSolver(500000 * COIN, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(500000 * COIN, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 500000 * COIN); // we should get the exact amount BOOST_CHECK_EQUAL(setCoinsRet.size(), 10U); // in ten coins @@ -485,79 +486,79 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // we need to try finding an exact subset anyway // sometimes it will fail, and so we use the next biggest coin: - empty_wallet(); - add_coin(MIN_CHANGE * 5 / 10); - add_coin(MIN_CHANGE * 6 / 10); - add_coin(MIN_CHANGE * 7 / 10); - add_coin(1111 * MIN_CHANGE); - BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + coins.clear(); + add_coin(coins, *wallet, MIN_CHANGE * 5 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 6 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 7 / 10); + add_coin(coins, *wallet, 1111 * MIN_CHANGE); + BOOST_CHECK(KnapsackSolver(1 * MIN_CHANGE, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1111 * MIN_CHANGE); // we get the bigger coin BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); // but sometimes it's possible, and we use an exact subset (0.4 + 0.6 = 1.0) - empty_wallet(); - add_coin(MIN_CHANGE * 4 / 10); - add_coin(MIN_CHANGE * 6 / 10); - add_coin(MIN_CHANGE * 8 / 10); - add_coin(1111 * MIN_CHANGE); - BOOST_CHECK(KnapsackSolver(MIN_CHANGE, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + coins.clear(); + add_coin(coins, *wallet, MIN_CHANGE * 4 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 6 / 10); + add_coin(coins, *wallet, MIN_CHANGE * 8 / 10); + add_coin(coins, *wallet, 1111 * MIN_CHANGE); + BOOST_CHECK(KnapsackSolver(MIN_CHANGE, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE); // we should get the exact amount BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); // in two coins 0.4+0.6 // test avoiding small change - empty_wallet(); - add_coin(MIN_CHANGE * 5 / 100); - add_coin(MIN_CHANGE * 1); - add_coin(MIN_CHANGE * 100); + coins.clear(); + add_coin(coins, *wallet, MIN_CHANGE * 5 / 100); + add_coin(coins, *wallet, MIN_CHANGE * 1); + add_coin(coins, *wallet, MIN_CHANGE * 100); // trying to make 100.01 from these three coins - BOOST_CHECK(KnapsackSolver(MIN_CHANGE * 10001 / 100, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(MIN_CHANGE * 10001 / 100, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, MIN_CHANGE * 10105 / 100); // we should get all coins BOOST_CHECK_EQUAL(setCoinsRet.size(), 3U); // but if we try to make 99.9, we should take the bigger of the two small coins to avoid small change - BOOST_CHECK(KnapsackSolver(MIN_CHANGE * 9990 / 100, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(MIN_CHANGE * 9990 / 100, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 101 * MIN_CHANGE); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); - } - - // test with many inputs - for (CAmount amt=1500; amt < COIN; amt*=10) { - empty_wallet(); - // Create 676 inputs (= (old MAX_STANDARD_TX_SIZE == 100000) / 148 bytes per input) - for (uint16_t j = 0; j < 676; j++) - add_coin(amt); - - // We only create the wallet once to save time, but we still run the coin selection RUN_TESTS times. - for (int i = 0; i < RUN_TESTS; i++) { - BOOST_CHECK(KnapsackSolver(2000, KnapsackGroupOutputs(filter_confirmed), setCoinsRet, nValueRet)); - - if (amt - 2000 < MIN_CHANGE) { - // needs more than one input: - uint16_t returnSize = std::ceil((2000.0 + MIN_CHANGE)/amt); - CAmount returnValue = amt * returnSize; - BOOST_CHECK_EQUAL(nValueRet, returnValue); - BOOST_CHECK_EQUAL(setCoinsRet.size(), returnSize); - } else { - // one input is sufficient: - BOOST_CHECK_EQUAL(nValueRet, amt); - BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); - } - } - } - - // test randomness - { - empty_wallet(); - for (int i2 = 0; i2 < 100; i2++) - add_coin(COIN); - - // Again, we only create the wallet once to save time, but we still run the coin selection RUN_TESTS times. - for (int i = 0; i < RUN_TESTS; i++) { + } + + // test with many inputs + for (CAmount amt=1500; amt < COIN; amt*=10) { + coins.clear(); + // Create 676 inputs (= (old MAX_STANDARD_TX_SIZE == 100000) / 148 bytes per input) + for (uint16_t j = 0; j < 676; j++) + add_coin(coins, *wallet, amt); + + // We only create the wallet once to save time, but we still run the coin selection RUN_TESTS times. + for (int i = 0; i < RUN_TESTS; i++) { + BOOST_CHECK(KnapsackSolver(2000, KnapsackGroupOutputs(coins, *wallet, filter_confirmed), setCoinsRet, nValueRet)); + + if (amt - 2000 < MIN_CHANGE) { + // needs more than one input: + uint16_t returnSize = std::ceil((2000.0 + MIN_CHANGE)/amt); + CAmount returnValue = amt * returnSize; + BOOST_CHECK_EQUAL(nValueRet, returnValue); + BOOST_CHECK_EQUAL(setCoinsRet.size(), returnSize); + } else { + // one input is sufficient: + BOOST_CHECK_EQUAL(nValueRet, amt); + BOOST_CHECK_EQUAL(setCoinsRet.size(), 1U); + } + } + } + + // test randomness + { + coins.clear(); + for (int i2 = 0; i2 < 100; i2++) + add_coin(coins, *wallet, COIN); + + // Again, we only create the wallet once to save time, but we still run the coin selection RUN_TESTS times. + for (int i = 0; i < RUN_TESTS; i++) { // picking 50 from 100 coins doesn't depend on the shuffle, // but does depend on randomness in the stochastic approximation code - BOOST_CHECK(KnapsackSolver(50 * COIN, GroupCoins(vCoins), setCoinsRet, nValueRet)); - BOOST_CHECK(KnapsackSolver(50 * COIN, GroupCoins(vCoins), setCoinsRet2, nValueRet)); + BOOST_CHECK(KnapsackSolver(50 * COIN, GroupCoins(coins), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(50 * COIN, GroupCoins(coins), setCoinsRet2, nValueRet)); BOOST_CHECK(!equal_sets(setCoinsRet, setCoinsRet2)); int fails = 0; @@ -567,66 +568,67 @@ BOOST_AUTO_TEST_CASE(knapsack_solver_test) // When choosing 1 from 100 identical coins, 1% of the time, this test will choose the same coin twice // which will cause it to fail. // To avoid that issue, run the test RANDOM_REPEATS times and only complain if all of them fail - BOOST_CHECK(KnapsackSolver(COIN, GroupCoins(vCoins), setCoinsRet, nValueRet)); - BOOST_CHECK(KnapsackSolver(COIN, GroupCoins(vCoins), setCoinsRet2, nValueRet)); + BOOST_CHECK(KnapsackSolver(COIN, GroupCoins(coins), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(COIN, GroupCoins(coins), setCoinsRet2, nValueRet)); if (equal_sets(setCoinsRet, setCoinsRet2)) fails++; } BOOST_CHECK_NE(fails, RANDOM_REPEATS); - } - - // add 75 cents in small change. not enough to make 90 cents, - // then try making 90 cents. there are multiple competing "smallest bigger" coins, - // one of which should be picked at random - add_coin(5 * CENT); - add_coin(10 * CENT); - add_coin(15 * CENT); - add_coin(20 * CENT); - add_coin(25 * CENT); - - for (int i = 0; i < RUN_TESTS; i++) { + } + + // add 75 cents in small change. not enough to make 90 cents, + // then try making 90 cents. there are multiple competing "smallest bigger" coins, + // one of which should be picked at random + add_coin(coins, *wallet, 5 * CENT); + add_coin(coins, *wallet, 10 * CENT); + add_coin(coins, *wallet, 15 * CENT); + add_coin(coins, *wallet, 20 * CENT); + add_coin(coins, *wallet, 25 * CENT); + + for (int i = 0; i < RUN_TESTS; i++) { int fails = 0; for (int j = 0; j < RANDOM_REPEATS; j++) { - BOOST_CHECK(KnapsackSolver(90*CENT, GroupCoins(vCoins), setCoinsRet, nValueRet)); - BOOST_CHECK(KnapsackSolver(90*CENT, GroupCoins(vCoins), setCoinsRet2, nValueRet)); + BOOST_CHECK(KnapsackSolver(90*CENT, GroupCoins(coins), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(90*CENT, GroupCoins(coins), setCoinsRet2, nValueRet)); if (equal_sets(setCoinsRet, setCoinsRet2)) fails++; } BOOST_CHECK_NE(fails, RANDOM_REPEATS); - } - } - - empty_wallet(); + } + } } BOOST_AUTO_TEST_CASE(ApproximateBestSubset) { + std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", CreateMockWalletDatabase()); + wallet->LoadWallet(); + LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetupDescriptorScriptPubKeyMans(); + CoinSet setCoinsRet; CAmount nValueRet; - - LOCK(testWallet.cs_wallet); - testWallet.SetupLegacyScriptPubKeyMan(); - - empty_wallet(); + std::vector<COutput> coins; // Test vValue sort order for (int i = 0; i < 1000; i++) - add_coin(1000 * COIN); - add_coin(3 * COIN); + add_coin(coins, *wallet, 1000 * COIN); + add_coin(coins, *wallet, 3 * COIN); - BOOST_CHECK(KnapsackSolver(1003 * COIN, KnapsackGroupOutputs(filter_standard), setCoinsRet, nValueRet)); + BOOST_CHECK(KnapsackSolver(1003 * COIN, KnapsackGroupOutputs(coins, *wallet, filter_standard), setCoinsRet, nValueRet)); BOOST_CHECK_EQUAL(nValueRet, 1003 * COIN); BOOST_CHECK_EQUAL(setCoinsRet.size(), 2U); - - empty_wallet(); } // Tests that with the ideal conditions, the coin selector will always be able to find a solution that can pay the target value BOOST_AUTO_TEST_CASE(SelectCoins_test) { - LOCK(testWallet.cs_wallet); - testWallet.SetupLegacyScriptPubKeyMan(); + std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", CreateMockWalletDatabase()); + wallet->LoadWallet(); + LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetupDescriptorScriptPubKeyMans(); // Random generator stuff std::default_random_engine generator; @@ -636,12 +638,15 @@ BOOST_AUTO_TEST_CASE(SelectCoins_test) // Run this test 100 times for (int i = 0; i < 100; ++i) { - empty_wallet(); + std::vector<COutput> coins; + CAmount balance{0}; // Make a wallet with 1000 exponentially distributed random inputs for (int j = 0; j < 1000; ++j) { - add_coin((CAmount)(distribution(generator)*10000000)); + CAmount val = distribution(generator)*10000000; + add_coin(coins, *wallet, val); + balance += val; } // Generate a random fee rate in the range of 100 - 400 @@ -658,7 +663,7 @@ BOOST_AUTO_TEST_CASE(SelectCoins_test) CoinSet out_set; CAmount out_value = 0; CCoinControl cc; - BOOST_CHECK(SelectCoins(testWallet, vCoins, target, out_set, out_value, cc, cs_params)); + BOOST_CHECK(SelectCoins(*wallet, coins, target, out_set, out_value, cc, cs_params)); BOOST_CHECK_GE(out_value, target); } } diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index 16cb7e0baf..dba3f35025 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -16,7 +16,7 @@ BOOST_FIXTURE_TEST_SUITE(db_tests, BasicTestingSetup) static std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& path, std::string& database_filename) { fs::path data_file = BDBDataFile(path); - database_filename = data_file.filename().string(); + database_filename = fs::PathToString(data_file.filename()); return GetBerkeleyEnv(data_file.parent_path()); } @@ -25,11 +25,7 @@ BOOST_AUTO_TEST_CASE(getwalletenv_file) std::string test_name = "test_name.dat"; const fs::path datadir = gArgs.GetDataDirNet(); fs::path file_path = datadir / test_name; -#if BOOST_VERSION >= 107700 - std::ofstream f(BOOST_FILESYSTEM_C_STR(file_path)); -#else - std::ofstream f(file_path.BOOST_FILESYSTEM_C_STR); -#endif // BOOST_VERSION >= 107700 + fs::ofstream f(file_path); f.close(); std::string filename; diff --git a/src/wallet/test/init_test_fixture.cpp b/src/wallet/test/init_test_fixture.cpp index 53c972c46d..170675c035 100644 --- a/src/wallet/test/init_test_fixture.cpp +++ b/src/wallet/test/init_test_fixture.cpp @@ -32,11 +32,7 @@ InitWalletDirTestingSetup::InitWalletDirTestingSetup(const std::string& chainNam fs::create_directories(m_walletdir_path_cases["default"]); fs::create_directories(m_walletdir_path_cases["custom"]); fs::create_directories(m_walletdir_path_cases["relative"]); -#if BOOST_VERSION >= 107700 - std::ofstream f(BOOST_FILESYSTEM_C_STR(m_walletdir_path_cases["file"])); -#else - std::ofstream f(m_walletdir_path_cases["file"].BOOST_FILESYSTEM_C_STR); -#endif // BOOST_VERSION >= 107700 + fs::ofstream f(m_walletdir_path_cases["file"]); f.close(); } @@ -50,5 +46,5 @@ InitWalletDirTestingSetup::~InitWalletDirTestingSetup() void InitWalletDirTestingSetup::SetWalletDir(const fs::path& walletdir_path) { - gArgs.ForceSetArg("-walletdir", walletdir_path.string()); + gArgs.ForceSetArg("-walletdir", fs::PathToString(walletdir_path)); } diff --git a/src/wallet/test/init_tests.cpp b/src/wallet/test/init_tests.cpp index 45e1b8c4b8..222c2bf4b7 100644 --- a/src/wallet/test/init_tests.cpp +++ b/src/wallet/test/init_tests.cpp @@ -17,7 +17,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_default) SetWalletDir(m_walletdir_path_cases["default"]); bool result = m_wallet_client->verify(); BOOST_CHECK(result == true); - fs::path walletdir = gArgs.GetArg("-walletdir", ""); + fs::path walletdir = fs::PathFromString(gArgs.GetArg("-walletdir", "")); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); BOOST_CHECK_EQUAL(walletdir, expected_path); } @@ -27,7 +27,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_custom) SetWalletDir(m_walletdir_path_cases["custom"]); bool result = m_wallet_client->verify(); BOOST_CHECK(result == true); - fs::path walletdir = gArgs.GetArg("-walletdir", ""); + fs::path walletdir = fs::PathFromString(gArgs.GetArg("-walletdir", "")); fs::path expected_path = fs::canonical(m_walletdir_path_cases["custom"]); BOOST_CHECK_EQUAL(walletdir, expected_path); } @@ -67,7 +67,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing) SetWalletDir(m_walletdir_path_cases["trailing"]); bool result = m_wallet_client->verify(); BOOST_CHECK(result == true); - fs::path walletdir = gArgs.GetArg("-walletdir", ""); + fs::path walletdir = fs::PathFromString(gArgs.GetArg("-walletdir", "")); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); BOOST_CHECK_EQUAL(walletdir, expected_path); } @@ -77,7 +77,7 @@ BOOST_AUTO_TEST_CASE(walletinit_verify_walletdir_no_trailing2) SetWalletDir(m_walletdir_path_cases["trailing2"]); bool result = m_wallet_client->verify(); BOOST_CHECK(result == true); - fs::path walletdir = gArgs.GetArg("-walletdir", ""); + fs::path walletdir = fs::PathFromString(gArgs.GetArg("-walletdir", "")); fs::path expected_path = fs::canonical(m_walletdir_path_cases["default"]); BOOST_CHECK_EQUAL(walletdir, expected_path); } diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 8a97f7779d..120a20749e 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -13,10 +13,21 @@ BOOST_FIXTURE_TEST_SUITE(psbt_wallet_tests, WalletTestingSetup) +static void import_descriptor(CWallet& wallet, const std::string& descriptor) +{ + LOCK(wallet.cs_wallet); + FlatSigningProvider provider; + std::string error; + std::unique_ptr<Descriptor> desc = Parse(descriptor, provider, error, /* require_checksum=*/ false); + assert(desc); + WalletDescriptor w_desc(std::move(desc), 0, 0, 10, 0); + wallet.AddWalletDescriptor(w_desc, provider, "", false); +} + BOOST_AUTO_TEST_CASE(psbt_updater_test) { - auto spk_man = m_wallet.GetOrCreateLegacyScriptPubKeyMan(); - LOCK2(m_wallet.cs_wallet, spk_man->cs_KeyStore); + LOCK(m_wallet.cs_wallet); + m_wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); // Create prevtxs and add to wallet CDataStream s_prev_tx1(ParseHex("0200000000010158e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd7501000000171600145f275f436b09a8cc9a2eb2a2f528485c68a56323feffffff02d8231f1b0100000017a914aed962d6654f9a2b36608eb9d64d2b260db4f1118700c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88702483045022100a22edcc6e5bc511af4cc4ae0de0fcd75c7e04d8c1c3a8aa9d820ed4b967384ec02200642963597b9b1bc22c75e9f3e117284a962188bf5e8a74c895089046a20ad770121035509a48eb623e10aace8bfd0212fdb8a8e5af3c94b0b133b95e114cab89e4f7965000000"), SER_NETWORK, PROTOCOL_VERSION); @@ -29,27 +40,10 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) s_prev_tx2 >> prev_tx2; m_wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(prev_tx2->GetHash()), std::forward_as_tuple(prev_tx2)); - // Add scripts - CScript rs1; - CDataStream s_rs1(ParseHex("475221029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f2102dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d752ae"), SER_NETWORK, PROTOCOL_VERSION); - s_rs1 >> rs1; - spk_man->AddCScript(rs1); - - CScript rs2; - CDataStream s_rs2(ParseHex("2200208c2353173743b595dfb4a07b72ba8e42e3797da74e87fe7d9d7497e3b2028903"), SER_NETWORK, PROTOCOL_VERSION); - s_rs2 >> rs2; - spk_man->AddCScript(rs2); - - CScript ws1; - CDataStream s_ws1(ParseHex("47522103089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc21023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7352ae"), SER_NETWORK, PROTOCOL_VERSION); - s_ws1 >> ws1; - spk_man->AddCScript(ws1); - - // Add hd seed - CKey key = DecodeSecret("5KSSJQ7UNfFGwVgpCZDSHm5rVNhMFcFtvWM3zQ8mW4qNDEN7LFd"); // Mainnet and uncompressed form of cUkG8i1RFfWGWy5ziR11zJ5V4U4W3viSFCfyJmZnvQaUsd1xuF3T - CPubKey master_pub_key = spk_man->DeriveNewSeed(key); - spk_man->SetHDSeed(master_pub_key); - spk_man->NewKeyPool(); + // Import descriptors for keys and scripts + import_descriptor(m_wallet, "sh(multi(2,xprv9s21ZrQH143K2LE7W4Xf3jATf9jECxSb7wj91ZnmY4qEJrS66Qru9RFqq8xbkgT32ya6HqYJweFdJUEDf5Q6JFV7jMiUws7kQfe6Tv4RbfN/0h/0h/0h,xprv9s21ZrQH143K2LE7W4Xf3jATf9jECxSb7wj91ZnmY4qEJrS66Qru9RFqq8xbkgT32ya6HqYJweFdJUEDf5Q6JFV7jMiUws7kQfe6Tv4RbfN/0h/0h/1h))"); + import_descriptor(m_wallet, "sh(wsh(multi(2,xprv9s21ZrQH143K2LE7W4Xf3jATf9jECxSb7wj91ZnmY4qEJrS66Qru9RFqq8xbkgT32ya6HqYJweFdJUEDf5Q6JFV7jMiUws7kQfe6Tv4RbfN/0h/0h/2h,xprv9s21ZrQH143K2LE7W4Xf3jATf9jECxSb7wj91ZnmY4qEJrS66Qru9RFqq8xbkgT32ya6HqYJweFdJUEDf5Q6JFV7jMiUws7kQfe6Tv4RbfN/0h/0h/3h)))"); + import_descriptor(m_wallet, "wpkh(xprv9s21ZrQH143K2LE7W4Xf3jATf9jECxSb7wj91ZnmY4qEJrS66Qru9RFqq8xbkgT32ya6HqYJweFdJUEDf5Q6JFV7jMiUws7kQfe6Tv4RbfN/0h/0h/*h)"); // Call FillPSBT PartiallySignedTransaction psbtx; @@ -71,7 +65,8 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) // Try to sign the mutated input SignatureData sigdata; - BOOST_CHECK(spk_man->FillPSBT(psbtx, PrecomputePSBTData(psbtx), SIGHASH_ALL, true, true) != TransactionError::OK); + BOOST_CHECK(m_wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, true, true) != TransactionError::OK); + //BOOST_CHECK(spk_man->FillPSBT(psbtx, PrecomputePSBTData(psbtx), SIGHASH_ALL, true, true) != TransactionError::OK); } BOOST_AUTO_TEST_CASE(parse_hd_keypath) diff --git a/src/wallet/test/spend_tests.cpp b/src/wallet/test/spend_tests.cpp index e779b2450f..d88d8eabdb 100644 --- a/src/wallet/test/spend_tests.cpp +++ b/src/wallet/test/spend_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <consensus/amount.h> #include <policy/fees.h> #include <validation.h> #include <wallet/coincontrol.h> @@ -32,6 +33,8 @@ BOOST_FIXTURE_TEST_CASE(SubtractFee, TestChain100Setup) CCoinControl coin_control; coin_control.m_feerate.emplace(10000); coin_control.fOverrideFeeRate = true; + // We need to use a change type with high cost of change so that the leftover amount will be dropped to fee instead of added as a change output + coin_control.m_change_type = OutputType::LEGACY; FeeCalculation fee_calc; BOOST_CHECK(CreateTransaction(*wallet, {recipient}, tx, fee, change_pos, error, coin_control, fee_calc)); BOOST_CHECK_EQUAL(tx->vout.size(), 1); diff --git a/src/wallet/test/util.cpp b/src/wallet/test/util.cpp index c3061b93c0..2990fc8f8d 100644 --- a/src/wallet/test/util.cpp +++ b/src/wallet/test/util.cpp @@ -6,6 +6,7 @@ #include <chain.h> #include <key.h> +#include <key_io.h> #include <test/util/setup_common.h> #include <wallet/wallet.h> #include <wallet/walletdb.h> @@ -23,9 +24,16 @@ std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cc } wallet->LoadWallet(); { - auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); - LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); - spk_man->AddKeyPubKey(key, key.GetPubKey()); + LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetupDescriptorScriptPubKeyMans(); + + FlatSigningProvider provider; + std::string error; + std::unique_ptr<Descriptor> desc = Parse("combo(" + EncodeSecret(key) + ")", provider, error, /* require_checksum=*/ false); + assert(desc); + WalletDescriptor w_desc(std::move(desc), 0, 0, 1, 1); + if (!wallet->AddWalletDescriptor(w_desc, provider, "", false)) assert(false); } WalletRescanReserver reserver(*wallet); reserver.reserve(); diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 9938380369..0965128ade 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -11,6 +11,7 @@ #include <vector> #include <interfaces/chain.h> +#include <key_io.h> #include <node/blockstorage.h> #include <node/context.h> #include <policy/policy.h> @@ -43,6 +44,7 @@ BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) static std::shared_ptr<CWallet> TestLoadWallet(WalletContext& context) { DatabaseOptions options; + options.create_flags = WALLET_FLAG_DESCRIPTORS; DatabaseStatus status; bilingual_str error; std::vector<bilingual_str> warnings; @@ -77,9 +79,13 @@ static CMutableTransaction TestSimpleSpend(const CTransaction& from, uint32_t in static void AddKey(CWallet& wallet, const CKey& key) { - auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); - LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); - spk_man->AddKeyPubKey(key, key.GetPubKey()); + LOCK(wallet.cs_wallet); + FlatSigningProvider provider; + std::string error; + std::unique_ptr<Descriptor> desc = Parse("combo(" + EncodeSecret(key) + ")", provider, error, /* require_checksum=*/ false); + assert(desc); + WalletDescriptor w_desc(std::move(desc), 0, 0, 1, 1); + if (!wallet.AddWalletDescriptor(w_desc, provider, "", false)) assert(false); } BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) @@ -95,6 +101,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet.SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); @@ -114,6 +121,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet.SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); @@ -140,6 +148,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet.SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); @@ -165,6 +174,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); { LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet.SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); } AddKey(wallet, coinbaseKey); @@ -258,7 +268,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) SetMockTime(KEY_TIME); m_coinbase_txns.emplace_back(CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); - std::string backup_file = (gArgs.GetDataDirNet() / "wallet.backup").string(); + std::string backup_file = fs::PathToString(gArgs.GetDataDirNet() / "wallet.backup"); // Import key into wallet and call dumpwallet to create backup file. { @@ -320,10 +330,12 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) { CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); - auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); CWalletTx wtx(m_coinbase_txns.back()); - LOCK2(wallet.cs_wallet, spk_man->cs_KeyStore); + LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet.SetupDescriptorScriptPubKeyMans(); + wallet.SetLastBlockProcessed(m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash()); CWalletTx::Confirmation confirm(CWalletTx::Status::CONFIRMED, m_node.chainman->ActiveChain().Height(), m_node.chainman->ActiveChain().Tip()->GetBlockHash(), 0); @@ -336,7 +348,7 @@ BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) // Invalidate the cached value, add the key, and make sure a new immature // credit amount is calculated. wtx.MarkDirty(); - BOOST_CHECK(spk_man->AddKeyPubKey(coinbaseKey, coinbaseKey.GetPubKey())); + AddKey(wallet, coinbaseKey); BOOST_CHECK_EQUAL(CachedTxGetImmatureCredit(wallet, wtx), 50*COIN); } @@ -593,14 +605,26 @@ BOOST_FIXTURE_TEST_CASE(ListCoinsTest, ListCoinsTestingSetup) BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) { - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); - wallet->SetupLegacyScriptPubKeyMan(); - wallet->SetMinVersion(FEATURE_LATEST); - wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); - BOOST_CHECK(!wallet->TopUpKeyPool(1000)); - CTxDestination dest; - bilingual_str error; - BOOST_CHECK(!wallet->GetNewDestination(OutputType::BECH32, "", dest, error)); + { + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); + wallet->SetupLegacyScriptPubKeyMan(); + wallet->SetMinVersion(FEATURE_LATEST); + wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); + BOOST_CHECK(!wallet->TopUpKeyPool(1000)); + CTxDestination dest; + bilingual_str error; + BOOST_CHECK(!wallet->GetNewDestination(OutputType::BECH32, "", dest, error)); + } + { + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); + LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetMinVersion(FEATURE_LATEST); + wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); + CTxDestination dest; + bilingual_str error; + BOOST_CHECK(!wallet->GetNewDestination(OutputType::BECH32, "", dest, error)); + } } // Explicit calculation which is used to test the wallet constant diff --git a/src/wallet/transaction.h b/src/wallet/transaction.h index 0cd91b9ebe..6fc1bd1eed 100644 --- a/src/wallet/transaction.h +++ b/src/wallet/transaction.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_WALLET_TRANSACTION_H #define BITCOIN_WALLET_TRANSACTION_H -#include <amount.h> +#include <consensus/amount.h> #include <primitives/transaction.h> #include <serialize.h> #include <wallet/ismine.h> @@ -216,9 +216,9 @@ public: } const auto it_op = mapValue.find("n"); - nOrderPos = (it_op != mapValue.end()) ? atoi64(it_op->second) : -1; + nOrderPos = (it_op != mapValue.end()) ? LocaleIndependentAtoi<int64_t>(it_op->second) : -1; const auto it_ts = mapValue.find("timesmart"); - nTimeSmart = (it_ts != mapValue.end()) ? static_cast<unsigned int>(atoi64(it_ts->second)) : 0; + nTimeSmart = (it_ts != mapValue.end()) ? static_cast<unsigned int>(LocaleIndependentAtoi<int64_t>(it_ts->second)) : 0; mapValue.erase("fromaccount"); mapValue.erase("spent"); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 1c18cdb1bc..803e88cda2 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -6,6 +6,7 @@ #include <wallet/wallet.h> #include <chain.h> +#include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/validation.h> #include <external_signer.h> @@ -1293,7 +1294,7 @@ void CWallet::updatedBlockTip() void CWallet::BlockUntilSyncedToCurrentChain() const { AssertLockNotHeld(cs_wallet); // Skip the queue-draining stuff if we know we're caught up with - // ::ChainActive().Tip(), otherwise put a callback in the validation interface queue and wait + // chain().Tip(), otherwise put a callback in the validation interface queue and wait // for the queue to drain enough to execute it (indicating we are caught up // at least with the time we entered this function). uint256 last_block_hash = WITH_LOCK(cs_wallet, return m_last_block_processed); @@ -1448,19 +1449,13 @@ bool CWallet::AddWalletFlags(uint64_t flags) // Helper for producing a max-sized low-S low-R signature (eg 71 bytes) // or a max-sized low-S signature (e.g. 72 bytes) if use_max_sig is true -bool CWallet::DummySignInput(CTxIn &tx_in, const CTxOut &txout, bool use_max_sig) const +bool DummySignInput(const SigningProvider& provider, CTxIn &tx_in, const CTxOut &txout, bool use_max_sig) { // Fill in dummy signatures for fee calculation. const CScript& scriptPubKey = txout.scriptPubKey; SignatureData sigdata; - std::unique_ptr<SigningProvider> provider = GetSolvingProvider(scriptPubKey); - if (!provider) { - // We don't know about this scriptpbuKey; - return false; - } - - if (!ProduceSignature(*provider, use_max_sig ? DUMMY_MAXIMUM_SIGNATURE_CREATOR : DUMMY_SIGNATURE_CREATOR, scriptPubKey, sigdata)) { + if (!ProduceSignature(provider, use_max_sig ? DUMMY_MAXIMUM_SIGNATURE_CREATOR : DUMMY_SIGNATURE_CREATOR, scriptPubKey, sigdata)) { return false; } UpdateInput(tx_in, sigdata); @@ -1468,14 +1463,21 @@ bool CWallet::DummySignInput(CTxIn &tx_in, const CTxOut &txout, bool use_max_sig } // Helper for producing a bunch of max-sized low-S low-R signatures (eg 71 bytes) -bool CWallet::DummySignTx(CMutableTransaction &txNew, const std::vector<CTxOut> &txouts, bool use_max_sig) const +bool CWallet::DummySignTx(CMutableTransaction &txNew, const std::vector<CTxOut> &txouts, const CCoinControl* coin_control) const { // Fill in dummy signatures for fee calculation. int nIn = 0; for (const auto& txout : txouts) { - if (!DummySignInput(txNew.vin[nIn], txout, use_max_sig)) { - return false; + CTxIn& txin = txNew.vin[nIn]; + // Use max sig if watch only inputs were used or if this particular input is an external input + // to ensure a sufficient fee is attained for the requested feerate. + const bool use_max_sig = coin_control && (coin_control->fAllowWatchOnly || coin_control->IsExternalSelected(txin.prevout)); + const std::unique_ptr<SigningProvider> provider = GetSolvingProvider(txout.scriptPubKey); + if (!provider || !DummySignInput(*provider, txin, txout, use_max_sig)) { + if (!coin_control || !DummySignInput(coin_control->m_external_provider, txin, txout, use_max_sig)) { + return false; + } } nIn++; @@ -2513,16 +2515,16 @@ std::unique_ptr<WalletDatabase> MakeWalletDatabase(const std::string& name, cons // 2. Path to an existing directory. // 3. Path to a symlink to a directory. // 4. For backwards compatibility, the name of a data file in -walletdir. - const fs::path wallet_path = fsbridge::AbsPathJoin(GetWalletDir(), name); + const fs::path wallet_path = fsbridge::AbsPathJoin(GetWalletDir(), fs::PathFromString(name)); fs::file_type path_type = fs::symlink_status(wallet_path).type(); if (!(path_type == fs::file_not_found || path_type == fs::directory_file || (path_type == fs::symlink_file && fs::is_directory(wallet_path)) || - (path_type == fs::regular_file && fs::path(name).filename() == name))) { + (path_type == fs::regular_file && fs::PathFromString(name).filename() == fs::PathFromString(name)))) { error_string = Untranslated(strprintf( "Invalid -wallet path '%s'. -wallet path should point to a directory where wallet.dat and " "database/log.?????????? files can be stored, a location where such a directory could be created, " "or (for backwards compatibility) the name of an existing data file in -walletdir (%s)", - name, GetWalletDir())); + name, fs::quoted(fs::PathToString(GetWalletDir())))); status = DatabaseStatus::FAILED_BAD_PATH; return nullptr; } @@ -3233,7 +3235,8 @@ void CWallet::LoadActiveScriptPubKeyMan(uint256 id, OutputType type, bool intern auto spk_man = m_spk_managers.at(id).get(); spk_mans[type] = spk_man; - if (spk_mans_other[type] == spk_man) { + const auto it = spk_mans_other.find(type); + if (it != spk_mans_other.end() && it->second == spk_man) { spk_mans_other.erase(type); } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index cedccf7d44..767b24adbb 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -6,7 +6,7 @@ #ifndef BITCOIN_WALLET_WALLET_H #define BITCOIN_WALLET_WALLET_H -#include <amount.h> +#include <consensus/amount.h> #include <interfaces/chain.h> #include <interfaces/handler.h> #include <outputtype.h> @@ -576,14 +576,13 @@ public: /** Pass this transaction to node for mempool insertion and relay to peers if flag set to true */ bool SubmitTxMemoryPoolAndRelay(const CWalletTx& wtx, std::string& err_string, bool relay) const; - bool DummySignTx(CMutableTransaction &txNew, const std::set<CTxOut> &txouts, bool use_max_sig = false) const + bool DummySignTx(CMutableTransaction &txNew, const std::set<CTxOut> &txouts, const CCoinControl* coin_control = nullptr) const { std::vector<CTxOut> v_txouts(txouts.size()); std::copy(txouts.begin(), txouts.end(), v_txouts.begin()); - return DummySignTx(txNew, v_txouts, use_max_sig); + return DummySignTx(txNew, v_txouts, coin_control); } - bool DummySignTx(CMutableTransaction &txNew, const std::vector<CTxOut> &txouts, bool use_max_sig = false) const; - bool DummySignInput(CTxIn &tx_in, const CTxOut &txout, bool use_max_sig = false) const; + bool DummySignTx(CMutableTransaction &txNew, const std::vector<CTxOut> &txouts, const CCoinControl* coin_control = nullptr) const; bool ImportScripts(const std::set<CScript> scripts, int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool ImportPrivKeys(const std::map<CKeyID, CKey>& privkey_map, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -928,4 +927,6 @@ bool AddWalletSetting(interfaces::Chain& chain, const std::string& wallet_name); //! Remove wallet name from persistent configuration so it will not be loaded on startup. bool RemoveWalletSetting(interfaces::Chain& chain, const std::string& wallet_name); +bool DummySignInput(const SigningProvider& provider, CTxIn &tx_in, const CTxOut &txout, bool use_max_sig); + #endif // BITCOIN_WALLET_WALLET_H diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 8ff09a0878..c920d4af51 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -1106,7 +1106,7 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas try { exists = fs::symlink_status(path).type() != fs::file_not_found; } catch (const fs::filesystem_error& e) { - error = Untranslated(strprintf("Failed to access database path '%s': %s", path.string(), fsbridge::get_filesystem_error_message(e))); + error = Untranslated(strprintf("Failed to access database path '%s': %s", fs::PathToString(path), fsbridge::get_filesystem_error_message(e))); status = DatabaseStatus::FAILED_BAD_PATH; return nullptr; } @@ -1118,33 +1118,33 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas } if (IsSQLiteFile(SQLiteDataFile(path))) { if (format) { - error = Untranslated(strprintf("Failed to load database path '%s'. Data is in ambiguous format.", path.string())); + error = Untranslated(strprintf("Failed to load database path '%s'. Data is in ambiguous format.", fs::PathToString(path))); status = DatabaseStatus::FAILED_BAD_FORMAT; return nullptr; } format = DatabaseFormat::SQLITE; } } else if (options.require_existing) { - error = Untranslated(strprintf("Failed to load database path '%s'. Path does not exist.", path.string())); + error = Untranslated(strprintf("Failed to load database path '%s'. Path does not exist.", fs::PathToString(path))); status = DatabaseStatus::FAILED_NOT_FOUND; return nullptr; } if (!format && options.require_existing) { - error = Untranslated(strprintf("Failed to load database path '%s'. Data is not in recognized format.", path.string())); + error = Untranslated(strprintf("Failed to load database path '%s'. Data is not in recognized format.", fs::PathToString(path))); status = DatabaseStatus::FAILED_BAD_FORMAT; return nullptr; } if (format && options.require_create) { - error = Untranslated(strprintf("Failed to create database path '%s'. Database already exists.", path.string())); + error = Untranslated(strprintf("Failed to create database path '%s'. Database already exists.", fs::PathToString(path))); status = DatabaseStatus::FAILED_ALREADY_EXISTS; return nullptr; } // A db already exists so format is set, but options also specifies the format, so make sure they agree if (format && options.require_format && format != options.require_format) { - error = Untranslated(strprintf("Failed to load database path '%s'. Data is not in required format.", path.string())); + error = Untranslated(strprintf("Failed to load database path '%s'. Data is not in required format.", fs::PathToString(path))); status = DatabaseStatus::FAILED_BAD_FORMAT; return nullptr; } @@ -1166,7 +1166,7 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas #ifdef USE_SQLITE return MakeSQLiteDatabase(path, options, status, error); #endif - error = Untranslated(strprintf("Failed to open database path '%s'. Build does not support SQLite database format.", path.string())); + error = Untranslated(strprintf("Failed to open database path '%s'. Build does not support SQLite database format.", fs::PathToString(path))); status = DatabaseStatus::FAILED_BAD_FORMAT; return nullptr; } @@ -1174,7 +1174,7 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas #ifdef USE_BDB return MakeBerkeleyDatabase(path, options, status, error); #endif - error = Untranslated(strprintf("Failed to open database path '%s'. Build does not support Berkeley DB database format.", path.string())); + error = Untranslated(strprintf("Failed to open database path '%s'. Build does not support Berkeley DB database format.", fs::PathToString(path))); status = DatabaseStatus::FAILED_BAD_FORMAT; return nullptr; } @@ -1188,9 +1188,9 @@ std::unique_ptr<WalletDatabase> CreateDummyWalletDatabase() /** Return object for accessing temporary in-memory database. */ std::unique_ptr<WalletDatabase> CreateMockWalletDatabase() { -#ifdef USE_BDB - return std::make_unique<BerkeleyDatabase>(std::make_shared<BerkeleyEnvironment>(), ""); -#elif USE_SQLITE +#ifdef USE_SQLITE return std::make_unique<SQLiteDatabase>("", "", true); +#elif USE_BDB + return std::make_unique<BerkeleyDatabase>(std::make_shared<BerkeleyEnvironment>(), ""); #endif } diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 715d9012ed..9c752623b3 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -6,7 +6,6 @@ #ifndef BITCOIN_WALLET_WALLETDB_H #define BITCOIN_WALLET_WALLETDB_H -#include <amount.h> #include <script/sign.h> #include <wallet/db.h> #include <wallet/walletutil.h> diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index e3cb5cee5d..88f0a2ce20 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -120,17 +120,33 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) tfm::format(std::cerr, "The -descriptors option can only be used with the 'create' command.\n"); return false; } + if (args.IsArgSet("-legacy") && command != "create") { + tfm::format(std::cerr, "The -legacy option can only be used with the 'create' command.\n"); + return false; + } if (command == "create" && !args.IsArgSet("-wallet")) { tfm::format(std::cerr, "Wallet name must be provided when creating a new wallet.\n"); return false; } const std::string name = args.GetArg("-wallet", ""); - const fs::path path = fsbridge::AbsPathJoin(GetWalletDir(), name); + const fs::path path = fsbridge::AbsPathJoin(GetWalletDir(), fs::PathFromString(name)); if (command == "create") { DatabaseOptions options; options.require_create = true; - if (args.GetBoolArg("-descriptors", false)) { + // If -legacy is set, use it. Otherwise default to false. + bool make_legacy = args.GetBoolArg("-legacy", false); + // If neither -legacy nor -descriptors is set, default to true. If -descriptors is set, use its value. + bool make_descriptors = (!args.IsArgSet("-descriptors") && !args.IsArgSet("-legacy")) || (args.IsArgSet("-descriptors") && args.GetBoolArg("-descriptors", true)); + if (make_legacy && make_descriptors) { + tfm::format(std::cerr, "Only one of -legacy or -descriptors can be set to true, not both\n"); + return false; + } + if (!make_legacy && !make_descriptors) { + tfm::format(std::cerr, "One of -legacy or -descriptors must be set to true (or omitted)\n"); + return false; + } + if (make_descriptors) { options.create_flags |= WALLET_FLAG_DESCRIPTORS; options.require_format = DatabaseFormat::SQLITE; } diff --git a/src/wallet/walletutil.cpp b/src/wallet/walletutil.cpp index 1c518daba6..7f813432b3 100644 --- a/src/wallet/walletutil.cpp +++ b/src/wallet/walletutil.cpp @@ -12,7 +12,7 @@ fs::path GetWalletDir() fs::path path; if (gArgs.IsArgSet("-walletdir")) { - path = gArgs.GetArg("-walletdir", ""); + path = fs::PathFromString(gArgs.GetArg("-walletdir", "")); if (!fs::is_directory(path)) { // If the path specified doesn't exist, we return the deliberately // invalid empty string. diff --git a/test/README.md b/test/README.md index acd68d8d8f..c9e15c4968 100644 --- a/test/README.md +++ b/test/README.md @@ -275,12 +275,15 @@ Use the `-v` option for verbose output. #### Dependencies -| Lint test | Dependency | Version [used by CI](../ci/lint/04_install.sh) | Installation -|-----------|:----------:|:-------------------------------------------:|-------------- -| [`lint-python.sh`](lint/lint-python.sh) | [flake8](https://gitlab.com/pycqa/flake8) | [3.8.3](https://github.com/bitcoin/bitcoin/pull/19348) | `pip3 install flake8==3.8.3` -| [`lint-python.sh`](lint/lint-python.sh) | [mypy](https://github.com/python/mypy) | [0.781](https://github.com/bitcoin/bitcoin/pull/19348) | `pip3 install mypy==0.781` -| [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck) | [0.7.2](https://github.com/bitcoin/bitcoin/pull/21749) | [details...](https://github.com/koalaman/shellcheck#installing) -| [`lint-spelling.sh`](lint/lint-spelling.sh) | [codespell](https://github.com/codespell-project/codespell) | [2.0.0](https://github.com/bitcoin/bitcoin/pull/20817) | `pip3 install codespell==2.0.0` +| Lint test | Dependency | +|-----------|:----------:| +| [`lint-python.sh`](lint/lint-python.sh) | [flake8](https://gitlab.com/pycqa/flake8) +| [`lint-python.sh`](lint/lint-python.sh) | [mypy](https://github.com/python/mypy) +| [`lint-python.sh`](lint/lint-python.sh) | [pyzmq](https://github.com/zeromq/pyzmq) +| [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck) +| [`lint-spelling.sh`](lint/lint-spelling.sh) | [codespell](https://github.com/codespell-project/codespell) + +In use versions and install instructions are available in the [CI setup](../ci/lint/04_install.sh). Please be aware that on Linux distributions all dependencies are usually available as packages, but could be outdated. diff --git a/test/config.ini.in b/test/config.ini.in index db80bba6f1..8bcba1b39c 100644 --- a/test/config.ini.in +++ b/test/config.ini.in @@ -24,3 +24,4 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py @ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true @ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true @ENABLE_EXTERNAL_SIGNER_TRUE@ENABLE_EXTERNAL_SIGNER=true +@ENABLE_SYSCALL_SANDBOX_TRUE@ENABLE_SYSCALL_SANDBOX=true diff --git a/test/functional/combine_logs.py b/test/functional/combine_logs.py index 00f2833f55..71dfb4c01a 100755 --- a/test/functional/combine_logs.py +++ b/test/functional/combine_logs.py @@ -188,7 +188,7 @@ def print_logs_plain(log_events, colors): def print_logs_html(log_events): """Renders the iterator of log events into html.""" try: - import jinja2 + import jinja2 #type:ignore except ImportError: print("jinja2 not found. Try `pip install jinja2`") sys.exit(1) diff --git a/test/functional/data/__init__.py b/test/functional/data/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/test/functional/data/__init__.py diff --git a/test/functional/feature_addrman.py b/test/functional/feature_addrman.py index 5a8394db2e..93d50c1369 100755 --- a/test/functional/feature_addrman.py +++ b/test/functional/feature_addrman.py @@ -109,7 +109,7 @@ class AddrmanTest(BitcoinTestFramework): self.stop_node(0) write_addrman(peers_dat, len_tried=-1) self.nodes[0].assert_start_raises_init_error( - expected_msg=init_error("Corrupt CAddrMan serialization: nTried=-1, should be in \\[0, 16384\\]:.*"), + expected_msg=init_error("Corrupt AddrMan serialization: nTried=-1, should be in \\[0, 16384\\]:.*"), match=ErrorMatch.FULL_REGEX, ) @@ -117,7 +117,7 @@ class AddrmanTest(BitcoinTestFramework): self.stop_node(0) write_addrman(peers_dat, len_new=-1) self.nodes[0].assert_start_raises_init_error( - expected_msg=init_error("Corrupt CAddrMan serialization: nNew=-1, should be in \\[0, 65536\\]:.*"), + expected_msg=init_error("Corrupt AddrMan serialization: nNew=-1, should be in \\[0, 65536\\]:.*"), match=ErrorMatch.FULL_REGEX, ) diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index ee2c71cd42..99ac1b5884 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -24,7 +24,6 @@ from test_framework.util import ( assert_equal, assert_greater_than, assert_raises_rpc_error, - satoshi_round, softfork_active, ) from test_framework.script_util import DUMMY_P2WPKH_SCRIPT @@ -94,7 +93,7 @@ class BIP68Test(BitcoinTestFramework): utxo = utxos[0] tx1 = CTransaction() - value = int(satoshi_round(utxo["amount"] - self.relayfee)*COIN) + value = int((utxo["amount"] - self.relayfee) * COIN) # Check that the disable flag disables relative locktime. # If sequence locks were used, this would require 1 block for the diff --git a/test/functional/feature_fee_estimation.py b/test/functional/feature_fee_estimation.py index 9c225dc687..ac00db8ff0 100755 --- a/test/functional/feature_fee_estimation.py +++ b/test/functional/feature_fee_estimation.py @@ -4,6 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test fee estimation code.""" from decimal import Decimal +import os import random from test_framework.messages import ( @@ -155,6 +156,21 @@ def check_estimates(node, fees_seen): check_raw_estimates(node, fees_seen) check_smart_estimates(node, fees_seen) + +def send_tx(node, utxo, feerate): + """Broadcast a 1in-1out transaction with a specific input and feerate (sat/vb).""" + overhead, op, scriptsig, nseq, value, spk = 10, 36, 5, 4, 8, 24 + tx_size = overhead + op + scriptsig + nseq + value + spk + fee = tx_size * feerate + + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(int(utxo["txid"], 16), utxo["vout"]), SCRIPT_SIG[utxo["vout"]])] + tx.vout = [CTxOut(int(utxo["amount"] * COIN) - fee, P2SH_1)] + txid = node.sendrawtransaction(tx.serialize().hex()) + + return txid + + class EstimateFeeTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 3 @@ -212,20 +228,16 @@ class EstimateFeeTest(BitcoinTestFramework): newmem.append(utx) self.memutxo = newmem - def run_test(self): - self.log.info("This test is time consuming, please be patient") - self.log.info("Splitting inputs so we can generate tx's") - - # Start node0 - self.start_node(0) + def initial_split(self, node): + """Split two coinbase UTxOs into many small coins""" self.txouts = [] self.txouts2 = [] # Split a coinbase into two transaction puzzle outputs - split_inputs(self.nodes[0], self.nodes[0].listunspent(0), self.txouts, True) + split_inputs(node, node.listunspent(0), self.txouts, True) # Mine - while len(self.nodes[0].getrawmempool()) > 0: - self.generate(self.nodes[0], 1) + while len(node.getrawmempool()) > 0: + self.generate(node, 1) # Repeatedly split those 2 outputs, doubling twice for each rep # Use txouts to monitor the available utxo, since these won't be tracked in wallet @@ -233,27 +245,19 @@ class EstimateFeeTest(BitcoinTestFramework): while reps < 5: # Double txouts to txouts2 while len(self.txouts) > 0: - split_inputs(self.nodes[0], self.txouts, self.txouts2) - while len(self.nodes[0].getrawmempool()) > 0: - self.generate(self.nodes[0], 1) + split_inputs(node, self.txouts, self.txouts2) + while len(node.getrawmempool()) > 0: + self.generate(node, 1) # Double txouts2 to txouts while len(self.txouts2) > 0: - split_inputs(self.nodes[0], self.txouts2, self.txouts) - while len(self.nodes[0].getrawmempool()) > 0: - self.generate(self.nodes[0], 1) + split_inputs(node, self.txouts2, self.txouts) + while len(node.getrawmempool()) > 0: + self.generate(node, 1) reps += 1 - self.log.info("Finished splitting") - - # Now we can connect the other nodes, didn't want to connect them earlier - # so the estimates would not be affected by the splitting transactions - self.start_node(1) - self.start_node(2) - self.connect_nodes(1, 0) - self.connect_nodes(0, 2) - self.connect_nodes(2, 1) - - self.sync_all() + def sanity_check_estimates_range(self): + """Populate estimation buckets, assert estimates are in a sane range and + are strictly increasing as the target decreases.""" self.fees_per_kb = [] self.memutxo = [] self.confutxo = self.txouts # Start with the set of confirmed txouts after splitting @@ -279,11 +283,100 @@ class EstimateFeeTest(BitcoinTestFramework): self.log.info("Final estimates after emptying mempools") check_estimates(self.nodes[1], self.fees_per_kb) - # check that the effective feerate is greater than or equal to the mempoolminfee even for high mempoolminfee - self.log.info("Test fee rate estimation after restarting node with high MempoolMinFee") + def test_feerate_mempoolminfee(self): high_val = 3*self.nodes[1].estimatesmartfee(1)['feerate'] self.restart_node(1, extra_args=[f'-minrelaytxfee={high_val}']) check_estimates(self.nodes[1], self.fees_per_kb) + self.restart_node(1) + + def sanity_check_rbf_estimates(self, utxos): + """During 5 blocks, broadcast low fee transactions. Only 10% of them get + confirmed and the remaining ones get RBF'd with a high fee transaction at + the next block. + The block policy estimator should return the high feerate. + """ + # The broadcaster and block producer + node = self.nodes[0] + miner = self.nodes[1] + # In sat/vb + low_feerate = 1 + high_feerate = 10 + # Cache the utxos of which to replace the spender after it failed to get + # confirmed + utxos_to_respend = [] + txids_to_replace = [] + + assert len(utxos) >= 250 + for _ in range(5): + # Broadcast 45 low fee transactions that will need to be RBF'd + for _ in range(45): + u = utxos.pop(0) + txid = send_tx(node, u, low_feerate) + utxos_to_respend.append(u) + txids_to_replace.append(txid) + # Broadcast 5 low fee transaction which don't need to + for _ in range(5): + send_tx(node, utxos.pop(0), low_feerate) + # Mine the transactions on another node + self.sync_mempools(wait=.1, nodes=[node, miner]) + for txid in txids_to_replace: + miner.prioritisetransaction(txid=txid, fee_delta=-COIN) + self.generate(miner, 1) + self.sync_blocks(wait=.1, nodes=[node, miner]) + # RBF the low-fee transactions + while True: + try: + u = utxos_to_respend.pop(0) + send_tx(node, u, high_feerate) + except IndexError: + break + + # Mine the last replacement txs + self.sync_mempools(wait=.1, nodes=[node, miner]) + self.generate(miner, 1) + self.sync_blocks(wait=.1, nodes=[node, miner]) + + # Only 10% of the transactions were really confirmed with a low feerate, + # the rest needed to be RBF'd. We must return the 90% conf rate feerate. + high_feerate_kvb = Decimal(high_feerate) / COIN * 10**3 + est_feerate = node.estimatesmartfee(2)["feerate"] + assert est_feerate == high_feerate_kvb + + def run_test(self): + self.log.info("This test is time consuming, please be patient") + self.log.info("Splitting inputs so we can generate tx's") + + # Split two coinbases into many small utxos + self.start_node(0) + self.initial_split(self.nodes[0]) + self.log.info("Finished splitting") + + # Now we can connect the other nodes, didn't want to connect them earlier + # so the estimates would not be affected by the splitting transactions + self.start_node(1) + self.start_node(2) + self.connect_nodes(1, 0) + self.connect_nodes(0, 2) + self.connect_nodes(2, 1) + self.sync_all() + + self.log.info("Testing estimates with single transactions.") + self.sanity_check_estimates_range() + + # check that the effective feerate is greater than or equal to the mempoolminfee even for high mempoolminfee + self.log.info("Test fee rate estimation after restarting node with high MempoolMinFee") + self.test_feerate_mempoolminfee() + + self.log.info("Restarting node with fresh estimation") + self.stop_node(0) + fee_dat = os.path.join(self.nodes[0].datadir, self.chain, "fee_estimates.dat") + os.remove(fee_dat) + self.start_node(0) + self.connect_nodes(0, 1) + self.connect_nodes(0, 2) + + self.log.info("Testing estimates with RBF.") + self.sanity_check_rbf_estimates(self.confutxo + self.memutxo) self.log.info("Testing that fee estimation is disabled in blocksonly.") self.restart_node(0, ["-blocksonly"]) diff --git a/test/functional/feature_notifications.py b/test/functional/feature_notifications.py index 4382022a7a..2a507c75c4 100755 --- a/test/functional/feature_notifications.py +++ b/test/functional/feature_notifications.py @@ -27,6 +27,9 @@ class NotificationsTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 self.setup_clean_chain = True + # The experimental syscall sandbox feature (-sandbox) is not compatible with -alertnotify, + # -blocknotify or -walletnotify (which all invoke execve). + self.disable_syscall_sandbox = True def setup_network(self): self.wallet = ''.join(chr(i) for i in range(FILE_CHAR_START, FILE_CHAR_END) if chr(i) not in FILE_CHARS_DISALLOWED) diff --git a/test/functional/feature_rbf.py b/test/functional/feature_rbf.py index d759a5aab5..420147542e 100755 --- a/test/functional/feature_rbf.py +++ b/test/functional/feature_rbf.py @@ -26,7 +26,7 @@ from test_framework.script_util import ( DUMMY_2_P2WPKH_SCRIPT, ) from test_framework.wallet import MiniWallet - +from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE MAX_REPLACEMENT_LIMIT = 100 class ReplaceByFeeTest(BitcoinTestFramework): @@ -44,9 +44,6 @@ class ReplaceByFeeTest(BitcoinTestFramework): ] self.supports_cli = False - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def run_test(self): self.wallet = MiniWallet(self.nodes[0]) # the pre-mined test framework chain contains coinbase outputs to the @@ -531,9 +528,9 @@ class ReplaceByFeeTest(BitcoinTestFramework): assert tx2b_txid in self.nodes[0].getrawmempool() def test_rpc(self): - us0 = self.nodes[0].listunspent()[0] + us0 = self.wallet.get_utxo() ins = [us0] - outs = {self.nodes[0].getnewaddress(): Decimal(1.0000000)} + outs = {ADDRESS_BCRT1_UNSPENDABLE: Decimal(1.0000000)} rawtx0 = self.nodes[0].createrawtransaction(ins, outs, 0, True) rawtx1 = self.nodes[0].createrawtransaction(ins, outs, 0, False) json0 = self.nodes[0].decoderawtransaction(rawtx0) @@ -541,14 +538,16 @@ class ReplaceByFeeTest(BitcoinTestFramework): assert_equal(json0["vin"][0]["sequence"], 4294967293) assert_equal(json1["vin"][0]["sequence"], 4294967295) - rawtx2 = self.nodes[0].createrawtransaction([], outs) - frawtx2a = self.nodes[0].fundrawtransaction(rawtx2, {"replaceable": True}) - frawtx2b = self.nodes[0].fundrawtransaction(rawtx2, {"replaceable": False}) + if self.is_wallet_compiled(): + self.init_wallet(node=0) + rawtx2 = self.nodes[0].createrawtransaction([], outs) + frawtx2a = self.nodes[0].fundrawtransaction(rawtx2, {"replaceable": True}) + frawtx2b = self.nodes[0].fundrawtransaction(rawtx2, {"replaceable": False}) - json0 = self.nodes[0].decoderawtransaction(frawtx2a['hex']) - json1 = self.nodes[0].decoderawtransaction(frawtx2b['hex']) - assert_equal(json0["vin"][0]["sequence"], 4294967293) - assert_equal(json1["vin"][0]["sequence"], 4294967294) + json0 = self.nodes[0].decoderawtransaction(frawtx2a['hex']) + json1 = self.nodes[0].decoderawtransaction(frawtx2b['hex']) + assert_equal(json0["vin"][0]["sequence"], 4294967293) + assert_equal(json1["vin"][0]["sequence"], 4294967294) def test_no_inherited_signaling(self): confirmed_utxo = self.wallet.get_utxo() diff --git a/test/functional/feature_segwit.py b/test/functional/feature_segwit.py index 25d1cb2bf1..2f9ab34e99 100755 --- a/test/functional/feature_segwit.py +++ b/test/functional/feature_segwit.py @@ -31,11 +31,11 @@ from test_framework.script import ( OP_1, OP_2, OP_CHECKMULTISIG, - OP_CHECKSIG, OP_DROP, OP_TRUE, ) from test_framework.script_util import ( + key_to_p2pk_script, key_to_p2pkh_script, key_to_p2wpkh_script, script_to_p2sh_script, @@ -459,7 +459,7 @@ class SegWitTest(BitcoinTestFramework): importlist.append(script_to_p2wsh_script(bare).hex()) else: pubkey = bytes.fromhex(v['pubkey']) - p2pk = CScript([pubkey, OP_CHECKSIG]) + p2pk = key_to_p2pk_script(pubkey) p2pkh = key_to_p2pkh_script(pubkey) importlist.append(p2pk.hex()) importlist.append(p2pkh.hex()) @@ -628,7 +628,7 @@ class SegWitTest(BitcoinTestFramework): pubkey = bytes.fromhex(v['pubkey']) p2wpkh = key_to_p2wpkh_script(pubkey) p2sh_p2wpkh = script_to_p2sh_script(p2wpkh) - p2pk = CScript([pubkey, OP_CHECKSIG]) + p2pk = key_to_p2pk_script(pubkey) p2pkh = CScript(bytes.fromhex(v['scriptPubKey'])) p2sh_p2pk = script_to_p2sh_script(p2pk) p2sh_p2pkh = script_to_p2sh_script(p2pkh) diff --git a/test/functional/feature_syscall_sandbox.py b/test/functional/feature_syscall_sandbox.py new file mode 100755 index 0000000000..caf7f1e7fc --- /dev/null +++ b/test/functional/feature_syscall_sandbox.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test bitcoind aborts if a disallowed syscall is used when compiled with the syscall sandbox.""" + +from test_framework.test_framework import BitcoinTestFramework, SkipTest + + +class SyscallSandboxTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + if not self.is_syscall_sandbox_compiled(): + raise SkipTest("bitcoind has not been built with syscall sandbox enabled.") + if self.options.nosandbox: + raise SkipTest("--nosandbox passed to test runner.") + + def run_test(self): + disallowed_syscall_terminated_bitcoind = False + expected_log_entry = 'ERROR: The syscall "getgroups" (syscall number 115) is not allowed by the syscall sandbox' + with self.nodes[0].assert_debug_log([expected_log_entry]): + self.log.info("Invoking disallowed syscall") + try: + self.nodes[0].invokedisallowedsyscall() + except ConnectionError: + disallowed_syscall_terminated_bitcoind = True + assert disallowed_syscall_terminated_bitcoind + self.nodes = [] + + +if __name__ == "__main__": + SyscallSandboxTest().main() diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index c44a48f15f..50a25ee1ef 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -76,6 +76,7 @@ from test_framework.script import ( taproot_construct, ) from test_framework.script_util import ( + key_to_p2pk_script, key_to_p2wpkh_script, keyhash_to_p2pkh_script, script_to_p2sh_script, @@ -1109,7 +1110,7 @@ def spenders_taproot_active(): for witv0 in [False, True]: for hashtype in VALID_SIGHASHES_ECDSA + [random.randrange(0x04, 0x80), random.randrange(0x84, 0x100)]: standard = (hashtype in VALID_SIGHASHES_ECDSA) and (compressed or not witv0) - add_spender(spenders, "legacy/pk-wrongkey", hashtype=hashtype, p2sh=p2sh, witv0=witv0, standard=standard, script=CScript([pubkey1, OP_CHECKSIG]), **SINGLE_SIG, key=eckey1, failure={"key": eckey2}, sigops_weight=4-3*witv0, **ERR_NO_SUCCESS) + add_spender(spenders, "legacy/pk-wrongkey", hashtype=hashtype, p2sh=p2sh, witv0=witv0, standard=standard, script=key_to_p2pk_script(pubkey1), **SINGLE_SIG, key=eckey1, failure={"key": eckey2}, sigops_weight=4-3*witv0, **ERR_NO_SUCCESS) add_spender(spenders, "legacy/pkh-sighashflip", hashtype=hashtype, p2sh=p2sh, witv0=witv0, standard=standard, pkh=pubkey1, key=eckey1, **SIGHASH_BITFLIP, sigops_weight=4-3*witv0, **ERR_NO_SUCCESS) # Verify that OP_CHECKSIGADD wasn't accidentally added to pre-taproot validation logic. diff --git a/test/functional/feature_versionbits_warning.py b/test/functional/feature_versionbits_warning.py index 311d871d49..d74ef5e088 100755 --- a/test/functional/feature_versionbits_warning.py +++ b/test/functional/feature_versionbits_warning.py @@ -28,6 +28,9 @@ class VersionBitsWarningTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 + # The experimental syscall sandbox feature (-sandbox) is not compatible with -alertnotify + # (which invokes execve). + self.disable_syscall_sandbox = True def setup_network(self): self.alert_filename = os.path.join(self.options.tmpdir, "alert.txt") diff --git a/test/functional/interface_rest.py b/test/functional/interface_rest.py index e0716fc54a..868bb42604 100755 --- a/test/functional/interface_rest.py +++ b/test/functional/interface_rest.py @@ -279,6 +279,13 @@ class RESTTest (BitcoinTestFramework): json_obj = self.test_rest_request(f"/headers/5/{bb_hash}") assert_equal(len(json_obj), 5) # now we should have 5 header objects + # Test number parsing + for num in ['5a', '-5', '0', '2001', '99999999999999999999999999999999999']: + assert_equal( + bytes(f'Header count out of range: {num}\r\n', 'ascii'), + self.test_rest_request(f"/headers/{num}/{bb_hash}", ret_type=RetType.BYTES, status=400), + ) + self.log.info("Test tx inclusion in the /mempool and /block URIs") # Make 3 tx and mine them on node 1 @@ -311,6 +318,15 @@ class RESTTest (BitcoinTestFramework): if 'coinbase' not in tx['vin'][0]} assert_equal(non_coinbase_txs, set(txs)) + # Verify that the non-coinbase tx has "prevout" key set + for tx_obj in json_obj["tx"]: + for vin in tx_obj["vin"]: + if "coinbase" not in vin: + assert "prevout" in vin + assert_equal(vin["prevout"]["generated"], False) + else: + assert "prevout" not in vin + # Check the same but without tx details json_obj = self.test_rest_request(f"/block/notxdetails/{newblockhash[0]}") for tx in txs: diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 4313b05f88..5a11a62ec4 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -583,7 +583,7 @@ class ZMQTest (BitcoinTestFramework): ], ipv6=True) # Generate 1 block in nodes[0] - self.nodes[0].generatetoaddress(1, ADDRESS_BCRT1_UNSPENDABLE) + self.generatetoaddress(self.nodes[0], 1, ADDRESS_BCRT1_UNSPENDABLE) # Should receive the same block hash assert_equal(self.nodes[0].getbestblockhash(), subscribers[0].receive().hex()) diff --git a/test/functional/mempool_package_limits.py b/test/functional/mempool_package_limits.py index 2217628858..89a5c83826 100755 --- a/test/functional/mempool_package_limits.py +++ b/test/functional/mempool_package_limits.py @@ -244,7 +244,7 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): assert_equal(txres["package-error"], "package-mempool-limits") # Clear mempool and check that the package passes now - node.generate(1) + self.generate(node, 1) assert all([res["allowed"] for res in node.testmempoolaccept(rawtxs=package_hex)]) def test_anc_count_limits(self): diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index c042961937..ff5e45519f 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -14,7 +14,6 @@ from test_framework.util import ( assert_equal, assert_raises_rpc_error, chain_transaction, - satoshi_round, ) # default limits @@ -66,8 +65,7 @@ class MempoolPackagesTest(BitcoinTestFramework): value = sent_value chain.append(txid) # We need the wtxids to check P2P announcements - fulltx = self.nodes[0].getrawtransaction(txid) - witnesstx = self.nodes[0].decoderawtransaction(fulltx, True) + witnesstx = self.nodes[0].gettransaction(txid=txid, verbose=True)['decoded'] witness_chain.append(witnesstx['hash']) # Check that listunspent ancestor{count, size, fees} yield the correct results @@ -209,10 +207,10 @@ class MempoolPackagesTest(BitcoinTestFramework): entry = self.nodes[0].getmempoolentry(x) descendant_fees += entry['fee'] if (x == chain[-1]): - assert_equal(entry['modifiedfee'], entry['fee']+satoshi_round(0.00002)) - assert_equal(entry['fees']['modified'], entry['fee']+satoshi_round(0.00002)) + assert_equal(entry['modifiedfee'], entry['fee'] + Decimal("0.00002")) + assert_equal(entry['fees']['modified'], entry['fee'] + Decimal("0.00002")) assert_equal(entry['descendantfees'], descendant_fees * COIN + 2000) - assert_equal(entry['fees']['descendant'], descendant_fees+satoshi_round(0.00002)) + assert_equal(entry['fees']['descendant'], descendant_fees + Decimal("0.00002")) # Check that node1's mempool is as expected (-> custom ancestor limit) mempool0 = self.nodes[0].getrawmempool(False) @@ -308,7 +306,7 @@ class MempoolPackagesTest(BitcoinTestFramework): value = utxo[0]['amount'] vout = utxo[0]['vout'] - send_value = satoshi_round((value - fee)/2) + send_value = (value - fee) / 2 inputs = [ {'txid' : txid, 'vout' : vout} ] outputs = {} for _ in range(2): diff --git a/test/functional/mining_prioritisetransaction.py b/test/functional/mining_prioritisetransaction.py index da85ee54be..35274d3500 100755 --- a/test/functional/mining_prioritisetransaction.py +++ b/test/functional/mining_prioritisetransaction.py @@ -13,7 +13,7 @@ from test_framework.util import assert_equal, assert_raises_rpc_error, create_co class PrioritiseTransactionTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True - self.num_nodes = 2 + self.num_nodes = 1 self.extra_args = [[ "-printpriority=1", "-acceptnonstdtxn=1", diff --git a/test/functional/p2p_compactblocks_blocksonly.py b/test/functional/p2p_compactblocks_blocksonly.py index 4073ec03a6..5f01fa4dfe 100755 --- a/test/functional/p2p_compactblocks_blocksonly.py +++ b/test/functional/p2p_compactblocks_blocksonly.py @@ -33,7 +33,7 @@ class P2PCompactBlocksBlocksOnly(BitcoinTestFramework): self.sync_all() def build_block_on_tip(self): - blockhash = self.nodes[2].generate(1)[0] + blockhash = self.generate(self.nodes[2], 1)[0] block_hex = self.nodes[2].getblock(blockhash=blockhash, verbosity=0) block = from_hex(CBlock(), block_hex) block.rehash() diff --git a/test/functional/p2p_invalid_messages.py b/test/functional/p2p_invalid_messages.py index f3b80abb59..82c7e94c59 100755 --- a/test/functional/p2p_invalid_messages.py +++ b/test/functional/p2p_invalid_messages.py @@ -209,7 +209,7 @@ class InvalidMessagesTest(BitcoinTestFramework): self.test_addrv2('unrecognized network', [ 'received: addrv2 (25 bytes)', - 'IP 9.9.9.9 mapped', + '9.9.9.9:8333 mapped', 'Added 1 addresses', ], bytes.fromhex( diff --git a/test/functional/p2p_ping.py b/test/functional/p2p_ping.py index 888e986fba..d67e97acf7 100755 --- a/test/functional/p2p_ping.py +++ b/test/functional/p2p_ping.py @@ -30,11 +30,16 @@ class NodeNoPong(P2PInterface): pass +TIMEOUT_INTERVAL = 20 * 60 + + class PingPongTest(BitcoinTestFramework): def set_test_params(self): self.setup_clean_chain = True self.num_nodes = 1 - self.extra_args = [['-peertimeout=3']] + # Set the peer connection timeout low. It does not matter for this + # test, as long as it is less than TIMEOUT_INTERVAL. + self.extra_args = [['-peertimeout=1']] def check_peer_info(self, *, pingtime, minping, pingwait): stats = self.nodes[0].getpeerinfo()[0] @@ -110,8 +115,11 @@ class PingPongTest(BitcoinTestFramework): self.nodes[0].ping() no_pong_node.wait_until(lambda: 'ping' in no_pong_node.last_message) with self.nodes[0].assert_debug_log(['ping timeout: 1201.000000s']): - self.mock_forward(20 * 60 + 1) - time.sleep(4) # peertimeout + 1 + self.mock_forward(TIMEOUT_INTERVAL // 2) + # Check that sending a ping does not prevent the disconnect + no_pong_node.sync_with_ping() + self.mock_forward(TIMEOUT_INTERVAL // 2 + 1) + no_pong_node.wait_for_disconnect() if __name__ == '__main__': diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index aa3b95fc4f..5195c9fd04 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -8,7 +8,12 @@ import random import struct import time -from test_framework.blocktools import create_block, create_coinbase, add_witness_commitment, WITNESS_COMMITMENT_HEADER +from test_framework.blocktools import ( + WITNESS_COMMITMENT_HEADER, + add_witness_commitment, + create_block, + create_coinbase, +) from test_framework.key import ECKey from test_framework.messages import ( BIP125_SEQUENCE_NUMBER, @@ -72,6 +77,7 @@ from test_framework.script import ( hash160, ) from test_framework.script_util import ( + key_to_p2pk_script, key_to_p2wpkh_script, keyhash_to_p2pkh_script, script_to_p2sh_script, @@ -193,7 +199,7 @@ class SegWitTest(BitcoinTestFramework): self.num_nodes = 2 # This test tests SegWit both pre and post-activation, so use the normal BIP9 activation. self.extra_args = [ - ["-acceptnonstdtxn=1", f"-testactivationheight=segwit@{SEGWIT_HEIGHT}", "-whitelist=noban@127.0.0.1"], + ["-acceptnonstdtxn=1", f"-testactivationheight=segwit@{SEGWIT_HEIGHT}", "-whitelist=noban@127.0.0.1", "-par=1"], ["-acceptnonstdtxn=0", f"-testactivationheight=segwit@{SEGWIT_HEIGHT}"], ] self.supports_cli = False @@ -504,8 +510,8 @@ class SegWitTest(BitcoinTestFramework): # 'block-validation-failed' (if script check threads > 1) or # 'non-mandatory-script-verify-flag (Witness program was passed an # empty witness)' (otherwise). - # TODO: support multiple acceptable reject reasons. - test_witness_block(self.nodes[0], self.test_node, block, accepted=False, with_witness=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, with_witness=False, + reason='non-mandatory-script-verify-flag (Witness program was passed an empty witness)') self.utxo.pop(0) self.utxo.append(UTXO(txid, 2, value)) @@ -785,7 +791,7 @@ class SegWitTest(BitcoinTestFramework): block_3.rehash() block_3.solve() - test_witness_block(self.nodes[0], self.test_node, block_3, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block_3, accepted=False, reason='bad-witness-merkle-match') # Add a different commitment with different nonce, but in the # right location, and with some funds burned(!). @@ -851,7 +857,7 @@ class SegWitTest(BitcoinTestFramework): # Change the nonce -- should not cause the block to be permanently # failed block.vtx[0].wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(1)] - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, reason='bad-witness-merkle-match') # Changing the witness reserved value doesn't change the block hash block.vtx[0].wit.vtxinwit[0].scriptWitness.stack = [ser_uint256(0)] @@ -860,7 +866,7 @@ class SegWitTest(BitcoinTestFramework): @subtest # type: ignore def test_witness_block_size(self): # TODO: Test that non-witness carrying blocks can't exceed 1MB - # Skipping this test for now; this is covered in p2p-fullblocktest.py + # Skipping this test for now; this is covered in feature_block.py # Test that witness-bearing blocks are limited at ceil(base + wit/4) <= 1MB. block = self.build_next_block() @@ -916,7 +922,7 @@ class SegWitTest(BitcoinTestFramework): # limit assert len(block.serialize()) > 2 * 1024 * 1024 - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, reason='bad-blk-weight') # Now resize the second transaction to make the block fit. cur_length = len(block.vtx[-1].wit.vtxinwit[0].scriptWitness.stack[0]) @@ -988,7 +994,8 @@ class SegWitTest(BitcoinTestFramework): self.update_witness_block_with_transactions(block, [tx]) # Extra witness data should not be allowed. - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, + reason='non-mandatory-script-verify-flag (Witness provided for non-witness script)') # Try extra signature data. Ok if we're not spending a witness output. block.vtx[1].wit.vtxinwit = [] @@ -1013,7 +1020,8 @@ class SegWitTest(BitcoinTestFramework): self.update_witness_block_with_transactions(block, [tx2]) # This has extra witness data, so it should fail. - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, + reason='non-mandatory-script-verify-flag (Stack size must be exactly one after execution)') # Now get rid of the extra witness, but add extra scriptSig data tx2.vin[0].scriptSig = CScript([OP_TRUE]) @@ -1025,7 +1033,8 @@ class SegWitTest(BitcoinTestFramework): block.solve() # This has extra signature data for a witness input, so it should fail. - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, + reason='non-mandatory-script-verify-flag (Witness requires empty scriptSig)') # Now get rid of the extra scriptsig on the witness input, and verify # success (even with extra scriptsig data in the non-witness input) @@ -1063,7 +1072,8 @@ class SegWitTest(BitcoinTestFramework): tx2.rehash() self.update_witness_block_with_transactions(block, [tx, tx2]) - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, + reason='non-mandatory-script-verify-flag (Push value size limit exceeded)') # Now reduce the length of the stack element tx2.wit.vtxinwit[0].scriptWitness.stack[0] = b'a' * (MAX_SCRIPT_ELEMENT_SIZE) @@ -1103,7 +1113,8 @@ class SegWitTest(BitcoinTestFramework): self.update_witness_block_with_transactions(block, [tx, tx2]) - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, + reason='non-mandatory-script-verify-flag (Script is too big)') # Try again with one less byte in the witness script witness_script = CScript([b'a' * MAX_SCRIPT_ELEMENT_SIZE] * 19 + [OP_DROP] * 62 + [OP_TRUE]) @@ -1175,7 +1186,7 @@ class SegWitTest(BitcoinTestFramework): block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx2]) - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, reason='bad-txnmrklroot') # Now try using a too short vtxinwit tx2.wit.vtxinwit.pop() @@ -1183,6 +1194,8 @@ class SegWitTest(BitcoinTestFramework): block.vtx = [block.vtx[0]] self.update_witness_block_with_transactions(block, [tx2]) + # This block doesn't result in a specific reject reason, but an iostream exception: + # "Exception 'CDataStream::read(): end of data: unspecified iostream_category error' (...) caught" test_witness_block(self.nodes[0], self.test_node, block, accepted=False) # Now make one of the intermediate witnesses be incorrect @@ -1192,7 +1205,8 @@ class SegWitTest(BitcoinTestFramework): block.vtx = [block.vtx[0]] self.update_witness_block_with_transactions(block, [tx2]) - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, + reason='non-mandatory-script-verify-flag (Operation not valid with the current stack size)') # Fix the broken witness and the block should be accepted. tx2.wit.vtxinwit[5].scriptWitness.stack = [b'a', witness_script] @@ -1414,7 +1428,7 @@ class SegWitTest(BitcoinTestFramework): self.sync_blocks() block2 = self.build_next_block() self.update_witness_block_with_transactions(block2, [spend_tx]) - test_witness_block(self.nodes[0], self.test_node, block2, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block2, accepted=False, reason='bad-txns-premature-spend-of-coinbase') # Advancing one more block should allow the spend. self.generate(self.nodes[0], 1) @@ -1455,7 +1469,7 @@ class SegWitTest(BitcoinTestFramework): # Now try to spend it. Send it to a P2WSH output, which we'll # use in the next test. - witness_script = CScript([pubkey, CScriptOp(OP_CHECKSIG)]) + witness_script = key_to_p2pk_script(pubkey) script_wsh = script_to_p2wsh_script(witness_script) tx2 = CTransaction() @@ -1533,7 +1547,7 @@ class SegWitTest(BitcoinTestFramework): key.generate() pubkey = key.get_pubkey().get_bytes() - witness_script = CScript([pubkey, CScriptOp(OP_CHECKSIG)]) + witness_script = key_to_p2pk_script(pubkey) script_pubkey = script_to_p2wsh_script(witness_script) # First create a witness output for use in the tests. @@ -1563,13 +1577,17 @@ class SegWitTest(BitcoinTestFramework): # Too-large input value sign_p2pk_witness_input(witness_script, tx, 0, hashtype, prev_utxo.nValue + 1, key) self.update_witness_block_with_transactions(block, [tx]) - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, + reason='non-mandatory-script-verify-flag (Script evaluated without error ' + 'but finished with a false/empty top stack element') # Too-small input value sign_p2pk_witness_input(witness_script, tx, 0, hashtype, prev_utxo.nValue - 1, key) block.vtx.pop() # remove last tx self.update_witness_block_with_transactions(block, [tx]) - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, + reason='non-mandatory-script-verify-flag (Script evaluated without error ' + 'but finished with a false/empty top stack element') # Now try correct value sign_p2pk_witness_input(witness_script, tx, 0, hashtype, prev_utxo.nValue, key) @@ -1671,7 +1689,8 @@ class SegWitTest(BitcoinTestFramework): tx2.vin[0].scriptSig = CScript([signature, pubkey]) block = self.build_next_block() self.update_witness_block_with_transactions(block, [tx, tx2]) - test_witness_block(self.nodes[0], self.test_node, block, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block, accepted=False, + reason='non-mandatory-script-verify-flag (Witness requires empty scriptSig)') # Move the signature to the witness. block.vtx.pop() @@ -1917,7 +1936,7 @@ class SegWitTest(BitcoinTestFramework): block_2 = self.build_next_block() self.update_witness_block_with_transactions(block_2, [tx2]) - test_witness_block(self.nodes[0], self.test_node, block_2, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block_2, accepted=False, reason='bad-blk-sigops') # Try dropping the last input in tx2, and add an output that has # too many sigops (contributing to legacy sigop count). @@ -1930,7 +1949,7 @@ class SegWitTest(BitcoinTestFramework): tx2.rehash() block_3 = self.build_next_block() self.update_witness_block_with_transactions(block_3, [tx2]) - test_witness_block(self.nodes[0], self.test_node, block_3, accepted=False) + test_witness_block(self.nodes[0], self.test_node, block_3, accepted=False, reason='bad-blk-sigops') # If we drop the last checksig in this output, the tx should succeed. block_4 = self.build_next_block() diff --git a/test/functional/rpc_blockchain.py b/test/functional/rpc_blockchain.py index c3c6ade684..eea9ee26cb 100755 --- a/test/functional/rpc_blockchain.py +++ b/test/functional/rpc_blockchain.py @@ -434,17 +434,55 @@ class BlockchainTest(BitcoinTestFramework): miniwallet.send_self_transfer(fee_rate=fee_per_kb, from_node=node) blockhash = self.generate(node, 1)[0] - self.log.info("Test getblock with verbosity 1 doesn't include fee") - block = node.getblock(blockhash, 1) - assert 'fee' not in block['tx'][1] - - self.log.info('Test getblock with verbosity 2 includes expected fee') - block = node.getblock(blockhash, 2) - tx = block['tx'][1] - assert 'fee' in tx - assert_equal(tx['fee'], tx['vsize'] * fee_per_byte) - - self.log.info("Test getblock with verbosity 2 still works with pruned Undo data") + def assert_fee_not_in_block(verbosity): + block = node.getblock(blockhash, verbosity) + assert 'fee' not in block['tx'][1] + + def assert_fee_in_block(verbosity): + block = node.getblock(blockhash, verbosity) + tx = block['tx'][1] + assert 'fee' in tx + assert_equal(tx['fee'], tx['vsize'] * fee_per_byte) + + def assert_vin_contains_prevout(verbosity): + block = node.getblock(blockhash, verbosity) + tx = block["tx"][1] + total_vin = Decimal("0.00000000") + total_vout = Decimal("0.00000000") + for vin in tx["vin"]: + assert "prevout" in vin + assert_equal(set(vin["prevout"].keys()), set(("value", "height", "generated", "scriptPubKey"))) + assert_equal(vin["prevout"]["generated"], True) + total_vin += vin["prevout"]["value"] + for vout in tx["vout"]: + total_vout += vout["value"] + assert_equal(total_vin, total_vout + tx["fee"]) + + def assert_vin_does_not_contain_prevout(verbosity): + block = node.getblock(blockhash, verbosity) + tx = block["tx"][1] + if isinstance(tx, str): + # In verbosity level 1, only the transaction hashes are written + pass + else: + for vin in tx["vin"]: + assert "prevout" not in vin + + self.log.info("Test that getblock with verbosity 1 doesn't include fee") + assert_fee_not_in_block(1) + + self.log.info('Test that getblock with verbosity 2 and 3 includes expected fee') + assert_fee_in_block(2) + assert_fee_in_block(3) + + self.log.info("Test that getblock with verbosity 1 and 2 does not include prevout") + assert_vin_does_not_contain_prevout(1) + assert_vin_does_not_contain_prevout(2) + + self.log.info("Test that getblock with verbosity 3 includes prevout") + assert_vin_contains_prevout(3) + + self.log.info("Test that getblock with verbosity 2 and 3 still works with pruned Undo data") datadir = get_datadir_path(self.options.tmpdir, 0) self.log.info("Test getblock with invalid verbosity type returns proper error message") @@ -458,8 +496,10 @@ class BlockchainTest(BitcoinTestFramework): # Move instead of deleting so we can restore chain state afterwards move_block_file('rev00000.dat', 'rev_wrong') - block = node.getblock(blockhash, 2) - assert 'fee' not in block['tx'][1] + assert_fee_not_in_block(2) + assert_fee_not_in_block(3) + assert_vin_does_not_contain_prevout(2) + assert_vin_does_not_contain_prevout(3) # Restore chain state move_block_file('rev_wrong', 'rev00000.dat') diff --git a/test/functional/rpc_fundrawtransaction.py b/test/functional/rpc_fundrawtransaction.py index cda0ae0eeb..b0e46c6ca7 100755 --- a/test/functional/rpc_fundrawtransaction.py +++ b/test/functional/rpc_fundrawtransaction.py @@ -8,6 +8,7 @@ from decimal import Decimal from itertools import product from test_framework.descriptors import descsum_create +from test_framework.key import ECKey from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_approx, @@ -19,6 +20,7 @@ from test_framework.util import ( count_bytes, find_vout_for_address, ) +from test_framework.wallet_util import bytes_to_wif def get_unspent(listunspent, amount): @@ -132,6 +134,7 @@ class RawTransactionsTest(BitcoinTestFramework): self.test_subtract_fee_with_presets() self.test_transaction_too_large() self.test_include_unsafe() + self.test_external_inputs() self.test_22670() def test_change_position(self): @@ -983,6 +986,57 @@ class RawTransactionsTest(BitcoinTestFramework): wallet.sendmany("", outputs) self.generate(self.nodes[0], 10) assert_raises_rpc_error(-4, "Transaction too large", recipient.fundrawtransaction, rawtx) + self.nodes[0].unloadwallet("large") + + def test_external_inputs(self): + self.log.info("Test funding with external inputs") + + eckey = ECKey() + eckey.generate() + privkey = bytes_to_wif(eckey.get_bytes()) + + self.nodes[2].createwallet("extfund") + wallet = self.nodes[2].get_wallet_rpc("extfund") + + # Make a weird but signable script. sh(pkh()) descriptor accomplishes this + desc = descsum_create("sh(pkh({}))".format(privkey)) + if self.options.descriptors: + res = self.nodes[0].importdescriptors([{"desc": desc, "timestamp": "now"}]) + else: + res = self.nodes[0].importmulti([{"desc": desc, "timestamp": "now"}]) + assert res[0]["success"] + addr = self.nodes[0].deriveaddresses(desc)[0] + addr_info = self.nodes[0].getaddressinfo(addr) + + self.nodes[0].sendtoaddress(addr, 10) + self.nodes[0].sendtoaddress(wallet.getnewaddress(), 10) + self.generate(self.nodes[0], 6) + self.sync_all() + ext_utxo = self.nodes[0].listunspent(addresses=[addr])[0] + + # An external input without solving data should result in an error + raw_tx = wallet.createrawtransaction([ext_utxo], {self.nodes[0].getnewaddress(): 15}) + assert_raises_rpc_error(-4, "Insufficient funds", wallet.fundrawtransaction, raw_tx) + + # Error conditions + assert_raises_rpc_error(-5, "'not a pubkey' is not hex", wallet.fundrawtransaction, raw_tx, {"solving_data": {"pubkeys":["not a pubkey"]}}) + assert_raises_rpc_error(-5, "'01234567890a0b0c0d0e0f' is not a valid public key", wallet.fundrawtransaction, raw_tx, {"solving_data": {"pubkeys":["01234567890a0b0c0d0e0f"]}}) + assert_raises_rpc_error(-5, "'not a script' is not hex", wallet.fundrawtransaction, raw_tx, {"solving_data": {"scripts":["not a script"]}}) + assert_raises_rpc_error(-8, "Unable to parse descriptor 'not a descriptor'", wallet.fundrawtransaction, raw_tx, {"solving_data": {"descriptors":["not a descriptor"]}}) + + # But funding should work when the solving data is provided + funded_tx = wallet.fundrawtransaction(raw_tx, {"solving_data": {"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}}) + signed_tx = wallet.signrawtransactionwithwallet(funded_tx['hex']) + assert not signed_tx['complete'] + signed_tx = self.nodes[0].signrawtransactionwithwallet(signed_tx['hex']) + assert signed_tx['complete'] + + funded_tx = wallet.fundrawtransaction(raw_tx, {"solving_data": {"descriptors": [desc]}}) + signed_tx = wallet.signrawtransactionwithwallet(funded_tx['hex']) + assert not signed_tx['complete'] + signed_tx = self.nodes[0].signrawtransactionwithwallet(signed_tx['hex']) + assert signed_tx['complete'] + self.nodes[2].unloadwallet("extfund") def test_include_unsafe(self): self.log.info("Test fundrawtxn with unsafe inputs") @@ -1017,6 +1071,7 @@ class RawTransactionsTest(BitcoinTestFramework): assert all((txin["txid"], txin["vout"]) in inputs for txin in tx_dec["vin"]) signedtx = wallet.signrawtransactionwithwallet(fundedtx['hex']) assert wallet.testmempoolaccept([signedtx['hex']])[0]["allowed"] + self.nodes[0].unloadwallet("unsafe") def test_22670(self): # In issue #22670, it was observed that ApproximateBestSubset may diff --git a/test/functional/rpc_invalid_address_message.py b/test/functional/rpc_invalid_address_message.py index e362642f0f..085f6582b5 100755 --- a/test/functional/rpc_invalid_address_message.py +++ b/test/functional/rpc_invalid_address_message.py @@ -29,9 +29,6 @@ class InvalidAddressErrorMessageTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 1 - def skip_test_if_missing_module(self): - self.skip_if_no_wallet() - def test_validateaddress(self): node = self.nodes[0] @@ -60,6 +57,10 @@ class InvalidAddressErrorMessageTest(BitcoinTestFramework): assert info['isvalid'] assert 'error' not in info + info = node.validateaddress(BECH32_INVALID_VERSION) + assert not info['isvalid'] + assert_equal(info['error'], 'Invalid Bech32 address witness version') + # Base58 info = node.validateaddress(BASE58_INVALID_PREFIX) assert not info['isvalid'] @@ -87,7 +88,10 @@ class InvalidAddressErrorMessageTest(BitcoinTestFramework): def run_test(self): self.test_validateaddress() - self.test_getaddressinfo() + + if self.is_wallet_compiled(): + self.init_wallet(node=0) + self.test_getaddressinfo() if __name__ == '__main__': diff --git a/test/functional/rpc_misc.py b/test/functional/rpc_misc.py index 13f33c321f..e32e562bce 100755 --- a/test/functional/rpc_misc.py +++ b/test/functional/rpc_misc.py @@ -57,7 +57,7 @@ class RpcMiscTest(BitcoinTestFramework): self.log.info("test logging rpc and help") # Test logging RPC returns the expected number of logging categories. - assert_equal(len(node.logging()), 25) + assert_equal(len(node.logging()), 27) # Test toggling a logging category on/off/on with the logging RPC. assert_equal(node.logging()['qt'], True) diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index f330bbf1c3..b132ac3d31 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -8,6 +8,8 @@ from decimal import Decimal from itertools import product +from test_framework.descriptors import descsum_create +from test_framework.key import ECKey from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_approx, @@ -16,6 +18,7 @@ from test_framework.util import ( assert_raises_rpc_error, find_output, ) +from test_framework.wallet_util import bytes_to_wif import json import os @@ -608,5 +611,43 @@ class PSBTTest(BitcoinTestFramework): assert_raises_rpc_error(-25, 'Inputs missing or spent', self.nodes[0].walletprocesspsbt, 'cHNidP8BAJoCAAAAAkvEW8NnDtdNtDpsmze+Ht2LH35IJcKv00jKAlUs21RrAwAAAAD/////S8Rbw2cO1020OmybN74e3Ysffkglwq/TSMoCVSzbVGsBAAAAAP7///8CwLYClQAAAAAWABSNJKzjaUb3uOxixsvh1GGE3fW7zQD5ApUAAAAAFgAUKNw0x8HRctAgmvoevm4u1SbN7XIAAAAAAAEAnQIAAAACczMa321tVHuN4GKWKRncycI22aX3uXgwSFUKM2orjRsBAAAAAP7///9zMxrfbW1Ue43gYpYpGdzJwjbZpfe5eDBIVQozaiuNGwAAAAAA/v///wIA+QKVAAAAABl2qRT9zXUVA8Ls5iVqynLHe5/vSe1XyYisQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAAAAAQEfQM0ClQAAAAAWABRmWQUcjSjghQ8/uH4Bn/zkakwLtAAAAA==') + # Test that we can fund psbts with external inputs specified + eckey = ECKey() + eckey.generate() + privkey = bytes_to_wif(eckey.get_bytes()) + + # Make a weird but signable script. sh(pkh()) descriptor accomplishes this + desc = descsum_create("sh(pkh({}))".format(privkey)) + if self.options.descriptors: + res = self.nodes[0].importdescriptors([{"desc": desc, "timestamp": "now"}]) + else: + res = self.nodes[0].importmulti([{"desc": desc, "timestamp": "now"}]) + assert res[0]["success"] + addr = self.nodes[0].deriveaddresses(desc)[0] + addr_info = self.nodes[0].getaddressinfo(addr) + + self.nodes[0].sendtoaddress(addr, 10) + self.generate(self.nodes[0], 6) + self.sync_all() + ext_utxo = self.nodes[0].listunspent(addresses=[addr])[0] + + # An external input without solving data should result in an error + assert_raises_rpc_error(-4, "Insufficient funds", self.nodes[1].walletcreatefundedpsbt, [ext_utxo], {self.nodes[0].getnewaddress(): 10 + ext_utxo['amount']}, 0, {'add_inputs': True}) + + # But funding should work when the solving data is provided + psbt = self.nodes[1].walletcreatefundedpsbt([ext_utxo], {self.nodes[0].getnewaddress(): 15}, 0, {'add_inputs': True, "solving_data": {"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}}) + signed = self.nodes[1].walletprocesspsbt(psbt['psbt']) + assert not signed['complete'] + signed = self.nodes[0].walletprocesspsbt(signed['psbt']) + assert signed['complete'] + self.nodes[0].finalizepsbt(signed['psbt']) + + psbt = self.nodes[1].walletcreatefundedpsbt([ext_utxo], {self.nodes[0].getnewaddress(): 15}, 0, {'add_inputs': True, "solving_data":{"descriptors": [desc]}}) + signed = self.nodes[1].walletprocesspsbt(psbt['psbt']) + assert not signed['complete'] + signed = self.nodes[0].walletprocesspsbt(signed['psbt']) + assert signed['complete'] + self.nodes[0].finalizepsbt(signed['psbt']) + if __name__ == '__main__': PSBTTest().main() diff --git a/test/functional/rpc_signer.py b/test/functional/rpc_signer.py index 9e963eba57..5c3722ef8f 100755 --- a/test/functional/rpc_signer.py +++ b/test/functional/rpc_signer.py @@ -27,6 +27,9 @@ class RPCSignerTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 4 + # The experimental syscall sandbox feature (-sandbox) is not compatible with -signer (which + # invokes execve). + self.disable_syscall_sandbox = True self.extra_args = [ [], diff --git a/test/functional/rpc_signrawtransaction.py b/test/functional/rpc_signrawtransaction.py index 18abece253..c519d0c7d1 100755 --- a/test/functional/rpc_signrawtransaction.py +++ b/test/functional/rpc_signrawtransaction.py @@ -25,12 +25,12 @@ from test_framework.messages import ( from test_framework.script import ( CScript, OP_CHECKLOCKTIMEVERIFY, - OP_CHECKSIG, OP_CHECKSEQUENCEVERIFY, OP_DROP, OP_TRUE, ) from test_framework.script_util import ( + key_to_p2pk_script, key_to_p2pkh_script, script_to_p2sh_p2wsh_script, script_to_p2wsh_script, @@ -229,7 +229,7 @@ class SignRawTransactionsTest(BitcoinTestFramework): embedded_pubkey = eckey.get_pubkey().get_bytes().hex() witness_script = { 'P2PKH': key_to_p2pkh_script(embedded_pubkey).hex(), - 'P2PK': CScript([bytes.fromhex(embedded_pubkey), OP_CHECKSIG]).hex() + 'P2PK': key_to_p2pk_script(embedded_pubkey).hex() }.get(tx_type, "Invalid tx_type") redeem_script = script_to_p2wsh_script(witness_script).hex() addr = script_to_p2sh(redeem_script) diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 6de372cd8e..85e3c2a383 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -33,11 +33,11 @@ from .script import ( CScriptOp, OP_1, OP_CHECKMULTISIG, - OP_CHECKSIG, OP_RETURN, OP_TRUE, ) from .script_util import ( + key_to_p2pk_script, key_to_p2wpkh_script, script_to_p2wsh_script, ) @@ -134,7 +134,7 @@ def create_coinbase(height, pubkey=None, extra_output_script=None, fees=0, nValu coinbaseoutput.nValue >>= halvings coinbaseoutput.nValue += fees if pubkey is not None: - coinbaseoutput.scriptPubKey = CScript([pubkey, OP_CHECKSIG]) + coinbaseoutput.scriptPubKey = key_to_p2pk_script(pubkey) else: coinbaseoutput.scriptPubKey = CScript([OP_TRUE]) coinbase.vout = [coinbaseoutput] diff --git a/test/functional/test_framework/script_util.py b/test/functional/test_framework/script_util.py index e0dfce4c2f..82a9067dd2 100755 --- a/test/functional/test_framework/script_util.py +++ b/test/functional/test_framework/script_util.py @@ -5,14 +5,14 @@ """Useful Script constants and utils.""" from test_framework.script import ( CScript, - hash160, - sha256, OP_0, - OP_DUP, - OP_HASH160, OP_CHECKSIG, + OP_DUP, OP_EQUAL, OP_EQUALVERIFY, + OP_HASH160, + hash160, + sha256, ) # To prevent a "tx-size-small" policy rule error, a transaction has to have a @@ -36,6 +36,11 @@ DUMMY_P2WPKH_SCRIPT = CScript([b'a' * 21]) DUMMY_2_P2WPKH_SCRIPT = CScript([b'b' * 21]) +def key_to_p2pk_script(key): + key = check_key(key) + return CScript([key, OP_CHECKSIG]) + + def keyhash_to_p2pkh_script(hash): assert len(hash) == 20 return CScript([OP_DUP, OP_HASH160, hash, OP_EQUALVERIFY, OP_CHECKSIG]) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 3aca93fab3..ec3561b1f2 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -101,6 +101,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.supports_cli = True self.bind_to_localhost_only = True self.parse_args() + self.disable_syscall_sandbox = self.options.nosandbox self.default_wallet_name = "default_wallet" if self.options.descriptors else "" self.wallet_data_filename = "wallet.dat" # Optional list of wallet names that can be set in set_test_params to @@ -159,6 +160,8 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): parser = argparse.ArgumentParser(usage="%(prog)s [options]") parser.add_argument("--nocleanup", dest="nocleanup", default=False, action="store_true", help="Leave bitcoinds and test.* datadir on exit or error") + parser.add_argument("--nosandbox", dest="nosandbox", default=False, action="store_true", + help="Don't use the syscall sandbox") parser.add_argument("--noshutdown", dest="noshutdown", default=False, action="store_true", help="Don't stop bitcoinds after the test execution") parser.add_argument("--cachedir", dest="cachedir", default=os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../../cache"), @@ -420,12 +423,12 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def import_deterministic_coinbase_privkeys(self): for i in range(self.num_nodes): - self.init_wallet(i) + self.init_wallet(node=i) - def init_wallet(self, i): - wallet_name = self.default_wallet_name if self.wallet_names is None else self.wallet_names[i] if i < len(self.wallet_names) else False + def init_wallet(self, *, node): + wallet_name = self.default_wallet_name if self.wallet_names is None else self.wallet_names[node] if node < len(self.wallet_names) else False if wallet_name is not False: - n = self.nodes[i] + n = self.nodes[node] if wallet_name is not None: n.createwallet(wallet_name=wallet_name, descriptors=self.options.descriptors, load_on_startup=True) n.importprivkey(privkey=n.get_deterministic_priv_key().key, label='coinbase') @@ -468,6 +471,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): extra_args = [[]] * num_nodes if versions is None: versions = [None] * num_nodes + if self.is_syscall_sandbox_compiled() and not self.disable_syscall_sandbox: + for i in range(len(extra_args)): + if versions[i] is None or versions[i] >= 219900: + extra_args[i] = extra_args[i] + ["-sandbox=log-and-abort"] if binary is None: binary = [get_bin_from_version(v, 'bitcoind', self.options.bitcoind) for v in versions] if binary_cli is None: @@ -621,19 +628,19 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): self.sync_all() def generate(self, generator, *args, **kwargs): - blocks = generator.generate(*args, **kwargs) + blocks = generator.generate(*args, invalid_call=False, **kwargs) return blocks def generateblock(self, generator, *args, **kwargs): - blocks = generator.generateblock(*args, **kwargs) + blocks = generator.generateblock(*args, invalid_call=False, **kwargs) return blocks def generatetoaddress(self, generator, *args, **kwargs): - blocks = generator.generatetoaddress(*args, **kwargs) + blocks = generator.generatetoaddress(*args, invalid_call=False, **kwargs) return blocks def generatetodescriptor(self, generator, *args, **kwargs): - blocks = generator.generatetodescriptor(*args, **kwargs) + blocks = generator.generatetodescriptor(*args, invalid_call=False, **kwargs) return blocks def sync_blocks(self, nodes=None, wait=1, timeout=60): @@ -887,3 +894,7 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass): def is_bdb_compiled(self): """Checks whether the wallet module was compiled with BDB support.""" return self.config["components"].getboolean("USE_BDB") + + def is_syscall_sandbox_compiled(self): + """Checks whether the syscall sandbox was compiled.""" + return self.config["components"].getboolean("ENABLE_SYSCALL_SANDBOX") diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index f9e2cfa2f5..e8ff41a46d 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -297,9 +297,21 @@ class TestNode(): time.sleep(1.0 / poll_per_s) self._raise_assertion_error("Unable to retrieve cookie credentials after {}s".format(self.rpc_timeout)) - def generate(self, nblocks, maxtries=1000000): + def generate(self, nblocks, maxtries=1000000, **kwargs): self.log.debug("TestNode.generate() dispatches `generate` call to `generatetoaddress`") - return self.generatetoaddress(nblocks=nblocks, address=self.get_deterministic_priv_key().address, maxtries=maxtries) + return self.generatetoaddress(nblocks=nblocks, address=self.get_deterministic_priv_key().address, maxtries=maxtries, **kwargs) + + def generateblock(self, *args, invalid_call, **kwargs): + assert not invalid_call + return self.__getattr__('generateblock')(*args, **kwargs) + + def generatetoaddress(self, *args, invalid_call, **kwargs): + assert not invalid_call + return self.__getattr__('generatetoaddress')(*args, **kwargs) + + def generatetodescriptor(self, *args, invalid_call, **kwargs): + assert not invalid_call + return self.__getattr__('generatetodescriptor')(*args, **kwargs) def get_wallet_rpc(self, wallet_name): if self.use_cli: diff --git a/test/functional/test_framework/wallet.py b/test/functional/test_framework/wallet.py index ef27cb3221..81aad20079 100644 --- a/test/functional/test_framework/wallet.py +++ b/test/functional/test_framework/wallet.py @@ -10,6 +10,7 @@ from enum import Enum from random import choice from typing import Optional from test_framework.address import ADDRESS_BCRT1_P2WSH_OP_TRUE +from test_framework.descriptors import descsum_create from test_framework.key import ECKey from test_framework.messages import ( COIN, @@ -23,16 +24,17 @@ from test_framework.messages import ( from test_framework.script import ( CScript, LegacySignatureHash, - OP_CHECKSIG, OP_TRUE, OP_NOP, SIGHASH_ALL, ) -from test_framework.script_util import key_to_p2wpkh_script +from test_framework.script_util import ( + key_to_p2pk_script, + key_to_p2wpkh_script, +) from test_framework.util import ( assert_equal, assert_greater_than_or_equal, - satoshi_round, ) DEFAULT_FEE = Decimal("0.0001") @@ -75,7 +77,7 @@ class MiniWallet: self._priv_key = ECKey() self._priv_key.set((1).to_bytes(32, 'big'), True) pub_key = self._priv_key.get_pubkey() - self._scriptPubKey = bytes(CScript([pub_key.get_bytes(), OP_CHECKSIG])) + self._scriptPubKey = key_to_p2pk_script(pub_key.get_bytes()) elif mode == MiniWalletMode.ADDRESS_OP_TRUE: self._address = ADDRESS_BCRT1_P2WSH_OP_TRUE self._scriptPubKey = bytes.fromhex(self._test_node.validateaddress(self._address)['scriptPubKey']) @@ -109,16 +111,16 @@ class MiniWallet: break tx.vin[0].scriptSig = CScript([der_sig + bytes(bytearray([SIGHASH_ALL]))]) - def generate(self, num_blocks): + def generate(self, num_blocks, **kwargs): """Generate blocks with coinbase outputs to the internal address, and append the outputs to the internal list""" - blocks = self._test_node.generatetodescriptor(num_blocks, self.get_descriptor()) + blocks = self._test_node.generatetodescriptor(num_blocks, self.get_descriptor(), **kwargs) for b in blocks: cb_tx = self._test_node.getblock(blockhash=b, verbosity=2)['tx'][0] self._utxos.append({'txid': cb_tx['txid'], 'vout': 0, 'value': cb_tx['vout'][0]['value']}) return blocks def get_descriptor(self): - return self._test_node.getdescriptorinfo(f'raw({self._scriptPubKey.hex()})')['descriptor'] + return descsum_create(f'raw({self._scriptPubKey.hex()})') def get_address(self): return self._address @@ -174,13 +176,12 @@ class MiniWallet: vsize = Decimal(96) # anyone-can-spend else: vsize = Decimal(168) # P2PK (73 bytes scriptSig + 35 bytes scriptPubKey + 60 bytes other) - send_value = satoshi_round(utxo_to_spend['value'] - fee_rate * (vsize / 1000)) - fee = utxo_to_spend['value'] - send_value + send_value = int(COIN * (utxo_to_spend['value'] - fee_rate * (vsize / 1000))) assert send_value > 0 tx = CTransaction() tx.vin = [CTxIn(COutPoint(int(utxo_to_spend['txid'], 16), utxo_to_spend['vout']), nSequence=sequence)] - tx.vout = [CTxOut(int(send_value * COIN), self._scriptPubKey)] + tx.vout = [CTxOut(send_value, self._scriptPubKey)] tx.nLockTime = locktime if not self._address: # raw script @@ -199,7 +200,7 @@ class MiniWallet: assert_equal(mempool_valid, tx_info['allowed']) if mempool_valid: assert_equal(tx_info['vsize'], vsize) - assert_equal(tx_info['fees']['base'], fee) + assert_equal(tx_info['fees']['base'], utxo_to_spend['value'] - Decimal(send_value) / COIN) return {'txid': tx_info['txid'], 'wtxid': tx_info['wtxid'], 'hex': tx_hex, 'tx': tx} def sendrawtransaction(self, *, from_node, tx_hex): diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index bb84962b75..b91b294108 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -40,7 +40,7 @@ except UnicodeDecodeError: CROSS = "x " CIRCLE = "o " -if os.name != 'nt' or sys.getwindowsversion() >= (10, 0, 14393): +if os.name != 'nt' or sys.getwindowsversion() >= (10, 0, 14393): #type:ignore if os.name == 'nt': import ctypes kernel32 = ctypes.windll.kernel32 # type: ignore @@ -171,6 +171,7 @@ BASE_SCRIPTS = [ 'rpc_users.py', 'rpc_whitelist.py', 'feature_proxy.py', + 'feature_syscall_sandbox.py', 'rpc_signrawtransaction.py --legacy-wallet', 'rpc_signrawtransaction.py --descriptors', 'rpc_rawtransaction.py --legacy-wallet', @@ -206,6 +207,7 @@ BASE_SCRIPTS = [ 'feature_assumevalid.py', 'example_test.py', 'wallet_txn_doublespend.py --legacy-wallet', + 'wallet_multisig_descriptor_psbt.py', 'wallet_txn_doublespend.py --descriptors', 'feature_backwards_compatibility.py --legacy-wallet', 'feature_backwards_compatibility.py --descriptors', diff --git a/test/functional/tool_wallet.py b/test/functional/tool_wallet.py index 4bf3927879..3a0daf04ad 100755 --- a/test/functional/tool_wallet.py +++ b/test/functional/tool_wallet.py @@ -31,8 +31,8 @@ class ToolWalletTest(BitcoinTestFramework): def bitcoin_wallet_process(self, *args): binary = self.config["environment"]["BUILDDIR"] + '/src/bitcoin-wallet' + self.config["environment"]["EXEEXT"] default_args = ['-datadir={}'.format(self.nodes[0].datadir), '-chain=%s' % self.chain] - if self.options.descriptors and 'create' in args: - default_args.append('-descriptors') + if not self.options.descriptors and 'create' in args: + default_args.append('-legacy') return subprocess.Popen([binary] + default_args + list(args), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) diff --git a/test/functional/wallet_abandonconflict.py b/test/functional/wallet_abandonconflict.py index 6365840041..8f54e50598 100755 --- a/test/functional/wallet_abandonconflict.py +++ b/test/functional/wallet_abandonconflict.py @@ -120,6 +120,14 @@ class AbandonConflictTest(BitcoinTestFramework): assert_equal(newbalance, balance + Decimal("30")) balance = newbalance + self.log.info("Check abandoned transactions in listsinceblock") + listsinceblock = self.nodes[0].listsinceblock() + txAB1_listsinceblock = [d for d in listsinceblock['transactions'] if d['txid'] == txAB1 and d['category'] == 'send'] + for tx in txAB1_listsinceblock: + assert_equal(tx['abandoned'], True) + assert_equal(tx['confirmations'], 0) + assert_equal(tx['trusted'], False) + # Verify that even with a low min relay fee, the tx is not reaccepted from wallet on startup once abandoned self.restart_node(0, extra_args=["-minrelaytxfee=0.00001"]) assert self.nodes[0].getmempoolinfo()['loaded'] @@ -149,6 +157,7 @@ class AbandonConflictTest(BitcoinTestFramework): assert_equal(newbalance, balance - Decimal("24.9996")) balance = newbalance + self.log.info("Test transactions conflicted by a double spend") # Create a double spend of AB1 by spending again from only A's 10 output # Mine double spend from node 1 inputs = [] @@ -163,6 +172,34 @@ class AbandonConflictTest(BitcoinTestFramework): self.connect_nodes(0, 1) self.sync_blocks() + tx_list = self.nodes[0].listtransactions() + + conflicted = [tx for tx in tx_list if tx["confirmations"] < 0] + assert_equal(4, len(conflicted)) + + wallet_conflicts = [tx for tx in conflicted if tx["walletconflicts"]] + assert_equal(2, len(wallet_conflicts)) + + double_spends = [tx for tx in tx_list if tx["walletconflicts"] and tx["confirmations"] > 0] + assert_equal(1, len(double_spends)) + double_spend = double_spends[0] + + # Test the properties of the conflicted transactions, i.e. with confirmations < 0. + for tx in conflicted: + assert_equal(tx["abandoned"], False) + assert_equal(tx["confirmations"], -1) + assert_equal(tx["trusted"], False) + + # Test the properties of the double-spend transaction, i.e. having wallet conflicts and confirmations > 0. + assert_equal(double_spend["abandoned"], False) + assert_equal(double_spend["confirmations"], 1) + assert "trusted" not in double_spend.keys() # "trusted" only returned if tx has 0 or negative confirmations. + + # Test the walletconflicts field of each. + for tx in wallet_conflicts: + assert_equal(double_spend["walletconflicts"], [tx["txid"]]) + assert_equal(tx["walletconflicts"], [double_spend["txid"]]) + # Verify that B and C's 10 BTC outputs are available for spending again because AB1 is now conflicted newbalance = self.nodes[0].getbalance() assert_equal(newbalance, balance + Decimal("20")) @@ -176,7 +213,7 @@ class AbandonConflictTest(BitcoinTestFramework): #assert_equal(newbalance, balance - Decimal("10")) self.log.info("If balance has not declined after invalidateblock then out of mempool wallet tx which is no longer") self.log.info("conflicted has not resumed causing its inputs to be seen as spent. See Issue #7315") - self.log.info(str(balance) + " -> " + str(newbalance) + " ?") + assert_equal(balance, newbalance) if __name__ == '__main__': diff --git a/test/functional/wallet_address_types.py b/test/functional/wallet_address_types.py index bdee22e62b..7a448e8590 100755 --- a/test/functional/wallet_address_types.py +++ b/test/functional/wallet_address_types.py @@ -204,8 +204,7 @@ class AddressTypeTest(BitcoinTestFramework): def test_change_output_type(self, node_sender, destinations, expected_type): txid = self.nodes[node_sender].sendmany(dummy="", amounts=dict.fromkeys(destinations, 0.001)) - raw_tx = self.nodes[node_sender].getrawtransaction(txid) - tx = self.nodes[node_sender].decoderawtransaction(raw_tx) + tx = self.nodes[node_sender].gettransaction(txid=txid, verbose=True)['decoded'] # Make sure the transaction has change: assert_equal(len(tx["vout"]), len(destinations) + 1) diff --git a/test/functional/wallet_backup.py b/test/functional/wallet_backup.py index bc6d6206e5..a07c28c8a4 100755 --- a/test/functional/wallet_backup.py +++ b/test/functional/wallet_backup.py @@ -124,9 +124,9 @@ class WalletBackupTest(BitcoinTestFramework): assert_raises_rpc_error(-8, "Wallet name already exists.", node.restorewallet, wallet_name, wallet_file) def init_three(self): - self.init_wallet(0) - self.init_wallet(1) - self.init_wallet(2) + self.init_wallet(node=0) + self.init_wallet(node=1) + self.init_wallet(node=2) def run_test(self): self.log.info("Generating initial blockchain") diff --git a/test/functional/wallet_basic.py b/test/functional/wallet_basic.py index 599e506f98..92da54d97c 100755 --- a/test/functional/wallet_basic.py +++ b/test/functional/wallet_basic.py @@ -666,7 +666,7 @@ class WalletTest(BitcoinTestFramework): self.generate(self.nodes[0], 1) destination = self.nodes[1].getnewaddress() txid = self.nodes[0].sendtoaddress(destination, 0.123) - tx = self.nodes[0].decoderawtransaction(self.nodes[0].gettransaction(txid)['hex']) + tx = self.nodes[0].gettransaction(txid=txid, verbose=True)['decoded'] output_addresses = [vout['scriptPubKey']['address'] for vout in tx["vout"]] assert len(output_addresses) > 1 for address in output_addresses: diff --git a/test/functional/wallet_bumpfee.py b/test/functional/wallet_bumpfee.py index a1676fffa5..46a5df4a8e 100755 --- a/test/functional/wallet_bumpfee.py +++ b/test/functional/wallet_bumpfee.py @@ -32,6 +32,8 @@ from test_framework.util import ( assert_greater_than, assert_raises_rpc_error, ) +from test_framework.wallet import MiniWallet + WALLET_PASSPHRASE = "test" WALLET_PASSPHRASE_TIMEOUT = 3600 @@ -265,6 +267,14 @@ def test_bumpfee_with_descendant_fails(self, rbf_node, rbf_node_address, dest_ad tx = rbf_node.signrawtransactionwithwallet(tx) rbf_node.sendrawtransaction(tx["hex"]) assert_raises_rpc_error(-8, "Transaction has descendants in the wallet", rbf_node.bumpfee, parent_id) + + # create tx with descendant in the mempool by using MiniWallet + miniwallet = MiniWallet(rbf_node) + parent_id = spend_one_input(rbf_node, miniwallet.get_address()) + tx = rbf_node.gettransaction(txid=parent_id, verbose=True)['decoded'] + miniwallet.scan_tx(tx) + miniwallet.send_self_transfer(from_node=rbf_node) + assert_raises_rpc_error(-8, "Transaction has descendants in the mempool", rbf_node.bumpfee, parent_id) self.clear_mempool() diff --git a/test/functional/wallet_create_tx.py b/test/functional/wallet_create_tx.py index c8b92ef1bf..00ee08002e 100755 --- a/test/functional/wallet_create_tx.py +++ b/test/functional/wallet_create_tx.py @@ -34,13 +34,13 @@ class CreateTxWalletTest(BitcoinTestFramework): self.log.info('Check that we have some (old) blocks and that anti-fee-sniping is disabled') assert_equal(self.nodes[0].getblockchaininfo()['blocks'], 200) txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) - tx = self.nodes[0].decoderawtransaction(self.nodes[0].gettransaction(txid)['hex']) + tx = self.nodes[0].gettransaction(txid=txid, verbose=True)['decoded'] assert_equal(tx['locktime'], 0) self.log.info('Check that anti-fee-sniping is enabled when we mine a recent block') self.generate(self.nodes[0], 1) txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 1) - tx = self.nodes[0].decoderawtransaction(self.nodes[0].gettransaction(txid)['hex']) + tx = self.nodes[0].gettransaction(txid=txid, verbose=True)['decoded'] assert 0 < tx['locktime'] <= 201 def test_tx_size_too_large(self): diff --git a/test/functional/wallet_descriptor.py b/test/functional/wallet_descriptor.py index 17a4c79da3..4ec44a8a6c 100755 --- a/test/functional/wallet_descriptor.py +++ b/test/functional/wallet_descriptor.py @@ -84,7 +84,7 @@ class WalletDescriptorTest(BitcoinTestFramework): send_wrpc = self.nodes[0].get_wallet_rpc("desc1") # Generate some coins - self.generatetoaddress(send_wrpc, COINBASE_MATURITY + 1, send_wrpc.getnewaddress()) + self.generatetoaddress(self.nodes[0], COINBASE_MATURITY + 1, send_wrpc.getnewaddress()) # Make transactions self.log.info("Test sending and receiving") diff --git a/test/functional/wallet_hd.py b/test/functional/wallet_hd.py index 974ce7f381..f54ae89c04 100755 --- a/test/functional/wallet_hd.py +++ b/test/functional/wallet_hd.py @@ -129,7 +129,7 @@ class WalletHDTest(BitcoinTestFramework): # send a tx and make sure its using the internal chain for the changeoutput txid = self.nodes[1].sendtoaddress(self.nodes[0].getnewaddress(), 1) - outs = self.nodes[1].decoderawtransaction(self.nodes[1].gettransaction(txid)['hex'])['vout'] + outs = self.nodes[1].gettransaction(txid=txid, verbose=True)['decoded']['vout'] keypath = "" for out in outs: if out['value'] != 1: diff --git a/test/functional/wallet_import_rescan.py b/test/functional/wallet_import_rescan.py index cbe3e9bfdd..27a2a42dac 100755 --- a/test/functional/wallet_import_rescan.py +++ b/test/functional/wallet_import_rescan.py @@ -99,7 +99,7 @@ class Variant(collections.namedtuple("Variant", "call data address_type rescan p assert_equal(tx["label"], self.label) assert_equal(tx["txid"], txid) assert_equal(tx["confirmations"], 1 + current_height - confirmation_height) - assert_equal("trusted" not in tx, True) + assert "trusted" not in tx address, = [ad for ad in addresses if txid in ad["txids"]] assert_equal(address["address"], self.address["address"]) diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index d86c3737fe..fc9eac1d74 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -74,7 +74,7 @@ class ImportDescriptorsTest(BitcoinTestFramework): assert_equal(wpriv.getwalletinfo()['keypoolsize'], 0) self.log.info('Mining coins') - self.generatetoaddress(w0, COINBASE_MATURITY + 1, w0.getnewaddress()) + self.generatetoaddress(self.nodes[0], COINBASE_MATURITY + 1, w0.getnewaddress()) # RPC importdescriptors ----------------------------------------------- @@ -405,7 +405,7 @@ class ImportDescriptorsTest(BitcoinTestFramework): solvable=True, ismine=True) txid = w0.sendtoaddress(address, 49.99995540) - self.generatetoaddress(w0, 6, w0.getnewaddress()) + self.generatetoaddress(self.nodes[0], 6, w0.getnewaddress()) self.sync_blocks() tx = wpriv.createrawtransaction([{"txid": txid, "vout": 0}], {w0.getnewaddress(): 49.999}) signed_tx = wpriv.signrawtransactionwithwallet(tx) @@ -454,7 +454,7 @@ class ImportDescriptorsTest(BitcoinTestFramework): self.generate(self.nodes[0], 6) self.sync_all() send_txid = wmulti_priv.sendtoaddress(w0.getnewaddress(), 8) - decoded = wmulti_priv.decoderawtransaction(wmulti_priv.gettransaction(send_txid)['hex']) + decoded = wmulti_priv.gettransaction(txid=send_txid, verbose=True)['decoded'] assert_equal(len(decoded['vin'][0]['txinwitness']), 4) self.generate(self.nodes[0], 6) self.sync_all() @@ -586,7 +586,7 @@ class ImportDescriptorsTest(BitcoinTestFramework): self.sync_all() # It is standard and would relay. txid = wmulti_priv_big.sendtoaddress(w0.getnewaddress(), 9.999) - decoded = wmulti_priv_big.decoderawtransaction(wmulti_priv_big.gettransaction(txid)['hex']) + decoded = wmulti_priv_big.gettransaction(txid=txid, verbose=True)['decoded'] # 20 sigs + dummy + witness script assert_equal(len(decoded['vin'][0]['txinwitness']), 22) @@ -620,12 +620,8 @@ class ImportDescriptorsTest(BitcoinTestFramework): self.generate(self.nodes[0], 6) self.sync_all() # It is standard and would relay. - txid = multi_priv_big.sendtoaddress(w0.getnewaddress(), 10, "", "", - True) - decoded = multi_priv_big.decoderawtransaction( - multi_priv_big.gettransaction(txid)['hex'] - ) - + txid = multi_priv_big.sendtoaddress(w0.getnewaddress(), 10, "", "", True) + decoded = multi_priv_big.gettransaction(txid=txid, verbose=True)['decoded'] self.log.info("Amending multisig with new private keys") self.nodes[1].createwallet(wallet_name="wmulti_priv3", descriptors=True) diff --git a/test/functional/wallet_keypool.py b/test/functional/wallet_keypool.py index c714993234..79235646b0 100755 --- a/test/functional/wallet_keypool.py +++ b/test/functional/wallet_keypool.py @@ -138,6 +138,20 @@ class KeyPoolTest(BitcoinTestFramework): assert_equal(wi['keypoolsize_hd_internal'], 100) assert_equal(wi['keypoolsize'], 100) + if not self.options.descriptors: + # Check that newkeypool entirely flushes the keypool + start_keypath = nodes[0].getaddressinfo(nodes[0].getnewaddress())['hdkeypath'] + start_change_keypath = nodes[0].getaddressinfo(nodes[0].getrawchangeaddress())['hdkeypath'] + # flush keypool and get new addresses + nodes[0].newkeypool() + end_keypath = nodes[0].getaddressinfo(nodes[0].getnewaddress())['hdkeypath'] + end_change_keypath = nodes[0].getaddressinfo(nodes[0].getrawchangeaddress())['hdkeypath'] + # The new keypath index should be 100 more than the old one + new_index = int(start_keypath.rsplit('/', 1)[1][:-1]) + 100 + new_change_index = int(start_change_keypath.rsplit('/', 1)[1][:-1]) + 100 + assert_equal(end_keypath, "m/0'/0'/" + str(new_index) + "'") + assert_equal(end_change_keypath, "m/0'/1'/" + str(new_change_index) + "'") + # create a blank wallet nodes[0].createwallet(wallet_name='w2', blank=True, disable_private_keys=True) w2 = nodes[0].get_wallet_rpc('w2') diff --git a/test/functional/wallet_listdescriptors.py b/test/functional/wallet_listdescriptors.py index 221f5262d9..436bbdcfcc 100755 --- a/test/functional/wallet_listdescriptors.py +++ b/test/functional/wallet_listdescriptors.py @@ -23,7 +23,7 @@ class ListDescriptorsTest(BitcoinTestFramework): self.skip_if_no_sqlite() # do not create any wallet by default - def init_wallet(self, i): + def init_wallet(self, *, node): return def run_test(self): diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index bd3b29c81c..f4a00a8ec8 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -44,6 +44,14 @@ class ListSinceBlockTest(BitcoinTestFramework): def test_no_blockhash(self): self.log.info("Test no blockhash") txid = self.nodes[2].sendtoaddress(self.nodes[0].getnewaddress(), 1) + self.sync_all() + assert_array_result(self.nodes[0].listtransactions(), {"txid": txid}, { + "category": "receive", + "amount": 1, + "confirmations": 0, + "trusted": False, + }) + blockhash, = self.generate(self.nodes[2], 1) blockheight = self.nodes[2].getblockheader(blockhash)['height'] self.sync_all() @@ -56,6 +64,9 @@ class ListSinceBlockTest(BitcoinTestFramework): "blockheight": blockheight, "confirmations": 1, }) + assert_equal(len(txs), 1) + assert "trusted" not in txs[0] + assert_equal( self.nodes[0].listsinceblock(), {"lastblock": blockhash, diff --git a/test/functional/wallet_listtransactions.py b/test/functional/wallet_listtransactions.py index a14bfe345c..ca6a6ab540 100755 --- a/test/functional/wallet_listtransactions.py +++ b/test/functional/wallet_listtransactions.py @@ -31,10 +31,10 @@ class ListTransactionsTest(BitcoinTestFramework): self.sync_all() assert_array_result(self.nodes[0].listtransactions(), {"txid": txid}, - {"category": "send", "amount": Decimal("-0.1"), "confirmations": 0}) + {"category": "send", "amount": Decimal("-0.1"), "confirmations": 0, "trusted": True}) assert_array_result(self.nodes[1].listtransactions(), {"txid": txid}, - {"category": "receive", "amount": Decimal("0.1"), "confirmations": 0}) + {"category": "receive", "amount": Decimal("0.1"), "confirmations": 0, "trusted": False}) self.log.info("Test confirmations change after mining a block") blockhash = self.generate(self.nodes[0], 1)[0] blockheight = self.nodes[0].getblockheader(blockhash)['height'] @@ -204,6 +204,15 @@ class ListTransactionsTest(BitcoinTestFramework): assert_equal(n.gettransaction(txid_3b)["bip125-replaceable"], "yes") assert_equal(n.gettransaction(txid_4)["bip125-replaceable"], "unknown") + self.log.info("Test bip125-replaceable status with listsinceblock") + for n in self.nodes[0:2]: + txs = {tx['txid']: tx['bip125-replaceable'] for tx in n.listsinceblock()['transactions']} + assert_equal(txs[txid_1], "no") + assert_equal(txs[txid_2], "no") + assert_equal(txs[txid_3], "yes") + assert_equal(txs[txid_3b], "yes") + assert_equal(txs[txid_4], "unknown") + self.log.info("Test mined transactions are no longer bip125-replaceable") self.generate(self.nodes[0], 1) assert txid_3b not in self.nodes[0].getrawmempool() diff --git a/test/functional/wallet_multisig_descriptor_psbt.py b/test/functional/wallet_multisig_descriptor_psbt.py new file mode 100755 index 0000000000..ed855d2525 --- /dev/null +++ b/test/functional/wallet_multisig_descriptor_psbt.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test a basic M-of-N multisig setup between multiple people using descriptor wallets and PSBTs, as well as a signing flow. + +This is meant to be documentation as much as functional tests, so it is kept as simple and readable as possible. +""" + +from test_framework.address import base58_to_byte +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_approx, + assert_equal, +) + + +class WalletMultisigDescriptorPSBTTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 3 + self.setup_clean_chain = True + self.wallet_names = [] + self.extra_args = [["-keypool=100"]] * self.num_nodes + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + self.skip_if_no_sqlite() + + @staticmethod + def _get_xpub(wallet): + """Extract the wallet's xpubs using `listdescriptors` and pick the one from the `pkh` descriptor since it's least likely to be accidentally reused (legacy addresses).""" + descriptor = next(filter(lambda d: d["desc"].startswith("pkh"), wallet.listdescriptors()["descriptors"])) + return descriptor["desc"].split("]")[-1].split("/")[0] + + @staticmethod + def _check_psbt(psbt, to, value, multisig): + """Helper function for any of the N participants to check the psbt with decodepsbt and verify it is OK before signing.""" + tx = multisig.decodepsbt(psbt)["tx"] + amount = 0 + for vout in tx["vout"]: + address = vout["scriptPubKey"]["address"] + assert_equal(multisig.getaddressinfo(address)["ischange"], address != to) + if address == to: + amount += vout["value"] + assert_approx(amount, float(value), vspan=0.001) + + def participants_create_multisigs(self, xpubs): + """The multisig is created by importing the following descriptors. The resulting wallet is watch-only and every participant can do this.""" + # some simple validation + assert_equal(len(xpubs), self.N) + # a sanity-check/assertion, this will throw if the base58 checksum of any of the provided xpubs are invalid + for xpub in xpubs: + base58_to_byte(xpub) + + for i, node in enumerate(self.nodes): + node.createwallet(wallet_name=f"{self.name}_{i}", blank=True, descriptors=True, disable_private_keys=True) + multisig = node.get_wallet_rpc(f"{self.name}_{i}") + external = multisig.getdescriptorinfo(f"wsh(sortedmulti({self.M},{f'/0/*,'.join(xpubs)}/0/*))") + internal = multisig.getdescriptorinfo(f"wsh(sortedmulti({self.M},{f'/1/*,'.join(xpubs)}/1/*))") + result = multisig.importdescriptors([ + { # receiving addresses (internal: False) + "desc": external["descriptor"], + "active": True, + "internal": False, + "timestamp": "now", + }, + { # change addresses (internal: True) + "desc": internal["descriptor"], + "active": True, + "internal": True, + "timestamp": "now", + }, + ]) + assert all(r["success"] for r in result) + yield multisig + + def run_test(self): + self.M = 2 + self.N = self.num_nodes + self.name = f"{self.M}_of_{self.N}_multisig" + self.log.info(f"Testing {self.name}...") + + participants = { + # Every participant generates an xpub. The most straightforward way is to create a new descriptor wallet. + # This wallet will be the participant's `signer` for the resulting multisig. Avoid reusing this wallet for any other purpose (for privacy reasons). + "signers": [node.get_wallet_rpc(node.createwallet(wallet_name=f"participant_{self.nodes.index(node)}", descriptors=True)["name"]) for node in self.nodes], + # After participants generate and exchange their xpubs they will each create their own watch-only multisig. + # Note: these multisigs are all the same, this justs highlights that each participant can independently verify everything on their own node. + "multisigs": [] + } + + self.log.info("Generate and exchange xpubs...") + xpubs = [self._get_xpub(signer) for signer in participants["signers"]] + + self.log.info("Every participant imports the following descriptors to create the watch-only multisig...") + participants["multisigs"] = list(self.participants_create_multisigs(xpubs)) + + self.log.info("Check that every participant's multisig generates the same addresses...") + for _ in range(10): # we check that the first 10 generated addresses are the same for all participant's multisigs + receive_addresses = [multisig.getnewaddress() for multisig in participants["multisigs"]] + all(address == receive_addresses[0] for address in receive_addresses) + change_addresses = [multisig.getrawchangeaddress() for multisig in participants["multisigs"]] + all(address == change_addresses[0] for address in change_addresses) + + self.log.info("Get a mature utxo to send to the multisig...") + coordinator_wallet = participants["signers"][0] + self.generatetoaddress(self.nodes[0], 101, coordinator_wallet.getnewaddress()) + + deposit_amount = 6.15 + multisig_receiving_address = participants["multisigs"][0].getnewaddress() + self.log.info("Send funds to the resulting multisig receiving address...") + coordinator_wallet.sendtoaddress(multisig_receiving_address, deposit_amount) + self.generate(self.nodes[0], 1) + self.sync_all() + for participant in participants["multisigs"]: + assert_approx(participant.getbalance(), deposit_amount, vspan=0.001) + + self.log.info("Send a transaction from the multisig!") + to = participants["signers"][self.N - 1].getnewaddress() + value = 1 + self.log.info("First, make a sending transaction, created using `walletcreatefundedpsbt` (anyone can initiate this)...") + psbt = participants["multisigs"][0].walletcreatefundedpsbt(inputs=[], outputs={to: value}, options={"feeRate": 0.00010}) + + psbts = [] + self.log.info("Now at least M users check the psbt with decodepsbt and (if OK) signs it with walletprocesspsbt...") + for m in range(self.M): + signers_multisig = participants["multisigs"][m] + self._check_psbt(psbt["psbt"], to, value, signers_multisig) + signing_wallet = participants["signers"][m] + partially_signed_psbt = signing_wallet.walletprocesspsbt(psbt["psbt"]) + psbts.append(partially_signed_psbt["psbt"]) + + self.log.info("Finally, collect the signed PSBTs with combinepsbt, finalizepsbt, then broadcast the resulting transaction...") + combined = coordinator_wallet.combinepsbt(psbts) + finalized = coordinator_wallet.finalizepsbt(combined) + coordinator_wallet.sendrawtransaction(finalized["hex"]) + + self.log.info("Check that balances are correct after the transaction has been included in a block.") + self.generate(self.nodes[0], 1) + self.sync_all() + assert_approx(participants["multisigs"][0].getbalance(), deposit_amount - value, vspan=0.001) + assert_equal(participants["signers"][self.N - 1].getbalance(), value) + + self.log.info("Send another transaction from the multisig, this time with a daisy chained signing flow (one after another in series)!") + psbt = participants["multisigs"][0].walletcreatefundedpsbt(inputs=[], outputs={to: value}, options={"feeRate": 0.00010}) + for m in range(self.M): + signers_multisig = participants["multisigs"][m] + self._check_psbt(psbt["psbt"], to, value, signers_multisig) + signing_wallet = participants["signers"][m] + psbt = signing_wallet.walletprocesspsbt(psbt["psbt"]) + assert_equal(psbt["complete"], m == self.M - 1) + finalized = coordinator_wallet.finalizepsbt(psbt["psbt"]) + coordinator_wallet.sendrawtransaction(finalized["hex"]) + + self.log.info("Check that balances are correct after the transaction has been included in a block.") + self.generate(self.nodes[0], 1) + self.sync_all() + assert_approx(participants["multisigs"][0].getbalance(), deposit_amount - (value * 2), vspan=0.001) + assert_equal(participants["signers"][self.N - 1].getbalance(), value * 2) + + +if __name__ == "__main__": + WalletMultisigDescriptorPSBTTest().main() diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py index aecdaf821f..c9daeabeb9 100755 --- a/test/functional/wallet_send.py +++ b/test/functional/wallet_send.py @@ -9,6 +9,7 @@ from itertools import product from test_framework.authproxy import JSONRPCException from test_framework.descriptors import descsum_create +from test_framework.key import ECKey from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -16,6 +17,7 @@ from test_framework.util import ( assert_greater_than, assert_raises_rpc_error, ) +from test_framework.wallet_util import bytes_to_wif class WalletSendTest(BitcoinTestFramework): def set_test_params(self): @@ -35,7 +37,7 @@ class WalletSendTest(BitcoinTestFramework): conf_target=None, estimate_mode=None, fee_rate=None, add_to_wallet=None, psbt=None, inputs=None, add_inputs=None, include_unsafe=None, change_address=None, change_position=None, change_type=None, include_watching=None, locktime=None, lock_unspents=None, replaceable=None, subtract_fee_from_outputs=None, - expect_error=None): + expect_error=None, solving_data=None): assert (amount is None) != (data is None) from_balance_before = from_wallet.getbalances()["mine"]["trusted"] @@ -94,6 +96,8 @@ class WalletSendTest(BitcoinTestFramework): options["replaceable"] = replaceable if subtract_fee_from_outputs is not None: options["subtract_fee_from_outputs"] = subtract_fee_from_outputs + if solving_data is not None: + options["solving_data"] = solving_data if len(options.keys()) == 0: options = None @@ -476,6 +480,47 @@ class WalletSendTest(BitcoinTestFramework): res = self.test_send(from_wallet=w5, to_wallet=w0, amount=1, include_unsafe=True) assert res["complete"] + self.log.info("External outputs") + eckey = ECKey() + eckey.generate() + privkey = bytes_to_wif(eckey.get_bytes()) + + self.nodes[1].createwallet("extsend") + ext_wallet = self.nodes[1].get_wallet_rpc("extsend") + self.nodes[1].createwallet("extfund") + ext_fund = self.nodes[1].get_wallet_rpc("extfund") + + # Make a weird but signable script. sh(pkh()) descriptor accomplishes this + desc = descsum_create("sh(pkh({}))".format(privkey)) + if self.options.descriptors: + res = ext_fund.importdescriptors([{"desc": desc, "timestamp": "now"}]) + else: + res = ext_fund.importmulti([{"desc": desc, "timestamp": "now"}]) + assert res[0]["success"] + addr = self.nodes[0].deriveaddresses(desc)[0] + addr_info = ext_fund.getaddressinfo(addr) + + self.nodes[0].sendtoaddress(addr, 10) + self.nodes[0].sendtoaddress(ext_wallet.getnewaddress(), 10) + self.generate(self.nodes[0], 6) + self.sync_all() + ext_utxo = ext_fund.listunspent(addresses=[addr])[0] + + # An external input without solving data should result in an error + self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, expect_error=(-4, "Insufficient funds")) + + # But funding should work when the solving data is provided + res = self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, solving_data={"pubkeys": [addr_info['pubkey']], "scripts": [addr_info["embedded"]["scriptPubKey"]]}) + signed = ext_wallet.walletprocesspsbt(res["psbt"]) + signed = ext_fund.walletprocesspsbt(res["psbt"]) + assert signed["complete"] + self.nodes[0].finalizepsbt(signed["psbt"]) + + res = self.test_send(from_wallet=ext_wallet, to_wallet=self.nodes[0], amount=15, inputs=[ext_utxo], add_inputs=True, psbt=True, include_watching=True, solving_data={"descriptors": [desc]}) + signed = ext_wallet.walletprocesspsbt(res["psbt"]) + signed = ext_fund.walletprocesspsbt(res["psbt"]) + assert signed["complete"] + self.nodes[0].finalizepsbt(signed["psbt"]) if __name__ == '__main__': WalletSendTest().main() diff --git a/test/functional/wallet_signer.py b/test/functional/wallet_signer.py index 7b77755d64..c6c1cc8784 100755 --- a/test/functional/wallet_signer.py +++ b/test/functional/wallet_signer.py @@ -27,6 +27,9 @@ class WalletSignerTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + # The experimental syscall sandbox feature (-sandbox) is not compatible with -signer (which + # invokes execve). + self.disable_syscall_sandbox = True self.extra_args = [ [], diff --git a/test/functional/wallet_taproot.py b/test/functional/wallet_taproot.py index 4f84dbd125..80c125df97 100755 --- a/test/functional/wallet_taproot.py +++ b/test/functional/wallet_taproot.py @@ -185,7 +185,7 @@ class WalletTaprootTest(BitcoinTestFramework): def setup_network(self): self.setup_nodes() - def init_wallet(self, i): + def init_wallet(self, *, node): pass @staticmethod diff --git a/test/functional/wallet_transactiontime_rescan.py b/test/functional/wallet_transactiontime_rescan.py index 78859e6131..afa5139da7 100755 --- a/test/functional/wallet_transactiontime_rescan.py +++ b/test/functional/wallet_transactiontime_rescan.py @@ -63,7 +63,7 @@ class TransactionTimeRescanTest(BitcoinTestFramework): # generate some btc to create transactions and check blockcount initial_mine = COINBASE_MATURITY + 1 - minernode.generatetoaddress(initial_mine, m1) + self.generatetoaddress(minernode, initial_mine, m1) assert_equal(minernode.getblockcount(), initial_mine + 200) # synchronize nodes and time @@ -76,7 +76,7 @@ class TransactionTimeRescanTest(BitcoinTestFramework): miner_wallet.sendtoaddress(wo1, 10) # generate blocks and check blockcount - minernode.generatetoaddress(COINBASE_MATURITY, m1) + self.generatetoaddress(minernode, COINBASE_MATURITY, m1) assert_equal(minernode.getblockcount(), initial_mine + 300) # synchronize nodes and time @@ -89,7 +89,7 @@ class TransactionTimeRescanTest(BitcoinTestFramework): miner_wallet.sendtoaddress(wo2, 5) # generate blocks and check blockcount - minernode.generatetoaddress(COINBASE_MATURITY, m1) + self.generatetoaddress(minernode, COINBASE_MATURITY, m1) assert_equal(minernode.getblockcount(), initial_mine + 400) # synchronize nodes and time @@ -102,7 +102,7 @@ class TransactionTimeRescanTest(BitcoinTestFramework): miner_wallet.sendtoaddress(wo3, 1) # generate more blocks and check blockcount - minernode.generatetoaddress(COINBASE_MATURITY, m1) + self.generatetoaddress(minernode, COINBASE_MATURITY, m1) assert_equal(minernode.getblockcount(), initial_mine + 500) self.log.info('Check user\'s final balance and transaction count') diff --git a/test/functional/wallet_upgradewallet.py b/test/functional/wallet_upgradewallet.py index ed98db55c9..5800880830 100755 --- a/test/functional/wallet_upgradewallet.py +++ b/test/functional/wallet_upgradewallet.py @@ -234,18 +234,13 @@ class UpgradeWalletTest(BitcoinTestFramework): assert_equal(1, hd_chain_version) seed_id = bytearray(seed_id) seed_id.reverse() - old_kvs = new_kvs - # First 2 keys should still be non-HD - for i in range(0, 2): - info = wallet.getaddressinfo(wallet.getnewaddress()) - assert 'hdkeypath' not in info - assert 'hdseedid' not in info - # Next key should be HD + + # New keys (including change) should be HD (the two old keys have been flushed) info = wallet.getaddressinfo(wallet.getnewaddress()) assert_equal(seed_id.hex(), info['hdseedid']) assert_equal('m/0\'/0\'/0\'', info['hdkeypath']) prev_seed_id = info['hdseedid'] - # Change key should be the same keypool + # Change key should be HD and from the same keypool info = wallet.getaddressinfo(wallet.getrawchangeaddress()) assert_equal(prev_seed_id, info['hdseedid']) assert_equal('m/0\'/0\'/1\'', info['hdkeypath']) @@ -291,14 +286,7 @@ class UpgradeWalletTest(BitcoinTestFramework): hd_chain_version, external_counter, seed_id, internal_counter = struct.unpack('<iI20sI', hd_chain) assert_equal(2, hd_chain_version) assert_equal(2, internal_counter) - # Drain the keypool by fetching one external key and one change key. Should still be the same keypool - info = wallet.getaddressinfo(wallet.getnewaddress()) - assert 'hdseedid' not in info - assert 'hdkeypath' not in info - info = wallet.getaddressinfo(wallet.getrawchangeaddress()) - assert 'hdseedid' not in info - assert 'hdkeypath' not in info - # The next addresses are HD and should be on different HD chains + # The next addresses are HD and should be on different HD chains (the one remaining key in each pool should have been flushed) info = wallet.getaddressinfo(wallet.getnewaddress()) ext_id = info['hdseedid'] assert_equal('m/0\'/0\'/0\'', info['hdkeypath']) diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index fcc4883d0b..b119cffec8 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -37,23 +37,15 @@ export LC_ALL=C # See https://doc.qt.io/qt-5/qcoreapplication.html#locale-settings and # https://stackoverflow.com/a/34878283 for more details. +# TODO: Reduce KNOWN_VIOLATIONS by replacing uses of locale dependent stoul/strtol with locale +# independent ToIntegral<T>(...) or the ParseInt*() functions. +# TODO: Reduce KNOWN_VIOLATIONS by replacing uses of locale dependent snprintf with strprintf. KNOWN_VIOLATIONS=( - "src/bitcoin-tx.cpp.*stoul" - "src/dbwrapper.cpp.*stoul" "src/dbwrapper.cpp:.*vsnprintf" - "src/node/blockstorage.cpp:.*atoi" - "src/qt/rpcconsole.cpp:.*atoi" - "src/rest.cpp:.*strtol" "src/test/dbwrapper_tests.cpp:.*snprintf" "src/test/fuzz/locale.cpp" - "src/test/fuzz/parse_numbers.cpp:.*atoi" "src/test/fuzz/string.cpp" - "src/torcontrol.cpp:.*atoi" "src/torcontrol.cpp:.*strtol" - "src/util/strencodings.cpp:.*atoi" - "src/util/strencodings.cpp:.*strtoll" - "src/util/strencodings.h:.*atoi" - "src/util/system.cpp:.*atoi" ) REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(crypto/ctaes/|leveldb/|secp256k1/|tinyformat.h|univalue/)" diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh index c448fa6f9a..3d22407fd1 100755 --- a/test/lint/lint-python.sh +++ b/test/lint/lint-python.sh @@ -102,7 +102,7 @@ if ! PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; e EXIT_CODE=1 fi -if ! mypy --ignore-missing-imports --show-error-codes $(git ls-files "test/functional/*.py" "contrib/devtools/*.py"); then +if ! mypy --show-error-codes $(git ls-files "test/functional/*.py" "contrib/devtools/*.py"); then EXIT_CODE=1 fi diff --git a/test/sanitizer_suppressions/ubsan b/test/sanitizer_suppressions/ubsan index b52e105a33..1d608b9ec1 100644 --- a/test/sanitizer_suppressions/ubsan +++ b/test/sanitizer_suppressions/ubsan @@ -22,7 +22,7 @@ unsigned-integer-overflow:arith_uint256.h unsigned-integer-overflow:basic_string.h unsigned-integer-overflow:bench/bench.h unsigned-integer-overflow:bitcoin-tx.cpp -unsigned-integer-overflow:bloom.cpp +unsigned-integer-overflow:common/bloom.cpp unsigned-integer-overflow:chain.cpp unsigned-integer-overflow:chain.h unsigned-integer-overflow:coded_stream.h @@ -48,7 +48,7 @@ implicit-integer-sign-change:*/new_allocator.h implicit-integer-sign-change:addrman.h implicit-integer-sign-change:arith_uint256.cpp implicit-integer-sign-change:bech32.cpp -implicit-integer-sign-change:bloom.cpp +implicit-integer-sign-change:common/bloom.cpp implicit-integer-sign-change:chain.cpp implicit-integer-sign-change:chain.h implicit-integer-sign-change:coins.h diff --git a/test/util/data/bitcoin-util-test.json b/test/util/data/bitcoin-util-test.json index a648c0287a..cca5732aa1 100644 --- a/test/util/data/bitcoin-util-test.json +++ b/test/util/data/bitcoin-util-test.json @@ -295,6 +295,12 @@ "description": "Create a new transaction with a single output script (OP_DROP) in a P2SH, wrapped in a P2SH (output as json)" }, { "exec": "./bitcoin-tx", + "args": ["-create", "outscript=0:999999999999999999999999999999"], + "return_code": 1, + "error_txt": "error: script parse error: decimal numeric value only allowed in the range -0xFFFFFFFF...0xFFFFFFFF", + "description": "Try to parse an output script with a decimal number above the allowed range" + }, + { "exec": "./bitcoin-tx", "args": ["-create", "outscript=0:9999999999"], "return_code": 1, "error_txt": "error: script parse error: decimal numeric value only allowed in the range -0xFFFFFFFF...0xFFFFFFFF", @@ -512,6 +518,30 @@ { "exec": "./bitcoin-tx", "args": ["-create", + "in=5897de6bd6027a475eadd57019d4e6872c396d0716c4875a5f1a6fcfdf385c1f:0:11aa"], + "return_code": 1, + "error_txt": "error: invalid TX sequence id '11aa'", + "description": "Try to parse a sequence number outside the allowed range" + }, + { "exec": "./bitcoin-tx", + "args": + ["-create", + "in=5897de6bd6027a475eadd57019d4e6872c396d0716c4875a5f1a6fcfdf385c1f:0:-1"], + "return_code": 1, + "error_txt": "error: invalid TX sequence id '-1'", + "description": "Try to parse a sequence number outside the allowed range" + }, + { "exec": "./bitcoin-tx", + "args": + ["-create", + "in=5897de6bd6027a475eadd57019d4e6872c396d0716c4875a5f1a6fcfdf385c1f:0:4294967296"], + "return_code": 1, + "error_txt": "error: invalid TX sequence id '4294967296'", + "description": "Try to parse a sequence number outside the allowed range" + }, + { "exec": "./bitcoin-tx", + "args": + ["-create", "in=5897de6bd6027a475eadd57019d4e6872c396d0716c4875a5f1a6fcfdf385c1f:0:4294967293", "outaddr=0.18:13tuJJDR2RgArmgfv6JScSdreahzgc4T6o"], "output_cmp": "txcreatedata_seq0.hex", @@ -519,6 +549,14 @@ }, { "exec": "./bitcoin-tx", "args": + ["-create", + "in=5897de6bd6027a475eadd57019d4e6872c396d0716c4875a5f1a6fcfdf385c1f:0: 4294967293 ", + "outaddr=0.18:13tuJJDR2RgArmgfv6JScSdreahzgc4T6o"], + "output_cmp": "txcreatedata_seq0.hex", + "description": "Creates a new transaction with one input with sequence number (+whitespace) and one address output" + }, + { "exec": "./bitcoin-tx", + "args": ["-json", "-create", "in=5897de6bd6027a475eadd57019d4e6872c396d0716c4875a5f1a6fcfdf385c1f:0:4294967293", @@ -542,14 +580,26 @@ "description": "Adds a new input with sequence number to a transaction (output in json)" }, { "exec": "./bitcoin-tx", + "args": ["-create", "outmultisig=1:-2:3:02a5:021:02df", "nversion=1"], + "return_code": 1, + "error_txt": "error: invalid multisig required number '-2'", + "description": "Try to parse a multisig number outside the allowed range" + }, + { "exec": "./bitcoin-tx", + "args": ["-create", "outmultisig=1:2:3a:02a5:021:02df", "nversion=1"], + "return_code": 1, + "error_txt": "error: invalid multisig total number '3a'", + "description": "Try to parse a multisig number outside the allowed range" + }, + { "exec": "./bitcoin-tx", "args": ["-create", "outmultisig=1:2:3:02a5613bd857b7048924264d1e70e08fb2a7e6527d32b7ab1bb993ac59964ff397:021ac43c7ff740014c3b33737ede99c967e4764553d1b2b83db77c83b8715fa72d:02df2089105c77f266fa11a9d33f05c735234075f2e8780824c6b709415f9fb485", "nversion=1"], "output_cmp": "txcreatemultisig1.hex", "description": "Creates a new transaction with a single 2-of-3 multisig output" }, { "exec": "./bitcoin-tx", - "args": ["-json", "-create", "outmultisig=1:2:3:02a5613bd857b7048924264d1e70e08fb2a7e6527d32b7ab1bb993ac59964ff397:021ac43c7ff740014c3b33737ede99c967e4764553d1b2b83db77c83b8715fa72d:02df2089105c77f266fa11a9d33f05c735234075f2e8780824c6b709415f9fb485", "nversion=1"], + "args": ["-json", "-create", "outmultisig=1: 2 : 3 :02a5613bd857b7048924264d1e70e08fb2a7e6527d32b7ab1bb993ac59964ff397:021ac43c7ff740014c3b33737ede99c967e4764553d1b2b83db77c83b8715fa72d:02df2089105c77f266fa11a9d33f05c735234075f2e8780824c6b709415f9fb485", "nversion=1"], "output_cmp": "txcreatemultisig1.json", - "description": "Creates a new transaction with a single 2-of-3 multisig output (output in json)" + "description": "Creates a new transaction with a single 2-of-3 multisig output (with whitespace, output in json)" }, { "exec": "./bitcoin-tx", "args": ["-create", "outmultisig=1:2:3:02a5613bd857b7048924264d1e70e08fb2a7e6527d32b7ab1bb993ac59964ff397:021ac43c7ff740014c3b33737ede99c967e4764553d1b2b83db77c83b8715fa72d:02df2089105c77f266fa11a9d33f05c735234075f2e8780824c6b709415f9fb485:S", "nversion=1"], |