diff options
192 files changed, 3081 insertions, 1814 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b92e3c66d0..44521c1af3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -295,3 +295,16 @@ jobs: TEST_RUNNER_EXTRA: ${{ github.event_name != 'pull_request' && '--extended' || '' }} shell: cmd run: py -3 test\functional\test_runner.py --jobs %NUMBER_OF_PROCESSORS% --ci --quiet --tmpdirprefix=%RUNNER_TEMP% --combinedlogslen=99999999 --timeout-factor=%TEST_RUNNER_TIMEOUT_FACTOR% %TEST_RUNNER_EXTRA% + + - name: Clone fuzz corpus + run: | + git clone --depth=1 https://github.com/bitcoin-core/qa-assets "$env:RUNNER_TEMP\qa-assets" + Set-Location "$env:RUNNER_TEMP\qa-assets" + Write-Host "Using qa-assets repo from commit ..." + git log -1 + + - name: Run fuzz binaries + env: + BITCOINFUZZ: "${{ github.workspace}}\\src\\fuzz.exe" + shell: cmd + run: py -3 test\fuzz\test_runner.py --par %NUMBER_OF_PROCESSORS% --loglevel DEBUG %RUNNER_TEMP%\qa-assets\fuzz_seed_corpus diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0ac6db76ed..76a1c2754a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -418,11 +418,8 @@ Backporting Security and bug fixes can be backported from `master` to release branches. -If the backport is non-trivial, it may be appropriate to open an -additional PR to backport the change, but only after the original PR -has been merged. -Otherwise, backports will be done in batches and -the maintainers will use the proper `Needs backport (...)` labels +Maintainers will do backports in batches and +use the proper `Needs backport (...)` labels when needed (the original author does not need to worry about it). A backport should contain the following metadata in the commit body: diff --git a/Makefile.am b/Makefile.am index cd1509a0ff..e79dc4e21b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -126,7 +126,7 @@ $(OSX_ZIP): deploydir cd $(APP_DIST_DIR) && find . | sort | $(ZIP) -X@ $@ $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt: $(OSX_APP_BUILT) $(OSX_PACKAGING) - INSTALL_NAME_TOOL=$(INSTALL_NAME_TOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) + OTOOL=$(OTOOL) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) deploydir: $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Bitcoin-Qt endif !BUILD_DARWIN diff --git a/build_msvc/bitcoin.sln b/build_msvc/bitcoin.sln index 0931bf5dfe..9fd6395f59 100644 --- a/build_msvc/bitcoin.sln +++ b/build_msvc/bitcoin.sln @@ -48,7 +48,9 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libtest_util", "libtest_uti EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test_bitcoin-qt", "test_bitcoin-qt\test_bitcoin-qt.vcxproj", "{51201D5E-D939-4854-AE9D-008F03FF518E}" EndProject -Project("{542007E3-BE0D-4B0D-A6B0-AA8813E2558D}") = "libminisketch", "libminisketch\libminisketch.vcxproj", "{542007E3-BE0D-4B0D-A6B0-AA8813E2558D}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libminisketch", "libminisketch\libminisketch.vcxproj", "{542007E3-BE0D-4B0D-A6B0-AA8813E2558D}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fuzz", "fuzz\fuzz.vcxproj", "{AFCEE6C1-89FB-49AB-A694-BA580A59E2D8}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -152,6 +154,10 @@ Global {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Debug|x64.Build.0 = Debug|x64 {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Release|x64.ActiveCfg = Release|x64 {542007E3-BE0D-4B0D-A6B0-AA8813E2558D}.Release|x64.Build.0 = Release|x64 + {AFCEE6C1-89FB-49AB-A694-BA580A59E2D8}.Debug|x64.ActiveCfg = Debug|x64 + {AFCEE6C1-89FB-49AB-A694-BA580A59E2D8}.Debug|x64.Build.0 = Debug|x64 + {AFCEE6C1-89FB-49AB-A694-BA580A59E2D8}.Release|x64.ActiveCfg = Release|x64 + {AFCEE6C1-89FB-49AB-A694-BA580A59E2D8}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/build_msvc/bitcoind/bitcoind.vcxproj b/build_msvc/bitcoind/bitcoind.vcxproj index a93723d8be..63337ca6a7 100644 --- a/build_msvc/bitcoind/bitcoind.vcxproj +++ b/build_msvc/bitcoind/bitcoind.vcxproj @@ -80,7 +80,7 @@ <ReplaceInFile FilePath="$(ConfigIniOut)" Replace="@BUILD_BITCOIND_TRUE@" By=""></ReplaceInFile> <ReplaceInFile FilePath="$(ConfigIniOut)" - Replace="@ENABLE_FUZZ_BINARY_TRUE@" By="#"></ReplaceInFile> + Replace="@ENABLE_FUZZ_BINARY_TRUE@" By=""></ReplaceInFile> <ReplaceInFile FilePath="$(ConfigIniOut)" Replace="@ENABLE_ZMQ_TRUE@" By=""></ReplaceInFile> <ReplaceInFile FilePath="$(ConfigIniOut)" diff --git a/build_msvc/common.init.vcxproj.in b/build_msvc/common.init.vcxproj.in index 950cc37f0a..71fceb6c66 100644 --- a/build_msvc/common.init.vcxproj.in +++ b/build_msvc/common.init.vcxproj.in @@ -87,10 +87,10 @@ <ClCompile> <WarningLevel>Level3</WarningLevel> <PrecompiledHeader>NotUsing</PrecompiledHeader> - <AdditionalOptions>/utf-8 /Zc:__cplusplus /std:c++20 %(AdditionalOptions)</AdditionalOptions> + <AdditionalOptions>/utf-8 /Zc:preprocessor /Zc:__cplusplus /std:c++20 %(AdditionalOptions)</AdditionalOptions> <DisableSpecificWarnings>4018;4244;4267;4715;4805</DisableSpecificWarnings> <TreatWarningAsError>true</TreatWarningAsError> - <PreprocessorDefinitions>_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;SECP256K1_STATIC;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;SECP256K1_STATIC;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;PROVIDE_FUZZ_MAIN_FUNCTION;%(PreprocessorDefinitions)</PreprocessorDefinitions> <AdditionalIncludeDirectories>..\..\src;..\..\src\minisketch\include;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> </ClCompile> <Link> diff --git a/build_msvc/fuzz/fuzz.vcxproj b/build_msvc/fuzz/fuzz.vcxproj new file mode 100644 index 0000000000..d1311fe9d9 --- /dev/null +++ b/build_msvc/fuzz/fuzz.vcxproj @@ -0,0 +1,86 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <Import Project="..\common.init.vcxproj" /> + <PropertyGroup Label="Globals"> + <ProjectGuid>{AFCEE6C1-89FB-49AB-A694-BA580A59E2D8}</ProjectGuid> + </PropertyGroup> + <PropertyGroup Label="Configuration"> + <ConfigurationType>Application</ConfigurationType> + <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir> + </PropertyGroup> + <ItemGroup> + <!-- TODO: Fix the code and remove miniscript.cpp exclusion. --> + <ClCompile Include="..\..\src\test\fuzz\*.cpp" Exclude="..\..\src\test\fuzz\miniscript.cpp" /> + <ClCompile Include="..\..\src\test\fuzz\util\descriptor.cpp"> + <ObjectFileName>$(IntDir)test_fuzz_util_descriptor.obj</ObjectFileName> + </ClCompile> + <ClCompile Include="..\..\src\test\fuzz\util\mempool.cpp"> + <ObjectFileName>$(IntDir)test_fuzz_util_mempool.obj</ObjectFileName> + </ClCompile> + <ClCompile Include="..\..\src\test\fuzz\util\net.cpp"> + <ObjectFileName>$(IntDir)test_fuzz_util_net.obj</ObjectFileName> + </ClCompile> + <ClCompile Include="..\..\src\wallet\test\fuzz\coincontrol.cpp"> + <ObjectFileName>$(IntDir)wallet_test_fuzz_coincontrol.obj</ObjectFileName> + </ClCompile> + <ClCompile Include="..\..\src\wallet\test\fuzz\coinselection.cpp"> + <ObjectFileName>$(IntDir)wallet_test_fuzz_coinselection.obj</ObjectFileName> + </ClCompile> + <ClCompile Include="..\..\src\wallet\test\fuzz\fees.cpp"> + <ObjectFileName>$(IntDir)wallet_test_fuzz_fees.obj</ObjectFileName> + </ClCompile> + <ClCompile Include="..\..\src\wallet\test\fuzz\notifications.cpp"> + <ObjectFileName>$(IntDir)wallet_test_fuzz_notifications.obj</ObjectFileName> + </ClCompile> + <ClCompile Include="..\..\src\wallet\test\fuzz\parse_iso8601.cpp"> + <ObjectFileName>$(IntDir)wallet_test_fuzz_parse_iso8601.obj</ObjectFileName> + </ClCompile> + <ClCompile Include="..\..\src\wallet\test\fuzz\scriptpubkeyman.cpp"> + <ObjectFileName>$(IntDir)wallet_test_fuzz_scriptpubkeyman.obj</ObjectFileName> + </ClCompile> + </ItemGroup> + <ItemGroup> + <ProjectReference Include="..\libminisketch\libminisketch.vcxproj"> + <Project>{542007e3-be0d-4b0d-a6b0-aa8813e2558d}</Project> + </ProjectReference> + <ProjectReference Include="..\libbitcoin_consensus\libbitcoin_consensus.vcxproj"> + <Project>{2b384fa8-9ee1-4544-93cb-0d733c25e8ce}</Project> + </ProjectReference> + <ProjectReference Include="..\libbitcoin_cli\libbitcoin_cli.vcxproj"> + <Project>{0667528c-d734-4009-adf9-c0d6c4a5a5a6}</Project> + </ProjectReference> + <ProjectReference Include="..\libbitcoin_common\libbitcoin_common.vcxproj"> + <Project>{7c87e378-df58-482e-aa2f-1bc129bc19ce}</Project> + </ProjectReference> + <ProjectReference Include="..\libbitcoin_crypto\libbitcoin_crypto.vcxproj"> + <Project>{6190199c-6cf4-4dad-bfbd-93fa72a760c1}</Project> + </ProjectReference> + <ProjectReference Include="..\libbitcoin_node\libbitcoin_node.vcxproj"> + <Project>{460fee33-1fe1-483f-b3bf-931ff8e969a5}</Project> + </ProjectReference> + <ProjectReference Include="..\libbitcoin_util\libbitcoin_util.vcxproj"> + <Project>{b53a5535-ee9d-4c6f-9a26-f79ee3bc3754}</Project> + </ProjectReference> + <ProjectReference Include="..\libbitcoin_wallet\libbitcoin_wallet.vcxproj"> + <Project>{93b86837-b543-48a5-a89b-7c87abb77df2}</Project> + </ProjectReference> + <ProjectReference Include="..\libbitcoin_zmq\libbitcoin_zmq.vcxproj"> + <Project>{792d487f-f14c-49fc-a9de-3fc150f31c3f}</Project> + </ProjectReference> + <ProjectReference Include="..\libtest_util\libtest_util.vcxproj"> + <Project>{1e065f03-3566-47d0-8fa9-daa72b084e7d}</Project> + </ProjectReference> + <ProjectReference Include="..\libunivalue\libunivalue.vcxproj"> + <Project>{5724ba7d-a09a-4ba8-800b-c4c1561b3d69}</Project> + </ProjectReference> + <ProjectReference Include="..\libsecp256k1\libsecp256k1.vcxproj"> + <Project>{bb493552-3b8c-4a8c-bf69-a6e7a51d2ea6}</Project> + </ProjectReference> + <ProjectReference Include="..\libleveldb\libleveldb.vcxproj"> + <Project>{18430fef-6b61-4c53-b396-718e02850f1b}</Project> + </ProjectReference> + </ItemGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> + <Import Project="..\common.vcxproj" /> +</Project> diff --git a/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in b/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in index 482e4333f7..b47d62b295 100644 --- a/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in +++ b/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in @@ -8,7 +8,6 @@ <ConfigurationType>StaticLibrary</ConfigurationType> </PropertyGroup> <ItemGroup> - <ClCompile Include="..\..\src\common\url.cpp" /> @SOURCE_FILES@ </ItemGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> diff --git a/build_msvc/test_bitcoin/test_bitcoin.vcxproj b/build_msvc/test_bitcoin/test_bitcoin.vcxproj index 0ae3819e50..b5aa58057f 100644 --- a/build_msvc/test_bitcoin/test_bitcoin.vcxproj +++ b/build_msvc/test_bitcoin/test_bitcoin.vcxproj @@ -59,11 +59,6 @@ <Project>{18430fef-6b61-4c53-b396-718e02850f1b}</Project> </ProjectReference> </ItemGroup> - <ItemDefinitionGroup> - <ClCompile> - <DisableSpecificWarnings>4018;4244;4267;4703;4715;4805</DisableSpecificWarnings> - </ClCompile> - </ItemDefinitionGroup> <Target Name="RawBenchHeaderGen" BeforeTargets="PrepareForBuild"> <PropertyGroup> <ErrorText>There was an error executing the JSON test header generation task.</ErrorText> diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh index 00a4d781c2..f6463438d3 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -14,5 +14,5 @@ export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" export TEST_RUNNER_EXTRA="--v2transport" export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' \ -CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE'" +CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS='-Wno-error=documentation'" export BITCOIND=bitcoin-node # Used in functional tests diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 36a172e42c..f1c358082d 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -17,7 +17,8 @@ export PACKAGES="ninja-build" # BDB generates false-positives and will be removed in future export DEP_OPTS="DEBUG=1 NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE'" +# _FORTIFY_SOURCE is not compatible with MSAN. +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE -U_FORTIFY_SOURCE'" export USE_MEMORY_SANITIZER="true" export RUN_UNIT_TESTS="false" export RUN_FUNCTIONAL_TESTS="false" diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index 431e4aba49..dd465cac2e 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -17,7 +17,8 @@ export PACKAGES="ninja-build" # BDB generates false-positives and will be removed in future export DEP_OPTS="DEBUG=1 NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening" +# _FORTIFY_SOURCE is not compatible with MSAN. +export BITCOIN_CONFIG="--with-sanitizers=memory CPPFLAGS='-U_FORTIFY_SOURCE'" export USE_MEMORY_SANITIZER="true" export RUN_FUNCTIONAL_TESTS="false" export CCACHE_MAXSIZE=250M diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh index 6f0b9cc285..6425120afb 100755 --- a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh +++ b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh @@ -8,8 +8,8 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" -# Use minimum supported python3.9 (or best-effort 3.10) and clang-14, see doc/dependencies.md -export PACKAGES="python3-zmq clang-14 llvm-14 libc++abi-14-dev libc++-14-dev" -export DEP_OPTS="NO_WALLET=1 CC=clang-14 CXX='clang++-14 -stdlib=libc++'" +# Use minimum supported python3.9 (or best-effort 3.10) and clang-15, see doc/dependencies.md +export PACKAGES="python3-zmq clang-15 llvm-15 libc++abi-15-dev libc++-15-dev" +export DEP_OPTS="NO_WALLET=1 CC=clang-15 CXX='clang++-15 -stdlib=libc++'" export GOAL="install" export BITCOIN_CONFIG="--enable-reduce-exports --enable-experimental-util-chainstate --with-experimental-kernel-lib --enable-shared" diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index 4c8658479b..5f422bbdb6 100755 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -16,5 +16,5 @@ export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=false export RUN_TIDY=true export GOAL="install" -export BITCOIN_CONFIG="CC=clang-${TIDY_LLVM_V} CXX=clang++-${TIDY_LLVM_V} --with-incompatible-bdb --disable-hardening CFLAGS='-O0 -g0' CXXFLAGS='-O0 -g0 -I/usr/lib/llvm-${TIDY_LLVM_V}/lib/clang/${TIDY_LLVM_V}/include'" +export BITCOIN_CONFIG="CC=clang-${TIDY_LLVM_V} CXX=clang++-${TIDY_LLVM_V} --with-incompatible-bdb --disable-hardening CFLAGS='-O0 -g0' CXXFLAGS='-O0 -g0'" export CCACHE_MAXSIZE=200M diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index bce5ee74b6..bf80d5d435 100755 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -16,4 +16,4 @@ export GOAL="deploy" # Prior to 11.0.0, the mingw-w64 headers were missing noreturn attributes, causing warnings when # cross-compiling for Windows. https://sourceforge.net/p/mingw-w64/bugs/306/ # https://github.com/mingw-w64/mingw-w64/commit/1690994f515910a31b9fb7c7bd3a52d4ba987abe -export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests CXXFLAGS=-Wno-return-type" +export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests CXXFLAGS='-Wno-return-type -Wno-error=maybe-uninitialized -Wno-error=array-bounds'" diff --git a/configure.ac b/configure.ac index febb352cdb..ee4400f21a 100644 --- a/configure.ac +++ b/configure.ac @@ -401,37 +401,35 @@ if test "$enable_werror" = "yes"; then ERROR_CXXFLAGS=$CXXFLAG_WERROR fi -if test "$CXXFLAGS_overridden" = "no"; then - AX_CHECK_COMPILE_FLAG([-Wall], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wextra], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wextra"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wgnu], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wgnu"], [], [$CXXFLAG_WERROR]) - dnl some compilers will ignore -Wformat-security without -Wformat, so just combine the two here. - AX_CHECK_COMPILE_FLAG([-Wformat -Wformat-security], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat -Wformat-security"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wvla], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wvla"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wshadow-field], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wshadow-field"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wthread-safety], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wloop-analysis], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wloop-analysis"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wredundant-decls], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wunused-member-function], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-member-function"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wdate-time], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdate-time"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wconditional-uninitialized], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wconditional-uninitialized"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wduplicated-branches], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-branches"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wduplicated-cond], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-cond"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wlogical-op], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wlogical-op"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Woverloaded-virtual], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Woverloaded-virtual"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wsuggest-override], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsuggest-override"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wimplicit-fallthrough"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wunreachable-code], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunreachable-code"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wdocumentation], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdocumentation"], [], [$CXXFLAG_WERROR]) - - dnl Some compilers (gcc) ignore unknown -Wno-* options, but warn about all - dnl unknown options if any other warning is produced. Test the -Wfoo case, and - dnl set the -Wno-foo case if it works. - AX_CHECK_COMPILE_FLAG([-Wunused-parameter], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-parameter"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wself-assign], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-self-assign"], [], [$CXXFLAG_WERROR]) - if test "$suppress_external_warnings" != "yes" ; then - AX_CHECK_COMPILE_FLAG([-Wdeprecated-copy], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-deprecated-copy"], [], [$CXXFLAG_WERROR]) - fi +AX_CHECK_COMPILE_FLAG([-Wall], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wall"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wextra], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wextra"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wgnu], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wgnu"], [], [$CXXFLAG_WERROR]) +dnl some compilers will ignore -Wformat-security without -Wformat, so just combine the two here. +AX_CHECK_COMPILE_FLAG([-Wformat -Wformat-security], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wformat -Wformat-security"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wvla], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wvla"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wshadow-field], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wshadow-field"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wthread-safety], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wloop-analysis], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wloop-analysis"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wredundant-decls], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wunused-member-function], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-member-function"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wdate-time], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdate-time"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wconditional-uninitialized], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wconditional-uninitialized"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wduplicated-branches], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-branches"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wduplicated-cond], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wduplicated-cond"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wlogical-op], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wlogical-op"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Woverloaded-virtual], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Woverloaded-virtual"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wsuggest-override], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wsuggest-override"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wimplicit-fallthrough], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wimplicit-fallthrough"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wunreachable-code], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunreachable-code"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wdocumentation], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdocumentation"], [], [$CXXFLAG_WERROR]) + +dnl Some compilers (gcc) ignore unknown -Wno-* options, but warn about all +dnl unknown options if any other warning is produced. Test the -Wfoo case, and +dnl set the -Wno-foo case if it works. +AX_CHECK_COMPILE_FLAG([-Wunused-parameter], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-unused-parameter"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-Wself-assign], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-self-assign"], [], [$CXXFLAG_WERROR]) +if test "$suppress_external_warnings" != "yes" ; then + AX_CHECK_COMPILE_FLAG([-Wdeprecated-copy], [NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Wno-deprecated-copy"], [], [$CXXFLAG_WERROR]) fi dnl Don't allow extended (non-ASCII) symbols in identifiers. This is easier for code review. @@ -696,6 +694,9 @@ case $host in TARGET_OS=darwin if test $cross_compiling != "yes"; then BUILD_OS=darwin + + AX_CHECK_LINK_FLAG([-Wl,-headerpad_max_install_names], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-headerpad_max_install_names"], [], [$LDFLAG_WERROR]) + AC_CHECK_PROG([BREW], [brew], [brew]) if test "$BREW" = "brew"; then dnl These Homebrew packages may be keg-only, meaning that they won't be found @@ -759,7 +760,6 @@ case $host in ;; *) AC_PATH_TOOL([DSYMUTIL], [dsymutil], [dsymutil]) - AC_PATH_TOOL([INSTALL_NAME_TOOL], [install_name_tool], [install_name_tool]) AC_PATH_TOOL([OTOOL], [otool], [otool]) AC_PATH_PROG([ZIP], [zip], [zip]) @@ -772,7 +772,6 @@ case $host in esac fi - AX_CHECK_LINK_FLAG([-Wl,-headerpad_max_install_names], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-headerpad_max_install_names"], [], [$LDFLAG_WERROR]) CORE_CPPFLAGS="$CORE_CPPFLAGS -DMAC_OSX -DOBJC_OLD_DISPATCH_PROTOTYPES=0" dnl ignore deprecated-declarations warnings coming from objcxx code @@ -1706,7 +1705,6 @@ AM_CONDITIONAL([ENABLE_QT_TESTS], [test "$BUILD_TEST_QT" = "yes"]) AM_CONDITIONAL([ENABLE_BENCH], [test "$use_bench" = "yes"]) AM_CONDITIONAL([USE_QRCODE], [test "$use_qr" = "yes"]) AM_CONDITIONAL([USE_LCOV], [test "$use_lcov" = "yes"]) -AM_CONDITIONAL([USE_LIBEVENT], [test "$use_libevent" = "yes"]) AM_CONDITIONAL([HARDEN], [test "$use_hardening" = "yes"]) AM_CONDITIONAL([ENABLE_SSE42], [test "$enable_sse42" = "yes"]) AM_CONDITIONAL([ENABLE_SSE41], [test "$enable_sse41" = "yes"]) diff --git a/contrib/devtools/gen-manpages.py b/contrib/devtools/gen-manpages.py index 2860e7db99..92acd9a403 100755 --- a/contrib/devtools/gen-manpages.py +++ b/contrib/devtools/gen-manpages.py @@ -62,6 +62,10 @@ with tempfile.NamedTemporaryFile('w', suffix='.h2m') as footer: # Copyright is the same for all binaries, so just use the first. footer.write('[COPYRIGHT]\n') footer.write('\n'.join(versions[0][2]).strip()) + # Create SEE ALSO section + footer.write('\n[SEE ALSO]\n') + footer.write(', '.join(s.rpartition('/')[2] + '(1)' for s in BINARIES)) + footer.write('\n') footer.flush() # Call the binaries through help2man to produce a manual page for each of them. diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 8f13c642d3..96818c7748 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -499,7 +499,6 @@ inspecting signatures in Mach-O binaries.") moreutils ;; Compression and archiving tar - bzip2 gzip xz ;; Build tools diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index ea599df3d8..d1df3062f8 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -66,9 +66,8 @@ building for macOS. Apple's version of `binutils` (called `cctools`) contains lots of functionality missing in the FSF's `binutils`. In addition to extra linker options for frameworks and sysroots, several -other tools are needed as well such as `install_name_tool`, `lipo`, and `nmedit`. These -do not build under Linux, so they have been patched to do so. The work here was used as -a starting point: [mingwandroid/toolchain4](https://github.com/mingwandroid/toolchain4). +other tools are needed as well. These do not build under Linux, so they have been patched to +do so. The work here was used as a starting point: [mingwandroid/toolchain4](https://github.com/mingwandroid/toolchain4). In order to build a working toolchain, the following source packages are needed from Apple: `cctools`, `dyld`, and `ld64`. diff --git a/depends/Makefile b/depends/Makefile index 88aae7ad81..005d9696fb 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -234,7 +234,6 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ -e 's|@NM@|$(host_NM)|' \ -e 's|@STRIP@|$(host_STRIP)|' \ -e 's|@OTOOL@|$(host_OTOOL)|' \ - -e 's|@INSTALL_NAME_TOOL@|$(host_INSTALL_NAME_TOOL)|' \ -e 's|@DSYMUTIL@|$(host_DSYMUTIL)|' \ -e 's|@build_os@|$(build_os)|' \ -e 's|@host_os@|$(host_os)|' \ diff --git a/depends/README.md b/depends/README.md index a8dfc83e3b..10e0985cf4 100644 --- a/depends/README.md +++ b/depends/README.md @@ -85,6 +85,10 @@ For linux S390X cross compilation: sudo apt-get install g++-s390x-linux-gnu binutils-s390x-linux-gnu +### Install the required dependencies: FreeBSD + + pkg install bash + ### Install the required dependencies: OpenBSD pkg_add bash gtar diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index a5f07643de..554bfd2c3e 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -5,7 +5,6 @@ build_darwin_RANLIB:=$(shell xcrun -f ranlib) build_darwin_STRIP:=$(shell xcrun -f strip) build_darwin_OTOOL:=$(shell xcrun -f otool) build_darwin_NM:=$(shell xcrun -f nm) -build_darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool) build_darwin_DSYMUTIL:=$(shell xcrun -f dsymutil) build_darwin_SHA256SUM=shasum -a 256 build_darwin_DOWNLOAD=curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o @@ -18,7 +17,6 @@ darwin_RANLIB:=$(shell xcrun -f ranlib) darwin_STRIP:=$(shell xcrun -f strip) darwin_OTOOL:=$(shell xcrun -f otool) darwin_NM:=$(shell xcrun -f nm) -darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool) darwin_DSYMUTIL:=$(shell xcrun -f dsymutil) darwin_native_binutils= darwin_native_toolchain= diff --git a/depends/builders/default.mk b/depends/builders/default.mk index 806d6e7c50..50869cd8a2 100644 --- a/depends/builders/default.mk +++ b/depends/builders/default.mk @@ -12,7 +12,7 @@ build_$(build_os)_$1 ?= $$(default_build_$1) build_$(build_arch)_$(build_os)_$1 ?= $$(build_$(build_os)_$1) build_$1=$$(build_$(build_arch)_$(build_os)_$1) endef -$(foreach var,CC CXX AR TAR RANLIB NM STRIP SHA256SUM DOWNLOAD OTOOL INSTALL_NAME_TOOL DSYMUTIL TOUCH,$(eval $(call add_build_tool_func,$(var)))) +$(foreach var,CC CXX AR TAR RANLIB NM STRIP SHA256SUM DOWNLOAD OTOOL DSYMUTIL TOUCH,$(eval $(call add_build_tool_func,$(var)))) define add_build_flags_func build_$(build_arch)_$(build_os)_$1 += $(build_$(build_os)_$1) build_$1=$$(build_$(build_arch)_$(build_os)_$1) diff --git a/depends/config.site.in b/depends/config.site.in index 29b2a67ed0..81975f02b9 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -123,11 +123,6 @@ if test "@host_os@" = darwin; then ac_cv_path_OTOOL="${OTOOL}" fi - if test -n "@INSTALL_NAME_TOOL@"; then - INSTALL_NAME_TOOL="@INSTALL_NAME_TOOL@" - ac_cv_path_INSTALL_NAME_TOOL="${INSTALL_NAME_TOOL}" - fi - if test -n "@DSYMUTIL@"; then DSYMUTIL="@DSYMUTIL@" ac_cv_path_DSYMUTIL="${DSYMUTIL}" diff --git a/depends/description.md b/depends/description.md index 69ee5bd36c..fa345bfe85 100644 --- a/depends/description.md +++ b/depends/description.md @@ -11,7 +11,7 @@ on new hosts. ### No reliance on timestamps File presence is used to determine what needs to be built. This makes the -results distributable and easily digestable by automated builders. +results distributable and easily digestible by automated builders. ### Each build only has its specified dependencies available at build-time. diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 8beedcc98a..639259ace3 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -39,7 +39,7 @@ llvm_config_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-config") llvm_lib_dir=$(shell $(llvm_config_prog) --libdir) endif -cctools_TOOLS=AR RANLIB STRIP NM OTOOL INSTALL_NAME_TOOL DSYMUTIL +cctools_TOOLS=AR RANLIB STRIP NM OTOOL DSYMUTIL # Make-only lowercase function lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1)))))))))))))))))))))))))) diff --git a/depends/hosts/default.mk b/depends/hosts/default.mk index cf3c90441d..6a6cab6cc6 100644 --- a/depends/hosts/default.mk +++ b/depends/hosts/default.mk @@ -38,5 +38,5 @@ host_$1 = $$($(host_arch)_$(host_os)_$1) host_$(release_type)_$1 = $$($(host_arch)_$(host_os)_$(release_type)_$1) endef -$(foreach tool,CC CXX AR RANLIB STRIP NM OBJCOPY OTOOL INSTALL_NAME_TOOL DSYMUTIL,$(eval $(call add_host_tool_func,$(tool)))) +$(foreach tool,CC CXX AR RANLIB STRIP NM OBJCOPY OTOOL DSYMUTIL,$(eval $(call add_host_tool_func,$(tool)))) $(foreach flags,CFLAGS CXXFLAGS CPPFLAGS LDFLAGS, $(eval $(call add_host_flags_func,$(flags)))) diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk index ebc097d686..7f0389b30d 100644 --- a/depends/packages/boost.mk +++ b/depends/packages/boost.mk @@ -1,8 +1,8 @@ package=boost $(package)_version=1.81.0 $(package)_download_path=https://boostorg.jfrog.io/artifactory/main/release/$($(package)_version)/source/ -$(package)_file_name=boost_$(subst .,_,$($(package)_version)).tar.bz2 -$(package)_sha256_hash=71feeed900fbccca04a3b4f2f84a7c217186f28a940ed8b7ed4725986baf99fa +$(package)_file_name=boost_$(subst .,_,$($(package)_version)).tar.gz +$(package)_sha256_hash=205666dea9f6a7cfed87c7a6dfbeb52a2c1b9de55712c9c1a87735d7181452b6 define $(package)_stage_cmds mkdir -p $($(package)_staging_prefix_dir)/include && \ diff --git a/depends/packages/fontconfig.mk b/depends/packages/fontconfig.mk index 444acfe36d..6baaecc55a 100644 --- a/depends/packages/fontconfig.mk +++ b/depends/packages/fontconfig.mk @@ -1,8 +1,8 @@ package=fontconfig $(package)_version=2.12.6 $(package)_download_path=https://www.freedesktop.org/software/fontconfig/release/ -$(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=cf0c30807d08f6a28ab46c61b8dbd55c97d2f292cf88f3a07d3384687f31f017 +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=064b9ebf060c9e77011733ac9dc0e2ce92870b574cca2405e11f5353a683c334 $(package)_dependencies=freetype expat $(package)_patches=gperf_header_regen.patch diff --git a/depends/packages/libXau.mk b/depends/packages/libXau.mk index aeb14dcd6e..6bafc4f41a 100644 --- a/depends/packages/libXau.mk +++ b/depends/packages/libXau.mk @@ -1,8 +1,8 @@ package=libXau $(package)_version=1.0.9 $(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/ -$(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=ccf8cbf0dbf676faa2ea0a6d64bcc3b6746064722b606c8c52917ed00dcb73ec +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=1f123d8304b082ad63a9e89376400a3b1d4c29e67e3ea07b3f659cccca690eea $(package)_dependencies=xproto # When updating this package, check the default value of diff --git a/depends/packages/libnatpmp.mk b/depends/packages/libnatpmp.mk index caa809ec0b..5a573a18e7 100644 --- a/depends/packages/libnatpmp.mk +++ b/depends/packages/libnatpmp.mk @@ -1,26 +1,20 @@ package=libnatpmp -$(package)_version=07004b97cf691774efebe70404cf22201e4d330d +$(package)_version=f2433bec24ca3d3f22a8a7840728a3ac177f94ba $(package)_download_path=https://github.com/miniupnp/libnatpmp/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=9321953ceb39d07c25463e266e50d0ae7b64676bb3a986d932b18881ed94f1fb -$(package)_patches=no_libtool.patch +$(package)_sha256_hash=ef84979950dfb3556705b63c9cd6c95501b75e887fba466234b187f3c9029669 +$(package)_build_subdir=build -define $(package)_set_vars - $(package)_build_opts=CC="$($(package)_cc)" - $(package)_build_opts_mingw32=CPPFLAGS=-DNATPMP_STATICLIB - $(package)_build_env+=CFLAGS="$($(package)_cflags) $($(package)_cppflags)" AR="$($(package)_ar)" -endef - -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/no_libtool.patch +define $(package)_config_cmds + $($(package)_cmake) -S .. -B . endef define $(package)_build_cmds - $(MAKE) libnatpmp.a $($(package)_build_opts) + $(MAKE) natpmp endef define $(package)_stage_cmds - mkdir -p $($(package)_staging_prefix_dir)/include $($(package)_staging_prefix_dir)/lib &&\ - install *.h $($(package)_staging_prefix_dir)/include &&\ + mkdir -p $($(package)_staging_prefix_dir)/include $($(package)_staging_prefix_dir)/lib && \ + install ../natpmp.h ../natpmp_declspec.h $($(package)_staging_prefix_dir)/include && \ install libnatpmp.a $($(package)_staging_prefix_dir)/lib endef diff --git a/depends/packages/libxcb_util.mk b/depends/packages/libxcb_util.mk index 6e4c7359b2..dc4456f85c 100644 --- a/depends/packages/libxcb_util.mk +++ b/depends/packages/libxcb_util.mk @@ -1,8 +1,8 @@ package=libxcb_util $(package)_version=0.4.0 $(package)_download_path=https://xcb.freedesktop.org/dist -$(package)_file_name=xcb-util-$($(package)_version).tar.bz2 -$(package)_sha256_hash=46e49469cb3b594af1d33176cd7565def2be3fa8be4371d62271fabb5eae50e9 +$(package)_file_name=xcb-util-$($(package)_version).tar.gz +$(package)_sha256_hash=0ed0934e2ef4ddff53fcc70fc64fb16fe766cd41ee00330312e20a985fd927a7 $(package)_dependencies=libxcb define $(package)_set_vars diff --git a/depends/packages/libxcb_util_image.mk b/depends/packages/libxcb_util_image.mk index d12d67e8e8..2228250fec 100644 --- a/depends/packages/libxcb_util_image.mk +++ b/depends/packages/libxcb_util_image.mk @@ -1,8 +1,8 @@ package=libxcb_util_image $(package)_version=0.4.0 $(package)_download_path=https://xcb.freedesktop.org/dist -$(package)_file_name=xcb-util-image-$($(package)_version).tar.bz2 -$(package)_sha256_hash=2db96a37d78831d643538dd1b595d7d712e04bdccf8896a5e18ce0f398ea2ffc +$(package)_file_name=xcb-util-image-$($(package)_version).tar.gz +$(package)_sha256_hash=cb2c86190cf6216260b7357a57d9100811bb6f78c24576a3a5bfef6ad3740a42 $(package)_dependencies=libxcb libxcb_util define $(package)_set_vars diff --git a/depends/packages/libxcb_util_keysyms.mk b/depends/packages/libxcb_util_keysyms.mk index d4f72dedbe..56bc33d258 100644 --- a/depends/packages/libxcb_util_keysyms.mk +++ b/depends/packages/libxcb_util_keysyms.mk @@ -1,8 +1,8 @@ package=libxcb_util_keysyms $(package)_version=0.4.0 $(package)_download_path=https://xcb.freedesktop.org/dist -$(package)_file_name=xcb-util-keysyms-$($(package)_version).tar.bz2 -$(package)_sha256_hash=0ef8490ff1dede52b7de533158547f8b454b241aa3e4dcca369507f66f216dd9 +$(package)_file_name=xcb-util-keysyms-$($(package)_version).tar.gz +$(package)_sha256_hash=0807cf078fbe38489a41d755095c58239e1b67299f14460dec2ec811e96caa96 $(package)_dependencies=libxcb xproto define $(package)_set_vars diff --git a/depends/packages/libxcb_util_render.mk b/depends/packages/libxcb_util_render.mk index 28f1fb073c..ee2883feda 100644 --- a/depends/packages/libxcb_util_render.mk +++ b/depends/packages/libxcb_util_render.mk @@ -1,8 +1,8 @@ package=libxcb_util_render $(package)_version=0.3.9 $(package)_download_path=https://xcb.freedesktop.org/dist -$(package)_file_name=xcb-util-renderutil-$($(package)_version).tar.bz2 -$(package)_sha256_hash=c6e97e48fb1286d6394dddb1c1732f00227c70bd1bedb7d1acabefdd340bea5b +$(package)_file_name=xcb-util-renderutil-$($(package)_version).tar.gz +$(package)_sha256_hash=55eee797e3214fe39d0f3f4d9448cc53cffe06706d108824ea37bb79fcedcad5 $(package)_dependencies=libxcb define $(package)_set_vars diff --git a/depends/packages/libxcb_util_wm.mk b/depends/packages/libxcb_util_wm.mk index 3b905ba4ec..a68fd23f8a 100644 --- a/depends/packages/libxcb_util_wm.mk +++ b/depends/packages/libxcb_util_wm.mk @@ -1,8 +1,8 @@ package=libxcb_util_wm $(package)_version=0.4.1 $(package)_download_path=https://xcb.freedesktop.org/dist -$(package)_file_name=xcb-util-wm-$($(package)_version).tar.bz2 -$(package)_sha256_hash=28bf8179640eaa89276d2b0f1ce4285103d136be6c98262b6151aaee1d3c2a3f +$(package)_file_name=xcb-util-wm-$($(package)_version).tar.gz +$(package)_sha256_hash=038b39c4bdc04a792d62d163ba7908f4bb3373057208c07110be73c1b04b8334 $(package)_dependencies=libxcb define $(package)_set_vars diff --git a/depends/packages/miniupnpc.mk b/depends/packages/miniupnpc.mk index 5698a7cbb1..077e3bb1ee 100644 --- a/depends/packages/miniupnpc.mk +++ b/depends/packages/miniupnpc.mk @@ -1,30 +1,31 @@ package=miniupnpc -$(package)_version=2.2.2 +$(package)_version=2.2.7 $(package)_download_path=https://miniupnp.tuxfamily.org/files/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=888fb0976ba61518276fe1eda988589c700a3f2a69d71089260d75562afd3687 -$(package)_patches=dont_leak_info.patch respect_mingw_cflags.patch no_libtool.patch +$(package)_sha256_hash=b0c3a27056840fd0ec9328a5a9bac3dc5e0ec6d2e8733349cf577b0aa1e70ac1 +$(package)_patches=dont_leak_info.patch cmake_get_src_addr.patch fix_windows_snprintf.patch +$(package)_build_subdir=build -# Next time this package is updated, ensure that _WIN32_WINNT is still properly set. -# See discussion in https://github.com/bitcoin/bitcoin/pull/25964. define $(package)_set_vars -$(package)_build_opts=CC="$($(package)_cc)" -$(package)_build_opts_mingw32=-f Makefile.mingw CFLAGS="$($(package)_cflags) -D_WIN32_WINNT=0x0601" -$(package)_build_env+=CFLAGS="$($(package)_cflags) $($(package)_cppflags)" AR="$($(package)_ar)" +$(package)_config_opts = -DUPNPC_BUILD_SAMPLE=OFF -DUPNPC_BUILD_SHARED=OFF +$(package)_config_opts += -DUPNPC_BUILD_STATIC=ON -DUPNPC_BUILD_TESTS=OFF +$(package)_config_opts_mingw32 += -DMINIUPNPC_TARGET_WINDOWS_VERSION=0x0601 endef define $(package)_preprocess_cmds patch -p1 < $($(package)_patch_dir)/dont_leak_info.patch && \ - patch -p1 < $($(package)_patch_dir)/respect_mingw_cflags.patch && \ - patch -p1 < $($(package)_patch_dir)/no_libtool.patch + patch -p1 < $($(package)_patch_dir)/cmake_get_src_addr.patch && \ + patch -p1 < $($(package)_patch_dir)/fix_windows_snprintf.patch +endef + +define $(package)_config_cmds + $($(package)_cmake) -S .. -B . endef define $(package)_build_cmds - $(MAKE) libminiupnpc.a $($(package)_build_opts) + $(MAKE) endef define $(package)_stage_cmds - mkdir -p $($(package)_staging_prefix_dir)/include/miniupnpc $($(package)_staging_prefix_dir)/lib &&\ - install *.h $($(package)_staging_prefix_dir)/include/miniupnpc &&\ - install libminiupnpc.a $($(package)_staging_prefix_dir)/lib + cmake --install . --prefix $($(package)_staging_prefix_dir) endef diff --git a/depends/packages/qrencode.mk b/depends/packages/qrencode.mk index 4d852d833d..4216646063 100644 --- a/depends/packages/qrencode.mk +++ b/depends/packages/qrencode.mk @@ -1,8 +1,8 @@ package=qrencode $(package)_version=4.1.1 $(package)_download_path=https://fukuchi.org/works/qrencode/ -$(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=e455d9732f8041cf5b9c388e345a641fd15707860f928e94507b1961256a6923 +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=da448ed4f52aba6bcb0cd48cac0dd51b8692bccc4cd127431402fca6f8171e8e $(package)_patches=cmake_fixups.patch define $(package)_set_vars diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 0acf4cf565..d057b2d410 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -22,6 +22,7 @@ $(package)_patches += fix-macos-linker.patch $(package)_patches += memory_resource.patch $(package)_patches += utc_from_string_no_optimize.patch $(package)_patches += windows_lto.patch +$(package)_patches += zlib-timebits64.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) $(package)_qttranslations_sha256_hash=24d4c58bc2a40c0f44f59ee64af4192c7d0038c1e45af61646cfc5b65058f271 @@ -251,6 +252,7 @@ define $(package)_preprocess_cmds patch -p1 -i $($(package)_patch_dir)/utc_from_string_no_optimize.patch && \ patch -p1 -i $($(package)_patch_dir)/guix_cross_lib_path.patch && \ patch -p1 -i $($(package)_patch_dir)/windows_lto.patch && \ + patch -p1 -i $($(package)_patch_dir)/zlib-timebits64.patch && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ cp -f qtbase/mkspecs/macx-clang/qplatformdefs.h qtbase/mkspecs/macx-clang-linux/ &&\ cp -f $($(package)_patch_dir)/mac-qmake.conf qtbase/mkspecs/macx-clang-linux/qmake.conf && \ diff --git a/depends/packages/xproto.mk b/depends/packages/xproto.mk index 7a43c52faf..29c349a21b 100644 --- a/depends/packages/xproto.mk +++ b/depends/packages/xproto.mk @@ -1,8 +1,8 @@ package=xproto $(package)_version=7.0.31 $(package)_download_path=https://xorg.freedesktop.org/releases/individual/proto -$(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=c6f9747da0bd3a95f86b17fb8dd5e717c8f3ab7f0ece3ba1b247899ec1ef7747 +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=6d755eaae27b45c5cc75529a12855fed5de5969b367ed05003944cf901ed43c7 define $(package)_set_vars $(package)_config_opts=--without-fop --without-xmlto --without-xsltproc --disable-specs diff --git a/depends/patches/libnatpmp/no_libtool.patch b/depends/patches/libnatpmp/no_libtool.patch deleted file mode 100644 index 2b9f01f6eb..0000000000 --- a/depends/patches/libnatpmp/no_libtool.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff -ruN libnatpmp-07004b97cf691774efebe70404cf22201e4d330d/Makefile libnatpmp-07004b97cf691774efebe70404cf22201e4d330d.new/Makefile ---- libnatpmp-07004b97cf691774efebe70404cf22201e4d330d/Makefile 2022-07-05 07:49:50.000000000 +0000 -+++ libnatpmp-07004b97cf691774efebe70404cf22201e4d330d.new/Makefile 2024-01-23 20:59:35.674821779 +0000 -@@ -197,11 +197,7 @@ - $(CC) $(LDFLAGS) -o $@ $^ $(LDLIBS) - - $(STATICLIB): $(LIBOBJS) --ifneq (, $(findstring darwin, $(OS))) -- $(LIBTOOL) -static -o $@ $? --else - $(AR) crs $@ $? --endif - - $(SHAREDLIB): $(LIBOBJS) - ifneq (, $(findstring darwin, $(OS))) diff --git a/depends/patches/miniupnpc/cmake_get_src_addr.patch b/depends/patches/miniupnpc/cmake_get_src_addr.patch new file mode 100644 index 0000000000..bae1b738f3 --- /dev/null +++ b/depends/patches/miniupnpc/cmake_get_src_addr.patch @@ -0,0 +1,22 @@ +commit cb2026239c2a3aff393952ccb0ee1c448189402d +Author: fanquake <fanquake@gmail.com> +Date: Fri Mar 22 14:03:54 2024 +0000 + + build: add MINIUPNPC_GET_SRC_ADDR to CMake build + + This mirrors the autotools build. + + See https://github.com/miniupnp/miniupnp/pull/721. + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 1aa95a8..0cacf3e 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -31,6 +31,7 @@ endif () + if (NOT WIN32) + target_compile_definitions(miniupnpc-private INTERFACE + MINIUPNPC_SET_SOCKET_TIMEOUT ++ MINIUPNPC_GET_SRC_ADDR + _BSD_SOURCE _DEFAULT_SOURCE) + if (NOT APPLE AND NOT CMAKE_SYSTEM_NAME MATCHES ".*BSD" AND NOT CMAKE_SYSTEM_NAME STREQUAL "SunOS") + # add_definitions (-D_POSIX_C_SOURCE=200112L) diff --git a/depends/patches/miniupnpc/dont_leak_info.patch b/depends/patches/miniupnpc/dont_leak_info.patch index 512f9c50ea..95a09a26dc 100644 --- a/depends/patches/miniupnpc/dont_leak_info.patch +++ b/depends/patches/miniupnpc/dont_leak_info.patch @@ -1,31 +1,31 @@ -commit 8815452257437ba36607d0e2381c01142d1c7bb0 +commit 51f6dd991c29af66fb4f64c6feb2787cce23a1a7 Author: fanquake <fanquake@gmail.com> -Date: Thu Nov 19 10:51:19 2020 +0800 +Date: Mon Jan 8 11:21:40 2024 +0000 Don't leak OS and miniupnpc version info in User-Agent -diff --git a//minisoap.c b/minisoap.c -index 7860667..775580b 100644 ---- a/minisoap.c -+++ b/minisoap.c +diff --git a/src/minisoap.c b/src/minisoap.c +index 903ac5f..046e0ea 100644 +--- a/src/minisoap.c ++++ b/src/minisoap.c @@ -90,7 +90,7 @@ int soapPostSubmit(SOCKET fd, headerssize = snprintf(headerbuf, sizeof(headerbuf), "POST %s HTTP/%s\r\n" "Host: %s%s\r\n" -- "User-Agent: " OS_STRING ", " UPNP_VERSION_STRING ", MiniUPnPc/" MINIUPNPC_VERSION_STRING "\r\n" +- "User-Agent: " OS_STRING " " UPNP_VERSION_STRING " MiniUPnPc/" MINIUPNPC_VERSION_STRING "\r\n" + "User-Agent: " UPNP_VERSION_STRING "\r\n" "Content-Length: %d\r\n" + #if (UPNP_VERSION_MAJOR == 1) && (UPNP_VERSION_MINOR == 0) "Content-Type: text/xml\r\n" - "SOAPAction: \"%s\"\r\n" -diff --git a/miniwget.c b/miniwget.c -index d5b7970..05aeb9c 100644 ---- a/miniwget.c -+++ b/miniwget.c +diff --git a/src/miniwget.c b/src/miniwget.c +index e76a5e5..0cc36fe 100644 +--- a/src/miniwget.c ++++ b/src/miniwget.c @@ -444,7 +444,7 @@ miniwget3(const char * host, "GET %s HTTP/%s\r\n" "Host: %s:%d\r\n" "Connection: Close\r\n" -- "User-Agent: " OS_STRING ", " UPNP_VERSION_STRING ", MiniUPnPc/" MINIUPNPC_VERSION_STRING "\r\n" +- "User-Agent: " OS_STRING " " UPNP_VERSION_STRING " MiniUPnPc/" MINIUPNPC_VERSION_STRING "\r\n" + "User-Agent: " UPNP_VERSION_STRING "\r\n" "\r\n", diff --git a/depends/patches/miniupnpc/fix_windows_snprintf.patch b/depends/patches/miniupnpc/fix_windows_snprintf.patch new file mode 100644 index 0000000000..ff9e26231e --- /dev/null +++ b/depends/patches/miniupnpc/fix_windows_snprintf.patch @@ -0,0 +1,25 @@ +commit a1e9de80ab99b4c956a6a4e21d3e0de6f7a1014d +Author: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> +Date: Sat Apr 20 15:14:47 2024 +0100 + + Fix macro expression that guards `snprintf` for Windows + + Otherwise, the `snprintf` is still wrongly emulated for the following + cases: + - mingw-w64 6.0.0 or new with ucrt + - mingw-w64 8.0.0 or new with iso c ext + +--- a/src/win32_snprintf.h ++++ b/src/win32_snprintf.h +@@ -23,9 +23,9 @@ + (defined(_MSC_VER) && _MSC_VER < 1900) /* Visual Studio older than 2015 */ || \ + (defined(__MINGW32__) && !defined(__MINGW64_VERSION_MAJOR) && defined(__NO_ISOCEXT)) /* mingw32 without iso c ext */ || \ + (defined(__MINGW64_VERSION_MAJOR) && /* mingw-w64 not ... */ !( \ +- (defined (__USE_MINGW_ANSI_STDIO) && __USE_MINGW_ANSI_STDIO != 0)) /* ... with ansi stdio */ || \ ++ (defined (__USE_MINGW_ANSI_STDIO) && __USE_MINGW_ANSI_STDIO != 0) /* ... with ansi stdio */ || \ + (__MINGW64_VERSION_MAJOR >= 6 && defined(_UCRT)) /* ... at least 6.0.0 with ucrt */ || \ +- (__MINGW64_VERSION_MAJOR >= 8 && !defined(__NO_ISOCEXT)) /* ... at least 8.0.0 with iso c ext */ || \ ++ (__MINGW64_VERSION_MAJOR >= 8 && !defined(__NO_ISOCEXT))) /* ... at least 8.0.0 with iso c ext */ || \ + 0) || \ + 0) + diff --git a/depends/patches/miniupnpc/no_libtool.patch b/depends/patches/miniupnpc/no_libtool.patch deleted file mode 100644 index bb7d4a87ef..0000000000 --- a/depends/patches/miniupnpc/no_libtool.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff -ruN miniupnpc-2.2.2/Makefile miniupnpc-2.2.2.new/Makefile ---- miniupnpc-2.2.2/Makefile 2020-11-27 18:25:02.000000000 +0000 -+++ miniupnpc-2.2.2.new/Makefile 2024-01-23 20:58:08.387188527 +0000 -@@ -298,11 +298,7 @@ - makedepend -Y -- $(CFLAGS) -- $(SRCS) 2>/dev/null - - $(LIBRARY): $(LIBOBJS) --ifneq (, $(findstring darwin, $(OS))) -- $(LIBTOOL) -static -o $@ $? --else - $(AR) crs $@ $? --endif - - $(SHAREDLIBRARY): $(LIBOBJS) - ifneq (, $(findstring darwin, $(OS))) diff --git a/depends/patches/miniupnpc/respect_mingw_cflags.patch b/depends/patches/miniupnpc/respect_mingw_cflags.patch deleted file mode 100644 index a44580ddab..0000000000 --- a/depends/patches/miniupnpc/respect_mingw_cflags.patch +++ /dev/null @@ -1,23 +0,0 @@ -commit fec515a7ac9991a0ee91068fda046b54b191155e -Author: fanquake <fanquake@gmail.com> -Date: Wed Jul 27 15:52:37 2022 +0100 - - build: respect CFLAGS in makefile.mingw - - Similar to the other Makefile. - - Cherry-pick of https://github.com/miniupnp/miniupnp/pull/619. - -diff --git a/Makefile.mingw b/Makefile.mingw -index 2bff7bd..88430d2 100644 ---- a/Makefile.mingw -+++ b/Makefile.mingw -@@ -19,7 +19,7 @@ else - RM = rm -f - endif - #CFLAGS = -Wall -g -DDEBUG -D_WIN32_WINNT=0X501 --CFLAGS = -Wall -W -Wstrict-prototypes -Os -DNDEBUG -D_WIN32_WINNT=0X501 -+CFLAGS ?= -Wall -W -Wstrict-prototypes -Os -DNDEBUG -D_WIN32_WINNT=0X501 - LDLIBS = -lws2_32 -liphlpapi - # -lwsock32 - # -liphlpapi is needed for GetBestRoute() and GetIpAddrTable() diff --git a/depends/patches/qt/zlib-timebits64.patch b/depends/patches/qt/zlib-timebits64.patch new file mode 100644 index 0000000000..139c1dfa77 --- /dev/null +++ b/depends/patches/qt/zlib-timebits64.patch @@ -0,0 +1,31 @@ +From a566e156b3fa07b566ddbf6801b517a9dba04fa3 Mon Sep 17 00:00:00 2001 +From: Mark Adler <madler@alumni.caltech.edu> +Date: Sat, 29 Jul 2023 22:13:09 -0700 +Subject: [PATCH] Avoid compiler complaints if _TIME_BITS defined when building + zlib. + +zlib does not use time_t, so _TIME_BITS is irrelevant. However it +may be defined anyway as part of a sledgehammer indiscriminately +applied to all builds. + +From https://github.com/madler/zlib/commit/a566e156b3fa07b566ddbf6801b517a9dba04fa3.patch +--- + qtbase/src/3rdparty/zlib/src/gzguts.h | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +diff --git a/qtbase/src/3rdparty/zlib/src/gzguts.h b/qtbase/src/3rdparty/zlib/src/gzguts.h +index e23f831f5..f9375047e 100644 +--- a/qtbase/src/3rdparty/zlib/src/gzguts.h ++++ b/qtbase/src/3rdparty/zlib/src/gzguts.h +@@ -26,9 +26,8 @@ + # ifndef _LARGEFILE_SOURCE + # define _LARGEFILE_SOURCE 1 + # endif +-# ifdef _FILE_OFFSET_BITS +-# undef _FILE_OFFSET_BITS +-# endif ++# undef _FILE_OFFSET_BITS ++# undef _TIME_BITS + #endif + + #ifdef HAVE_HIDDEN diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index aa10e4a891..bf86a0ee4b 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -1,6 +1,6 @@ # FreeBSD Build Guide -**Updated for FreeBSD [12.3](https://www.freebsd.org/releases/12.3R/announce/)** +**Updated for FreeBSD [14.0](https://www.freebsd.org/releases/14.0R/announce/)** This guide describes how to build bitcoind, command-line utilities, and GUI on FreeBSD. @@ -64,10 +64,11 @@ sh/bash: export BDB_PREFIX=[path displayed above] #### GUI Dependencies ###### Qt5 -Bitcoin Core includes a GUI built with the cross-platform Qt Framework. To compile the GUI, we need to install `qt5`. Skip if you don't intend to use the GUI. +Bitcoin Core includes a GUI built with the cross-platform Qt Framework. To compile the GUI, we need to install the necessary parts of Qt. Skip if you don't intend to use the GUI. ```bash -pkg install qt5 +pkg install qt5-buildtools qt5-core qt5-gui qt5-linguisttools qt5-testlib qt5-widgets ``` + ###### libqrencode The GUI can encode addresses in a QR Code. To build in QR support for the GUI, install `libqrencode`. Skip if not using the GUI or don't want QR code functionality. diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index 7ed83853a8..264d5f2d08 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -1,6 +1,6 @@ # OpenBSD Build Guide -**Updated for OpenBSD [7.4](https://www.openbsd.org/74.html)** +**Updated for OpenBSD [7.5](https://www.openbsd.org/75.html)** This guide describes how to build bitcoind, command-line utilities, and GUI on OpenBSD. @@ -63,7 +63,7 @@ export BDB_PREFIX="/path/to/bitcoin/depends/x86_64-unknown-openbsd" Bitcoin Core includes a GUI built with the cross-platform Qt Framework. To compile the GUI, Qt 5 is required. ```bash -pkg_add qt5 +pkg_add qtbase qttools ``` ## Building Bitcoin Core diff --git a/doc/build-osx.md b/doc/build-osx.md index 5c3dc1ac7f..20c92ab7a4 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -51,6 +51,20 @@ To install, run the following from your terminal: brew install automake libtool boost pkg-config libevent ``` +For macOS 11 (Big Sur) and 12 (Monterey) you need to install a more recent version of llvm. + +``` bash +brew install llvm +``` + +And append the following to the configure commands below: + +``` bash +CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ +``` + +Try `llvm@17` if compilation fails with the default version of llvm. + ### 4. Clone Bitcoin repository `git` should already be installed by default on your system. diff --git a/doc/dependencies.md b/doc/dependencies.md index 1e48d225b0..2b9d9128d3 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -8,7 +8,7 @@ You can find installation instructions in the `build-*.md` file for your platfor | --- | --- | | [Autoconf](https://www.gnu.org/software/autoconf/) | [2.69](https://github.com/bitcoin/bitcoin/pull/17769) | | [Automake](https://www.gnu.org/software/automake/) | [1.13](https://github.com/bitcoin/bitcoin/pull/18290) | -| [Clang](https://clang.llvm.org) | [14.0](https://github.com/bitcoin/bitcoin/pull/29208) | +| [Clang](https://clang.llvm.org) | [15.0](https://github.com/bitcoin/bitcoin/pull/29165) | | [GCC](https://gcc.gnu.org) | [11.1](https://github.com/bitcoin/bitcoin/pull/29091) | | [Python](https://www.python.org) (scripts, tests) | [3.9](https://github.com/bitcoin/bitcoin/pull/28211) | | [systemtap](https://sourceware.org/systemtap/) ([tracing](tracing.md))| N/A | @@ -35,8 +35,8 @@ You can find installation instructions in the `build-*.md` file for your platfor ### Networking | Dependency | Releases | Version used | Minimum required | Runtime | | --- | --- | --- | --- | --- | -| [libnatpmp](../depends/packages/libnatpmp.mk) | [link](https://github.com/miniupnp/libnatpmp/) | commit [07004b9...](https://github.com/bitcoin/bitcoin/pull/25917) | | No | -| [MiniUPnPc](../depends/packages/miniupnpc.mk) | [link](https://miniupnp.tuxfamily.org/) | [2.2.2](https://github.com/bitcoin/bitcoin/pull/20421) | 2.1 | No | +| [libnatpmp](../depends/packages/libnatpmp.mk) | [link](https://github.com/miniupnp/libnatpmp/) | commit [f2433be...](https://github.com/bitcoin/bitcoin/pull/29708) | | No | +| [MiniUPnPc](../depends/packages/miniupnpc.mk) | [link](https://miniupnp.tuxfamily.org/) | [2.2.7](https://github.com/bitcoin/bitcoin/pull/29707) | 2.1 | No | ### Notifications | Dependency | Releases | Version used | Minimum required | Runtime | diff --git a/doc/external-signer.md b/doc/external-signer.md index de44cdd880..1468e852ef 100644 --- a/doc/external-signer.md +++ b/doc/external-signer.md @@ -150,6 +150,9 @@ Example, display the first native SegWit receive address on Testnet: The command MUST be able to figure out the address type from the descriptor. +The command MUST return an object containing `{"address": "[the address]"}`. +As a sanity check, for devices that support this, it SHOULD ask the device to derive the address. + If <descriptor> contains a master key fingerprint, the command MUST fail if it does not match the fingerprint known by the device. If <descriptor> contains an xpub, the command MUST fail if it does not match the xpub known by the device. diff --git a/doc/i2p.md b/doc/i2p.md index b6c07388b7..b769a74df4 100644 --- a/doc/i2p.md +++ b/doc/i2p.md @@ -79,8 +79,8 @@ one of the networks has issues. ## Persistent vs transient I2P addresses The first time Bitcoin Core connects to the I2P router, it automatically -generates a persistent I2P address and its corresponding private key by default -or if `-i2pacceptincoming=1` is set. The private key is saved in a file named +generates a persistent I2P address and its corresponding private key by default, +unless `-i2pacceptincoming=0` is set. The private key is saved in a file named `i2p_private_key` in the Bitcoin Core data directory. The persistent I2P address is used for making outbound connections and accepting inbound connections. diff --git a/doc/managing-wallets.md b/doc/managing-wallets.md index 22e006c963..b99d88877b 100644 --- a/doc/managing-wallets.md +++ b/doc/managing-wallets.md @@ -122,6 +122,22 @@ $ bitcoin-cli -rpcwallet="restored-wallet" getwalletinfo The restored wallet can also be loaded in the GUI via `File` ->`Open wallet`. +## Wallet Passphrase + +Understanding wallet security is crucial for safely storing your Bitcoin. A key aspect is the wallet passphrase, used for encryption. Let's explore its nuances, role, encryption process, and limitations. + +- **Not the Seed:** +The wallet passphrase and the seed are two separate components in wallet security. The seed, or HD seed, functions as a master key for deriving private and public keys in a hierarchical deterministic (HD) wallet. In contrast, the passphrase serves as an additional layer of security specifically designed to secure the private keys within the wallet. The passphrase serves as a safeguard, demanding an additional layer of authentication to access funds in the wallet. + +- **Protection Against Unauthorized Access:** +The passphrase serves as a protective measure, securing your funds in situations where an unauthorized user gains access to your unlocked computer or device while your wallet application is active. Without the passphrase, they would be unable to access your wallet's funds or execute transactions. However, it's essential to be aware that someone with access can potentially compromise the security of your passphrase by installing a keylogger. + +- **Doesn't Encrypt Metadata or Public Keys:** +It's important to note that the passphrase primarily secures the private keys and access to funds within the wallet. It does not encrypt metadata associated with transactions or public keys. Information about your transaction history and the public keys involved may still be visible. + +- **Risk of Fund Loss if Forgotten or Lost:** +If the wallet passphrase is too complex and is subsequently forgotten or lost, there is a risk of losing access to the funds permanently. A forgotten passphrase will result in the inability to unlock the wallet and access the funds. + ## Migrating Legacy Wallets to Descriptor Wallets Legacy wallets (traditional non-descriptor wallets) can be migrated to become Descriptor wallets diff --git a/doc/release-notes-27679.md b/doc/release-notes-27679.md new file mode 100644 index 0000000000..dbbb30428c --- /dev/null +++ b/doc/release-notes-27679.md @@ -0,0 +1,2 @@ +- unix socket paths are now accepted for `-zmqpubrawblock` and `-zmqpubrawtx` with +the format `-zmqpubrawtx=unix:/path/to/file`
\ No newline at end of file diff --git a/doc/release-process.md b/doc/release-process.md index 95ef08a26f..1e6d49100e 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -5,17 +5,17 @@ Release Process ### Before every release candidate -* Update translations see [translation_process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md#synchronising-translations). * Update release candidate version in `configure.ac` (`CLIENT_VERSION_RC`). * Update manpages (after rebuilding the binaries), see [gen-manpages.py](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-manpagespy). -* Update bitcoin.conf and commit, see [gen-bitcoin-conf.sh](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-bitcoin-confsh). +* Update bitcoin.conf and commit changes if they exist, see [gen-bitcoin-conf.sh](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-bitcoin-confsh). ### Before every major and minor release * Update [bips.md](bips.md) to account for changes since the last release. * Update version in `configure.ac` (don't forget to set `CLIENT_VERSION_RC` to `0`). * Update manpages (see previous section) -* Write release notes (see "Write the release notes" below). +* Write release notes (see "Write the release notes" below) in doc/release-notes.md. If necessary, + archive the previous release notes as doc/release-notes/release-notes-${VERSION}.md. ### Before every major release @@ -28,6 +28,7 @@ Release Process #### Before branch-off +* Update translations see [translation_process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md#synchronising-translations). * Update hardcoded [seeds](/contrib/seeds/README.md), see [this pull request](https://github.com/bitcoin/bitcoin/pull/27488) for an example. * Update the following variables in [`src/kernel/chainparams.cpp`](/src/kernel/chainparams.cpp) for mainnet, testnet, and signet: - `m_assumed_blockchain_size` and `m_assumed_chain_state_size` with the current size plus some overhead (see @@ -161,6 +162,8 @@ Then open a Pull Request to the [guix.sigs repository](https://github.com/bitcoi ### macOS codesigner only: Create detached macOS signatures (assuming [signapple](https://github.com/achow101/signapple/) is installed and up to date with master branch) +In the `guix-build-${VERSION}/output/x86_64-apple-darwin` and `guix-build-${VERSION}/output/arm64-apple-darwin` directories: + tar xf bitcoin-osx-unsigned.tar.gz ./detached-sig-create.sh /path/to/codesign.p12 Enter the keychain password and authorize the signature @@ -168,6 +171,8 @@ Then open a Pull Request to the [guix.sigs repository](https://github.com/bitcoi ### Windows codesigner only: Create detached Windows signatures +In the `guix-build-${VERSION}/output/x86_64-w64-mingw32` directory: + tar xf bitcoin-win-unsigned.tar.gz ./detached-sig-create.sh -key /path/to/codesign.key Enter the passphrase for the key when prompted @@ -175,18 +180,22 @@ Then open a Pull Request to the [guix.sigs repository](https://github.com/bitcoi ### Windows and macOS codesigners only: test code signatures It is advised to test that the code signature attaches properly prior to tagging by performing the `guix-codesign` step. -However if this is done, once the release has been tagged in the bitcoin-detached-sigs repo, the `guix-codesign` step must be performed again in order for the guix attestation to be valid when compared against the attestations of non-codesigner builds. +However if this is done, once the release has been tagged in the bitcoin-detached-sigs repo, the `guix-codesign` step must be performed again in order for the guix attestation to be valid when compared against the attestations of non-codesigner builds. The directories created by `guix-codesign` will need to be cleared prior to running `guix-codesign` again. ### Windows and macOS codesigners only: Commit the detached codesign payloads ```sh pushd ./bitcoin-detached-sigs -# checkout the appropriate branch for this release series -rm -rf ./* +# checkout or create the appropriate branch for this release series +git checkout --orphan <branch> +# if you are the macOS codesigner +rm -rf osx tar xf signature-osx.tar.gz +# if you are the windows codesigner +rm -rf win tar xf signature-win.tar.gz git add -A -git commit -m "point to ${VERSION}" +git commit -m "<version>: {osx,win} signature for {rc,final}" git tag -s "v${VERSION}" HEAD git push the current branch and new tag popd @@ -216,45 +225,36 @@ popd Then open a Pull Request to the [guix.sigs repository](https://github.com/bitcoin-core/guix.sigs). -## After 3 or more people have guix-built and their results match +## After 6 or more people have guix-built and their results match -Combine the `all.SHA256SUMS.asc` file from all signers into `SHA256SUMS.asc`: +After verifying signatures, combine the `all.SHA256SUMS.asc` file from all signers into `SHA256SUMS.asc`: ```bash cat "$VERSION"/*/all.SHA256SUMS.asc > SHA256SUMS.asc ``` -- Upload to the bitcoincore.org server (`/var/www/bin/bitcoin-core-${VERSION}/`): - 1. The contents of each `./bitcoin/guix-build-${VERSION}/output/${HOST}/` directory, except for - `*-debug*` files. +- Upload to the bitcoincore.org server: + 1. The contents of each `./bitcoin/guix-build-${VERSION}/output/${HOST}/` directory. Guix will output all of the results into host subdirectories, but the SHA256SUMS file does not include these subdirectories. In order for downloads via torrent to verify without directory structure modification, all of the uploaded files need to be in the same directory as the SHA256SUMS file. - The `*-debug*` files generated by the guix build contain debug symbols - for troubleshooting by developers. It is assumed that anyone that is - interested in debugging can run guix to generate the files for - themselves. To avoid end-user confusion about which file to pick, as well - as save storage space *do not upload these to the bitcoincore.org server, - nor put them in the torrent*. - - ```sh - find guix-build-${VERSION}/output/ -maxdepth 2 -type f -not -name "SHA256SUMS.part" -and -not -name "*debug*" -exec scp {} user@bitcoincore.org:/var/www/bin/bitcoin-core-${VERSION} \; - ``` + Wait until all of these files have finished uploading before uploading the SHA256SUMS(.asc) files. 2. The `SHA256SUMS` file - 3. The `SHA256SUMS.asc` combined signature file you just created + 3. The `SHA256SUMS.asc` combined signature file you just created. -- Create a torrent of the `/var/www/bin/bitcoin-core-${VERSION}` directory such - that at the top level there is only one file: the `bitcoin-core-${VERSION}` - directory containing everything else. Name the torrent - `bitcoin-${VERSION}.torrent` (note that there is no `-core-` in this name). +- After uploading release candidate binaries, notify the bitcoin-core-dev mailing list and + bitcoin-dev group that a release candidate is available for testing. Include a link to the release + notes draft. - Optionally help seed this torrent. To get the `magnet:` URI use: +- The server will automatically create an OpenTimestamps file and torrent of the directory. + +- Optionally help seed this torrent. To get the `magnet:` URI use: ```sh transmission-show -m <torrent file> @@ -265,32 +265,33 @@ cat "$VERSION"/*/all.SHA256SUMS.asc > SHA256SUMS.asc Also put it into the `optional_magnetlink:` slot in the YAML file for bitcoincore.org. -- Update other repositories and websites for new version - - - bitcoincore.org blog post +- Archive the release notes for the new version to `doc/release-notes/release-notes-${VERSION}.md` + (branch `master` and branch of the release). - - bitcoincore.org maintained versions update: - [table](https://github.com/bitcoin-core/bitcoincore.org/commits/master/_includes/posts/maintenance-table.md) +- Update the bitcoincore.org website - - Delete post-EOL [release branches](https://github.com/bitcoin/bitcoin/branches/all) and create a tag `v${branch_name}-final`. + - blog post - - Delete ["Needs backport" labels](https://github.com/bitcoin/bitcoin/labels?q=backport) for non-existing branches. + - maintained versions [table](https://github.com/bitcoin-core/bitcoincore.org/commits/master/_includes/posts/maintenance-table.md) - - bitcoincore.org RPC documentation update + - RPC documentation update - See https://github.com/bitcoin-core/bitcoincore.org/blob/master/contrib/doc-gen/ + +- Update repositories + + - Delete post-EOL [release branches](https://github.com/bitcoin/bitcoin/branches/all) and create a tag `v${branch_name}-final`. + + - Delete ["Needs backport" labels](https://github.com/bitcoin/bitcoin/labels?q=backport) for non-existing branches. + - Update packaging repo - Push the flatpak to flathub, e.g. https://github.com/flathub/org.bitcoincore.bitcoin-qt/pull/2 - Push the snap, see https://github.com/bitcoin-core/packaging/blob/main/snap/local/build.md - - This repo - - - Archive the release notes for the new version to `doc/release-notes/` (branch `master` and branch of the release) - - - Create a [new GitHub release](https://github.com/bitcoin/bitcoin/releases/new) with a link to the archived release notes + - Create a [new GitHub release](https://github.com/bitcoin/bitcoin/releases/new) with a link to the archived release notes - Announce the release: diff --git a/share/rpcauth/README.md b/share/rpcauth/README.md index 6f627b867b..1b3acb1dac 100644 --- a/share/rpcauth/README.md +++ b/share/rpcauth/README.md @@ -15,4 +15,5 @@ positional arguments: optional arguments: -h, --help show this help message and exit + -j, --json output data in json format ``` diff --git a/share/rpcauth/rpcauth.py b/share/rpcauth/rpcauth.py index cc7bba1f8b..506fcf9d91 100755 --- a/share/rpcauth/rpcauth.py +++ b/share/rpcauth/rpcauth.py @@ -7,6 +7,7 @@ from argparse import ArgumentParser from getpass import getpass from secrets import token_hex, token_urlsafe import hmac +import json def generate_salt(size): """Create size byte hex salt""" @@ -24,6 +25,7 @@ def main(): parser = ArgumentParser(description='Create login credentials for a JSON-RPC user') parser.add_argument('username', help='the username for authentication') parser.add_argument('password', help='leave empty to generate a random password or specify "-" to prompt for password', nargs='?') + parser.add_argument("-j", "--json", help="output to json instead of plain-text", action='store_true') args = parser.parse_args() if not args.password: @@ -35,9 +37,13 @@ def main(): salt = generate_salt(16) password_hmac = password_to_hmac(salt, args.password) - print('String to be appended to bitcoin.conf:') - print(f'rpcauth={args.username}:{salt}${password_hmac}') - print(f'Your password:\n{args.password}') + if args.json: + odict={'username':args.username, 'password':args.password, 'rpcauth':f'{args.username}:{salt}${password_hmac}'} + print(json.dumps(odict)) + else: + print('String to be appended to bitcoin.conf:') + print(f'rpcauth={args.username}:{salt}${password_hmac}') + print(f'Your password:\n{args.password}') if __name__ == '__main__': main() diff --git a/src/Makefile.am b/src/Makefile.am index 7673d2f545..639aecf3b3 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -230,6 +230,7 @@ BITCOIN_CORE_H = \ node/peerman_args.h \ node/protocol_version.h \ node/psbt.h \ + node/timeoffsets.h \ node/transaction.h \ node/txreconciliation.h \ node/utxo_snapshot.h \ @@ -280,7 +281,6 @@ BITCOIN_CORE_H = \ support/lockedpool.h \ sync.h \ threadsafety.h \ - timedata.h \ torcontrol.h \ txdb.h \ txmempool.h \ @@ -320,7 +320,7 @@ BITCOIN_CORE_H = \ util/sock.h \ util/spanparsing.h \ util/string.h \ - util/subprocess.hpp \ + util/subprocess.h \ util/syserror.h \ util/task_runner.h \ util/thread.h \ @@ -434,6 +434,7 @@ libbitcoin_node_a_SOURCES = \ node/minisketchwrapper.cpp \ node/peerman_args.cpp \ node/psbt.cpp \ + node/timeoffsets.cpp \ node/transaction.cpp \ node/txreconciliation.cpp \ node/utxo_snapshot.cpp \ @@ -461,7 +462,6 @@ libbitcoin_node_a_SOURCES = \ rpc/txoutproof.cpp \ script/sigcache.cpp \ signet.cpp \ - timedata.cpp \ torcontrol.cpp \ txdb.cpp \ txmempool.cpp \ @@ -679,6 +679,7 @@ libbitcoin_common_a_SOURCES = \ common/run_command.cpp \ common/settings.cpp \ common/system.cpp \ + common/url.cpp \ compressor.cpp \ core_read.cpp \ core_write.cpp \ @@ -711,11 +712,6 @@ libbitcoin_common_a_SOURCES = \ script/solver.cpp \ warnings.cpp \ $(BITCOIN_CORE_H) - -if USE_LIBEVENT -libbitcoin_common_a_CPPFLAGS += $(EVENT_CFLAGS) -libbitcoin_common_a_SOURCES += common/url.cpp -endif # # util # diff --git a/src/Makefile.test.include b/src/Makefile.test.include index cf88a02b95..cfd28b0a4d 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -85,6 +85,7 @@ BITCOIN_TESTS =\ test/checkqueue_tests.cpp \ test/coins_tests.cpp \ test/coinstatsindex_tests.cpp \ + test/common_url_tests.cpp \ test/compilerbug_tests.cpp \ test/compress_tests.cpp \ test/crypto_tests.cpp \ @@ -152,7 +153,7 @@ BITCOIN_TESTS =\ test/streams_tests.cpp \ test/sync_tests.cpp \ test/system_tests.cpp \ - test/timedata_tests.cpp \ + test/timeoffsets_tests.cpp \ test/torcontrol_tests.cpp \ test/transaction_tests.cpp \ test/translation_tests.cpp \ @@ -383,7 +384,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/string.cpp \ test/fuzz/strprintf.cpp \ test/fuzz/system.cpp \ - test/fuzz/timedata.cpp \ + test/fuzz/timeoffsets.cpp \ test/fuzz/torcontrol.cpp \ test/fuzz/transaction.cpp \ test/fuzz/tx_in.cpp \ diff --git a/src/addrman_impl.h b/src/addrman_impl.h index 867c894d01..dd7f7b318f 100644 --- a/src/addrman_impl.h +++ b/src/addrman_impl.h @@ -11,7 +11,6 @@ #include <protocol.h> #include <serialize.h> #include <sync.h> -#include <timedata.h> #include <uint256.h> #include <util/time.h> diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 129deeec60..8901d10ef6 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -11,7 +11,6 @@ #include <clientversion.h> #include <common/args.h> #include <common/system.h> -#include <common/url.h> #include <compat/compat.h> #include <compat/stdin.h> #include <policy/feerate.h> @@ -51,7 +50,6 @@ using CliClock = std::chrono::system_clock; const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; -UrlDecodeFn* const URL_DECODE = urlDecode; static const char DEFAULT_RPCCONNECT[] = "127.0.0.1"; static const int DEFAULT_HTTP_CLIENT_TIMEOUT=900; @@ -551,7 +549,7 @@ public: peer.is_outbound ? "out" : "in", ConnectionTypeForNetinfo(peer.conn_type), peer.network, - peer.transport_protocol_type.starts_with('v') ? peer.transport_protocol_type[1] : ' ', + (peer.transport_protocol_type.size() == 2 && peer.transport_protocol_type[0] == 'v') ? peer.transport_protocol_type[1] : ' ', PingTimeToString(peer.min_ping), PingTimeToString(peer.ping), peer.last_send ? ToString(time_now - peer.last_send) : "", diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index d5dfbbec27..fe90958a5f 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -11,7 +11,6 @@ #include <clientversion.h> #include <common/args.h> #include <common/system.h> -#include <common/url.h> #include <compat/compat.h> #include <interfaces/init.h> #include <key.h> @@ -28,7 +27,6 @@ #include <tuple> const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; -UrlDecodeFn* const URL_DECODE = nullptr; static void SetupWalletToolArgs(ArgsManager& argsman) { diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 4f0a816388..54796c5abb 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -12,7 +12,6 @@ #include <common/args.h> #include <common/init.h> #include <common/system.h> -#include <common/url.h> #include <compat/compat.h> #include <init.h> #include <interfaces/chain.h> @@ -35,7 +34,6 @@ using node::NodeContext; const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; -UrlDecodeFn* const URL_DECODE = urlDecode; #if HAVE_DECL_FORK diff --git a/src/common/run_command.cpp b/src/common/run_command.cpp index e5356490ef..347b486095 100644 --- a/src/common/run_command.cpp +++ b/src/common/run_command.cpp @@ -12,7 +12,7 @@ #include <univalue.h> #ifdef ENABLE_EXTERNAL_SIGNER -#include <util/subprocess.hpp> +#include <util/subprocess.h> #endif // ENABLE_EXTERNAL_SIGNER UniValue RunCommandParseJSON(const std::string& str_command, const std::string& str_std_in) diff --git a/src/common/url.cpp b/src/common/url.cpp index 053e1a825c..ecf88d07ea 100644 --- a/src/common/url.cpp +++ b/src/common/url.cpp @@ -4,19 +4,36 @@ #include <common/url.h> -#include <event2/http.h> - -#include <cstdlib> +#include <charconv> #include <string> +#include <string_view> +#include <system_error> -std::string urlDecode(const std::string &urlEncoded) { +std::string UrlDecode(std::string_view url_encoded) +{ std::string res; - if (!urlEncoded.empty()) { - char *decoded = evhttp_uridecode(urlEncoded.c_str(), false, nullptr); - if (decoded) { - res = std::string(decoded); - free(decoded); + res.reserve(url_encoded.size()); + + for (size_t i = 0; i < url_encoded.size(); ++i) { + char c = url_encoded[i]; + // Special handling for percent which should be followed by two hex digits + // representing an octet values, see RFC 3986, Section 2.1 Percent-Encoding + if (c == '%' && i + 2 < url_encoded.size()) { + unsigned int decoded_value{0}; + auto [p, ec] = std::from_chars(url_encoded.data() + i + 1, url_encoded.data() + i + 3, decoded_value, 16); + + // Only if there is no error and the pointer is set to the end of + // the string, we can be sure both characters were valid hex + if (ec == std::errc{} && p == url_encoded.data() + i + 3) { + res += static_cast<char>(decoded_value); + // Next two characters are part of the percent encoding + i += 2; + continue; + } + // In case of invalid percent encoding, add the '%' and continue } + res += c; } + return res; } diff --git a/src/common/url.h b/src/common/url.h index b16b8241af..203f41c70f 100644 --- a/src/common/url.h +++ b/src/common/url.h @@ -6,9 +6,12 @@ #define BITCOIN_COMMON_URL_H #include <string> +#include <string_view> -using UrlDecodeFn = std::string(const std::string& url_encoded); -UrlDecodeFn urlDecode; -extern UrlDecodeFn* const URL_DECODE; +/* Decode a URL. + * + * Notably this implementation does not decode a '+' to a ' '. + */ +std::string UrlDecode(std::string_view url_encoded); #endif // BITCOIN_COMMON_URL_H diff --git a/src/core_read.cpp b/src/core_read.cpp index e32e46d1b9..5956d9df5f 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -256,6 +256,6 @@ util::Result<int> SighashFromStr(const std::string& sighash) if (it != map_sighash_values.end()) { return it->second; } else { - return util::Error{Untranslated(sighash + " is not a valid sighash parameter.")}; + return util::Error{Untranslated("'" + sighash + "' is not a valid sighash parameter.")}; } } diff --git a/src/index/base.cpp b/src/index/base.cpp index a203ce4a9f..e66c89f9e4 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -160,12 +160,24 @@ void BaseIndex::Sync() } const CBlockIndex* pindex_next = WITH_LOCK(cs_main, return NextSyncBlock(pindex, m_chainstate->m_chain)); + // If pindex_next is null, it means pindex is the chain tip, so + // commit data indexed so far. if (!pindex_next) { SetBestBlockIndex(pindex); // No need to handle errors in Commit. See rationale above. Commit(); - m_synced = true; - break; + + // If pindex is still the chain tip after committing, exit the + // sync loop. It is important for cs_main to be locked while + // setting m_synced = true, otherwise a new block could be + // attached while m_synced is still false, and it would not be + // indexed. + LOCK(::cs_main); + pindex_next = NextSyncBlock(pindex, m_chainstate->m_chain); + if (!pindex_next) { + m_synced = true; + break; + } } if (pindex_next->pprev != pindex && !Rewind(pindex, pindex_next->pprev)) { FatalErrorf("%s: Failed to rewind index %s to a previous chain tip", __func__, GetName()); diff --git a/src/init.cpp b/src/init.cpp index e8e6391af7..1e6d296551 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -67,7 +67,6 @@ #include <scheduler.h> #include <script/sigcache.h> #include <sync.h> -#include <timedata.h> #include <torcontrol.h> #include <txdb.h> #include <txmempool.h> @@ -523,7 +522,6 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-maxconnections=<n>", strprintf("Maintain at most <n> automatic connections to peers (default: %u). This limit does not apply to connections manually added via -addnode or the addnode RPC, which have a separate limit of %u.", DEFAULT_MAX_PEER_CONNECTIONS, MAX_ADDNODE_CONNECTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxreceivebuffer=<n>", strprintf("Maximum per-connection receive buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXRECEIVEBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxsendbuffer=<n>", strprintf("Maximum per-connection memory usage for the send buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXSENDBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by outbound peers forward or backward by this amount (default: %u seconds).", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxuploadtarget=<n>", strprintf("Tries to keep outbound traffic under the given target per 24h. Limit does not apply to peers with 'download' permission or blocks created within past week. 0 = no limit (default: %s). Optional suffix units [k|K|m|M|g|G|t|T] (default: M). Lowercase is 1000 base while uppercase is 1024 base", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); #if HAVE_SOCKADDR_UN argsman.AddArg("-onion=<ip:port|path>", "Use separate SOCKS5 proxy to reach peers via Tor onion services, set -noonion to disable (default: -proxy). May be a local file path prefixed with 'unix:'.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -546,7 +544,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-proxy=<ip:port>", "Connect through SOCKS5 proxy, set -noproxy to disable (default: disabled)", ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_ELISION, OptionsCategory::CONNECTION); #endif argsman.AddArg("-proxyrandomize", strprintf("Randomize credentials for every proxy connection. This enables Tor stream isolation (default: %u)", DEFAULT_PROXYRANDOMIZE), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-seednode=<ip>", "Connect to a node to retrieve peer addresses, and disconnect. This option can be specified multiple times to connect to multiple nodes.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-seednode=<ip>", "Connect to a node to retrieve peer addresses, and disconnect. This option can be specified multiple times to connect to multiple nodes. During startup, seednodes will be tried before dnsseeds.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-networkactive", "Enable all P2P network activity (default: 1). Can be changed by the setnetworkactive RPC command", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-timeout=<n>", strprintf("Specify socket connection timeout in milliseconds. If an initial attempt to connect is unsuccessful after this amount of time, drop it (minimum: 1, default: %d)", DEFAULT_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peertimeout=<n>", strprintf("Specify a p2p connection timeout delay in seconds. After connecting to a peer, wait this amount of time before considering disconnection based on inactivity (minimum: 1, default: %d)", DEFAULT_PEER_CONNECT_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); @@ -636,7 +634,7 @@ void SetupServerArgs(ArgsManager& argsman) MAX_OP_RETURN_RELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); argsman.AddArg("-mempoolfullrbf", strprintf("Accept transaction replace-by-fee without requiring replaceability signaling (default: %u)", DEFAULT_MEMPOOL_FULL_RBF), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); - argsman.AddArg("-permitbaremultisig", strprintf("Relay non-P2SH multisig (default: %u)", DEFAULT_PERMIT_BAREMULTISIG), ArgsManager::ALLOW_ANY, + argsman.AddArg("-permitbaremultisig", strprintf("Relay transactions creating non-P2SH multisig outputs (default: %u)", DEFAULT_PERMIT_BAREMULTISIG), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); argsman.AddArg("-minrelaytxfee=<amt>", strprintf("Fees (in %s/kvB) smaller than this are considered zero fee for relaying, mining and transaction creation (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_MIN_RELAY_TX_FEE)), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); @@ -992,10 +990,8 @@ bool AppInitParameterInteraction(const ArgsManager& args) InitWarning(strprintf(_("Reducing -maxconnections from %d to %d, because of system limitations."), nUserMaxConnections, nMaxConnections)); // ********************************************************* Step 3: parameter-to-internal-flags - auto result = init::SetLoggingCategories(args); - if (!result) return InitError(util::ErrorString(result)); - result = init::SetLoggingLevel(args); - if (!result) return InitError(util::ErrorString(result)); + if (auto result{init::SetLoggingCategories(args)}; !result) return InitError(util::ErrorString(result)); + if (auto result{init::SetLoggingLevel(args)}; !result) return InitError(util::ErrorString(result)); nConnectTimeout = args.GetIntArg("-timeout", DEFAULT_CONNECT_TIMEOUT); if (nConnectTimeout <= 0) { @@ -1301,30 +1297,31 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } } - for (const std::string port_option : { - "-i2psam", - "-onion", - "-proxy", - "-rpcbind", - "-torcontrol", - "-whitebind", - "-zmqpubhashblock", - "-zmqpubhashtx", - "-zmqpubrawblock", - "-zmqpubrawtx", - "-zmqpubsequence", + for ([[maybe_unused]] const auto& [arg, unix] : std::vector<std::pair<std::string, bool>>{ + // arg name UNIX socket support + {"-i2psam", false}, + {"-onion", true}, + {"-proxy", true}, + {"-rpcbind", false}, + {"-torcontrol", false}, + {"-whitebind", false}, + {"-zmqpubhashblock", true}, + {"-zmqpubhashtx", true}, + {"-zmqpubrawblock", true}, + {"-zmqpubrawtx", true}, + {"-zmqpubsequence", true}, }) { - for (const std::string& socket_addr : args.GetArgs(port_option)) { + for (const std::string& socket_addr : args.GetArgs(arg)) { std::string host_out; uint16_t port_out{0}; if (!SplitHostPort(socket_addr, port_out, host_out)) { #if HAVE_SOCKADDR_UN - // Allow unix domain sockets for -proxy and -onion e.g. unix:/some/file/path - if ((port_option != "-proxy" && port_option != "-onion") || socket_addr.find(ADDR_PREFIX_UNIX) != 0) { - return InitError(InvalidPortErrMsg(port_option, socket_addr)); + // Allow unix domain sockets for some options e.g. unix:/some/file/path + if (!unix || socket_addr.find(ADDR_PREFIX_UNIX) != 0) { + return InitError(InvalidPortErrMsg(arg, socket_addr)); } #else - return InitError(InvalidPortErrMsg(port_option, socket_addr)); + return InitError(InvalidPortErrMsg(arg, socket_addr)); #endif } } diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index 6114236623..c41f35829d 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -127,7 +127,7 @@ public: virtual bool setAddressReceiveRequest(const CTxDestination& dest, const std::string& id, const std::string& value) = 0; //! Display address on external signer - virtual bool displayAddress(const CTxDestination& dest) = 0; + virtual util::Result<void> displayAddress(const CTxDestination& dest) = 0; //! Lock coin. virtual bool lockCoin(const COutPoint& output, const bool write_to_db) = 0; diff --git a/src/net.cpp b/src/net.cpp index e388f05b03..1eda98253a 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -412,6 +412,9 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo // Resolve const uint16_t default_port{pszDest != nullptr ? GetDefaultPort(pszDest) : m_params.GetDefaultPort()}; + + // Collection of addresses to try to connect to: either all dns resolved addresses if a domain name (pszDest) is provided, or addrConnect otherwise. + std::vector<CAddress> connect_to{}; if (pszDest) { std::vector<CService> resolved{Lookup(pszDest, default_port, fNameLookup && !HaveNameProxy(), 256)}; if (!resolved.empty()) { @@ -432,8 +435,16 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo LogPrintf("Not opening a connection to %s, already connected to %s\n", pszDest, addrConnect.ToStringAddrPort()); return nullptr; } + // Add the address to the resolved addresses vector so we can try to connect to it later on + connect_to.push_back(addrConnect); } + } else { + // For resolution via proxy + connect_to.push_back(addrConnect); } + } else { + // Connect via addrConnect directly + connect_to.push_back(addrConnect); } // Connect @@ -443,94 +454,99 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo assert(!addr_bind.IsValid()); std::unique_ptr<i2p::sam::Session> i2p_transient_session; - if (addrConnect.IsValid()) { - const bool use_proxy{GetProxy(addrConnect.GetNetwork(), proxy)}; - bool proxyConnectionFailed = false; + for (auto& target_addr: connect_to) { + if (target_addr.IsValid()) { + const bool use_proxy{GetProxy(target_addr.GetNetwork(), proxy)}; + bool proxyConnectionFailed = false; - if (addrConnect.IsI2P() && use_proxy) { - i2p::Connection conn; - bool connected{false}; + if (target_addr.IsI2P() && use_proxy) { + i2p::Connection conn; + bool connected{false}; - if (m_i2p_sam_session) { - connected = m_i2p_sam_session->Connect(addrConnect, conn, proxyConnectionFailed); - } else { - { - LOCK(m_unused_i2p_sessions_mutex); - if (m_unused_i2p_sessions.empty()) { - i2p_transient_session = - std::make_unique<i2p::sam::Session>(proxy, &interruptNet); - } else { - i2p_transient_session.swap(m_unused_i2p_sessions.front()); - m_unused_i2p_sessions.pop(); + if (m_i2p_sam_session) { + connected = m_i2p_sam_session->Connect(target_addr, conn, proxyConnectionFailed); + } else { + { + LOCK(m_unused_i2p_sessions_mutex); + if (m_unused_i2p_sessions.empty()) { + i2p_transient_session = + std::make_unique<i2p::sam::Session>(proxy, &interruptNet); + } else { + i2p_transient_session.swap(m_unused_i2p_sessions.front()); + m_unused_i2p_sessions.pop(); + } } - } - connected = i2p_transient_session->Connect(addrConnect, conn, proxyConnectionFailed); - if (!connected) { - LOCK(m_unused_i2p_sessions_mutex); - if (m_unused_i2p_sessions.size() < MAX_UNUSED_I2P_SESSIONS_SIZE) { - m_unused_i2p_sessions.emplace(i2p_transient_session.release()); + connected = i2p_transient_session->Connect(target_addr, conn, proxyConnectionFailed); + if (!connected) { + LOCK(m_unused_i2p_sessions_mutex); + if (m_unused_i2p_sessions.size() < MAX_UNUSED_I2P_SESSIONS_SIZE) { + m_unused_i2p_sessions.emplace(i2p_transient_session.release()); + } } } - } - if (connected) { - sock = std::move(conn.sock); - addr_bind = CAddress{conn.me, NODE_NONE}; + if (connected) { + sock = std::move(conn.sock); + addr_bind = CAddress{conn.me, NODE_NONE}; + } + } else if (use_proxy) { + LogPrintLevel(BCLog::PROXY, BCLog::Level::Debug, "Using proxy: %s to connect to %s\n", proxy.ToString(), target_addr.ToStringAddrPort()); + sock = ConnectThroughProxy(proxy, target_addr.ToStringAddr(), target_addr.GetPort(), proxyConnectionFailed); + } else { + // no proxy needed (none set for target network) + sock = ConnectDirectly(target_addr, conn_type == ConnectionType::MANUAL); } - } else if (use_proxy) { - LogPrintLevel(BCLog::PROXY, BCLog::Level::Debug, "Using proxy: %s to connect to %s:%s\n", proxy.ToString(), addrConnect.ToStringAddr(), addrConnect.GetPort()); - sock = ConnectThroughProxy(proxy, addrConnect.ToStringAddr(), addrConnect.GetPort(), proxyConnectionFailed); - } else { - // no proxy needed (none set for target network) - sock = ConnectDirectly(addrConnect, conn_type == ConnectionType::MANUAL); + if (!proxyConnectionFailed) { + // If a connection to the node was attempted, and failure (if any) is not caused by a problem connecting to + // the proxy, mark this as an attempt. + addrman.Attempt(target_addr, fCountFailure); + } + } else if (pszDest && GetNameProxy(proxy)) { + std::string host; + uint16_t port{default_port}; + SplitHostPort(std::string(pszDest), port, host); + bool proxyConnectionFailed; + sock = ConnectThroughProxy(proxy, host, port, proxyConnectionFailed); + } + // Check any other resolved address (if any) if we fail to connect + if (!sock) { + continue; } - if (!proxyConnectionFailed) { - // If a connection to the node was attempted, and failure (if any) is not caused by a problem connecting to - // the proxy, mark this as an attempt. - addrman.Attempt(addrConnect, fCountFailure); + + NetPermissionFlags permission_flags = NetPermissionFlags::None; + std::vector<NetWhitelistPermissions> whitelist_permissions = conn_type == ConnectionType::MANUAL ? vWhitelistedRangeOutgoing : std::vector<NetWhitelistPermissions>{}; + AddWhitelistPermissionFlags(permission_flags, target_addr, whitelist_permissions); + + // Add node + NodeId id = GetNewNodeId(); + uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); + if (!addr_bind.IsValid()) { + addr_bind = GetBindAddress(*sock); } - } else if (pszDest && GetNameProxy(proxy)) { - std::string host; - uint16_t port{default_port}; - SplitHostPort(std::string(pszDest), port, host); - bool proxyConnectionFailed; - sock = ConnectThroughProxy(proxy, host, port, proxyConnectionFailed); - } - if (!sock) { - return nullptr; - } + CNode* pnode = new CNode(id, + std::move(sock), + target_addr, + CalculateKeyedNetGroup(target_addr), + nonce, + addr_bind, + pszDest ? pszDest : "", + conn_type, + /*inbound_onion=*/false, + CNodeOptions{ + .permission_flags = permission_flags, + .i2p_sam_session = std::move(i2p_transient_session), + .recv_flood_size = nReceiveFloodSize, + .use_v2transport = use_v2transport, + }); + pnode->AddRef(); - NetPermissionFlags permission_flags = NetPermissionFlags::None; - std::vector<NetWhitelistPermissions> whitelist_permissions = conn_type == ConnectionType::MANUAL ? vWhitelistedRangeOutgoing : std::vector<NetWhitelistPermissions>{}; - AddWhitelistPermissionFlags(permission_flags, addrConnect, whitelist_permissions); + // We're making a new connection, harvest entropy from the time (and our peer count) + RandAddEvent((uint32_t)id); - // Add node - NodeId id = GetNewNodeId(); - uint64_t nonce = GetDeterministicRandomizer(RANDOMIZER_ID_LOCALHOSTNONCE).Write(id).Finalize(); - if (!addr_bind.IsValid()) { - addr_bind = GetBindAddress(*sock); + return pnode; } - CNode* pnode = new CNode(id, - std::move(sock), - addrConnect, - CalculateKeyedNetGroup(addrConnect), - nonce, - addr_bind, - pszDest ? pszDest : "", - conn_type, - /*inbound_onion=*/false, - CNodeOptions{ - .permission_flags = permission_flags, - .i2p_sam_session = std::move(i2p_transient_session), - .recv_flood_size = nReceiveFloodSize, - .use_v2transport = use_v2transport, - }); - pnode->AddRef(); - // We're making a new connection, harvest entropy from the time (and our peer count) - RandAddEvent((uint32_t)id); - - return pnode; + return nullptr; } void CNode::CloseSocketDisconnect() @@ -599,7 +615,6 @@ void CNode::CopyStats(CNodeStats& stats) X(m_last_tx_time); X(m_last_block_time); X(m_connected); - X(nTimeOffset); X(m_addr_name); X(nVersion); { @@ -2168,11 +2183,36 @@ void CConnman::WakeMessageHandler() void CConnman::ThreadDNSAddressSeed() { + constexpr int TARGET_OUTBOUND_CONNECTIONS = 2; + int outbound_connection_count = 0; + + if (gArgs.IsArgSet("-seednode")) { + auto start = NodeClock::now(); + constexpr std::chrono::seconds SEEDNODE_TIMEOUT = 30s; + LogPrintf("-seednode enabled. Trying the provided seeds for %d seconds before defaulting to the dnsseeds.\n", SEEDNODE_TIMEOUT.count()); + while (!interruptNet) { + if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) + return; + + // Abort if we have spent enough time without reaching our target. + // Giving seed nodes 30 seconds so this does not become a race against fixedseeds (which triggers after 1 min) + if (NodeClock::now() > start + SEEDNODE_TIMEOUT) { + LogPrintf("Couldn't connect to enough peers via seed nodes. Handing fetch logic to the DNS seeds.\n"); + break; + } + + outbound_connection_count = GetFullOutboundConnCount(); + if (outbound_connection_count >= TARGET_OUTBOUND_CONNECTIONS) { + LogPrintf("P2P peers available. Finished fetching data from seed nodes.\n"); + break; + } + } + } + FastRandomContext rng; std::vector<std::string> seeds = m_params.DNSSeeds(); Shuffle(seeds.begin(), seeds.end(), rng); int seeds_right_now = 0; // Number of seeds left before testing if we have enough connections - int found = 0; if (gArgs.GetBoolArg("-forcednsseed", DEFAULT_FORCEDNSSEED)) { // When -forcednsseed is provided, query all. @@ -2184,98 +2224,101 @@ void CConnman::ThreadDNSAddressSeed() seeds_right_now = seeds.size(); } - // goal: only query DNS seed if address need is acute - // * If we have a reasonable number of peers in addrman, spend - // some time trying them first. This improves user privacy by - // creating fewer identifying DNS requests, reduces trust by - // giving seeds less influence on the network topology, and - // reduces traffic to the seeds. - // * When querying DNS seeds query a few at once, this ensures - // that we don't give DNS seeds the ability to eclipse nodes - // that query them. - // * If we continue having problems, eventually query all the - // DNS seeds, and if that fails too, also try the fixed seeds. - // (done in ThreadOpenConnections) - const std::chrono::seconds seeds_wait_time = (addrman.Size() >= DNSSEEDS_DELAY_PEER_THRESHOLD ? DNSSEEDS_DELAY_MANY_PEERS : DNSSEEDS_DELAY_FEW_PEERS); - - for (const std::string& seed : seeds) { - if (seeds_right_now == 0) { - seeds_right_now += DNSSEEDS_TO_QUERY_AT_ONCE; - - if (addrman.Size() > 0) { - LogPrintf("Waiting %d seconds before querying DNS seeds.\n", seeds_wait_time.count()); - std::chrono::seconds to_wait = seeds_wait_time; - while (to_wait.count() > 0) { - // if sleeping for the MANY_PEERS interval, wake up - // early to see if we have enough peers and can stop - // this thread entirely freeing up its resources - std::chrono::seconds w = std::min(DNSSEEDS_DELAY_FEW_PEERS, to_wait); - if (!interruptNet.sleep_for(w)) return; - to_wait -= w; - - int nRelevant = 0; - { - LOCK(m_nodes_mutex); - for (const CNode* pnode : m_nodes) { - if (pnode->fSuccessfullyConnected && pnode->IsFullOutboundConn()) ++nRelevant; + // Proceed with dnsseeds if seednodes hasn't reached the target or if forcednsseed is set + if (outbound_connection_count < TARGET_OUTBOUND_CONNECTIONS || seeds_right_now) { + // goal: only query DNS seed if address need is acute + // * If we have a reasonable number of peers in addrman, spend + // some time trying them first. This improves user privacy by + // creating fewer identifying DNS requests, reduces trust by + // giving seeds less influence on the network topology, and + // reduces traffic to the seeds. + // * When querying DNS seeds query a few at once, this ensures + // that we don't give DNS seeds the ability to eclipse nodes + // that query them. + // * If we continue having problems, eventually query all the + // DNS seeds, and if that fails too, also try the fixed seeds. + // (done in ThreadOpenConnections) + int found = 0; + const std::chrono::seconds seeds_wait_time = (addrman.Size() >= DNSSEEDS_DELAY_PEER_THRESHOLD ? DNSSEEDS_DELAY_MANY_PEERS : DNSSEEDS_DELAY_FEW_PEERS); + + for (const std::string& seed : seeds) { + if (seeds_right_now == 0) { + seeds_right_now += DNSSEEDS_TO_QUERY_AT_ONCE; + + if (addrman.Size() > 0) { + LogPrintf("Waiting %d seconds before querying DNS seeds.\n", seeds_wait_time.count()); + std::chrono::seconds to_wait = seeds_wait_time; + while (to_wait.count() > 0) { + // if sleeping for the MANY_PEERS interval, wake up + // early to see if we have enough peers and can stop + // this thread entirely freeing up its resources + std::chrono::seconds w = std::min(DNSSEEDS_DELAY_FEW_PEERS, to_wait); + if (!interruptNet.sleep_for(w)) return; + to_wait -= w; + + if (GetFullOutboundConnCount() >= TARGET_OUTBOUND_CONNECTIONS) { + if (found > 0) { + LogPrintf("%d addresses found from DNS seeds\n", found); + LogPrintf("P2P peers available. Finished DNS seeding.\n"); + } else { + LogPrintf("P2P peers available. Skipped DNS seeding.\n"); + } + return; } } - if (nRelevant >= 2) { - if (found > 0) { - LogPrintf("%d addresses found from DNS seeds\n", found); - LogPrintf("P2P peers available. Finished DNS seeding.\n"); - } else { - LogPrintf("P2P peers available. Skipped DNS seeding.\n"); - } - return; - } } } - } - - if (interruptNet) return; - // hold off on querying seeds if P2P network deactivated - if (!fNetworkActive) { - LogPrintf("Waiting for network to be reactivated before querying DNS seeds.\n"); - do { - if (!interruptNet.sleep_for(std::chrono::seconds{1})) return; - } while (!fNetworkActive); - } + if (interruptNet) return; - LogPrintf("Loading addresses from DNS seed %s\n", seed); - // If -proxy is in use, we make an ADDR_FETCH connection to the DNS resolved peer address - // for the base dns seed domain in chainparams - if (HaveNameProxy()) { - AddAddrFetch(seed); - } else { - std::vector<CAddress> vAdd; - constexpr ServiceFlags requiredServiceBits{SeedsServiceFlags()}; - std::string host = strprintf("x%x.%s", requiredServiceBits, seed); - CNetAddr resolveSource; - if (!resolveSource.SetInternal(host)) { - continue; + // hold off on querying seeds if P2P network deactivated + if (!fNetworkActive) { + LogPrintf("Waiting for network to be reactivated before querying DNS seeds.\n"); + do { + if (!interruptNet.sleep_for(std::chrono::seconds{1})) return; + } while (!fNetworkActive); } - unsigned int nMaxIPs = 256; // Limits number of IPs learned from a DNS seed - const auto addresses{LookupHost(host, nMaxIPs, true)}; - if (!addresses.empty()) { - for (const CNetAddr& ip : addresses) { - CAddress addr = CAddress(CService(ip, m_params.GetDefaultPort()), requiredServiceBits); - addr.nTime = rng.rand_uniform_delay(Now<NodeSeconds>() - 3 * 24h, -4 * 24h); // use a random age between 3 and 7 days old - vAdd.push_back(addr); - found++; - } - addrman.Add(vAdd, resolveSource); - } else { - // If the seed does not support a subdomain with our desired service bits, - // we make an ADDR_FETCH connection to the DNS resolved peer address for the - // base dns seed domain in chainparams + + LogPrintf("Loading addresses from DNS seed %s\n", seed); + // If -proxy is in use, we make an ADDR_FETCH connection to the DNS resolved peer address + // for the base dns seed domain in chainparams + if (HaveNameProxy()) { AddAddrFetch(seed); + } else { + std::vector<CAddress> vAdd; + constexpr ServiceFlags requiredServiceBits{SeedsServiceFlags()}; + std::string host = strprintf("x%x.%s", requiredServiceBits, seed); + CNetAddr resolveSource; + if (!resolveSource.SetInternal(host)) { + continue; + } + // Limit number of IPs learned from a single DNS seed. This limit exists to prevent the results from + // one DNS seed from dominating AddrMan. Note that the number of results from a UDP DNS query is + // bounded to 33 already, but it is possible for it to use TCP where a larger number of results can be + // returned. + unsigned int nMaxIPs = 32; + const auto addresses{LookupHost(host, nMaxIPs, true)}; + if (!addresses.empty()) { + for (const CNetAddr& ip : addresses) { + CAddress addr = CAddress(CService(ip, m_params.GetDefaultPort()), requiredServiceBits); + addr.nTime = rng.rand_uniform_delay(Now<NodeSeconds>() - 3 * 24h, -4 * 24h); // use a random age between 3 and 7 days old + vAdd.push_back(addr); + found++; + } + addrman.Add(vAdd, resolveSource); + } else { + // If the seed does not support a subdomain with our desired service bits, + // we make an ADDR_FETCH connection to the DNS resolved peer address for the + // base dns seed domain in chainparams + AddAddrFetch(seed); + } } + --seeds_right_now; } - --seeds_right_now; + LogPrintf("%d addresses found from DNS seeds\n", found); + } else { + LogPrintf("Skipping DNS seeds. Enough peers have been found\n"); } - LogPrintf("%d addresses found from DNS seeds\n", found); } void CConnman::DumpAddresses() @@ -2326,6 +2369,19 @@ void CConnman::StartExtraBlockRelayPeers() m_start_extra_block_relay_peers = true; } +// Return the number of outbound connections that are full relay (not blocks only) +int CConnman::GetFullOutboundConnCount() const +{ + int nRelevant = 0; + { + LOCK(m_nodes_mutex); + for (const CNode* pnode : m_nodes) { + if (pnode->fSuccessfullyConnected && pnode->IsFullOutboundConn()) ++nRelevant; + } + } + return nRelevant; +} + // Return the number of peers we have over our outbound connection limit // Exclude peers that are marked for disconnect, or are going to be // disconnected soon (eg ADDR_FETCH and FEELER) @@ -191,7 +191,6 @@ public: std::chrono::seconds m_last_tx_time; std::chrono::seconds m_last_block_time; std::chrono::seconds m_connected; - int64_t nTimeOffset; std::string m_addr_name; int nVersion; std::string cleanSubVer; @@ -703,7 +702,6 @@ public: std::atomic<std::chrono::seconds> m_last_recv{0s}; //! Unix epoch time at peer connection const std::chrono::seconds m_connected; - std::atomic<int64_t> nTimeOffset{0}; // Address of this peer const CAddress addr; // Bind address of our side of the connection @@ -1175,6 +1173,8 @@ public: void StartExtraBlockRelayPeers(); + // Count the number of full-relay peer we have. + int GetFullOutboundConnCount() const; // Return the number of outbound peers we have in excess of our target (eg, // if we previously called SetTryNewOutboundPeer(true), and have since set // to false, we may have extra peers that we wish to disconnect). This may diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 39ffff97d2..9b5983b9d0 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -23,6 +23,7 @@ #include <netbase.h> #include <netmessagemaker.h> #include <node/blockstorage.h> +#include <node/timeoffsets.h> #include <node/txreconciliation.h> #include <policy/fees.h> #include <policy/policy.h> @@ -34,7 +35,6 @@ #include <scheduler.h> #include <streams.h> #include <sync.h> -#include <timedata.h> #include <tinyformat.h> #include <txmempool.h> #include <txorphanage.h> @@ -390,6 +390,10 @@ struct Peer { /** Whether this peer wants invs or headers (when possible) for block announcements */ bool m_prefers_headers GUARDED_BY(NetEventsInterface::g_msgproc_mutex){false}; + /** Time offset computed during the version handshake based on the + * timestamp the peer sent in the version message. */ + std::atomic<std::chrono::seconds> m_time_offset{0s}; + explicit Peer(NodeId id, ServiceFlags our_services) : m_id{id} , m_our_services{our_services} @@ -513,7 +517,7 @@ public: std::optional<std::string> FetchBlock(NodeId peer_id, const CBlockIndex& block_index) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) const override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); - bool IgnoresIncomingTxs() override { return m_opts.ignore_incoming_txs; } + PeerManagerInfo GetInfo() const override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); void SendPings() override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); void RelayTransaction(const uint256& txid, const uint256& wtxid) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); void SetBestBlock(int height, std::chrono::seconds time) override @@ -586,7 +590,7 @@ private: * @param[in] maybe_add_extra_compact_tx Whether this tx should be added to vExtraTxnForCompact. * Set to false if the tx has already been rejected before, * e.g. is an orphan, to avoid adding duplicate entries. - * Updates m_txrequest, m_recent_rejects, m_orphanage, and vExtraTxnForCompact. */ + * Updates m_txrequest, m_recent_rejects, m_recent_rejects_reconsiderable, m_orphanage, and vExtraTxnForCompact. */ void ProcessInvalidTx(NodeId nodeid, const CTransactionRef& tx, const TxValidationState& result, bool maybe_add_extra_compact_tx) EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, cs_main); @@ -596,6 +600,42 @@ private: void ProcessValidTx(NodeId nodeid, const CTransactionRef& tx, const std::list<CTransactionRef>& replaced_transactions) EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, cs_main); + struct PackageToValidate { + const Package m_txns; + const std::vector<NodeId> m_senders; + /** Construct a 1-parent-1-child package. */ + explicit PackageToValidate(const CTransactionRef& parent, + const CTransactionRef& child, + NodeId parent_sender, + NodeId child_sender) : + m_txns{parent, child}, + m_senders {parent_sender, child_sender} + {} + + std::string ToString() const { + Assume(m_txns.size() == 2); + return strprintf("parent %s (wtxid=%s, sender=%d) + child %s (wtxid=%s, sender=%d)", + m_txns.front()->GetHash().ToString(), + m_txns.front()->GetWitnessHash().ToString(), + m_senders.front(), + m_txns.back()->GetHash().ToString(), + m_txns.back()->GetWitnessHash().ToString(), + m_senders.back()); + } + }; + + /** Handle the results of package validation: calls ProcessValidTx and ProcessInvalidTx for + * individual transactions, and caches rejection for the package as a group. + */ + void ProcessPackageResult(const PackageToValidate& package_to_validate, const PackageMempoolAcceptResult& package_result) + EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, cs_main); + + /** Look for a child of this transaction in the orphanage to form a 1-parent-1-child package, + * skipping any combinations that have already been tried. Return the resulting package along with + * the senders of its respective transactions, or std::nullopt if no package is found. */ + std::optional<PackageToValidate> Find1P1CPackage(const CTransactionRef& ptx, NodeId nodeid) + EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, cs_main); + /** * Reconsider orphan transactions after a parent has been accepted to the mempool. * @@ -749,6 +789,8 @@ private: /** Next time to check for stale tip */ std::chrono::seconds m_stale_tip_check_time GUARDED_BY(cs_main){0s}; + TimeOffsets m_outbound_time_offsets; + const Options m_opts; bool RejectIncomingTxs(const CNode& peer) const; @@ -806,7 +848,16 @@ private: /** Stalling timeout for blocks in IBD */ std::atomic<std::chrono::seconds> m_block_stalling_timeout{BLOCK_STALLING_TIMEOUT_DEFAULT}; - bool AlreadyHaveTx(const GenTxid& gtxid) + /** Check whether we already have this gtxid in: + * - mempool + * - orphanage + * - m_recent_rejects + * - m_recent_rejects_reconsiderable (if include_reconsiderable = true) + * - m_recent_confirmed_transactions + * Also responsible for resetting m_recent_rejects and m_recent_rejects_reconsiderable if the + * chain tip has changed. + * */ + bool AlreadyHaveTx(const GenTxid& gtxid, bool include_reconsiderable) EXCLUSIVE_LOCKS_REQUIRED(cs_main, !m_recent_confirmed_transactions_mutex); /** @@ -844,8 +895,32 @@ private: * Memory used: 1.3 MB */ CRollingBloomFilter m_recent_rejects GUARDED_BY(::cs_main){120'000, 0.000'001}; + /** Block hash of chain tip the last time we reset m_recent_rejects and + * m_recent_rejects_reconsiderable. */ uint256 hashRecentRejectsChainTip GUARDED_BY(cs_main); + /** + * Filter for: + * (1) wtxids of transactions that were recently rejected by the mempool but are + * eligible for reconsideration if submitted with other transactions. + * (2) packages (see GetPackageHash) we have already rejected before and should not retry. + * + * Similar to m_recent_rejects, this filter is used to save bandwidth when e.g. all of our peers + * have larger mempools and thus lower minimum feerates than us. + * + * When a transaction's error is TxValidationResult::TX_RECONSIDERABLE (in a package or by + * itself), add its wtxid to this filter. When a package fails for any reason, add the combined + * hash to this filter. + * + * Upon receiving an announcement for a transaction, if it exists in this filter, do not + * download the txdata. When considering packages, if it exists in this filter, drop it. + * + * Reset this filter when the chain tip changes. + * + * Parameters are picked to be the same as m_recent_rejects, with the same rationale. + */ + CRollingBloomFilter m_recent_rejects_reconsiderable GUARDED_BY(::cs_main){120'000, 0.000'001}; + /* * Filter for transactions that have been recently confirmed. * We use this to avoid requesting transactions that have already been @@ -1792,10 +1867,19 @@ bool PeerManagerImpl::GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) c stats.presync_height = peer->m_headers_sync->GetPresyncHeight(); } } + stats.time_offset = peer->m_time_offset; return true; } +PeerManagerInfo PeerManagerImpl::GetInfo() const +{ + return PeerManagerInfo{ + .median_outbound_time_offset = m_outbound_time_offsets.Median(), + .ignores_incoming_txs = m_opts.ignore_incoming_txs, + }; +} + void PeerManagerImpl::AddToCompactExtraTransactions(const CTransactionRef& tx) { if (m_opts.max_extra_txs <= 0) @@ -2194,7 +2278,7 @@ void PeerManagerImpl::BlockChecked(const CBlock& block, const BlockValidationSta // -bool PeerManagerImpl::AlreadyHaveTx(const GenTxid& gtxid) +bool PeerManagerImpl::AlreadyHaveTx(const GenTxid& gtxid, bool include_reconsiderable) { if (m_chainman.ActiveChain().Tip()->GetBlockHash() != hashRecentRejectsChainTip) { // If the chain tip has changed previously rejected transactions @@ -2203,12 +2287,15 @@ bool PeerManagerImpl::AlreadyHaveTx(const GenTxid& gtxid) // txs a second chance. hashRecentRejectsChainTip = m_chainman.ActiveChain().Tip()->GetBlockHash(); m_recent_rejects.reset(); + m_recent_rejects_reconsiderable.reset(); } const uint256& hash = gtxid.GetHash(); if (m_orphanage.HaveTx(gtxid)) return true; + if (include_reconsiderable && m_recent_rejects_reconsiderable.contains(hash)) return true; + { LOCK(m_recent_confirmed_transactions_mutex); if (m_recent_confirmed_transactions.contains(hash)) return true; @@ -3097,7 +3184,14 @@ void PeerManagerImpl::ProcessInvalidTx(NodeId nodeid, const CTransactionRef& ptx // See also comments in https://github.com/bitcoin/bitcoin/pull/18044#discussion_r443419034 // for concerns around weakening security of unupgraded nodes // if we start doing this too early. - m_recent_rejects.insert(ptx->GetWitnessHash().ToUint256()); + if (state.GetResult() == TxValidationResult::TX_RECONSIDERABLE) { + // If the result is TX_RECONSIDERABLE, add it to m_recent_rejects_reconsiderable + // because we should not download or submit this transaction by itself again, but may + // submit it as part of a package later. + m_recent_rejects_reconsiderable.insert(ptx->GetWitnessHash().ToUint256()); + } else { + m_recent_rejects.insert(ptx->GetWitnessHash().ToUint256()); + } m_txrequest.ForgetTxHash(ptx->GetWitnessHash()); // If the transaction failed for TX_INPUTS_NOT_STANDARD, // then we know that the witness was irrelevant to the policy @@ -3107,6 +3201,8 @@ void PeerManagerImpl::ProcessInvalidTx(NodeId nodeid, const CTransactionRef& ptx // processing of this transaction in the event that child // transactions are later received (resulting in // parent-fetching by txid via the orphan-handling logic). + // We only add the txid if it differs from the wtxid, to avoid wasting entries in the + // rolling bloom filter. if (state.GetResult() == TxValidationResult::TX_INPUTS_NOT_STANDARD && ptx->HasWitness()) { m_recent_rejects.insert(ptx->GetHash().ToUint256()); m_txrequest.ForgetTxHash(ptx->GetHash()); @@ -3153,6 +3249,116 @@ void PeerManagerImpl::ProcessValidTx(NodeId nodeid, const CTransactionRef& tx, c } } +void PeerManagerImpl::ProcessPackageResult(const PackageToValidate& package_to_validate, const PackageMempoolAcceptResult& package_result) +{ + AssertLockNotHeld(m_peer_mutex); + AssertLockHeld(g_msgproc_mutex); + AssertLockHeld(cs_main); + + const auto& package = package_to_validate.m_txns; + const auto& senders = package_to_validate.m_senders; + + if (package_result.m_state.IsInvalid()) { + m_recent_rejects_reconsiderable.insert(GetPackageHash(package)); + } + // We currently only expect to process 1-parent-1-child packages. Remove if this changes. + if (!Assume(package.size() == 2)) return; + + // Iterate backwards to erase in-package descendants from the orphanage before they become + // relevant in AddChildrenToWorkSet. + auto package_iter = package.rbegin(); + auto senders_iter = senders.rbegin(); + while (package_iter != package.rend()) { + const auto& tx = *package_iter; + const NodeId nodeid = *senders_iter; + const auto it_result{package_result.m_tx_results.find(tx->GetWitnessHash())}; + + // It is not guaranteed that a result exists for every transaction. + if (it_result != package_result.m_tx_results.end()) { + const auto& tx_result = it_result->second; + switch (tx_result.m_result_type) { + case MempoolAcceptResult::ResultType::VALID: + { + ProcessValidTx(nodeid, tx, tx_result.m_replaced_transactions); + break; + } + case MempoolAcceptResult::ResultType::INVALID: + case MempoolAcceptResult::ResultType::DIFFERENT_WITNESS: + { + // Don't add to vExtraTxnForCompact, as these transactions should have already been + // added there when added to the orphanage or rejected for TX_RECONSIDERABLE. + // This should be updated if package submission is ever used for transactions + // that haven't already been validated before. + ProcessInvalidTx(nodeid, tx, tx_result.m_state, /*maybe_add_extra_compact_tx=*/false); + break; + } + case MempoolAcceptResult::ResultType::MEMPOOL_ENTRY: + { + // AlreadyHaveTx() should be catching transactions that are already in mempool. + Assume(false); + break; + } + } + } + package_iter++; + senders_iter++; + } +} + +std::optional<PeerManagerImpl::PackageToValidate> PeerManagerImpl::Find1P1CPackage(const CTransactionRef& ptx, NodeId nodeid) +{ + AssertLockNotHeld(m_peer_mutex); + AssertLockHeld(g_msgproc_mutex); + AssertLockHeld(cs_main); + + const auto& parent_wtxid{ptx->GetWitnessHash()}; + + Assume(m_recent_rejects_reconsiderable.contains(parent_wtxid.ToUint256())); + + // Prefer children from this peer. This helps prevent censorship attempts in which an attacker + // sends lots of fake children for the parent, and we (unluckily) keep selecting the fake + // children instead of the real one provided by the honest peer. + const auto cpfp_candidates_same_peer{m_orphanage.GetChildrenFromSamePeer(ptx, nodeid)}; + + // These children should be sorted from newest to oldest. In the (probably uncommon) case + // of children that replace each other, this helps us accept the highest feerate (probably the + // most recent) one efficiently. + for (const auto& child : cpfp_candidates_same_peer) { + Package maybe_cpfp_package{ptx, child}; + if (!m_recent_rejects_reconsiderable.contains(GetPackageHash(maybe_cpfp_package))) { + return PeerManagerImpl::PackageToValidate{ptx, child, nodeid, nodeid}; + } + } + + // If no suitable candidate from the same peer is found, also try children that were provided by + // a different peer. This is useful because sometimes multiple peers announce both transactions + // to us, and we happen to download them from different peers (we wouldn't have known that these + // 2 transactions are related). We still want to find 1p1c packages then. + // + // If we start tracking all announcers of orphans, we can restrict this logic to parent + child + // pairs in which both were provided by the same peer, i.e. delete this step. + const auto cpfp_candidates_different_peer{m_orphanage.GetChildrenFromDifferentPeer(ptx, nodeid)}; + + // Find the first 1p1c that hasn't already been rejected. We randomize the order to not + // create a bias that attackers can use to delay package acceptance. + // + // Create a random permutation of the indices. + std::vector<size_t> tx_indices(cpfp_candidates_different_peer.size()); + std::iota(tx_indices.begin(), tx_indices.end(), 0); + Shuffle(tx_indices.begin(), tx_indices.end(), m_rng); + + for (const auto index : tx_indices) { + // If we already tried a package and failed for any reason, the combined hash was + // cached in m_recent_rejects_reconsiderable. + const auto [child_tx, child_sender] = cpfp_candidates_different_peer.at(index); + Package maybe_cpfp_package{ptx, child_tx}; + if (!m_recent_rejects_reconsiderable.contains(GetPackageHash(maybe_cpfp_package))) { + return PeerManagerImpl::PackageToValidate{ptx, child_tx, nodeid, child_sender}; + } + } + return std::nullopt; +} + bool PeerManagerImpl::ProcessOrphanTx(Peer& peer) { AssertLockHeld(g_msgproc_mutex); @@ -3168,9 +3374,7 @@ bool PeerManagerImpl::ProcessOrphanTx(Peer& peer) if (result.m_result_type == MempoolAcceptResult::ResultType::VALID) { LogPrint(BCLog::TXPACKAGES, " accepted orphan tx %s (wtxid=%s)\n", orphanHash.ToString(), orphan_wtxid.ToString()); - Assume(result.m_replaced_transactions.has_value()); - std::list<CTransactionRef> empty_replacement_list; - ProcessValidTx(peer.m_id, porphanTx, result.m_replaced_transactions.value_or(empty_replacement_list)); + ProcessValidTx(peer.m_id, porphanTx, result.m_replaced_transactions); return true; } else if (state.GetResult() != TxValidationResult::TX_MISSING_INPUTS) { LogPrint(BCLog::TXPACKAGES, " invalid orphan tx %s (wtxid=%s) from peer=%d. %s\n", @@ -3666,12 +3870,12 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, peer->m_starting_height, addrMe.ToStringAddrPort(), fRelay, pfrom.GetId(), remoteAddr, (mapped_as ? strprintf(", mapped_as=%d", mapped_as) : "")); - int64_t nTimeOffset = nTime - GetTime(); - pfrom.nTimeOffset = nTimeOffset; + peer->m_time_offset = NodeSeconds{std::chrono::seconds{nTime}} - Now<NodeSeconds>(); if (!pfrom.IsInboundConn()) { // Don't use timedata samples from inbound peers to make it - // harder for others to tamper with our adjusted time. - AddTimeData(pfrom.addr, nTimeOffset); + // harder for others to create false warnings about our clock being out of sync. + m_outbound_time_offsets.Add(peer->m_time_offset); + m_outbound_time_offsets.WarnIfOutOfSync(); } // If the peer is old enough to have the old alert system, send it the final alert. @@ -4013,7 +4217,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } const GenTxid gtxid = ToGenTxid(inv); - const bool fAlreadyHave = AlreadyHaveTx(gtxid); + const bool fAlreadyHave = AlreadyHaveTx(gtxid, /*include_reconsiderable=*/true); LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom.GetId()); AddKnownTx(*peer, inv.hash); @@ -4318,7 +4522,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // already; and an adversary can already relay us old transactions // (older than our recency filter) if trying to DoS us, without any need // for witness malleation. - if (AlreadyHaveTx(GenTxid::Wtxid(wtxid))) { + if (AlreadyHaveTx(GenTxid::Wtxid(wtxid), /*include_reconsiderable=*/true)) { if (pfrom.HasPermission(NetPermissionFlags::ForceRelay)) { // Always relay transactions received from peers with forcerelay // permission, even if they were already in the mempool, allowing @@ -4332,6 +4536,20 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, RelayTransaction(tx.GetHash(), tx.GetWitnessHash()); } } + + if (m_recent_rejects_reconsiderable.contains(wtxid)) { + // When a transaction is already in m_recent_rejects_reconsiderable, we shouldn't submit + // it by itself again. However, look for a matching child in the orphanage, as it is + // possible that they succeed as a package. + LogPrint(BCLog::TXPACKAGES, "found tx %s (wtxid=%s) in reconsiderable rejects, looking for child in orphanage\n", + txid.ToString(), wtxid.ToString()); + if (auto package_to_validate{Find1P1CPackage(ptx, pfrom.GetId())}) { + const auto package_result{ProcessNewPackage(m_chainman.ActiveChainstate(), m_mempool, package_to_validate->m_txns, /*test_accept=*/false, /*client_maxfeerate=*/std::nullopt)}; + LogDebug(BCLog::TXPACKAGES, "package evaluation for %s: %s\n", package_to_validate->ToString(), + package_result.m_state.IsValid() ? "package accepted" : "package rejected"); + ProcessPackageResult(package_to_validate.value(), package_result); + } + } // If a tx is detected by m_recent_rejects it is ignored. Because we haven't // submitted the tx to our mempool, we won't have computed a DoS // score for it or determined exactly why we consider it invalid. @@ -4354,7 +4572,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, const TxValidationState& state = result.m_state; if (result.m_result_type == MempoolAcceptResult::ResultType::VALID) { - ProcessValidTx(pfrom.GetId(), ptx, result.m_replaced_transactions.value()); + ProcessValidTx(pfrom.GetId(), ptx, result.m_replaced_transactions); pfrom.m_last_tx_time = GetTime<std::chrono::seconds>(); } else if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) @@ -4371,10 +4589,23 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } std::sort(unique_parents.begin(), unique_parents.end()); unique_parents.erase(std::unique(unique_parents.begin(), unique_parents.end()), unique_parents.end()); + + // Distinguish between parents in m_recent_rejects and m_recent_rejects_reconsiderable. + // We can tolerate having up to 1 parent in m_recent_rejects_reconsiderable since we + // submit 1p1c packages. However, fail immediately if any are in m_recent_rejects. + std::optional<uint256> rejected_parent_reconsiderable; for (const uint256& parent_txid : unique_parents) { if (m_recent_rejects.contains(parent_txid)) { fRejectedParents = true; break; + } else if (m_recent_rejects_reconsiderable.contains(parent_txid) && !m_mempool.exists(GenTxid::Txid(parent_txid))) { + // More than 1 parent in m_recent_rejects_reconsiderable: 1p1c will not be + // sufficient to accept this package, so just give up here. + if (rejected_parent_reconsiderable.has_value()) { + fRejectedParents = true; + break; + } + rejected_parent_reconsiderable = parent_txid; } } if (!fRejectedParents) { @@ -4388,7 +4619,9 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // protocol for getting all unconfirmed parents. const auto gtxid{GenTxid::Txid(parent_txid)}; AddKnownTx(*peer, parent_txid); - if (!AlreadyHaveTx(gtxid)) AddTxAnnouncement(pfrom, gtxid, current_time); + // Exclude m_recent_rejects_reconsiderable: the missing parent may have been + // previously rejected for being too low feerate. This orphan might CPFP it. + if (!AlreadyHaveTx(gtxid, /*include_reconsiderable=*/false)) AddTxAnnouncement(pfrom, gtxid, current_time); } if (m_orphanage.AddTx(ptx, pfrom.GetId())) { @@ -4420,6 +4653,19 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (state.IsInvalid()) { ProcessInvalidTx(pfrom.GetId(), ptx, state, /*maybe_add_extra_compact_tx=*/true); } + // When a transaction fails for TX_RECONSIDERABLE, look for a matching child in the + // orphanage, as it is possible that they succeed as a package. + if (state.GetResult() == TxValidationResult::TX_RECONSIDERABLE) { + LogPrint(BCLog::TXPACKAGES, "tx %s (wtxid=%s) failed but reconsiderable, looking for child in orphanage\n", + txid.ToString(), wtxid.ToString()); + if (auto package_to_validate{Find1P1CPackage(ptx, pfrom.GetId())}) { + const auto package_result{ProcessNewPackage(m_chainman.ActiveChainstate(), m_mempool, package_to_validate->m_txns, /*test_accept=*/false, /*client_maxfeerate=*/std::nullopt)}; + LogDebug(BCLog::TXPACKAGES, "package evaluation for %s: %s\n", package_to_validate->ToString(), + package_result.m_state.IsValid() ? "package accepted" : "package rejected"); + ProcessPackageResult(package_to_validate.value(), package_result); + } + } + return; } @@ -4951,7 +5197,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, std::vector<unsigned char> vData; vRecv >> vData; - // Nodes must NEVER send a data item > 520 bytes (the max size for a script data object, + // Nodes must NEVER send a data item > MAX_SCRIPT_ELEMENT_SIZE bytes (the max size for a script data object, // and thus, the maximum size any matched object can have) in a filteradd message bool bad = false; if (vData.size() > MAX_SCRIPT_ELEMENT_SIZE) { @@ -6029,7 +6275,9 @@ bool PeerManagerImpl::SendMessages(CNode* pto) entry.second.GetHash().ToString(), entry.first); } for (const GenTxid& gtxid : requestable) { - if (!AlreadyHaveTx(gtxid)) { + // Exclude m_recent_rejects_reconsiderable: we may be requesting a missing parent + // that was previously rejected for being too low feerate. + if (!AlreadyHaveTx(gtxid, /*include_reconsiderable=*/false)) { LogPrint(BCLog::NET, "Requesting %s %s peer=%d\n", gtxid.IsWtxid() ? "wtx" : "tx", gtxid.GetHash().ToString(), pto->GetId()); vGetData.emplace_back(gtxid.IsWtxid() ? MSG_WTX : (MSG_TX | GetFetchFlags(*peer)), gtxid.GetHash()); diff --git a/src/net_processing.h b/src/net_processing.h index f8d7a8f511..85e399d948 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -9,6 +9,8 @@ #include <net.h> #include <validationinterface.h> +#include <chrono> + class AddrMan; class CChainParams; class CTxMemPool; @@ -41,6 +43,12 @@ struct CNodeStateStats { bool m_addr_relay_enabled{false}; ServiceFlags their_services; int64_t presync_height{-1}; + std::chrono::seconds time_offset{0}; +}; + +struct PeerManagerInfo { + std::chrono::seconds median_outbound_time_offset{0s}; + bool ignores_incoming_txs{false}; }; class PeerManager : public CValidationInterface, public NetEventsInterface @@ -83,8 +91,8 @@ public: /** Get statistics from node state */ virtual bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) const = 0; - /** Whether this node ignores txs received over p2p. */ - virtual bool IgnoresIncomingTxs() = 0; + /** Get peer manager info. */ + virtual PeerManagerInfo GetInfo() const = 0; /** Relay transaction to all peers. */ virtual void RelayTransaction(const uint256& txid, const uint256& wtxid) = 0; diff --git a/src/netbase.cpp b/src/netbase.cpp index 3ca1a5227a..f0fa298378 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -632,10 +632,7 @@ std::unique_ptr<Sock> ConnectDirectly(const CService& dest, bool manual_connecti std::unique_ptr<Sock> Proxy::Connect() const { - if (!IsValid()) { - LogPrintf("Cannot connect to invalid Proxy\n"); - return {}; - } + if (!IsValid()) return {}; if (!m_is_unix_socket) return ConnectDirectly(proxy, /*manual_connection=*/true); @@ -656,7 +653,6 @@ std::unique_ptr<Sock> Proxy::Connect() const socklen_t len = sizeof(addrun); if(!ConnectToSocket(*sock, (struct sockaddr*)&addrun, len, path, /*manual_connection=*/true)) { - LogPrintf("Cannot connect to socket for %s\n", path); return {}; } diff --git a/src/node/timeoffsets.cpp b/src/node/timeoffsets.cpp new file mode 100644 index 0000000000..62f527be8a --- /dev/null +++ b/src/node/timeoffsets.cpp @@ -0,0 +1,69 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <logging.h> +#include <node/interface_ui.h> +#include <node/timeoffsets.h> +#include <sync.h> +#include <tinyformat.h> +#include <util/time.h> +#include <util/translation.h> +#include <warnings.h> + +#include <algorithm> +#include <chrono> +#include <cstdint> +#include <deque> +#include <limits> +#include <optional> + +using namespace std::chrono_literals; + +void TimeOffsets::Add(std::chrono::seconds offset) +{ + LOCK(m_mutex); + + if (m_offsets.size() >= MAX_SIZE) { + m_offsets.pop_front(); + } + m_offsets.push_back(offset); + LogDebug(BCLog::NET, "Added time offset %+ds, total samples %d\n", + Ticks<std::chrono::seconds>(offset), m_offsets.size()); +} + +std::chrono::seconds TimeOffsets::Median() const +{ + LOCK(m_mutex); + + // Only calculate the median if we have 5 or more offsets + if (m_offsets.size() < 5) return 0s; + + auto sorted_copy = m_offsets; + std::sort(sorted_copy.begin(), sorted_copy.end()); + return sorted_copy[sorted_copy.size() / 2]; // approximate median is good enough, keep it simple +} + +bool TimeOffsets::WarnIfOutOfSync() const +{ + // when median == std::numeric_limits<int64_t>::min(), calling std::chrono::abs is UB + auto median{std::max(Median(), std::chrono::seconds(std::numeric_limits<int64_t>::min() + 1))}; + if (std::chrono::abs(median) <= WARN_THRESHOLD) { + SetMedianTimeOffsetWarning(std::nullopt); + uiInterface.NotifyAlertChanged(); + return false; + } + + bilingual_str msg{strprintf(_( + "Your computer's date and time appear to be more than %d minutes out of sync with the network, " + "this may lead to consensus failure. After you've confirmed your computer's clock, this message " + "should no longer appear when you restart your node. Without a restart, it should stop showing " + "automatically after you've connected to a sufficient number of new outbound peers, which may " + "take some time. You can inspect the `timeoffset` field of the `getpeerinfo` and `getnetworkinfo` " + "RPC methods to get more info." + ), Ticks<std::chrono::minutes>(WARN_THRESHOLD))}; + LogWarning("%s\n", msg.original); + SetMedianTimeOffsetWarning(msg); + uiInterface.NotifyAlertChanged(); + return true; +} diff --git a/src/node/timeoffsets.h b/src/node/timeoffsets.h new file mode 100644 index 0000000000..2b12584e12 --- /dev/null +++ b/src/node/timeoffsets.h @@ -0,0 +1,39 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_TIMEOFFSETS_H +#define BITCOIN_NODE_TIMEOFFSETS_H + +#include <sync.h> + +#include <chrono> +#include <cstddef> +#include <deque> + +class TimeOffsets +{ + //! Maximum number of timeoffsets stored. + static constexpr size_t MAX_SIZE{50}; + //! Minimum difference between system and network time for a warning to be raised. + static constexpr std::chrono::minutes WARN_THRESHOLD{10}; + + mutable Mutex m_mutex; + /** The observed time differences between our local clock and those of our outbound peers. A + * positive offset means our peer's clock is ahead of our local clock. */ + std::deque<std::chrono::seconds> m_offsets GUARDED_BY(m_mutex){}; + +public: + /** Add a new time offset sample. */ + void Add(std::chrono::seconds offset) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); + + /** Compute and return the median of the collected time offset samples. The median is returned + * as 0 when there are less than 5 samples. */ + std::chrono::seconds Median() const EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); + + /** Raise warnings if the median time offset exceeds the warnings threshold. Returns true if + * warnings were raised. */ + bool WarnIfOutOfSync() const EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); +}; + +#endif // BITCOIN_NODE_TIMEOFFSETS_H diff --git a/src/outputtype.cpp b/src/outputtype.cpp index 566e5ec55a..c72d9deacb 100644 --- a/src/outputtype.cpp +++ b/src/outputtype.cpp @@ -85,7 +85,7 @@ CTxDestination AddAndGetDestinationForScript(FillableSigningProvider& keystore, { // Add script to keystore keystore.AddCScript(script); - // Note that scripts over 520 bytes are not yet supported. + // Note that scripts over MAX_SCRIPT_ELEMENT_SIZE bytes are not yet supported. switch (type) { case OutputType::LEGACY: return ScriptHash(script); diff --git a/src/policy/packages.cpp b/src/policy/packages.cpp index 3a63a9fe46..693adcdfd0 100644 --- a/src/policy/packages.cpp +++ b/src/policy/packages.cpp @@ -147,3 +147,24 @@ bool IsChildWithParentsTree(const Package& package) return true; }); } + +uint256 GetPackageHash(const std::vector<CTransactionRef>& transactions) +{ + // Create a vector of the wtxids. + std::vector<Wtxid> wtxids_copy; + std::transform(transactions.cbegin(), transactions.cend(), std::back_inserter(wtxids_copy), + [](const auto& tx){ return tx->GetWitnessHash(); }); + + // Sort in ascending order + std::sort(wtxids_copy.begin(), wtxids_copy.end(), [](const auto& lhs, const auto& rhs) { + return std::lexicographical_compare(std::make_reverse_iterator(lhs.end()), std::make_reverse_iterator(lhs.begin()), + std::make_reverse_iterator(rhs.end()), std::make_reverse_iterator(rhs.begin())); + }); + + // Get sha256 hash of the wtxids concatenated in this order + HashWriter hashwriter; + for (const auto& wtxid : wtxids_copy) { + hashwriter << wtxid; + } + return hashwriter.GetSHA256(); +} diff --git a/src/policy/packages.h b/src/policy/packages.h index 537d8476e2..3050320122 100644 --- a/src/policy/packages.h +++ b/src/policy/packages.h @@ -88,4 +88,9 @@ bool IsChildWithParents(const Package& package); * other (the package is a "tree"). */ bool IsChildWithParentsTree(const Package& package); + +/** Get the hash of these transactions' wtxids, concatenated in lexicographical order (treating the + * wtxids as little endian encoded uint256, smallest to largest). */ +uint256 GetPackageHash(const std::vector<CTransactionRef>& transactions); + #endif // BITCOIN_POLICY_PACKAGES_H diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index d08ec4fb7f..d8b4b907e4 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -111,7 +111,7 @@ bool IsStandardTx(const CTransaction& tx, const std::optional<unsigned>& max_dat for (const CTxIn& txin : tx.vin) { // Biggest 'standard' txin involving only keys is a 15-of-15 P2SH - // multisig with compressed keys (remember the 520 byte limit on + // multisig with compressed keys (remember the MAX_SCRIPT_ELEMENT_SIZE byte limit on // redeemScript size). That works out to a (15*(33+1))+3=513 byte // redeemScript, 513+1+15*(73+1)+3=1627 bytes of scriptSig, which // we round off to 1650(MAX_STANDARD_SCRIPTSIG_SIZE) bytes for diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index 84c3352b9d..2ad79b6f99 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -191,13 +191,13 @@ std::optional<std::pair<DiagramCheckError, std::string>> ImprovesFeerateDiagram( int64_t replacement_vsize) { // Require that the replacement strictly improves the mempool's feerate diagram. - const auto diagram_results{pool.CalculateFeerateDiagramsForRBF(replacement_fees, replacement_vsize, direct_conflicts, all_conflicts)}; + const auto chunk_results{pool.CalculateChunksForRBF(replacement_fees, replacement_vsize, direct_conflicts, all_conflicts)}; - if (!diagram_results.has_value()) { - return std::make_pair(DiagramCheckError::UNCALCULABLE, util::ErrorString(diagram_results).original); + if (!chunk_results.has_value()) { + return std::make_pair(DiagramCheckError::UNCALCULABLE, util::ErrorString(chunk_results).original); } - if (!std::is_gt(CompareFeerateDiagram(diagram_results.value().second, diagram_results.value().first))) { + if (!std::is_gt(CompareChunks(chunk_results.value().second, chunk_results.value().first))) { return std::make_pair(DiagramCheckError::FAILURE, "insufficient feerate: does not improve feerate diagram"); } return std::nullopt; diff --git a/src/qt/addresstablemodel.cpp b/src/qt/addresstablemodel.cpp index c52ef7cd67..efdc3966d1 100644 --- a/src/qt/addresstablemodel.cpp +++ b/src/qt/addresstablemodel.cpp @@ -369,21 +369,22 @@ QString AddressTableModel::addRow(const QString &type, const QString &label, con else if(type == Receive) { // Generate a new address to associate with given label - auto op_dest = walletModel->wallet().getNewDestination(address_type, strLabel); - if (!op_dest) { + if (auto dest{walletModel->wallet().getNewDestination(address_type, strLabel)}) { + strAddress = EncodeDestination(*dest); + } else { WalletModel::UnlockContext ctx(walletModel->requestUnlock()); if (!ctx.isValid()) { // Unlock wallet failed or was cancelled editStatus = WALLET_UNLOCK_FAILURE; return QString(); } - op_dest = walletModel->wallet().getNewDestination(address_type, strLabel); - if (!op_dest) { + if (auto dest_retry{walletModel->wallet().getNewDestination(address_type, strLabel)}) { + strAddress = EncodeDestination(*dest_retry); + } else { editStatus = KEY_GENERATION_FAILURE; return QString(); } } - strAddress = EncodeDestination(*op_dest); } else { diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 1d1a84e375..e094b686d4 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -775,9 +775,9 @@ QString formatPingTime(std::chrono::microseconds ping_time) QObject::tr("%1 ms").arg(QString::number((int)(count_microseconds(ping_time) / 1000), 10)); } -QString formatTimeOffset(int64_t nTimeOffset) +QString formatTimeOffset(int64_t time_offset) { - return QObject::tr("%1 s").arg(QString::number((int)nTimeOffset, 10)); + return QObject::tr("%1 s").arg(QString::number((int)time_offset, 10)); } QString formatNiceTimeOffset(qint64 secs) diff --git a/src/qt/guiutil.h b/src/qt/guiutil.h index 7f06fdfe37..3e28e54557 100644 --- a/src/qt/guiutil.h +++ b/src/qt/guiutil.h @@ -240,8 +240,8 @@ namespace GUIUtil /** Format a CNodeStats.m_last_ping_time into a user-readable string or display N/A, if 0 */ QString formatPingTime(std::chrono::microseconds ping_time); - /** Format a CNodeCombinedStats.nTimeOffset into a user-readable string */ - QString formatTimeOffset(int64_t nTimeOffset); + /** Format a CNodeStateStats.time_offset into a user-readable string */ + QString formatTimeOffset(int64_t time_offset); QString formatNiceTimeOffset(qint64 secs); diff --git a/src/qt/main.cpp b/src/qt/main.cpp index ded057dbfa..16befd99e8 100644 --- a/src/qt/main.cpp +++ b/src/qt/main.cpp @@ -4,7 +4,6 @@ #include <qt/bitcoin.h> -#include <common/url.h> #include <compat/compat.h> #include <util/translation.h> @@ -17,7 +16,6 @@ extern const std::function<std::string(const char*)> G_TRANSLATION_FUN = [](const char* psz) { return QCoreApplication::translate("bitcoin-core", psz).toStdString(); }; -UrlDecodeFn* const URL_DECODE = urlDecode; const std::function<std::string()> G_TEST_GET_FULL_NAME{}; diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index d2b184ebdf..bafef62945 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -23,6 +23,7 @@ #include <rpc/server.h> #include <util/strencodings.h> #include <util/string.h> +#include <util/time.h> #include <util/threadnames.h> #include <univalue.h> @@ -1196,7 +1197,6 @@ void RPCConsole::updateDetailWidget() ui->peerBytesRecv->setText(GUIUtil::formatBytes(stats->nodeStats.nRecvBytes)); ui->peerPingTime->setText(GUIUtil::formatPingTime(stats->nodeStats.m_last_ping_time)); ui->peerMinPing->setText(GUIUtil::formatPingTime(stats->nodeStats.m_min_ping_time)); - ui->timeoffset->setText(GUIUtil::formatTimeOffset(stats->nodeStats.nTimeOffset)); if (stats->nodeStats.nVersion) { ui->peerVersion->setText(QString::number(stats->nodeStats.nVersion)); } @@ -1228,6 +1228,7 @@ void RPCConsole::updateDetailWidget() // This check fails for example if the lock was busy and // nodeStateStats couldn't be fetched. if (stats->fNodeStateStatsAvailable) { + ui->timeoffset->setText(GUIUtil::formatTimeOffset(Ticks<std::chrono::seconds>(stats->nodeStateStats.time_offset))); ui->peerServices->setText(GUIUtil::formatServicesStr(stats->nodeStateStats.their_services)); // Sync height is init to -1 if (stats->nodeStateStats.nSyncHeight > -1) { diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index 1bdf94d3b5..fe000bcbb8 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -569,16 +569,17 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) return true; } -bool WalletModel::displayAddress(std::string sAddress) const +void WalletModel::displayAddress(std::string sAddress) const { CTxDestination dest = DecodeDestination(sAddress); - bool res = false; try { - res = m_wallet->displayAddress(dest); + util::Result<void> result = m_wallet->displayAddress(dest); + if (!result) { + QMessageBox::warning(nullptr, tr("Signer error"), QString::fromStdString(util::ErrorString(result).translated)); + } } catch (const std::runtime_error& e) { QMessageBox::critical(nullptr, tr("Can't display address"), e.what()); } - return res; } bool WalletModel::isWalletEnabled() diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 503ee16823..ab2096c1fe 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -130,7 +130,7 @@ public: UnlockContext requestUnlock(); bool bumpFee(uint256 hash, uint256& new_hash); - bool displayAddress(std::string sAddress) const; + void displayAddress(std::string sAddress) const; static bool isWalletEnabled(); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index a1135c27d4..eed004806a 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2174,7 +2174,8 @@ static RPCHelpMan scantxoutset() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { UniValue result(UniValue::VOBJ); - if (request.params[0].get_str() == "status") { + const auto action{self.Arg<std::string>("action")}; + if (action == "status") { CoinsViewScanReserver reserver; if (reserver.reserve()) { // no scan in progress @@ -2182,7 +2183,7 @@ static RPCHelpMan scantxoutset() } result.pushKV("progress", g_scan_progress.load()); return result; - } else if (request.params[0].get_str() == "abort") { + } else if (action == "abort") { CoinsViewScanReserver reserver; if (reserver.reserve()) { // reserve was possible which means no scan was running @@ -2191,7 +2192,7 @@ static RPCHelpMan scantxoutset() // set the abort flag g_should_abort_scan = true; return true; - } else if (request.params[0].get_str() == "start") { + } else if (action == "start") { CoinsViewScanReserver reserver; if (!reserver.reserve()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Scan already in progress, use action \"abort\" or \"status\""); @@ -2260,7 +2261,7 @@ static RPCHelpMan scantxoutset() result.pushKV("unspents", unspents); result.pushKV("total_amount", ValueFromAmount(total_in)); } else { - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid action '%s'", request.params[0].get_str())); + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid action '%s'", action)); } return result; }, diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 920bb9ea7f..e6fe3c7ebd 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -994,10 +994,8 @@ static RPCHelpMan submitpackage() fees.pushKV("effective-includes", effective_includes_res); } result_inner.pushKV("fees", fees); - if (it->second.m_replaced_transactions.has_value()) { - for (const auto& ptx : it->second.m_replaced_transactions.value()) { - replaced_txids.insert(ptx->GetHash()); - } + for (const auto& ptx : it->second.m_replaced_transactions) { + replaced_txids.insert(ptx->GetHash()); } break; } diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index f7cdbf52dd..454c262803 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -122,7 +122,7 @@ static RPCHelpMan getnetworkhashps() { ChainstateManager& chainman = EnsureAnyChainman(request.context); LOCK(cs_main); - return GetNetworkHashPS(self.Arg<int>(0), self.Arg<int>(1), chainman.ActiveChain()); + return GetNetworkHashPS(self.Arg<int>("nblocks"), self.Arg<int>("height"), chainman.ActiveChain()); }, }; } @@ -229,12 +229,12 @@ static RPCHelpMan generatetodescriptor() "\nGenerate 11 blocks to mydesc\n" + HelpExampleCli("generatetodescriptor", "11 \"mydesc\"")}, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - const auto num_blocks{self.Arg<int>(0)}; - const auto max_tries{self.Arg<uint64_t>(2)}; + const auto num_blocks{self.Arg<int>("num_blocks")}; + const auto max_tries{self.Arg<uint64_t>("maxtries")}; CScript coinbase_script; std::string error; - if (!getScriptFromDescriptor(self.Arg<std::string>(1), coinbase_script, error)) { + if (!getScriptFromDescriptor(self.Arg<std::string>("descriptor"), coinbase_script, error)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error); } diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index f935a3b08f..739ff1a48e 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -23,7 +23,6 @@ #include <rpc/server_util.h> #include <rpc/util.h> #include <sync.h> -#include <timedata.h> #include <util/chaintype.h> #include <util/strencodings.h> #include <util/string.h> @@ -239,7 +238,7 @@ static RPCHelpMan getpeerinfo() obj.pushKV("bytessent", stats.nSendBytes); obj.pushKV("bytesrecv", stats.nRecvBytes); obj.pushKV("conntime", count_seconds(stats.m_connected)); - obj.pushKV("timeoffset", stats.nTimeOffset); + obj.pushKV("timeoffset", Ticks<std::chrono::seconds>(statestats.time_offset)); if (stats.m_last_ping_time > 0us) { obj.pushKV("pingtime", Ticks<SecondsDouble>(stats.m_last_ping_time)); } @@ -322,7 +321,7 @@ static RPCHelpMan addnode() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - const std::string command{request.params[1].get_str()}; + const auto command{self.Arg<std::string>("command")}; if (command != "onetry" && command != "add" && command != "remove") { throw std::runtime_error( self.ToString()); @@ -331,9 +330,9 @@ static RPCHelpMan addnode() NodeContext& node = EnsureAnyNodeContext(request.context); CConnman& connman = EnsureConnman(node); - const std::string node_arg{request.params[0].get_str()}; + const auto node_arg{self.Arg<std::string>("node")}; bool node_v2transport = connman.GetLocalServices() & NODE_P2P_V2; - bool use_v2transport = self.MaybeArg<bool>(2).value_or(node_v2transport); + bool use_v2transport = self.MaybeArg<bool>("v2transport").value_or(node_v2transport); if (use_v2transport && !node_v2transport) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Error: v2transport requested but not enabled (see -v2transport)"); @@ -679,9 +678,10 @@ static RPCHelpMan getnetworkinfo() obj.pushKV("localservicesnames", GetServicesNames(services)); } if (node.peerman) { - obj.pushKV("localrelay", !node.peerman->IgnoresIncomingTxs()); + auto peerman_info{node.peerman->GetInfo()}; + obj.pushKV("localrelay", !peerman_info.ignores_incoming_txs); + obj.pushKV("timeoffset", Ticks<std::chrono::seconds>(peerman_info.median_outbound_time_offset)); } - obj.pushKV("timeoffset", GetTimeOffset()); if (node.connman) { obj.pushKV("networkactive", node.connman->GetNetworkActive()); obj.pushKV("connections", node.connman->GetNodeCount(ConnectionDirection::Both)); diff --git a/src/rpc/signmessage.cpp b/src/rpc/signmessage.cpp index 8c752ba1fd..9f3c24efcf 100644 --- a/src/rpc/signmessage.cpp +++ b/src/rpc/signmessage.cpp @@ -38,9 +38,9 @@ static RPCHelpMan verifymessage() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::string strAddress = request.params[0].get_str(); - std::string strSign = request.params[1].get_str(); - std::string strMessage = request.params[2].get_str(); + std::string strAddress = self.Arg<std::string>("address"); + std::string strSign = self.Arg<std::string>("signature"); + std::string strMessage = self.Arg<std::string>("message"); switch (MessageVerify(strAddress, strSign, strMessage)) { case MessageVerificationResult::ERR_INVALID_ADDRESS: diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 6e332e3855..f683878054 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -24,6 +24,8 @@ #include <util/string.h> #include <util/translation.h> +#include <algorithm> +#include <iterator> #include <string_view> #include <tuple> @@ -729,6 +731,16 @@ std::vector<std::pair<std::string, bool>> RPCHelpMan::GetArgNames() const return ret; } +size_t RPCHelpMan::GetParamIndex(std::string_view key) const +{ + auto it{std::find_if( + m_args.begin(), m_args.end(), [&key](const auto& arg) { return arg.GetName() == key;} + )}; + + CHECK_NONFATAL(it != m_args.end()); // TODO: ideally this is checked at compile time + return std::distance(m_args.begin(), it); +} + std::string RPCHelpMan::ToString() const { std::string ret; diff --git a/src/rpc/util.h b/src/rpc/util.h index f6ee6a317a..177af90c05 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -404,18 +404,25 @@ public: UniValue HandleRequest(const JSONRPCRequest& request) const; /** - * Helper to get a request argument. - * This function only works during m_fun(), i.e. it should only be used in - * RPC method implementations. The helper internally checks whether the - * user-passed argument isNull() and parses (from JSON) and returns the - * user-passed argument, or the default value derived from the RPCArg - * documentation, or a falsy value if no default was given. + * @brief Helper to get a required or default-valued request argument. * - * Use Arg<Type>(i) to get the argument or its default value. Otherwise, - * use MaybeArg<Type>(i) to get the optional argument or a falsy value. + * Use this function when the argument is required or when it has a default value. If the + * argument is optional and may not be provided, use MaybeArg instead. * - * The Type passed to this helper must match the corresponding - * RPCArg::Type. + * This function only works during m_fun(), i.e., it should only be used in + * RPC method implementations. It internally checks whether the user-passed + * argument isNull() and parses (from JSON) and returns the user-passed argument, + * or the default value derived from the RPCArg documentation. + * + * There are two overloads of this function: + * - Use Arg<Type>(size_t i) to get the argument (or the default value) by index. + * - Use Arg<Type>(const std::string& key) to get the argument (or the default value) by key. + * + * The Type passed to this helper must match the corresponding RPCArg::Type. + * + * @return The value of the RPC argument (or the default value) cast to type Type. + * + * @see MaybeArg for handling optional arguments without default values. */ template <typename R> auto Arg(size_t i) const @@ -429,6 +436,34 @@ public: return ArgValue<const R&>(i); } } + template<typename R> + auto Arg(std::string_view key) const + { + return Arg<R>(GetParamIndex(key)); + } + /** + * @brief Helper to get an optional request argument. + * + * Use this function when the argument is optional and does not have a default value. If the + * argument is required or has a default value, use Arg instead. + * + * This function only works during m_fun(), i.e., it should only be used in + * RPC method implementations. It internally checks whether the user-passed + * argument isNull() and parses (from JSON) and returns the user-passed argument, + * or a falsy value if no argument was passed. + * + * There are two overloads of this function: + * - Use MaybeArg<Type>(size_t i) to get the optional argument by index. + * - Use MaybeArg<Type>(const std::string& key) to get the optional argument by key. + * + * The Type passed to this helper must match the corresponding RPCArg::Type. + * + * @return For integral and floating-point types, a std::optional<Type> is returned. + * For other types, a Type* pointer to the argument is returned. If the + * argument is not provided, std::nullopt or a null pointer is returned. + * + * @see Arg for handling arguments that are required or have a default value. + */ template <typename R> auto MaybeArg(size_t i) const { @@ -441,6 +476,11 @@ public: return ArgValue<const R*>(i); } } + template<typename R> + auto MaybeArg(std::string_view key) const + { + return MaybeArg<R>(GetParamIndex(key)); + } std::string ToString() const; /** Return the named args that need to be converted from string to another JSON type */ UniValue GetArgMap() const; @@ -460,6 +500,8 @@ private: mutable const JSONRPCRequest* m_req{nullptr}; // A pointer to the request for the duration of m_fun() template <typename R> R ArgValue(size_t i) const; + //! Return positional index of a parameter using its name as key. + size_t GetParamIndex(std::string_view key) const; }; /** diff --git a/src/script/sign.cpp b/src/script/sign.cpp index be4b357568..22ac062a63 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -295,7 +295,7 @@ struct TapSatisfier: Satisfier<XOnlyPubKey> { //! Conversion from a raw xonly public key. template <typename I> std::optional<XOnlyPubKey> FromPKBytes(I first, I last) const { - CHECK_NONFATAL(last - first == 32); + if (last - first != 32) return {}; XOnlyPubKey pubkey; std::copy(first, last, pubkey.begin()); return pubkey; diff --git a/src/test/common_url_tests.cpp b/src/test/common_url_tests.cpp new file mode 100644 index 0000000000..065c7d97bc --- /dev/null +++ b/src/test/common_url_tests.cpp @@ -0,0 +1,75 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit/. + +#include <common/url.h> + +#include <string> + +#include <boost/test/unit_test.hpp> + +BOOST_AUTO_TEST_SUITE(common_url_tests) + +// These test vectors were ported from test/regress.c in the libevent library +// which used to be a dependency of the UrlDecode function. + +BOOST_AUTO_TEST_CASE(encode_decode_test) { + BOOST_CHECK_EQUAL(UrlDecode("Hello"), "Hello"); + BOOST_CHECK_EQUAL(UrlDecode("99"), "99"); + BOOST_CHECK_EQUAL(UrlDecode(""), ""); + BOOST_CHECK_EQUAL(UrlDecode("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ123456789-.~_"), + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ123456789-.~_"); + BOOST_CHECK_EQUAL(UrlDecode("%20"), " "); + BOOST_CHECK_EQUAL(UrlDecode("%FF%F0%E0"), "\xff\xf0\xe0"); + BOOST_CHECK_EQUAL(UrlDecode("%01%19"), "\x01\x19"); + BOOST_CHECK_EQUAL(UrlDecode("http%3A%2F%2Fwww.ietf.org%2Frfc%2Frfc3986.txt"), + "http://www.ietf.org/rfc/rfc3986.txt"); + BOOST_CHECK_EQUAL(UrlDecode("1%2B2%3D3"), "1+2=3"); +} + +BOOST_AUTO_TEST_CASE(decode_malformed_test) { + BOOST_CHECK_EQUAL(UrlDecode("%%xhello th+ere \xff"), "%%xhello th+ere \xff"); + + BOOST_CHECK_EQUAL(UrlDecode("%"), "%"); + BOOST_CHECK_EQUAL(UrlDecode("%%"), "%%"); + BOOST_CHECK_EQUAL(UrlDecode("%%%"), "%%%"); + BOOST_CHECK_EQUAL(UrlDecode("%%%%"), "%%%%"); + + BOOST_CHECK_EQUAL(UrlDecode("+"), "+"); + BOOST_CHECK_EQUAL(UrlDecode("++"), "++"); + + BOOST_CHECK_EQUAL(UrlDecode("?"), "?"); + BOOST_CHECK_EQUAL(UrlDecode("??"), "??"); + + BOOST_CHECK_EQUAL(UrlDecode("%G1"), "%G1"); + BOOST_CHECK_EQUAL(UrlDecode("%2"), "%2"); + BOOST_CHECK_EQUAL(UrlDecode("%ZX"), "%ZX"); + + BOOST_CHECK_EQUAL(UrlDecode("valid%20string%G1"), "valid string%G1"); + BOOST_CHECK_EQUAL(UrlDecode("%20invalid%ZX"), " invalid%ZX"); + BOOST_CHECK_EQUAL(UrlDecode("%20%G1%ZX"), " %G1%ZX"); + + BOOST_CHECK_EQUAL(UrlDecode("%1 "), "%1 "); + BOOST_CHECK_EQUAL(UrlDecode("% 9"), "% 9"); + BOOST_CHECK_EQUAL(UrlDecode(" %Z "), " %Z "); + BOOST_CHECK_EQUAL(UrlDecode(" % X"), " % X"); + + BOOST_CHECK_EQUAL(UrlDecode("%%ffg"), "%\xffg"); + BOOST_CHECK_EQUAL(UrlDecode("%fg"), "%fg"); + + BOOST_CHECK_EQUAL(UrlDecode("%-1"), "%-1"); + BOOST_CHECK_EQUAL(UrlDecode("%1-"), "%1-"); +} + +BOOST_AUTO_TEST_CASE(decode_lowercase_hex_test) { + BOOST_CHECK_EQUAL(UrlDecode("%f0%a0%b0"), "\xf0\xa0\xb0"); +} + +BOOST_AUTO_TEST_CASE(decode_internal_nulls_test) { + std::string result1{"\0\0x\0\0", 5}; + BOOST_CHECK_EQUAL(UrlDecode("%00%00x%00%00"), result1); + std::string result2{"abc\0\0", 5}; + BOOST_CHECK_EQUAL(UrlDecode("abc%00%00"), result2); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 0fef8c5906..5e9ae78681 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -16,7 +16,6 @@ #include <test/util/net.h> #include <test/util/random.h> #include <test/util/setup_common.h> -#include <timedata.h> #include <util/string.h> #include <util/time.h> #include <validation.h> @@ -72,7 +71,6 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) /*local_services=*/ServiceFlags(NODE_NETWORK | NODE_WITNESS), /*version=*/PROTOCOL_VERSION, /*relay_txs=*/true); - TestOnlyResetTimeData(); // This test requires that we have a chain with non-zero work. { diff --git a/src/test/feefrac_tests.cpp b/src/test/feefrac_tests.cpp index 2e015b382e..5af3c3d7ed 100644 --- a/src/test/feefrac_tests.cpp +++ b/src/test/feefrac_tests.cpp @@ -82,43 +82,4 @@ BOOST_AUTO_TEST_CASE(feefrac_operators) } -BOOST_AUTO_TEST_CASE(build_diagram_test) -{ - FeeFrac p1{1000, 100}; - FeeFrac empty{0, 0}; - FeeFrac zero_fee{0, 1}; - FeeFrac oversized_1{4611686000000, 4000000}; - FeeFrac oversized_2{184467440000000, 100000}; - - // Diagram-building will reorder chunks - std::vector<FeeFrac> chunks; - chunks.push_back(p1); - chunks.push_back(zero_fee); - chunks.push_back(empty); - chunks.push_back(oversized_1); - chunks.push_back(oversized_2); - - // Caller in charge of sorting chunks in case chunk size limit - // differs from cluster size limit - std::sort(chunks.begin(), chunks.end(), [](const FeeFrac& a, const FeeFrac& b) { return a > b; }); - - // Chunks are now sorted in reverse order (largest first) - BOOST_CHECK(chunks[0] == empty); // empty is considered "highest" fee - BOOST_CHECK(chunks[1] == oversized_2); - BOOST_CHECK(chunks[2] == oversized_1); - BOOST_CHECK(chunks[3] == p1); - BOOST_CHECK(chunks[4] == zero_fee); - - std::vector<FeeFrac> generated_diagram{BuildDiagramFromChunks(chunks)}; - BOOST_CHECK(generated_diagram.size() == 1 + chunks.size()); - - // Prepended with an empty, then the chunks summed in order as above - BOOST_CHECK(generated_diagram[0] == empty); - BOOST_CHECK(generated_diagram[1] == empty); - BOOST_CHECK(generated_diagram[2] == oversized_2); - BOOST_CHECK(generated_diagram[3] == oversized_2 + oversized_1); - BOOST_CHECK(generated_diagram[4] == oversized_2 + oversized_1 + p1); - BOOST_CHECK(generated_diagram[5] == oversized_2 + oversized_1 + p1 + zero_fee); -} - BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/fuzz/addition_overflow.cpp b/src/test/fuzz/addition_overflow.cpp index 5100b6f438..071e5fb029 100644 --- a/src/test/fuzz/addition_overflow.cpp +++ b/src/test/fuzz/addition_overflow.cpp @@ -24,12 +24,14 @@ void TestAdditionOverflow(FuzzedDataProvider& fuzzed_data_provider) assert(is_addition_overflow_custom == AdditionOverflow(j, i)); assert(maybe_add == CheckedAdd(j, i)); assert(sat_add == SaturatingAdd(j, i)); +#ifndef _MSC_VER T result_builtin; const bool is_addition_overflow_builtin = __builtin_add_overflow(i, j, &result_builtin); assert(is_addition_overflow_custom == is_addition_overflow_builtin); if (!is_addition_overflow_custom) { assert(i + j == result_builtin); } +#endif if (is_addition_overflow_custom) { assert(sat_add == std::numeric_limits<T>::min() || sat_add == std::numeric_limits<T>::max()); } else { diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index ebc5673e71..c9a3bc86ac 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -33,7 +33,6 @@ #include <optional> #include <stdexcept> #include <stdint.h> -#include <unistd.h> using node::SnapshotMetadata; diff --git a/src/test/fuzz/feeratediagram.cpp b/src/test/fuzz/feeratediagram.cpp index 6d710093cb..1a9c5ee946 100644 --- a/src/test/fuzz/feeratediagram.cpp +++ b/src/test/fuzz/feeratediagram.cpp @@ -16,6 +16,20 @@ namespace { +/** Takes the pre-computed and topologically-valid chunks and generates a fee diagram which starts at FeeFrac of (0, 0) */ +std::vector<FeeFrac> BuildDiagramFromChunks(const Span<const FeeFrac> chunks) +{ + std::vector<FeeFrac> diagram; + diagram.reserve(chunks.size() + 1); + + diagram.emplace_back(0, 0); + for (auto& chunk : chunks) { + diagram.emplace_back(diagram.back() + chunk); + } + return diagram; +} + + /** Evaluate a diagram at a specific size, returning the fee as a fraction. * * Fees in diagram cannot exceed 2^32, as the returned evaluation could overflow @@ -103,7 +117,7 @@ FUZZ_TARGET(build_and_compare_feerate_diagram) assert(diagram1.front() == empty); assert(diagram2.front() == empty); - auto real = CompareFeerateDiagram(diagram1, diagram2); + auto real = CompareChunks(chunks1, chunks2); auto sim = CompareDiagrams(diagram1, diagram2); assert(real == sim); diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index a8e490b459..f9915187bd 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -25,7 +25,6 @@ #include <memory> #include <string> #include <tuple> -#include <unistd.h> #include <utility> #include <vector> @@ -135,9 +134,9 @@ void initialize() #if defined(PROVIDE_FUZZ_MAIN_FUNCTION) static bool read_stdin(std::vector<uint8_t>& data) { - uint8_t buffer[1024]; - ssize_t length = 0; - while ((length = read(STDIN_FILENO, buffer, 1024)) > 0) { + std::istream::char_type buffer[1024]; + std::streamsize length; + while ((std::cin.read(buffer, 1024), length = std::cin.gcount()) > 0) { data.insert(data.end(), buffer, buffer + length); } return length == 0; diff --git a/src/test/fuzz/multiplication_overflow.cpp b/src/test/fuzz/multiplication_overflow.cpp index aeef4f24b7..a762a4dfe3 100644 --- a/src/test/fuzz/multiplication_overflow.cpp +++ b/src/test/fuzz/multiplication_overflow.cpp @@ -17,12 +17,18 @@ void TestMultiplicationOverflow(FuzzedDataProvider& fuzzed_data_provider) const T i = fuzzed_data_provider.ConsumeIntegral<T>(); const T j = fuzzed_data_provider.ConsumeIntegral<T>(); const bool is_multiplication_overflow_custom = MultiplicationOverflow(i, j); +#ifndef _MSC_VER T result_builtin; const bool is_multiplication_overflow_builtin = __builtin_mul_overflow(i, j, &result_builtin); assert(is_multiplication_overflow_custom == is_multiplication_overflow_builtin); if (!is_multiplication_overflow_custom) { assert(i * j == result_builtin); } +#else + if (!is_multiplication_overflow_custom) { + (void)(i * j); + } +#endif } } // namespace diff --git a/src/test/fuzz/partially_downloaded_block.cpp b/src/test/fuzz/partially_downloaded_block.cpp index ab75afe066..2bf47930f4 100644 --- a/src/test/fuzz/partially_downloaded_block.cpp +++ b/src/test/fuzz/partially_downloaded_block.cpp @@ -72,7 +72,7 @@ FUZZ_TARGET(partially_downloaded_block, .init = initialize_pdb) available.insert(i); } - if (add_to_mempool) { + if (add_to_mempool && !pool.exists(GenTxid::Txid(tx->GetHash()))) { LOCK2(cs_main, pool.cs); pool.addUnchecked(ConsumeTxMemPoolEntry(fuzzed_data_provider, *tx)); available.insert(i); diff --git a/src/test/fuzz/poolresource.cpp b/src/test/fuzz/poolresource.cpp index ce64ef6472..f764d9f8db 100644 --- a/src/test/fuzz/poolresource.cpp +++ b/src/test/fuzz/poolresource.cpp @@ -63,9 +63,9 @@ public: { if (m_total_allocated > 0x1000000) return; size_t alignment_bits = m_provider.ConsumeIntegralInRange<size_t>(0, 7); - size_t alignment = 1 << alignment_bits; + size_t alignment = size_t{1} << alignment_bits; size_t size_bits = m_provider.ConsumeIntegralInRange<size_t>(0, 16 - alignment_bits); - size_t size = m_provider.ConsumeIntegralInRange<size_t>(1U << size_bits, (1U << (size_bits + 1)) - 1U) << alignment_bits; + size_t size = m_provider.ConsumeIntegralInRange<size_t>(size_t{1} << size_bits, (size_t{1} << (size_bits + 1)) - 1U) << alignment_bits; Allocate(size, alignment); } diff --git a/src/test/fuzz/rbf.cpp b/src/test/fuzz/rbf.cpp index 754aff4e70..64785948f6 100644 --- a/src/test/fuzz/rbf.cpp +++ b/src/test/fuzz/rbf.cpp @@ -82,17 +82,6 @@ FUZZ_TARGET(rbf, .init = initialize_rbf) } } -void CheckDiagramConcave(std::vector<FeeFrac>& diagram) -{ - // Diagrams are in monotonically-decreasing feerate order. - FeeFrac last_chunk = diagram.front(); - for (size_t i = 1; i<diagram.size(); ++i) { - FeeFrac next_chunk = diagram[i] - diagram[i-1]; - assert(next_chunk <= last_chunk); - last_chunk = next_chunk; - } -} - FUZZ_TARGET(package_rbf, .init = initialize_package_rbf) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); @@ -107,6 +96,12 @@ FUZZ_TARGET(package_rbf, .init = initialize_package_rbf) std::vector<CTransaction> mempool_txs; size_t iter{0}; + int32_t replacement_vsize = fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(1, 1000000); + + // Keep track of the total vsize of CTxMemPoolEntry's being added to the mempool to avoid overflow + // Add replacement_vsize since this is added to new diagram during RBF check + int64_t running_vsize_total{replacement_vsize}; + LOCK2(cs_main, pool.cs); LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), NUM_ITERS) @@ -114,19 +109,33 @@ FUZZ_TARGET(package_rbf, .init = initialize_package_rbf) // Make sure txns only have one input, and that a unique input is given to avoid circular references std::optional<CMutableTransaction> parent = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!parent) { - continue; + return; } assert(iter <= g_outpoints.size()); parent->vin.resize(1); parent->vin[0].prevout = g_outpoints[iter++]; mempool_txs.emplace_back(*parent); - pool.addUnchecked(ConsumeTxMemPoolEntry(fuzzed_data_provider, mempool_txs.back())); + const auto parent_entry = ConsumeTxMemPoolEntry(fuzzed_data_provider, mempool_txs.back()); + running_vsize_total += parent_entry.GetTxSize(); + if (running_vsize_total > std::numeric_limits<int32_t>::max()) { + // We aren't adding this final tx to mempool, so we don't want to conflict with it + mempool_txs.pop_back(); + break; + } + pool.addUnchecked(parent_entry); if (fuzzed_data_provider.ConsumeBool() && !child->vin.empty()) { child->vin[0].prevout = COutPoint{mempool_txs.back().GetHash(), 0}; } mempool_txs.emplace_back(*child); - pool.addUnchecked(ConsumeTxMemPoolEntry(fuzzed_data_provider, mempool_txs.back())); + const auto child_entry = ConsumeTxMemPoolEntry(fuzzed_data_provider, mempool_txs.back()); + running_vsize_total += child_entry.GetTxSize(); + if (running_vsize_total > std::numeric_limits<int32_t>::max()) { + // We aren't adding this final tx to mempool, so we don't want to conflict with it + mempool_txs.pop_back(); + break; + } + pool.addUnchecked(child_entry); if (fuzzed_data_provider.ConsumeBool()) { pool.PrioritiseTransaction(mempool_txs.back().GetHash().ToUint256(), fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(-100000, 100000)); @@ -147,33 +156,33 @@ FUZZ_TARGET(package_rbf, .init = initialize_package_rbf) pool.CalculateDescendants(txiter, all_conflicts); } - // Calculate the feerate diagrams for a replacement. + // Calculate the chunks for a replacement. CAmount replacement_fees = ConsumeMoney(fuzzed_data_provider); - int64_t replacement_vsize = fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(1, 1000000); - auto calc_results{pool.CalculateFeerateDiagramsForRBF(replacement_fees, replacement_vsize, direct_conflicts, all_conflicts)}; + auto calc_results{pool.CalculateChunksForRBF(replacement_fees, replacement_vsize, direct_conflicts, all_conflicts)}; if (calc_results.has_value()) { - // Sanity checks on the diagrams. + // Sanity checks on the chunks. - // Diagrams start at 0. - assert(calc_results->first.front().size == 0); - assert(calc_results->first.front().fee == 0); - assert(calc_results->second.front().size == 0); - assert(calc_results->second.front().fee == 0); - - CheckDiagramConcave(calc_results->first); - CheckDiagramConcave(calc_results->second); + // Feerates are monotonically decreasing. + FeeFrac first_sum; + for (size_t i = 0; i < calc_results->first.size(); ++i) { + first_sum += calc_results->first[i]; + if (i) assert(!(calc_results->first[i - 1] << calc_results->first[i])); + } + FeeFrac second_sum; + for (size_t i = 0; i < calc_results->second.size(); ++i) { + second_sum += calc_results->second[i]; + if (i) assert(!(calc_results->second[i - 1] << calc_results->second[i])); + } - CAmount replaced_fee{0}; - int64_t replaced_size{0}; + FeeFrac replaced; for (auto txiter : all_conflicts) { - replaced_fee += txiter->GetModifiedFee(); - replaced_size += txiter->GetTxSize(); + replaced.fee += txiter->GetModifiedFee(); + replaced.size += txiter->GetTxSize(); } - // The total fee of the new diagram should be the total fee of the old - // diagram - replaced_fee + replacement_fees - assert(calc_results->first.back().fee - replaced_fee + replacement_fees == calc_results->second.back().fee); - assert(calc_results->first.back().size - replaced_size + replacement_vsize == calc_results->second.back().size); + // The total fee & size of the new diagram minus replaced fee & size should be the total + // fee & size of the old diagram minus replacement fee & size. + assert((first_sum - replaced) == (second_sum - FeeFrac{replacement_fees, replacement_vsize})); } // If internals report error, wrapper should too @@ -182,10 +191,12 @@ FUZZ_TARGET(package_rbf, .init = initialize_package_rbf) assert(err_tuple.value().first == DiagramCheckError::UNCALCULABLE); } else { // Diagram check succeeded + auto old_sum = std::accumulate(calc_results->first.begin(), calc_results->first.end(), FeeFrac{}); + auto new_sum = std::accumulate(calc_results->second.begin(), calc_results->second.end(), FeeFrac{}); if (!err_tuple.has_value()) { // New diagram's final fee should always match or exceed old diagram's - assert(calc_results->first.back().fee <= calc_results->second.back().fee); - } else if (calc_results->first.back().fee > calc_results->second.back().fee) { + assert(old_sum.fee <= new_sum.fee); + } else if (old_sum.fee > new_sum.fee) { // Or it failed, and if old diagram had higher fees, it should be a failure assert(err_tuple.value().first == DiagramCheckError::FAILURE); } diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index e81efac6e0..631da13803 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -90,7 +90,7 @@ FUZZ_TARGET(string) (void)ToUpper(random_string_1); (void)TrimString(random_string_1); (void)TrimString(random_string_1, random_string_2); - (void)urlDecode(random_string_1); + (void)UrlDecode(random_string_1); (void)ContainsNoNUL(random_string_1); (void)_(random_string_1.c_str()); try { diff --git a/src/test/fuzz/timedata.cpp b/src/test/fuzz/timedata.cpp deleted file mode 100644 index f5d005296b..0000000000 --- a/src/test/fuzz/timedata.cpp +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) 2020-2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <test/fuzz/FuzzedDataProvider.h> -#include <test/fuzz/fuzz.h> -#include <test/fuzz/util.h> -#include <timedata.h> - -#include <cstdint> -#include <string> -#include <vector> - -FUZZ_TARGET(timedata) -{ - FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - const unsigned int max_size = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, 1000); - // A max_size of 0 implies no limit, so cap the max number of insertions to avoid timeouts - auto max_to_insert = fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 4000); - // Divide by 2 to avoid signed integer overflow in .median() - const int64_t initial_value = fuzzed_data_provider.ConsumeIntegral<int64_t>() / 2; - CMedianFilter<int64_t> median_filter{max_size, initial_value}; - while (fuzzed_data_provider.remaining_bytes() > 0 && --max_to_insert >= 0) { - (void)median_filter.median(); - assert(median_filter.size() > 0); - assert(static_cast<size_t>(median_filter.size()) == median_filter.sorted().size()); - assert(static_cast<unsigned int>(median_filter.size()) <= max_size || max_size == 0); - // Divide by 2 to avoid signed integer overflow in .median() - median_filter.input(fuzzed_data_provider.ConsumeIntegral<int64_t>() / 2); - } -} diff --git a/src/test/fuzz/timeoffsets.cpp b/src/test/fuzz/timeoffsets.cpp new file mode 100644 index 0000000000..019337a94a --- /dev/null +++ b/src/test/fuzz/timeoffsets.cpp @@ -0,0 +1,28 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/timeoffsets.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/util/setup_common.h> + +#include <chrono> +#include <cstdint> +#include <functional> + +void initialize_timeoffsets() +{ + static const auto testing_setup = MakeNoLogFileContext<>(ChainType::MAIN); +} + +FUZZ_TARGET(timeoffsets, .init = initialize_timeoffsets) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + TimeOffsets offsets{}; + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes() > 0, 4'000) { + (void)offsets.Median(); + offsets.Add(std::chrono::seconds{fuzzed_data_provider.ConsumeIntegral<std::chrono::seconds::rep>()}); + offsets.WarnIfOutOfSync(); + } +} diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index 0b4019d5eb..9f0aedf29b 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -139,7 +139,6 @@ void CheckATMPInvariants(const MempoolAcceptResult& res, bool txid_in_mempool, b Assert(wtxid_in_mempool); Assert(res.m_state.IsValid()); Assert(!res.m_state.IsInvalid()); - Assert(res.m_replaced_transactions); Assert(res.m_vsize); Assert(res.m_base_fees); Assert(res.m_effective_feerate); @@ -154,7 +153,6 @@ void CheckATMPInvariants(const MempoolAcceptResult& res, bool txid_in_mempool, b Assert(res.m_state.IsInvalid()); const bool is_reconsiderable{res.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE}; - Assert(!res.m_replaced_transactions); Assert(!res.m_vsize); Assert(!res.m_base_fees); // Fee information is provided if the failure is TX_RECONSIDERABLE. diff --git a/src/test/fuzz/txorphan.cpp b/src/test/fuzz/txorphan.cpp index 5423ba8920..a44f47b00d 100644 --- a/src/test/fuzz/txorphan.cpp +++ b/src/test/fuzz/txorphan.cpp @@ -45,6 +45,8 @@ FUZZ_TARGET(txorphan, .init = initialize_orphanage) // if true, allow duplicate input when constructing tx const bool duplicate_input = fuzzed_data_provider.ConsumeBool(); + CTransactionRef ptx_potential_parent = nullptr; + LIMITED_WHILE(outpoints.size() < 200'000 && fuzzed_data_provider.ConsumeBool(), 10 * DEFAULT_MAX_ORPHAN_TRANSACTIONS) { // construct transaction @@ -78,6 +80,27 @@ FUZZ_TARGET(txorphan, .init = initialize_orphanage) return new_tx; }(); + // Trigger orphanage functions that are called using parents. ptx_potential_parent is a tx we constructed in a + // previous loop and potentially the parent of this tx. + if (ptx_potential_parent) { + // Set up future GetTxToReconsider call. + orphanage.AddChildrenToWorkSet(*ptx_potential_parent); + + // Check that all txns returned from GetChildrenFrom* are indeed a direct child of this tx. + NodeId peer_id = fuzzed_data_provider.ConsumeIntegral<NodeId>(); + for (const auto& child : orphanage.GetChildrenFromSamePeer(ptx_potential_parent, peer_id)) { + assert(std::any_of(child->vin.cbegin(), child->vin.cend(), [&](const auto& input) { + return input.prevout.hash == ptx_potential_parent->GetHash(); + })); + } + for (const auto& [child, peer] : orphanage.GetChildrenFromDifferentPeer(ptx_potential_parent, peer_id)) { + assert(std::any_of(child->vin.cbegin(), child->vin.cend(), [&](const auto& input) { + return input.prevout.hash == ptx_potential_parent->GetHash(); + })); + assert(peer != peer_id); + } + } + // trigger orphanage functions LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10 * DEFAULT_MAX_ORPHAN_TRANSACTIONS) { @@ -86,9 +109,6 @@ FUZZ_TARGET(txorphan, .init = initialize_orphanage) CallOneOf( fuzzed_data_provider, [&] { - orphanage.AddChildrenToWorkSet(*tx); - }, - [&] { { CTransactionRef ref = orphanage.GetTxToReconsider(peer_id); if (ref) { @@ -136,6 +156,12 @@ FUZZ_TARGET(txorphan, .init = initialize_orphanage) orphanage.LimitOrphans(limit, limit_orphans_rng); Assert(orphanage.Size() <= limit); }); + + } + // Set tx as potential parent to be used for future GetChildren() calls. + if (!ptx_potential_parent || fuzzed_data_provider.ConsumeBool()) { + ptx_potential_parent = tx; } + } } diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 217e4a6d22..9f8d434213 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -202,9 +202,11 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) tx7.vout[1].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx7.vout[1].nValue = 1 * COIN; - auto ancestors_calculated{pool.CalculateMemPoolAncestors(entry.Fee(2000000LL).FromTx(tx7), CTxMemPool::Limits::NoLimits())}; - BOOST_REQUIRE(ancestors_calculated.has_value()); - BOOST_CHECK(*ancestors_calculated == setAncestors); + { + auto ancestors_calculated{pool.CalculateMemPoolAncestors(entry.Fee(2000000LL).FromTx(tx7), CTxMemPool::Limits::NoLimits())}; + BOOST_REQUIRE(ancestors_calculated.has_value()); + BOOST_CHECK(*ancestors_calculated == setAncestors); + } pool.addUnchecked(entry.FromTx(tx7), setAncestors); BOOST_CHECK_EQUAL(pool.size(), 7U); @@ -260,9 +262,11 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) tx10.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx10.vout[0].nValue = 10 * COIN; - ancestors_calculated = pool.CalculateMemPoolAncestors(entry.Fee(200000LL).Time(NodeSeconds{4s}).FromTx(tx10), CTxMemPool::Limits::NoLimits()); - BOOST_REQUIRE(ancestors_calculated); - BOOST_CHECK(*ancestors_calculated == setAncestors); + { + auto ancestors_calculated{pool.CalculateMemPoolAncestors(entry.Fee(200000LL).Time(NodeSeconds{4s}).FromTx(tx10), CTxMemPool::Limits::NoLimits())}; + BOOST_REQUIRE(ancestors_calculated); + BOOST_CHECK(*ancestors_calculated == setAncestors); + } pool.addUnchecked(entry.FromTx(tx10), setAncestors); diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 70a8279f04..b9dff96610 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -19,7 +19,6 @@ #include <test/util/random.h> #include <test/util/setup_common.h> #include <test/util/validation.h> -#include <timedata.h> #include <util/strencodings.h> #include <util/string.h> #include <validation.h> @@ -902,10 +901,6 @@ BOOST_AUTO_TEST_CASE(initial_advertise_from_version_message) chainman.ResetIbd(); m_node.args->ForceSetArg("-capturemessages", "0"); m_node.args->ForceSetArg("-bind", ""); - // PeerManager::ProcessMessage() calls AddTimeData() which changes the internal state - // in timedata.cpp and later confuses the test "timedata_tests/addtimedata". Thus reset - // that state as it was before our test was run. - TestOnlyResetTimeData(); } diff --git a/src/test/orphanage_tests.cpp b/src/test/orphanage_tests.cpp index 4231fcc909..b2643cf678 100644 --- a/src/test/orphanage_tests.cpp +++ b/src/test/orphanage_tests.cpp @@ -38,14 +38,56 @@ public: } }; -static void MakeNewKeyWithFastRandomContext(CKey& key) +static void MakeNewKeyWithFastRandomContext(CKey& key, FastRandomContext& rand_ctx = g_insecure_rand_ctx) { std::vector<unsigned char> keydata; - keydata = g_insecure_rand_ctx.randbytes(32); + keydata = rand_ctx.randbytes(32); key.Set(keydata.data(), keydata.data() + keydata.size(), /*fCompressedIn=*/true); assert(key.IsValid()); } +// Creates a transaction with 2 outputs. Spends all outpoints. If outpoints is empty, spends a random one. +static CTransactionRef MakeTransactionSpending(const std::vector<COutPoint>& outpoints, FastRandomContext& det_rand) +{ + CKey key; + MakeNewKeyWithFastRandomContext(key, det_rand); + CMutableTransaction tx; + // If no outpoints are given, create a random one. + if (outpoints.empty()) { + tx.vin.emplace_back(Txid::FromUint256(det_rand.rand256()), 0); + } else { + for (const auto& outpoint : outpoints) { + tx.vin.emplace_back(outpoint); + } + } + // Ensure txid != wtxid + tx.vin[0].scriptWitness.stack.push_back({1}); + tx.vout.resize(2); + tx.vout[0].nValue = CENT; + tx.vout[0].scriptPubKey = GetScriptForDestination(PKHash(key.GetPubKey())); + tx.vout[1].nValue = 3 * CENT; + tx.vout[1].scriptPubKey = GetScriptForDestination(WitnessV0KeyHash(key.GetPubKey())); + return MakeTransactionRef(tx); +} + +static bool EqualTxns(const std::set<CTransactionRef>& set_txns, const std::vector<CTransactionRef>& vec_txns) +{ + if (vec_txns.size() != set_txns.size()) return false; + for (const auto& tx : vec_txns) { + if (!set_txns.contains(tx)) return false; + } + return true; +} +static bool EqualTxns(const std::set<CTransactionRef>& set_txns, + const std::vector<std::pair<CTransactionRef, NodeId>>& vec_txns) +{ + if (vec_txns.size() != set_txns.size()) return false; + for (const auto& [tx, nodeid] : vec_txns) { + if (!set_txns.contains(tx)) return false; + } + return true; +} + BOOST_AUTO_TEST_CASE(DoS_mapOrphans) { // This test had non-deterministic coverage due to @@ -138,4 +180,105 @@ BOOST_AUTO_TEST_CASE(DoS_mapOrphans) BOOST_CHECK(orphanage.CountOrphans() == 0); } +BOOST_AUTO_TEST_CASE(get_children) +{ + FastRandomContext det_rand{true}; + std::vector<COutPoint> empty_outpoints; + + auto parent1 = MakeTransactionSpending(empty_outpoints, det_rand); + auto parent2 = MakeTransactionSpending(empty_outpoints, det_rand); + + // Make sure these parents have different txids otherwise this test won't make sense. + while (parent1->GetHash() == parent2->GetHash()) { + parent2 = MakeTransactionSpending(empty_outpoints, det_rand); + } + + // Create children to go into orphanage. + auto child_p1n0 = MakeTransactionSpending({{parent1->GetHash(), 0}}, det_rand); + auto child_p2n1 = MakeTransactionSpending({{parent2->GetHash(), 1}}, det_rand); + // Spends the same tx twice. Should not cause duplicates. + auto child_p1n0_p1n1 = MakeTransactionSpending({{parent1->GetHash(), 0}, {parent1->GetHash(), 1}}, det_rand); + // Spends the same outpoint as previous tx. Should still be returned; don't assume outpoints are unique. + auto child_p1n0_p2n0 = MakeTransactionSpending({{parent1->GetHash(), 0}, {parent2->GetHash(), 0}}, det_rand); + + const NodeId node1{1}; + const NodeId node2{2}; + + // All orphans provided by node1 + { + TxOrphanage orphanage; + BOOST_CHECK(orphanage.AddTx(child_p1n0, node1)); + BOOST_CHECK(orphanage.AddTx(child_p2n1, node1)); + BOOST_CHECK(orphanage.AddTx(child_p1n0_p1n1, node1)); + BOOST_CHECK(orphanage.AddTx(child_p1n0_p2n0, node1)); + + std::set<CTransactionRef> expected_parent1_children{child_p1n0, child_p1n0_p2n0, child_p1n0_p1n1}; + std::set<CTransactionRef> expected_parent2_children{child_p2n1, child_p1n0_p2n0}; + + BOOST_CHECK(EqualTxns(expected_parent1_children, orphanage.GetChildrenFromSamePeer(parent1, node1))); + BOOST_CHECK(EqualTxns(expected_parent2_children, orphanage.GetChildrenFromSamePeer(parent2, node1))); + + BOOST_CHECK(EqualTxns(expected_parent1_children, orphanage.GetChildrenFromDifferentPeer(parent1, node2))); + BOOST_CHECK(EqualTxns(expected_parent2_children, orphanage.GetChildrenFromDifferentPeer(parent2, node2))); + + // The peer must match + BOOST_CHECK(orphanage.GetChildrenFromSamePeer(parent1, node2).empty()); + BOOST_CHECK(orphanage.GetChildrenFromSamePeer(parent2, node2).empty()); + + // There shouldn't be any children of this tx in the orphanage + BOOST_CHECK(orphanage.GetChildrenFromSamePeer(child_p1n0_p2n0, node1).empty()); + BOOST_CHECK(orphanage.GetChildrenFromSamePeer(child_p1n0_p2n0, node2).empty()); + BOOST_CHECK(orphanage.GetChildrenFromDifferentPeer(child_p1n0_p2n0, node1).empty()); + BOOST_CHECK(orphanage.GetChildrenFromDifferentPeer(child_p1n0_p2n0, node2).empty()); + } + + // Orphans provided by node1 and node2 + { + TxOrphanage orphanage; + BOOST_CHECK(orphanage.AddTx(child_p1n0, node1)); + BOOST_CHECK(orphanage.AddTx(child_p2n1, node1)); + BOOST_CHECK(orphanage.AddTx(child_p1n0_p1n1, node2)); + BOOST_CHECK(orphanage.AddTx(child_p1n0_p2n0, node2)); + + // +----------------+---------------+----------------------------------+ + // | | sender=node1 | sender=node2 | + // +----------------+---------------+----------------------------------+ + // | spends parent1 | child_p1n0 | child_p1n0_p1n1, child_p1n0_p2n0 | + // | spends parent2 | child_p2n1 | child_p1n0_p2n0 | + // +----------------+---------------+----------------------------------+ + + // Children of parent1 from node1: + { + std::set<CTransactionRef> expected_parent1_node1{child_p1n0}; + + BOOST_CHECK(EqualTxns(expected_parent1_node1, orphanage.GetChildrenFromSamePeer(parent1, node1))); + BOOST_CHECK(EqualTxns(expected_parent1_node1, orphanage.GetChildrenFromDifferentPeer(parent1, node2))); + } + + // Children of parent2 from node1: + { + std::set<CTransactionRef> expected_parent2_node1{child_p2n1}; + + BOOST_CHECK(EqualTxns(expected_parent2_node1, orphanage.GetChildrenFromSamePeer(parent2, node1))); + BOOST_CHECK(EqualTxns(expected_parent2_node1, orphanage.GetChildrenFromDifferentPeer(parent2, node2))); + } + + // Children of parent1 from node2: + { + std::set<CTransactionRef> expected_parent1_node2{child_p1n0_p1n1, child_p1n0_p2n0}; + + BOOST_CHECK(EqualTxns(expected_parent1_node2, orphanage.GetChildrenFromSamePeer(parent1, node2))); + BOOST_CHECK(EqualTxns(expected_parent1_node2, orphanage.GetChildrenFromDifferentPeer(parent1, node1))); + } + + // Children of parent2 from node2: + { + std::set<CTransactionRef> expected_parent2_node2{child_p1n0_p2n0}; + + BOOST_CHECK(EqualTxns(expected_parent2_node2, orphanage.GetChildrenFromSamePeer(parent2, node2))); + BOOST_CHECK(EqualTxns(expected_parent2_node2, orphanage.GetChildrenFromDifferentPeer(parent2, node1))); + } + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/rbf_tests.cpp b/src/test/rbf_tests.cpp index ed33969710..19e45c550a 100644 --- a/src/test/rbf_tests.cpp +++ b/src/test/rbf_tests.cpp @@ -426,21 +426,21 @@ BOOST_FIXTURE_TEST_CASE(calc_feerate_diagram_rbf, TestChain100Setup) // Replacement of size 1 { - const auto replace_one{pool.CalculateFeerateDiagramsForRBF(/*replacement_fees=*/0, /*replacement_vsize=*/1, {entry_low}, {entry_low})}; + const auto replace_one{pool.CalculateChunksForRBF(/*replacement_fees=*/0, /*replacement_vsize=*/1, {entry_low}, {entry_low})}; BOOST_CHECK(replace_one.has_value()); - std::vector<FeeFrac> expected_old_diagram{FeeFrac(0, 0), FeeFrac(low_fee, low_size)}; - BOOST_CHECK(replace_one->first == expected_old_diagram); - std::vector<FeeFrac> expected_new_diagram{FeeFrac(0, 0), FeeFrac(0, 1)}; - BOOST_CHECK(replace_one->second == expected_new_diagram); + std::vector<FeeFrac> expected_old_chunks{{low_fee, low_size}}; + BOOST_CHECK(replace_one->first == expected_old_chunks); + std::vector<FeeFrac> expected_new_chunks{{0, 1}}; + BOOST_CHECK(replace_one->second == expected_new_chunks); } // Non-zero replacement fee/size { - const auto replace_one_fee{pool.CalculateFeerateDiagramsForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {entry_low}, {entry_low})}; + const auto replace_one_fee{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {entry_low}, {entry_low})}; BOOST_CHECK(replace_one_fee.has_value()); - std::vector<FeeFrac> expected_old_diagram{FeeFrac(0, 0), FeeFrac(low_fee, low_size)}; + std::vector<FeeFrac> expected_old_diagram{{low_fee, low_size}}; BOOST_CHECK(replace_one_fee->first == expected_old_diagram); - std::vector<FeeFrac> expected_new_diagram{FeeFrac(0, 0), FeeFrac(high_fee, low_size)}; + std::vector<FeeFrac> expected_new_diagram{{high_fee, low_size}}; BOOST_CHECK(replace_one_fee->second == expected_new_diagram); } @@ -451,22 +451,22 @@ BOOST_FIXTURE_TEST_CASE(calc_feerate_diagram_rbf, TestChain100Setup) const auto high_size = entry_high->GetTxSize(); { - const auto replace_single_chunk{pool.CalculateFeerateDiagramsForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {entry_low}, {entry_low, entry_high})}; + const auto replace_single_chunk{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {entry_low}, {entry_low, entry_high})}; BOOST_CHECK(replace_single_chunk.has_value()); - std::vector<FeeFrac> expected_old_diagram{FeeFrac(0, 0), FeeFrac(low_fee + high_fee, low_size + high_size)}; - BOOST_CHECK(replace_single_chunk->first == expected_old_diagram); - std::vector<FeeFrac> expected_new_diagram{FeeFrac(0, 0), FeeFrac(high_fee, low_size)}; - BOOST_CHECK(replace_single_chunk->second == expected_new_diagram); + std::vector<FeeFrac> expected_old_chunks{{low_fee + high_fee, low_size + high_size}}; + BOOST_CHECK(replace_single_chunk->first == expected_old_chunks); + std::vector<FeeFrac> expected_new_chunks{{high_fee, low_size}}; + BOOST_CHECK(replace_single_chunk->second == expected_new_chunks); } // Conflict with the 2nd tx, resulting in new diagram with three entries { - const auto replace_cpfp_child{pool.CalculateFeerateDiagramsForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {entry_high}, {entry_high})}; + const auto replace_cpfp_child{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {entry_high}, {entry_high})}; BOOST_CHECK(replace_cpfp_child.has_value()); - std::vector<FeeFrac> expected_old_diagram{FeeFrac(0, 0), FeeFrac(low_fee + high_fee, low_size + high_size)}; - BOOST_CHECK(replace_cpfp_child->first == expected_old_diagram); - std::vector<FeeFrac> expected_new_diagram{FeeFrac(0, 0), FeeFrac(high_fee, low_size), FeeFrac(low_fee + high_fee, low_size + low_size)}; - BOOST_CHECK(replace_cpfp_child->second == expected_new_diagram); + std::vector<FeeFrac> expected_old_chunks{{low_fee + high_fee, low_size + high_size}}; + BOOST_CHECK(replace_cpfp_child->first == expected_old_chunks); + std::vector<FeeFrac> expected_new_chunks{{high_fee, low_size}, {low_fee, low_size}}; + BOOST_CHECK(replace_cpfp_child->second == expected_new_chunks); } // third transaction causes the topology check to fail @@ -476,7 +476,7 @@ BOOST_FIXTURE_TEST_CASE(calc_feerate_diagram_rbf, TestChain100Setup) const auto normal_size = entry_normal->GetTxSize(); { - const auto replace_too_large{pool.CalculateFeerateDiagramsForRBF(/*replacement_fees=*/normal_fee, /*replacement_vsize=*/normal_size, {entry_low}, {entry_low, entry_high, entry_normal})}; + const auto replace_too_large{pool.CalculateChunksForRBF(/*replacement_fees=*/normal_fee, /*replacement_vsize=*/normal_size, {entry_low}, {entry_low, entry_high, entry_normal})}; BOOST_CHECK(!replace_too_large.has_value()); BOOST_CHECK_EQUAL(util::ErrorString(replace_too_large).original, strprintf("%s has 2 descendants, max 1 allowed", low_tx->GetHash().GetHex())); } @@ -493,12 +493,12 @@ BOOST_FIXTURE_TEST_CASE(calc_feerate_diagram_rbf, TestChain100Setup) const auto low_size_2 = entry_low_2->GetTxSize(); { - const auto replace_two_chunks_single_cluster{pool.CalculateFeerateDiagramsForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {entry_high_2}, {entry_high_2, entry_low_2})}; + const auto replace_two_chunks_single_cluster{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {entry_high_2}, {entry_high_2, entry_low_2})}; BOOST_CHECK(replace_two_chunks_single_cluster.has_value()); - std::vector<FeeFrac> expected_old_diagram{FeeFrac(0, 0), FeeFrac(high_fee, high_size_2), FeeFrac(low_fee + high_fee, low_size_2 + high_size_2)}; - BOOST_CHECK(replace_two_chunks_single_cluster->first == expected_old_diagram); - std::vector<FeeFrac> expected_new_diagram{FeeFrac(0, 0), FeeFrac(high_fee, low_size_2)}; - BOOST_CHECK(replace_two_chunks_single_cluster->second == expected_new_diagram); + std::vector<FeeFrac> expected_old_chunks{{high_fee, high_size_2}, {low_fee, low_size_2}}; + BOOST_CHECK(replace_two_chunks_single_cluster->first == expected_old_chunks); + std::vector<FeeFrac> expected_new_chunks{{high_fee, low_size_2}}; + BOOST_CHECK(replace_two_chunks_single_cluster->second == expected_new_chunks); } // You can have more than two direct conflicts if the there are multiple affected clusters, all of size 2 or less @@ -515,10 +515,10 @@ BOOST_FIXTURE_TEST_CASE(calc_feerate_diagram_rbf, TestChain100Setup) const auto conflict_3_entry = pool.GetIter(conflict_3->GetHash()).value(); { - const auto replace_multiple_clusters{pool.CalculateFeerateDiagramsForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {conflict_1_entry, conflict_2_entry, conflict_3_entry}, {conflict_1_entry, conflict_2_entry, conflict_3_entry})}; + const auto replace_multiple_clusters{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {conflict_1_entry, conflict_2_entry, conflict_3_entry}, {conflict_1_entry, conflict_2_entry, conflict_3_entry})}; BOOST_CHECK(replace_multiple_clusters.has_value()); - BOOST_CHECK(replace_multiple_clusters->first.size() == 4); - BOOST_CHECK(replace_multiple_clusters->second.size() == 2); + BOOST_CHECK(replace_multiple_clusters->first.size() == 3); + BOOST_CHECK(replace_multiple_clusters->second.size() == 1); } // Add a child transaction to conflict_1 and make it cluster size 2, two chunks due to same feerate @@ -527,11 +527,11 @@ BOOST_FIXTURE_TEST_CASE(calc_feerate_diagram_rbf, TestChain100Setup) const auto conflict_1_child_entry = pool.GetIter(conflict_1_child->GetHash()).value(); { - const auto replace_multiple_clusters_2{pool.CalculateFeerateDiagramsForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {conflict_1_entry, conflict_2_entry, conflict_3_entry}, {conflict_1_entry, conflict_2_entry, conflict_3_entry, conflict_1_child_entry})}; + const auto replace_multiple_clusters_2{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {conflict_1_entry, conflict_2_entry, conflict_3_entry}, {conflict_1_entry, conflict_2_entry, conflict_3_entry, conflict_1_child_entry})}; BOOST_CHECK(replace_multiple_clusters_2.has_value()); - BOOST_CHECK(replace_multiple_clusters_2->first.size() == 5); - BOOST_CHECK(replace_multiple_clusters_2->second.size() == 2); + BOOST_CHECK(replace_multiple_clusters_2->first.size() == 4); + BOOST_CHECK(replace_multiple_clusters_2->second.size() == 1); } // Add another descendant to conflict_1, making the cluster size > 2 should fail at this point. @@ -540,86 +540,86 @@ BOOST_FIXTURE_TEST_CASE(calc_feerate_diagram_rbf, TestChain100Setup) const auto conflict_1_grand_child_entry = pool.GetIter(conflict_1_child->GetHash()).value(); { - const auto replace_cluster_size_3{pool.CalculateFeerateDiagramsForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {conflict_1_entry, conflict_2_entry, conflict_3_entry}, {conflict_1_entry, conflict_2_entry, conflict_3_entry, conflict_1_child_entry, conflict_1_grand_child_entry})}; + const auto replace_cluster_size_3{pool.CalculateChunksForRBF(/*replacement_fees=*/high_fee, /*replacement_vsize=*/low_size, {conflict_1_entry, conflict_2_entry, conflict_3_entry}, {conflict_1_entry, conflict_2_entry, conflict_3_entry, conflict_1_child_entry, conflict_1_grand_child_entry})}; BOOST_CHECK(!replace_cluster_size_3.has_value()); BOOST_CHECK_EQUAL(util::ErrorString(replace_cluster_size_3).original, strprintf("%s has 2 descendants, max 1 allowed", conflict_1->GetHash().GetHex())); } } -BOOST_AUTO_TEST_CASE(feerate_diagram_utilities) +BOOST_AUTO_TEST_CASE(feerate_chunks_utilities) { - // Sanity check the correctness of the feerate diagram comparison. + // Sanity check the correctness of the feerate chunks comparison. // A strictly better case. - std::vector<FeeFrac> old_diagram{{FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}}}; - std::vector<FeeFrac> new_diagram{{FeeFrac{0, 0}, FeeFrac{1000, 300}, FeeFrac{1050, 400}}}; + std::vector<FeeFrac> old_chunks{{{950, 300}, {100, 100}}}; + std::vector<FeeFrac> new_chunks{{{1000, 300}, {50, 100}}}; - BOOST_CHECK(std::is_lt(CompareFeerateDiagram(old_diagram, new_diagram))); - BOOST_CHECK(std::is_gt(CompareFeerateDiagram(new_diagram, old_diagram))); + BOOST_CHECK(std::is_lt(CompareChunks(old_chunks, new_chunks))); + BOOST_CHECK(std::is_gt(CompareChunks(new_chunks, old_chunks))); // Incomparable diagrams - old_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}}; - new_diagram = {FeeFrac{0, 0}, FeeFrac{1000, 300}, FeeFrac{1000, 400}}; + old_chunks = {{950, 300}, {100, 100}}; + new_chunks = {{1000, 300}, {0, 100}}; - BOOST_CHECK(CompareFeerateDiagram(old_diagram, new_diagram) == std::partial_ordering::unordered); - BOOST_CHECK(CompareFeerateDiagram(new_diagram, old_diagram) == std::partial_ordering::unordered); + BOOST_CHECK(CompareChunks(old_chunks, new_chunks) == std::partial_ordering::unordered); + BOOST_CHECK(CompareChunks(new_chunks, old_chunks) == std::partial_ordering::unordered); // Strictly better but smaller size. - old_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}}; - new_diagram = {FeeFrac{0, 0}, FeeFrac{1100, 300}}; + old_chunks = {{950, 300}, {100, 100}}; + new_chunks = {{1100, 300}}; - BOOST_CHECK(std::is_lt(CompareFeerateDiagram(old_diagram, new_diagram))); - BOOST_CHECK(std::is_gt(CompareFeerateDiagram(new_diagram, old_diagram))); + BOOST_CHECK(std::is_lt(CompareChunks(old_chunks, new_chunks))); + BOOST_CHECK(std::is_gt(CompareChunks(new_chunks, old_chunks))); // New diagram is strictly better due to the first chunk, even though // second chunk contributes no fees - old_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}}; - new_diagram = {FeeFrac{0, 0}, FeeFrac{1100, 100}, FeeFrac{1100, 200}}; + old_chunks = {{950, 300}, {100, 100}}; + new_chunks = {{1100, 100}, {0, 100}}; - BOOST_CHECK(std::is_lt(CompareFeerateDiagram(old_diagram, new_diagram))); - BOOST_CHECK(std::is_gt(CompareFeerateDiagram(new_diagram, old_diagram))); + BOOST_CHECK(std::is_lt(CompareChunks(old_chunks, new_chunks))); + BOOST_CHECK(std::is_gt(CompareChunks(new_chunks, old_chunks))); // Feerate of first new chunk is better with, but second chunk is worse - old_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}}; - new_diagram = {FeeFrac{0, 0}, FeeFrac{750, 100}, FeeFrac{999, 350}, FeeFrac{1150, 1000}}; + old_chunks = {{950, 300}, {100, 100}}; + new_chunks = {{750, 100}, {249, 250}, {151, 650}}; - BOOST_CHECK(CompareFeerateDiagram(old_diagram, new_diagram) == std::partial_ordering::unordered); - BOOST_CHECK(CompareFeerateDiagram(new_diagram, old_diagram) == std::partial_ordering::unordered); + BOOST_CHECK(CompareChunks(old_chunks, new_chunks) == std::partial_ordering::unordered); + BOOST_CHECK(CompareChunks(new_chunks, old_chunks) == std::partial_ordering::unordered); // If we make the second chunk slightly better, the new diagram now wins. - old_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}}; - new_diagram = {FeeFrac{0, 0}, FeeFrac{750, 100}, FeeFrac{1000, 350}, FeeFrac{1150, 500}}; + old_chunks = {{950, 300}, {100, 100}}; + new_chunks = {{750, 100}, {250, 250}, {150, 150}}; - BOOST_CHECK(std::is_lt(CompareFeerateDiagram(old_diagram, new_diagram))); - BOOST_CHECK(std::is_gt(CompareFeerateDiagram(new_diagram, old_diagram))); + BOOST_CHECK(std::is_lt(CompareChunks(old_chunks, new_chunks))); + BOOST_CHECK(std::is_gt(CompareChunks(new_chunks, old_chunks))); // Identical diagrams, cannot be strictly better - old_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}}; - new_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}}; + old_chunks = {{950, 300}, {100, 100}}; + new_chunks = {{950, 300}, {100, 100}}; - BOOST_CHECK(std::is_eq(CompareFeerateDiagram(old_diagram, new_diagram))); - BOOST_CHECK(std::is_eq(CompareFeerateDiagram(new_diagram, old_diagram))); + BOOST_CHECK(std::is_eq(CompareChunks(old_chunks, new_chunks))); + BOOST_CHECK(std::is_eq(CompareChunks(new_chunks, old_chunks))); // Same aggregate fee, but different total size (trigger single tail fee check step) - old_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 399}}; - new_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}}; + old_chunks = {{950, 300}, {100, 99}}; + new_chunks = {{950, 300}, {100, 100}}; // No change in evaluation when tail check needed. - BOOST_CHECK(std::is_gt(CompareFeerateDiagram(old_diagram, new_diagram))); - BOOST_CHECK(std::is_lt(CompareFeerateDiagram(new_diagram, old_diagram))); + BOOST_CHECK(std::is_gt(CompareChunks(old_chunks, new_chunks))); + BOOST_CHECK(std::is_lt(CompareChunks(new_chunks, old_chunks))); // Trigger multiple tail fee check steps - old_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 399}}; - new_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}, FeeFrac{1050, 401}, FeeFrac{1050, 402}}; + old_chunks = {{950, 300}, {100, 99}}; + new_chunks = {{950, 300}, {100, 100}, {0, 1}, {0, 1}}; - BOOST_CHECK(std::is_gt(CompareFeerateDiagram(old_diagram, new_diagram))); - BOOST_CHECK(std::is_lt(CompareFeerateDiagram(new_diagram, old_diagram))); + BOOST_CHECK(std::is_gt(CompareChunks(old_chunks, new_chunks))); + BOOST_CHECK(std::is_lt(CompareChunks(new_chunks, old_chunks))); // Multiple tail fee check steps, unordered result - new_diagram = {FeeFrac{0, 0}, FeeFrac{950, 300}, FeeFrac{1050, 400}, FeeFrac{1050, 401}, FeeFrac{1050, 402}, FeeFrac{1051, 403}}; - BOOST_CHECK(CompareFeerateDiagram(old_diagram, new_diagram) == std::partial_ordering::unordered); - BOOST_CHECK(CompareFeerateDiagram(new_diagram, old_diagram) == std::partial_ordering::unordered); + new_chunks = {{950, 300}, {100, 100}, {0, 1}, {0, 1}, {1, 1}}; + BOOST_CHECK(CompareChunks(old_chunks, new_chunks) == std::partial_ordering::unordered); + BOOST_CHECK(CompareChunks(new_chunks, old_chunks) == std::partial_ordering::unordered); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 3a1cb45e8d..acacb6257d 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -582,4 +582,72 @@ BOOST_AUTO_TEST_CASE(help_example) BOOST_CHECK_NE(HelpExampleRpcNamed("foo", {{"arg", true}}), HelpExampleRpcNamed("foo", {{"arg", "true"}})); } +static void CheckRpc(const std::vector<RPCArg>& params, const UniValue& args, RPCHelpMan::RPCMethodImpl test_impl) +{ + auto null_result{RPCResult{RPCResult::Type::NONE, "", "None"}}; + const RPCHelpMan rpc{"dummy", "dummy description", params, null_result, RPCExamples{""}, test_impl}; + JSONRPCRequest req; + req.params = args; + + rpc.HandleRequest(req); +} + +BOOST_AUTO_TEST_CASE(rpc_arg_helper) +{ + constexpr bool DEFAULT_BOOL = true; + constexpr auto DEFAULT_STRING = "default"; + constexpr uint64_t DEFAULT_UINT64_T = 3; + + //! Parameters with which the RPCHelpMan is instantiated + const std::vector<RPCArg> params{ + // Required arg + {"req_int", RPCArg::Type::NUM, RPCArg::Optional::NO, ""}, + {"req_str", RPCArg::Type::STR, RPCArg::Optional::NO, ""}, + // Default arg + {"def_uint64_t", RPCArg::Type::NUM, RPCArg::Default{DEFAULT_UINT64_T}, ""}, + {"def_string", RPCArg::Type::STR, RPCArg::Default{DEFAULT_STRING}, ""}, + {"def_bool", RPCArg::Type::BOOL, RPCArg::Default{DEFAULT_BOOL}, ""}, + // Optional arg without default + {"opt_double", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, ""}, + {"opt_string", RPCArg::Type::STR, RPCArg::Optional::OMITTED, ""} + }; + + //! Check that `self.Arg` returns the same value as the `request.params` accessors + RPCHelpMan::RPCMethodImpl check_positional = [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + BOOST_CHECK_EQUAL(self.Arg<int>(0), request.params[0].getInt<int>()); + BOOST_CHECK_EQUAL(self.Arg<std::string>(1), request.params[1].get_str()); + BOOST_CHECK_EQUAL(self.Arg<uint64_t>(2), request.params[2].isNull() ? DEFAULT_UINT64_T : request.params[2].getInt<uint64_t>()); + BOOST_CHECK_EQUAL(self.Arg<std::string>(3), request.params[3].isNull() ? DEFAULT_STRING : request.params[3].get_str()); + BOOST_CHECK_EQUAL(self.Arg<bool>(4), request.params[4].isNull() ? DEFAULT_BOOL : request.params[4].get_bool()); + if (!request.params[5].isNull()) { + BOOST_CHECK_EQUAL(self.MaybeArg<double>(5).value(), request.params[5].get_real()); + } else { + BOOST_CHECK(!self.MaybeArg<double>(5)); + } + if (!request.params[6].isNull()) { + BOOST_CHECK(self.MaybeArg<std::string>(6)); + BOOST_CHECK_EQUAL(*self.MaybeArg<std::string>(6), request.params[6].get_str()); + } else { + BOOST_CHECK(!self.MaybeArg<std::string>(6)); + } + return UniValue{}; + }; + CheckRpc(params, UniValue{JSON(R"([5, "hello", null, null, null, null, null])")}, check_positional); + CheckRpc(params, UniValue{JSON(R"([5, "hello", 4, "test", true, 1.23, "world"])")}, check_positional); + + //! Check that `self.Arg` returns the same value when using index and key + RPCHelpMan::RPCMethodImpl check_named = [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + BOOST_CHECK_EQUAL(self.Arg<int>(0), self.Arg<int>("req_int")); + BOOST_CHECK_EQUAL(self.Arg<std::string>(1), self.Arg<std::string>("req_str")); + BOOST_CHECK_EQUAL(self.Arg<uint64_t>(2), self.Arg<uint64_t>("def_uint64_t")); + BOOST_CHECK_EQUAL(self.Arg<std::string>(3), self.Arg<std::string>("def_string")); + BOOST_CHECK_EQUAL(self.Arg<bool>(4), self.Arg<bool>("def_bool")); + BOOST_CHECK(self.MaybeArg<double>(5) == self.MaybeArg<double>("opt_double")); + BOOST_CHECK(self.MaybeArg<std::string>(6) == self.MaybeArg<std::string>("opt_string")); + return UniValue{}; + }; + CheckRpc(params, UniValue{JSON(R"([5, "hello", null, null, null, null, null])")}, check_named); + CheckRpc(params, UniValue{JSON(R"([5, "hello", 4, "test", true, 1.23, "world"])")}, check_named); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 0af2fdce08..e4142e203c 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -1254,6 +1254,30 @@ BOOST_AUTO_TEST_CASE(script_combineSigs) BOOST_CHECK(combined.scriptSig == partial3c); } +/** + * Reproduction of an exception incorrectly raised when parsing a public key inside a TapMiniscript. + */ +BOOST_AUTO_TEST_CASE(sign_invalid_miniscript) +{ + FillableSigningProvider keystore; + SignatureData sig_data; + CMutableTransaction prev, curr; + + // Create a Taproot output which contains a leaf in which a non-32 bytes push is used where a public key is expected + // by the Miniscript parser. This offending Script was found by the RPC fuzzer. + const auto invalid_pubkey{ParseHex("173d36c8c9c9c9ffffffffffff0200000000021e1e37373721361818181818181e1e1e1e19000000000000000000b19292929292926b006c9b9b9292")}; + TaprootBuilder builder; + builder.Add(0, {invalid_pubkey}, 0xc0); + XOnlyPubKey nums{ParseHex("50929b74c1a04954b78b4b6035e97a5e078a5a0f28ec96d547bfee9ace803ac0")}; + builder.Finalize(nums); + prev.vout.emplace_back(0, GetScriptForDestination(builder.GetOutput())); + curr.vin.emplace_back(COutPoint{prev.GetHash(), 0}); + sig_data.tr_spenddata = builder.GetSpendData(); + + // SignSignature can fail but it shouldn't raise an exception (nor crash). + BOOST_CHECK(!SignSignature(keystore, CTransaction(prev), curr, 0, SIGHASH_ALL, sig_data)); +} + BOOST_AUTO_TEST_CASE(script_standard_push) { ScriptError err; diff --git a/src/test/system_tests.cpp b/src/test/system_tests.cpp index 8aab2b565c..2de147deea 100644 --- a/src/test/system_tests.cpp +++ b/src/test/system_tests.cpp @@ -11,7 +11,7 @@ #include <univalue.h> #ifdef ENABLE_EXTERNAL_SIGNER -#include <util/subprocess.hpp> +#include <util/subprocess.h> #endif // ENABLE_EXTERNAL_SIGNER #include <boost/test/unit_test.hpp> diff --git a/src/test/timedata_tests.cpp b/src/test/timedata_tests.cpp deleted file mode 100644 index 2814dbf4c0..0000000000 --- a/src/test/timedata_tests.cpp +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (c) 2011-2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. -// - -#include <netaddress.h> -#include <noui.h> -#include <test/util/logging.h> -#include <test/util/setup_common.h> -#include <timedata.h> -#include <util/string.h> -#include <util/translation.h> -#include <warnings.h> - -#include <string> - -#include <boost/test/unit_test.hpp> - -BOOST_FIXTURE_TEST_SUITE(timedata_tests, BasicTestingSetup) - -BOOST_AUTO_TEST_CASE(util_MedianFilter) -{ - CMedianFilter<int> filter(5, 15); - - BOOST_CHECK_EQUAL(filter.median(), 15); - - filter.input(20); // [15 20] - BOOST_CHECK_EQUAL(filter.median(), 17); - - filter.input(30); // [15 20 30] - BOOST_CHECK_EQUAL(filter.median(), 20); - - filter.input(3); // [3 15 20 30] - BOOST_CHECK_EQUAL(filter.median(), 17); - - filter.input(7); // [3 7 15 20 30] - BOOST_CHECK_EQUAL(filter.median(), 15); - - filter.input(18); // [3 7 18 20 30] - BOOST_CHECK_EQUAL(filter.median(), 18); - - filter.input(0); // [0 3 7 18 30] - BOOST_CHECK_EQUAL(filter.median(), 7); -} - -static void MultiAddTimeData(int n, int64_t offset) -{ - static int cnt = 0; - for (int i = 0; i < n; ++i) { - CNetAddr addr; - addr.SetInternal(ToString(++cnt)); - AddTimeData(addr, offset); - } -} - - -BOOST_AUTO_TEST_CASE(addtimedata) -{ - BOOST_CHECK_EQUAL(GetTimeOffset(), 0); - - //Part 1: Add large offsets to test a warning message that our clock may be wrong. - MultiAddTimeData(3, DEFAULT_MAX_TIME_ADJUSTMENT + 1); - // Filter size is 1 + 3 = 4: It is always initialized with a single element (offset 0) - - { - ASSERT_DEBUG_LOG("Please check that your computer's date and time are correct!"); - MultiAddTimeData(1, DEFAULT_MAX_TIME_ADJUSTMENT + 1); //filter size 5 - } - - BOOST_CHECK(GetWarnings(true).original.find("clock is wrong") != std::string::npos); - - // nTimeOffset is not changed if the median of offsets exceeds DEFAULT_MAX_TIME_ADJUSTMENT - BOOST_CHECK_EQUAL(GetTimeOffset(), 0); - - // Part 2: Test positive and negative medians by adding more offsets - MultiAddTimeData(4, 100); // filter size 9 - BOOST_CHECK_EQUAL(GetTimeOffset(), 100); - MultiAddTimeData(10, -100); //filter size 19 - BOOST_CHECK_EQUAL(GetTimeOffset(), -100); - - // Part 3: Test behaviour when filter has reached maximum number of offsets - const int MAX_SAMPLES = 200; - int nfill = (MAX_SAMPLES - 3 - 19) / 2; //89 - MultiAddTimeData(nfill, 100); - MultiAddTimeData(nfill, -100); //filter size MAX_SAMPLES - 3 - BOOST_CHECK_EQUAL(GetTimeOffset(), -100); - - MultiAddTimeData(2, 100); - //filter size MAX_SAMPLES -1, median is the initial 0 offset - //since we added same number of positive/negative offsets - - BOOST_CHECK_EQUAL(GetTimeOffset(), 0); - - // After the number of offsets has reached MAX_SAMPLES -1 (=199), nTimeOffset will never change - // because it is only updated when the number of elements in the filter becomes odd. It was decided - // not to fix this because it prevents possible attacks. See the comment in AddTimeData() or issue #4521 - // for a more detailed explanation. - MultiAddTimeData(2, 100); // filter median is 100 now, but nTimeOffset will not change - // We want this test to end with nTimeOffset==0, otherwise subsequent tests of the suite will fail. - BOOST_CHECK_EQUAL(GetTimeOffset(), 0); - - TestOnlyResetTimeData(); -} - -BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/timeoffsets_tests.cpp b/src/test/timeoffsets_tests.cpp new file mode 100644 index 0000000000..008f1a9db9 --- /dev/null +++ b/src/test/timeoffsets_tests.cpp @@ -0,0 +1,69 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// + +#include <node/timeoffsets.h> +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +#include <chrono> +#include <vector> + +using namespace std::chrono_literals; + +static void AddMulti(TimeOffsets& offsets, const std::vector<std::chrono::seconds>& to_add) +{ + for (auto offset : to_add) { + offsets.Add(offset); + } +} + +BOOST_FIXTURE_TEST_SUITE(timeoffsets_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(timeoffsets) +{ + TimeOffsets offsets{}; + BOOST_CHECK(offsets.Median() == 0s); + + AddMulti(offsets, {{0s, -1s, -2s, -3s}}); + // median should be zero for < 5 offsets + BOOST_CHECK(offsets.Median() == 0s); + + offsets.Add(-4s); + // we now have 5 offsets: [-4, -3, -2, -1, 0] + BOOST_CHECK(offsets.Median() == -2s); + + AddMulti(offsets, {4, 5s}); + // we now have 9 offsets: [-4, -3, -2, -1, 0, 5, 5, 5, 5] + BOOST_CHECK(offsets.Median() == 0s); + + AddMulti(offsets, {41, 10s}); + // the TimeOffsets is now at capacity with 50 offsets, oldest offsets is discarded for any additional offset + BOOST_CHECK(offsets.Median() == 10s); + + AddMulti(offsets, {25, 15s}); + // we now have 25 offsets of 10s followed by 25 offsets of 15s + BOOST_CHECK(offsets.Median() == 15s); +} + +static bool IsWarningRaised(const std::vector<std::chrono::seconds>& check_offsets) +{ + TimeOffsets offsets{}; + AddMulti(offsets, check_offsets); + return offsets.WarnIfOutOfSync(); +} + + +BOOST_AUTO_TEST_CASE(timeoffsets_warning) +{ + BOOST_CHECK(IsWarningRaised({{-60min, -40min, -30min, 0min, 10min}})); + BOOST_CHECK(IsWarningRaised({5, 11min})); + + BOOST_CHECK(!IsWarningRaised({4, 60min})); + BOOST_CHECK(!IsWarningRaised({100, 3min})); +} + + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txpackage_tests.cpp b/src/test/txpackage_tests.cpp index b948ea8acb..8112f5f685 100644 --- a/src/test/txpackage_tests.cpp +++ b/src/test/txpackage_tests.cpp @@ -8,9 +8,12 @@ #include <policy/policy.h> #include <primitives/transaction.h> #include <script/script.h> +#include <serialize.h> +#include <streams.h> #include <test/util/random.h> #include <test/util/script.h> #include <test/util/setup_common.h> +#include <util/strencodings.h> #include <test/util/txmempool.h> #include <validation.h> @@ -40,6 +43,93 @@ inline CTransactionRef create_placeholder_tx(size_t num_inputs, size_t num_outpu return MakeTransactionRef(mtx); } +// Create a Wtxid from a hex string +inline Wtxid WtxidFromString(std::string_view str) +{ + return Wtxid::FromUint256(uint256S(str.data())); +} + +BOOST_FIXTURE_TEST_CASE(package_hash_tests, TestChain100Setup) +{ + // Random real segwit transaction + DataStream stream_1{ + ParseHex("02000000000101964b8aa63509579ca6086e6012eeaa4c2f4dd1e283da29b67c8eea38b3c6fd220000000000fdffffff0294c618000000000017a9145afbbb42f4e83312666d0697f9e66259912ecde38768fa2c0000000000160014897388a0889390fd0e153a22bb2cf9d8f019faf50247304402200547406380719f84d68cf4e96cc3e4a1688309ef475b150be2b471c70ea562aa02206d255f5acc40fd95981874d77201d2eb07883657ce1c796513f32b6079545cdf0121023ae77335cefcb5ab4c1dc1fb0d2acfece184e593727d7d5906c78e564c7c11d125cf0c00"), + }; + CTransaction tx_1(deserialize, TX_WITH_WITNESS, stream_1); + CTransactionRef ptx_1{MakeTransactionRef(tx_1)}; + + // Random real nonsegwit transaction + DataStream stream_2{ + ParseHex("01000000010b26e9b7735eb6aabdf358bab62f9816a21ba9ebdb719d5299e88607d722c190000000008b4830450220070aca44506c5cef3a16ed519d7c3c39f8aab192c4e1c90d065f37b8a4af6141022100a8e160b856c2d43d27d8fba71e5aef6405b8643ac4cb7cb3c462aced7f14711a0141046d11fee51b0e60666d5049a9101a72741df480b96ee26488a4d3466b95c9a40ac5eeef87e10a5cd336c19a84565f80fa6c547957b7700ff4dfbdefe76036c339ffffffff021bff3d11000000001976a91404943fdd508053c75000106d3bc6e2754dbcff1988ac2f15de00000000001976a914a266436d2965547608b9e15d9032a7b9d64fa43188ac00000000"), + }; + CTransaction tx_2(deserialize, TX_WITH_WITNESS, stream_2); + CTransactionRef ptx_2{MakeTransactionRef(tx_2)}; + + // Random real segwit transaction + DataStream stream_3{ + ParseHex("0200000000010177862801f77c2c068a70372b4c435ef8dd621291c36a64eb4dd491f02218f5324600000000fdffffff014a0100000000000022512035ea312034cfac01e956a269f3bf147f569c2fbb00180677421262da042290d803402be713325ff285e66b0380f53f2fae0d0fb4e16f378a440fed51ce835061437566729d4883bc917632f3cff474d6384bc8b989961a1d730d4a87ed38ad28bd337b20f1d658c6c138b1c312e072b4446f50f01ae0da03a42e6274f8788aae53416a7fac0063036f7264010118746578742f706c61696e3b636861727365743d7574662d3800357b2270223a226272632d3230222c226f70223a226d696e74222c227469636b223a224342414c222c22616d74223a2236393639227d6821c1f1d658c6c138b1c312e072b4446f50f01ae0da03a42e6274f8788aae53416a7f00000000"), + }; + CTransaction tx_3(deserialize, TX_WITH_WITNESS, stream_3); + CTransactionRef ptx_3{MakeTransactionRef(tx_3)}; + + // It's easy to see that wtxids are sorted in lexicographical order: + Wtxid wtxid_1{WtxidFromString("0x85cd1a31eb38f74ed5742ec9cb546712ab5aaf747de28a9168b53e846cbda17f")}; + Wtxid wtxid_2{WtxidFromString("0xb4749f017444b051c44dfd2720e88f314ff94f3dd6d56d40ef65854fcd7fff6b")}; + Wtxid wtxid_3{WtxidFromString("0xe065bac15f62bb4e761d761db928ddee65a47296b2b776785abb912cdec474e3")}; + BOOST_CHECK_EQUAL(tx_1.GetWitnessHash(), wtxid_1); + BOOST_CHECK_EQUAL(tx_2.GetWitnessHash(), wtxid_2); + BOOST_CHECK_EQUAL(tx_3.GetWitnessHash(), wtxid_3); + + BOOST_CHECK(wtxid_1.GetHex() < wtxid_2.GetHex()); + BOOST_CHECK(wtxid_2.GetHex() < wtxid_3.GetHex()); + + // The txids are not (we want to test that sorting and hashing use wtxid, not txid): + Txid txid_1{TxidFromString("0xbd0f71c1d5e50589063e134fad22053cdae5ab2320db5bf5e540198b0b5a4e69")}; + Txid txid_2{TxidFromString("0xb4749f017444b051c44dfd2720e88f314ff94f3dd6d56d40ef65854fcd7fff6b")}; + Txid txid_3{TxidFromString("0xee707be5201160e32c4fc715bec227d1aeea5940fb4295605e7373edce3b1a93")}; + BOOST_CHECK_EQUAL(tx_1.GetHash(), txid_1); + BOOST_CHECK_EQUAL(tx_2.GetHash(), txid_2); + BOOST_CHECK_EQUAL(tx_3.GetHash(), txid_3); + + BOOST_CHECK(txid_2.GetHex() < txid_1.GetHex()); + + BOOST_CHECK(txid_1.ToUint256() != wtxid_1.ToUint256()); + BOOST_CHECK(txid_2.ToUint256() == wtxid_2.ToUint256()); + BOOST_CHECK(txid_3.ToUint256() != wtxid_3.ToUint256()); + + // We are testing that both functions compare using GetHex() and not uint256. + // (in this pair of wtxids, hex string order != uint256 order) + BOOST_CHECK(wtxid_2 < wtxid_1); + // (in this pair of wtxids, hex string order == uint256 order) + BOOST_CHECK(wtxid_2 < wtxid_3); + + // All permutations of the package containing ptx_1, ptx_2, ptx_3 have the same package hash + std::vector<CTransactionRef> package_123{ptx_1, ptx_2, ptx_3}; + std::vector<CTransactionRef> package_132{ptx_1, ptx_3, ptx_2}; + std::vector<CTransactionRef> package_231{ptx_2, ptx_3, ptx_1}; + std::vector<CTransactionRef> package_213{ptx_2, ptx_1, ptx_3}; + std::vector<CTransactionRef> package_312{ptx_3, ptx_1, ptx_2}; + std::vector<CTransactionRef> package_321{ptx_3, ptx_2, ptx_1}; + + uint256 calculated_hash_123 = (HashWriter() << wtxid_1 << wtxid_2 << wtxid_3).GetSHA256(); + + uint256 hash_if_by_txid = (HashWriter() << wtxid_2 << wtxid_1 << wtxid_3).GetSHA256(); + BOOST_CHECK(hash_if_by_txid != calculated_hash_123); + + uint256 hash_if_use_txid = (HashWriter() << txid_2 << txid_1 << txid_3).GetSHA256(); + BOOST_CHECK(hash_if_use_txid != calculated_hash_123); + + uint256 hash_if_use_int_order = (HashWriter() << wtxid_2 << wtxid_1 << wtxid_3).GetSHA256(); + BOOST_CHECK(hash_if_use_int_order != calculated_hash_123); + + BOOST_CHECK_EQUAL(calculated_hash_123, GetPackageHash(package_123)); + BOOST_CHECK_EQUAL(calculated_hash_123, GetPackageHash(package_132)); + BOOST_CHECK_EQUAL(calculated_hash_123, GetPackageHash(package_231)); + BOOST_CHECK_EQUAL(calculated_hash_123, GetPackageHash(package_213)); + BOOST_CHECK_EQUAL(calculated_hash_123, GetPackageHash(package_312)); + BOOST_CHECK_EQUAL(calculated_hash_123, GetPackageHash(package_321)); +} + BOOST_FIXTURE_TEST_CASE(package_sanitization_tests, TestChain100Setup) { // Packages can't have more than 25 transactions. @@ -190,6 +280,9 @@ BOOST_FIXTURE_TEST_CASE(noncontextual_package_tests, TestChain100Setup) BOOST_CHECK_EQUAL(state.GetRejectReason(), "package-not-sorted"); BOOST_CHECK(IsChildWithParents({tx_parent, tx_child})); BOOST_CHECK(IsChildWithParentsTree({tx_parent, tx_child})); + BOOST_CHECK(GetPackageHash({tx_parent}) != GetPackageHash({tx_child})); + BOOST_CHECK(GetPackageHash({tx_child, tx_child}) != GetPackageHash({tx_child})); + BOOST_CHECK(GetPackageHash({tx_child, tx_parent}) != GetPackageHash({tx_child, tx_child})); } // 24 Parents and 1 Child @@ -450,6 +543,8 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) BOOST_CHECK_EQUAL(ptx_child1->GetHash(), ptx_child2->GetHash()); // child1 and child2 have different wtxids BOOST_CHECK(ptx_child1->GetWitnessHash() != ptx_child2->GetWitnessHash()); + // Check that they have different package hashes + BOOST_CHECK(GetPackageHash({ptx_parent, ptx_child1}) != GetPackageHash({ptx_parent, ptx_child2})); // Try submitting Package1{parent, child1} and Package2{parent, child2} where the children are // same-txid-different-witness. @@ -503,7 +598,8 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) /*output_destination=*/grandchild_locking_script, /*output_amount=*/CAmount(47 * COIN), /*submit=*/false); CTransactionRef ptx_grandchild = MakeTransactionRef(mtx_grandchild); - + // Check that they have different package hashes + BOOST_CHECK(GetPackageHash({ptx_child1, ptx_grandchild}) != GetPackageHash({ptx_child2, ptx_grandchild})); // We already submitted child1 above. { Package package_child2_grandchild{ptx_child2, ptx_grandchild}; diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 2c18184261..38350b33cc 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -14,7 +14,6 @@ #include <banman.h> #include <chainparams.h> #include <common/system.h> -#include <common/url.h> #include <consensus/consensus.h> #include <consensus/params.h> #include <consensus/validation.h> @@ -81,7 +80,6 @@ using node::RegenerateCommitments; using node::VerifyLoadedChainstate; const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr; -UrlDecodeFn* const URL_DECODE = nullptr; /** Random context to get unique temp data dirs. Separate from g_insecure_rand_ctx, which can be seeded from a const env var */ static FastRandomContext g_insecure_rand_ctx_temp_path; diff --git a/src/test/util/txmempool.cpp b/src/test/util/txmempool.cpp index 71cf8aca60..870cdd0b32 100644 --- a/src/test/util/txmempool.cpp +++ b/src/test/util/txmempool.cpp @@ -68,12 +68,6 @@ std::optional<std::string> CheckPackageMempoolAcceptResult(const Package& txns, return strprintf("tx %s unexpectedly failed: %s", wtxid.ToString(), atmp_result.m_state.ToString()); } - //m_replaced_transactions should exist iff the result was VALID - if (atmp_result.m_replaced_transactions.has_value() != valid) { - return strprintf("tx %s result should %shave m_replaced_transactions", - wtxid.ToString(), valid ? "" : "not "); - } - // m_vsize and m_base_fees should exist iff the result was VALID or MEMPOOL_ENTRY const bool mempool_entry{atmp_result.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY}; if (atmp_result.m_base_fees.has_value() != (valid || mempool_entry)) { diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index fe2d2ba592..1c02066047 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -12,6 +12,7 @@ #include <test/util/random.h> #include <test/util/setup_common.h> #include <uint256.h> +#include <util/check.h> #include <validation.h> #include <vector> @@ -102,14 +103,14 @@ BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) BOOST_CHECK_EQUAL(chainman.GetAll().size(), 2); - Chainstate& background_cs{*[&] { + Chainstate& background_cs{*Assert([&]() -> Chainstate* { for (Chainstate* cs : chainman.GetAll()) { if (cs != &chainman.ActiveChainstate()) { return cs; } } - assert(false); - }()}; + return nullptr; + }())}; // Append the first block to the background chain. BlockValidationState state; diff --git a/src/timedata.cpp b/src/timedata.cpp deleted file mode 100644 index d948de976f..0000000000 --- a/src/timedata.cpp +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright (c) 2014-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - -#include <timedata.h> - -#include <common/args.h> -#include <logging.h> -#include <netaddress.h> -#include <node/interface_ui.h> -#include <sync.h> -#include <tinyformat.h> -#include <util/translation.h> -#include <warnings.h> - -static GlobalMutex g_timeoffset_mutex; -static int64_t nTimeOffset GUARDED_BY(g_timeoffset_mutex) = 0; - -/** - * "Never go to sea with two chronometers; take one or three." - * Our three time sources are: - * - System clock - * - Median of other nodes clocks - * - The user (asking the user to fix the system clock if the first two disagree) - */ -int64_t GetTimeOffset() -{ - LOCK(g_timeoffset_mutex); - return nTimeOffset; -} - -#define BITCOIN_TIMEDATA_MAX_SAMPLES 200 - -static std::set<CNetAddr> g_sources; -static CMedianFilter<int64_t> g_time_offsets{BITCOIN_TIMEDATA_MAX_SAMPLES, 0}; -static bool g_warning_emitted; - -void AddTimeData(const CNetAddr& ip, int64_t nOffsetSample) -{ - LOCK(g_timeoffset_mutex); - // Ignore duplicates - if (g_sources.size() == BITCOIN_TIMEDATA_MAX_SAMPLES) - return; - if (!g_sources.insert(ip).second) - return; - - // Add data - g_time_offsets.input(nOffsetSample); - LogPrint(BCLog::NET, "added time data, samples %d, offset %+d (%+d minutes)\n", g_time_offsets.size(), nOffsetSample, nOffsetSample / 60); - - // There is a known issue here (see issue #4521): - // - // - The structure g_time_offsets contains up to 200 elements, after which - // any new element added to it will not increase its size, replacing the - // oldest element. - // - // - The condition to update nTimeOffset includes checking whether the - // number of elements in g_time_offsets is odd, which will never happen after - // there are 200 elements. - // - // But in this case the 'bug' is protective against some attacks, and may - // actually explain why we've never seen attacks which manipulate the - // clock offset. - // - // So we should hold off on fixing this and clean it up as part of - // a timing cleanup that strengthens it in a number of other ways. - // - if (g_time_offsets.size() >= 5 && g_time_offsets.size() % 2 == 1) { - int64_t nMedian = g_time_offsets.median(); - std::vector<int64_t> vSorted = g_time_offsets.sorted(); - // Only let other nodes change our time by so much - int64_t max_adjustment = std::max<int64_t>(0, gArgs.GetIntArg("-maxtimeadjustment", DEFAULT_MAX_TIME_ADJUSTMENT)); - if (nMedian >= -max_adjustment && nMedian <= max_adjustment) { - nTimeOffset = nMedian; - } else { - nTimeOffset = 0; - - if (!g_warning_emitted) { - // If nobody has a time different than ours but within 5 minutes of ours, give a warning - bool fMatch = false; - for (const int64_t nOffset : vSorted) { - if (nOffset != 0 && nOffset > -5 * 60 && nOffset < 5 * 60) fMatch = true; - } - - if (!fMatch) { - g_warning_emitted = true; - bilingual_str strMessage = strprintf(_("Please check that your computer's date and time are correct! If your clock is wrong, %s will not work properly."), PACKAGE_NAME); - SetMiscWarning(strMessage); - uiInterface.ThreadSafeMessageBox(strMessage, "", CClientUIInterface::MSG_WARNING); - } - } - } - - if (LogAcceptCategory(BCLog::NET, BCLog::Level::Debug)) { - std::string log_message{"time data samples: "}; - for (const int64_t n : vSorted) { - log_message += strprintf("%+d ", n); - } - log_message += strprintf("| median offset = %+d (%+d minutes)", nTimeOffset, nTimeOffset / 60); - LogPrint(BCLog::NET, "%s\n", log_message); - } - } -} - -void TestOnlyResetTimeData() -{ - LOCK(g_timeoffset_mutex); - nTimeOffset = 0; - g_sources.clear(); - g_time_offsets = CMedianFilter<int64_t>{BITCOIN_TIMEDATA_MAX_SAMPLES, 0}; - g_warning_emitted = false; -} diff --git a/src/timedata.h b/src/timedata.h deleted file mode 100644 index 3e76f80452..0000000000 --- a/src/timedata.h +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) 2014-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_TIMEDATA_H -#define BITCOIN_TIMEDATA_H - -#include <algorithm> -#include <cassert> -#include <cstdint> -#include <vector> - -static const int64_t DEFAULT_MAX_TIME_ADJUSTMENT = 70 * 60; - -class CNetAddr; - -/** - * Median filter over a stream of values. - * Returns the median of the last N numbers - */ -template <typename T> -class CMedianFilter -{ -private: - std::vector<T> vValues; - std::vector<T> vSorted; - unsigned int nSize; - -public: - CMedianFilter(unsigned int _size, T initial_value) : nSize(_size) - { - vValues.reserve(_size); - vValues.push_back(initial_value); - vSorted = vValues; - } - - void input(T value) - { - if (vValues.size() == nSize) { - vValues.erase(vValues.begin()); - } - vValues.push_back(value); - - vSorted.resize(vValues.size()); - std::copy(vValues.begin(), vValues.end(), vSorted.begin()); - std::sort(vSorted.begin(), vSorted.end()); - } - - T median() const - { - int vSortedSize = vSorted.size(); - assert(vSortedSize > 0); - if (vSortedSize & 1) // Odd number of elements - { - return vSorted[vSortedSize / 2]; - } else // Even number of elements - { - return (vSorted[vSortedSize / 2 - 1] + vSorted[vSortedSize / 2]) / 2; - } - } - - int size() const - { - return vValues.size(); - } - - std::vector<T> sorted() const - { - return vSorted; - } -}; - -/** Functions to keep track of adjusted P2P time */ -int64_t GetTimeOffset(); -void AddTimeData(const CNetAddr& ip, int64_t nTime); - -/** - * Reset the internal state of GetTimeOffset() and AddTimeData(). - */ -void TestOnlyResetTimeData(); - -#endif // BITCOIN_TIMEDATA_H diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 82eec6241f..06066e38b2 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -1280,7 +1280,7 @@ std::optional<std::string> CTxMemPool::CheckConflictTopology(const setEntries& d return std::nullopt; } -util::Result<std::pair<std::vector<FeeFrac>, std::vector<FeeFrac>>> CTxMemPool::CalculateFeerateDiagramsForRBF(CAmount replacement_fees, int64_t replacement_vsize, const setEntries& direct_conflicts, const setEntries& all_conflicts) +util::Result<std::pair<std::vector<FeeFrac>, std::vector<FeeFrac>>> CTxMemPool::CalculateChunksForRBF(CAmount replacement_fees, int64_t replacement_vsize, const setEntries& direct_conflicts, const setEntries& all_conflicts) { Assume(replacement_vsize > 0); @@ -1335,7 +1335,6 @@ util::Result<std::pair<std::vector<FeeFrac>, std::vector<FeeFrac>>> CTxMemPool:: // No topology restrictions post-chunking; sort std::sort(old_chunks.begin(), old_chunks.end(), std::greater()); - std::vector<FeeFrac> old_diagram = BuildDiagramFromChunks(old_chunks); std::vector<FeeFrac> new_chunks; @@ -1365,6 +1364,5 @@ util::Result<std::pair<std::vector<FeeFrac>, std::vector<FeeFrac>>> CTxMemPool:: // No topology restrictions post-chunking; sort std::sort(new_chunks.begin(), new_chunks.end(), std::greater()); - std::vector<FeeFrac> new_diagram = BuildDiagramFromChunks(new_chunks); - return std::make_pair(old_diagram, new_diagram); + return std::make_pair(old_chunks, new_chunks); } diff --git a/src/txmempool.h b/src/txmempool.h index 9dd4d56da7..52a3dc2d7d 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -738,7 +738,7 @@ public: } /** - * Calculate the old and new mempool feerate diagrams relating to the + * Calculate the sorted chunks for the old and new mempool relating to the * clusters that would be affected by a potential replacement transaction. * (replacement_fees, replacement_vsize) values are gathered from a * proposed set of replacement transactions that are considered as a single @@ -752,7 +752,7 @@ public: * @param[in] all_conflicts All transactions that would be removed * @return old and new diagram pair respectively, or an error string if the conflicts don't match a calculable topology */ - util::Result<std::pair<std::vector<FeeFrac>, std::vector<FeeFrac>>> CalculateFeerateDiagramsForRBF(CAmount replacement_fees, int64_t replacement_vsize, const setEntries& direct_conflicts, const setEntries& all_conflicts) EXCLUSIVE_LOCKS_REQUIRED(cs); + util::Result<std::pair<std::vector<FeeFrac>, std::vector<FeeFrac>>> CalculateChunksForRBF(CAmount replacement_fees, int64_t replacement_vsize, const setEntries& direct_conflicts, const setEntries& all_conflicts) EXCLUSIVE_LOCKS_REQUIRED(cs); /* Check that all direct conflicts are in a cluster size of two or less. Each * direct conflict may be in a separate cluster. diff --git a/src/txorphanage.cpp b/src/txorphanage.cpp index e907fffd4f..92055ded72 100644 --- a/src/txorphanage.cpp +++ b/src/txorphanage.cpp @@ -241,3 +241,77 @@ void TxOrphanage::EraseForBlock(const CBlock& block) LogPrint(BCLog::TXPACKAGES, "Erased %d orphan tx included or conflicted by block\n", nErased); } } + +std::vector<CTransactionRef> TxOrphanage::GetChildrenFromSamePeer(const CTransactionRef& parent, NodeId nodeid) const +{ + LOCK(m_mutex); + + // First construct a vector of iterators to ensure we do not return duplicates of the same tx + // and so we can sort by nTimeExpire. + std::vector<OrphanMap::iterator> iters; + + // For each output, get all entries spending this prevout, filtering for ones from the specified peer. + for (unsigned int i = 0; i < parent->vout.size(); i++) { + const auto it_by_prev = m_outpoint_to_orphan_it.find(COutPoint(parent->GetHash(), i)); + if (it_by_prev != m_outpoint_to_orphan_it.end()) { + for (const auto& elem : it_by_prev->second) { + if (elem->second.fromPeer == nodeid) { + iters.emplace_back(elem); + } + } + } + } + + // Sort by address so that duplicates can be deleted. At the same time, sort so that more recent + // orphans (which expire later) come first. Break ties based on address, as nTimeExpire is + // quantified in seconds and it is possible for orphans to have the same expiry. + std::sort(iters.begin(), iters.end(), [](const auto& lhs, const auto& rhs) { + if (lhs->second.nTimeExpire == rhs->second.nTimeExpire) { + return &(*lhs) < &(*rhs); + } else { + return lhs->second.nTimeExpire > rhs->second.nTimeExpire; + } + }); + // Erase duplicates + iters.erase(std::unique(iters.begin(), iters.end()), iters.end()); + + // Convert to a vector of CTransactionRef + std::vector<CTransactionRef> children_found; + children_found.reserve(iters.size()); + for (const auto& child_iter : iters) { + children_found.emplace_back(child_iter->second.tx); + } + return children_found; +} + +std::vector<std::pair<CTransactionRef, NodeId>> TxOrphanage::GetChildrenFromDifferentPeer(const CTransactionRef& parent, NodeId nodeid) const +{ + LOCK(m_mutex); + + // First construct vector of iterators to ensure we do not return duplicates of the same tx. + std::vector<OrphanMap::iterator> iters; + + // For each output, get all entries spending this prevout, filtering for ones not from the specified peer. + for (unsigned int i = 0; i < parent->vout.size(); i++) { + const auto it_by_prev = m_outpoint_to_orphan_it.find(COutPoint(parent->GetHash(), i)); + if (it_by_prev != m_outpoint_to_orphan_it.end()) { + for (const auto& elem : it_by_prev->second) { + if (elem->second.fromPeer != nodeid) { + iters.emplace_back(elem); + } + } + } + } + + // Erase duplicates + std::sort(iters.begin(), iters.end(), IteratorComparator()); + iters.erase(std::unique(iters.begin(), iters.end()), iters.end()); + + // Convert iterators to pair<CTransactionRef, NodeId> + std::vector<std::pair<CTransactionRef, NodeId>> children_found; + children_found.reserve(iters.size()); + for (const auto& child_iter : iters) { + children_found.emplace_back(child_iter->second.tx, child_iter->second.fromPeer); + } + return children_found; +} diff --git a/src/txorphanage.h b/src/txorphanage.h index 2fd14e6fd2..a3c8edaa2a 100644 --- a/src/txorphanage.h +++ b/src/txorphanage.h @@ -51,6 +51,14 @@ public: /** Does this peer have any work to do? */ bool HaveTxToReconsider(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex);; + /** Get all children that spend from this tx and were received from nodeid. Sorted from most + * recent to least recent. */ + std::vector<CTransactionRef> GetChildrenFromSamePeer(const CTransactionRef& parent, NodeId nodeid) const EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); + + /** Get all children that spend from this tx but were not received from nodeid. Also return + * which peer provided each tx. */ + std::vector<std::pair<CTransactionRef, NodeId>> GetChildrenFromDifferentPeer(const CTransactionRef& parent, NodeId nodeid) const EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); + /** Return how many entries exist in the orphange */ size_t Size() EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) { diff --git a/src/util/bitdeque.h b/src/util/bitdeque.h index 1e34b72475..ac9d302420 100644 --- a/src/util/bitdeque.h +++ b/src/util/bitdeque.h @@ -14,18 +14,17 @@ /** Class that mimics std::deque<bool>, but with std::vector<bool>'s bit packing. * - * BlobSize selects the (minimum) number of bits that are allocated at once. + * BITS_PER_WORD selects the (minimum) number of bits that are allocated at once. * Larger values reduce the asymptotic memory usage overhead, at the cost of * needing larger up-front allocations. The default is 4096 bytes. */ -template<int BlobSize = 4096 * 8> +template<int BITS_PER_WORD = 4096 * 8> class bitdeque { // Internal definitions - using word_type = std::bitset<BlobSize>; + using word_type = std::bitset<BITS_PER_WORD>; using deque_type = std::deque<word_type>; - static_assert(BlobSize > 0); - static constexpr int BITS_PER_WORD = BlobSize; + static_assert(BITS_PER_WORD > 0); // Forward and friend declarations of iterator types. template<bool Const> class Iterator; diff --git a/src/util/feefrac.cpp b/src/util/feefrac.cpp index 68fb836936..5b6173835c 100644 --- a/src/util/feefrac.cpp +++ b/src/util/feefrac.cpp @@ -7,39 +7,26 @@ #include <array> #include <vector> -std::vector<FeeFrac> BuildDiagramFromChunks(const Span<const FeeFrac> chunks) -{ - std::vector<FeeFrac> diagram; - diagram.reserve(chunks.size() + 1); - - diagram.emplace_back(0, 0); - for (auto& chunk : chunks) { - diagram.emplace_back(diagram.back() + chunk); - } - return diagram; -} - -std::partial_ordering CompareFeerateDiagram(Span<const FeeFrac> dia0, Span<const FeeFrac> dia1) +std::partial_ordering CompareChunks(Span<const FeeFrac> chunks0, Span<const FeeFrac> chunks1) { /** Array to allow indexed access to input diagrams. */ - const std::array<Span<const FeeFrac>, 2> dias = {dia0, dia1}; + const std::array<Span<const FeeFrac>, 2> chunk = {chunks0, chunks1}; /** How many elements we have processed in each input. */ - size_t next_index[2] = {1, 1}; + size_t next_index[2] = {0, 0}; + /** Accumulated fee/sizes in diagrams, up to next_index[i] - 1. */ + FeeFrac accum[2]; /** Whether the corresponding input is strictly better than the other at least in one place. */ bool better_somewhere[2] = {false, false}; /** Get the first unprocessed point in diagram number dia. */ - const auto next_point = [&](int dia) { return dias[dia][next_index[dia]]; }; + const auto next_point = [&](int dia) { return chunk[dia][next_index[dia]] + accum[dia]; }; /** Get the last processed point in diagram number dia. */ - const auto prev_point = [&](int dia) { return dias[dia][next_index[dia] - 1]; }; - - // Diagrams should be non-empty, and first elements zero in size and fee - Assert(!dia0.empty() && !dia1.empty()); - Assert(prev_point(0).IsEmpty()); - Assert(prev_point(1).IsEmpty()); + const auto prev_point = [&](int dia) { return accum[dia]; }; + /** Move to the next point in diagram number dia. */ + const auto advance = [&](int dia) { accum[dia] += chunk[dia][next_index[dia]++]; }; do { - bool done_0 = next_index[0] == dias[0].size(); - bool done_1 = next_index[1] == dias[1].size(); + bool done_0 = next_index[0] == chunk[0].size(); + bool done_1 = next_index[1] == chunk[1].size(); if (done_0 && done_1) break; // Determine which diagram has the first unprocessed point. If a single side is finished, use the @@ -69,17 +56,16 @@ std::partial_ordering CompareFeerateDiagram(Span<const FeeFrac> dia0, Span<const // If B and P have the same size, B can be marked as processed (in addition to P, see // below), as we've already performed a comparison at this size. - if (point_b.size == point_p.size) ++next_index[!unproc_side]; + if (point_b.size == point_p.size) advance(!unproc_side); } // If P lies above AB, unproc_side is better in P. If P lies below AB, then !unproc_side is // better in P. if (std::is_gt(cmp)) better_somewhere[unproc_side] = true; if (std::is_lt(cmp)) better_somewhere[!unproc_side] = true; - ++next_index[unproc_side]; + advance(unproc_side); // If both diagrams are better somewhere, they are incomparable. if (better_somewhere[0] && better_somewhere[1]) return std::partial_ordering::unordered; - } while(true); // Otherwise compare the better_somewhere values. diff --git a/src/util/feefrac.h b/src/util/feefrac.h index 7102f85f88..9772162010 100644 --- a/src/util/feefrac.h +++ b/src/util/feefrac.h @@ -146,15 +146,14 @@ struct FeeFrac } }; -/** Takes the pre-computed and topologically-valid chunks and generates a fee diagram which starts at FeeFrac of (0, 0) */ -std::vector<FeeFrac> BuildDiagramFromChunks(Span<const FeeFrac> chunks); - -/** Compares two feerate diagrams. The shorter one is implicitly - * extended with a horizontal straight line. +/** Compare the feerate diagrams implied by the provided sorted chunks data. + * + * The implied diagram for each starts at (0, 0), then contains for each chunk the cumulative fee + * and size up to that chunk, and then extends infinitely to the right with a horizontal line. * - * A feerate diagram consists of a list of (fee, size) points with the property that size - * is strictly increasing and that the first entry is (0, 0). + * The caller must guarantee that the sum of the FeeFracs in either of the chunks' data set do not + * overflow (so sum fees < 2^63, and sum sizes < 2^31). */ -std::partial_ordering CompareFeerateDiagram(Span<const FeeFrac> dia0, Span<const FeeFrac> dia1); +std::partial_ordering CompareChunks(Span<const FeeFrac> chunks0, Span<const FeeFrac> chunks1); #endif // BITCOIN_UTIL_FEEFRAC_H diff --git a/src/util/result.h b/src/util/result.h index b99995c7e5..122a7638fa 100644 --- a/src/util/result.h +++ b/src/util/result.h @@ -39,6 +39,16 @@ private: std::variant<bilingual_str, T> m_variant; + //! Disallow copy constructor, require Result to be moved for efficiency. + Result(const Result&) = delete; + + //! Disallow operator= to avoid confusion in the future when the Result + //! class gains support for richer error reporting, and callers should have + //! ability to set a new result value without clearing existing error + //! messages. + Result& operator=(const Result&) = delete; + Result& operator=(Result&&) = delete; + template <typename FT> friend bilingual_str ErrorString(const Result<FT>& result); @@ -46,6 +56,8 @@ public: Result() : m_variant{std::in_place_index_t<1>{}, std::monostate{}} {} // constructor for void Result(T obj) : m_variant{std::in_place_index_t<1>{}, std::move(obj)} {} Result(Error error) : m_variant{std::in_place_index_t<0>{}, std::move(error.message)} {} + Result(Result&&) = default; + ~Result() = default; //! std::optional methods, so functions returning optional<T> can change to //! return Result<T> with minimal changes to existing code, and vice versa. diff --git a/src/util/subprocess.hpp b/src/util/subprocess.h index e660aa143d..af408b31d4 100644 --- a/src/util/subprocess.hpp +++ b/src/util/subprocess.h @@ -33,8 +33,10 @@ Documentation for C++ subprocessing library. @version 1.0.0 */ -#ifndef SUBPROCESS_HPP -#define SUBPROCESS_HPP +#ifndef BITCOIN_UTIL_SUBPROCESS_H +#define BITCOIN_UTIL_SUBPROCESS_H + +#include <util/syserror.h> #include <algorithm> #include <cassert> @@ -150,32 +152,13 @@ class OSError: public std::runtime_error { public: OSError(const std::string& err_msg, int err_code): - std::runtime_error( err_msg + ": " + std::strerror(err_code) ) + std::runtime_error(err_msg + ": " + SysErrorString(err_code)) {} }; //-------------------------------------------------------------------- - -//Environment Variable types -#ifndef _MSC_VER - using env_string_t = std::string; - using env_char_t = char; -#else - using env_string_t = std::wstring; - using env_char_t = wchar_t; -#endif -using env_map_t = std::map<env_string_t, env_string_t>; -using env_vector_t = std::vector<env_char_t>; - -//-------------------------------------------------------------------- namespace util { - template <typename R> - inline bool is_ready(std::shared_future<R> const &f) - { - return f.wait_for(std::chrono::seconds(0)) == std::future_status::ready; - } - inline void quote_argument(const std::wstring &argument, std::wstring &command_line, bool force) { @@ -305,100 +288,6 @@ namespace util if (!SetHandleInformation(*child_handle, HANDLE_FLAG_INHERIT, 0)) throw OSError("SetHandleInformation", 0); } - - // env_map_t MapFromWindowsEnvironment() - // * Imports current Environment in a C-string table - // * Parses the strings by splitting on the first "=" per line - // * Creates a map of the variables - // * Returns the map - inline env_map_t MapFromWindowsEnvironment(){ - wchar_t *variable_strings_ptr; - wchar_t *environment_strings_ptr; - std::wstring delimiter(L"="); - int del_len = delimiter.length(); - env_map_t mapped_environment; - - // Get a pointer to the environment block. - environment_strings_ptr = GetEnvironmentStringsW(); - // If the returned pointer is NULL, exit. - if (environment_strings_ptr == NULL) - { - throw OSError("GetEnvironmentStringsW", 0); - } - - // Variable strings are separated by NULL byte, and the block is - // terminated by a NULL byte. - - variable_strings_ptr = (wchar_t *) environment_strings_ptr; - - //Since the environment map ends with a null, we can loop until we find it. - while (*variable_strings_ptr) - { - // Create a string from Variable String - env_string_t current_line(variable_strings_ptr); - // Find the first "equals" sign. - auto pos = current_line.find(delimiter); - // Assuming it's not missing ... - if(pos!=std::wstring::npos){ - // ... parse the key and value. - env_string_t key = current_line.substr(0, pos); - env_string_t value = current_line.substr(pos + del_len); - // Map the entry. - mapped_environment[key] = value; - } - // Jump to next line in the environment map. - variable_strings_ptr += std::wcslen(variable_strings_ptr) + 1; - } - // We're done with the old environment map buffer. - FreeEnvironmentStringsW(environment_strings_ptr); - - // Return the map. - return mapped_environment; - } - - // env_vector_t WindowsEnvironmentVectorFromMap(const env_map_t &source_map) - // * Creates a vector buffer for the new environment string table - // * Copies in the mapped variables - // * Returns the vector - inline env_vector_t WindowsEnvironmentVectorFromMap(const env_map_t &source_map) - { - // Make a new environment map buffer. - env_vector_t environment_map_buffer; - // Give it some space. - environment_map_buffer.reserve(4096); - - // And fill'er up. - for(auto kv: source_map){ - // Create the line - env_string_t current_line(kv.first); current_line += L"="; current_line += kv.second; - // Add the line to the buffer. - std::copy(current_line.begin(), current_line.end(), std::back_inserter(environment_map_buffer)); - // Append a null - environment_map_buffer.push_back(0); - } - // Append one last null because of how Windows does it's environment maps. - environment_map_buffer.push_back(0); - - return environment_map_buffer; - } - - // env_vector_t CreateUpdatedWindowsEnvironmentVector(const env_map_t &changes_map) - // * Merges host environment with new mapped variables - // * Creates and returns string vector based on map - inline env_vector_t CreateUpdatedWindowsEnvironmentVector(const env_map_t &changes_map){ - // Import the environment map - env_map_t environment_map = MapFromWindowsEnvironment(); - // Merge in the changes with overwrite - for(auto& it: changes_map) - { - environment_map[it.first] = it.second; - } - // Create a Windows-usable Environment Map Buffer - env_vector_t environment_map_strings_vector = WindowsEnvironmentVectorFromMap(environment_map); - - return environment_map_strings_vector; - } - #endif /*! @@ -431,26 +320,6 @@ namespace util } - /*! - * Function: join - * Parameters: - * [in] vec : Vector of strings which needs to be joined to form - * a single string with words separated by a separator char. - * [in] sep : String used to separate 2 words in the joined string. - * Default constructed to ' ' (space). - * [out] string: Joined string. - */ - static inline - std::string join(const std::vector<std::string>& vec, - const std::string& sep = " ") - { - std::string res; - for (auto& elem : vec) res.append(elem + sep); - res.erase(--res.end()); - return res; - } - - #ifndef __USING_WINDOWS__ /*! * Function: set_clo_on_exec @@ -652,56 +521,6 @@ namespace util */ /*! - * The buffer size of the stdin/stdout/stderr - * streams of the child process. - * Default value is 0. - */ -struct bufsize { - explicit bufsize(int sz): bufsiz(sz) {} - int bufsiz = 0; -}; - -/*! - * Option to defer spawning of the child process - * till `Popen::start_process` API is called. - * Default value is false. - */ -struct defer_spawn { - explicit defer_spawn(bool d): defer(d) {} - bool defer = false; -}; - -/*! - * Option to close all file descriptors - * when the child process is spawned. - * The close fd list does not include - * input/output/error if they are explicitly - * set as part of the Popen arguments. - * - * Default value is false. - */ -struct close_fds { - explicit close_fds(bool c): close_all(c) {} - bool close_all = false; -}; - -/*! - * Option to make the child process as the - * session leader and thus the process - * group leader. - * Default value is false. - */ -struct session_leader { - explicit session_leader(bool sl): leader_(sl) {} - bool leader_ = false; -}; - -struct shell { - explicit shell(bool s): shell_(s) {} - bool shell_ = false; -}; - -/*! * Base class for all arguments involving string value. */ struct string_arg @@ -727,34 +546,6 @@ struct executable: string_arg }; /*! - * Option to set the current working directory - * of the spawned process. - * - * Eg: cwd{"/some/path/x"} - */ -struct cwd: string_arg -{ - template <typename T> - cwd(T&& arg): string_arg(std::forward<T>(arg)) {} -}; - -/*! - * Option to specify environment variables required by - * the spawned process. - * - * Eg: environment{{ {"K1", "V1"}, {"K2", "V2"},... }} - */ -struct environment -{ - environment(env_map_t&& env): - env_(std::move(env)) {} - explicit environment(const env_map_t& env): - env_(env) {} - env_map_t env_; -}; - - -/*! * Used for redirecting input/output/error */ enum IOTYPE { @@ -870,44 +661,6 @@ struct error int wr_ch_ = -1; }; -// Impoverished, meager, needy, truly needy -// version of type erasure to store function pointers -// needed to provide the functionality of preexec_func -// ATTN: Can be used only to execute functions with no -// arguments and returning void. -// Could have used more efficient methods, ofcourse, but -// that won't yield me the consistent syntax which I am -// aiming for. If you know, then please do let me know. - -class preexec_func -{ -public: - preexec_func() {} - - template <typename Func> - explicit preexec_func(Func f): holder_(new FuncHolder<Func>(std::move(f))) - {} - - void operator()() { - (*holder_)(); - } - -private: - struct HolderBase { - virtual void operator()() const = 0; - virtual ~HolderBase(){}; - }; - template <typename T> - struct FuncHolder: HolderBase { - FuncHolder(T func): func_(std::move(func)) {} - void operator()() const override { func_(); } - // The function pointer/reference - T func_; - }; - - std::unique_ptr<HolderBase> holder_ = nullptr; -}; - // ~~~~ End Popen Args ~~~~ @@ -917,8 +670,8 @@ private: * This is basically used to determine the length of the actual * data stored inside the dynamically resized vector. * - * This is what is returned as the output to communicate and check_output - * functions, so, users must know about this class. + * This is what is returned as the output to the communicate + * function, so, users must know about this class. * * OutBuffer and ErrBuffer are just different typedefs to this class. */ @@ -929,22 +682,6 @@ public: explicit Buffer(size_t cap) { buf.resize(cap); } void add_cap(size_t cap) { buf.resize(cap); } -#if 0 - Buffer(const Buffer& other): - buf(other.buf), - length(other.length) - { - std::cout << "COPY" << std::endl; - } - - Buffer(Buffer&& other): - buf(std::move(other.buf)), - length(other.length) - { - std::cout << "MOVE" << std::endl; - } -#endif - public: std::vector<char> buf; size_t length = 0; @@ -965,39 +702,9 @@ class Popen; */ namespace detail { - -// Metaprogram for searching a type within -// a variadic parameter pack -// This is particularly required to do a compile time -// checking of the arguments provided to 'check_output' function -// wherein the user is not expected to provide an 'output' option. - -template <typename... T> struct param_pack{}; - -template <typename F, typename T> struct has_type; - -template <typename F> -struct has_type<F, param_pack<>> { - static constexpr bool value = false; -}; - -template <typename F, typename... T> -struct has_type<F, param_pack<F, T...>> { - static constexpr bool value = true; -}; - -template <typename F, typename H, typename... T> -struct has_type<F, param_pack<H,T...>> { - static constexpr bool value = - std::is_same<F, typename std::decay<H>::type>::value ? true : has_type<F, param_pack<T...>>::value; -}; - -//---- - /*! * A helper class to Popen class for setting - * options as provided in the Popen constructor - * or in check_output arguments. + * options as provided in the Popen constructor. * This design allows us to _not_ have any fixed position * to any arguments and specify them in a way similar to what * can be done in python. @@ -1007,17 +714,9 @@ struct ArgumentDeducer ArgumentDeducer(Popen* p): popen_(p) {} void set_option(executable&& exe); - void set_option(cwd&& cwdir); - void set_option(bufsize&& bsiz); - void set_option(environment&& env); - void set_option(defer_spawn&& defer); - void set_option(shell&& sh); void set_option(input&& inp); void set_option(output&& out); void set_option(error&& err); - void set_option(close_fds&& cfds); - void set_option(preexec_func&& prefunc); - void set_option(session_leader&& sleader); private: Popen* popen_ = nullptr; @@ -1168,9 +867,6 @@ public:// Yes they are public HANDLE g_hChildStd_ERR_Wr = nullptr; #endif - // Buffer size for the IO streams - int bufsiz_ = 0; - // Pipes for communicating with child // Emulates stdin @@ -1200,26 +896,23 @@ private: * interface to the client. * * API's provided by the class: - * 1. Popen({"cmd"}, output{..}, error{..}, cwd{..}, ....) + * Popen({"cmd"}, output{..}, error{..}, ....) * Command provided as a sequence. - * 2. Popen("cmd arg1"m output{..}, error{..}, cwd{..}, ....) + * Popen("cmd arg1", output{..}, error{..}, ....) * Command provided in a single string. - * 3. wait() - Wait for the child to exit. - * 4. retcode() - The return code of the exited child. - * 5. pid() - PID of the spawned child. - * 6. poll() - Check the status of the running child. - * 7. kill(sig_num) - Kill the child. SIGTERM used by default. - * 8. send(...) - Send input to the input channel of the child. - * 9. communicate(...) - Get the output/error from the child and close the channels - * from the parent side. - *10. input() - Get the input channel/File pointer. Can be used for - * customizing the way of sending input to child. - *11. output() - Get the output channel/File pointer. Usually used - in case of redirection. See piping examples. - *12. error() - Get the error channel/File pointer. Usually used - in case of redirection. - *13. start_process() - Start the child process. Only to be used when - * `defer_spawn` option was provided in Popen constructor. + * wait() - Wait for the child to exit. + * retcode() - The return code of the exited child. + * pid() - PID of the spawned child. + * poll() - Check the status of the running child. + * send(...) - Send input to the input channel of the child. + * communicate(...) - Get the output/error from the child and close the channels + * from the parent side. + * input() - Get the input channel/File pointer. Can be used for + * customizing the way of sending input to child. + * output() - Get the output channel/File pointer. Usually used + in case of redirection. See piping examples. + * error() - Get the error channel/File pointer. Usually used + in case of redirection. */ class Popen { @@ -1237,7 +930,7 @@ public: // Setup the communication channels of the Popen class stream_.setup_comm_channels(); - if (!defer_process_start_) execute_process(); + execute_process(); } template <typename... Args> @@ -1249,7 +942,7 @@ public: // Setup the communication channels of the Popen class stream_.setup_comm_channels(); - if (!defer_process_start_) execute_process(); + execute_process(); } template <typename... Args> @@ -1260,20 +953,9 @@ public: // Setup the communication channels of the Popen class stream_.setup_comm_channels(); - if (!defer_process_start_) execute_process(); + execute_process(); } -/* - ~Popen() - { -#ifdef __USING_WINDOWS__ - CloseHandle(this->process_handle_); -#endif - } -*/ - - void start_process() noexcept(false); - int pid() const noexcept { return child_pid_; } int retcode() const noexcept { return retcode_; } @@ -1282,10 +964,6 @@ public: int poll() noexcept(false); - // Does not fail, Caller is expected to recheck the - // status with a call to poll() - void kill(int sig_num = 9); - void set_out_buf_cap(size_t cap) { stream_.set_out_buf_cap(cap); } void set_err_buf_cap(size_t cap) { stream_.set_err_buf_cap(cap); } @@ -1347,16 +1025,7 @@ private: std::future<void> cleanup_future_; #endif - bool defer_process_start_ = false; - bool close_fds_ = false; - bool has_preexec_fn_ = false; - bool shell_ = false; - bool session_leader_ = false; - std::string exe_name_; - std::string cwd_; - env_map_t env_; - preexec_func preexec_fn_; // Command in string format std::string args_; @@ -1391,20 +1060,6 @@ inline void Popen::populate_c_argv() cargv_.push_back(nullptr); } -inline void Popen::start_process() noexcept(false) -{ - // The process was started/tried to be started - // in the constructor itself. - // For explicitly calling this API to start the - // process, 'defer_spawn' argument must be set to - // true in the constructor. - if (!defer_process_start_) { - assert (0); - return; - } - execute_process(); -} - inline int Popen::wait() noexcept(false) { #ifdef __USING_WINDOWS__ @@ -1476,33 +1131,9 @@ inline int Popen::poll() noexcept(false) #endif } -inline void Popen::kill(int sig_num) -{ -#ifdef __USING_WINDOWS__ - if (!TerminateProcess(this->process_handle_, (UINT)sig_num)) { - throw OSError("TerminateProcess", 0); - } -#else - if (session_leader_) killpg(child_pid_, sig_num); - else ::kill(child_pid_, sig_num); -#endif -} - - inline void Popen::execute_process() noexcept(false) { #ifdef __USING_WINDOWS__ - if (this->shell_) { - throw OSError("shell not currently supported on windows", 0); - } - - void* environment_string_table_ptr = nullptr; - env_vector_t environment_string_vector; - if(this->env_.size()){ - environment_string_vector = util::CreateUpdatedWindowsEnvironmentVector(env_); - environment_string_table_ptr = (void*)environment_string_vector.data(); - } - if (exe_name_.length()) { this->vargs_.insert(this->vargs_.begin(), this->exe_name_); this->populate_c_argv(); @@ -1549,7 +1180,7 @@ inline void Popen::execute_process() noexcept(false) NULL, // primary thread security attributes TRUE, // handles are inherited creation_flags, // creation flags - environment_string_table_ptr, // use provided environment + NULL, // use parent's environment NULL, // use parent's current directory &siStartInfo, // STARTUPINFOW pointer &piProcInfo); // receives PROCESS_INFORMATION @@ -1588,14 +1219,6 @@ inline void Popen::execute_process() noexcept(false) int err_rd_pipe, err_wr_pipe; std::tie(err_rd_pipe, err_wr_pipe) = util::pipe_cloexec(); - if (shell_) { - auto new_cmd = util::join(vargs_); - vargs_.clear(); - vargs_.insert(vargs_.begin(), {"/bin/sh", "-c"}); - vargs_.push_back(new_cmd); - populate_c_argv(); - } - if (exe_name_.length()) { vargs_.insert(vargs_.begin(), exe_name_); populate_c_argv(); @@ -1662,30 +1285,6 @@ namespace detail { popen_->exe_name_ = std::move(exe.arg_value); } - inline void ArgumentDeducer::set_option(cwd&& cwdir) { - popen_->cwd_ = std::move(cwdir.arg_value); - } - - inline void ArgumentDeducer::set_option(bufsize&& bsiz) { - popen_->stream_.bufsiz_ = bsiz.bufsiz; - } - - inline void ArgumentDeducer::set_option(environment&& env) { - popen_->env_ = std::move(env.env_); - } - - inline void ArgumentDeducer::set_option(defer_spawn&& defer) { - popen_->defer_process_start_ = defer.defer; - } - - inline void ArgumentDeducer::set_option(shell&& sh) { - popen_->shell_ = sh.shell_; - } - - inline void ArgumentDeducer::set_option(session_leader&& sleader) { - popen_->session_leader_ = sleader.leader_; - } - inline void ArgumentDeducer::set_option(input&& inp) { if (inp.rd_ch_ != -1) popen_->stream_.read_from_parent_ = inp.rd_ch_; if (inp.wr_ch_ != -1) popen_->stream_.write_to_child_ = inp.wr_ch_; @@ -1708,15 +1307,6 @@ namespace detail { if (err.rd_ch_ != -1) popen_->stream_.err_read_ = err.rd_ch_; } - inline void ArgumentDeducer::set_option(close_fds&& cfds) { - popen_->close_fds_ = cfds.close_all; - } - - inline void ArgumentDeducer::set_option(preexec_func&& prefunc) { - popen_->preexec_fn_ = std::move(prefunc); - popen_->has_preexec_fn_ = true; - } - inline void Child::execute_child() { #ifndef __USING_WINDOWS__ @@ -1763,41 +1353,8 @@ namespace detail { if (stream.err_write_ != -1 && stream.err_write_ > 2) close(stream.err_write_); - // Close all the inherited fd's except the error write pipe - if (parent_->close_fds_) { - int max_fd = sysconf(_SC_OPEN_MAX); - if (max_fd == -1) throw OSError("sysconf failed", errno); - - for (int i = 3; i < max_fd; i++) { - if (i == err_wr_pipe_) continue; - close(i); - } - } - - // Change the working directory if provided - if (parent_->cwd_.length()) { - sys_ret = chdir(parent_->cwd_.c_str()); - if (sys_ret == -1) throw OSError("chdir failed", errno); - } - - if (parent_->has_preexec_fn_) { - parent_->preexec_fn_(); - } - - if (parent_->session_leader_) { - sys_ret = setsid(); - if (sys_ret == -1) throw OSError("setsid failed", errno); - } - // Replace the current image with the executable - if (parent_->env_.size()) { - for (auto& kv : parent_->env_) { - setenv(kv.first.c_str(), kv.second.c_str(), 1); - } - sys_ret = execvp(parent_->exe_name_.c_str(), parent_->cargv_.data()); - } else { - sys_ret = execvp(parent_->exe_name_.c_str(), parent_->cargv_.data()); - } + sys_ret = execvp(parent_->exe_name_.c_str(), parent_->cargv_.data()); if (sys_ret == -1) throw OSError("execve failed", errno); @@ -1840,16 +1397,7 @@ namespace detail { for (auto& h : handles) { if (h == nullptr) continue; - switch (bufsiz_) { - case 0: - setvbuf(h, nullptr, _IONBF, BUFSIZ); - break; - case 1: - setvbuf(h, nullptr, _IONBF, BUFSIZ); - break; - default: - setvbuf(h, nullptr, _IOFBF, bufsiz_); - }; + setvbuf(h, nullptr, _IONBF, BUFSIZ); } #endif } @@ -1985,4 +1533,4 @@ namespace detail { } -#endif // SUBPROCESS_HPP +#endif // BITCOIN_UTIL_SUBPROCESS_H diff --git a/src/util/time.cpp b/src/util/time.cpp index 456662bd84..f08eb5300a 100644 --- a/src/util/time.cpp +++ b/src/util/time.cpp @@ -16,11 +16,11 @@ void UninterruptibleSleep(const std::chrono::microseconds& n) { std::this_thread::sleep_for(n); } -static std::atomic<int64_t> nMockTime(0); //!< For testing +static std::atomic<std::chrono::seconds> g_mock_time{}; //!< For testing NodeClock::time_point NodeClock::now() noexcept { - const std::chrono::seconds mocktime{nMockTime.load(std::memory_order_relaxed)}; + const auto mocktime{g_mock_time.load(std::memory_order_relaxed)}; const auto ret{ mocktime.count() ? mocktime : @@ -29,20 +29,16 @@ NodeClock::time_point NodeClock::now() noexcept return time_point{ret}; }; -void SetMockTime(int64_t nMockTimeIn) -{ - Assert(nMockTimeIn >= 0); - nMockTime.store(nMockTimeIn, std::memory_order_relaxed); -} - +void SetMockTime(int64_t nMockTimeIn) { SetMockTime(std::chrono::seconds{nMockTimeIn}); } void SetMockTime(std::chrono::seconds mock_time_in) { - nMockTime.store(mock_time_in.count(), std::memory_order_relaxed); + Assert(mock_time_in >= 0s); + g_mock_time.store(mock_time_in, std::memory_order_relaxed); } std::chrono::seconds GetMockTime() { - return std::chrono::seconds(nMockTime.load(std::memory_order_relaxed)); + return g_mock_time.load(std::memory_order_relaxed); } int64_t GetTime() { return GetTime<std::chrono::seconds>().count(); } diff --git a/src/validation.cpp b/src/validation.cpp index 903f9caf13..f57851b4f7 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -946,8 +946,9 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) maybe_rbf_limits.descendant_size_vbytes += conflict->GetSizeWithDescendants(); } - auto ancestors{m_pool.CalculateMemPoolAncestors(*entry, maybe_rbf_limits)}; - if (!ancestors) { + if (auto ancestors{m_pool.CalculateMemPoolAncestors(*entry, maybe_rbf_limits)}) { + ws.m_ancestors = std::move(*ancestors); + } else { // If CalculateMemPoolAncestors fails second time, we want the original error string. // Contracting/payment channels CPFP carve-out: // If the new transaction is relatively small (up to 40k weight) @@ -970,11 +971,13 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) if (ws.m_vsize > EXTRA_DESCENDANT_TX_SIZE_LIMIT) { return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "too-long-mempool-chain", error_message); } - ancestors = m_pool.CalculateMemPoolAncestors(*entry, cpfp_carve_out_limits); - if (!ancestors) return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "too-long-mempool-chain", error_message); + if (auto ancestors_retry{m_pool.CalculateMemPoolAncestors(*entry, cpfp_carve_out_limits)}) { + ws.m_ancestors = std::move(*ancestors_retry); + } else { + return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "too-long-mempool-chain", error_message); + } } - ws.m_ancestors = *ancestors; // Even though just checking direct mempool parents for inheritance would be sufficient, we // check using the full ancestor set here because it's more convenient to use what we have // already calculated. diff --git a/src/validation.h b/src/validation.h index e3b2a2d59b..28b045fe80 100644 --- a/src/validation.h +++ b/src/validation.h @@ -113,7 +113,6 @@ void PruneBlockFilesManual(Chainstate& active_chainstate, int nManualPruneHeight *| txid in mempool? | yes | no | no* | yes | yes | *| wtxid in mempool? | yes | no | no* | yes | no | *| m_state | yes, IsValid() | yes, IsInvalid() | yes, IsInvalid() | yes, IsValid() | yes, IsValid() | -*| m_replaced_transactions | yes | no | no | no | no | *| m_vsize | yes | no | no | yes | no | *| m_base_fees | yes | no | no | yes | no | *| m_effective_feerate | yes | yes | no | no | no | @@ -139,7 +138,7 @@ struct MempoolAcceptResult { const TxValidationState m_state; /** Mempool transactions replaced by the tx. */ - const std::optional<std::list<CTransactionRef>> m_replaced_transactions; + const std::list<CTransactionRef> m_replaced_transactions; /** Virtual size as used by the mempool, calculated using serialized size and sigops. */ const std::optional<int64_t> m_vsize; /** Raw base fees in satoshis. */ diff --git a/src/wallet/external_signer_scriptpubkeyman.cpp b/src/wallet/external_signer_scriptpubkeyman.cpp index a71f8f9fbc..b5703fa54a 100644 --- a/src/wallet/external_signer_scriptpubkeyman.cpp +++ b/src/wallet/external_signer_scriptpubkeyman.cpp @@ -9,9 +9,11 @@ #include <wallet/external_signer_scriptpubkeyman.h> #include <iostream> +#include <key_io.h> #include <memory> #include <stdexcept> #include <string> +#include <univalue.h> #include <utility> #include <vector> @@ -51,15 +53,26 @@ ExternalSigner ExternalSignerScriptPubKeyMan::GetExternalSigner() { return signers[0]; } -bool ExternalSignerScriptPubKeyMan::DisplayAddress(const CScript scriptPubKey, const ExternalSigner &signer) const +util::Result<void> ExternalSignerScriptPubKeyMan::DisplayAddress(const CTxDestination& dest, const ExternalSigner &signer) const { // TODO: avoid the need to infer a descriptor from inside a descriptor wallet + const CScript& scriptPubKey = GetScriptForDestination(dest); auto provider = GetSolvingProvider(scriptPubKey); auto descriptor = InferDescriptor(scriptPubKey, *provider); - signer.DisplayAddress(descriptor->ToString()); - // TODO inspect result - return true; + const UniValue& result = signer.DisplayAddress(descriptor->ToString()); + + const UniValue& error = result.find_value("error"); + if (error.isStr()) return util::Error{strprintf(_("Signer returned error: %s"), error.getValStr())}; + + const UniValue& ret_address = result.find_value("address"); + if (!ret_address.isStr()) return util::Error{_("Signer did not echo address")}; + + if (ret_address.getValStr() != EncodeDestination(dest)) { + return util::Error{strprintf(_("Signer echoed unexpected address %s"), ret_address.getValStr())}; + } + + return util::Result<void>(); } // If sign is true, transaction must previously have been filled diff --git a/src/wallet/external_signer_scriptpubkeyman.h b/src/wallet/external_signer_scriptpubkeyman.h index c052ce6129..44286456b6 100644 --- a/src/wallet/external_signer_scriptpubkeyman.h +++ b/src/wallet/external_signer_scriptpubkeyman.h @@ -9,6 +9,8 @@ #include <memory> +struct bilingual_str; + namespace wallet { class ExternalSignerScriptPubKeyMan : public DescriptorScriptPubKeyMan { @@ -27,7 +29,11 @@ class ExternalSignerScriptPubKeyMan : public DescriptorScriptPubKeyMan static ExternalSigner GetExternalSigner(); - bool DisplayAddress(const CScript scriptPubKey, const ExternalSigner &signer) const; + /** + * Display address on the device and verify that the returned value matches. + * @returns nothing or an error message + */ + util::Result<void> DisplayAddress(const CTxDestination& dest, const ExternalSigner& signer) const; TransactionError FillPSBT(PartiallySignedTransaction& psbt, const PrecomputedTransactionData& txdata, int sighash_type = 1 /* SIGHASH_ALL */, bool sign = true, bool bip32derivs = false, int* n_signed = nullptr, bool finalize = true) const override; }; diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index a866666f64..0c1cae7253 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -247,7 +247,7 @@ public: return value.empty() ? m_wallet->EraseAddressReceiveRequest(batch, dest, id) : m_wallet->SetAddressReceiveRequest(batch, dest, id, value); } - bool displayAddress(const CTxDestination& dest) override + util::Result<void> displayAddress(const CTxDestination& dest) override { LOCK(m_wallet->cs_wallet); return m_wallet->DisplayAddress(dest); diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index 7f068c05ef..bed9ec029a 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -789,9 +789,8 @@ RPCHelpMan walletdisplayaddress() throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); } - if (!pwallet->DisplayAddress(dest)) { - throw JSONRPCError(RPC_MISC_ERROR, "Failed to display address"); - } + util::Result<void> res = pwallet->DisplayAddress(dest); + if (!res) throw JSONRPCError(RPC_MISC_ERROR, util::ErrorString(res).original); UniValue result(UniValue::VOBJ); result.pushKV("address", request.params[0].get_str()); diff --git a/src/wallet/rpc/coins.cpp b/src/wallet/rpc/coins.cpp index 0cb0891141..b6c7396f4b 100644 --- a/src/wallet/rpc/coins.cpp +++ b/src/wallet/rpc/coins.cpp @@ -194,15 +194,12 @@ RPCHelpMan getbalance() LOCK(pwallet->cs_wallet); - const auto dummy_value{self.MaybeArg<std::string>(0)}; + const auto dummy_value{self.MaybeArg<std::string>("dummy")}; if (dummy_value && *dummy_value != "*") { throw JSONRPCError(RPC_METHOD_DEPRECATED, "dummy first argument must be excluded or set to \"*\"."); } - int min_depth = 0; - if (!request.params[1].isNull()) { - min_depth = request.params[1].getInt<int>(); - } + const auto min_depth{self.Arg<int>("minconf")}; bool include_watchonly = ParseIncludeWatchonly(request.params[2], *pwallet); diff --git a/src/wallet/rpc/util.cpp b/src/wallet/rpc/util.cpp index 06ec7db1bc..1252843e9d 100644 --- a/src/wallet/rpc/util.cpp +++ b/src/wallet/rpc/util.cpp @@ -11,6 +11,7 @@ #include <wallet/context.h> #include <wallet/wallet.h> +#include <string_view> #include <univalue.h> #include <boost/date_time/posix_time/posix_time.hpp> @@ -61,9 +62,9 @@ bool ParseIncludeWatchonly(const UniValue& include_watchonly, const CWallet& wal bool GetWalletNameFromJSONRPCRequest(const JSONRPCRequest& request, std::string& wallet_name) { - if (URL_DECODE && request.URI.substr(0, WALLET_ENDPOINT_BASE.size()) == WALLET_ENDPOINT_BASE) { + if (request.URI.starts_with(WALLET_ENDPOINT_BASE)) { // wallet endpoint was used - wallet_name = URL_DECODE(request.URI.substr(WALLET_ENDPOINT_BASE.size())); + wallet_name = UrlDecode(std::string_view{request.URI}.substr(WALLET_ENDPOINT_BASE.size())); return true; } return false; diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 4575881d96..2c1ab8d44a 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -27,7 +27,6 @@ #include <unordered_map> enum class OutputType; -struct bilingual_str; namespace wallet { struct MigrationData; diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 5d23ebd35a..9a7e166e68 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -683,11 +683,11 @@ util::Result<SelectionResult> ChooseSelectionResult(interfaces::Chain& chain, co // Vector of results. We will choose the best one based on waste. std::vector<SelectionResult> results; std::vector<util::Result<SelectionResult>> errors; - auto append_error = [&] (const util::Result<SelectionResult>& result) { + auto append_error = [&] (util::Result<SelectionResult>&& result) { // If any specific error message appears here, then something different from a simple "no selection found" happened. // Let's save it, so it can be retrieved to the user if no other selection algorithm succeeded. if (HasErrorMsg(result)) { - errors.emplace_back(result); + errors.emplace_back(std::move(result)); } }; @@ -698,7 +698,7 @@ util::Result<SelectionResult> ChooseSelectionResult(interfaces::Chain& chain, co if (!coin_selection_params.m_subtract_fee_outputs) { if (auto bnb_result{SelectCoinsBnB(groups.positive_group, nTargetValue, coin_selection_params.m_cost_of_change, max_inputs_weight)}) { results.push_back(*bnb_result); - } else append_error(bnb_result); + } else append_error(std::move(bnb_result)); } // As Knapsack and SRD can create change, also deduce change weight. @@ -707,25 +707,25 @@ util::Result<SelectionResult> ChooseSelectionResult(interfaces::Chain& chain, co // The knapsack solver has some legacy behavior where it will spend dust outputs. We retain this behavior, so don't filter for positive only here. if (auto knapsack_result{KnapsackSolver(groups.mixed_group, nTargetValue, coin_selection_params.m_min_change_target, coin_selection_params.rng_fast, max_inputs_weight)}) { results.push_back(*knapsack_result); - } else append_error(knapsack_result); + } else append_error(std::move(knapsack_result)); if (coin_selection_params.m_effective_feerate > CFeeRate{3 * coin_selection_params.m_long_term_feerate}) { // Minimize input set for feerates of at least 3×LTFRE (default: 30 ṩ/vB+) if (auto cg_result{CoinGrinder(groups.positive_group, nTargetValue, coin_selection_params.m_min_change_target, max_inputs_weight)}) { cg_result->ComputeAndSetWaste(coin_selection_params.min_viable_change, coin_selection_params.m_cost_of_change, coin_selection_params.m_change_fee); results.push_back(*cg_result); } else { - append_error(cg_result); + append_error(std::move(cg_result)); } } if (auto srd_result{SelectCoinsSRD(groups.positive_group, nTargetValue, coin_selection_params.m_change_fee, coin_selection_params.rng_fast, max_inputs_weight)}) { results.push_back(*srd_result); - } else append_error(srd_result); + } else append_error(std::move(srd_result)); if (results.empty()) { // No solution found, retrieve the first explicit error (if any). // future: add 'severity level' to errors so the worst one can be retrieved instead of the first one. - return errors.empty() ? util::Error() : errors.front(); + return errors.empty() ? util::Error() : std::move(errors.front()); } // If the chosen input set has unconfirmed inputs, check for synergies from overlapping ancestry @@ -818,7 +818,7 @@ util::Result<SelectionResult> AutomaticCoinSelection(const CWallet& wallet, Coin // Coin Selection attempts to select inputs from a pool of eligible UTXOs to fund the // transaction at a target feerate. If an attempt fails, more attempts may be made using a more // permissive CoinEligibilityFilter. - util::Result<SelectionResult> res = [&] { + { // Place coins eligibility filters on a scope increasing order. std::vector<SelectionFilter> ordered_filters{ // If possible, fund the transaction with confirmed UTXOs only. Prefer at least six @@ -866,9 +866,9 @@ util::Result<SelectionResult> AutomaticCoinSelection(const CWallet& wallet, Coin if (CAmount total_amount = available_coins.GetTotalAmount() - total_discarded < value_to_select) { // Special case, too-long-mempool cluster. if (total_amount + total_unconf_long_chain > value_to_select) { - return util::Result<SelectionResult>({_("Unconfirmed UTXOs are available, but spending them creates a chain of transactions that will be rejected by the mempool")}); + return util::Error{_("Unconfirmed UTXOs are available, but spending them creates a chain of transactions that will be rejected by the mempool")}; } - return util::Result<SelectionResult>(util::Error()); // General "Insufficient Funds" + return util::Error{}; // General "Insufficient Funds" } // Walk-through the filters until the solution gets found. @@ -885,19 +885,17 @@ util::Result<SelectionResult> AutomaticCoinSelection(const CWallet& wallet, Coin // If any specific error message appears here, then something particularly wrong might have happened. // Save the error and continue the selection process. So if no solutions gets found, we can return // the detailed error to the upper layers. - if (HasErrorMsg(res)) res_detailed_errors.emplace_back(res); + if (HasErrorMsg(res)) res_detailed_errors.emplace_back(std::move(res)); } } // Return right away if we have a detailed error - if (!res_detailed_errors.empty()) return res_detailed_errors.front(); + if (!res_detailed_errors.empty()) return std::move(res_detailed_errors.front()); // General "Insufficient Funds" - return util::Result<SelectionResult>(util::Error()); - }(); - - return res; + return util::Error{}; + } } static bool IsCurrentForAntiFeeSniping(interfaces::Chain& chain, const uint256& block_hash) diff --git a/src/wallet/test/fuzz/coinselection.cpp b/src/wallet/test/fuzz/coinselection.cpp index 297432de9e..331590df7f 100644 --- a/src/wallet/test/fuzz/coinselection.cpp +++ b/src/wallet/test/fuzz/coinselection.cpp @@ -291,7 +291,10 @@ FUZZ_TARGET(coinselection) } std::vector<COutput> utxos; - std::vector<util::Result<SelectionResult>> results{result_srd, result_knapsack, result_bnb}; + std::vector<util::Result<SelectionResult>> results; + results.emplace_back(std::move(result_srd)); + results.emplace_back(std::move(result_knapsack)); + results.emplace_back(std::move(result_bnb)); CAmount new_total_balance{CreateCoins(fuzzed_data_provider, utxos, coin_params, next_locktime)}; if (new_total_balance > 0) { std::set<std::shared_ptr<COutput>> new_utxo_pool; diff --git a/src/wallet/test/fuzz/notifications.cpp b/src/wallet/test/fuzz/notifications.cpp index 9a515828fe..792079e6c6 100644 --- a/src/wallet/test/fuzz/notifications.cpp +++ b/src/wallet/test/fuzz/notifications.cpp @@ -106,13 +106,11 @@ struct FuzzedWallet { CTxDestination GetDestination(FuzzedDataProvider& fuzzed_data_provider) { auto type{fuzzed_data_provider.PickValueInArray(OUTPUT_TYPES)}; - util::Result<CTxDestination> op_dest{util::Error{}}; if (fuzzed_data_provider.ConsumeBool()) { - op_dest = wallet->GetNewDestination(type, ""); + return *Assert(wallet->GetNewDestination(type, "")); } else { - op_dest = wallet->GetNewChangeDestination(type); + return *Assert(wallet->GetNewChangeDestination(type)); } - return *Assert(op_dest); } CScript GetScriptPubKey(FuzzedDataProvider& fuzzed_data_provider) { return GetScriptForDestination(GetDestination(fuzzed_data_provider)); } void FundTx(FuzzedDataProvider& fuzzed_data_provider, CMutableTransaction tx) diff --git a/src/wallet/test/spend_tests.cpp b/src/wallet/test/spend_tests.cpp index 3509bc116f..963c0f838b 100644 --- a/src/wallet/test/spend_tests.cpp +++ b/src/wallet/test/spend_tests.cpp @@ -97,13 +97,11 @@ BOOST_FIXTURE_TEST_CASE(wallet_duplicated_preset_inputs_test, TestChain100Setup) // so that the recipient's amount is no longer equal to the user's selected target of 299 BTC. // First case, use 'subtract_fee_from_outputs=true' - util::Result<CreatedTransactionResult> res_tx = CreateTransaction(*wallet, recipients, /*change_pos=*/std::nullopt, coin_control); - BOOST_CHECK(!res_tx.has_value()); + BOOST_CHECK(!CreateTransaction(*wallet, recipients, /*change_pos=*/std::nullopt, coin_control)); // Second case, don't use 'subtract_fee_from_outputs'. recipients[0].fSubtractFeeFromAmount = false; - res_tx = CreateTransaction(*wallet, recipients, /*change_pos=*/std::nullopt, coin_control); - BOOST_CHECK(!res_tx.has_value()); + BOOST_CHECK(!CreateTransaction(*wallet, recipients, /*change_pos=*/std::nullopt, coin_control)); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 96c4397504..8f4171eb15 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2667,7 +2667,7 @@ void ReserveDestination::ReturnDestination() address = CNoDestination(); } -bool CWallet::DisplayAddress(const CTxDestination& dest) +util::Result<void> CWallet::DisplayAddress(const CTxDestination& dest) { CScript scriptPubKey = GetScriptForDestination(dest); for (const auto& spk_man : GetScriptPubKeyMans(scriptPubKey)) { @@ -2676,9 +2676,9 @@ bool CWallet::DisplayAddress(const CTxDestination& dest) continue; } ExternalSigner signer = ExternalSignerScriptPubKeyMan::GetExternalSigner(); - return signer_spk_man->DisplayAddress(scriptPubKey, signer); + return signer_spk_man->DisplayAddress(dest, signer); } - return false; + return util::Error{_("There is no ScriptPubKeyManager for this address")}; } bool CWallet::LockCoin(const COutPoint& output, WalletBatch* batch) diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index b49b5a7d0d..6a998fa398 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -537,8 +537,8 @@ public: bool IsSpentKey(const CScript& scriptPubKey) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void SetSpentKeyState(WalletBatch& batch, const uint256& hash, unsigned int n, bool used, std::set<CTxDestination>& tx_destinations) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - /** Display address on an external signer. Returns false if external signer support is not compiled */ - bool DisplayAddress(const CTxDestination& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /** Display address on an external signer. */ + util::Result<void> DisplayAddress(const CTxDestination& dest) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool IsLockedCoin(const COutPoint& output) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool LockCoin(const COutPoint& output, WalletBatch* batch = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); diff --git a/src/warnings.cpp b/src/warnings.cpp index 84b021dad5..d55eecc48d 100644 --- a/src/warnings.cpp +++ b/src/warnings.cpp @@ -14,11 +14,13 @@ #include <util/string.h> #include <util/translation.h> +#include <optional> #include <vector> static GlobalMutex g_warnings_mutex; static bilingual_str g_misc_warnings GUARDED_BY(g_warnings_mutex); static bool fLargeWorkInvalidChainFound GUARDED_BY(g_warnings_mutex) = false; +static std::optional<bilingual_str> g_timeoffset_warning GUARDED_BY(g_warnings_mutex){}; void SetMiscWarning(const bilingual_str& warning) { @@ -32,6 +34,11 @@ void SetfLargeWorkInvalidChainFound(bool flag) fLargeWorkInvalidChainFound = flag; } +void SetMedianTimeOffsetWarning(std::optional<bilingual_str> warning) +{ + LOCK(g_warnings_mutex); + g_timeoffset_warning = warning; +} bilingual_str GetWarnings(bool verbose) { bilingual_str warnings_concise; @@ -56,6 +63,10 @@ bilingual_str GetWarnings(bool verbose) warnings_verbose.emplace_back(warnings_concise); } + if (g_timeoffset_warning) { + warnings_verbose.emplace_back(g_timeoffset_warning.value()); + } + if (verbose) { return Join(warnings_verbose, Untranslated("<hr />")); } diff --git a/src/warnings.h b/src/warnings.h index b21e2ea2b8..866bce6246 100644 --- a/src/warnings.h +++ b/src/warnings.h @@ -6,12 +6,15 @@ #ifndef BITCOIN_WARNINGS_H #define BITCOIN_WARNINGS_H +#include <optional> #include <string> struct bilingual_str; void SetMiscWarning(const bilingual_str& warning); void SetfLargeWorkInvalidChainFound(bool flag); +/** Pass std::nullopt to disable the warning */ +void SetMedianTimeOffsetWarning(std::optional<bilingual_str> warning); /** Format a string that describes several potential problems detected by the core. * @param[in] verbose bool * - if true, get all warnings separated by <hr /> diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index d10db046f5..536e471053 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -8,6 +8,7 @@ #include <kernel/chain.h> #include <kernel/mempool_entry.h> #include <logging.h> +#include <netbase.h> #include <primitives/block.h> #include <primitives/transaction.h> #include <validationinterface.h> @@ -57,7 +58,12 @@ std::unique_ptr<CZMQNotificationInterface> CZMQNotificationInterface::Create(std { std::string arg("-zmq" + entry.first); const auto& factory = entry.second; - for (const std::string& address : gArgs.GetArgs(arg)) { + for (std::string& address : gArgs.GetArgs(arg)) { + // libzmq uses prefix "ipc://" for UNIX domain sockets + if (address.substr(0, ADDR_PREFIX_UNIX.length()) == ADDR_PREFIX_UNIX) { + address.replace(0, ADDR_PREFIX_UNIX.length(), ADDR_PREFIX_IPC); + } + std::unique_ptr<CZMQAbstractNotifier> notifier = factory(); notifier->SetType(entry.first); notifier->SetAddress(address); diff --git a/src/zmq/zmqutil.h b/src/zmq/zmqutil.h index 334b51aa91..bec48c0a56 100644 --- a/src/zmq/zmqutil.h +++ b/src/zmq/zmqutil.h @@ -9,4 +9,7 @@ void zmqError(const std::string& str); +/** Prefix for unix domain socket addresses (which are local filesystem paths) */ +const std::string ADDR_PREFIX_IPC = "ipc://"; // used by libzmq, example "ipc:///root/path/to/file" + #endif // BITCOIN_ZMQ_ZMQUTIL_H diff --git a/test/functional/feature_assumeutxo.py b/test/functional/feature_assumeutxo.py index 19cbbcffdb..2842d82d80 100755 --- a/test/functional/feature_assumeutxo.py +++ b/test/functional/feature_assumeutxo.py @@ -13,8 +13,6 @@ The assumeutxo value generated and used here is committed to in Interesting test cases could be loading an assumeutxo snapshot file with: -- TODO: Valid hash but invalid snapshot file (bad coin height or - bad other serialization) - TODO: Valid snapshot file, but referencing a snapshot block that turns out to be invalid, or has an invalid parent - TODO: Valid snapshot file and snapshot block, but the block is not on the @@ -98,18 +96,23 @@ class AssumeutxoTest(BitcoinTestFramework): self.log.info(" - snapshot file with alternated UTXO data") cases = [ - [b"\xff" * 32, 0, "7d52155c9a9fdc4525b637ef6170568e5dad6fabd0b1fdbb9432010b8453095b"], # wrong outpoint hash - [(1).to_bytes(4, "little"), 32, "9f4d897031ab8547665b4153317ae2fdbf0130c7840b66427ebc48b881cb80ad"], # wrong outpoint index - [b"\x81", 36, "3da966ba9826fb6d2604260e01607b55ba44e1a5de298606b08704bc62570ea8"], # wrong coin code VARINT((coinbase ? 1 : 0) | (height << 1)) - [b"\x80", 36, "091e893b3ccb4334378709578025356c8bcb0a623f37c7c4e493133c988648e5"], # another wrong coin code + # (content, offset, wrong_hash, custom_message) + [b"\xff" * 32, 0, "7d52155c9a9fdc4525b637ef6170568e5dad6fabd0b1fdbb9432010b8453095b", None], # wrong outpoint hash + [(1).to_bytes(4, "little"), 32, "9f4d897031ab8547665b4153317ae2fdbf0130c7840b66427ebc48b881cb80ad", None], # wrong outpoint index + [b"\x81", 36, "3da966ba9826fb6d2604260e01607b55ba44e1a5de298606b08704bc62570ea8", None], # wrong coin code VARINT + [b"\x80", 36, "091e893b3ccb4334378709578025356c8bcb0a623f37c7c4e493133c988648e5", None], # another wrong coin code + [b"\x84\x58", 36, None, "[snapshot] bad snapshot data after deserializing 0 coins"], # wrong coin case with height 364 and coinbase 0 + [b"\xCA\xD2\x8F\x5A", 41, None, "[snapshot] bad snapshot data after deserializing 0 coins - bad tx out value"], # Amount exceeds MAX_MONEY ] - for content, offset, wrong_hash in cases: + for content, offset, wrong_hash, custom_message in cases: with open(bad_snapshot_path, "wb") as f: f.write(valid_snapshot_contents[:(32 + 8 + offset)]) f.write(content) f.write(valid_snapshot_contents[(32 + 8 + offset + len(content)):]) - expected_error(log_msg=f"[snapshot] bad snapshot content hash: expected a4bf3407ccb2cc0145c49ebba8fa91199f8a3903daf0883875941497d2493c27, got {wrong_hash}") + + log_msg = custom_message if custom_message is not None else f"[snapshot] bad snapshot content hash: expected a4bf3407ccb2cc0145c49ebba8fa91199f8a3903daf0883875941497d2493c27, got {wrong_hash}" + expected_error(log_msg=log_msg) def test_headers_not_synced(self, valid_snapshot_path): for node in self.nodes[1:]: diff --git a/test/functional/feature_framework_unit_tests.py b/test/functional/feature_framework_unit_tests.py new file mode 100755 index 0000000000..c9754e083c --- /dev/null +++ b/test/functional/feature_framework_unit_tests.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017-2024 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Framework unit tests + +Unit tests for the test framework. +""" + +import sys +import unittest + +from test_framework.test_framework import TEST_EXIT_PASSED, TEST_EXIT_FAILED + +# List of framework modules containing unit tests. Should be kept in sync with +# the output of `git grep unittest.TestCase ./test/functional/test_framework` +TEST_FRAMEWORK_MODULES = [ + "address", + "crypto.bip324_cipher", + "blocktools", + "crypto.chacha20", + "crypto.ellswift", + "key", + "messages", + "crypto.muhash", + "crypto.poly1305", + "crypto.ripemd160", + "script", + "segwit_addr", + "wallet_util", +] + + +def run_unit_tests(): + test_framework_tests = unittest.TestSuite() + for module in TEST_FRAMEWORK_MODULES: + test_framework_tests.addTest( + unittest.TestLoader().loadTestsFromName(f"test_framework.{module}") + ) + result = unittest.TextTestRunner(stream=sys.stdout, verbosity=1, failfast=True).run( + test_framework_tests + ) + if not result.wasSuccessful(): + sys.exit(TEST_EXIT_FAILED) + sys.exit(TEST_EXIT_PASSED) + + +if __name__ == "__main__": + run_unit_tests() + diff --git a/test/functional/feature_maxtipage.py b/test/functional/feature_maxtipage.py index 51f37ef1e0..a1774a5395 100755 --- a/test/functional/feature_maxtipage.py +++ b/test/functional/feature_maxtipage.py @@ -43,6 +43,10 @@ class MaxTipAgeTest(BitcoinTestFramework): self.generate(node_miner, 1) assert_equal(node_ibd.getblockchaininfo()['initialblockdownload'], False) + # reset time to system time so we don't have a time offset with the ibd node the next + # time we connect to it, ensuring TimeOffsets::WarnIfOutOfSync() doesn't output to stderr + node_miner.setmocktime(0) + def run_test(self): self.log.info("Test IBD with maximum tip age of 24 hours (default).") self.test_maxtipage(DEFAULT_MAX_TIP_AGE, set_parameter=False) diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index e85541d0ec..e7d65b4539 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -10,7 +10,6 @@ from test_framework.blocktools import ( create_block, add_witness_commitment, MAX_BLOCK_SIGOPS_WEIGHT, - WITNESS_SCALE_FACTOR, ) from test_framework.messages import ( COutPoint, @@ -20,6 +19,7 @@ from test_framework.messages import ( CTxOut, SEQUENCE_FINAL, tx_from_hex, + WITNESS_SCALE_FACTOR, ) from test_framework.script import ( ANNEX_TAG, diff --git a/test/functional/interface_zmq.py b/test/functional/interface_zmq.py index 3c3ff1e4a0..9f6f8919de 100755 --- a/test/functional/interface_zmq.py +++ b/test/functional/interface_zmq.py @@ -3,7 +3,9 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test the ZMQ notification interface.""" +import os import struct +import tempfile from time import sleep from io import BytesIO @@ -30,7 +32,7 @@ from test_framework.util import ( from test_framework.wallet import ( MiniWallet, ) -from test_framework.netutil import test_ipv6_local +from test_framework.netutil import test_ipv6_local, test_unix_socket # Test may be skipped and not have zmq installed @@ -119,6 +121,10 @@ class ZMQTest (BitcoinTestFramework): self.ctx = zmq.Context() try: self.test_basic() + if test_unix_socket(): + self.test_basic(unix=True) + else: + self.log.info("Skipping ipc test, because UNIX sockets are not supported.") self.test_sequence() self.test_mempool_sync() self.test_reorg() @@ -139,7 +145,7 @@ class ZMQTest (BitcoinTestFramework): socket.setsockopt(zmq.IPV6, 1) subscribers.append(ZMQSubscriber(socket, topic.encode())) - self.restart_node(0, [f"-zmqpub{topic}={address}" for topic, address in services]) + self.restart_node(0, [f"-zmqpub{topic}={address.replace('ipc://', 'unix:')}" for topic, address in services]) for i, sub in enumerate(subscribers): sub.socket.connect(services[i][1]) @@ -176,12 +182,19 @@ class ZMQTest (BitcoinTestFramework): return subscribers - def test_basic(self): + def test_basic(self, unix = False): + self.log.info(f"Running basic test with {'ipc' if unix else 'tcp'} protocol") # Invalid zmq arguments don't take down the node, see #17185. self.restart_node(0, ["-zmqpubrawtx=foo", "-zmqpubhashtx=bar"]) address = f"tcp://127.0.0.1:{self.zmq_port_base}" + + if unix: + # Use the shortest temp path possible since paths may have as little as 92-char limit + socket_path = tempfile.NamedTemporaryFile().name + address = f"ipc://{socket_path}" + subs = self.setup_zmq_test([(topic, address) for topic in ["hashblock", "hashtx", "rawblock", "rawtx"]]) hashblock = subs[0] @@ -247,6 +260,8 @@ class ZMQTest (BitcoinTestFramework): ]) assert_equal(self.nodes[1].getzmqnotifications(), []) + if unix: + os.unlink(socket_path) def test_reorg(self): diff --git a/test/functional/mempool_accept.py b/test/functional/mempool_accept.py index 272e932fcc..b00be5f4f0 100755 --- a/test/functional/mempool_accept.py +++ b/test/functional/mempool_accept.py @@ -28,6 +28,8 @@ from test_framework.script import ( OP_HASH160, OP_RETURN, OP_TRUE, + SIGHASH_ALL, + sign_input_legacy, ) from test_framework.script_util import ( DUMMY_MIN_OP_RETURN_SCRIPT, @@ -386,5 +388,24 @@ class MempoolAcceptanceTest(BitcoinTestFramework): maxfeerate=0, ) + self.log.info('Spending a confirmed bare multisig is okay') + address = self.wallet.get_address() + tx = tx_from_hex(raw_tx_reference) + privkey, pubkey = generate_keypair() + tx.vout[0].scriptPubKey = keys_to_multisig_script([pubkey] * 3, k=1) # Some bare multisig script (1-of-3) + tx.rehash() + self.generateblock(node, address, [tx.serialize().hex()]) + tx_spend = CTransaction() + tx_spend.vin.append(CTxIn(COutPoint(tx.sha256, 0), b"")) + tx_spend.vout.append(CTxOut(tx.vout[0].nValue - int(fee*COIN), script_to_p2wsh_script(CScript([OP_TRUE])))) + tx_spend.rehash() + sign_input_legacy(tx_spend, 0, tx.vout[0].scriptPubKey, privkey, sighash_type=SIGHASH_ALL) + tx_spend.vin[0].scriptSig = bytes(CScript([OP_0])) + tx_spend.vin[0].scriptSig + self.check_mempool_result( + result_expected=[{'txid': tx_spend.rehash(), 'allowed': True, 'vsize': tx_spend.get_vsize(), 'fees': { 'base': Decimal('0.00000700')}}], + rawtxs=[tx_spend.serialize().hex()], + maxfeerate=0, + ) + if __name__ == '__main__': MempoolAcceptanceTest().main() diff --git a/test/functional/mempool_accept_v3.py b/test/functional/mempool_accept_v3.py index 1b55cd0a0d..8285b82c19 100755 --- a/test/functional/mempool_accept_v3.py +++ b/test/functional/mempool_accept_v3.py @@ -533,10 +533,10 @@ class MempoolAcceptV3(BitcoinTestFramework): tx_unrelated_replacee = self.wallet.send_self_transfer(from_node=node, utxo_to_spend=utxo_unrelated_conflict) assert tx_unrelated_replacee["txid"] in node.getrawmempool() - fee_to_beat_child2 = int(tx_v3_child_2["fee"] * COIN) + fee_to_beat = max(int(tx_v3_child_2["fee"] * COIN), int(tx_unrelated_replacee["fee"]*COIN)) tx_v3_child_3 = self.wallet.create_self_transfer_multi( - utxos_to_spend=[tx_v3_parent["new_utxos"][0], utxo_unrelated_conflict], fee_per_output=fee_to_beat_child2*5, version=3 + utxos_to_spend=[tx_v3_parent["new_utxos"][0], utxo_unrelated_conflict], fee_per_output=fee_to_beat*2, version=3 ) node.sendrawtransaction(tx_v3_child_3["hex"]) self.check_mempool(txids_v2_100 + [tx_v3_parent["txid"], tx_v3_child_3["txid"]]) diff --git a/test/functional/mining_basic.py b/test/functional/mining_basic.py index da796d3f70..5f2dde8eac 100755 --- a/test/functional/mining_basic.py +++ b/test/functional/mining_basic.py @@ -308,7 +308,7 @@ class MiningTest(BitcoinTestFramework): # Should ask for the block from a p2p node, if they announce the header as well: peer = node.add_p2p_connection(P2PDataStore()) - peer.wait_for_getheaders(timeout=5) # Drop the first getheaders + peer.wait_for_getheaders(timeout=5, block_hash=block.hashPrevBlock) peer.send_blocks_and_test(blocks=[block], node=node) # Must be active now: assert chain_tip(block.hash, status='active', branchlen=0) in node.getchaintips() diff --git a/test/functional/mocks/signer.py b/test/functional/mocks/signer.py index 5f4fad6380..23d163aac3 100755 --- a/test/functional/mocks/signer.py +++ b/test/functional/mocks/signer.py @@ -25,35 +25,36 @@ def getdescriptors(args): sys.stdout.write(json.dumps({ "receive": [ - "pkh([00000001/44'/1'/" + args.account + "']" + xpub + "/0/*)#vt6w3l3j", - "sh(wpkh([00000001/49'/1'/" + args.account + "']" + xpub + "/0/*))#r0grqw5x", - "wpkh([00000001/84'/1'/" + args.account + "']" + xpub + "/0/*)#x30uthjs", - "tr([00000001/86'/1'/" + args.account + "']" + xpub + "/0/*)#sng9rd4t" + "pkh([00000001/44h/1h/" + args.account + "']" + xpub + "/0/*)#aqllu46s", + "sh(wpkh([00000001/49h/1h/" + args.account + "']" + xpub + "/0/*))#5dh56mgg", + "wpkh([00000001/84h/1h/" + args.account + "']" + xpub + "/0/*)#h62dxaej", + "tr([00000001/86h/1h/" + args.account + "']" + xpub + "/0/*)#pcd5w87f" ], "internal": [ - "pkh([00000001/44'/1'/" + args.account + "']" + xpub + "/1/*)#all0v2p2", - "sh(wpkh([00000001/49'/1'/" + args.account + "']" + xpub + "/1/*))#kwx4c3pe", - "wpkh([00000001/84'/1'/" + args.account + "']" + xpub + "/1/*)#h92akzzg", - "tr([00000001/86'/1'/" + args.account + "']" + xpub + "/1/*)#p8dy7c9n" + "pkh([00000001/44h/1h/" + args.account + "']" + xpub + "/1/*)#v567pq2g", + "sh(wpkh([00000001/49h/1h/" + args.account + "']" + xpub + "/1/*))#pvezzyah", + "wpkh([00000001/84h/1h/" + args.account + "']" + xpub + "/1/*)#xw0vmgf2", + "tr([00000001/86h/1h/" + args.account + "']" + xpub + "/1/*)#svg4njw3" ] })) def displayaddress(args): - # Several descriptor formats are acceptable, so allowing for potential - # changes to InferDescriptor: if args.fingerprint != "00000001": return sys.stdout.write(json.dumps({"error": "Unexpected fingerprint", "fingerprint": args.fingerprint})) - expected_desc = [ - "wpkh([00000001/84'/1'/0'/0/0]02c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#0yneg42r", - "tr([00000001/86'/1'/0'/0/0]c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#4vdj9jqk", - ] + expected_desc = { + "wpkh([00000001/84h/1h/0h/0/0]02c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#3te6hhy7": "bcrt1qm90ugl4d48jv8n6e5t9ln6t9zlpm5th68x4f8g", + "sh(wpkh([00000001/49h/1h/0h/0/0]02c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7))#kz9y5w82": "2N2gQKzjUe47gM8p1JZxaAkTcoHPXV6YyVp", + "pkh([00000001/44h/1h/0h/0/0]02c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#q3pqd8wh": "n1LKejAadN6hg2FrBXoU1KrwX4uK16mco9", + "tr([00000001/86h/1h/0h/0/0]c97dc3f4420402e01a113984311bf4a1b8de376cac0bdcfaf1b3ac81f13433c7)#puqqa90m": "tb1phw4cgpt6cd30kz9k4wkpwm872cdvhss29jga2xpmftelhqll62mscq0k4g", + "wpkh([00000001/84h/1h/0h/0/1]03a20a46308be0b8ded6dff0a22b10b4245c587ccf23f3b4a303885be3a524f172)#aqpjv5xr": "wrong_address", + } if args.desc not in expected_desc: return sys.stdout.write(json.dumps({"error": "Unexpected descriptor", "desc": args.desc})) - return sys.stdout.write(json.dumps({"address": "bcrt1qm90ugl4d48jv8n6e5t9ln6t9zlpm5th68x4f8g"})) + return sys.stdout.write(json.dumps({"address": expected_desc[args.desc]})) def signtx(args): if args.fingerprint != "00000001": diff --git a/test/functional/p2p_1p1c_network.py b/test/functional/p2p_1p1c_network.py new file mode 100755 index 0000000000..e88c826962 --- /dev/null +++ b/test/functional/p2p_1p1c_network.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test that 1p1c package submission allows a 1p1c package to propagate in a "network" of nodes. Send +various packages from different nodes on a network in which some nodes have already received some of +the transactions (and submitted them to mempool, kept them as orphans or rejected them as +too-low-feerate transactions). The packages should be received and accepted by all nodes. +""" + +from decimal import Decimal +from math import ceil + +from test_framework.messages import ( + msg_tx, +) +from test_framework.p2p import ( + P2PInterface, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + fill_mempool, +) +from test_framework.wallet import ( + MiniWallet, + MiniWalletMode, +) + +# 1sat/vB feerate denominated in BTC/KvB +FEERATE_1SAT_VB = Decimal("0.00001000") + +class PackageRelayTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 4 + # hugely speeds up the test, as it involves multiple hops of tx relay. + self.noban_tx_relay = True + self.extra_args = [[ + "-datacarriersize=100000", + "-maxmempool=5", + ]] * self.num_nodes + self.supports_cli = False + + def raise_network_minfee(self): + filler_wallet = MiniWallet(self.nodes[0]) + fill_mempool(self, self.nodes[0], filler_wallet) + + self.log.debug("Wait for the network to sync mempools") + self.sync_mempools() + + self.log.debug("Check that all nodes' mempool minimum feerates are above min relay feerate") + for node in self.nodes: + assert_equal(node.getmempoolinfo()['minrelaytxfee'], FEERATE_1SAT_VB) + assert_greater_than(node.getmempoolinfo()['mempoolminfee'], FEERATE_1SAT_VB) + + def create_basic_1p1c(self, wallet): + low_fee_parent = wallet.create_self_transfer(fee_rate=FEERATE_1SAT_VB, confirmed_only=True) + high_fee_child = wallet.create_self_transfer(utxo_to_spend=low_fee_parent["new_utxo"], fee_rate=999*FEERATE_1SAT_VB) + package_hex_basic = [low_fee_parent["hex"], high_fee_child["hex"]] + return package_hex_basic, low_fee_parent["tx"], high_fee_child["tx"] + + def create_package_2outs(self, wallet): + # First create a tester tx to see the vsize, and then adjust the fees + utxo_for_2outs = wallet.get_utxo(confirmed_only=True) + + low_fee_parent_2outs_tester = wallet.create_self_transfer_multi( + utxos_to_spend=[utxo_for_2outs], + num_outputs=2, + ) + + # Target 1sat/vB so the number of satoshis is equal to the vsize. + # Round up. The goal is to be between min relay feerate and mempool min feerate. + fee_2outs = ceil(low_fee_parent_2outs_tester["tx"].get_vsize() / 2) + + low_fee_parent_2outs = wallet.create_self_transfer_multi( + utxos_to_spend=[utxo_for_2outs], + num_outputs=2, + fee_per_output=fee_2outs, + ) + + # Now create the child + high_fee_child_2outs = wallet.create_self_transfer_multi( + utxos_to_spend=low_fee_parent_2outs["new_utxos"][::-1], + fee_per_output=fee_2outs*100, + ) + return [low_fee_parent_2outs["hex"], high_fee_child_2outs["hex"]], low_fee_parent_2outs["tx"], high_fee_child_2outs["tx"] + + def create_package_2p1c(self, wallet): + parent1 = wallet.create_self_transfer(fee_rate=FEERATE_1SAT_VB*10, confirmed_only=True) + parent2 = wallet.create_self_transfer(fee_rate=FEERATE_1SAT_VB*20, confirmed_only=True) + child = wallet.create_self_transfer_multi( + utxos_to_spend=[parent1["new_utxo"], parent2["new_utxo"]], + fee_per_output=999*parent1["tx"].get_vsize(), + ) + return [parent1["hex"], parent2["hex"], child["hex"]], parent1["tx"], parent2["tx"], child["tx"] + + def create_packages(self): + # 1: Basic 1-parent-1-child package, parent 1sat/vB, child 999sat/vB + package_hex_1, parent_1, child_1 = self.create_basic_1p1c(self.wallet) + + # 2: same as 1, parent's txid is the same as its wtxid. + package_hex_2, parent_2, child_2 = self.create_basic_1p1c(self.wallet_nonsegwit) + + # 3: 2-parent-1-child package. Both parents are above mempool min feerate. No package submission happens. + # We require packages to be child-with-unconfirmed-parents and only allow 1-parent-1-child packages. + package_hex_3, parent_31, parent_32, child_3 = self.create_package_2p1c(self.wallet) + + # 4: parent + child package where the child spends 2 different outputs from the parent. + package_hex_4, parent_4, child_4 = self.create_package_2outs(self.wallet) + + # Assemble return results + packages_to_submit = [package_hex_1, package_hex_2, package_hex_3, package_hex_4] + # node0: sender + # node1: pre-received the children (orphan) + # node3: pre-received the parents (too low fee) + # All nodes receive parent_31 ahead of time. + txns_to_send = [ + [], + [child_1, child_2, parent_31, child_3, child_4], + [parent_31], + [parent_1, parent_2, parent_31, parent_4] + ] + + return packages_to_submit, txns_to_send + + def run_test(self): + self.wallet = MiniWallet(self.nodes[1]) + self.wallet_nonsegwit = MiniWallet(self.nodes[2], mode=MiniWalletMode.RAW_P2PK) + self.generate(self.wallet_nonsegwit, 10) + self.generate(self.wallet, 120) + + self.log.info("Fill mempools with large transactions to raise mempool minimum feerates") + self.raise_network_minfee() + + # Create the transactions. + self.wallet.rescan_utxos(include_mempool=True) + packages_to_submit, transactions_to_presend = self.create_packages() + + self.peers = [self.nodes[i].add_p2p_connection(P2PInterface()) for i in range(self.num_nodes)] + + self.log.info("Pre-send some transactions to nodes") + for (i, peer) in enumerate(self.peers): + for tx in transactions_to_presend[i]: + peer.send_and_ping(msg_tx(tx)) + # This disconnect removes any sent orphans from the orphanage (EraseForPeer) and times + # out the in-flight requests. It is currently required for the test to pass right now, + # because the node will not reconsider an orphan tx and will not (re)try requesting + # orphan parents from multiple peers if the first one didn't respond. + # TODO: remove this in the future if the node tries orphan resolution with multiple peers. + peer.peer_disconnect() + + self.log.info("Submit full packages to node0") + for package_hex in packages_to_submit: + submitpackage_result = self.nodes[0].submitpackage(package_hex) + assert_equal(submitpackage_result["package_msg"], "success") + + self.log.info("Wait for mempools to sync") + self.sync_mempools(timeout=20) + + +if __name__ == '__main__': + PackageRelayTest().main() diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index 0950579580..9e314db110 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -387,7 +387,7 @@ class CompactBlocksTest(BitcoinTestFramework): if announce == "inv": test_node.send_message(msg_inv([CInv(MSG_BLOCK, block.sha256)])) - test_node.wait_until(lambda: "getheaders" in test_node.last_message, timeout=30) + test_node.wait_for_getheaders(timeout=30) test_node.send_header_for_blocks([block]) else: test_node.send_header_for_blocks([block]) diff --git a/test/functional/p2p_disconnect_ban.py b/test/functional/p2p_disconnect_ban.py index c389ff732f..678b006886 100755 --- a/test/functional/p2p_disconnect_ban.py +++ b/test/functional/p2p_disconnect_ban.py @@ -77,6 +77,7 @@ class DisconnectBanTest(BitcoinTestFramework): self.nodes[1].setmocktime(old_time) self.nodes[1].setban("127.0.0.0/32", "add") self.nodes[1].setban("127.0.0.0/24", "add") + self.nodes[1].setban("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion", "add") self.nodes[1].setban("192.168.0.1", "add", 1) # ban for 1 seconds self.nodes[1].setban("2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/19", "add", 1000) # ban for 1000 seconds listBeforeShutdown = self.nodes[1].listbanned() @@ -85,13 +86,13 @@ class DisconnectBanTest(BitcoinTestFramework): self.log.info("setban: test banning with absolute timestamp") self.nodes[1].setban("192.168.0.2", "add", old_time + 120, True) - # Move time forward by 3 seconds so the third ban has expired + # Move time forward by 3 seconds so the fourth ban has expired self.nodes[1].setmocktime(old_time + 3) - assert_equal(len(self.nodes[1].listbanned()), 4) + assert_equal(len(self.nodes[1].listbanned()), 5) self.log.info("Test ban_duration and time_remaining") for ban in self.nodes[1].listbanned(): - if ban["address"] in ["127.0.0.0/32", "127.0.0.0/24"]: + if ban["address"] in ["127.0.0.0/32", "127.0.0.0/24", "pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion"]: assert_equal(ban["ban_duration"], 86400) assert_equal(ban["time_remaining"], 86397) elif ban["address"] == "2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/19": @@ -108,6 +109,7 @@ class DisconnectBanTest(BitcoinTestFramework): assert_equal("127.0.0.0/32", listAfterShutdown[1]['address']) assert_equal("192.168.0.2/32", listAfterShutdown[2]['address']) assert_equal("/19" in listAfterShutdown[3]['address'], True) + assert_equal("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion", listAfterShutdown[4]['address']) # Clear ban lists self.nodes[1].clearbanned() diff --git a/test/functional/p2p_handshake.py b/test/functional/p2p_handshake.py index f0b62e291d..dd19fe9333 100755 --- a/test/functional/p2p_handshake.py +++ b/test/functional/p2p_handshake.py @@ -41,6 +41,7 @@ class P2PHandshakeTest(BitcoinTestFramework): peer.sync_with_ping() peer.peer_disconnect() peer.wait_for_disconnect() + self.wait_until(lambda: len(node.getpeerinfo()) == 0) def test_desirable_service_flags(self, node, service_flag_tests, desirable_service_flags, expect_disconnect): """Check that connecting to a peer either fails or succeeds depending on its offered diff --git a/test/functional/p2p_initial_headers_sync.py b/test/functional/p2p_initial_headers_sync.py index e67c384da7..bc6e0fb355 100755 --- a/test/functional/p2p_initial_headers_sync.py +++ b/test/functional/p2p_initial_headers_sync.py @@ -38,9 +38,10 @@ class HeadersSyncTest(BitcoinTestFramework): def run_test(self): self.log.info("Adding a peer to node0") peer1 = self.nodes[0].add_p2p_connection(P2PInterface()) + best_block_hash = int(self.nodes[0].getbestblockhash(), 16) # Wait for peer1 to receive a getheaders - peer1.wait_for_getheaders() + peer1.wait_for_getheaders(block_hash=best_block_hash) # An empty reply will clear the outstanding getheaders request, # allowing additional getheaders requests to be sent to this peer in # the future. @@ -60,17 +61,12 @@ class HeadersSyncTest(BitcoinTestFramework): assert "getheaders" not in peer2.last_message assert "getheaders" not in peer3.last_message - with p2p_lock: - peer1.last_message.pop("getheaders", None) - self.log.info("Have all peers announce a new block") self.announce_random_block(all_peers) self.log.info("Check that peer1 receives a getheaders in response") - peer1.wait_for_getheaders() + peer1.wait_for_getheaders(block_hash=best_block_hash) peer1.send_message(msg_headers()) # Send empty response, see above - with p2p_lock: - peer1.last_message.pop("getheaders", None) self.log.info("Check that exactly 1 of {peer2, peer3} received a getheaders in response") count = 0 @@ -80,7 +76,6 @@ class HeadersSyncTest(BitcoinTestFramework): if "getheaders" in p.last_message: count += 1 peer_receiving_getheaders = p - p.last_message.pop("getheaders", None) p.send_message(msg_headers()) # Send empty response, see above assert_equal(count, 1) @@ -89,14 +84,14 @@ class HeadersSyncTest(BitcoinTestFramework): self.announce_random_block(all_peers) self.log.info("Check that peer1 receives a getheaders in response") - peer1.wait_for_getheaders() + peer1.wait_for_getheaders(block_hash=best_block_hash) self.log.info("Check that the remaining peer received a getheaders as well") expected_peer = peer2 if peer2 == peer_receiving_getheaders: expected_peer = peer3 - expected_peer.wait_for_getheaders() + expected_peer.wait_for_getheaders(block_hash=best_block_hash) self.log.info("Success!") diff --git a/test/functional/p2p_opportunistic_1p1c.py b/test/functional/p2p_opportunistic_1p1c.py new file mode 100755 index 0000000000..e07acd5481 --- /dev/null +++ b/test/functional/p2p_opportunistic_1p1c.py @@ -0,0 +1,414 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test opportunistic 1p1c package submission logic. +""" + +from decimal import Decimal +import time +from test_framework.messages import ( + CInv, + CTxInWitness, + MAX_BIP125_RBF_SEQUENCE, + MSG_WTX, + msg_inv, + msg_tx, + tx_from_hex, +) +from test_framework.p2p import ( + P2PInterface, +) +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + fill_mempool, +) +from test_framework.wallet import ( + MiniWallet, + MiniWalletMode, +) + +# 1sat/vB feerate denominated in BTC/KvB +FEERATE_1SAT_VB = Decimal("0.00001000") +# Number of seconds to wait to ensure no getdata is received +GETDATA_WAIT = 60 + +def cleanup(func): + def wrapper(self, *args, **kwargs): + try: + func(self, *args, **kwargs) + finally: + self.nodes[0].disconnect_p2ps() + # Do not clear the node's mempool, as each test requires mempool min feerate > min + # relay feerate. However, do check that this is the case. + assert self.nodes[0].getmempoolinfo()["mempoolminfee"] > self.nodes[0].getnetworkinfo()["relayfee"] + # Ensure we do not try to spend the same UTXOs in subsequent tests, as they will look like RBF attempts. + self.wallet.rescan_utxos(include_mempool=True) + + # Resets if mocktime was used + self.nodes[0].setmocktime(0) + return wrapper + +class PackageRelayTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + self.extra_args = [[ + "-datacarriersize=100000", + "-maxmempool=5", + ]] + self.supports_cli = False + + def create_tx_below_mempoolminfee(self, wallet): + """Create a 1-input 1sat/vB transaction using a confirmed UTXO. Decrement and use + self.sequence so that subsequent calls to this function result in unique transactions.""" + + self.sequence -= 1 + assert_greater_than(self.nodes[0].getmempoolinfo()["mempoolminfee"], FEERATE_1SAT_VB) + + return wallet.create_self_transfer(fee_rate=FEERATE_1SAT_VB, sequence=self.sequence, confirmed_only=True) + + @cleanup + def test_basic_child_then_parent(self): + node = self.nodes[0] + self.log.info("Check that opportunistic 1p1c logic works when child is received before parent") + + low_fee_parent = self.create_tx_below_mempoolminfee(self.wallet) + high_fee_child = self.wallet.create_self_transfer(utxo_to_spend=low_fee_parent["new_utxo"], fee_rate=20*FEERATE_1SAT_VB) + + peer_sender = node.add_p2p_connection(P2PInterface()) + + # 1. Child is received first (perhaps the low feerate parent didn't meet feefilter or the requests were sent to different nodes). It is missing an input. + high_child_wtxid_int = int(high_fee_child["tx"].getwtxid(), 16) + peer_sender.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=high_child_wtxid_int)])) + peer_sender.wait_for_getdata([high_child_wtxid_int]) + peer_sender.send_and_ping(msg_tx(high_fee_child["tx"])) + + # 2. Node requests the missing parent by txid. + parent_txid_int = int(low_fee_parent["txid"], 16) + peer_sender.wait_for_getdata([parent_txid_int]) + + # 3. Sender relays the parent. Parent+Child are evaluated as a package and accepted. + peer_sender.send_and_ping(msg_tx(low_fee_parent["tx"])) + + # 4. Both transactions should now be in mempool. + node_mempool = node.getrawmempool() + assert low_fee_parent["txid"] in node_mempool + assert high_fee_child["txid"] in node_mempool + + node.disconnect_p2ps() + + @cleanup + def test_basic_parent_then_child(self, wallet): + node = self.nodes[0] + low_fee_parent = self.create_tx_below_mempoolminfee(wallet) + high_fee_child = wallet.create_self_transfer(utxo_to_spend=low_fee_parent["new_utxo"], fee_rate=20*FEERATE_1SAT_VB) + + peer_sender = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=1, connection_type="outbound-full-relay") + peer_ignored = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=2, connection_type="outbound-full-relay") + + # 1. Parent is relayed first. It is too low feerate. + parent_wtxid_int = int(low_fee_parent["tx"].getwtxid(), 16) + peer_sender.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=parent_wtxid_int)])) + peer_sender.wait_for_getdata([parent_wtxid_int]) + peer_sender.send_and_ping(msg_tx(low_fee_parent["tx"])) + assert low_fee_parent["txid"] not in node.getrawmempool() + + # Send again from peer_ignored, check that it is ignored + peer_ignored.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=parent_wtxid_int)])) + assert "getdata" not in peer_ignored.last_message + + # 2. Child is relayed next. It is missing an input. + high_child_wtxid_int = int(high_fee_child["tx"].getwtxid(), 16) + peer_sender.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=high_child_wtxid_int)])) + peer_sender.wait_for_getdata([high_child_wtxid_int]) + peer_sender.send_and_ping(msg_tx(high_fee_child["tx"])) + + # 3. Node requests the missing parent by txid. + # It should do so even if it has previously rejected that parent for being too low feerate. + parent_txid_int = int(low_fee_parent["txid"], 16) + peer_sender.wait_for_getdata([parent_txid_int]) + + # 4. Sender re-relays the parent. Parent+Child are evaluated as a package and accepted. + peer_sender.send_and_ping(msg_tx(low_fee_parent["tx"])) + + # 5. Both transactions should now be in mempool. + node_mempool = node.getrawmempool() + assert low_fee_parent["txid"] in node_mempool + assert high_fee_child["txid"] in node_mempool + + @cleanup + def test_low_and_high_child(self, wallet): + node = self.nodes[0] + low_fee_parent = self.create_tx_below_mempoolminfee(wallet) + # This feerate is above mempoolminfee, but not enough to also bump the low feerate parent. + feerate_just_above = node.getmempoolinfo()["mempoolminfee"] + med_fee_child = wallet.create_self_transfer(utxo_to_spend=low_fee_parent["new_utxo"], fee_rate=feerate_just_above) + high_fee_child = wallet.create_self_transfer(utxo_to_spend=low_fee_parent["new_utxo"], fee_rate=999*FEERATE_1SAT_VB) + + peer_sender = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=1, connection_type="outbound-full-relay") + peer_ignored = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=2, connection_type="outbound-full-relay") + + self.log.info("Check that tx caches low fee parent + low fee child package rejections") + + # 1. Send parent, rejected for being low feerate. + parent_wtxid_int = int(low_fee_parent["tx"].getwtxid(), 16) + peer_sender.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=parent_wtxid_int)])) + peer_sender.wait_for_getdata([parent_wtxid_int]) + peer_sender.send_and_ping(msg_tx(low_fee_parent["tx"])) + assert low_fee_parent["txid"] not in node.getrawmempool() + + # Send again from peer_ignored, check that it is ignored + peer_ignored.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=parent_wtxid_int)])) + assert "getdata" not in peer_ignored.last_message + + # 2. Send an (orphan) child that has a higher feerate, but not enough to bump the parent. + med_child_wtxid_int = int(med_fee_child["tx"].getwtxid(), 16) + peer_sender.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=med_child_wtxid_int)])) + peer_sender.wait_for_getdata([med_child_wtxid_int]) + peer_sender.send_and_ping(msg_tx(med_fee_child["tx"])) + + # 3. Node requests the orphan's missing parent. + parent_txid_int = int(low_fee_parent["txid"], 16) + peer_sender.wait_for_getdata([parent_txid_int]) + + # 4. The low parent + low child are submitted as a package. They are not accepted due to low package feerate. + peer_sender.send_and_ping(msg_tx(low_fee_parent["tx"])) + + assert low_fee_parent["txid"] not in node.getrawmempool() + assert med_fee_child["txid"] not in node.getrawmempool() + + # If peer_ignored announces the low feerate child, it should be ignored + peer_ignored.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=med_child_wtxid_int)])) + assert "getdata" not in peer_ignored.last_message + # If either peer sends the parent again, package evaluation should not be attempted + peer_sender.send_and_ping(msg_tx(low_fee_parent["tx"])) + peer_ignored.send_and_ping(msg_tx(low_fee_parent["tx"])) + + assert low_fee_parent["txid"] not in node.getrawmempool() + assert med_fee_child["txid"] not in node.getrawmempool() + + # 5. Send the high feerate (orphan) child + high_child_wtxid_int = int(high_fee_child["tx"].getwtxid(), 16) + peer_sender.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=high_child_wtxid_int)])) + peer_sender.wait_for_getdata([high_child_wtxid_int]) + peer_sender.send_and_ping(msg_tx(high_fee_child["tx"])) + + # 6. Node requests the orphan's parent, even though it has already been rejected, both by + # itself and with a child. This is necessary, otherwise high_fee_child can be censored. + parent_txid_int = int(low_fee_parent["txid"], 16) + peer_sender.wait_for_getdata([parent_txid_int]) + + # 7. The low feerate parent + high feerate child are submitted as a package. + peer_sender.send_and_ping(msg_tx(low_fee_parent["tx"])) + + # 8. Both transactions should now be in mempool + node_mempool = node.getrawmempool() + assert low_fee_parent["txid"] in node_mempool + assert high_fee_child["txid"] in node_mempool + assert med_fee_child["txid"] not in node_mempool + + @cleanup + def test_orphan_consensus_failure(self): + self.log.info("Check opportunistic 1p1c logic with consensus-invalid orphan causes disconnect of the correct peer") + node = self.nodes[0] + low_fee_parent = self.create_tx_below_mempoolminfee(self.wallet) + coin = low_fee_parent["new_utxo"] + address = node.get_deterministic_priv_key().address + # Create raw transaction spending the parent, but with no signature (a consensus error). + hex_orphan_no_sig = node.createrawtransaction([{"txid": coin["txid"], "vout": coin["vout"]}], {address : coin["value"] - Decimal("0.0001")}) + tx_orphan_bad_wit = tx_from_hex(hex_orphan_no_sig) + tx_orphan_bad_wit.wit.vtxinwit.append(CTxInWitness()) + tx_orphan_bad_wit.wit.vtxinwit[0].scriptWitness.stack = [b'garbage'] + + bad_orphan_sender = node.add_p2p_connection(P2PInterface()) + parent_sender = node.add_p2p_connection(P2PInterface()) + + # 1. Child is received first. It is missing an input. + child_wtxid_int = int(tx_orphan_bad_wit.getwtxid(), 16) + bad_orphan_sender.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=child_wtxid_int)])) + bad_orphan_sender.wait_for_getdata([child_wtxid_int]) + bad_orphan_sender.send_and_ping(msg_tx(tx_orphan_bad_wit)) + + # 2. Node requests the missing parent by txid. + parent_txid_int = int(low_fee_parent["txid"], 16) + bad_orphan_sender.wait_for_getdata([parent_txid_int]) + + # 3. A different peer relays the parent. Parent+Child are evaluated as a package and rejected. + parent_sender.send_message(msg_tx(low_fee_parent["tx"])) + + # 4. Transactions should not be in mempool. + node_mempool = node.getrawmempool() + assert low_fee_parent["txid"] not in node_mempool + assert tx_orphan_bad_wit.rehash() not in node_mempool + + # 5. Peer that sent a consensus-invalid transaction should be disconnected. + bad_orphan_sender.wait_for_disconnect() + + # The peer that didn't provide the orphan should not be disconnected. + parent_sender.sync_with_ping() + + @cleanup + def test_parent_consensus_failure(self): + self.log.info("Check opportunistic 1p1c logic with consensus-invalid parent causes disconnect of the correct peer") + node = self.nodes[0] + low_fee_parent = self.create_tx_below_mempoolminfee(self.wallet) + high_fee_child = self.wallet.create_self_transfer(utxo_to_spend=low_fee_parent["new_utxo"], fee_rate=999*FEERATE_1SAT_VB) + + # Create invalid version of parent with a bad signature. + tx_parent_bad_wit = tx_from_hex(low_fee_parent["hex"]) + tx_parent_bad_wit.wit.vtxinwit.append(CTxInWitness()) + tx_parent_bad_wit.wit.vtxinwit[0].scriptWitness.stack = [b'garbage'] + + package_sender = node.add_p2p_connection(P2PInterface()) + fake_parent_sender = node.add_p2p_connection(P2PInterface()) + + # 1. Child is received first. It is missing an input. + child_wtxid_int = int(high_fee_child["tx"].getwtxid(), 16) + package_sender.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=child_wtxid_int)])) + package_sender.wait_for_getdata([child_wtxid_int]) + package_sender.send_and_ping(msg_tx(high_fee_child["tx"])) + + # 2. Node requests the missing parent by txid. + parent_txid_int = int(tx_parent_bad_wit.rehash(), 16) + package_sender.wait_for_getdata([parent_txid_int]) + + # 3. A different node relays the parent. The parent is first evaluated by itself and + # rejected for being too low feerate. Then it is evaluated as a package and, after passing + # feerate checks, rejected for having a bad signature (consensus error). + fake_parent_sender.send_message(msg_tx(tx_parent_bad_wit)) + + # 4. Transactions should not be in mempool. + node_mempool = node.getrawmempool() + assert tx_parent_bad_wit.rehash() not in node_mempool + assert high_fee_child["txid"] not in node_mempool + + # 5. Peer sent a consensus-invalid transaction. + fake_parent_sender.wait_for_disconnect() + + self.log.info("Check that fake parent does not cause orphan to be deleted and real package can still be submitted") + # 6. Child-sending should not have been punished and the orphan should remain in orphanage. + # It can send the "real" parent transaction, and the package is accepted. + parent_wtxid_int = int(low_fee_parent["tx"].getwtxid(), 16) + package_sender.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=parent_wtxid_int)])) + package_sender.wait_for_getdata([parent_wtxid_int]) + package_sender.send_and_ping(msg_tx(low_fee_parent["tx"])) + + node_mempool = node.getrawmempool() + assert low_fee_parent["txid"] in node_mempool + assert high_fee_child["txid"] in node_mempool + + @cleanup + def test_multiple_parents(self): + self.log.info("Check that node does not request more than 1 previously-rejected low feerate parent") + + node = self.nodes[0] + node.setmocktime(int(time.time())) + + # 2-parent-1-child package where both parents are below mempool min feerate + parent_low_1 = self.create_tx_below_mempoolminfee(self.wallet_nonsegwit) + parent_low_2 = self.create_tx_below_mempoolminfee(self.wallet_nonsegwit) + child_bumping = self.wallet_nonsegwit.create_self_transfer_multi( + utxos_to_spend=[parent_low_1["new_utxo"], parent_low_2["new_utxo"]], + fee_per_output=999*parent_low_1["tx"].get_vsize(), + ) + + peer_sender = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=1, connection_type="outbound-full-relay") + + # 1. Send both parents. Each should be rejected for being too low feerate. + # Send unsolicited so that we can later check that no "getdata" was ever received. + peer_sender.send_and_ping(msg_tx(parent_low_1["tx"])) + peer_sender.send_and_ping(msg_tx(parent_low_2["tx"])) + + # parent_low_1 and parent_low_2 are rejected for being low feerate. + assert parent_low_1["txid"] not in node.getrawmempool() + assert parent_low_2["txid"] not in node.getrawmempool() + + # 2. Send child. + peer_sender.send_and_ping(msg_tx(child_bumping["tx"])) + + # 3. Node should not request any parents, as it should recognize that it will not accept + # multi-parent-1-child packages. + node.bumpmocktime(GETDATA_WAIT) + peer_sender.sync_with_ping() + assert "getdata" not in peer_sender.last_message + + @cleanup + def test_other_parent_in_mempool(self): + self.log.info("Check opportunistic 1p1c fails if child already has another parent in mempool") + node = self.nodes[0] + + # This parent needs CPFP + parent_low = self.create_tx_below_mempoolminfee(self.wallet) + # This parent does not need CPFP and can be submitted alone ahead of time + parent_high = self.wallet.create_self_transfer(fee_rate=FEERATE_1SAT_VB*10, confirmed_only=True) + child = self.wallet.create_self_transfer_multi( + utxos_to_spend=[parent_high["new_utxo"], parent_low["new_utxo"]], + fee_per_output=999*parent_low["tx"].get_vsize(), + ) + + peer_sender = node.add_outbound_p2p_connection(P2PInterface(), p2p_idx=1, connection_type="outbound-full-relay") + + # 1. Send first parent which will be accepted. + peer_sender.send_and_ping(msg_tx(parent_high["tx"])) + assert parent_high["txid"] in node.getrawmempool() + + # 2. Send child. + peer_sender.send_and_ping(msg_tx(child["tx"])) + + # 3. Node requests parent_low. However, 1p1c fails because package-not-child-with-unconfirmed-parents + parent_low_txid_int = int(parent_low["txid"], 16) + peer_sender.wait_for_getdata([parent_low_txid_int]) + peer_sender.send_and_ping(msg_tx(parent_low["tx"])) + + node_mempool = node.getrawmempool() + assert parent_high["txid"] in node_mempool + assert parent_low["txid"] not in node_mempool + assert child["txid"] not in node_mempool + + # Same error if submitted through submitpackage without parent_high + package_hex_missing_parent = [parent_low["hex"], child["hex"]] + result_missing_parent = node.submitpackage(package_hex_missing_parent) + assert_equal(result_missing_parent["package_msg"], "package-not-child-with-unconfirmed-parents") + + def run_test(self): + node = self.nodes[0] + # To avoid creating transactions with the same txid (can happen if we set the same feerate + # and reuse the same input as a previous transaction that wasn't successfully submitted), + # we give each subtest a different nSequence for its transactions. + self.sequence = MAX_BIP125_RBF_SEQUENCE + + self.wallet = MiniWallet(node) + self.wallet_nonsegwit = MiniWallet(node, mode=MiniWalletMode.RAW_P2PK) + self.generate(self.wallet_nonsegwit, 10) + self.generate(self.wallet, 20) + + filler_wallet = MiniWallet(node) + fill_mempool(self, node, filler_wallet) + + self.log.info("Check opportunistic 1p1c logic when parent (txid != wtxid) is received before child") + self.test_basic_parent_then_child(self.wallet) + + self.log.info("Check opportunistic 1p1c logic when parent (txid == wtxid) is received before child") + self.test_basic_parent_then_child(self.wallet_nonsegwit) + + self.log.info("Check opportunistic 1p1c logic when child is received before parent") + self.test_basic_child_then_parent() + + self.log.info("Check opportunistic 1p1c logic when 2 candidate children exist (parent txid != wtxid)") + self.test_low_and_high_child(self.wallet) + + self.log.info("Check opportunistic 1p1c logic when 2 candidate children exist (parent txid == wtxid)") + self.test_low_and_high_child(self.wallet_nonsegwit) + + self.test_orphan_consensus_failure() + self.test_parent_consensus_failure() + self.test_multiple_parents() + self.test_other_parent_in_mempool() + + +if __name__ == '__main__': + PackageRelayTest().main() diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index af47c6d9f0..45bbd7f1c3 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -191,14 +191,13 @@ class TestP2PConn(P2PInterface): def announce_block_and_wait_for_getdata(self, block, use_header, timeout=60): with p2p_lock: self.last_message.pop("getdata", None) - self.last_message.pop("getheaders", None) msg = msg_headers() msg.headers = [CBlockHeader(block)] if use_header: self.send_message(msg) else: self.send_message(msg_inv(inv=[CInv(MSG_BLOCK, block.sha256)])) - self.wait_for_getheaders(timeout=timeout) + self.wait_for_getheaders(block_hash=block.hashPrevBlock, timeout=timeout) self.send_message(msg) self.wait_for_getdata([block.sha256], timeout=timeout) @@ -1055,7 +1054,7 @@ class SegWitTest(BitcoinTestFramework): @subtest def test_max_witness_push_length(self): - """Test that witness stack can only allow up to 520 byte pushes.""" + """Test that witness stack can only allow up to MAX_SCRIPT_ELEMENT_SIZE byte pushes.""" block = self.build_next_block() diff --git a/test/functional/p2p_sendheaders.py b/test/functional/p2p_sendheaders.py index 508d6fe403..27a3aa8fb9 100755 --- a/test/functional/p2p_sendheaders.py +++ b/test/functional/p2p_sendheaders.py @@ -311,6 +311,7 @@ class SendHeadersTest(BitcoinTestFramework): # Now that we've synced headers, headers announcements should work tip = self.mine_blocks(1) + expected_hash = tip inv_node.check_last_inv_announcement(inv=[tip]) test_node.check_last_headers_announcement(headers=[tip]) @@ -334,7 +335,10 @@ class SendHeadersTest(BitcoinTestFramework): if j == 0: # Announce via inv test_node.send_block_inv(tip) - test_node.wait_for_getheaders() + if i == 0: + test_node.wait_for_getheaders(block_hash=expected_hash) + else: + assert "getheaders" not in test_node.last_message # Should have received a getheaders now test_node.send_header_for_blocks(blocks) # Test that duplicate inv's won't result in duplicate @@ -521,6 +525,7 @@ class SendHeadersTest(BitcoinTestFramework): self.log.info("Part 5: Testing handling of unconnecting headers") # First we test that receipt of an unconnecting header doesn't prevent # chain sync. + expected_hash = tip for i in range(10): self.log.debug("Part 5.{}: starting...".format(i)) test_node.last_message.pop("getdata", None) @@ -533,15 +538,14 @@ class SendHeadersTest(BitcoinTestFramework): block_time += 1 height += 1 # Send the header of the second block -> this won't connect. - with p2p_lock: - test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[1]]) - test_node.wait_for_getheaders() + test_node.wait_for_getheaders(block_hash=expected_hash) test_node.send_header_for_blocks(blocks) test_node.wait_for_getdata([x.sha256 for x in blocks]) [test_node.send_message(msg_block(x)) for x in blocks] test_node.sync_with_ping() assert_equal(int(self.nodes[0].getbestblockhash(), 16), blocks[1].sha256) + expected_hash = blocks[1].sha256 blocks = [] # Now we test that if we repeatedly don't send connecting headers, we @@ -556,13 +560,12 @@ class SendHeadersTest(BitcoinTestFramework): for i in range(1, MAX_NUM_UNCONNECTING_HEADERS_MSGS): # Send a header that doesn't connect, check that we get a getheaders. - with p2p_lock: - test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[i]]) - test_node.wait_for_getheaders() + test_node.wait_for_getheaders(block_hash=expected_hash) # Next header will connect, should re-set our count: test_node.send_header_for_blocks([blocks[0]]) + expected_hash = blocks[0].sha256 # Remove the first two entries (blocks[1] would connect): blocks = blocks[2:] @@ -571,10 +574,8 @@ class SendHeadersTest(BitcoinTestFramework): # before we get disconnected. Should be 5*MAX_NUM_UNCONNECTING_HEADERS_MSGS for i in range(5 * MAX_NUM_UNCONNECTING_HEADERS_MSGS - 1): # Send a header that doesn't connect, check that we get a getheaders. - with p2p_lock: - test_node.last_message.pop("getheaders", None) test_node.send_header_for_blocks([blocks[i % len(blocks)]]) - test_node.wait_for_getheaders() + test_node.wait_for_getheaders(block_hash=expected_hash) # Eventually this stops working. test_node.send_header_for_blocks([blocks[-1]]) diff --git a/test/functional/p2p_tx_download.py b/test/functional/p2p_tx_download.py index 0e463c5072..7a50f1e605 100755 --- a/test/functional/p2p_tx_download.py +++ b/test/functional/p2p_tx_download.py @@ -5,6 +5,7 @@ """ Test transaction download behavior """ +from decimal import Decimal import time from test_framework.messages import ( @@ -14,6 +15,7 @@ from test_framework.messages import ( MSG_WTX, msg_inv, msg_notfound, + msg_tx, ) from test_framework.p2p import ( P2PInterface, @@ -22,6 +24,7 @@ from test_framework.p2p import ( from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, + fill_mempool, ) from test_framework.wallet import MiniWallet @@ -54,6 +57,7 @@ MAX_GETDATA_INBOUND_WAIT = GETDATA_TX_INTERVAL + INBOUND_PEER_TX_DELAY + TXID_RE class TxDownloadTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + self.extra_args= [['-datacarriersize=100000', '-maxmempool=5', '-persistmempool=0']] * self.num_nodes def test_tx_requests(self): self.log.info("Test that we request transactions from all our peers, eventually") @@ -241,6 +245,29 @@ class TxDownloadTest(BitcoinTestFramework): self.log.info('Check that spurious notfound is ignored') self.nodes[0].p2ps[0].send_message(msg_notfound(vec=[CInv(MSG_TX, 1)])) + def test_rejects_filter_reset(self): + self.log.info('Check that rejected tx is not requested again') + node = self.nodes[0] + fill_mempool(self, node, self.wallet) + self.wallet.rescan_utxos() + mempoolminfee = node.getmempoolinfo()['mempoolminfee'] + peer = node.add_p2p_connection(TestP2PConn()) + low_fee_tx = self.wallet.create_self_transfer(fee_rate=Decimal("0.9")*mempoolminfee) + assert_equal(node.testmempoolaccept([low_fee_tx['hex']])[0]["reject-reason"], "mempool min fee not met") + peer.send_and_ping(msg_tx(low_fee_tx['tx'])) + peer.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=int(low_fee_tx['wtxid'], 16))])) + node.setmocktime(int(time.time())) + node.bumpmocktime(MAX_GETDATA_INBOUND_WAIT) + peer.sync_with_ping() + assert_equal(peer.tx_getdata_count, 0) + + self.log.info('Check that rejection filter is cleared after new block comes in') + self.generate(self.wallet, 1, sync_fun=self.no_op) + peer.sync_with_ping() + peer.send_and_ping(msg_inv([CInv(t=MSG_WTX, h=int(low_fee_tx['wtxid'], 16))])) + node.bumpmocktime(MAX_GETDATA_INBOUND_WAIT) + peer.wait_for_getdata([int(low_fee_tx['wtxid'], 16)]) + def run_test(self): self.wallet = MiniWallet(self.nodes[0]) @@ -257,16 +284,22 @@ class TxDownloadTest(BitcoinTestFramework): # Run each test against new bitcoind instances, as setting mocktimes has long-term effects on when # the next trickle relay event happens. - for test in [self.test_in_flight_max, self.test_inv_block, self.test_tx_requests]: + for test, with_inbounds in [ + (self.test_in_flight_max, True), + (self.test_inv_block, True), + (self.test_tx_requests, True), + (self.test_rejects_filter_reset, False), + ]: self.stop_nodes() self.start_nodes() self.connect_nodes(1, 0) # Setup the p2p connections self.peers = [] - for node in self.nodes: - for _ in range(NUM_INBOUND): - self.peers.append(node.add_p2p_connection(TestP2PConn())) - self.log.info("Nodes are setup with {} incoming connections each".format(NUM_INBOUND)) + if with_inbounds: + for node in self.nodes: + for _ in range(NUM_INBOUND): + self.peers.append(node.add_p2p_connection(TestP2PConn())) + self.log.info("Nodes are setup with {} incoming connections each".format(NUM_INBOUND)) test() diff --git a/test/functional/rpc_packages.py b/test/functional/rpc_packages.py index e061030da3..37c42f2533 100755 --- a/test/functional/rpc_packages.py +++ b/test/functional/rpc_packages.py @@ -386,12 +386,14 @@ class RPCPackagesTest(BitcoinTestFramework): "-maxmempool=5", "-persistmempool=0", ]) + self.wallet.rescan_utxos() fill_mempool(self, node, self.wallet) minrelay = node.getmempoolinfo()["minrelaytxfee"] parent = self.wallet.create_self_transfer( fee_rate=minrelay, + confirmed_only=True, ) child = self.wallet.create_self_transfer( @@ -411,6 +413,7 @@ class RPCPackagesTest(BitcoinTestFramework): # Reset maxmempool, datacarriersize, reset dynamic mempool minimum feerate, and empty mempool. self.restart_node(0) + self.wallet.rescan_utxos() assert_equal(node.getrawmempool(), []) @@ -420,7 +423,10 @@ class RPCPackagesTest(BitcoinTestFramework): assert_equal(node.getrawmempool(), []) self.log.info("Submitpackage maxburnamount arg testing") - chained_txns_burn = self.wallet.create_self_transfer_chain(chain_length=2) + chained_txns_burn = self.wallet.create_self_transfer_chain( + chain_length=2, + utxo_to_spend=self.wallet.get_utxo(confirmed_only=True), + ) chained_burn_hex = [t["hex"] for t in chained_txns_burn] tx = tx_from_hex(chained_burn_hex[1]) diff --git a/test/functional/rpc_psbt.py b/test/functional/rpc_psbt.py index 016aa3ba11..6ee7e56886 100755 --- a/test/functional/rpc_psbt.py +++ b/test/functional/rpc_psbt.py @@ -16,8 +16,6 @@ from test_framework.messages import ( CTxIn, CTxOut, MAX_BIP125_RBF_SEQUENCE, - WITNESS_SCALE_FACTOR, - ser_compact_size, ) from test_framework.psbt import ( PSBT, @@ -42,6 +40,7 @@ from test_framework.util import ( find_vout_for_address, ) from test_framework.wallet_util import ( + calculate_input_weight, generate_keypair, get_generate_key, ) @@ -752,17 +751,9 @@ class PSBTTest(BitcoinTestFramework): input_idx = i break psbt_in = dec["inputs"][input_idx] - # Calculate the input weight - # (prevout + sequence + length of scriptSig + scriptsig) * WITNESS_SCALE_FACTOR + len of num scriptWitness stack items + (length of stack item + stack item) * N stack items - # Note that occasionally this weight estimate may be slightly larger or smaller than the real weight - # as sometimes ECDSA signatures are one byte shorter than expected with a probability of 1/128 - len_scriptsig = len(psbt_in["final_scriptSig"]["hex"]) // 2 if "final_scriptSig" in psbt_in else 0 - len_scriptsig += len(ser_compact_size(len_scriptsig)) - len_scriptwitness = (sum([(len(x) // 2) + len(ser_compact_size(len(x) // 2)) for x in psbt_in["final_scriptwitness"]]) + len(ser_compact_size(len(psbt_in["final_scriptwitness"])))) if "final_scriptwitness" in psbt_in else 0 - len_prevout_txid = 32 - len_prevout_index = 4 - len_sequence = 4 - input_weight = ((len_prevout_txid + len_prevout_index + len_sequence + len_scriptsig) * WITNESS_SCALE_FACTOR) + len_scriptwitness + scriptsig_hex = psbt_in["final_scriptSig"]["hex"] if "final_scriptSig" in psbt_in else "" + witness_stack_hex = psbt_in["final_scriptwitness"] if "final_scriptwitness" in psbt_in else None + input_weight = calculate_input_weight(scriptsig_hex, witness_stack_hex) low_input_weight = input_weight // 2 high_input_weight = input_weight * 2 @@ -886,7 +877,7 @@ class PSBTTest(BitcoinTestFramework): assert_equal(comb_psbt, psbt) self.log.info("Test walletprocesspsbt raises if an invalid sighashtype is passed") - assert_raises_rpc_error(-8, "all is not a valid sighash parameter.", self.nodes[0].walletprocesspsbt, psbt, sighashtype="all") + assert_raises_rpc_error(-8, "'all' is not a valid sighash parameter.", self.nodes[0].walletprocesspsbt, psbt, sighashtype="all") self.log.info("Test decoding PSBT with per-input preimage types") # note that the decodepsbt RPC doesn't check whether preimages and hashes match @@ -992,7 +983,7 @@ class PSBTTest(BitcoinTestFramework): self.nodes[2].sendrawtransaction(processed_psbt['hex']) self.log.info("Test descriptorprocesspsbt raises if an invalid sighashtype is passed") - assert_raises_rpc_error(-8, "all is not a valid sighash parameter.", self.nodes[2].descriptorprocesspsbt, psbt, [descriptor], sighashtype="all") + assert_raises_rpc_error(-8, "'all' is not a valid sighash parameter.", self.nodes[2].descriptorprocesspsbt, psbt, [descriptor], sighashtype="all") if __name__ == '__main__': diff --git a/test/functional/rpc_setban.py b/test/functional/rpc_setban.py index bc426d7371..ba86b278bd 100755 --- a/test/functional/rpc_setban.py +++ b/test/functional/rpc_setban.py @@ -64,20 +64,10 @@ class SetBanTests(BitcoinTestFramework): assert self.is_banned(node, tor_addr) assert not self.is_banned(node, ip_addr) - self.log.info("Test the ban list is preserved through restart") - - self.restart_node(1) - assert self.is_banned(node, tor_addr) - assert not self.is_banned(node, ip_addr) - node.setban(tor_addr, "remove") assert not self.is_banned(self.nodes[1], tor_addr) assert not self.is_banned(node, ip_addr) - self.restart_node(1) - assert not self.is_banned(node, tor_addr) - assert not self.is_banned(node, ip_addr) - self.log.info("Test -bantime") self.restart_node(1, ["-bantime=1234"]) self.nodes[1].setban("127.0.0.1", "add") diff --git a/test/functional/rpc_signrawtransactionwithkey.py b/test/functional/rpc_signrawtransactionwithkey.py index 0913f5057e..268584331e 100755 --- a/test/functional/rpc_signrawtransactionwithkey.py +++ b/test/functional/rpc_signrawtransactionwithkey.py @@ -124,7 +124,7 @@ class SignRawTransactionWithKeyTest(BitcoinTestFramework): self.log.info("Test signing transaction with invalid sighashtype") tx = self.nodes[0].createrawtransaction(INPUTS, OUTPUTS) privkeys = [self.nodes[0].get_deterministic_priv_key().key] - assert_raises_rpc_error(-8, "all is not a valid sighash parameter.", self.nodes[0].signrawtransactionwithkey, tx, privkeys, sighashtype="all") + assert_raises_rpc_error(-8, "'all' is not a valid sighash parameter.", self.nodes[0].signrawtransactionwithkey, tx, privkeys, sighashtype="all") def run_test(self): self.successful_signing_test() diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index cfd923bab3..f0dc866f69 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -28,6 +28,7 @@ from .messages import ( ser_uint256, tx_from_hex, uint256_from_str, + WITNESS_SCALE_FACTOR, ) from .script import ( CScript, @@ -45,7 +46,6 @@ from .script_util import ( ) from .util import assert_equal -WITNESS_SCALE_FACTOR = 4 MAX_BLOCK_SIGOPS = 20000 MAX_BLOCK_SIGOPS_WEIGHT = MAX_BLOCK_SIGOPS * WITNESS_SCALE_FACTOR diff --git a/test/functional/test_framework/p2p.py b/test/functional/test_framework/p2p.py index ce76008c46..00bd1e4017 100755 --- a/test/functional/test_framework/p2p.py +++ b/test/functional/test_framework/p2p.py @@ -644,15 +644,17 @@ class P2PInterface(P2PConnection): self.wait_until(test_function, timeout=timeout) - def wait_for_getheaders(self, *, timeout=60): - """Waits for a getheaders message. + def wait_for_getheaders(self, block_hash=None, *, timeout=60): + """Waits for a getheaders message containing a specific block hash. - Receiving any getheaders message will satisfy the predicate. the last_message["getheaders"] - value must be explicitly cleared before calling this method, or this will return - immediately with success. TODO: change this method to take a hash value and only - return true if the correct block header has been requested.""" + If no block hash is provided, checks whether any getheaders message has been received by the node.""" def test_function(): - return self.last_message.get("getheaders") + last_getheaders = self.last_message.pop("getheaders", None) + if block_hash is None: + return last_getheaders + if last_getheaders is None: + return False + return block_hash == last_getheaders.locator.vHave[0] self.wait_until(test_function, timeout=timeout) diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 3275517888..7b19d31e17 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -483,7 +483,7 @@ class CScript(bytes): i = 0 while i < len(self): sop_idx = i - opcode = self[i] + opcode = CScriptOp(self[i]) i += 1 if opcode > OP_PUSHDATA4: @@ -590,7 +590,7 @@ class CScript(bytes): n += 1 elif opcode in (OP_CHECKMULTISIG, OP_CHECKMULTISIGVERIFY): if fAccurate and (OP_1 <= lastOpcode <= OP_16): - n += opcode.decode_op_n() + n += lastOpcode.decode_op_n() else: n += 20 lastOpcode = opcode @@ -782,6 +782,20 @@ class TestFrameworkScript(unittest.TestCase): for value in values: self.assertEqual(CScriptNum.decode(CScriptNum.encode(CScriptNum(value))), value) + def test_legacy_sigopcount(self): + # test repeated single sig ops + for n_ops in range(1, 100, 10): + for singlesig_op in (OP_CHECKSIG, OP_CHECKSIGVERIFY): + singlesigs_script = CScript([singlesig_op]*n_ops) + self.assertEqual(singlesigs_script.GetSigOpCount(fAccurate=False), n_ops) + self.assertEqual(singlesigs_script.GetSigOpCount(fAccurate=True), n_ops) + # test multisig op (including accurate counting, i.e. BIP16) + for n in range(1, 16+1): + for multisig_op in (OP_CHECKMULTISIG, OP_CHECKMULTISIGVERIFY): + multisig_script = CScript([CScriptOp.encode_op_n(n), multisig_op]) + self.assertEqual(multisig_script.GetSigOpCount(fAccurate=False), 20) + self.assertEqual(multisig_script.GetSigOpCount(fAccurate=True), n) + def BIP341_sha_prevouts(txTo): return sha256(b"".join(i.prevout.serialize() for i in txTo.vin)) diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index dbaf42fdaa..0de09b6440 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -505,6 +505,8 @@ def fill_mempool(test_framework, node, miniwallet): It will not ensure mempools become synced as it is based on a single node and assumes -minrelaytxfee is 1 sat/vbyte. + To avoid unintentional tx dependencies, it is recommended to use separate miniwallets for + mempool filling vs transactions in tests. """ test_framework.log.info("Fill the mempool until eviction is triggered and the mempoolminfee rises") txouts = gen_return_txouts() @@ -522,8 +524,14 @@ def fill_mempool(test_framework, node, miniwallet): # Mine COINBASE_MATURITY - 1 blocks so that the UTXOs are allowed to be spent test_framework.generate(node, 100 - 1) + # Get all UTXOs up front to ensure none of the transactions spend from each other, as that may + # change their effective feerate and thus the order in which they are selected for eviction. + confirmed_utxos = [miniwallet.get_utxo(confirmed_only=True) for _ in range(num_of_batches * tx_batch_size + 1)] + assert_equal(len(confirmed_utxos), num_of_batches * tx_batch_size + 1) + test_framework.log.debug("Create a mempool tx that will be evicted") - tx_to_be_evicted_id = miniwallet.send_self_transfer(from_node=node, fee_rate=relayfee)["txid"] + tx_to_be_evicted_id = miniwallet.send_self_transfer(from_node=node, utxo_to_spend=confirmed_utxos[0], fee_rate=relayfee)["txid"] + del confirmed_utxos[0] # Increase the tx fee rate to give the subsequent transactions a higher priority in the mempool # The tx has an approx. vsize of 65k, i.e. multiplying the previous fee rate (in sats/kvB) @@ -534,7 +542,9 @@ def fill_mempool(test_framework, node, miniwallet): with node.assert_debug_log(["rolling minimum fee bumped"]): for batch_of_txid in range(num_of_batches): fee = (batch_of_txid + 1) * base_fee - create_lots_of_big_transactions(miniwallet, node, fee, tx_batch_size, txouts) + utxos = confirmed_utxos[:tx_batch_size] + create_lots_of_big_transactions(miniwallet, node, fee, tx_batch_size, txouts, utxos) + del confirmed_utxos[:tx_batch_size] test_framework.log.debug("The tx should be evicted by now") # The number of transactions created should be greater than the ones present in the mempool diff --git a/test/functional/test_framework/wallet_util.py b/test/functional/test_framework/wallet_util.py index 44811918bf..2168e607b2 100755 --- a/test/functional/test_framework/wallet_util.py +++ b/test/functional/test_framework/wallet_util.py @@ -4,6 +4,7 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Useful util functions for testing the wallet""" from collections import namedtuple +import unittest from test_framework.address import ( byte_to_base58, @@ -15,6 +16,11 @@ from test_framework.address import ( script_to_p2wsh, ) from test_framework.key import ECKey +from test_framework.messages import ( + CTxIn, + CTxInWitness, + WITNESS_SCALE_FACTOR, +) from test_framework.script_util import ( key_to_p2pkh_script, key_to_p2wpkh_script, @@ -123,6 +129,19 @@ def generate_keypair(compressed=True, wif=False): privkey = bytes_to_wif(privkey.get_bytes(), compressed) return privkey, pubkey +def calculate_input_weight(scriptsig_hex, witness_stack_hex=None): + """Given a scriptSig and a list of witness stack items for an input in hex format, + calculate the total input weight. If the input has no witness data, + `witness_stack_hex` can be set to None.""" + tx_in = CTxIn(scriptSig=bytes.fromhex(scriptsig_hex)) + witness_size = 0 + if witness_stack_hex is not None: + tx_inwit = CTxInWitness() + for witness_item_hex in witness_stack_hex: + tx_inwit.scriptWitness.stack.append(bytes.fromhex(witness_item_hex)) + witness_size = len(tx_inwit.serialize()) + return len(tx_in.serialize()) * WITNESS_SCALE_FACTOR + witness_size + class WalletUnlock(): """ A context manager for unlocking a wallet with a passphrase and automatically locking it afterward. @@ -141,3 +160,42 @@ class WalletUnlock(): def __exit__(self, *args): _ = args self.wallet.walletlock() + + +class TestFrameworkWalletUtil(unittest.TestCase): + def test_calculate_input_weight(self): + SKELETON_BYTES = 32 + 4 + 4 # prevout-txid, prevout-index, sequence + SMALL_LEN_BYTES = 1 # bytes needed for encoding scriptSig / witness item lengths < 253 + LARGE_LEN_BYTES = 3 # bytes needed for encoding scriptSig / witness item lengths >= 253 + + # empty scriptSig, no witness + self.assertEqual(calculate_input_weight(""), + (SKELETON_BYTES + SMALL_LEN_BYTES) * WITNESS_SCALE_FACTOR) + self.assertEqual(calculate_input_weight("", None), + (SKELETON_BYTES + SMALL_LEN_BYTES) * WITNESS_SCALE_FACTOR) + # small scriptSig, no witness + scriptSig_small = "00"*252 + self.assertEqual(calculate_input_weight(scriptSig_small, None), + (SKELETON_BYTES + SMALL_LEN_BYTES + 252) * WITNESS_SCALE_FACTOR) + # small scriptSig, empty witness stack + self.assertEqual(calculate_input_weight(scriptSig_small, []), + (SKELETON_BYTES + SMALL_LEN_BYTES + 252) * WITNESS_SCALE_FACTOR + SMALL_LEN_BYTES) + # large scriptSig, no witness + scriptSig_large = "00"*253 + self.assertEqual(calculate_input_weight(scriptSig_large, None), + (SKELETON_BYTES + LARGE_LEN_BYTES + 253) * WITNESS_SCALE_FACTOR) + # large scriptSig, empty witness stack + self.assertEqual(calculate_input_weight(scriptSig_large, []), + (SKELETON_BYTES + LARGE_LEN_BYTES + 253) * WITNESS_SCALE_FACTOR + SMALL_LEN_BYTES) + # empty scriptSig, 5 small witness stack items + self.assertEqual(calculate_input_weight("", ["00", "11", "22", "33", "44"]), + ((SKELETON_BYTES + SMALL_LEN_BYTES) * WITNESS_SCALE_FACTOR) + SMALL_LEN_BYTES + 5 * SMALL_LEN_BYTES + 5) + # empty scriptSig, 253 small witness stack items + self.assertEqual(calculate_input_weight("", ["00"]*253), + ((SKELETON_BYTES + SMALL_LEN_BYTES) * WITNESS_SCALE_FACTOR) + LARGE_LEN_BYTES + 253 * SMALL_LEN_BYTES + 253) + # small scriptSig, 3 large witness stack items + self.assertEqual(calculate_input_weight(scriptSig_small, ["00"*253]*3), + ((SKELETON_BYTES + SMALL_LEN_BYTES + 252) * WITNESS_SCALE_FACTOR) + SMALL_LEN_BYTES + 3 * LARGE_LEN_BYTES + 3*253) + # large scriptSig, 3 large witness stack items + self.assertEqual(calculate_input_weight(scriptSig_large, ["00"*253]*3), + ((SKELETON_BYTES + LARGE_LEN_BYTES + 253) * WITNESS_SCALE_FACTOR) + SMALL_LEN_BYTES + 3 * LARGE_LEN_BYTES + 3*253) diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 3f6e47d410..2b0b24ec05 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -26,7 +26,6 @@ import sys import tempfile import re import logging -import unittest os.environ["REQUIRE_WALLET_TYPE_SET"] = "1" @@ -70,22 +69,7 @@ if platform.system() != 'Windows' or sys.getwindowsversion() >= (10, 0, 14393): TEST_EXIT_PASSED = 0 TEST_EXIT_SKIPPED = 77 -# List of framework modules containing unit tests. Should be kept in sync with -# the output of `git grep unittest.TestCase ./test/functional/test_framework` -TEST_FRAMEWORK_MODULES = [ - "address", - "crypto.bip324_cipher", - "blocktools", - "crypto.chacha20", - "crypto.ellswift", - "key", - "messages", - "crypto.muhash", - "crypto.poly1305", - "crypto.ripemd160", - "script", - "segwit_addr", -] +TEST_FRAMEWORK_UNIT_TESTS = 'feature_framework_unit_tests.py' EXTENDED_SCRIPTS = [ # These tests are not run by default. @@ -199,6 +183,8 @@ BASE_SCRIPTS = [ 'wallet_txn_clone.py --segwit', 'rpc_getchaintips.py', 'rpc_misc.py', + 'p2p_1p1c_network.py', + 'p2p_opportunistic_1p1c.py', 'interface_rest.py', 'mempool_spend_coinbase.py', 'wallet_avoid_mixing_output_types.py --descriptors', @@ -254,6 +240,7 @@ BASE_SCRIPTS = [ 'wallet_keypool.py --descriptors', 'wallet_descriptor.py --descriptors', 'p2p_nobloomfilter_messages.py', + TEST_FRAMEWORK_UNIT_TESTS, 'p2p_filter.py', 'rpc_setban.py --v1transport', 'rpc_setban.py --v2transport', @@ -439,7 +426,6 @@ def main(): parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs") parser.add_argument('--failfast', '-F', action='store_true', help='stop execution after the first test failure') parser.add_argument('--filter', help='filter scripts to run by regular expression') - parser.add_argument('--skipunit', '-u', action='store_true', help='skip unit tests for the test framework') args, unknown_args = parser.parse_known_args() @@ -551,10 +537,9 @@ def main(): combined_logs_len=args.combinedlogslen, failfast=args.failfast, use_term_control=args.ansi, - skipunit=args.skipunit, ) -def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False, use_term_control, skipunit=False): +def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage=False, args=None, combined_logs_len=0, failfast=False, use_term_control): args = args or [] # Warn if bitcoind is already running @@ -577,15 +562,6 @@ def run_tests(*, test_list, src_dir, build_dir, tmpdir, jobs=1, enable_coverage= # a hard link or a copy on any platform. See https://github.com/bitcoin/bitcoin/pull/27561. sys.path.append(tests_dir) - if not skipunit: - print("Running Unit Tests for Test Framework Modules") - test_framework_tests = unittest.TestSuite() - for module in TEST_FRAMEWORK_MODULES: - test_framework_tests.addTest(unittest.TestLoader().loadTestsFromName("test_framework.{}".format(module))) - result = unittest.TextTestRunner(verbosity=1, failfast=True).run(test_framework_tests) - if not result.wasSuccessful(): - sys.exit("Early exiting after failure in TestFramework unit tests") - flags = ['--cachedir={}'.format(cache_dir)] + args if enable_coverage: diff --git a/test/functional/wallet_send.py b/test/functional/wallet_send.py index e4ca341b49..0a0a8dba0d 100755 --- a/test/functional/wallet_send.py +++ b/test/functional/wallet_send.py @@ -9,10 +9,6 @@ from itertools import product from test_framework.authproxy import JSONRPCException from test_framework.descriptors import descsum_create -from test_framework.messages import ( - ser_compact_size, - WITNESS_SCALE_FACTOR, -) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import ( assert_equal, @@ -21,7 +17,10 @@ from test_framework.util import ( assert_raises_rpc_error, count_bytes, ) -from test_framework.wallet_util import generate_keypair +from test_framework.wallet_util import ( + calculate_input_weight, + generate_keypair, +) class WalletSendTest(BitcoinTestFramework): @@ -543,17 +542,9 @@ class WalletSendTest(BitcoinTestFramework): input_idx = i break psbt_in = dec["inputs"][input_idx] - # Calculate the input weight - # (prevout + sequence + length of scriptSig + scriptsig) * WITNESS_SCALE_FACTOR + len of num scriptWitness stack items + (length of stack item + stack item) * N stack items - # Note that occasionally this weight estimate may be slightly larger or smaller than the real weight - # as sometimes ECDSA signatures are one byte shorter than expected with a probability of 1/128 - len_scriptsig = len(psbt_in["final_scriptSig"]["hex"]) // 2 if "final_scriptSig" in psbt_in else 0 - len_scriptsig += len(ser_compact_size(len_scriptsig)) - len_scriptwitness = (sum([(len(x) // 2) + len(ser_compact_size(len(x) // 2)) for x in psbt_in["final_scriptwitness"]]) + len(ser_compact_size(len(psbt_in["final_scriptwitness"])))) if "final_scriptwitness" in psbt_in else 0 - len_prevout_txid = 32 - len_prevout_index = 4 - len_sequence = 4 - input_weight = ((len_prevout_txid + len_prevout_index + len_sequence + len_scriptsig) * WITNESS_SCALE_FACTOR) + len_scriptwitness + scriptsig_hex = psbt_in["final_scriptSig"]["hex"] if "final_scriptSig" in psbt_in else "" + witness_stack_hex = psbt_in["final_scriptwitness"] if "final_scriptwitness" in psbt_in else None + input_weight = calculate_input_weight(scriptsig_hex, witness_stack_hex) # Input weight error conditions assert_raises_rpc_error( diff --git a/test/functional/wallet_signer.py b/test/functional/wallet_signer.py index 32a1887153..abfc3c1ba1 100755 --- a/test/functional/wallet_signer.py +++ b/test/functional/wallet_signer.py @@ -130,8 +130,9 @@ class WalletSignerTest(BitcoinTestFramework): assert_equal(address_info['hdkeypath'], "m/86h/1h/0h/0/0") self.log.info('Test walletdisplayaddress') - result = hww.walletdisplayaddress(address1) - assert_equal(result, {"address": address1}) + for address in [address1, address2, address3]: + result = hww.walletdisplayaddress(address) + assert_equal(result, {"address": address}) # Handle error thrown by script self.set_mock_result(self.nodes[1], "2") @@ -140,6 +141,13 @@ class WalletSignerTest(BitcoinTestFramework): ) self.clear_mock_result(self.nodes[1]) + # Returned address MUST match: + address_fail = hww.getnewaddress(address_type="bech32") + assert_equal(address_fail, "bcrt1ql7zg7ukh3dwr25ex2zn9jse926f27xy2jz58tm") + assert_raises_rpc_error(-1, 'Signer echoed unexpected address wrong_address', + hww.walletdisplayaddress, address_fail + ) + self.log.info('Prepare mock PSBT') self.nodes[0].sendtoaddress(address4, 1) self.generate(self.nodes[0], 1) diff --git a/test/functional/wallet_signrawtransactionwithwallet.py b/test/functional/wallet_signrawtransactionwithwallet.py index b0517f951d..612a2542e7 100755 --- a/test/functional/wallet_signrawtransactionwithwallet.py +++ b/test/functional/wallet_signrawtransactionwithwallet.py @@ -55,7 +55,7 @@ class SignRawTransactionWithWalletTest(BitcoinTestFramework): def test_with_invalid_sighashtype(self): self.log.info("Test signrawtransactionwithwallet raises if an invalid sighashtype is passed") - assert_raises_rpc_error(-8, "all is not a valid sighash parameter.", self.nodes[0].signrawtransactionwithwallet, hexstring=RAW_TX, sighashtype="all") + assert_raises_rpc_error(-8, "'all' is not a valid sighash parameter.", self.nodes[0].signrawtransactionwithwallet, hexstring=RAW_TX, sighashtype="all") def script_verification_error_test(self): """Create and sign a raw transaction with valid (vin 0), invalid (vin 1) and one missing (vin 2) input script. diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index 558d63e85c..a635175e7c 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -11,6 +11,7 @@ import argparse import configparser import logging import os +import platform import random import subprocess import sys @@ -18,7 +19,7 @@ import sys def get_fuzz_env(*, target, source_dir): symbolizer = os.environ.get('LLVM_SYMBOLIZER_PATH', "/usr/bin/llvm-symbolizer") - return { + fuzz_env = { 'FUZZ': target, 'UBSAN_OPTIONS': f'suppressions={source_dir}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1', @@ -27,6 +28,10 @@ def get_fuzz_env(*, target, source_dir): 'ASAN_SYMBOLIZER_PATH':symbolizer, 'MSAN_SYMBOLIZER_PATH':symbolizer, } + if platform.system() == "Windows": + # On Windows, `env` option must include valid `SystemRoot`. + fuzz_env = {**fuzz_env, 'SystemRoot': os.environ.get('SystemRoot')} + return fuzz_env def main(): diff --git a/test/lint/commit-script-check.sh b/test/lint/commit-script-check.sh index 55c9528dea..fe845ed19e 100755 --- a/test/lint/commit-script-check.sh +++ b/test/lint/commit-script-check.sh @@ -22,6 +22,11 @@ if ! sed --help 2>&1 | grep -q 'GNU'; then exit 1; fi +if ! grep --help 2>&1 | grep -q 'GNU'; then + echo "Error: the installed grep package is not compatible. Please make sure you have GNU grep installed in your system."; + exit 1; +fi + RET=0 PREV_BRANCH=$(git name-rev --name-only HEAD) PREV_HEAD=$(git rev-parse HEAD) diff --git a/test/lint/test_runner/src/main.rs b/test/lint/test_runner/src/main.rs index e22e047e4b..f054f99011 100644 --- a/test/lint/test_runner/src/main.rs +++ b/test/lint/test_runner/src/main.rs @@ -137,9 +137,9 @@ fn lint_trailing_whitespace() -> LintResult { if trailing_space { Err(r#" ^^^ -Trailing whitespace is problematic, because git may warn about it, or editors may remove it by -default, forcing developers in the future to either undo the changes manually or spend time on -review. +Trailing whitespace (including Windows line endings [CR LF]) is problematic, because git may warn +about it, or editors may remove it by default, forcing developers in the future to either undo the +changes manually or spend time on review. Thus, it is best to remove the trailing space now. |